regexp_parser 0.1.6 → 0.2.0

data/lib/regexp_parser/expression/methods/tests.rb

@@ -0,0 +1,116 @@
+module Regexp::Expression
+  class Base
+
+    # Test if this expression has the given test_type, which can be either
+    # a symbol or an array of symbols to check against the expression's type.
+    #
+    #   # is it a :group expression
+    #   exp.type? :group
+    #
+    #   # is it a :set, :subset, or :meta
+    #   exp.type? [:set, :subset, :meta]
+    #
+    def type?(test_type)
+      case test_type
+      when Array
+        if test_type.include?(:*)
+          return (test_type.include?(type) or test_type.include?(:*))
+        else
+          return test_type.include?(type)
+        end
+      when Symbol
+        return (type == test_type or test_type == :*)
+      else
+        raise "Array or Symbol expected, #{test_type.class.name} given"
+      end
+    end
+
+    # Test if this expression has the given test_token, and optionally a given
+    # test_type.
+    #
+    #   # Any expressions
+    #   exp.is? :*  # always returns true
+    #
+    #   # is it a :capture
+    #   exp.is? :capture
+    #
+    #   # is it a :character and a :set
+    #   exp.is? :character, :set
+    #
+    #   # is it a :meta :dot
+    #   exp.is? :dot, :meta
+    #
+    #   # is it a :meta or :escape :dot
+    #   exp.is? :dot, [:meta, :escape]
+    #
+    def is?(test_token, test_type = nil)
+      return true if test_token === :*
+      token == test_token and (test_type ? type?(test_type) : true)
+    end
+
+    # Test if this expression matches an entry in the given scope spec.
+    #
+    # A scope spec can be one of:
+    #
+    #   . An array: Interpreted as a set of tokens, tested for inclusion
+    #               of the expression's token.
+    #
+    #   . A hash:   Where the key is interpreted as the expression type
+    #               and the value is either a symbol or an array. In this
+    #               case, when the scope is a hash, one_of? calls itself to
+    #               evaluate the key's value.
+    #
+    #   . A symbol: matches the expression's token or type, depending on
+    #               the level of the call. If one_of? is called directly with
+    #               a symbol then it will always be checked against the
+    #               type of the expression. If it's being called for a value
+    #               from a hash, it will be checked against the token of the
+    #               expression.
+    #
+    #   # any expression
+    #   exp.one_of?(:*) # always true
+    #
+    #   # like exp.type?(:group)
+    #   exp.one_of?(:group)
+    #
+    #   # any expression of type meta
+    #   exp.one_of?(:meta => :*)
+    #
+    #   # meta dots and alternations
+    #   exp.one_of?(:meta => [:dot, :alternation])
+    #
+    #   # meta dots and any set tokens
+    #   exp.one_of?({meta: [:dot], set: :*})
+    #
+    def one_of?(scope, top = true)
+      case scope
+      when Array
+        if scope.include?(:*)
+          return (scope.include?(token) or scope.include?(:*))
+        else
+          return scope.include?(token)
+        end
+
+      when Hash
+        if scope.has_key?(:*)
+          test_type = scope.has_key?(type) ? type : :*
+          return one_of?(scope[test_type], false)
+        else
+          return (scope.has_key?(type) and one_of?(scope[type], false))
+        end
+
+      when Symbol
+        return true if scope == :*
+
+        return is?(scope) unless top
+        return type?(scope) if top
+
+      else
+        raise "Array, Hash, or Symbol expected, #{scope.class.name} given"
+      end
+
+      false
+    end
+
+  end
+end

data/lib/regexp_parser/expression/methods/traverse.rb

@@ -0,0 +1,63 @@
+module Regexp::Expression
+  class Subexpression < Regexp::Expression::Base
+
+    # Traverses the subexpression (depth-first, pre-order) and calls the given
+    # block for each expression with three arguments; the traversal event,
+    # the expression, and the index of the expression within its parent.
+    #
+    # The event argument is passed as follows:
+    #
+    # - For subexpressions, :enter upon entrering the subexpression, and
+    #   :exit upon exiting it.
+    #
+    # - For terminal expressions, :visit is called once.
+    #
+    # Returns self.
+    def traverse(include_self = false, &block)
+      raise 'traverse requires a block' unless block_given?
+
+      block.call(:enter, self, 0) if include_self
+
+      each_with_index do |exp, index|
+        if exp.terminal?
+          block.call(:visit, exp, index)
+        else
+          block.call(:enter, exp, index)
+          exp.traverse(&block)
+          block.call(:exit, exp, index)
+        end
+      end
+
+      block.call(:exit, self, 0) if include_self
+
+      self
+    end
+    alias :walk :traverse
+
+    # Iterates over the expressions of this expression as an array, passing
+    # the expression and its index within its parent to the given block.
+    def each_expression(include_self = false, &block)
+      traverse(include_self) do |event, exp, index|
+        yield(exp, index) unless event == :exit
+      end
+    end
+
+    # Returns a new array with the results of calling the given block once
+    # for every expression. If a block is not given, returns an array with
+    # each expression and its level index as an array.
+    def map(include_self = false, &block)
+      result = []
+
+      each_expression(include_self) do |exp, index|
+        if block_given?
+          result << yield(exp, index)
+        else
+          result << [exp, index]
+        end
+      end
+
+      result
+    end
+
+  end
+end

data/lib/regexp_parser/expression/quantifier.rb

@@ -21,6 +21,16 @@ module Regexp::Expression
       @text.dup
     end
     alias :to_str :to_s
+
+    def to_h
+      {
+        :token => token,
+        :text  => text,
+        :mode  => mode,
+        :min   =>  min,
+        :max   =>  max
+      }
+    end
   end
 
 end

data/lib/regexp_parser/expression/sequence.rb

@@ -0,0 +1,45 @@
+module Regexp::Expression
+
+  # A sequence of expressions. Differs from a Subexpressions by how it handles
+  # quantifiers, as it applies them to its last element instead of itself as
+  # a whole subexpression.
+  #
+  # Used as the base class for the Alternation alternatives and Conditional
+  # branches.
+  class Sequence < Regexp::Expression::Subexpression
+    def initialize(level, set_level, conditional_level)
+      super Regexp::Token.new(
+        :expression,
+        :sequence,
+        '',
+        nil, # ts
+        nil, # te
+        level,
+        set_level,
+        conditional_level
+      )
+    end
+
+    def text
+      to_s
+    end
+
+    def starts_at
+      @expressions.first.starts_at
+    end
+
+    def quantify(token, text, min = nil, max = nil, mode = :greedy)
+      offset = -1
+      target = expressions[offset]
+      while target and target.is_a?(FreeSpace)
+        target = expressions[offset -= 1]
+      end
+
+      raise ArgumentError.new("No valid target found for '#{text}' " +
+                              "quantifier") unless target
+
+      target.quantify(token, text, min, max, mode)
+    end
+  end
+
+end

data/lib/regexp_parser/expression/subexpression.rb

@@ -17,7 +17,12 @@ module Regexp::Expression
     end
 
     def <<(exp)
-      @expressions << exp
+      if exp.is_a?(WhiteSpace) and @expressions.last and
+        @expressions.last.is_a?(WhiteSpace)
+        @expressions.last.merge(exp)
+      else
+        @expressions << exp
+      end
     end
 
     def insert(exp)
@@ -48,6 +53,22 @@ module Regexp::Expression
       @expressions.length
     end
 
+    def empty?
+      @expressions.empty?
+    end
+
+    def all?(&block)
+      @expressions.all? {|exp| yield(exp) }
+    end
+
+    def ts
+      starts_at
+    end
+
+    def te
+      ts + to_s.length
+    end
+
     def to_s(format = :full)
       s = ''
 
@@ -64,6 +85,13 @@ module Regexp::Expression
 
       s
     end
+
+    def to_h
+      h = super
+      h[:text] = to_s(:base)
+      h[:expressions] = @expressions.map(&:to_h)
+      h
+    end
   end
 
 end

data/lib/regexp_parser/lexer.rb

@@ -10,11 +10,11 @@ module Regexp::Lexer
 
   CLOSING_TOKENS = [:close].freeze
 
-  def self.scan(input, syntax = "ruby/#{RUBY_VERSION}", &block)
+  def self.lex(input, syntax = "ruby/#{RUBY_VERSION}", &block)
     syntax = Regexp::Syntax.new(syntax)
 
     @tokens = []
-    @nesting, @set_nesting = 0, 0
+    @nesting, @set_nesting, @conditional_nesting = 0, 0, 0
 
     last = nil
     Regexp::Scanner.scan(input) do |type, token, text, ts, te|
@@ -27,11 +27,14 @@ module Regexp::Lexer
         last and last.type == :literal
 
       current = Regexp::Token.new(type, token, text, ts, te,
-                                  @nesting, @set_nesting)
+                @nesting, @set_nesting, @conditional_nesting)
 
       current = merge_literal(current) if type == :literal and
         last and last.type == :literal
 
+      current = merge_condition(current) if type == :conditional and
+        [:condition, :condition_close].include?(token)
+
       last.next(current) if last
       current.previous(last) if last
 
@@ -42,12 +45,18 @@ module Regexp::Lexer
     end
 
     if block_given?
-      @tokens.each {|t| block.call(t)}
+      @tokens.map {|t| block.call(t)}
     else
       @tokens
     end
   end
 
+  class << self
+    alias :scan :lex
+  end
+
+  protected
+
   def self.ascend(type, token)
     if type == :group or type == :assertion
       @nesting -= 1 if CLOSING_TOKENS.include?(token)
@@ -56,6 +65,10 @@ module Regexp::Lexer
     if type == :set or type == :subset
       @set_nesting -= 1 if token == :close
     end
+
+    if type == :conditional
+      @conditional_nesting -= 1 if token == :close
+    end
   end
 
   def self.descend(type, token)
@@ -66,6 +79,10 @@ module Regexp::Lexer
     if type == :set or type == :subset
       @set_nesting += 1 if token == :open
     end
+
+    if type == :conditional
+      @conditional_nesting += 1 if token == :open
+    end
   end
 
   # called by scan to break a literal run that is longer than one character
@@ -86,11 +103,11 @@ module Regexp::Lexer
 
       @tokens.pop
       @tokens << Regexp::Token.new(:literal, :literal, lead, token.ts,
-                                   (token.te - last_length), @nesting, @set_nesting)
+                  (token.te - last_length), @nesting, @set_nesting, @conditional_nesting)
 
       @tokens << Regexp::Token.new(:literal, :literal, last,
-                                   (token.ts + lead_length),
-                                   token.te, @nesting, @set_nesting)
+                  (token.ts + lead_length),
+                  token.te, @nesting, @set_nesting, @conditional_nesting)
     end
   end
 
@@ -99,7 +116,13 @@ module Regexp::Lexer
   def self.merge_literal(current)
     last = @tokens.pop
     replace = Regexp::Token.new(:literal, :literal, last.text + current.text,
-                                   last.ts, current.te, @nesting, @set_nesting)
+                last.ts, current.te, @nesting, @set_nesting, @conditional_nesting)
+  end
+
+  def self.merge_condition(current)
+    last = @tokens.pop
+    Regexp::Token.new(:conditional, :condition, last.text + current.text,
+      last.ts, current.te, @nesting, @set_nesting, @conditional_nesting)
   end
 
 end # module Regexp::Lexer

data/lib/regexp_parser/parser.rb

@@ -1,18 +1,14 @@
-require File.expand_path('../expression', __FILE__)
+require 'regexp_parser/expression'
 
 module Regexp::Parser
   include Regexp::Expression
   include Regexp::Syntax
 
-  class ParserError < StandardError
-    def initialize(what)
-      super what
-    end
-  end
+  class ParserError < StandardError; end
 
   class UnknownTokenTypeError < ParserError
     def initialize(type, token)
-      super "Unknown #{type} type #{token.inspect}"
+      super "Unknown token type #{type} #{token.inspect}"
     end
   end
 
@@ -25,8 +21,10 @@ module Regexp::Parser
   def self.parse(input, syntax = "ruby/#{RUBY_VERSION}", &block)
     @nesting = [@root = @node = Root.new]
 
+    @conditional_nesting = []
+
     Regexp::Lexer.scan(input, syntax) do |token|
-      self.parse_token token
+      parse_token token
     end
 
     if block_given?
@@ -43,23 +41,34 @@ module Regexp::Parser
     @node  = exp
   end
 
+  def self.nest_conditional(exp)
+    @conditional_nesting.push exp
+
+    @node << exp
+    @node  = exp
+  end
+
   def self.parse_token(token)
     case token.type
-    when :meta;         self.meta(token)
-    when :quantifier;   self.quantifier(token)
-    when :anchor;       self.anchor(token)
-    when :escape;       self.escape(token)
-    when :group;        self.group(token)
-    when :assertion;    self.group(token)
-    when :set, :subset; self.set(token)
-    when :type;         self.type(token)
-    when :backref;      self.backref(token)
+    when :meta;         meta(token)
+    when :quantifier;   quantifier(token)
+    when :anchor;       anchor(token)
+    when :escape;       escape(token)
+    when :group;        group(token)
+    when :assertion;    group(token)
+    when :set, :subset; set(token)
+    when :type;         type(token)
+    when :backref;      backref(token)
+    when :conditional;  conditional(token)
+    when :keep;         keep(token)
 
     when :property, :nonproperty
-      self.property(token)
+      property(token)
 
     when :literal
       @node << Literal.new(token)
+    when :free_space
+      free_space(token)
 
     else
       raise UnknownTokenTypeError.new(token.type, token)
@@ -69,19 +78,19 @@ module Regexp::Parser
   def self.set(token)
     case token.token
     when :open
-      self.open_set(token)
+      open_set(token)
     when :close
-      self.close_set(token)
+      close_set(token)
     when :negate
-      self.negate_set
+      negate_set
     when :member, :range, :escape, :collation, :equivalent
-      self.append_set(token)
+      append_set(token)
     when *Token::Escape::All
-      self.append_set(token)
+      append_set(token)
     when *Token::CharacterSet::All
-      self.append_set(token)
+      append_set(token)
     when *Token::UnicodeProperty::All
-      self.append_set(token)
+      append_set(token)
     else
       raise UnknownTokenError.new('CharacterSet', token)
     end
@@ -95,7 +104,7 @@ module Regexp::Parser
       unless @node.token == :alternation
         unless @node.last.is_a?(Alternation)
           alt = Alternation.new(token)
-          seq = Sequence.new
+          seq = Alternative.new(alt.level, alt.set_level, alt.conditional_level)
 
           while @node.expressions.last
             seq.insert @node.expressions.pop
@@ -163,6 +172,30 @@ module Regexp::Parser
     end
   end
 
+  def self.conditional(token)
+    case token.token
+    when :open
+      nest_conditional(Conditional::Expression.new(token))
+    when :condition
+      @conditional_nesting.last.condition(Conditional::Condition.new(token))
+      @conditional_nesting.last.branch
+    when :separator
+      @conditional_nesting.last.branch
+      @node = @conditional_nesting.last.branches.last
+    when :close
+      @conditional_nesting.pop
+
+      @node = if @conditional_nesting.empty?
+        @nesting.last
+      else
+        @conditional_nesting.last
+      end
+
+    else
+      raise UnknownTokenError.new('Conditional', token)
+    end
+  end
+
   def self.property(token)
     include Regexp::Expression::UnicodeProperty
 
@@ -291,14 +324,50 @@ module Regexp::Parser
     when :control
       @node << EscapeSequence::Control.new(token)
 
+    when :meta_sequence
+      if token.text =~ /\A\\M-\\C/
+        @node << EscapeSequence::MetaControl.new(token)
+      else
+        @node << EscapeSequence::Meta.new(token)
+      end
+
     else
       # treating everything else as a literal
       @node << EscapeSequence::Literal.new(token)
     end
   end
 
+
+  def self.keep(token)
+    @node << Keep::Mark.new(token)
+  end
+
+  def self.free_space(token)
+    case token.token
+    when :comment
+      @node << Comment.new(token)
+    when :whitespace
+      if @node.last and @node.last.is_a?(WhiteSpace)
+        @node.last.merge(WhiteSpace.new(token))
+      else
+        @node << WhiteSpace.new(token)
+      end
+    else
+      raise UnknownTokenError.new('FreeSpace', token)
+    end
+  end
+
   def self.quantifier(token)
-    unless @node.expressions.last
+    offset = -1
+    target_node = @node.expressions[offset]
+    while target_node and target_node.is_a?(FreeSpace)
+      target_node = @node.expressions[offset -= 1]
+    end
+
+    raise ArgumentError.new("No valid target found for '#{token.text}' "+
+                            "quantifier") unless target_node
+
+    unless target_node
       if token.token == :zero_or_one
         raise "Quantifier given without a target, or the syntax of the group " +
               "or its options is incorrect"
@@ -309,35 +378,36 @@ module Regexp::Parser
 
     case token.token
     when :zero_or_one
-      @node.expressions.last.quantify(:zero_or_one, token.text, 0, 1, :greedy)
+      target_node.quantify(:zero_or_one, token.text, 0, 1, :greedy)
     when :zero_or_one_reluctant
-      @node.expressions.last.quantify(:zero_or_one, token.text, 0, 1, :reluctant)
+      target_node.quantify(:zero_or_one, token.text, 0, 1, :reluctant)
     when :zero_or_one_possessive
-      @node.expressions.last.quantify(:zero_or_one, token.text, 0, 1, :possessive)
+      target_node.quantify(:zero_or_one, token.text, 0, 1, :possessive)
 
     when :zero_or_more
-      @node.expressions.last.quantify(:zero_or_more, token.text, 0, -1, :greedy)
+      target_node.quantify(:zero_or_more, token.text, 0, -1, :greedy)
     when :zero_or_more_reluctant
-      @node.expressions.last.quantify(:zero_or_more, token.text, 0, -1, :reluctant)
+      target_node.quantify(:zero_or_more, token.text, 0, -1, :reluctant)
     when :zero_or_more_possessive
-      @node.expressions.last.quantify(:zero_or_more, token.text, 0, -1, :possessive)
+      target_node.quantify(:zero_or_more, token.text, 0, -1, :possessive)
 
     when :one_or_more
-      @node.expressions.last.quantify(:one_or_more, token.text, 1, -1, :greedy)
+      target_node.quantify(:one_or_more, token.text, 1, -1, :greedy)
     when :one_or_more_reluctant
-      @node.expressions.last.quantify(:one_or_more, token.text, 1, -1, :reluctant)
+      target_node.quantify(:one_or_more, token.text, 1, -1, :reluctant)
     when :one_or_more_possessive
-      @node.expressions.last.quantify(:one_or_more, token.text, 1, -1, :possessive)
+      target_node.quantify(:one_or_more, token.text, 1, -1, :possessive)
 
     when :interval
-      self.interval(token.text)
+      interval(target_node, token)
 
     else
       raise UnknownTokenError.new('Quantifier', token)
     end
   end
 
-  def self.interval(text)
+  def self.interval(target_node, token)
+    text = token.text
     mchr = text[text.length-1].chr =~ /[?+]/ ? text[text.length-1].chr : nil
     mode = case mchr
     when '?'; text.chop!; :reluctant
@@ -349,19 +419,19 @@ module Regexp::Parser
     min = range[0].empty? ? 0 : range[0]
     max = range[1] ? (range[1].empty? ? -1 : range[1]) : min
 
-    @node.expressions.last.quantify(:interval, text, min.to_i, max.to_i, mode)
+    target_node.quantify(:interval, text, min.to_i, max.to_i, mode)
   end
 
   def self.group(token)
     case token.token
     when :options
-      self.options(token)
+      options(token)
     when :close
-      self.close_group
+      close_group
     when :comment
       @node << Group::Comment.new(token)
     else
-      self.open_group(token)
+      open_group(token)
     end
   end
 
@@ -372,10 +442,13 @@ module Regexp::Parser
     exp.options = {
       :m => opt[0].include?('m') ? true : false,
       :i => opt[0].include?('i') ? true : false,
-      :x => opt[0].include?('x') ? true : false
+      :x => opt[0].include?('x') ? true : false,
+      :d => opt[0].include?('d') ? true : false,
+      :a => opt[0].include?('a') ? true : false,
+      :u => opt[0].include?('u') ? true : false
     }
 
-    self.nest exp
+    nest(exp)
   end
 
   def self.open_group(token)
@@ -402,7 +475,7 @@ module Regexp::Parser
       raise UnknownTokenError.new('Group type open', token)
     end
 
-    self.nest exp
+    nest(exp)
   end
 
   def self.close_group