regexp_parser 0.4.6 → 0.4.7

data/lib/regexp_parser/scanner/scanner.rl

@@ -448,7 +448,7 @@
     };
 
     alternation {
-      if in_conditional and conditional_stack.length > 0 and 
+      if in_conditional and conditional_stack.length > 0 and
          conditional_stack.last[1] == @group_depth
         emit(:conditional, :separator, *text(data, ts, te))
       else
@@ -538,7 +538,7 @@
 
     # (?#...) comments: parsed as a single expression, without introducing a
     # new nesting level. Comments may not include parentheses, escaped or not.
-    # special case for close, action performed on all transitions to get the 
+    # special case for close, action performed on all transitions to get the
     # correct closing count.
     # ------------------------------------------------------------------------
     group_open . group_comment $group_closed {
@@ -622,14 +622,10 @@
         end
       else
         if @spacing_stack.length > 1 and
-          @spacing_stack.last[1] == (@group_depth + 1)
+          @spacing_stack.last[:depth] == (@group_depth + 1)
           @spacing_stack.pop
 
-          @free_spacing = @spacing_stack.last[0]
-
-          if @spacing_stack.length == 1
-            @in_options = false
-          end
+          @free_spacing = @spacing_stack.last[:free_spacing]
         end
 
         emit(:group, :close, *text(data, ts, te))
@@ -777,9 +773,7 @@
 # THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
 # This file was generated from lib/regexp_parser/scanner/scanner.rl
 
-module Regexp::Scanner
-  %% write data;
-
+class Regexp::Scanner
   # General scanner error (catch all)
   class ScannerError < StandardError; end
 
@@ -839,6 +833,10 @@ module Regexp::Scanner
   # This method may raise errors if a syntax error is encountered.
   # --------------------------------------------------------------------------
   def self.scan(input_object, &block)
+    new.scan(input_object, &block)
+  end
+
+  def scan(input_object, &block)
     @literal, top, stack = nil, 0, []
 
     if input_object.is_a?(Regexp)
@@ -857,11 +855,12 @@ module Regexp::Scanner
     @block  = block_given? ? block : nil
 
     @in_group, @group_depth = false, 0
-    @in_options, @spacing_stack = false, [[@free_spacing, 0]]
+    @spacing_stack = [{:free_spacing => @free_spacing, :depth => 0}]
 
     in_set,   set_depth, set_type   = false, 0, :set
     in_conditional, conditional_depth, conditional_stack = false, 0, []
 
+    %% write data;
     %% write init;
     %% write exec;
 
@@ -881,12 +880,25 @@ module Regexp::Scanner
     @tokens
   end
 
+  # Emits an array with the details of the scanned pattern
+  def emit(type, token, text, ts, te)
+    #puts "EMIT: type: #{type}, token: #{token}, text: #{text}, ts: #{ts}, te: #{te}"
+
+    emit_literal if @literal
+
+    if @block
+      @block.call type, token, text, ts, te
+    end
+
+    @tokens << [type, token, text, ts, te]
+  end
+
   private
 
   # Ragel's regex-based scan of the group options introduced a lot of
   # ambiguity, so we just ask it to find the beginning of what looks
   # like an options run and handle the rest in here.
-  def self.scan_options(p, data, ts, te)
+  def scan_options(p, data, ts, te)
     text = text(data, ts, te).first
 
     options_char, options_length = true, 0
@@ -938,26 +950,26 @@ module Regexp::Scanner
   end
 
   # Copy from ts to te from data as text
-  def self.copy(data, range)
+  def copy(data, range)
     data[range].pack('c*')
   end
 
   # Copy from ts to te from data as text, returning an array with the text
   #  and the offsets used to copy it.
-  def self.text(data, ts, te, soff = 0)
+  def text(data, ts, te, soff = 0)
     [copy(data, ts-soff..te-1), ts-soff, te]
   end
 
   # Appends one or more characters to the literal buffer, to be emitted later
   # by a call to emit_literal. Contents can be a mix of ASCII and UTF-8.
-  def self.append_literal(data, ts, te)
+  def append_literal(data, ts, te)
     @literal ||= []
     @literal << text(data, ts, te)
   end
 
   # Emits the literal run collected by calls to the append_literal method,
   # using the total start (ts) and end (te) offsets of the run.
-  def self.emit_literal
+  def emit_literal
     ts, te = @literal.first[1], @literal.last[2]
     text = @literal.map {|t| t[0]}.join
 
@@ -967,43 +979,34 @@ module Regexp::Scanner
     emit(:literal, :literal, text, ts, te)
   end
 
-  def self.emit_options(text, ts, te)
-    if text =~ /\(\?([mixdau]+)?-?([mix]+)?:/
-      positive, negative = $1, $2
+  def emit_options(text, ts, te)
+    if text =~ /\(\?([mixdau]*)-?([mix]*)(:)?/
+      positive, negative, group_local = $1, $2, $3
 
-      if positive =~ /x/
+      if positive.include?('x')
         @free_spacing = true
       end
 
       # If the x appears in both, treat it like ruby does, the second cancels
       # the first.
-      if negative =~ /x/
+      if negative.include?('x')
         @free_spacing = false
       end
-    end
-
-    @in_options = true
-    @spacing_stack << [@free_spacing, @group_depth]
-
-    emit(:group, :options, text, ts, te)
-  end
-
-  # Emits an array with the details of the scanned pattern
-  def self.emit(type, token, text, ts, te)
-    #puts "EMIT: type: #{type}, token: #{token}, text: #{text}, ts: #{ts}, te: #{te}"
 
-    emit_literal if @literal
-
-    if @block
-      @block.call type, token, text, ts, te
+      if group_local
+        @spacing_stack << {:free_spacing => @free_spacing, :depth => @group_depth}
+      else
+        # switch for parent group level
+        @spacing_stack.last[:free_spacing] = @free_spacing
+      end
     end
 
-    @tokens << [type, token, text, ts, te]
+    emit(:group, :options, text, ts, te)
   end
 
   # Centralizes and unifies the handling of validation related
   # errors.
-  def self.validation_error(type, what, reason)
+  def validation_error(type, what, reason)
     case type
     when :group
       error = InvalidGroupError.new(what, reason)
@@ -1019,12 +1022,12 @@ module Regexp::Scanner
   end
 
   # Used for references with an empty name or number
-  def self.empty_backref_error(type, what)
+  def empty_backref_error(type, what)
     validation_error(:backref, what, 'ref ID is empty')
   end
 
   # Used for named expressions with an empty name
-  def self.empty_name_error(type, what)
+  def empty_name_error(type, what)
     validation_error(type, what, 'name is empty')
   end
 

data/lib/regexp_parser/syntax/tokens/backref.rb

@@ -3,7 +3,7 @@ module Regexp::Syntax
 
     module Backreference
       Name      = [:name_ref]
-      Number    = [:number_ref, :number_rel_ref]
+      Number    = [:number, :number_ref, :number_rel_ref]
 
       NestLevel = [:name_nest_ref, :number_nest_ref]
 

data/lib/regexp_parser/version.rb

@@ -1,5 +1,5 @@
 class Regexp
-  module Parser
-    VERSION = '0.4.6'
+  class Parser
+    VERSION = '0.4.7'
   end
 end

data/test/expression/test_to_h.rb

@@ -14,7 +14,7 @@ class ExpressionToH < Test::Unit::TestCase
       :starts_at         => 0,
       :length            => 3,
       :quantifier        => nil,
-      :options           => nil,
+      :options           => {},
       :level             => nil,
       :set_level         => nil,
       :conditional_level => nil,
@@ -26,7 +26,7 @@ class ExpressionToH < Test::Unit::TestCase
           :starts_at         => 0,
           :length            => 3,
           :quantifier        => nil,
-          :options           => nil,
+          :options           => {},
           :level             => 0,
           :set_level         => 0,
           :conditional_level => 0

data/test/lexer/test_refcalls.rb

@@ -3,6 +3,9 @@ require File.expand_path("../../helpers", __FILE__)
 class LexerRefCalls < Test::Unit::TestCase
 
   tests = {
+    # Traditional numerical group back-reference
+    '(abc)\1'         => [3, :backref, :number,             '\1',         5,  7, 0, 0, 0],
+
     # Group back-references, named, numbered, and relative
     '(?<X>abc)\k<X>'  => [3, :backref, :name_ref,           '\k<X>',      9, 14, 0, 0, 0],
     "(?<X>abc)\\k'X'" => [3, :backref, :name_ref,           "\\k'X'",     9, 14, 0, 0, 0],

data/test/parser/test_errors.rb

@@ -1,58 +1,62 @@
 require File.expand_path("../../helpers", __FILE__)
 
 class ParserErrors < Test::Unit::TestCase
+  def setup
+    @rp = Regexp::Parser.new
+    @rp.parse(/foo/)
+  end
 
   def test_parser_unknown_token_type
     assert_raise( Regexp::Parser::UnknownTokenTypeError ) {
-      RP.parse_token(Regexp::Token.new(:foo, :bar))
+      @rp.__send__(:parse_token, Regexp::Token.new(:foo, :bar))
     }
   end
 
   def test_parser_unknown_set_token
     assert_raise( Regexp::Parser::UnknownTokenError ) {
-      RP.parse_token(Regexp::Token.new(:set, :foo))
+      @rp.__send__(:parse_token, Regexp::Token.new(:set, :foo))
     }
   end
 
   def test_parser_unknown_meta_token
     assert_raise( Regexp::Parser::UnknownTokenError ) {
-      RP.parse_token(Regexp::Token.new(:meta, :foo))
+      @rp.__send__(:parse_token, Regexp::Token.new(:meta, :foo))
     }
   end
 
   def test_parser_unknown_character_type_token
     assert_raise( Regexp::Parser::UnknownTokenError ) {
-      RP.parse_token(Regexp::Token.new(:type, :foo))
+      @rp.__send__(:parse_token, Regexp::Token.new(:type, :foo))
     }
   end
 
   def test_parser_unknown_unicode_property_token
     assert_raise( Regexp::Parser::UnknownTokenError ) {
-      RP.parse_token(Regexp::Token.new(:property, :foo))
+      @rp.__send__(:parse_token, Regexp::Token.new(:property, :foo))
     }
   end
 
   def test_parser_unknown_unicode_nonproperty_token
     assert_raise( Regexp::Parser::UnknownTokenError ) {
-      RP.parse_token(Regexp::Token.new(:nonproperty, :foo))
+      @rp.__send__(:parse_token, Regexp::Token.new(:nonproperty, :foo))
     }
   end
 
   def test_parser_unknown_anchor_token
     assert_raise( Regexp::Parser::UnknownTokenError ) {
-      RP.parse_token(Regexp::Token.new(:anchor, :foo))
+      @rp.__send__(:parse_token, Regexp::Token.new(:anchor, :foo))
     }
   end
 
   def test_parser_unknown_quantifier_token
     assert_raise( Regexp::Parser::UnknownTokenError ) {
-      RP.parse_token(Regexp::Token.new(:quantifier, :foo))
+      @rp.__send__(:parse_token, Regexp::Token.new(:quantifier, :foo))
     }
   end
 
   def test_parser_unknown_group_open_token
     assert_raise( Regexp::Parser::UnknownTokenError ) {
-      RP.parse_token(Regexp::Token.new(:group, :foo))
+      @rp.__send__(:parse_token, Regexp::Token.new(:group, :foo))
     }
   end
 

data/test/parser/test_groups.rb

@@ -3,44 +3,170 @@ require File.expand_path("../../helpers", __FILE__)
 class TestParserGroups < Test::Unit::TestCase
 
   def test_parse_root_options_mi
-    t = RP.parse((/[abc]/mi).to_s, 'ruby/1.8')
+    t = RP.parse(/[abc]/mi, 'ruby/1.8')
 
     assert_equal true,  t.m?
     assert_equal true,  t.i?
     assert_equal false, t.x?
   end
 
-  def test_parse_nested_options_m
-    t = RP.parse('(?xi-m:a(?m-ix:b))', 'ruby/1.8')
+  def test_parse_option_group
+    t = RP.parse(/(?m:a)/, 'ruby/1.8')
+
+    assert_equal Group::Options, t.expressions[0].class
+    assert_equal :options,       t.expressions[0].token
+
+    assert_equal true,  t.expressions[0].m?
+    assert_equal false, t.expressions[0].i?
+    assert_equal false, t.expressions[0].x?
+  end
+
+  def test_parse_self_defeating_option_group
+    t = RP.parse(/(?m-m:a)/, 'ruby/1.8')
+
+    assert_equal false, t.expressions[0].m?
+    assert_equal false, t.expressions[0].i?
+    assert_equal false, t.expressions[0].x?
+  end
+
+  def test_parse_nested_options_activate_one
+    t = RP.parse(/(?x-mi:a(?m:b))/, 'ruby/1.8')
+
+    assert_equal false, t.expressions[0].m?
+    assert_equal false, t.expressions[0].i?
+    assert_equal true,  t.expressions[0].x?
 
     assert_equal true,  t.expressions[0].expressions[1].m?
     assert_equal false, t.expressions[0].expressions[1].i?
-    assert_equal false, t.expressions[0].expressions[1].x?
+    assert_equal true,  t.expressions[0].expressions[1].x?
   end
 
-  def test_parse_nested_options_xm
-    t = RP.parse(/(?i-xm:a(?mx-i:b))/, 'ruby/1.8')
+  def test_parse_nested_options_deactivate_one
+    t = RP.parse(/(?ix-m:a(?-i:b))/, 'ruby/1.8')
 
-    assert_equal true,  t.expressions[0].expressions[1].m?
+    assert_equal false, t.expressions[0].m?
+    assert_equal true,  t.expressions[0].i?
+    assert_equal true,  t.expressions[0].x?
+
+    assert_equal false, t.expressions[0].expressions[1].m?
     assert_equal false, t.expressions[0].expressions[1].i?
     assert_equal true,  t.expressions[0].expressions[1].x?
   end
 
-  def test_parse_nested_options_im
-    t = RP.parse(/(?x-mi:a(?mi-x:b))/, 'ruby/1.8')
+  def test_parse_nested_options_invert_all
+    t = RP.parse('(?xi-m:a(?m-ix:b))', 'ruby/1.8')
+
+    assert_equal false, t.expressions[0].m?
+    assert_equal true,  t.expressions[0].i?
+    assert_equal true,  t.expressions[0].x?
 
     assert_equal true,  t.expressions[0].expressions[1].m?
-    assert_equal true,  t.expressions[0].expressions[1].i?
+    assert_equal false, t.expressions[0].expressions[1].i?
     assert_equal false, t.expressions[0].expressions[1].x?
   end
 
+  def test_parse_nested_options_affect_literal_subexpressions
+    t = RP.parse(/(?x-mi:a(?m:b))/, 'ruby/1.8')
+
+    # a
+    assert_equal false, t.expressions[0].expressions[0].m?
+    assert_equal false, t.expressions[0].expressions[0].i?
+    assert_equal true,  t.expressions[0].expressions[0].x?
+
+    # b
+    assert_equal true,  t.expressions[0].expressions[1].expressions[0].m?
+    assert_equal false, t.expressions[0].expressions[1].expressions[0].i?
+    assert_equal true,  t.expressions[0].expressions[1].expressions[0].x?
+  end
+
+  def test_parse_option_switch_group
+    t = RP.parse(/a(?i-m)b/m, 'ruby/1.8')
+
+    assert_equal Group::Options, t.expressions[1].class
+    assert_equal :options,       t.expressions[1].token
+    # TODO: change this ^ to :options_switch in v1.0.0
+
+    assert_equal false, t.expressions[1].m?
+    assert_equal true,  t.expressions[1].i?
+    assert_equal false, t.expressions[1].x?
+  end
+
+  def test_parse_option_switch_affects_following_expressions
+    t = RP.parse(/a(?i-m)b/m, 'ruby/1.8')
+
+    # a
+    assert_equal true,  t.expressions[0].m?
+    assert_equal false, t.expressions[0].i?
+    assert_equal false, t.expressions[0].x?
+
+    # b
+    assert_equal false, t.expressions[2].m?
+    assert_equal true,  t.expressions[2].i?
+    assert_equal false, t.expressions[2].x?
+  end
+
+  def test_parse_option_switch_in_group
+    t = RP.parse(/(a(?i-m)b)c/m, 'ruby/1.8')
+
+    group1 = t.expressions[0]
+
+    assert_equal true,  group1.m?
+    assert_equal false, group1.i?
+    assert_equal false, group1.x?
+
+    # a
+    assert_equal true,  group1.expressions[0].m?
+    assert_equal false, group1.expressions[0].i?
+    assert_equal false, group1.expressions[0].x?
+
+    # (?i-m)
+    assert_equal false, group1.expressions[1].m?
+    assert_equal true,  group1.expressions[1].i?
+    assert_equal false, group1.expressions[1].x?
+
+    # b
+    assert_equal false, group1.expressions[2].m?
+    assert_equal true,  group1.expressions[2].i?
+    assert_equal false, group1.expressions[2].x?
+
+    # c
+    assert_equal true,  t.expressions[1].m?
+    assert_equal false, t.expressions[1].i?
+    assert_equal false, t.expressions[1].x?
+  end
+
+  def test_parse_nested_option_switch_in_group
+    t = RP.parse(/((?i-m)(a(?-i)b))/m, 'ruby/1.8')
+
+    group2 = t.expressions[0].expressions[1]
+
+    assert_equal false, group2.m?
+    assert_equal true,  group2.i?
+    assert_equal false, group2.x?
+
+    # a
+    assert_equal false, group2.expressions[0].m?
+    assert_equal true,  group2.expressions[0].i?
+    assert_equal false, group2.expressions[0].x?
+
+    # (?-i)
+    assert_equal false, group2.expressions[1].m?
+    assert_equal false, group2.expressions[1].i?
+    assert_equal false, group2.expressions[1].x?
+
+    # b
+    assert_equal false, group2.expressions[2].m?
+    assert_equal false, group2.expressions[2].i?
+    assert_equal false, group2.expressions[2].x?
+  end
+
   if RUBY_VERSION >= '2.0'
     def test_parse_options_dau
       t = RP.parse('(?dua:abc)')
 
-      assert_equal true,  t.expressions[0].d?
+      assert_equal false, t.expressions[0].d?
       assert_equal true,  t.expressions[0].a?
-      assert_equal true,  t.expressions[0].u?
+      assert_equal false, t.expressions[0].u?
     end
 
     def test_parse_nested_options_dau
@@ -65,12 +191,12 @@ class TestParserGroups < Test::Unit::TestCase
       assert_equal false, t.expressions[0].a?
       assert_equal false, t.expressions[0].u?
 
-      assert_equal true,  t.expressions[0].expressions[1].d?
+      assert_equal false, t.expressions[0].expressions[1].d?
       assert_equal true,  t.expressions[0].expressions[1].a?
       assert_equal false, t.expressions[0].expressions[1].u?
       assert_equal false, t.expressions[0].expressions[1].x?
       assert_equal false, t.expressions[0].expressions[1].m?
-      assert_equal false, t.expressions[0].expressions[1].i?
+      assert_equal true,  t.expressions[0].expressions[1].i?
     end
   end