regexp_parser 1.6.0 → 1.8.2

data/lib/regexp_parser/scanner/scanner.rl

@@ -21,7 +21,7 @@
   set_close             = ']';
   brackets              = set_open | set_close;
 
-  comment               = ('#' . [^\n]* . '\n');
+  comment               = ('#' . [^\n]* . '\n'?);
 
   class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                           'cntrl' | 'digit' | 'graph' |
@@ -62,13 +62,17 @@
   quantifier_possessive = '?+' | '*+' | '++';
   quantifier_mode       = '?'  | '+';
 
-  quantifier_interval   = range_open . (digit+)? . ','? . (digit+)? .
-                          range_close . quantifier_mode?;
+  quantity_exact        = (digit+);
+  quantity_minimum      = (digit+) . ',';
+  quantity_maximum      = ',' . (digit+);
+  quantity_range        = (digit+) . ',' . (digit+);
+  quantifier_interval   = range_open . ( quantity_exact | quantity_minimum |
+                          quantity_maximum | quantity_range ) . range_close .
+                          quantifier_mode?;
 
   quantifiers           = quantifier_greedy | quantifier_reluctant |
                           quantifier_possessive | quantifier_interval;
 
-
   conditional           = '(?(';
 
   group_comment         = '?#' . [^)]* . group_close;
@@ -114,7 +118,9 @@
                           curlies | parantheses | brackets |
                           line_anchor | quantifier_greedy;
 
-  ascii_print           = ((0x20..0x7e) - meta_char);
+  literal_delimiters    = ']' | '}';
+
+  ascii_print           = ((0x20..0x7e) - meta_char - '#');
   ascii_nonprint        = (0x01..0x1f | 0x7f);
 
   utf8_2_byte           = (0xc2..0xdf 0x80..0xbf);
@@ -122,7 +128,7 @@
   utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf);
 
   non_literal_escape    = char_type_char | anchor_char | escaped_ascii |
-                          group_ref | keep_mark | [xucCM];
+                          keep_mark | [xucCM];
 
   non_set_escape        = (anchor_char - 'b') | group_ref | keep_mark |
                           multi_codepoint_char_type | [0-9cCM];
@@ -417,6 +423,10 @@
       end
     };
 
+    literal_delimiters {
+      append_literal(data, ts, te)
+    };
+
     # Character sets
     # ------------------------------------------------------------------------
     set_open >set_opened {
@@ -620,10 +630,15 @@
       end
     };
 
-    quantifier_interval  @err(premature_end_error) {
+    quantifier_interval  {
       emit(:quantifier, :interval, *text(data, ts, te))
     };
 
+    # Catch unmatched curly braces as literals
+    range_open {
+      append_literal(data, ts, te)
+    };
+
     # Escaped sequences
     # ------------------------------------------------------------------------
     backslash > (backslashed, 1) {
@@ -634,7 +649,9 @@
       if free_spacing
         emit(:free_space, :comment, *text(data, ts, te))
       else
-        append_literal(data, ts, te)
+        # consume only the pound sign (#) and backtrack to do regular scanning
+        append_literal(data, ts, ts + 1)
+        fexec ts + 1;
       end
     };
 
@@ -722,21 +739,16 @@ class Regexp::Scanner
   #
   # This method may raise errors if a syntax error is encountered.
   # --------------------------------------------------------------------------
-  def self.scan(input_object, &block)
-    new.scan(input_object, &block)
+  def self.scan(input_object, options: nil, &block)
+    new.scan(input_object, options: options, &block)
   end
 
-  def scan(input_object, &block)
+  def scan(input_object, options: nil, &block)
     self.literal = nil
     stack = []
 
-    if input_object.is_a?(Regexp)
-      input = input_object.source
-      self.free_spacing = (input_object.options & Regexp::EXTENDED != 0)
-    else
-      input = input_object
-      self.free_spacing = false
-    end
+    input = input_object.is_a?(Regexp) ? input_object.source : input_object
+    self.free_spacing = free_spacing?(input_object, options)
     self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]
 
     data  = input.unpack("c*") if input.is_a?(String)
@@ -802,6 +814,18 @@ class Regexp::Scanner
   attr_accessor :tokens, :literal, :block, :free_spacing, :spacing_stack,
                 :group_depth, :set_depth, :conditional_stack
 
+  def free_spacing?(input_object, options)
+    if options && !input_object.is_a?(String)
+      raise ArgumentError, 'options cannot be supplied unless scanning a String'
+    end
+
+    options = input_object.options if input_object.is_a?(::Regexp)
+
+    return false unless options
+
+    options & Regexp::EXTENDED != 0
+  end
+
   def in_group?
     group_depth > 0
   end

data/lib/regexp_parser/syntax/version_lookup.rb

@@ -74,9 +74,9 @@ module Regexp::Syntax
   end
 
   def warn_if_future_version(const_name)
-    return if comparable_version(const_name) < comparable_version('3.0.0')
+    return if comparable_version(const_name) < comparable_version('4.0.0')
 
-    warn('This library has only been tested up to Ruby 2.x, '\
+    warn('This library has only been tested up to Ruby 3.x, '\
          "but you are running with #{const_get(const_name).inspect}")
   end
 end

data/lib/regexp_parser/version.rb

@@ -1,5 +1,5 @@
 class Regexp
   class Parser
-    VERSION = '1.6.0'
+    VERSION = '1.8.2'
   end
 end

data/regexp_parser.gemspec

@@ -32,5 +32,5 @@ Gem::Specification.new do |gem|
 
   gem.platform      = Gem::Platform::RUBY
 
-  gem.required_ruby_version = '>= 1.9.1'
+  gem.required_ruby_version = '>= 2.0.0'
 end

data/spec/expression/methods/match_length_spec.rb

@@ -120,6 +120,13 @@ RSpec.describe(Regexp::MatchLength) do
       expect { result.next }.to raise_error(StopIteration)
     end
 
+    it 'is aware of limit option even if called without a block' do
+      result = ML.of(/a?/).each(limit: 1)
+      expect(result).to be_a(Enumerator)
+      expect(result.next).to eq 0
+      expect { result.next }.to raise_error(StopIteration)
+    end
+
     it 'is limited to 1000 iterations in case there are infinite match lengths' do
       expect(ML.of(/a*/).first(3000).size).to eq 1000
     end

data/spec/expression/methods/traverse_spec.rb

@@ -39,6 +39,17 @@ RSpec.describe('Subexpression#traverse') do
     expect(visits).to eq 9
   end
 
+  specify('Subexpression#traverse without a block') do
+    root = RP.parse(/abc/)
+    enum = root.traverse
+
+    expect(enum).to be_a(Enumerator)
+    event, expr, idx = enum.next
+    expect(event).to eq(:visit)
+    expect(expr).to be_a(Regexp::Expression::Literal)
+    expect(idx).to eq(0)
+  end
+
   specify('Subexpression#walk alias') do
     root = RP.parse(/abc/)
 
@@ -81,6 +92,16 @@ RSpec.describe('Subexpression#traverse') do
     expect(indices).to eq [0, 0, 1, 0, 2]
   end
 
+  specify('Subexpression#each_expression without a block') do
+    root = RP.parse(/abc/)
+    enum = root.each_expression
+
+    expect(enum).to be_a(Enumerator)
+    expr, idx = enum.next
+    expect(expr).to be_a(Regexp::Expression::Literal)
+    expect(idx).to eq(0)
+  end
+
   specify('Subexpression#flat_map without block') do
     root = RP.parse(/a(b([c-e]+))?/)
 

data/spec/expression/options_spec.rb

@@ -85,44 +85,44 @@ RSpec.describe('Expression#options') do
       .and change { exp.unicode_classes? }.from(false).to(true)
   end
 
-  RSpec.shared_examples '#options' do |regexp, klass, at: []|
+  RSpec.shared_examples '#options' do |regexp, path, klass|
     it "works for expression class #{klass}" do
-      exp = RP.parse(/#{regexp.source}/i).dig(*at)
+      exp = RP.parse(/#{regexp.source}/i).dig(*path)
       expect(exp).to be_a(klass)
       expect(exp).to be_i
       expect(exp).not_to be_x
     end
   end
 
-  include_examples '#options', //, Root
-  include_examples '#options', /a/, Literal, at: [0]
-  include_examples '#options', /\A/, Anchor::Base, at: [0]
-  include_examples '#options', /\d/, CharacterType::Base, at: [0]
-  include_examples '#options', /\n/, EscapeSequence::Base, at: [0]
-  include_examples '#options', /\K/, Keep::Mark, at: [0]
-  include_examples '#options', /./, CharacterType::Any, at: [0]
-  include_examples '#options', /(a)/, Group::Base, at: [0]
-  include_examples '#options', /(a)/, Literal, at: [0, 0]
-  include_examples '#options', /(?=a)/, Assertion::Base, at: [0]
-  include_examples '#options', /(?=a)/, Literal, at: [0, 0]
-  include_examples '#options', /(a|b)/, Group::Base, at: [0]
-  include_examples '#options', /(a|b)/, Alternation, at: [0, 0]
-  include_examples '#options', /(a|b)/, Alternative, at: [0, 0, 0]
-  include_examples '#options', /(a|b)/, Literal, at: [0, 0, 0, 0]
-  include_examples '#options', /(a)\1/, Backreference::Base, at: [1]
-  include_examples '#options', /(a)\k<1>/, Backreference::Number, at: [1]
-  include_examples '#options', /(a)\g<1>/, Backreference::NumberCall, at: [1]
-  include_examples '#options', /[a]/, CharacterSet, at: [0]
-  include_examples '#options', /[a]/, Literal, at: [0, 0]
-  include_examples '#options', /[a-z]/, CharacterSet::Range, at: [0, 0]
-  include_examples '#options', /[a-z]/, Literal, at: [0, 0, 0]
-  include_examples '#options', /[a&&z]/, CharacterSet::Intersection, at: [0, 0]
-  include_examples '#options', /[a&&z]/, CharacterSet::IntersectedSequence, at: [0, 0, 0]
-  include_examples '#options', /[a&&z]/, Literal, at: [0, 0, 0, 0]
-  include_examples '#options', /[[:ascii:]]/, PosixClass, at: [0, 0]
-  include_examples '#options', /\p{word}/, UnicodeProperty::Base, at: [0]
-  include_examples '#options', /(a)(?(1)b|c)/, Conditional::Expression, at: [1]
-  include_examples '#options', /(a)(?(1)b|c)/, Conditional::Condition, at: [1, 0]
-  include_examples '#options', /(a)(?(1)b|c)/, Conditional::Branch, at: [1, 1]
-  include_examples '#options', /(a)(?(1)b|c)/, Literal, at: [1, 1, 0]
+  include_examples '#options', //, [], Root
+  include_examples '#options', /a/, [0], Literal
+  include_examples '#options', /\A/, [0], Anchor::Base
+  include_examples '#options', /\d/, [0], CharacterType::Base
+  include_examples '#options', /\n/, [0], EscapeSequence::Base
+  include_examples '#options', /\K/, [0], Keep::Mark
+  include_examples '#options', /./, [0], CharacterType::Any
+  include_examples '#options', /(a)/, [0], Group::Base
+  include_examples '#options', /(a)/, [0, 0], Literal
+  include_examples '#options', /(?=a)/, [0], Assertion::Base
+  include_examples '#options', /(?=a)/, [0, 0], Literal
+  include_examples '#options', /(a|b)/, [0], Group::Base
+  include_examples '#options', /(a|b)/, [0, 0], Alternation
+  include_examples '#options', /(a|b)/, [0, 0, 0], Alternative
+  include_examples '#options', /(a|b)/, [0, 0, 0, 0], Literal
+  include_examples '#options', /(a)\1/, [1], Backreference::Base
+  include_examples '#options', /(a)\k<1>/, [1], Backreference::Number
+  include_examples '#options', /(a)\g<1>/, [1], Backreference::NumberCall
+  include_examples '#options', /[a]/, [0], CharacterSet
+  include_examples '#options', /[a]/, [0, 0], Literal
+  include_examples '#options', /[a-z]/, [0, 0], CharacterSet::Range
+  include_examples '#options', /[a-z]/, [0, 0, 0], Literal
+  include_examples '#options', /[a&&z]/, [0, 0], CharacterSet::Intersection
+  include_examples '#options', /[a&&z]/, [0, 0, 0], CharacterSet::IntersectedSequence
+  include_examples '#options', /[a&&z]/, [0, 0, 0, 0], Literal
+  include_examples '#options', /[[:ascii:]]/, [0, 0], PosixClass
+  include_examples '#options', /\p{word}/, [0], UnicodeProperty::Base
+  include_examples '#options', /(a)(?(1)b|c)/, [1], Conditional::Expression
+  include_examples '#options', /(a)(?(1)b|c)/, [1, 0], Conditional::Condition
+  include_examples '#options', /(a)(?(1)b|c)/, [1, 1], Conditional::Branch
+  include_examples '#options', /(a)(?(1)b|c)/, [1, 1, 0], Literal
 end

data/spec/lexer/delimiters_spec.rb

@@ -0,0 +1,68 @@
+require 'spec_helper'
+
+RSpec.describe('Literal delimiter lexing') do
+  include_examples 'lex', '}',
+    0 => [:literal,     :literal,       '}',       0,  1,  0, 0, 0]
+
+  include_examples 'lex', '}}',
+    0 => [:literal,     :literal,       '}}',      0,  2,  0, 0, 0]
+
+  include_examples 'lex', '{',
+    0 => [:literal,     :literal,       '{',       0,  1,  0, 0, 0]
+
+  include_examples 'lex', '{{',
+    0 => [:literal,     :literal,       '{{',      0,  2,  0, 0, 0]
+
+  include_examples 'lex', '{}',
+    0 => [:literal,     :literal,       '{}',      0,  2,  0, 0, 0]
+
+  include_examples 'lex', '}{',
+    0 => [:literal,     :literal,       '}{',      0,  2,  0, 0, 0]
+
+  include_examples 'lex', '}{+',
+    0 => [:literal,     :literal,       '}',       0,  1,  0, 0, 0],
+    1 => [:literal,     :literal,       '{',       1,  2,  0, 0, 0],
+    2 => [:quantifier,  :one_or_more,   '+',       2,  3,  0, 0, 0]
+
+  include_examples 'lex', '{{var}}',
+    0 => [:literal,     :literal,       '{{var}}',  0,  7,  0, 0, 0]
+
+  include_examples 'lex', 'a{b}c',
+    0 => [:literal,     :literal,       'a{b}c',    0,  5,  0, 0, 0]
+
+  include_examples 'lex', 'a{1,2',
+    0 => [:literal,     :literal,       'a{1,2',    0,  5,  0, 0, 0]
+
+  include_examples 'lex', '({.+})',
+    0 => [:group,       :capture,       '(',    0,  1,  0, 0, 0],
+    1 => [:literal,     :literal,       '{',    1,  2,  1, 0, 0],
+    2 => [:meta,        :dot,           '.',    2,  3,  1, 0, 0],
+    3 => [:quantifier,  :one_or_more,   '+',    3,  4,  1, 0, 0],
+    4 => [:literal,     :literal,       '}',    4,  5,  1, 0, 0],
+    5 => [:group,       :close,         ')',    5,  6,  0, 0, 0]
+
+  include_examples 'lex', ']',
+    0 => [:literal,     :literal,       ']',        0,  1,  0, 0, 0]
+
+  include_examples 'lex', ']]',
+    0 => [:literal,     :literal,       ']]',       0,  2,  0, 0, 0]
+
+  include_examples 'lex', ']\[',
+    0 => [:literal,     :literal,       ']',        0,  1,  0, 0, 0],
+    1 => [:escape,      :set_open,      '\[',       1,  3,  0, 0, 0]
+
+  include_examples 'lex', '()',
+    0 => [:group,       :capture,       '(',        0,  1,  0, 0, 0],
+    1 => [:group,       :close,         ')',        1,  2,  0, 0, 0]
+
+  include_examples 'lex', '{abc:.+}}}[^}]]}',
+    0 => [:literal,     :literal,       '{abc:',    0,  5,  0, 0, 0],
+    1 => [:meta,        :dot,           '.',        5,  6,  0, 0, 0],
+    2 => [:quantifier,  :one_or_more,   '+',        6,  7,  0, 0, 0],
+    3 => [:literal,     :literal,       '}}}',      7,  10, 0, 0, 0],
+    4 => [:set,         :open,          '[',        10, 11, 0, 0, 0],
+    5 => [:set,         :negate,        '^',        11, 12, 0, 1, 0],
+    6 => [:literal,     :literal,       '}',        12, 13, 0, 1, 0],
+    7 => [:set,         :close,         ']',        13, 14, 0, 0, 0],
+    8 => [:literal,     :literal,       ']}',       14, 16, 0, 0, 0]
+end

data/spec/parser/options_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+RSpec.describe('passing options to parse') do
+  it 'raises if if parsing from a Regexp and options are passed' do
+    expect { RP.parse(/a+/, options: ::Regexp::EXTENDED) }.to raise_error(
+      ArgumentError,
+      'options cannot be supplied unless parsing a String'
+    )
+  end
+
+  it 'sets options if parsing from a String' do
+    root = RP.parse('a+', options: ::Regexp::MULTILINE | ::Regexp::EXTENDED)
+
+    expect(root.options).to eq(m: true, x: true)
+  end
+
+  it 'allows options to not be supplied when parsing from a Regexp' do
+    root = RP.parse(/a+/ix)
+
+    expect(root.options).to eq(i: true, x: true)
+  end
+
+  it 'has an empty option-hash when parsing from a String and passing no options' do
+    root = RP.parse('a+')
+
+    expect(root.options).to be_empty
+  end
+end

data/spec/parser/quantifiers_spec.rb

@@ -35,6 +35,7 @@ RSpec.describe('Quantifier parsing') do
   include_examples 'quantifier', /a{4}b/,    '{4}',    :greedy,     :interval,     4, 4
   include_examples 'quantifier', /a{4}?b/,   '{4}?',   :reluctant,  :interval,     4, 4
   include_examples 'quantifier', /a{4}+b/,   '{4}+',   :possessive, :interval,     4, 4
+  include_examples 'quantifier', /a{004}+b/, '{004}+', :possessive, :interval,     4, 4
 
   specify('mode-checking methods') do
     exp = RP.parse(/a??/).first

data/spec/scanner/delimiters_spec.rb

@@ -0,0 +1,52 @@
+require 'spec_helper'
+
+RSpec.describe('Literal delimiter scanning') do
+  include_examples 'scan', '}',
+    0 => [:literal,     :literal,       '}',        0,  1]
+
+  include_examples 'scan', '}}',
+    0 => [:literal,     :literal,       '}}',       0,  2]
+
+  include_examples 'scan', '{',
+    0 => [:literal,     :literal,       '{',        0,  1]
+
+  include_examples 'scan', '{{',
+    0 => [:literal,     :literal,       '{{',       0,  2]
+
+  include_examples 'scan', '{}',
+    0 => [:literal,     :literal,       '{}',       0,  2]
+
+  include_examples 'scan', '}{',
+    0 => [:literal,     :literal,       '}{',       0,  2]
+
+  include_examples 'scan', '}{+',
+    0 => [:literal,     :literal,       '}{',       0,  2]
+
+  include_examples 'scan', '{{var}}',
+    0 => [:literal,     :literal,       '{{var}}',  0,  7]
+
+  include_examples 'scan', 'a{1,2',
+    0 => [:literal,     :literal,       'a{1,2',    0,  5]
+
+  include_examples 'scan', '({.+})',
+    0 => [:group,       :capture,       '(',        0,  1],
+    1 => [:literal,     :literal,       '{',        1,  2],
+    2 => [:meta,        :dot,           '.',        2,  3],
+    3 => [:quantifier,  :one_or_more,   '+',        3,  4],
+    4 => [:literal,     :literal,       '}',        4,  5],
+    5 => [:group,       :close,         ')',        5,  6]
+
+  include_examples 'scan', ']',
+    0 => [:literal,     :literal,       ']',        0,  1]
+
+  include_examples 'scan', ']]',
+    0 => [:literal,     :literal,       ']]',       0,  2]
+
+  include_examples 'scan', ']\[',
+    0 => [:literal,     :literal,       ']',        0,  1],
+    1 => [:escape,      :set_open,      '\[',       1,  3]
+
+  include_examples 'scan', '()',
+    0 => [:group,       :capture,       '(',        0,  1],
+    1 => [:group,       :close,         ')',        1,  2]
+end