regexp_parser 1.7.1 → 1.8.2

data/lib/regexp_parser/scanner/scanner.rl

@@ -21,7 +21,7 @@
   set_close             = ']';
   brackets              = set_open | set_close;
 
-  comment               = ('#' . [^\n]* . '\n');
+  comment               = ('#' . [^\n]* . '\n'?);
 
   class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                           'cntrl' | 'digit' | 'graph' |
@@ -120,7 +120,7 @@
 
   literal_delimiters    = ']' | '}';
 
-  ascii_print           = ((0x20..0x7e) - meta_char);
+  ascii_print           = ((0x20..0x7e) - meta_char - '#');
   ascii_nonprint        = (0x01..0x1f | 0x7f);
 
   utf8_2_byte           = (0xc2..0xdf 0x80..0xbf);
@@ -128,7 +128,7 @@
   utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf);
 
   non_literal_escape    = char_type_char | anchor_char | escaped_ascii |
-                          group_ref | keep_mark | [xucCM];
+                          keep_mark | [xucCM];
 
   non_set_escape        = (anchor_char - 'b') | group_ref | keep_mark |
                           multi_codepoint_char_type | [0-9cCM];
@@ -649,7 +649,9 @@
       if free_spacing
         emit(:free_space, :comment, *text(data, ts, te))
       else
-        append_literal(data, ts, te)
+        # consume only the pound sign (#) and backtrack to do regular scanning
+        append_literal(data, ts, ts + 1)
+        fexec ts + 1;
       end
     };
 
@@ -737,21 +739,16 @@ class Regexp::Scanner
   #
   # This method may raise errors if a syntax error is encountered.
   # --------------------------------------------------------------------------
-  def self.scan(input_object, &block)
-    new.scan(input_object, &block)
+  def self.scan(input_object, options: nil, &block)
+    new.scan(input_object, options: options, &block)
   end
 
-  def scan(input_object, &block)
+  def scan(input_object, options: nil, &block)
     self.literal = nil
     stack = []
 
-    if input_object.is_a?(Regexp)
-      input = input_object.source
-      self.free_spacing = (input_object.options & Regexp::EXTENDED != 0)
-    else
-      input = input_object
-      self.free_spacing = false
-    end
+    input = input_object.is_a?(Regexp) ? input_object.source : input_object
+    self.free_spacing = free_spacing?(input_object, options)
     self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]
 
     data  = input.unpack("c*") if input.is_a?(String)
@@ -817,6 +814,18 @@ class Regexp::Scanner
   attr_accessor :tokens, :literal, :block, :free_spacing, :spacing_stack,
                 :group_depth, :set_depth, :conditional_stack
 
+  def free_spacing?(input_object, options)
+    if options && !input_object.is_a?(String)
+      raise ArgumentError, 'options cannot be supplied unless scanning a String'
+    end
+
+    options = input_object.options if input_object.is_a?(::Regexp)
+
+    return false unless options
+
+    options & Regexp::EXTENDED != 0
+  end
+
   def in_group?
     group_depth > 0
   end

data/lib/regexp_parser/syntax/version_lookup.rb

@@ -74,9 +74,9 @@ module Regexp::Syntax
   end
 
   def warn_if_future_version(const_name)
-    return if comparable_version(const_name) < comparable_version('3.0.0')
+    return if comparable_version(const_name) < comparable_version('4.0.0')
 
-    warn('This library has only been tested up to Ruby 2.x, '\
+    warn('This library has only been tested up to Ruby 3.x, '\
          "but you are running with #{const_get(const_name).inspect}")
   end
 end

data/lib/regexp_parser/version.rb

@@ -1,5 +1,5 @@
 class Regexp
   class Parser
-    VERSION = '1.7.1'
+    VERSION = '1.8.2'
   end
 end

data/regexp_parser.gemspec

@@ -32,5 +32,5 @@ Gem::Specification.new do |gem|
 
   gem.platform      = Gem::Platform::RUBY
 
-  gem.required_ruby_version = '>= 1.9.1'
+  gem.required_ruby_version = '>= 2.0.0'
 end

data/spec/parser/options_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+RSpec.describe('passing options to parse') do
+  it 'raises if if parsing from a Regexp and options are passed' do
+    expect { RP.parse(/a+/, options: ::Regexp::EXTENDED) }.to raise_error(
+      ArgumentError,
+      'options cannot be supplied unless parsing a String'
+    )
+  end
+
+  it 'sets options if parsing from a String' do
+    root = RP.parse('a+', options: ::Regexp::MULTILINE | ::Regexp::EXTENDED)
+
+    expect(root.options).to eq(m: true, x: true)
+  end
+
+  it 'allows options to not be supplied when parsing from a Regexp' do
+    root = RP.parse(/a+/ix)
+
+    expect(root.options).to eq(i: true, x: true)
+  end
+
+  it 'has an empty option-hash when parsing from a String and passing no options' do
+    root = RP.parse('a+')
+
+    expect(root.options).to be_empty
+  end
+end

data/spec/scanner/escapes_spec.rb

@@ -13,6 +13,10 @@ RSpec.describe('Escape scanning') do
 
   include_examples 'scan', 'c\qt',            1 => [:escape,  :literal,          '\q',             1,  3]
 
+  # these incomplete ref/call sequences are treated as literal escapes by Ruby
+  include_examples 'scan', 'c\gt',            1 => [:escape,  :literal,          '\g',             1,  3]
+  include_examples 'scan', 'c\kt',            1 => [:escape,  :literal,          '\k',             1,  3]
+
   include_examples 'scan', 'a\012c',          1 => [:escape,  :octal,            '\012',           1,  5]
   include_examples 'scan', 'a\0124',          1 => [:escape,  :octal,            '\012',           1,  5]
   include_examples 'scan', '\712+7',          0 => [:escape,  :octal,            '\712',           0,  4]

data/spec/scanner/free_space_spec.rb

@@ -39,6 +39,17 @@ RSpec.describe('FreeSpace scanning') do
       11 => [:free_space, :comment,  "# B ? comment\n",      37,  51],
       17 => [:free_space, :comment,  "# C {2,3} comment\n",  66,  84],
       29 => [:free_space, :comment,  "# D|E comment\n",     100, 114]
+
+    # single line / no trailing newline (c.f. issue #66)
+    include_examples 'scan', /a # b/x,
+      0 => [:literal,    :literal,    'a',   0,  1],
+      1 => [:free_space, :whitespace, ' ',   1,  2],
+      2 => [:free_space, :comment,    "# b", 2,  5]
+
+    # without spaces (c.f. issue #66)
+    include_examples 'scan', /a#b/x,
+      0 => [:literal,    :literal,  'a',   0,  1],
+      1 => [:free_space, :comment,  "#b", 1,  3]
   end
 
   describe('scan free space inlined') do
@@ -130,4 +141,25 @@ RSpec.describe('FreeSpace scanning') do
       26 => [:literal,    :literal,        'i j',   35, 38],
       27 => [:group,      :close,          ')',     38, 39]
   end
+
+  describe('scanning `#` in regular (non-x mode)') do # c.f. issue 70
+    include_examples 'scan', /a#bcd/,
+      0 => [:literal, :literal, 'a#bcd', 0, 5]
+    include_examples 'scan', /a # bcd/,
+      0 => [:literal, :literal, 'a # bcd', 0, 7]
+
+    include_examples 'scan', /a#\d/,
+      0 => [:literal, :literal, 'a#', 0, 2],
+      1 => [:type,    :digit,   '\d', 2, 4]
+    include_examples 'scan', /a # \d/,
+      0 => [:literal, :literal, 'a # ', 0, 4],
+      1 => [:type,    :digit,   '\d',   4, 6]
+
+    include_examples 'scan', /a#()/,
+      0 => [:literal, :literal, 'a#', 0, 2],
+      1 => [:group,   :capture, '(',  2, 3]
+    include_examples 'scan', /a # ()/,
+      0 => [:literal, :literal, 'a # ', 0, 4],
+      1 => [:group,   :capture, '(',    4, 5]
+  end
 end

data/spec/scanner/options_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+RSpec.describe('passing options to scan') do
+  def expect_type_tokens(tokens, type_tokens)
+    expect(tokens.map { |type, token, *| [type, token] }).to eq(type_tokens)
+  end
+
+  it 'raises if if scanning from a Regexp and options are passed' do
+    expect { RS.scan(/a+/, options: ::Regexp::EXTENDED) }.to raise_error(
+      ArgumentError,
+      'options cannot be supplied unless scanning a String'
+    )
+  end
+
+  it 'sets free_spacing based on options if scanning from a String' do
+    expect_type_tokens(
+      RS.scan('a+#c', options: ::Regexp::MULTILINE | ::Regexp::EXTENDED),
+      [
+        %i[literal literal],
+        %i[quantifier one_or_more],
+        %i[free_space comment]
+      ]
+    )
+  end
+
+  it 'does not set free_spacing if scanning from a String and passing no options' do
+    expect_type_tokens(
+      RS.scan('a+#c'),
+      [
+        %i[literal literal],
+        %i[quantifier one_or_more],
+        %i[literal literal]
+      ]
+    )
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: regexp_parser
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.8.2
 platform: ruby
 authors:
 - Ammar Ali
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-06-07 00:00:00.000000000 Z
+date: 2020-10-11 00:00:00.000000000 Z
 dependencies: []
 description: A library for tokenizing, lexing, and parsing Ruby regular expressions.
 email:
@@ -124,6 +124,7 @@ files:
 - spec/parser/free_space_spec.rb
 - spec/parser/groups_spec.rb
 - spec/parser/keep_spec.rb
+- spec/parser/options_spec.rb
 - spec/parser/posix_classes_spec.rb
 - spec/parser/properties_spec.rb
 - spec/parser/quantifiers_spec.rb
@@ -143,6 +144,7 @@ files:
 - spec/scanner/keep_spec.rb
 - spec/scanner/literals_spec.rb
 - spec/scanner/meta_spec.rb
+- spec/scanner/options_spec.rb
 - spec/scanner/properties_spec.rb
 - spec/scanner/quantifiers_spec.rb
 - spec/scanner/refcalls_spec.rb
@@ -176,84 +178,86 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.1
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.2.0.rc.1
 signing_key:
 specification_version: 4
 summary: Scanner, lexer, parser for ruby's regular expressions
 test_files:
-- spec/token/token_spec.rb
-- spec/spec_helper.rb
-- spec/lexer/escapes_spec.rb
-- spec/lexer/keep_spec.rb
+- spec/expression/base_spec.rb
+- spec/expression/clone_spec.rb
+- spec/expression/conditional_spec.rb
+- spec/expression/free_space_spec.rb
+- spec/expression/methods/match_length_spec.rb
+- spec/expression/methods/match_spec.rb
+- spec/expression/methods/strfregexp_spec.rb
+- spec/expression/methods/tests_spec.rb
+- spec/expression/methods/traverse_spec.rb
+- spec/expression/options_spec.rb
+- spec/expression/root_spec.rb
+- spec/expression/sequence_spec.rb
+- spec/expression/subexpression_spec.rb
+- spec/expression/to_h_spec.rb
+- spec/expression/to_s_spec.rb
 - spec/lexer/all_spec.rb
 - spec/lexer/conditionals_spec.rb
-- spec/lexer/nesting_spec.rb
 - spec/lexer/delimiters_spec.rb
-- spec/lexer/refcalls_spec.rb
+- spec/lexer/escapes_spec.rb
+- spec/lexer/keep_spec.rb
 - spec/lexer/literals_spec.rb
-- spec/parser/escapes_spec.rb
-- spec/parser/properties_spec.rb
-- spec/parser/sets_spec.rb
-- spec/parser/free_space_spec.rb
-- spec/parser/keep_spec.rb
+- spec/lexer/nesting_spec.rb
+- spec/lexer/refcalls_spec.rb
 - spec/parser/all_spec.rb
-- spec/parser/conditionals_spec.rb
-- spec/parser/types_spec.rb
-- spec/parser/anchors_spec.rb
 - spec/parser/alternation_spec.rb
-- spec/parser/posix_classes_spec.rb
-- spec/parser/set/ranges_spec.rb
-- spec/parser/set/intersections_spec.rb
+- spec/parser/anchors_spec.rb
+- spec/parser/conditionals_spec.rb
 - spec/parser/errors_spec.rb
-- spec/parser/refcalls_spec.rb
+- spec/parser/escapes_spec.rb
+- spec/parser/free_space_spec.rb
 - spec/parser/groups_spec.rb
+- spec/parser/keep_spec.rb
+- spec/parser/options_spec.rb
+- spec/parser/posix_classes_spec.rb
+- spec/parser/properties_spec.rb
 - spec/parser/quantifiers_spec.rb
-- spec/support/warning_extractor.rb
-- spec/support/shared_examples.rb
-- spec/support/runner.rb
-- spec/expression/subexpression_spec.rb
-- spec/expression/methods/match_spec.rb
-- spec/expression/methods/match_length_spec.rb
-- spec/expression/methods/traverse_spec.rb
-- spec/expression/methods/strfregexp_spec.rb
-- spec/expression/methods/tests_spec.rb
-- spec/expression/free_space_spec.rb
-- spec/expression/options_spec.rb
-- spec/expression/to_s_spec.rb
-- spec/expression/root_spec.rb
-- spec/expression/sequence_spec.rb
-- spec/expression/clone_spec.rb
-- spec/expression/to_h_spec.rb
-- spec/expression/conditional_spec.rb
-- spec/expression/base_spec.rb
-- spec/syntax/syntax_spec.rb
-- spec/syntax/syntax_token_map_spec.rb
-- spec/syntax/versions/1.9.3_spec.rb
-- spec/syntax/versions/2.2.0_spec.rb
-- spec/syntax/versions/1.9.1_spec.rb
-- spec/syntax/versions/2.0.0_spec.rb
-- spec/syntax/versions/1.8.6_spec.rb
-- spec/syntax/versions/aliases_spec.rb
-- spec/scanner/escapes_spec.rb
-- spec/scanner/properties_spec.rb
-- spec/scanner/sets_spec.rb
-- spec/scanner/free_space_spec.rb
-- spec/scanner/keep_spec.rb
+- spec/parser/refcalls_spec.rb
+- spec/parser/set/intersections_spec.rb
+- spec/parser/set/ranges_spec.rb
+- spec/parser/sets_spec.rb
+- spec/parser/types_spec.rb
 - spec/scanner/all_spec.rb
-- spec/scanner/conditionals_spec.rb
-- spec/scanner/types_spec.rb
 - spec/scanner/anchors_spec.rb
-- spec/scanner/meta_spec.rb
-- spec/scanner/errors_spec.rb
+- spec/scanner/conditionals_spec.rb
 - spec/scanner/delimiters_spec.rb
-- spec/scanner/refcalls_spec.rb
+- spec/scanner/errors_spec.rb
+- spec/scanner/escapes_spec.rb
+- spec/scanner/free_space_spec.rb
 - spec/scanner/groups_spec.rb
+- spec/scanner/keep_spec.rb
 - spec/scanner/literals_spec.rb
+- spec/scanner/meta_spec.rb
+- spec/scanner/options_spec.rb
+- spec/scanner/properties_spec.rb
 - spec/scanner/quantifiers_spec.rb
+- spec/scanner/refcalls_spec.rb
+- spec/scanner/sets_spec.rb
+- spec/scanner/types_spec.rb
+- spec/spec_helper.rb
+- spec/support/runner.rb
+- spec/support/shared_examples.rb
+- spec/support/warning_extractor.rb
+- spec/syntax/syntax_spec.rb
+- spec/syntax/syntax_token_map_spec.rb
+- spec/syntax/versions/1.8.6_spec.rb
+- spec/syntax/versions/1.9.1_spec.rb
+- spec/syntax/versions/1.9.3_spec.rb
+- spec/syntax/versions/2.0.0_spec.rb
+- spec/syntax/versions/2.2.0_spec.rb
+- spec/syntax/versions/aliases_spec.rb
+- spec/token/token_spec.rb