regexp_parser 2.0.0 → 2.1.1

data/lib/regexp_parser/scanner/scanner.rl

@@ -3,6 +3,11 @@
   include re_char_type "char_type.rl";
   include re_property  "property.rl";
 
+  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf);
+  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf);
+  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf);
+  utf8_multibyte        = utf8_2_byte | utf8_3_byte | utf8_4_byte;
+
   dot                   = '.';
   backslash             = '\\';
   alternation           = '|';
@@ -15,7 +20,7 @@
 
   group_open            = '(';
   group_close           = ')';
-  parantheses           = group_open | group_close;
+  parentheses           = group_open | group_close;
 
   set_open              = '[';
   set_close             = ']';
@@ -32,7 +37,7 @@
   class_posix           = ('[:' . '^'? . class_name_posix . ':]');
 
 
-  # these are not supported in ruby, and need verification
+  # these are not supported in ruby at the moment
   collating_sequence    = '[.' . (alpha | [\-])+ . '.]';
   character_equivalent  = '[=' . alpha . '=]';
 
@@ -53,6 +58,8 @@
 
   meta_sequence         = 'M-' . (backslash . ('c' | 'C-'))? . backslash? . any;
 
+  sequence_char         = [CMcux];
+
   zero_or_one           = '?' | '??' | '?+';
   zero_or_more          = '*' | '*?' | '*+';
   one_or_more           = '+' | '+?' | '++';
@@ -90,21 +97,26 @@
   group_options         = '?' . ( [^!#'():<=>~]+ . ':'? ) ?;
 
   group_ref             = [gk];
-  group_name_char       = (alnum | '_');
-  group_name_id         = (group_name_char . (group_name_char+)?)?;
-  group_number          = '-'? . [1-9] . ([0-9]+)?;
+  group_name_id_ab      = ([^0-9\->] | utf8_multibyte) . ([^>] | utf8_multibyte)*;
+  group_name_id_sq      = ([^0-9\-'] | utf8_multibyte) . ([^'] | utf8_multibyte)*;
+  group_number          = '-'? . [1-9] . [0-9]*;
   group_level           = [+\-] . [0-9]+;
 
-  group_name            = ('<' . group_name_id . '>') | ("'" . group_name_id . "'");
+  group_name            = ('<' . group_name_id_ab? . '>') |
+                          ("'" . group_name_id_sq? . "'");
   group_lookup          = group_name | group_number;
 
   group_named           = ('?' . group_name );
 
-  group_name_ref        = group_ref . (('<' . group_name_id . group_level? '>') |
-                                       ("'" . group_name_id . group_level? "'"));
+  group_name_backref    = 'k' . (('<' . group_name_id_ab? . group_level? '>') |
+                                 ("'" . group_name_id_sq? . group_level? "'"));
+  group_name_call       = 'g' . (('<' . group_name_id_ab? . group_level? '>') |
+                                 ("'" . group_name_id_sq? . group_level? "'"));
 
-  group_number_ref      = group_ref . (('<' . group_number . group_level? '>') |
-                                       ("'" . group_number . group_level? "'"));
+  group_number_backref  = 'k' . (('<' . group_number . group_level? '>') |
+                                 ("'" . group_number . group_level? "'"));
+  group_number_call     = 'g' . (('<' . ((group_number . group_level?) | '0') '>') |
+                                 ("'" . ((group_number . group_level?) | '0') "'"));
 
   group_type            = group_atomic | group_passive | group_absence | group_named;
 
@@ -115,7 +127,7 @@
 
   # characters that 'break' a literal
   meta_char             = dot | backslash | alternation |
-                          curlies | parantheses | brackets |
+                          curlies | parentheses | brackets |
                           line_anchor | quantifier_greedy;
 
   literal_delimiters    = ']' | '}';
@@ -123,15 +135,13 @@
   ascii_print           = ((0x20..0x7e) - meta_char - '#');
   ascii_nonprint        = (0x01..0x1f | 0x7f);
 
-  utf8_2_byte           = (0xc2..0xdf 0x80..0xbf);
-  utf8_3_byte           = (0xe0..0xef 0x80..0xbf 0x80..0xbf);
-  utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf);
-
   non_literal_escape    = char_type_char | anchor_char | escaped_ascii |
-                          keep_mark | [xucCM];
+                          keep_mark | sequence_char;
+
+  # escapes that also work within a character set
+  set_escape            = backslash | brackets | escaped_ascii | property_char |
+                          sequence_char | single_codepoint_char_type;
 
-  non_set_escape        = (anchor_char - 'b') | group_ref | keep_mark |
-                          multi_codepoint_char_type | [0-9cCM];
 
   # EOF error, used where it can be detected
   action premature_end_error {
@@ -226,23 +236,19 @@
       emit(type, class_name.to_sym, text)
     };
 
-    collating_sequence >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
-      emit(:set, :collation, copy(data, ts, te))
-    };
-
-    character_equivalent >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
-      emit(:set, :equivalent, copy(data, ts, te))
-    };
+    # These are not supported in ruby at the moment. Enable them if they are.
+    # collating_sequence >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
+    #   emit(:set, :collation, copy(data, ts, te))
+    # };
+    # character_equivalent >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
+    #   emit(:set, :equivalent, copy(data, ts, te))
+    # };
 
     meta_char > (set_meta, 1) {
       emit(:literal, :literal, copy(data, ts, te))
     };
 
-    any            |
-    ascii_nonprint |
-    utf8_2_byte    |
-    utf8_3_byte    |
-    utf8_4_byte    {
+    any | ascii_nonprint | utf8_multibyte {
       text = copy(data, ts, te)
       emit(:literal, :literal, text)
     };
@@ -251,16 +257,16 @@
   # set escapes scanner
   # --------------------------------------------------------------------------
   set_escape_sequence := |*
-    non_set_escape > (escaped_set_alpha, 2) {
-      emit(:escape, :literal, copy(data, ts-1, te))
-      fret;
-    };
-
-    any > (escaped_set_alpha, 1) {
+    set_escape > (escaped_set_alpha, 2) {
       fhold;
       fnext character_set;
       fcall escape_sequence;
     };
+
+    any > (escaped_set_alpha, 1) {
+      emit(:escape, :literal, copy(data, ts-1, te))
+      fret;
+    };
   *|;
 
 
@@ -325,7 +331,7 @@
       fret;
     };
 
-    hex_sequence > (escaped_alpha, 5) $eof(premature_end_error) {
+    hex_sequence > (escaped_alpha, 5) @eof(premature_end_error) {
       emit(:escape, :hex, copy(data, ts-1, te))
       fret;
     };
@@ -356,10 +362,7 @@
       fcall unicode_property;
     };
 
-    (any -- non_literal_escape) |
-    utf8_2_byte                 |
-    utf8_3_byte                 |
-    utf8_4_byte                   > (escaped_alpha, 1) {
+    (any -- non_literal_escape) | utf8_multibyte > (escaped_alpha, 1) {
       emit(:escape, :literal, copy(data, ts-1, te))
       fret;
     };
@@ -511,10 +514,10 @@
       when /^\(\?(?:<>|'')/
         validation_error(:group, 'named group', 'name is empty')
 
-      when /^\(\?<\w*>/
+      when /^\(\?<[^>]+>/
         emit(:group, :named_ab,  text)
 
-      when /^\(\?'\w*'/
+      when /^\(\?'[^']+'/
         emit(:group, :named_sq,  text)
 
       end
@@ -543,65 +546,35 @@
 
     # Group backreference, named and numbered
     # ------------------------------------------------------------------------
-    backslash . (group_name_ref | group_number_ref) > (backslashed, 4) {
+    backslash . (group_name_backref | group_number_backref) > (backslashed, 4) {
       case text = copy(data, ts, te)
-      when /^\\([gk])(<>|'')/ # angle brackets
-        validation_error(:backref, 'ref/call', 'ref ID is empty')
-
-      when /^\\([gk])<[^\d+-]\w*>/ # angle-brackets
-        if $1 == 'k'
-          emit(:backref, :name_ref_ab, text)
-        else
-          emit(:backref, :name_call_ab, text)
-        end
-
-      when /^\\([gk])'[^\d+-]\w*'/ #single quotes
-        if $1 == 'k'
-          emit(:backref, :name_ref_sq, text)
-        else
-          emit(:backref, :name_call_sq, text)
-        end
-
-      when /^\\([gk])<\d+>/ # angle-brackets
-        if $1 == 'k'
-          emit(:backref, :number_ref_ab, text)
-        else
-          emit(:backref, :number_call_ab, text)
-        end
-
-      when /^\\([gk])'\d+'/ # single quotes
-        if $1 == 'k'
-          emit(:backref, :number_ref_sq, text)
-        else
-          emit(:backref, :number_call_sq, text)
-        end
-
-      when /^\\(?:g<\+|g<-|(k)<-)\d+>/ # angle-brackets
-        if $1 == 'k'
-          emit(:backref, :number_rel_ref_ab, text)
-        else
-          emit(:backref, :number_rel_call_ab, text)
-        end
-
-      when /^\\(?:g'\+|g'-|(k)'-)\d+'/ # single quotes
-        if $1 == 'k'
-          emit(:backref, :number_rel_ref_sq, text)
-        else
-          emit(:backref, :number_rel_call_sq, text)
-        end
-
-      when /^\\k<[^\d+\-]\w*[+\-]\d+>/ # angle-brackets
-        emit(:backref, :name_recursion_ref_ab, text)
-
-      when /^\\k'[^\d+\-]\w*[+\-]\d+'/ # single-quotes
-        emit(:backref, :name_recursion_ref_sq, text)
-
-      when /^\\([gk])<[+\-]?\d+[+\-]\d+>/ # angle-brackets
-        emit(:backref, :number_recursion_ref_ab, text)
-
-      when /^\\([gk])'[+\-]?\d+[+\-]\d+'/ # single-quotes
-        emit(:backref, :number_recursion_ref_sq, text)
+      when /^\\k(<>|'')/
+        validation_error(:backref, 'backreference', 'ref ID is empty')
+      when /^\\k(.)[^\p{digit}\-][^+\-]*\D$/
+        emit(:backref, $1 == '<' ? :name_ref_ab : :name_ref_sq, text)
+      when /^\\k(.)\d+\D$/
+        emit(:backref, $1 == '<' ? :number_ref_ab : :number_ref_sq, text)
+      when /^\\k(.)-\d+\D$/
+        emit(:backref, $1 == '<' ? :number_rel_ref_ab : :number_rel_ref_sq, text)
+      when /^\\k(.)[^\p{digit}\-].*[+\-]\d+\D$/
+        emit(:backref, $1 == '<' ? :name_recursion_ref_ab : :name_recursion_ref_sq, text)
+      when /^\\k(.)-?\d+[+\-]\d+\D$/
+        emit(:backref, $1 == '<' ? :number_recursion_ref_ab : :number_recursion_ref_sq, text)
+      end
+    };
 
+    # Group call, named and numbered
+    # ------------------------------------------------------------------------
+    backslash . (group_name_call | group_number_call) > (backslashed, 4) {
+      case text = copy(data, ts, te)
+      when /^\\g(<>|'')/
+        validation_error(:backref, 'subexpression call', 'ref ID is empty')
+      when /^\\g(.)[^\p{digit}+\->][^+\-]*/
+        emit(:backref, $1 == '<' ? :name_call_ab : :name_call_sq, text)
+      when /^\\g(.)\d+\D$/
+        emit(:backref, $1 == '<' ? :number_call_ab : :number_call_sq, text)
+      when /^\\g(.)[+-]\d+/
+        emit(:backref, $1 == '<' ? :number_rel_call_ab : :number_rel_call_sq, text)
       end
     };
 
@@ -668,11 +641,7 @@
     # Literal: any run of ASCII (pritable or non-printable), and/or UTF-8,
     # except meta characters.
     # ------------------------------------------------------------------------
-    (ascii_print -- space)+    |
-    ascii_nonprint+ |
-    utf8_2_byte+    |
-    utf8_3_byte+    |
-    utf8_4_byte+    {
+    (ascii_print -- space)+ | ascii_nonprint+ | utf8_multibyte+ {
       append_literal(data, ts, te)
     };
 
@@ -682,12 +651,14 @@
 # THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
 # This file was generated from lib/regexp_parser/scanner/scanner.rl
 
+require 'regexp_parser/error'
+
 class Regexp::Scanner
   # General scanner error (catch all)
-  class ScannerError < StandardError; end
+  class ScannerError < Regexp::Parser::Error; end
 
   # Base for all scanner validation errors
-  class ValidationError < StandardError
+  class ValidationError < Regexp::Parser::Error
     def initialize(reason)
       super reason
     end
@@ -789,14 +760,13 @@ class Regexp::Scanner
 
   # lazy-load property maps when first needed
   require 'yaml'
-  PROP_MAPS_DIR = File.expand_path('../scanner/properties', __FILE__)
 
   def self.short_prop_map
-    @short_prop_map ||= YAML.load_file("#{PROP_MAPS_DIR}/short.yml")
+    @short_prop_map ||= YAML.load_file("#{__dir__}/scanner/properties/short.yml")
   end
 
   def self.long_prop_map
-    @long_prop_map ||= YAML.load_file("#{PROP_MAPS_DIR}/long.yml")
+    @long_prop_map ||= YAML.load_file("#{__dir__}/scanner/properties/long.yml")
   end
 
   # Emits an array with the details of the scanned pattern

data/lib/regexp_parser/syntax.rb

@@ -1,9 +1,11 @@
-require File.expand_path('../syntax/tokens', __FILE__)
-require File.expand_path('../syntax/base', __FILE__)
-require File.expand_path('../syntax/any', __FILE__)
-require File.expand_path('../syntax/version_lookup', __FILE__)
-require File.expand_path('../syntax/versions', __FILE__)
+require 'regexp_parser/error'
 
 module Regexp::Syntax
-  class SyntaxError < StandardError; end
+  class SyntaxError < Regexp::Parser::Error; end
 end
+
+require_relative 'syntax/tokens'
+require_relative 'syntax/base'
+require_relative 'syntax/any'
+require_relative 'syntax/version_lookup'
+require_relative 'syntax/versions'

data/lib/regexp_parser/syntax/any.rb

@@ -4,12 +4,12 @@ module Regexp::Syntax
   # is useful during development, testing, and should be useful for some types
   # of transformations as well.
   class Any < Base
-    def initialize
+    def initialize # rubocop:disable Lint/MissingSuper
       @implements = { :* => [:*] }
     end
 
-    def implements?(type, token) true end
-    def implements!(type, token) true end
+    def implements?(_type, _token) true end
+    def implements!(_type, _token) true end
   end
 
 end

data/lib/regexp_parser/syntax/base.rb

@@ -1,7 +1,7 @@
 require 'set'
 
 module Regexp::Syntax
-  class NotImplementedError < SyntaxError
+  class NotImplementedError < Regexp::Syntax::SyntaxError
     def initialize(syntax, type, token)
       super "#{syntax.class.name} does not implement: [#{type}:#{token}]"
     end

data/lib/regexp_parser/syntax/version_lookup.rb

@@ -3,13 +3,13 @@ module Regexp::Syntax
   VERSION_REGEXP = /#{VERSION_FORMAT}/
   VERSION_CONST_REGEXP = /\AV\d+_\d+(?:_\d+)?\z/
 
-  class InvalidVersionNameError < SyntaxError
+  class InvalidVersionNameError < Regexp::Syntax::SyntaxError
     def initialize(name)
       super "Invalid version name '#{name}'. Expected format is '#{VERSION_FORMAT}'"
     end
   end
 
-  class UnknownSyntaxNameError < SyntaxError
+  class UnknownSyntaxNameError < Regexp::Syntax::SyntaxError
     def initialize(name)
       super "Unknown syntax name '#{name}'."
     end

data/lib/regexp_parser/version.rb

@@ -1,5 +1,5 @@
 class Regexp
   class Parser
-    VERSION = '2.0.0'
+    VERSION = '2.1.1'
   end
 end

data/spec/expression/clone_spec.rb

@@ -27,8 +27,8 @@ RSpec.describe('Expression#clone') do
     expect(root_2.quantifier.object_id).not_to eq copy_2.quantifier.object_id
 
     # regression test
-    expect { root_2.clone }.not_to change { root_2.quantifier.object_id }
-    expect { root_2.clone }.not_to change { root_2.quantifier.text.object_id }
+    expect { root_2.clone }.not_to(change { root_2.quantifier.object_id })
+    expect { root_2.clone }.not_to(change { root_2.quantifier.text.object_id })
   end
 
   specify('Subexpression#clone') do
@@ -48,7 +48,7 @@ RSpec.describe('Expression#clone') do
     end
 
     # regression test
-    expect { root.clone }.not_to change { root.expressions.object_id }
+    expect { root.clone }.not_to(change { root.expressions.object_id })
   end
 
   specify('Group::Named#clone') do
@@ -69,7 +69,39 @@ RSpec.describe('Expression#clone') do
     end
 
     # regression test
-    expect { root_1.clone }.not_to change { root_1.name.object_id }
+    expect { root_1.clone }.not_to(change { root_1.name.object_id })
+  end
+
+  specify('Group::Options#clone') do
+    root = RP.parse('foo(?i)bar')
+    copy = root.clone
+
+    expect(copy.to_s).to eq root.to_s
+
+    root_1 = root[1]
+    copy_1 = copy[1]
+
+    expect(root_1.option_changes).to eq copy_1.option_changes
+    expect(root_1.option_changes.object_id).not_to eq copy_1.option_changes.object_id
+
+    # regression test
+    expect { root_1.clone }.not_to(change { root_1.option_changes.object_id })
+  end
+
+  specify('Backreference::Base#clone') do
+    root = RP.parse('(foo)\1')
+    copy = root.clone
+
+    expect(copy.to_s).to eq root.to_s
+
+    root_1 = root[1]
+    copy_1 = copy[1]
+
+    expect(root_1.referenced_expression.to_s).to eq copy_1.referenced_expression.to_s
+    expect(root_1.referenced_expression.object_id).not_to eq copy_1.referenced_expression.object_id
+
+    # regression test
+    expect { root_1.clone }.not_to(change { root_1.referenced_expression.object_id })
   end
 
   specify('Sequence#clone') do