regexp_parser 1.5.1 → 1.8.1

data/lib/regexp_parser/scanner/properties/long.yml

@@ -6,6 +6,8 @@ adlam: adlam
 age=1.1: age=1.1
 age=10.0: age=10.0
 age=11.0: age=11.0
+age=12.0: age=12.0
+age=12.1: age=12.1
 age=2.0: age=2.0
 age=2.1: age=2.1
 age=3.0: age=3.0
@@ -64,7 +66,6 @@ changeswhenuppercased: changes_when_uppercased
 cherokee: cherokee
 closepunctuation: close_punctuation
 cntrl: cntrl
-combiningmark: combining_mark
 common: common
 connectorpunctuation: connector_punctuation
 control: control
@@ -86,6 +87,7 @@ dogra: dogra
 duployan: duployan
 egyptianhieroglyphs: egyptian_hieroglyphs
 elbasan: elbasan
+elymaic: elymaic
 emoji: emoji
 emojicomponent: emoji_component
 emojimodifier: emoji_modifier
@@ -206,8 +208,10 @@ indogra: in_dogra
 indominotiles: in_domino_tiles
 induployan: in_duployan
 inearlydynasticcuneiform: in_early_dynastic_cuneiform
+inegyptianhieroglyphformatcontrols: in_egyptian_hieroglyph_format_controls
 inegyptianhieroglyphs: in_egyptian_hieroglyphs
 inelbasan: in_elbasan
+inelymaic: in_elymaic
 inemoticons: in_emoticons
 inenclosedalphanumerics: in_enclosed_alphanumerics
 inenclosedalphanumericsupplement: in_enclosed_alphanumeric_supplement
@@ -322,12 +326,14 @@ inmyanmar: in_myanmar
 inmyanmarextendeda: in_myanmar_extended_a
 inmyanmarextendedb: in_myanmar_extended_b
 innabataean: in_nabataean
+innandinagari: in_nandinagari
 innewa: in_newa
 innewtailue: in_new_tai_lue
 innko: in_nko
 innoblock: in_no_block
 innumberforms: in_number_forms
 innushu: in_nushu
+innyiakengpuachuehmong: in_nyiakeng_puachue_hmong
 inogham: in_ogham
 inolchiki: in_ol_chiki
 inoldhungarian: in_old_hungarian
@@ -343,6 +349,7 @@ inoriya: in_oriya
 inornamentaldingbats: in_ornamental_dingbats
 inosage: in_osage
 inosmanya: in_osmanya
+inottomansiyaqnumbers: in_ottoman_siyaq_numbers
 inpahawhhmong: in_pahawh_hmong
 inpalmyrene: in_palmyrene
 inpaucinhau: in_pau_cin_hau
@@ -368,6 +375,7 @@ insiddham: in_siddham
 insinhala: in_sinhala
 insinhalaarchaicnumbers: in_sinhala_archaic_numbers
 insmallformvariants: in_small_form_variants
+insmallkanaextension: in_small_kana_extension
 insogdian: in_sogdian
 insorasompeng: in_sora_sompeng
 insoyombo: in_soyombo
@@ -386,6 +394,7 @@ insupplementaryprivateuseareaa: in_supplementary_private_use_area_a
 insupplementaryprivateuseareab: in_supplementary_private_use_area_b
 insuttonsignwriting: in_sutton_signwriting
 insylotinagri: in_syloti_nagri
+insymbolsandpictographsextendeda: in_symbols_and_pictographs_extended_a
 insyriac: in_syriac
 insyriacsupplement: in_syriac_supplement
 intagalog: in_tagalog
@@ -397,6 +406,7 @@ intaiviet: in_tai_viet
 intaixuanjingsymbols: in_tai_xuan_jing_symbols
 intakri: in_takri
 intamil: in_tamil
+intamilsupplement: in_tamil_supplement
 intangut: in_tangut
 intangutcomponents: in_tangut_components
 intelugu: in_telugu
@@ -414,6 +424,7 @@ invariationselectors: in_variation_selectors
 invariationselectorssupplement: in_variation_selectors_supplement
 invedicextensions: in_vedic_extensions
 inverticalforms: in_vertical_forms
+inwancho: in_wancho
 inwarangciti: in_warang_citi
 inyijinghexagramsymbols: in_yijing_hexagram_symbols
 inyiradicals: in_yi_radicals
@@ -469,6 +480,7 @@ mro: mro
 multani: multani
 myanmar: myanmar
 nabataean: nabataean
+nandinagari: nandinagari
 newa: newa
 newline: newline
 newtailue: new_tai_lue
@@ -477,6 +489,7 @@ noncharactercodepoint: noncharacter_code_point
 nonspacingmark: nonspacing_mark
 number: number
 nushu: nushu
+nyiakengpuachuehmong: nyiakeng_puachue_hmong
 ogham: ogham
 olchiki: ol_chiki
 oldhungarian: old_hungarian
@@ -569,6 +582,7 @@ uppercase: uppercase
 uppercaseletter: uppercase_letter
 vai: vai
 variationselector: variation_selector
+wancho: wancho
 warangciti: warang_citi
 whitespace: white_space
 word: word

data/lib/regexp_parser/scanner/properties/short.yml

@@ -31,6 +31,7 @@ cher: cherokee
 ci: case_ignorable
 cn: unassigned
 co: private_use
+combiningmark: mark
 copt: coptic
 cprt: cypriot
 cs: surrogate
@@ -49,6 +50,7 @@ dsrt: deseret
 dupl: duployan
 egyp: egyptian_hieroglyphs
 elba: elbasan
+elym: elymaic
 ethi: ethiopic
 ext: extender
 geor: georgian
@@ -72,6 +74,7 @@ hex: hex_digit
 hira: hiragana
 hluw: anatolian_hieroglyphs
 hmng: pahawh_hmong
+hmnp: nyiakeng_puachue_hmong
 hung: old_hungarian
 idc: id_continue
 ideo: ideographic
@@ -125,6 +128,7 @@ mtei: meetei_mayek
 mult: multani
 mymr: myanmar
 n: number
+nand: nandinagari
 narb: old_north_arabian
 nbat: nabataean
 nchar: noncharacter_code_point
@@ -216,6 +220,7 @@ uideo: unified_ideograph
 vaii: vai
 vs: variation_selector
 wara: warang_citi
+wcho: wancho
 wspace: white_space
 xidc: xid_continue
 xids: xid_start

data/lib/regexp_parser/scanner/scanner.rl

@@ -21,7 +21,7 @@
   set_close             = ']';
   brackets              = set_open | set_close;
 
-  comment               = ('#' . [^\n]* . '\n');
+  comment               = ('#' . [^\n]* . '\n'?);
 
   class_name_posix      = 'alnum' | 'alpha' | 'blank' |
                           'cntrl' | 'digit' | 'graph' |
@@ -62,13 +62,17 @@
   quantifier_possessive = '?+' | '*+' | '++';
   quantifier_mode       = '?'  | '+';
 
-  quantifier_interval   = range_open . (digit+)? . ','? . (digit+)? .
-                          range_close . quantifier_mode?;
+  quantity_exact        = (digit+);
+  quantity_minimum      = (digit+) . ',';
+  quantity_maximum      = ',' . (digit+);
+  quantity_range        = (digit+) . ',' . (digit+);
+  quantifier_interval   = range_open . ( quantity_exact | quantity_minimum |
+                          quantity_maximum | quantity_range ) . range_close .
+                          quantifier_mode?;
 
   quantifiers           = quantifier_greedy | quantifier_reluctant |
                           quantifier_possessive | quantifier_interval;
 
-
   conditional           = '(?(';
 
   group_comment         = '?#' . [^)]* . group_close;
@@ -114,7 +118,9 @@
                           curlies | parantheses | brackets |
                           line_anchor | quantifier_greedy;
 
-  ascii_print           = ((0x20..0x7e) - meta_char);
+  literal_delimiters    = ']' | '}';
+
+  ascii_print           = ((0x20..0x7e) - meta_char - '#');
   ascii_nonprint        = (0x01..0x1f | 0x7f);
 
   utf8_2_byte           = (0xc2..0xdf 0x80..0xbf);
@@ -122,7 +128,7 @@
   utf8_4_byte           = (0xf0..0xf4 0x80..0xbf 0x80..0xbf 0x80..0xbf);
 
   non_literal_escape    = char_type_char | anchor_char | escaped_ascii |
-                          group_ref | keep_mark | [xucCM];
+                          keep_mark | [xucCM];
 
   non_set_escape        = (anchor_char - 'b') | group_ref | keep_mark |
                           multi_codepoint_char_type | [0-9cCM];
@@ -417,6 +423,10 @@
       end
     };
 
+    literal_delimiters {
+      append_literal(data, ts, te)
+    };
+
     # Character sets
     # ------------------------------------------------------------------------
     set_open >set_opened {
@@ -620,10 +630,15 @@
       end
     };
 
-    quantifier_interval  @err(premature_end_error) {
+    quantifier_interval  {
       emit(:quantifier, :interval, *text(data, ts, te))
     };
 
+    # Catch unmatched curly braces as literals
+    range_open {
+      append_literal(data, ts, te)
+    };
+
     # Escaped sequences
     # ------------------------------------------------------------------------
     backslash > (backslashed, 1) {
@@ -634,7 +649,9 @@
       if free_spacing
         emit(:free_space, :comment, *text(data, ts, te))
       else
-        append_literal(data, ts, te)
+        # consume only the pound sign (#) and backtrack to do regular scanning
+        append_literal(data, ts, ts + 1)
+        fexec ts + 1;
       end
     };
 
@@ -722,21 +739,16 @@ class Regexp::Scanner
   #
   # This method may raise errors if a syntax error is encountered.
   # --------------------------------------------------------------------------
-  def self.scan(input_object, &block)
-    new.scan(input_object, &block)
+  def self.scan(input_object, options: nil, &block)
+    new.scan(input_object, options: options, &block)
   end
 
-  def scan(input_object, &block)
+  def scan(input_object, options: nil, &block)
     self.literal = nil
     stack = []
 
-    if input_object.is_a?(Regexp)
-      input = input_object.source
-      self.free_spacing = (input_object.options & Regexp::EXTENDED != 0)
-    else
-      input = input_object
-      self.free_spacing = false
-    end
+    input = input_object.is_a?(Regexp) ? input_object.source : input_object
+    self.free_spacing = free_spacing?(input_object, options)
     self.spacing_stack = [{:free_spacing => free_spacing, :depth => 0}]
 
     data  = input.unpack("c*") if input.is_a?(String)
@@ -802,6 +814,18 @@ class Regexp::Scanner
   attr_accessor :tokens, :literal, :block, :free_spacing, :spacing_stack,
                 :group_depth, :set_depth, :conditional_stack
 
+  def free_spacing?(input_object, options)
+    if options && !input_object.is_a?(String)
+      raise ArgumentError, 'options cannot be supplied unless scanning a String'
+    end
+
+    options = input_object.options if input_object.is_a?(::Regexp)
+
+    return false unless options
+
+    options & Regexp::EXTENDED != 0
+  end
+
   def in_group?
     group_depth > 0
   end

data/lib/regexp_parser/syntax/tokens/unicode_property.rb

@@ -53,6 +53,10 @@ module Regexp::Syntax
 
       Age_V2_6_0 = [:'age=11.0']
 
+      Age_V2_6_2 = [:'age=12.0']
+
+      Age_V2_6_3 = [:'age=12.1']
+
       Age = all[:Age_V]
 
       Derived_V1_9_0 = [
@@ -297,6 +301,18 @@ module Regexp::Syntax
         :sogdian,
       ]
 
+      Script_V2_6_2 = [
+        :egyptian_hieroglyph_format_controls,
+        :elymaic,
+        :nandinagari,
+        :nyiakeng_puachue_hmong,
+        :ottoman_siyaq_numbers,
+        :small_kana_extension,
+        :symbols_and_pictographs_extended_a,
+        :tamil_supplement,
+        :wancho,
+      ]
+
       Script = all[:Script_V]
 
       UnicodeBlock_V1_9_0 = [
@@ -612,6 +628,18 @@ module Regexp::Syntax
         :in_sogdian,
       ]
 
+      UnicodeBlock_V2_6_2 = [
+        :in_egyptian_hieroglyph_format_controls,
+        :in_elymaic,
+        :in_nandinagari,
+        :in_nyiakeng_puachue_hmong,
+        :in_ottoman_siyaq_numbers,
+        :in_small_kana_extension,
+        :in_symbols_and_pictographs_extended_a,
+        :in_tamil_supplement,
+        :in_wancho,
+      ]
+
       UnicodeBlock = all[:UnicodeBlock_V]
 
       Emoji_V2_5_0 = [
@@ -632,6 +660,8 @@ module Regexp::Syntax
       V2_4_0 = all[:V2_4_0]
       V2_5_0 = all[:V2_5_0]
       V2_6_0 = all[:V2_6_0]
+      V2_6_2 = all[:V2_6_2]
+      V2_6_3 = all[:V2_6_3]
 
       All = all[/^V\d+_\d+_\d+$/]
 

data/lib/regexp_parser/syntax/versions/2.6.2.rb

@@ -0,0 +1,10 @@
+module Regexp::Syntax
+  class V2_6_2 < Regexp::Syntax::V2_6_0
+    def initialize
+      super
+
+      implements :property,    UnicodeProperty::V2_6_2
+      implements :nonproperty, UnicodeProperty::V2_6_2
+    end
+  end
+end

data/lib/regexp_parser/syntax/versions/2.6.3.rb

@@ -0,0 +1,10 @@
+module Regexp::Syntax
+  class V2_6_3 < Regexp::Syntax::V2_6_2
+    def initialize
+      super
+
+      implements :property,    UnicodeProperty::V2_6_3
+      implements :nonproperty, UnicodeProperty::V2_6_3
+    end
+  end
+end

data/lib/regexp_parser/version.rb

@@ -1,5 +1,5 @@
 class Regexp
   class Parser
-    VERSION = '1.5.1'
+    VERSION = '1.8.1'
   end
 end

data/regexp_parser.gemspec

@@ -32,5 +32,5 @@ Gem::Specification.new do |gem|
 
   gem.platform      = Gem::Platform::RUBY
 
-  gem.required_ruby_version = '>= 1.9.1'
+  gem.required_ruby_version = '>= 2.0.0'
 end

data/spec/expression/methods/match_length_spec.rb

@@ -120,6 +120,13 @@ RSpec.describe(Regexp::MatchLength) do
       expect { result.next }.to raise_error(StopIteration)
     end
 
+    it 'is aware of limit option even if called without a block' do
+      result = ML.of(/a?/).each(limit: 1)
+      expect(result).to be_a(Enumerator)
+      expect(result.next).to eq 0
+      expect { result.next }.to raise_error(StopIteration)
+    end
+
     it 'is limited to 1000 iterations in case there are infinite match lengths' do
       expect(ML.of(/a*/).first(3000).size).to eq 1000
     end

data/spec/expression/methods/traverse_spec.rb

@@ -39,6 +39,17 @@ RSpec.describe('Subexpression#traverse') do
     expect(visits).to eq 9
   end
 
+  specify('Subexpression#traverse without a block') do
+    root = RP.parse(/abc/)
+    enum = root.traverse
+
+    expect(enum).to be_a(Enumerator)
+    event, expr, idx = enum.next
+    expect(event).to eq(:visit)
+    expect(expr).to be_a(Regexp::Expression::Literal)
+    expect(idx).to eq(0)
+  end
+
   specify('Subexpression#walk alias') do
     root = RP.parse(/abc/)
 
@@ -81,6 +92,16 @@ RSpec.describe('Subexpression#traverse') do
     expect(indices).to eq [0, 0, 1, 0, 2]
   end
 
+  specify('Subexpression#each_expression without a block') do
+    root = RP.parse(/abc/)
+    enum = root.each_expression
+
+    expect(enum).to be_a(Enumerator)
+    expr, idx = enum.next
+    expect(expr).to be_a(Regexp::Expression::Literal)
+    expect(idx).to eq(0)
+  end
+
   specify('Subexpression#flat_map without block') do
     root = RP.parse(/a(b([c-e]+))?/)
 

data/spec/expression/options_spec.rb

@@ -85,44 +85,44 @@ RSpec.describe('Expression#options') do
       .and change { exp.unicode_classes? }.from(false).to(true)
   end
 
-  RSpec.shared_examples '#options' do |regexp, klass, at: []|
+  RSpec.shared_examples '#options' do |regexp, path, klass|
     it "works for expression class #{klass}" do
-      exp = RP.parse(/#{regexp.source}/i).dig(*at)
+      exp = RP.parse(/#{regexp.source}/i).dig(*path)
       expect(exp).to be_a(klass)
       expect(exp).to be_i
       expect(exp).not_to be_x
     end
   end
 
-  include_examples '#options', //, Root
-  include_examples '#options', /a/, Literal, at: [0]
-  include_examples '#options', /\A/, Anchor::Base, at: [0]
-  include_examples '#options', /\d/, CharacterType::Base, at: [0]
-  include_examples '#options', /\n/, EscapeSequence::Base, at: [0]
-  include_examples '#options', /\K/, Keep::Mark, at: [0]
-  include_examples '#options', /./, CharacterType::Any, at: [0]
-  include_examples '#options', /(a)/, Group::Base, at: [0]
-  include_examples '#options', /(a)/, Literal, at: [0, 0]
-  include_examples '#options', /(?=a)/, Assertion::Base, at: [0]
-  include_examples '#options', /(?=a)/, Literal, at: [0, 0]
-  include_examples '#options', /(a|b)/, Group::Base, at: [0]
-  include_examples '#options', /(a|b)/, Alternation, at: [0, 0]
-  include_examples '#options', /(a|b)/, Alternative, at: [0, 0, 0]
-  include_examples '#options', /(a|b)/, Literal, at: [0, 0, 0, 0]
-  include_examples '#options', /(a)\1/, Backreference::Base, at: [1]
-  include_examples '#options', /(a)\k<1>/, Backreference::Number, at: [1]
-  include_examples '#options', /(a)\g<1>/, Backreference::NumberCall, at: [1]
-  include_examples '#options', /[a]/, CharacterSet, at: [0]
-  include_examples '#options', /[a]/, Literal, at: [0, 0]
-  include_examples '#options', /[a-z]/, CharacterSet::Range, at: [0, 0]
-  include_examples '#options', /[a-z]/, Literal, at: [0, 0, 0]
-  include_examples '#options', /[a&&z]/, CharacterSet::Intersection, at: [0, 0]
-  include_examples '#options', /[a&&z]/, CharacterSet::IntersectedSequence, at: [0, 0, 0]
-  include_examples '#options', /[a&&z]/, Literal, at: [0, 0, 0, 0]
-  include_examples '#options', /[[:ascii:]]/, PosixClass, at: [0, 0]
-  include_examples '#options', /\p{word}/, UnicodeProperty::Base, at: [0]
-  include_examples '#options', /(a)(?(1)b|c)/, Conditional::Expression, at: [1]
-  include_examples '#options', /(a)(?(1)b|c)/, Conditional::Condition, at: [1, 0]
-  include_examples '#options', /(a)(?(1)b|c)/, Conditional::Branch, at: [1, 1]
-  include_examples '#options', /(a)(?(1)b|c)/, Literal, at: [1, 1, 0]
+  include_examples '#options', //, [], Root
+  include_examples '#options', /a/, [0], Literal
+  include_examples '#options', /\A/, [0], Anchor::Base
+  include_examples '#options', /\d/, [0], CharacterType::Base
+  include_examples '#options', /\n/, [0], EscapeSequence::Base
+  include_examples '#options', /\K/, [0], Keep::Mark
+  include_examples '#options', /./, [0], CharacterType::Any
+  include_examples '#options', /(a)/, [0], Group::Base
+  include_examples '#options', /(a)/, [0, 0], Literal
+  include_examples '#options', /(?=a)/, [0], Assertion::Base
+  include_examples '#options', /(?=a)/, [0, 0], Literal
+  include_examples '#options', /(a|b)/, [0], Group::Base
+  include_examples '#options', /(a|b)/, [0, 0], Alternation
+  include_examples '#options', /(a|b)/, [0, 0, 0], Alternative
+  include_examples '#options', /(a|b)/, [0, 0, 0, 0], Literal
+  include_examples '#options', /(a)\1/, [1], Backreference::Base
+  include_examples '#options', /(a)\k<1>/, [1], Backreference::Number
+  include_examples '#options', /(a)\g<1>/, [1], Backreference::NumberCall
+  include_examples '#options', /[a]/, [0], CharacterSet
+  include_examples '#options', /[a]/, [0, 0], Literal
+  include_examples '#options', /[a-z]/, [0, 0], CharacterSet::Range
+  include_examples '#options', /[a-z]/, [0, 0, 0], Literal
+  include_examples '#options', /[a&&z]/, [0, 0], CharacterSet::Intersection
+  include_examples '#options', /[a&&z]/, [0, 0, 0], CharacterSet::IntersectedSequence
+  include_examples '#options', /[a&&z]/, [0, 0, 0, 0], Literal
+  include_examples '#options', /[[:ascii:]]/, [0, 0], PosixClass
+  include_examples '#options', /\p{word}/, [0], UnicodeProperty::Base
+  include_examples '#options', /(a)(?(1)b|c)/, [1], Conditional::Expression
+  include_examples '#options', /(a)(?(1)b|c)/, [1, 0], Conditional::Condition
+  include_examples '#options', /(a)(?(1)b|c)/, [1, 1], Conditional::Branch
+  include_examples '#options', /(a)(?(1)b|c)/, [1, 1, 0], Literal
 end