regexp_parser 2.1.1 → 2.5.0

data/lib/regexp_parser/scanner/properties/short.csv

@@ -0,0 +1,246 @@
+# THIS FILE IS AUTO-GENERATED BY `rake props:update` - DO NOT EDIT
+adlm,adlam
+aghb,caucasian_albanian
+ahex,ascii_hex_digit
+arab,arabic
+armi,imperial_aramaic
+armn,armenian
+avst,avestan
+bali,balinese
+bamu,bamum
+bass,bassa_vah
+batk,batak
+beng,bengali
+bhks,bhaiksuki
+bidic,bidi_control
+bopo,bopomofo
+brah,brahmi
+brai,braille
+bugi,buginese
+buhd,buhid
+c,other
+cakm,chakma
+cans,canadian_aboriginal
+cari,carian
+cc,control
+cf,format
+cher,cherokee
+chrs,chorasmian
+ci,case_ignorable
+cn,unassigned
+co,private_use
+combiningmark,mark
+copt,coptic
+cpmn,cypro_minoan
+cprt,cypriot
+cs,surrogate
+cwcf,changes_when_casefolded
+cwcm,changes_when_casemapped
+cwl,changes_when_lowercased
+cwt,changes_when_titlecased
+cwu,changes_when_uppercased
+cyrl,cyrillic
+dep,deprecated
+deva,devanagari
+di,default_ignorable_code_point
+dia,diacritic
+diak,dives_akuru
+dogr,dogra
+dsrt,deseret
+dupl,duployan
+ebase,emoji_modifier_base
+ecomp,emoji_component
+egyp,egyptian_hieroglyphs
+elba,elbasan
+elym,elymaic
+emod,emoji_modifier
+epres,emoji_presentation
+ethi,ethiopic
+ext,extender
+geor,georgian
+glag,glagolitic
+gong,gunjala_gondi
+gonm,masaram_gondi
+goth,gothic
+gran,grantha
+grbase,grapheme_base
+grek,greek
+grext,grapheme_extend
+grlink,grapheme_link
+gujr,gujarati
+guru,gurmukhi
+hang,hangul
+hani,han
+hano,hanunoo
+hatr,hatran
+hebr,hebrew
+hex,hex_digit
+hira,hiragana
+hluw,anatolian_hieroglyphs
+hmng,pahawh_hmong
+hmnp,nyiakeng_puachue_hmong
+hung,old_hungarian
+idc,id_continue
+ideo,ideographic
+ids,id_start
+idsb,ids_binary_operator
+idst,ids_trinary_operator
+ital,old_italic
+java,javanese
+joinc,join_control
+kali,kayah_li
+kana,katakana
+khar,kharoshthi
+khmr,khmer
+khoj,khojki
+kits,khitan_small_script
+knda,kannada
+kthi,kaithi
+l,letter
+lana,tai_tham
+laoo,lao
+latn,latin
+lc,cased_letter
+lepc,lepcha
+limb,limbu
+lina,linear_a
+linb,linear_b
+ll,lowercase_letter
+lm,modifier_letter
+lo,other_letter
+loe,logical_order_exception
+lt,titlecase_letter
+lu,uppercase_letter
+lyci,lycian
+lydi,lydian
+m,mark
+mahj,mahajani
+maka,makasar
+mand,mandaic
+mani,manichaean
+marc,marchen
+mc,spacing_mark
+me,enclosing_mark
+medf,medefaidrin
+mend,mende_kikakui
+merc,meroitic_cursive
+mero,meroitic_hieroglyphs
+mlym,malayalam
+mn,nonspacing_mark
+mong,mongolian
+mroo,mro
+mtei,meetei_mayek
+mult,multani
+mymr,myanmar
+n,number
+nand,nandinagari
+narb,old_north_arabian
+nbat,nabataean
+nchar,noncharacter_code_point
+nd,decimal_number
+nkoo,nko
+nl,letter_number
+no,other_number
+nshu,nushu
+oalpha,other_alphabetic
+odi,other_default_ignorable_code_point
+ogam,ogham
+ogrext,other_grapheme_extend
+oidc,other_id_continue
+oids,other_id_start
+olck,ol_chiki
+olower,other_lowercase
+omath,other_math
+orkh,old_turkic
+orya,oriya
+osge,osage
+osma,osmanya
+ougr,old_uyghur
+oupper,other_uppercase
+p,punctuation
+palm,palmyrene
+patsyn,pattern_syntax
+patws,pattern_white_space
+pauc,pau_cin_hau
+pc,connector_punctuation
+pcm,prepended_concatenation_mark
+pd,dash_punctuation
+pe,close_punctuation
+perm,old_permic
+pf,final_punctuation
+phag,phags_pa
+phli,inscriptional_pahlavi
+phlp,psalter_pahlavi
+phnx,phoenician
+pi,initial_punctuation
+plrd,miao
+po,other_punctuation
+prti,inscriptional_parthian
+ps,open_punctuation
+qaac,coptic
+qaai,inherited
+qmark,quotation_mark
+ri,regional_indicator
+rjng,rejang
+rohg,hanifi_rohingya
+runr,runic
+s,symbol
+samr,samaritan
+sarb,old_south_arabian
+saur,saurashtra
+sc,currency_symbol
+sd,soft_dotted
+sgnw,signwriting
+shaw,shavian
+shrd,sharada
+sidd,siddham
+sind,khudawadi
+sinh,sinhala
+sk,modifier_symbol
+sm,math_symbol
+so,other_symbol
+sogd,sogdian
+sogo,old_sogdian
+sora,sora_sompeng
+soyo,soyombo
+sterm,sentence_terminal
+sund,sundanese
+sylo,syloti_nagri
+syrc,syriac
+tagb,tagbanwa
+takr,takri
+tale,tai_le
+talu,new_tai_lue
+taml,tamil
+tang,tangut
+tavt,tai_viet
+telu,telugu
+term,terminal_punctuation
+tfng,tifinagh
+tglg,tagalog
+thaa,thaana
+tibt,tibetan
+tirh,tirhuta
+tnsa,tangsa
+ugar,ugaritic
+uideo,unified_ideograph
+vaii,vai
+vith,vithkuqi
+vs,variation_selector
+wara,warang_citi
+wcho,wancho
+wspace,white_space
+xidc,xid_continue
+xids,xid_start
+xpeo,old_persian
+xsux,cuneiform
+yezi,yezidi
+yiii,yi
+z,separator
+zanb,zanabazar_square
+zinh,inherited
+zl,line_separator
+zp,paragraph_separator
+zs,space_separator
+zyyy,common
+zzzz,unknown

data/lib/regexp_parser/scanner/property.rl

@@ -20,7 +20,7 @@
       name = data[ts+2..te-2].pack('c*').gsub(/[\^\s_\-]/, '').downcase
 
       token = self.class.short_prop_map[name] || self.class.long_prop_map[name]
-      raise UnknownUnicodePropertyError.new(name) unless token
+      validation_error(:property, name) unless token
 
       self.emit(type, token.to_sym, text)
 

data/lib/regexp_parser/scanner/scanner.rl

@@ -28,13 +28,7 @@
 
   comment               = ('#' . [^\n]* . '\n'?);
 
-  class_name_posix      = 'alnum' | 'alpha' | 'blank' |
-                          'cntrl' | 'digit' | 'graph' |
-                          'lower' | 'print' | 'punct' |
-                          'space' | 'upper' | 'xdigit' |
-                          'word'  | 'ascii';
-
-  class_posix           = ('[:' . '^'? . class_name_posix . ':]');
+  class_posix           = ('[:' . '^'? . [^\[\]]* . ':]');
 
 
   # these are not supported in ruby at the moment
@@ -74,8 +68,7 @@
   quantity_maximum      = ',' . (digit+);
   quantity_range        = (digit+) . ',' . (digit+);
   quantifier_interval   = range_open . ( quantity_exact | quantity_minimum |
-                          quantity_maximum | quantity_range ) . range_close .
-                          quantifier_mode?;
+                          quantity_maximum | quantity_range ) . range_close;
 
   quantifiers           = quantifier_greedy | quantifier_reluctant |
                           quantifier_possessive | quantifier_interval;
@@ -223,24 +216,28 @@
       fcall character_set;
     };
 
-    class_posix >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
+    class_posix >(open_bracket, 1) @set_closed @eof(premature_end_error) {
       text = copy(data, ts, te)
 
       type = :posixclass
       class_name = text[2..-3]
-      if class_name[0].chr == '^'
+      if class_name[0] == '^'
         class_name = class_name[1..-1]
         type = :nonposixclass
       end
 
+      unless self.class.posix_classes.include?(class_name)
+        validation_error(:posix_class, text)
+      end
+
       emit(type, class_name.to_sym, text)
     };
 
     # These are not supported in ruby at the moment. Enable them if they are.
-    # collating_sequence >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
+    # collating_sequence >(open_bracket, 1) @set_closed @eof(premature_end_error) {
     #   emit(:set, :collation, copy(data, ts, te))
     # };
-    # character_equivalent >(open_bracket, 1) @set_closed @eof(premature_end_error)  {
+    # character_equivalent >(open_bracket, 1) @set_closed @eof(premature_end_error) {
     #   emit(:set, :equivalent, copy(data, ts, te))
     # };
 
@@ -323,7 +320,7 @@
 
     codepoint_sequence > (escaped_alpha, 6) $eof(premature_end_error) {
       text = copy(data, ts-1, te)
-      if text[2].chr == '{'
+      if text[2] == '{'
         emit(:escape, :codepoint_list, text)
       else
         emit(:escape, :codepoint,      text)
@@ -419,12 +416,12 @@
 
     backslash . anchor_char > (backslashed, 3) {
       case text = copy(data, ts, te)
-      when '\\A'; emit(:anchor, :bos,                text)
-      when '\\z'; emit(:anchor, :eos,                text)
-      when '\\Z'; emit(:anchor, :eos_ob_eol,         text)
-      when '\\b'; emit(:anchor, :word_boundary,      text)
-      when '\\B'; emit(:anchor, :nonword_boundary,   text)
-      when '\\G'; emit(:anchor, :match_start,        text)
+      when '\A';  emit(:anchor, :bos,                text)
+      when '\z';  emit(:anchor, :eos,                text)
+      when '\Z';  emit(:anchor, :eos_ob_eol,         text)
+      when '\b';  emit(:anchor, :word_boundary,      text)
+      when '\B';  emit(:anchor, :nonword_boundary,   text)
+      when '\G';  emit(:anchor, :match_start,        text)
       end
     };
 
@@ -477,7 +474,7 @@
     group_open . group_options >group_opened {
       text = copy(data, ts, te)
       if text[2..-1] =~ /([^\-mixdau:]|^$)|-.*([dau])/
-        raise InvalidGroupOption.new($1 || "-#{$2}", text)
+        validation_error(:group_option, $1 || "-#{$2}", text)
       end
       emit_options(text)
     };
@@ -605,7 +602,7 @@
       end
     };
 
-    quantifier_interval  {
+    quantifier_interval {
       emit(:quantifier, :interval, copy(data, ts, te))
     };
 
@@ -686,6 +683,7 @@ class Regexp::Scanner
   end
 
   # Invalid groupOption. Used for inline options.
+  # TODO: should become InvalidGroupOptionError in v3.0.0 for consistency
   class InvalidGroupOption < ValidationError
     def initialize(option, text)
       super "Invalid group option #{option} in #{text}"
@@ -706,6 +704,13 @@ class Regexp::Scanner
     end
   end
 
+  # The POSIX class name was not recognized by the scanner.
+  class UnknownPosixClassError < ValidationError
+    def initialize(text)
+      super "Unknown POSIX class #{text}"
+    end
+  end
+
   # Scans the given regular expression text, or Regexp object and collects the
   # emitted token into an array that gets returned at the end. If a block is
   # given, it gets called for each emitted token.
@@ -759,14 +764,21 @@ class Regexp::Scanner
   end
 
   # lazy-load property maps when first needed
-  require 'yaml'
-
   def self.short_prop_map
-    @short_prop_map ||= YAML.load_file("#{__dir__}/scanner/properties/short.yml")
+    @short_prop_map ||= parse_prop_map('short')
   end
 
   def self.long_prop_map
-    @long_prop_map ||= YAML.load_file("#{__dir__}/scanner/properties/long.yml")
+    @long_prop_map ||= parse_prop_map('long')
+  end
+
+  def self.parse_prop_map(name)
+    File.read("#{__dir__}/scanner/properties/#{name}.csv").scan(/(.+),(.+)/).to_h
+  end
+
+  def self.posix_classes
+    %w[alnum alpha ascii blank cntrl digit graph
+       lower print punct space upper word xdigit]
   end
 
   # Emits an array with the details of the scanned pattern
@@ -871,15 +883,16 @@ class Regexp::Scanner
 
   # Centralizes and unifies the handling of validation related
   # errors.
-  def validation_error(type, what, reason)
-    case type
-    when :group
-      error = InvalidGroupError.new(what, reason)
-    when :backref
-      error = InvalidBackrefError.new(what, reason)
-    when :sequence
-      error = InvalidSequenceError.new(what, reason)
-    end
+  def validation_error(type, what, reason = nil)
+    error =
+      case type
+      when :backref      then InvalidBackrefError.new(what, reason)
+      when :group        then InvalidGroupError.new(what, reason)
+      when :group_option then InvalidGroupOption.new(what, reason)
+      when :posix_class  then UnknownPosixClassError.new(what)
+      when :property     then UnknownUnicodePropertyError.new(what)
+      when :sequence     then InvalidSequenceError.new(what, reason)
+      end
 
     raise error # unless @@config.validation_ignore
   end