regexp_parser 2.2.1 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 381a794200168f95ff6329cc8a01330d21a05e02b75e0b06dcc6bd8f763c111d
-  data.tar.gz: bd7617cb3763e6d759c8e1364aed037ae2fff85af3cf28823476cadd14ff080e
+  metadata.gz: 369b108d8410e12bd6af5c659f58cb56c583e48780c1b35b6270bb21cc6a4ee7
+  data.tar.gz: 30cd2c0823ae154a2db04c705f898f252774ec8ab9ef304833c5e3546ba7406a
 SHA512:
-  metadata.gz: 0a039012013e9b57329fd685aaf29386d8b848071e514f59df0acc3437a1dae5c76b6bf94158cc3deece08f3a1fec9437ac84590d97f8590d8dcee1e0dc6c726
-  data.tar.gz: 4d67da41fbef9b9336ccfd02e3a742286bf4ef96d469c8aa2bbb9a6a55ed4aa6027a28b10ba6c9993b15937e3fe51a349632bcf5808f6237cf77a1d29ceb74f2
+  metadata.gz: 4104bec7dd02a7ea099de9aeacb766fb1a2db50cb52bd84f44e4bde93431d436b75d0f1b3f4d62242713a1eeca3f4d8c0be034270d515979aad8ad2d504880b0
+  data.tar.gz: 11deb2d7c8a6fad3fa9cb18b3f29cae15bab7e12e6cbbc968706dd02c16b0d1a6b1d69f05a5f665f7b46947315b0ea4ecda62dab8ddca8b5ef71f521b877da74

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## [Unreleased]
+
+## [2.3.0] - 2022-04-08 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- improved parsing performance through `Syntax` refactoring
+  - instead of fresh `Syntax` instances, pre-loaded constants are now re-used
+  - this approximately doubles the parsing speed for simple regexps
+- added methods to `Syntax` classes to show relative feature sets
+  - e.g. `Regexp::Syntax::V3_2_0.added_features`
+- support for new unicode properties of Ruby 3.2 / Unicode 14.0
+
 ## [2.2.1] - 2022-02-11 - [Janosch Müller](mailto:janosch84@gmail.com)
 
 ### Fixed

data/Gemfile

@@ -5,7 +5,7 @@ gemspec
 group :development, :test do
   gem 'ice_nine', '~> 0.11.2'
   gem 'rake', '~> 13.0'
-  gem 'regexp_property_values', '~> 1.0'
+  gem 'regexp_property_values', '~> 1.3'
   gem 'rspec', '~> 3.10'
   if RUBY_VERSION.to_f >= 2.7
     gem 'gouteur'

data/README.md

@@ -157,31 +157,41 @@ flavor). Syntax classes act as lookup tables, and are layered to create
 flavor variations. Syntax only comes into play in the lexer.
 
 #### Example
-The following instantiates syntax objects for Ruby 2.0, 1.9, 1.8, and
+The following fetches syntax objects for Ruby 2.0, 1.9, 1.8, and
 checks a few of their implementation features.
 
 ```ruby
 require 'regexp_parser'
 
-ruby_20 = Regexp::Syntax.new 'ruby/2.0'
+ruby_20 = Regexp::Syntax.for 'ruby/2.0'
 ruby_20.implements? :quantifier,  :zero_or_one             # => true
 ruby_20.implements? :quantifier,  :zero_or_one_reluctant   # => true
 ruby_20.implements? :quantifier,  :zero_or_one_possessive  # => true
 ruby_20.implements? :conditional, :condition               # => true
 
-ruby_19 = Regexp::Syntax.new 'ruby/1.9'
+ruby_19 = Regexp::Syntax.for 'ruby/1.9'
 ruby_19.implements? :quantifier,  :zero_or_one             # => true
 ruby_19.implements? :quantifier,  :zero_or_one_reluctant   # => true
 ruby_19.implements? :quantifier,  :zero_or_one_possessive  # => true
 ruby_19.implements? :conditional, :condition               # => false
 
-ruby_18 = Regexp::Syntax.new 'ruby/1.8'
+ruby_18 = Regexp::Syntax.for 'ruby/1.8'
 ruby_18.implements? :quantifier,  :zero_or_one             # => true
 ruby_18.implements? :quantifier,  :zero_or_one_reluctant   # => true
 ruby_18.implements? :quantifier,  :zero_or_one_possessive  # => false
 ruby_18.implements? :conditional, :condition               # => false
 ```
 
+Syntax objects can also be queried about their complete and relative feature sets.
+
+```ruby
+require 'regexp_parser'
+
+ruby_20 = Regexp::Syntax.for 'ruby/2.0' # => Regexp::Syntax::V2_0_0
+ruby_20.added_features                  # => { conditional: [...], ... }
+ruby_20.removed_features                # => { property: [:newline], ... }
+ruby_20.features                        # => { anchor: [...], ... }
+```
 
 #### Notes
   * Variations on a token, for example a named group with angle brackets (< and >)

data/Rakefile

@@ -5,9 +5,7 @@ require 'rake'
 require 'rake/testtask'
 require 'rspec/core/rake_task'
 
-RAGEL_SOURCE_DIR = File.join(__dir__, 'lib/regexp_parser/scanner')
-RAGEL_OUTPUT_DIR = File.join(__dir__, 'lib/regexp_parser')
-RAGEL_SOURCE_FILES = %w[scanner] # scanner.rl imports the other files
+Dir['tasks/**/*.rake'].each { |file| load(file) }
 
 Bundler::GemHelper.install_tasks
 
@@ -19,60 +17,7 @@ namespace :test do
   task full: [:'ragel:rb', :spec]
 end
 
-namespace :ragel do
-  desc "Process the ragel source files and output ruby code"
-  task :rb do
-    RAGEL_SOURCE_FILES.each do |source_file|
-      output_file = "#{RAGEL_OUTPUT_DIR}/#{source_file}.rb"
-      # using faster flat table driven FSM, about 25% larger code, but about 30% faster
-      sh "ragel -F1 -R #{RAGEL_SOURCE_DIR}/#{source_file}.rl -o #{output_file}"
-
-      contents = File.read(output_file)
-
-      File.open(output_file, 'r+') do |file|
-        contents = "# -*- warn-indent:false;  -*-\n" + contents
-
-        file.write(contents)
-      end
-    end
-  end
-
-  desc "Delete the ragel generated source file(s)"
-  task :clean do
-    RAGEL_SOURCE_FILES.each do |file|
-      sh "rm -f #{RAGEL_OUTPUT_DIR}/#{file}.rb"
-    end
-  end
-end
-
 # Add ragel task as a prerequisite for building the gem to ensure that the
 # latest scanner code is generated and included in the build.
 desc "Runs ragel:rb before building the gem"
 task :build => ['ragel:rb']
-
-namespace :props do
-  desc 'Write new property value hashes for the properties scanner'
-  task :update do
-    require 'regexp_property_values'
-    RegexpPropertyValues.update
-    dir = File.join(__dir__, 'lib/regexp_parser/scanner/properties')
-
-    write_hash_to_file = ->(hash, path) do
-      File.open(path, 'w') do |f|
-        f.puts "# THIS FILE IS AUTO-GENERATED BY `rake props:update` - DO NOT EDIT",
-               *hash.sort.map { |pair| pair.join(',') }
-      end
-      puts "Wrote #{hash.count} aliases to `#{path}`"
-    end
-
-    long_names_to_tokens = RegexpPropertyValues.all.map do |val|
-      [val.identifier, val.full_name.downcase]
-    end
-    write_hash_to_file.call(long_names_to_tokens, "#{dir}/long.csv")
-
-    short_names_to_tokens = RegexpPropertyValues.alias_hash.map do |k, v|
-      [k.identifier, v.full_name.downcase]
-    end
-    write_hash_to_file.call(short_names_to_tokens, "#{dir}/short.csv")
-  end
-end

data/lib/regexp_parser/lexer.rb

@@ -18,7 +18,7 @@ class Regexp::Lexer
   end
 
   def lex(input, syntax = "ruby/#{RUBY_VERSION}", options: nil, &block)
-    syntax = Regexp::Syntax.new(syntax)
+    syntax = Regexp::Syntax.for(syntax)
 
     self.tokens = []
     self.nesting = 0

data/lib/regexp_parser/scanner/properties/long.csv

@@ -6,6 +6,7 @@ age=11.0,age=11.0
 age=12.0,age=12.0
 age=12.1,age=12.1
 age=13.0,age=13.0
+age=14.0,age=14.0
 age=2.0,age=2.0
 age=2.1,age=2.1
 age=3.0,age=3.0
@@ -72,6 +73,7 @@ coptic,coptic
 cuneiform,cuneiform
 currencysymbol,currency_symbol
 cypriot,cypriot
+cyprominoan,cypro_minoan
 cyrillic,cyrillic
 dash,dash
 dashpunctuation,dash_punctuation
@@ -136,6 +138,7 @@ inancientgreeknumbers,in_ancient_greek_numbers
 inancientsymbols,in_ancient_symbols
 inarabic,in_arabic
 inarabicextendeda,in_arabic_extended_a
+inarabicextendedb,in_arabic_extended_b
 inarabicmathematicalalphabeticsymbols,in_arabic_mathematical_alphabetic_symbols
 inarabicpresentationformsa,in_arabic_presentation_forms_a
 inarabicpresentationformsb,in_arabic_presentation_forms_b
@@ -197,6 +200,7 @@ incuneiform,in_cuneiform
 incuneiformnumbersandpunctuation,in_cuneiform_numbers_and_punctuation
 incurrencysymbols,in_currency_symbols
 incypriotsyllabary,in_cypriot_syllabary
+incyprominoan,in_cypro_minoan
 incyrillic,in_cyrillic
 incyrillicextendeda,in_cyrillic_extended_a
 incyrillicextendedb,in_cyrillic_extended_b
@@ -223,6 +227,7 @@ inenclosedideographicsupplement,in_enclosed_ideographic_supplement
 inethiopic,in_ethiopic
 inethiopicextended,in_ethiopic_extended
 inethiopicextendeda,in_ethiopic_extended_a
+inethiopicextendedb,in_ethiopic_extended_b
 inethiopicsupplement,in_ethiopic_supplement
 ingeneralpunctuation,in_general_punctuation
 ingeometricshapes,in_geometric_shapes
@@ -264,6 +269,7 @@ initialpunctuation,initial_punctuation
 injavanese,in_javanese
 inkaithi,in_kaithi
 inkanaextendeda,in_kana_extended_a
+inkanaextendedb,in_kana_extended_b
 inkanasupplement,in_kana_supplement
 inkanbun,in_kanbun
 inkangxiradicals,in_kangxi_radicals
@@ -285,6 +291,8 @@ inlatinextendedb,in_latin_extended_b
 inlatinextendedc,in_latin_extended_c
 inlatinextendedd,in_latin_extended_d
 inlatinextendede,in_latin_extended_e
+inlatinextendedf,in_latin_extended_f
+inlatinextendedg,in_latin_extended_g
 inlepcha,in_lepcha
 inletterlikesymbols,in_letterlike_symbols
 inlimbu,in_limbu
@@ -349,6 +357,7 @@ inoldpersian,in_old_persian
 inoldsogdian,in_old_sogdian
 inoldsoutharabian,in_old_south_arabian
 inoldturkic,in_old_turkic
+inolduyghur,in_old_uyghur
 inopticalcharacterrecognition,in_optical_character_recognition
 inoriya,in_oriya
 inornamentaldingbats,in_ornamental_dingbats
@@ -413,6 +422,7 @@ intaixuanjingsymbols,in_tai_xuan_jing_symbols
 intakri,in_takri
 intamil,in_tamil
 intamilsupplement,in_tamil_supplement
+intangsa,in_tangsa
 intangut,in_tangut
 intangutcomponents,in_tangut_components
 intangutsupplement,in_tangut_supplement
@@ -422,15 +432,18 @@ inthai,in_thai
 intibetan,in_tibetan
 intifinagh,in_tifinagh
 intirhuta,in_tirhuta
+intoto,in_toto
 intransportandmapsymbols,in_transport_and_map_symbols
 inugaritic,in_ugaritic
 inunifiedcanadianaboriginalsyllabics,in_unified_canadian_aboriginal_syllabics
 inunifiedcanadianaboriginalsyllabicsextended,in_unified_canadian_aboriginal_syllabics_extended
+inunifiedcanadianaboriginalsyllabicsextendeda,in_unified_canadian_aboriginal_syllabics_extended_a
 invai,in_vai
 invariationselectors,in_variation_selectors
 invariationselectorssupplement,in_variation_selectors_supplement
 invedicextensions,in_vedic_extensions
 inverticalforms,in_vertical_forms
+invithkuqi,in_vithkuqi
 inwancho,in_wancho
 inwarangciti,in_warang_citi
 inyezidi,in_yezidi
@@ -438,6 +451,7 @@ inyijinghexagramsymbols,in_yijing_hexagram_symbols
 inyiradicals,in_yi_radicals
 inyisyllables,in_yi_syllables
 inzanabazarsquare,in_zanabazar_square
+inznamennymusicalnotation,in_znamenny_musical_notation
 javanese,javanese
 joincontrol,join_control
 kaithi,kaithi
@@ -509,6 +523,7 @@ oldpersian,old_persian
 oldsogdian,old_sogdian
 oldsoutharabian,old_south_arabian
 oldturkic,old_turkic
+olduyghur,old_uyghur
 openpunctuation,open_punctuation
 oriya,oriya
 osage,osage
@@ -573,6 +588,7 @@ taitham,tai_tham
 taiviet,tai_viet
 takri,takri
 tamil,tamil
+tangsa,tangsa
 tangut,tangut
 telugu,telugu
 terminalpunctuation,terminal_punctuation
@@ -582,6 +598,7 @@ tibetan,tibetan
 tifinagh,tifinagh
 tirhuta,tirhuta
 titlecaseletter,titlecase_letter
+toto,toto
 ugaritic,ugaritic
 unassigned,unassigned
 unifiedideograph,unified_ideograph
@@ -591,6 +608,7 @@ uppercase,uppercase
 uppercaseletter,uppercase_letter
 vai,vai
 variationselector,variation_selector
+vithkuqi,vithkuqi
 wancho,wancho
 warangciti,warang_citi
 whitespace,white_space

data/lib/regexp_parser/scanner/properties/short.csv

@@ -31,6 +31,7 @@ cn,unassigned
 co,private_use
 combiningmark,mark
 copt,coptic
+cpmn,cypro_minoan
 cprt,cypriot
 cs,surrogate
 cwcf,changes_when_casefolded
@@ -154,6 +155,7 @@ orkh,old_turkic
 orya,oriya
 osge,osage
 osma,osmanya
+ougr,old_uyghur
 oupper,other_uppercase
 p,punctuation
 palm,palmyrene
@@ -219,9 +221,11 @@ tglg,tagalog
 thaa,thaana
 tibt,tibetan
 tirh,tirhuta
+tnsa,tangsa
 ugar,ugaritic
 uideo,unified_ideograph
 vaii,vai
+vith,vithkuqi
 vs,variation_selector
 wara,warang_citi
 wcho,wancho