regexp_parser 2.1.1 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 077b8a0c90d90cf46e44671ec1335a5373eef72c61a0bcf4de43ba5217a188c3
-  data.tar.gz: b9aed868af73adcdf40c09720c5d10091b25a53b25a792717ceb5591039a2931
+  metadata.gz: 369b108d8410e12bd6af5c659f58cb56c583e48780c1b35b6270bb21cc6a4ee7
+  data.tar.gz: 30cd2c0823ae154a2db04c705f898f252774ec8ab9ef304833c5e3546ba7406a
 SHA512:
-  metadata.gz: 9c04d9a6434c6e3f322e97e8e2a1c86b3ddda88bd8821368a37b92f5836e4c3df1dc27a79165303420c3e8d5eea31bda1483824da01a40ce30961b645ba65ddd
-  data.tar.gz: 01e5c261e9dca0c4df7c696128dbc0520ca40aa6b9393cc8d6c3bdb8386470aeb773566000b811f98c1407038216c8d2c0b444c7955ea5a881ac759796f8a440
+  metadata.gz: 4104bec7dd02a7ea099de9aeacb766fb1a2db50cb52bd84f44e4bde93431d436b75d0f1b3f4d62242713a1eeca3f4d8c0be034270d515979aad8ad2d504880b0
+  data.tar.gz: 11deb2d7c8a6fad3fa9cb18b3f29cae15bab7e12e6cbbc968706dd02c16b0d1a6b1d69f05a5f665f7b46947315b0ea4ecda62dab8ddca8b5ef71f521b877da74

data/CHANGELOG.md

@@ -1,5 +1,36 @@
 ## [Unreleased]
 
+## [2.3.0] - 2022-04-08 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- improved parsing performance through `Syntax` refactoring
+  - instead of fresh `Syntax` instances, pre-loaded constants are now re-used
+  - this approximately doubles the parsing speed for simple regexps
+- added methods to `Syntax` classes to show relative feature sets
+  - e.g. `Regexp::Syntax::V3_2_0.added_features`
+- support for new unicode properties of Ruby 3.2 / Unicode 14.0
+
+## [2.2.1] - 2022-02-11 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Fixed
+
+- fixed Syntax version of absence groups (`(?~...)`)
+  - the lexer accepted them for any Ruby version
+  - now they are only recognized for Ruby >= 2.4.1 in which they were introduced
+- reduced gem size by excluding specs from package
+- removed deprecated `test_files` gemspec setting
+- no longer depend on `yaml`/`psych` (except for Ruby <= 2.4)
+- no longer depend on `set`
+  - `set` was removed from the stdlib and made a standalone gem as of Ruby 3
+  - this made it a hidden/undeclared dependency of `regexp_parser`
+
+## [2.2.0] - 2021-12-04 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- added support for 13 new unicode properties introduced in Ruby 3.1.0
+
 ## [2.1.1] - 2021-02-23 - [Janosch Müller](mailto:janosch84@gmail.com)
 
 ### Fixed

data/Gemfile

@@ -5,7 +5,7 @@ gemspec
 group :development, :test do
   gem 'ice_nine', '~> 0.11.2'
   gem 'rake', '~> 13.0'
-  gem 'regexp_property_values', '~> 1.0'
+  gem 'regexp_property_values', '~> 1.3'
   gem 'rspec', '~> 3.10'
   if RUBY_VERSION.to_f >= 2.7
     gem 'gouteur'

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010, 2012-2015,  Ammar Ali
+Copyright (c) 2010, 2012-2022,  Ammar Ali
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation

data/README.md

@@ -1,6 +1,9 @@
 # Regexp::Parser
 
-[![Gem Version](https://badge.fury.io/rb/regexp_parser.svg)](http://badge.fury.io/rb/regexp_parser) [![Build Status](https://github.com/ammar/regexp_parser/workflows/tests/badge.svg)](https://github.com/ammar/regexp_parser/actions) [![Build Status](https://github.com/ammar/regexp_parser/workflows/gouteur/badge.svg)](https://github.com/ammar/regexp_parser/actions) [![Code Climate](https://codeclimate.com/github/ammar/regexp_parser.svg)](https://codeclimate.com/github/ammar/regexp_parser/badges)
+[![Gem Version](https://badge.fury.io/rb/regexp_parser.svg)](http://badge.fury.io/rb/regexp_parser)
+[![Build Status](https://github.com/ammar/regexp_parser/workflows/tests/badge.svg)](https://github.com/ammar/regexp_parser/actions)
+[![Build Status](https://github.com/ammar/regexp_parser/workflows/gouteur/badge.svg)](https://github.com/ammar/regexp_parser/actions)
+[![Code Climate](https://codeclimate.com/github/ammar/regexp_parser.svg)](https://codeclimate.com/github/ammar/regexp_parser/badges)
 
 A Ruby gem for tokenizing, parsing, and transforming regular expressions.
 
@@ -154,31 +157,41 @@ flavor). Syntax classes act as lookup tables, and are layered to create
 flavor variations. Syntax only comes into play in the lexer.
 
 #### Example
-The following instantiates syntax objects for Ruby 2.0, 1.9, 1.8, and
+The following fetches syntax objects for Ruby 2.0, 1.9, 1.8, and
 checks a few of their implementation features.
 
 ```ruby
 require 'regexp_parser'
 
-ruby_20 = Regexp::Syntax.new 'ruby/2.0'
+ruby_20 = Regexp::Syntax.for 'ruby/2.0'
 ruby_20.implements? :quantifier,  :zero_or_one             # => true
 ruby_20.implements? :quantifier,  :zero_or_one_reluctant   # => true
 ruby_20.implements? :quantifier,  :zero_or_one_possessive  # => true
 ruby_20.implements? :conditional, :condition               # => true
 
-ruby_19 = Regexp::Syntax.new 'ruby/1.9'
+ruby_19 = Regexp::Syntax.for 'ruby/1.9'
 ruby_19.implements? :quantifier,  :zero_or_one             # => true
 ruby_19.implements? :quantifier,  :zero_or_one_reluctant   # => true
 ruby_19.implements? :quantifier,  :zero_or_one_possessive  # => true
 ruby_19.implements? :conditional, :condition               # => false
 
-ruby_18 = Regexp::Syntax.new 'ruby/1.8'
+ruby_18 = Regexp::Syntax.for 'ruby/1.8'
 ruby_18.implements? :quantifier,  :zero_or_one             # => true
 ruby_18.implements? :quantifier,  :zero_or_one_reluctant   # => true
 ruby_18.implements? :quantifier,  :zero_or_one_possessive  # => false
 ruby_18.implements? :conditional, :condition               # => false
 ```
 
+Syntax objects can also be queried about their complete and relative feature sets.
+
+```ruby
+require 'regexp_parser'
+
+ruby_20 = Regexp::Syntax.for 'ruby/2.0' # => Regexp::Syntax::V2_0_0
+ruby_20.added_features                  # => { conditional: [...], ... }
+ruby_20.removed_features                # => { property: [:newline], ... }
+ruby_20.features                        # => { anchor: [...], ... }
+```
 
 #### Notes
   * Variations on a token, for example a named group with angle brackets (< and >)
@@ -357,12 +370,12 @@ _Note that not all of these are available in all versions of Ruby_
 | &emsp;&nbsp;_**Reluctant** (Lazy)_    | `??`, `*?`, `+?`, `{m,M}?`                              | &#x2713; |
 | &emsp;&nbsp;_**Possessive**_          | `?+`, `*+`, `++`, `{m,M}+`                              | &#x2713; |
 | **String Escapes**                    |                                                         | &#x22f1; |
-| &emsp;&nbsp;_**Control**_             | `\C-C`, `\cD`                                           | &#x2713; |
+| &emsp;&nbsp;_**Control** \[1\]_       | `\C-C`, `\cD`                                           | &#x2713; |
 | &emsp;&nbsp;_**Hex**_                 | `\x20`, `\x{701230}`                                    | &#x2713; |
-| &emsp;&nbsp;_**Meta**_                | `\M-c`, `\M-\C-C`, `\M-\cC`, `\C-\M-C`, `\c\M-C`        | &#x2713; |
+| &emsp;&nbsp;_**Meta** \[1\]_          | `\M-c`, `\M-\C-C`, `\M-\cC`, `\C-\M-C`, `\c\M-C`        | &#x2713; |
 | &emsp;&nbsp;_**Octal**_               | `\0`, `\01`, `\012`                                     | &#x2713; |
 | &emsp;&nbsp;_**Unicode**_             | `\uHHHH`, `\u{H+ H+}`                                   | &#x2713; |
-| **Unicode Properties**                | _<sub>([Unicode 11.0.0](http://www.unicode.org/versions/Unicode11.0.0/))</sub>_ | &#x22f1; |
+| **Unicode Properties**                | _<sub>([Unicode 13.0.0](https://www.unicode.org/versions/Unicode13.0.0/))</sub>_ | &#x22f1; |
 | &emsp;&nbsp;_**Age**_                 | `\p{Age=5.2}`, `\P{age=7.0}`, `\p{^age=8.0}`            | &#x2713; |
 | &emsp;&nbsp;_**Blocks**_              | `\p{InArmenian}`, `\P{InKhmer}`, `\p{^InThai}`          | &#x2713; |
 | &emsp;&nbsp;_**Classes**_             | `\p{Alpha}`, `\P{Space}`, `\p{^Alnum}`                  | &#x2713; |
@@ -371,6 +384,10 @@ _Note that not all of these are available in all versions of Ruby_
 | &emsp;&nbsp;_**Scripts**_             | `\p{Arabic}`, `\P{Hiragana}`, `\p{^Greek}`              | &#x2713; |
 | &emsp;&nbsp;_**Simple**_              | `\p{Dash}`, `\p{Extender}`, `\p{^Hyphen}`               | &#x2713; |
 
+**\[1\]**: As of Ruby 3.1, meta and control sequences are [pre-processed to hex escapes when used in Regexp literals](
+ https://github.com/ruby/ruby/commit/11ae581a4a7f5d5f5ec6378872eab8f25381b1b9 ), so they will only reach the
+scanner and will only be emitted if a String or a Regexp that has been built with the `::new` constructor is scanned.
+
 ##### Inapplicable Features
 
 Some modifiers, like `o` and `s`, apply to the **Regexp** object itself and do not
@@ -384,7 +401,6 @@ expressions library (Onigmo). They are not supported by the scanner.
   - **Quotes**: `\Q...\E` _[[See]](https://github.com/k-takata/Onigmo/blob/7911409/doc/RE#L499)_
   - **Capture History**: `(?@...)`, `(?@<name>...)` _[[See]](https://github.com/k-takata/Onigmo/blob/7911409/doc/RE#L550)_
 
-
 See something missing? Please submit an [issue](https://github.com/ammar/regexp_parser/issues)
 
 _**Note**: Attempting to process expressions with unsupported syntax features can raise an error,
@@ -392,26 +408,14 @@ or incorrectly return tokens/objects as literals._
 
 
 ## Testing
-To run the tests simply run rake from the root directory, as 'test' is the default task.
-
-It generates the scanner's code from the Ragel source files and runs all the tests, thus it requires Ragel to be installed.
+To run the tests simply run rake from the root directory.
 
-The tests use RSpec. They can also be run with the test runner that whitelists some warnings:
-
-```
-bin/test
-```
-
-You can run a specific test like so:
-
-```
-bin/test spec/scanner/properties_spec.rb
-```
+The default task generates the scanner's code from the Ragel source files and runs all the specs, thus it requires Ragel to be installed.
 
-Note that changes to Ragel files will not be reflected when running `rspec` or `bin/test`, so you might want to run:
+Note that changes to Ragel files will not be reflected when running `rspec` on its own, so to run individual tests you might want to run:
 
 ```
-rake ragel:rb && bin/test spec/scanner/properties_spec.rb
+rake ragel:rb && rspec spec/scanner/properties_spec.rb
 ```
 
 ## Building
@@ -443,7 +447,7 @@ Projects using regexp_parser.
 
 - [meta_re](https://github.com/ammar/meta_re) is a regular expression preprocessor with alias support.
 
-- [mutant](https://github.com/mbj/mutant) (before v0.9.0) manipulates your regular expressions (amongst others) to see if your tests cover their behavior.
+- [mutant](https://github.com/mbj/mutant) manipulates your regular expressions (amongst others) to see if your tests cover their behavior.
 
 - [rubocop](https://github.com/rubocop-hq/rubocop) is a linter for Ruby that uses regexp_parser to lint Regexps.
 
@@ -476,4 +480,4 @@ Documentation and books used while working on this project.
 
 ---
 ##### Copyright
-_Copyright (c) 2010-2020 Ammar Ali. See LICENSE file for details._
+_Copyright (c) 2010-2022 Ammar Ali. See LICENSE file for details._

data/Rakefile

@@ -1,87 +1,23 @@
+require 'bundler'
 require 'rubygems'
-
+require 'rubygems/package_task'
 require 'rake'
 require 'rake/testtask'
+require 'rspec/core/rake_task'
 
-require 'bundler'
-require 'rubygems/package_task'
-
-
-RAGEL_SOURCE_DIR = File.join(__dir__, 'lib/regexp_parser/scanner')
-RAGEL_OUTPUT_DIR = File.join(__dir__, 'lib/regexp_parser')
-RAGEL_SOURCE_FILES = %w{scanner} # scanner.rl includes property.rl
-
+Dir['tasks/**/*.rake'].each { |file| load(file) }
 
 Bundler::GemHelper.install_tasks
 
+RSpec::Core::RakeTask.new(:spec)
 
 task :default => [:'test:full']
 
 namespace :test do
-  task full: :'ragel:rb' do
-    sh 'bin/test'
-  end
+  task full: [:'ragel:rb', :spec]
 end
 
-namespace :ragel do
-  desc "Process the ragel source files and output ruby code"
-  task :rb do
-    RAGEL_SOURCE_FILES.each do |source_file|
-      output_file = "#{RAGEL_OUTPUT_DIR}/#{source_file}.rb"
-      # using faster flat table driven FSM, about 25% larger code, but about 30% faster
-      sh "ragel -F1 -R #{RAGEL_SOURCE_DIR}/#{source_file}.rl -o #{output_file}"
-
-      contents = File.read(output_file)
-
-      File.open(output_file, 'r+') do |file|
-        contents = "# -*- warn-indent:false;  -*-\n" + contents
-
-        file.write(contents)
-      end
-    end
-  end
-
-  desc "Delete the ragel generated source file(s)"
-  task :clean do
-    RAGEL_SOURCE_FILES.each do |file|
-      sh "rm -f #{RAGEL_OUTPUT_DIR}/#{file}.rb"
-    end
-  end
-end
-
-
 # Add ragel task as a prerequisite for building the gem to ensure that the
 # latest scanner code is generated and included in the build.
 desc "Runs ragel:rb before building the gem"
 task :build => ['ragel:rb']
-
-
-namespace :props do
-  desc 'Write new property value hashes for the properties scanner'
-  task :update do
-    require 'regexp_property_values'
-    RegexpPropertyValues.update
-    dir = File.join(__dir__, 'lib/regexp_parser/scanner/properties')
-
-    require 'psych'
-    write_hash_to_file = ->(hash, path) do
-      File.open(path, 'w') do |f|
-        f.puts '#',
-               "# THIS FILE IS AUTO-GENERATED BY `rake props:update`, DO NOT EDIT",
-               '#',
-               hash.sort.to_h.to_yaml
-      end
-      puts "Wrote #{hash.count} aliases to `#{path}`"
-    end
-
-    long_names_to_tokens = RegexpPropertyValues.all.map do |val|
-      [val.identifier, val.full_name.downcase]
-    end
-    write_hash_to_file.call(long_names_to_tokens, "#{dir}/long.yml")
-
-    short_names_to_tokens = RegexpPropertyValues.alias_hash.map do |k, v|
-      [k.identifier, v.full_name.downcase]
-    end
-    write_hash_to_file.call(short_names_to_tokens, "#{dir}/short.yml")
-  end
-end

data/lib/regexp_parser/expression/base.rb

@@ -0,0 +1,123 @@
+module Regexp::Expression
+  class Base
+    attr_accessor :type, :token
+    attr_accessor :text, :ts
+    attr_accessor :level, :set_level, :conditional_level, :nesting_level
+
+    attr_accessor :quantifier
+    attr_accessor :options
+
+    def initialize(token, options = {})
+      self.type              = token.type
+      self.token             = token.token
+      self.text              = token.text
+      self.ts                = token.ts
+      self.level             = token.level
+      self.set_level         = token.set_level
+      self.conditional_level = token.conditional_level
+      self.nesting_level     = 0
+      self.quantifier        = nil
+      self.options           = options
+    end
+
+    def initialize_copy(orig)
+      self.text       = (orig.text       ? orig.text.dup         : nil)
+      self.options    = (orig.options    ? orig.options.dup      : nil)
+      self.quantifier = (orig.quantifier ? orig.quantifier.clone : nil)
+      super
+    end
+
+    def to_re(format = :full)
+      ::Regexp.new(to_s(format))
+    end
+
+    alias :starts_at :ts
+
+    def base_length
+      to_s(:base).length
+    end
+
+    def full_length
+      to_s.length
+    end
+
+    def offset
+      [starts_at, full_length]
+    end
+
+    def coded_offset
+      '@%d+%d' % offset
+    end
+
+    def to_s(format = :full)
+      "#{text}#{quantifier_affix(format)}"
+    end
+
+    def quantifier_affix(expression_format)
+      quantifier.to_s if quantified? && expression_format != :base
+    end
+
+    def terminal?
+      !respond_to?(:expressions)
+    end
+
+    def quantify(token, text, min = nil, max = nil, mode = :greedy)
+      self.quantifier = Quantifier.new(token, text, min, max, mode)
+    end
+
+    def unquantified_clone
+      clone.tap { |exp| exp.quantifier = nil }
+    end
+
+    def quantified?
+      !quantifier.nil?
+    end
+
+    # Deprecated. Prefer `#repetitions` which has a more uniform interface.
+    def quantity
+      return [nil,nil] unless quantified?
+      [quantifier.min, quantifier.max]
+    end
+
+    def repetitions
+      return 1..1 unless quantified?
+      min = quantifier.min
+      max = quantifier.max < 0 ? Float::INFINITY : quantifier.max
+      range = min..max
+      # fix Range#minmax on old Rubies - https://bugs.ruby-lang.org/issues/15807
+      if RUBY_VERSION.to_f < 2.7
+        range.define_singleton_method(:minmax) { [min, max] }
+      end
+      range
+    end
+
+    def greedy?
+      quantified? and quantifier.greedy?
+    end
+
+    def reluctant?
+      quantified? and quantifier.reluctant?
+    end
+    alias :lazy? :reluctant?
+
+    def possessive?
+      quantified? and quantifier.possessive?
+    end
+
+    def attributes
+      {
+        type:              type,
+        token:             token,
+        text:              to_s(:base),
+        starts_at:         ts,
+        length:            full_length,
+        level:             level,
+        set_level:         set_level,
+        conditional_level: conditional_level,
+        options:           options,
+        quantifier:        quantified? ? quantifier.to_h : nil,
+      }
+    end
+    alias :to_h :attributes
+  end
+end

data/lib/regexp_parser/expression/classes/anchor.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   module Anchor
     class Base < Regexp::Expression::Base; end
 
@@ -22,5 +21,4 @@ module Regexp::Expression
     EOS      = EndOfString
     EOSobEOL = EndOfStringOrBeforeEndOfLine
   end
-
 end

data/lib/regexp_parser/expression/classes/{escape.rb → escape_sequence.rb}

@@ -1,16 +1,22 @@
 module Regexp::Expression
+  # TODO: unify naming with Token::Escape, on way or the other, in v3.0.0
   module EscapeSequence
     class Base < Regexp::Expression::Base
-      require 'yaml'
-
-      def char
-        # poor man's unescape without using eval
-        YAML.load(%Q(---\n"#{text}"\n))
-      end
-
       def codepoint
         char.ord
       end
+
+      if ''.respond_to?(:undump)
+        def char
+          %("#{text}").undump
+        end
+      else
+        # poor man's unescape without using eval
+        require 'yaml'
+        def char
+          YAML.load(%Q(---\n"#{text}"\n))
+        end
+      end
     end
 
     class Literal < EscapeSequence::Base

data/lib/regexp_parser/expression/classes/free_space.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   class FreeSpace < Regexp::Expression::Base
     def quantify(_token, _text, _min = nil, _max = nil, _mode = :greedy)
       raise Regexp::Parser::Error, 'Can not quantify a free space object'
@@ -13,5 +12,4 @@ module Regexp::Expression
       text << exp.text
     end
   end
-
 end

data/lib/regexp_parser/expression/classes/literal.rb

@@ -1,7 +1,3 @@
 module Regexp::Expression
-
-  class Literal < Regexp::Expression::Base
-    # Obviously nothing special here, yet.
-  end
-
+  class Literal < Regexp::Expression::Base; end
 end

data/lib/regexp_parser/expression/classes/property.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   module UnicodeProperty
     class Base < Regexp::Expression::Base
       def negative?
@@ -116,5 +115,4 @@ module Regexp::Expression
     class Script  < UnicodeProperty::Base; end
     class Block   < UnicodeProperty::Base; end
   end
-
 end # module Regexp::Expression

data/lib/regexp_parser/expression/classes/root.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   class Root < Regexp::Expression::Subexpression
     def self.build(options = {})
       new(build_token, options)

data/lib/regexp_parser/expression/classes/type.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   module CharacterType
     class Base < Regexp::Expression::Base; end
 
@@ -15,5 +14,4 @@ module Regexp::Expression
     class Linebreak        < CharacterType::Base; end
     class ExtendedGrapheme < CharacterType::Base; end
   end
-
 end

data/lib/regexp_parser/expression/methods/strfregexp.rb

@@ -43,7 +43,7 @@ module Regexp::Expression
 
       # Order is important! Fields that use other fields in their
       # definition must appear before the fields they use.
-      part_keys = %w{a m b o i l x s e S y k c q Q z Z t ~t T >}
+      part_keys = %w[a m b o i l x s e S y k c q Q z Z t ~t T >]
       part.keys.each {|k| part[k] = "<?#{k}?>"}
 
       part['>'] = print_level ? ('  ' * (print_level + indent_offset)) : ''

data/lib/regexp_parser/expression/quantifier.rb

@@ -1,6 +1,6 @@
 module Regexp::Expression
   class Quantifier
-    MODES = [:greedy, :possessive, :reluctant]
+    MODES = %i[greedy possessive reluctant]
 
     attr_reader :token, :text, :min, :max, :mode
 

data/lib/regexp_parser/expression/sequence.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   # A sequence of expressions. Differs from a Subexpressions by how it handles
   # quantifiers, as it applies them to its last element instead of itself as
   # a whole subexpression.

data/lib/regexp_parser/expression/subexpression.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   class Subexpression < Regexp::Expression::Base
     include Enumerable