regexp_parser 2.0.2 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3d9161fb969c7c5aac0798a31054ed4f0d5da8992b8167afa7025492d1042c9
-  data.tar.gz: 8321d42545c5dbfc810ad3f32fb03677e95d1dba3f3145f20af2e655f1ca45c3
+  metadata.gz: 332259c898b9b344e10961053bb2b761f4dd5530182a5f6195639dba9cbb99f9
+  data.tar.gz: b537f9bd23db799ee562494633f1e8423501651540a04b634ae07dfe8f3b19c3
 SHA512:
-  metadata.gz: def5d282e5720c260bcb581e666704d04579534f8e04986867382836e58f4949d1eea0d3f7b3b2efdb71c5796b3bc6e5c4f92594aab9c9cd61985de18627c026
-  data.tar.gz: 3e2671325e18a6b4d61b5cd9da410f21a3cdc24382f415b9517498437d2c5e8e473ebe481aeb610667f33aa9d538d05d43b3fae5565b1fb856690e81377e82e5
+  metadata.gz: 393ecc1cc20189e4a79252e6acf6dab7dd6dc07ba9c47ae7479746eaf8ebe2ccfd1ebcb82fd027edc2c5c938eb490f2f36a93587d2405a54017e0e2727a35a15
+  data.tar.gz: 6c961232ce5f3f409c91d0b66dd23c809e92f47aa6c1f94f2f1929e8eeccfb4bc25fcdf5935fc968d7e0c0ae632992a6d38bc8e982858f2da996a8eac54d3c89

data/CHANGELOG.md

@@ -1,5 +1,59 @@
 ## [Unreleased]
 
+## [2.2.0] - 2021-12-04 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- Added support for 13 new unicode properties introduced in Ruby 3.1.0-dev
+
+## [2.1.1] - 2021-02-23 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Fixed
+
+- fixed `NameError` when requiring only `'regexp_parser/scanner'` in v2.1.0
+  * thanks to [Jared White and Sam Ruby](https://github.com/ruby2js/ruby2js) for the report
+
+## [2.1.0] - 2021-02-22 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- common ancestor for all scanning/parsing/lexing errors
+  * `Regexp::Parser::Error` can now be rescued as a catch-all
+  * the following errors (and their many descendants) now inherit from it:
+    - `Regexp::Expression::Conditional::TooManyBranches`
+    - `Regexp::Parser::ParserError`
+    - `Regexp::Scanner::ScannerError`
+    - `Regexp::Scanner::ValidationError`
+    - `Regexp::Syntax::SyntaxError`
+  * it replaces `ArgumentError` in some rare cases (`Regexp::Parser.parse('?')`)
+  * thanks to [sandstrom](https://github.com/sandstrom) for the cue
+
+### Fixed
+
+- fixed scanning of whole-pattern recursion calls `\g<0>` and `\g'0'`
+  * a regression in v2.0.1 had caused them to be scanned as literals
+- fixed scanning of some backreference and subexpression call edge cases
+  * e.g. `\k<+1>`, `\g<x-1>`
+- fixed tokenization of some escapes in character sets
+  * `.`, `|`, `{`, `}`, `(`, `)`, `^`, `$`, `?`, `+`, `*`
+  * all of these correctly emitted `#type` `:literal` and `#token` `:literal` if *not* escaped
+  * if escaped, they emitted e.g. `#type` `:escape` and `#token` `:group_open` for `[\(]`
+  * the escaped versions now correctly emit `#type` `:escape` and `#token` `:literal`
+- fixed handling of control/metacontrol escapes in character sets
+  * e.g. `[\cX]`, `[\M-\C-X]`
+  * they were misread as bunch of individual literals, escapes, and ranges
+- fixed some cases where calling `#dup`/`#clone` on expressions led to shared state
+
+## [2.0.3] - 2020-12-28 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Fixed
+
+- fixed error when scanning some unlikely and redundant but valid charset patterns
+  * e.g. `/[[.a-b.]]/`, `/[[=e=]]/`,
+- fixed ancestry of some error classes related to syntax version lookup
+  * `NotImplementedError`, `InvalidVersionNameError`, `UnknownSyntaxNameError`
+  * they now correctly inherit from `Regexp::Syntax::SyntaxError` instead of Rubys `::SyntaxError`
+
 ## [2.0.2] - 2020-12-25 - [Janosch Müller](mailto:janosch84@gmail.com)
 
 ### Fixed

data/Gemfile

@@ -6,5 +6,9 @@ group :development, :test do
   gem 'ice_nine', '~> 0.11.2'
   gem 'rake', '~> 13.0'
   gem 'regexp_property_values', '~> 1.0'
-  gem 'rspec', '~> 3.8'
+  gem 'rspec', '~> 3.10'
+  if RUBY_VERSION.to_f >= 2.7
+    gem 'gouteur'
+    gem 'rubocop', '~> 1.7'
+  end
 end

data/README.md

@@ -1,6 +1,9 @@
 # Regexp::Parser
 
-[![Gem Version](https://badge.fury.io/rb/regexp_parser.svg)](http://badge.fury.io/rb/regexp_parser) [![Build Status](https://github.com/ammar/regexp_parser/workflows/tests/badge.svg)](https://github.com/ammar/regexp_parser/actions) [![Code Climate](https://codeclimate.com/github/ammar/regexp_parser.svg)](https://codeclimate.com/github/ammar/regexp_parser/badges)
+[![Gem Version](https://badge.fury.io/rb/regexp_parser.svg)](http://badge.fury.io/rb/regexp_parser)
+[![Build Status](https://github.com/ammar/regexp_parser/workflows/tests/badge.svg)](https://github.com/ammar/regexp_parser/actions)
+[![Build Status](https://github.com/ammar/regexp_parser/workflows/gouteur/badge.svg)](https://github.com/ammar/regexp_parser/actions)
+[![Code Climate](https://codeclimate.com/github/ammar/regexp_parser.svg)](https://codeclimate.com/github/ammar/regexp_parser/badges)
 
 A Ruby gem for tokenizing, parsing, and transforming regular expressions.
 
@@ -357,12 +360,12 @@ _Note that not all of these are available in all versions of Ruby_
 |   _**Reluctant** (Lazy)_    | `??`, `*?`, `+?`, `{m,M}?`                              | ✓ |
 |   _**Possessive**_          | `?+`, `*+`, `++`, `{m,M}+`                              | ✓ |
 | **String Escapes**                    |                                                         | ⋱ |
-|   _**Control**_             | `\C-C`, `\cD`                                           | ✓ |
+|   _**Control** \[1\]_       | `\C-C`, `\cD`                                           | ✓ |
 |   _**Hex**_                 | `\x20`, `\x{701230}`                                    | ✓ |
-|   _**Meta**_                | `\M-c`, `\M-\C-C`, `\M-\cC`, `\C-\M-C`, `\c\M-C`        | ✓ |
+|   _**Meta** \[1\]_          | `\M-c`, `\M-\C-C`, `\M-\cC`, `\C-\M-C`, `\c\M-C`        | ✓ |
 |   _**Octal**_               | `\0`, `\01`, `\012`                                     | ✓ |
 |   _**Unicode**_             | `\uHHHH`, `\u{H+ H+}`                                   | ✓ |
-| **Unicode Properties**                | _<sub>([Unicode 11.0.0](http://www.unicode.org/versions/Unicode11.0.0/))</sub>_ | &#x22f1; |
+| **Unicode Properties**                | _<sub>([Unicode 13.0.0](https://www.unicode.org/versions/Unicode13.0.0/))</sub>_ | &#x22f1; |
 | &emsp;&nbsp;_**Age**_                 | `\p{Age=5.2}`, `\P{age=7.0}`, `\p{^age=8.0}`            | &#x2713; |
 | &emsp;&nbsp;_**Blocks**_              | `\p{InArmenian}`, `\P{InKhmer}`, `\p{^InThai}`          | &#x2713; |
 | &emsp;&nbsp;_**Classes**_             | `\p{Alpha}`, `\P{Space}`, `\p{^Alnum}`                  | &#x2713; |
@@ -371,6 +374,10 @@ _Note that not all of these are available in all versions of Ruby_
 | &emsp;&nbsp;_**Scripts**_             | `\p{Arabic}`, `\P{Hiragana}`, `\p{^Greek}`              | &#x2713; |
 | &emsp;&nbsp;_**Simple**_              | `\p{Dash}`, `\p{Extender}`, `\p{^Hyphen}`               | &#x2713; |
 
+**\[1\]**: As of Ruby 3.1, meta and control sequences are [pre-processed to hex escapes when used in Regexp literals](
+ https://github.com/ruby/ruby/commit/11ae581a4a7f5d5f5ec6378872eab8f25381b1b9 ), so they will only reach the
+scanner and will only be emitted if a String or a Regexp that has been built with the `::new` constructor is scanned.
+
 ##### Inapplicable Features
 
 Some modifiers, like `o` and `s`, apply to the **Regexp** object itself and do not
@@ -384,7 +391,6 @@ expressions library (Onigmo). They are not supported by the scanner.
   - **Quotes**: `\Q...\E` _[[See]](https://github.com/k-takata/Onigmo/blob/7911409/doc/RE#L499)_
   - **Capture History**: `(?@...)`, `(?@<name>...)` _[[See]](https://github.com/k-takata/Onigmo/blob/7911409/doc/RE#L550)_
 
-
 See something missing? Please submit an [issue](https://github.com/ammar/regexp_parser/issues)
 
 _**Note**: Attempting to process expressions with unsupported syntax features can raise an error,
@@ -392,26 +398,14 @@ or incorrectly return tokens/objects as literals._
 
 
 ## Testing
-To run the tests simply run rake from the root directory, as 'test' is the default task.
-
-It generates the scanner's code from the Ragel source files and runs all the tests, thus it requires Ragel to be installed.
-
-The tests use RSpec. They can also be run with the test runner that whitelists some warnings:
-
-```
-bin/test
-```
-
-You can run a specific test like so:
+To run the tests simply run rake from the root directory.
 
-```
-bin/test spec/scanner/properties_spec.rb
-```
+The default task generates the scanner's code from the Ragel source files and runs all the specs, thus it requires Ragel to be installed.
 
-Note that changes to Ragel files will not be reflected when running `rspec` or `bin/test`, so you might want to run:
+Note that changes to Ragel files will not be reflected when running `rspec` on its own, so to run individual tests you might want to run:
 
 ```
-rake ragel:rb && bin/test spec/scanner/properties_spec.rb
+rake ragel:rb && rspec spec/scanner/properties_spec.rb
 ```
 
 ## Building

data/Rakefile

@@ -1,35 +1,31 @@
+require 'bundler'
 require 'rubygems'
-
+require 'rubygems/package_task'
 require 'rake'
 require 'rake/testtask'
+require 'rspec/core/rake_task'
 
-require 'bundler'
-require 'rubygems/package_task'
-
-
-RAGEL_SOURCE_DIR = File.expand_path '../lib/regexp_parser/scanner', __FILE__
-RAGEL_OUTPUT_DIR = File.expand_path '../lib/regexp_parser', __FILE__
+RAGEL_SOURCE_DIR = File.join(__dir__, 'lib/regexp_parser/scanner')
+RAGEL_OUTPUT_DIR = File.join(__dir__, 'lib/regexp_parser')
 RAGEL_SOURCE_FILES = %w{scanner} # scanner.rl includes property.rl
 
-
 Bundler::GemHelper.install_tasks
 
+RSpec::Core::RakeTask.new(:spec)
 
 task :default => [:'test:full']
 
 namespace :test do
-  task full: :'ragel:rb' do
-    sh 'bin/test'
-  end
+  task full: [:'ragel:rb', :spec]
 end
 
 namespace :ragel do
   desc "Process the ragel source files and output ruby code"
   task :rb do
-    RAGEL_SOURCE_FILES.each do |file|
-      output_file = "#{RAGEL_OUTPUT_DIR}/#{file}.rb"
+    RAGEL_SOURCE_FILES.each do |source_file|
+      output_file = "#{RAGEL_OUTPUT_DIR}/#{source_file}.rb"
       # using faster flat table driven FSM, about 25% larger code, but about 30% faster
-      sh "ragel -F1 -R #{RAGEL_SOURCE_DIR}/#{file}.rl -o #{output_file}"
+      sh "ragel -F1 -R #{RAGEL_SOURCE_DIR}/#{source_file}.rl -o #{output_file}"
 
       contents = File.read(output_file)
 
@@ -49,19 +45,17 @@ namespace :ragel do
   end
 end
 
-
 # Add ragel task as a prerequisite for building the gem to ensure that the
 # latest scanner code is generated and included in the build.
 desc "Runs ragel:rb before building the gem"
 task :build => ['ragel:rb']
 
-
 namespace :props do
   desc 'Write new property value hashes for the properties scanner'
   task :update do
     require 'regexp_property_values'
     RegexpPropertyValues.update
-    dir = File.expand_path('../lib/regexp_parser/scanner/properties', __FILE__)
+    dir = File.join(__dir__, 'lib/regexp_parser/scanner/properties')
 
     require 'psych'
     write_hash_to_file = ->(hash, path) do

data/lib/regexp_parser/error.rb

@@ -0,0 +1,4 @@
+class Regexp::Parser
+  # base class for all gem-specific errors (inherited but never raised itself)
+  class Error < StandardError; end
+end

data/lib/regexp_parser/expression/base.rb

@@ -0,0 +1,123 @@
+module Regexp::Expression
+  class Base
+    attr_accessor :type, :token
+    attr_accessor :text, :ts
+    attr_accessor :level, :set_level, :conditional_level, :nesting_level
+
+    attr_accessor :quantifier
+    attr_accessor :options
+
+    def initialize(token, options = {})
+      self.type              = token.type
+      self.token             = token.token
+      self.text              = token.text
+      self.ts                = token.ts
+      self.level             = token.level
+      self.set_level         = token.set_level
+      self.conditional_level = token.conditional_level
+      self.nesting_level     = 0
+      self.quantifier        = nil
+      self.options           = options
+    end
+
+    def initialize_copy(orig)
+      self.text       = (orig.text       ? orig.text.dup         : nil)
+      self.options    = (orig.options    ? orig.options.dup      : nil)
+      self.quantifier = (orig.quantifier ? orig.quantifier.clone : nil)
+      super
+    end
+
+    def to_re(format = :full)
+      ::Regexp.new(to_s(format))
+    end
+
+    alias :starts_at :ts
+
+    def base_length
+      to_s(:base).length
+    end
+
+    def full_length
+      to_s.length
+    end
+
+    def offset
+      [starts_at, full_length]
+    end
+
+    def coded_offset
+      '@%d+%d' % offset
+    end
+
+    def to_s(format = :full)
+      "#{text}#{quantifier_affix(format)}"
+    end
+
+    def quantifier_affix(expression_format)
+      quantifier.to_s if quantified? && expression_format != :base
+    end
+
+    def terminal?
+      !respond_to?(:expressions)
+    end
+
+    def quantify(token, text, min = nil, max = nil, mode = :greedy)
+      self.quantifier = Quantifier.new(token, text, min, max, mode)
+    end
+
+    def unquantified_clone
+      clone.tap { |exp| exp.quantifier = nil }
+    end
+
+    def quantified?
+      !quantifier.nil?
+    end
+
+    # Deprecated. Prefer `#repetitions` which has a more uniform interface.
+    def quantity
+      return [nil,nil] unless quantified?
+      [quantifier.min, quantifier.max]
+    end
+
+    def repetitions
+      return 1..1 unless quantified?
+      min = quantifier.min
+      max = quantifier.max < 0 ? Float::INFINITY : quantifier.max
+      range = min..max
+      # fix Range#minmax on old Rubies - https://bugs.ruby-lang.org/issues/15807
+      if RUBY_VERSION.to_f < 2.7
+        range.define_singleton_method(:minmax) { [min, max] }
+      end
+      range
+    end
+
+    def greedy?
+      quantified? and quantifier.greedy?
+    end
+
+    def reluctant?
+      quantified? and quantifier.reluctant?
+    end
+    alias :lazy? :reluctant?
+
+    def possessive?
+      quantified? and quantifier.possessive?
+    end
+
+    def attributes
+      {
+        type:              type,
+        token:             token,
+        text:              to_s(:base),
+        starts_at:         ts,
+        length:            full_length,
+        level:             level,
+        set_level:         set_level,
+        conditional_level: conditional_level,
+        options:           options,
+        quantifier:        quantified? ? quantifier.to_h : nil,
+      }
+    end
+    alias :to_h :attributes
+  end
+end

data/lib/regexp_parser/expression/classes/anchor.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   module Anchor
     class Base < Regexp::Expression::Base; end
 
@@ -22,5 +21,4 @@ module Regexp::Expression
     EOS      = EndOfString
     EOSobEOL = EndOfStringOrBeforeEndOfLine
   end
-
 end

data/lib/regexp_parser/expression/classes/{backref.rb → backreference.rb}

@@ -2,6 +2,11 @@ module Regexp::Expression
   module Backreference
     class Base < Regexp::Expression::Base
       attr_accessor :referenced_expression
+
+      def initialize_copy(orig)
+        self.referenced_expression = orig.referenced_expression.dup
+        super
+      end
     end
 
     class Number < Backreference::Base

data/lib/regexp_parser/expression/classes/{set → character_set}/range.rb

@@ -7,7 +7,8 @@ module Regexp::Expression
       alias :ts :starts_at
 
       def <<(exp)
-        complete? && raise("Can't add more than 2 expressions to a Range")
+        complete? and raise Regexp::Parser::Error,
+          "Can't add more than 2 expressions to a Range"
         super
       end
 

data/lib/regexp_parser/expression/classes/conditional.rb

@@ -1,6 +1,6 @@
 module Regexp::Expression
   module Conditional
-    class TooManyBranches < StandardError
+    class TooManyBranches < Regexp::Parser::Error
       def initialize
         super('The conditional expression has more than 2 branches')
       end
@@ -15,6 +15,11 @@ module Regexp::Expression
         ref = text.tr("'<>()", "")
         ref =~ /\D/ ? ref : Integer(ref)
       end
+
+      def initialize_copy(orig)
+        self.referenced_expression = orig.referenced_expression.dup
+        super
+      end
     end
 
     class Branch < Regexp::Expression::Sequence; end
@@ -53,6 +58,11 @@ module Regexp::Expression
       def to_s(format = :full)
         "#{text}#{condition}#{branches.join('|')})#{quantifier_affix(format)}"
       end
+
+      def initialize_copy(orig)
+        self.referenced_expression = orig.referenced_expression.dup
+        super
+      end
     end
   end
 end

data/lib/regexp_parser/expression/classes/{escape.rb → escape_sequence.rb}

@@ -1,4 +1,5 @@
 module Regexp::Expression
+  # TODO: unify naming with Token::Escape, on way or the other, in v3.0.0
   module EscapeSequence
     class Base < Regexp::Expression::Base
       require 'yaml'

data/lib/regexp_parser/expression/classes/free_space.rb

@@ -1,8 +1,7 @@
 module Regexp::Expression
-
   class FreeSpace < Regexp::Expression::Base
     def quantify(_token, _text, _min = nil, _max = nil, _mode = :greedy)
-      raise "Can not quantify a free space object"
+      raise Regexp::Parser::Error, 'Can not quantify a free space object'
     end
   end
 
@@ -13,5 +12,4 @@ module Regexp::Expression
       text << exp.text
     end
   end
-
 end

data/lib/regexp_parser/expression/classes/group.rb

@@ -35,6 +35,11 @@ module Regexp::Expression
     class Atomic  < Group::Base; end
     class Options < Group::Base
       attr_accessor :option_changes
+
+      def initialize_copy(orig)
+        self.option_changes = orig.option_changes.dup
+        super
+      end
     end
 
     class Capture < Group::Base
@@ -53,7 +58,7 @@ module Regexp::Expression
         super
       end
 
-      def initialize_clone(orig)
+      def initialize_copy(orig)
         @name = orig.name.dup
         super
       end

data/lib/regexp_parser/expression/classes/literal.rb

@@ -1,7 +1,3 @@
 module Regexp::Expression
-
-  class Literal < Regexp::Expression::Base
-    # Obviously nothing special here, yet.
-  end
-
+  class Literal < Regexp::Expression::Base; end
 end

data/lib/regexp_parser/expression/classes/property.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   module UnicodeProperty
     class Base < Regexp::Expression::Base
       def negative?
@@ -7,7 +6,7 @@ module Regexp::Expression
       end
 
       def name
-        text =~ /\A\\[pP]\{([^}]+)\}\z/; $1
+        text[/\A\\[pP]\{([^}]+)\}\z/, 1]
       end
 
       def shortcut
@@ -116,5 +115,4 @@ module Regexp::Expression
     class Script  < UnicodeProperty::Base; end
     class Block   < UnicodeProperty::Base; end
   end
-
 end # module Regexp::Expression

data/lib/regexp_parser/expression/classes/root.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   class Root < Regexp::Expression::Subexpression
     def self.build(options = {})
       new(build_token, options)

data/lib/regexp_parser/expression/classes/type.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   module CharacterType
     class Base < Regexp::Expression::Base; end
 
@@ -15,5 +14,4 @@ module Regexp::Expression
     class Linebreak        < CharacterType::Base; end
     class ExtendedGrapheme < CharacterType::Base; end
   end
-
 end

data/lib/regexp_parser/expression/quantifier.rb

@@ -1,6 +1,6 @@
 module Regexp::Expression
   class Quantifier
-    MODES = [:greedy, :possessive, :reluctant]
+    MODES = %i[greedy possessive reluctant]
 
     attr_reader :token, :text, :min, :max, :mode
 
@@ -12,7 +12,7 @@ module Regexp::Expression
       @max   = max
     end
 
-    def initialize_clone(orig)
+    def initialize_copy(orig)
       @text = orig.text.dup
       super
     end

data/lib/regexp_parser/expression/sequence.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   # A sequence of expressions. Differs from a Subexpressions by how it handles
   # quantifiers, as it applies them to its last element instead of itself as
   # a whole subexpression.
@@ -41,17 +40,11 @@ module Regexp::Expression
     alias :ts :starts_at
 
     def quantify(token, text, min = nil, max = nil, mode = :greedy)
-      offset = -1
-      target = expressions[offset]
-      while target.is_a?(FreeSpace)
-        target = expressions[offset -= 1]
-      end
-
-      target || raise(ArgumentError, "No valid target found for '#{text}' "\
-                                     'quantifier')
+      target = expressions.reverse.find { |exp| !exp.is_a?(FreeSpace) }
+      target or raise Regexp::Parser::Error,
+        "No valid target found for '#{text}' quantifier"
 
       target.quantify(token, text, min, max, mode)
     end
   end
-
 end

data/lib/regexp_parser/expression/subexpression.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   class Subexpression < Regexp::Expression::Base
     include Enumerable
 
@@ -12,7 +11,7 @@ module Regexp::Expression
     end
 
     # Override base method to clone the expressions as well.
-    def initialize_clone(orig)
+    def initialize_copy(orig)
       self.expressions = orig.expressions.map(&:clone)
       super
     end