regexp_parser 2.1.1 → 2.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 077b8a0c90d90cf46e44671ec1335a5373eef72c61a0bcf4de43ba5217a188c3
-  data.tar.gz: b9aed868af73adcdf40c09720c5d10091b25a53b25a792717ceb5591039a2931
+  metadata.gz: f871ec3cdea5a594f72f5386f1b344710e6204f7307ba40d966653197f526be8
+  data.tar.gz: dd93c880f29ec77531faa2379fbfc8e34a9b67680664c6a3477d38afeaa1809a
 SHA512:
-  metadata.gz: 9c04d9a6434c6e3f322e97e8e2a1c86b3ddda88bd8821368a37b92f5836e4c3df1dc27a79165303420c3e8d5eea31bda1483824da01a40ce30961b645ba65ddd
-  data.tar.gz: 01e5c261e9dca0c4df7c696128dbc0520ca40aa6b9393cc8d6c3bdb8386470aeb773566000b811f98c1407038216c8d2c0b444c7955ea5a881ac759796f8a440
+  metadata.gz: 45e52ab0ce7bec3e4a275efa3828532778c49e8d36eec1ea82a43755a87abc9eee97e986027aa8f5c64fd604f15164d2ad4f37e5d6e22a5a1e3e9da6788271b9
+  data.tar.gz: 1f5514f3252294d9fe0877cff1d8b0db0400838c97ed78d15bbb794b94595c20d081681e4b1fe9bb6c89be7749514d8b2b8cf385360d002cd89e2a76ce6d2e63

data/CHANGELOG.md

@@ -1,5 +1,93 @@
 ## [Unreleased]
 
+### Added
+
+- `Regexp::Expression::Base.construct` and `.token_class` methods
+
+## [2.4.0] - 2022-05-09 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Fixed
+
+- fixed interpretation of `+` and `?` after interval quantifiers (`{n,n}`)
+  - they used to be treated as reluctant or possessive mode indicators
+  - however, Ruby does not support these modes for interval quantifiers
+  - they are now treated as chained quantifiers instead, as Ruby does it
+  - c.f. [#3](https://github.com/ammar/regexp_parser/issues/3)
+- fixed `Expression::Base#nesting_level` for some tree rewrite cases
+  - e.g. the alternatives in `/a|[b]/` had an inconsistent nesting_level
+- fixed `Scanner` accepting invalid posix classes, e.g. `[[:foo:]]`
+  - they raise a `SyntaxError` when used in a Regexp, so could only be passed as String
+  - they now raise a `Regexp::Scanner::ValidationError` in the `Scanner`
+
+### Added
+
+- added `Expression::Base#==` for (deep) comparison of expressions
+- added `Expression::Base#parts`
+  - returns the text elements and subexpressions of an expression
+  - e.g. `parse(/(a)/)[0].parts # => ["(", #<Literal @text="a"...>, ")"]`
+- added `Expression::Base#te` (a.k.a. token end index)
+  - `Expression::Subexpression` always had `#te`, only terminal nodes lacked it so far
+- made some `Expression::Base` methods available on `Quantifier` instances, too
+  - `#type`, `#type?`, `#is?`, `#one_of?`, `#options`, `#terminal?`
+  - `#base_length`, `#full_length`, `#starts_at`, `#te`, `#ts`, `#offset`
+  - `#conditional_level`, `#level`, `#nesting_level` , `#set_level`
+  - this allows a more unified handling with `Expression::Base` instances
+- allowed `Quantifier#initialize` to take a token and options Hash like other nodes
+- added a deprecation warning for initializing Quantifiers with 4+ arguments:
+
+    Calling `Expression::Base#quantify` or `Quantifier.new` with 4+ arguments
+    is deprecated.
+
+    It will no longer be supported in regexp_parser v3.0.0.
+
+    Please pass a Regexp::Token instead, e.g. replace `token, text, min, max, mode`
+    with `::Regexp::Token.new(:quantifier, token, text)`. min, max, and mode
+    will be derived automatically.
+
+    Or do `exp.quantifier = Quantifier.construct(token: token, text: str)`.
+
+    This is consistent with how Expression::Base instances are created.
+
+
+## [2.3.1] - 2022-04-24 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Fixed
+
+- removed five inexistent unicode properties from `Syntax#features`
+  - these were never supported by Ruby or the `Regexp::Scanner`
+  - thanks to [Markus Schirp](https://github.com/mbj) for the report
+
+## [2.3.0] - 2022-04-08 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- improved parsing performance through `Syntax` refactoring
+  - instead of fresh `Syntax` instances, pre-loaded constants are now re-used
+  - this approximately doubles the parsing speed for simple regexps
+- added methods to `Syntax` classes to show relative feature sets
+  - e.g. `Regexp::Syntax::V3_2_0.added_features`
+- support for new unicode properties of Ruby 3.2 / Unicode 14.0
+
+## [2.2.1] - 2022-02-11 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Fixed
+
+- fixed Syntax version of absence groups (`(?~...)`)
+  - the lexer accepted them for any Ruby version
+  - now they are only recognized for Ruby >= 2.4.1 in which they were introduced
+- reduced gem size by excluding specs from package
+- removed deprecated `test_files` gemspec setting
+- no longer depend on `yaml`/`psych` (except for Ruby <= 2.4)
+- no longer depend on `set`
+  - `set` was removed from the stdlib and made a standalone gem as of Ruby 3
+  - this made it a hidden/undeclared dependency of `regexp_parser`
+
+## [2.2.0] - 2021-12-04 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- added support for 13 new unicode properties introduced in Ruby 3.1.0
+
 ## [2.1.1] - 2021-02-23 - [Janosch Müller](mailto:janosch84@gmail.com)
 
 ### Fixed
@@ -149,7 +237,7 @@
 
 ### Added
 
-- `Expression#each_expression` and `#traverse` can now be called without a block
+- `Expression::Base#each_expression` and `#traverse` can now be called without a block
   * this returns an `Enumerator` and allows chaining, e.g. `each_expression.select`
   * thanks to [Masataka Kuwabara](https://github.com/pocke)
 
@@ -175,7 +263,7 @@
 - Fixed `Group#option_changes` not accounting for indirectly disabled (overridden) encoding flags
 - Fixed `Scanner` allowing negative encoding options if there were no positive options, e.g. '(?-u)'
 - Fixed `ScannerError` for some valid meta/control sequences such as '\\C-\\\\'
-- Fixed `Expression#match` and `#=~` not working with a single argument
+- Fixed `Expression::Base#match` and `#=~` not working with a single argument
 
 ### [1.5.0] - 2019-05-14 - [Janosch Müller](mailto:janosch84@gmail.com)
 
@@ -183,15 +271,15 @@
 
 - Added `#referenced_expression` for backrefs, subexp calls and conditionals
   * returns the `Group` expression that is being referenced via name or number
-- Added `Expression#repetitions`
+- Added `Expression::Base#repetitions`
   * returns a `Range` of allowed repetitions (`1..1` if there is no quantifier)
   * like `#quantity` but with a more uniform interface
-- Added `Expression#match_length`
+- Added `Expression::Base#match_length`
   * allows to inspect and iterate over String lengths matched by the Expression
 
 ### Fixed
 
-- Fixed `Expression#clone` "direction"
+- Fixed `Expression::Base#clone` "direction"
   * it used to dup ivars onto the callee, leaving only the clone referencing the original objects
   * this will affect you if you call `#eql?`/`#equal?` on expressions or use them as Hash keys
 - Fixed `#clone` results for `Sequences`, e.g. alternations and conditionals
@@ -353,7 +441,7 @@ This release includes several breaking changes, mostly to character sets, #map a
 - Fixed a thread safety issue (issue #45)
 - Some public class methods that were only reliable for
   internal use are now private instance methods (PR #46)
-- Improved the usefulness of Expression#options (issue #43) -
+- Improved the usefulness of Expression::Base#options (issue #43) -
   #options and derived methods such as #i?, #m? and #x? are now
   defined for all Expressions that are affected by such flags.
 - Fixed scanning of whitespace following (?x) (commit 5c94bd2)

data/Gemfile

@@ -5,9 +5,10 @@ gemspec
 group :development, :test do
   gem 'ice_nine', '~> 0.11.2'
   gem 'rake', '~> 13.0'
-  gem 'regexp_property_values', '~> 1.0'
+  gem 'regexp_property_values', '~> 1.3'
   gem 'rspec', '~> 3.10'
   if RUBY_VERSION.to_f >= 2.7
+    gem 'benchmark-ips', '~> 2.1'
     gem 'gouteur'
     gem 'rubocop', '~> 1.7'
   end

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010, 2012-2015,  Ammar Ali
+Copyright (c) 2010, 2012-2022,  Ammar Ali
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation

data/README.md

@@ -1,6 +1,9 @@
 # Regexp::Parser
 
-[![Gem Version](https://badge.fury.io/rb/regexp_parser.svg)](http://badge.fury.io/rb/regexp_parser) [![Build Status](https://github.com/ammar/regexp_parser/workflows/tests/badge.svg)](https://github.com/ammar/regexp_parser/actions) [![Build Status](https://github.com/ammar/regexp_parser/workflows/gouteur/badge.svg)](https://github.com/ammar/regexp_parser/actions) [![Code Climate](https://codeclimate.com/github/ammar/regexp_parser.svg)](https://codeclimate.com/github/ammar/regexp_parser/badges)
+[![Gem Version](https://badge.fury.io/rb/regexp_parser.svg)](http://badge.fury.io/rb/regexp_parser)
+[![Build Status](https://github.com/ammar/regexp_parser/workflows/tests/badge.svg)](https://github.com/ammar/regexp_parser/actions)
+[![Build Status](https://github.com/ammar/regexp_parser/workflows/gouteur/badge.svg)](https://github.com/ammar/regexp_parser/actions)
+[![Code Climate](https://codeclimate.com/github/ammar/regexp_parser.svg)](https://codeclimate.com/github/ammar/regexp_parser/badges)
 
 A Ruby gem for tokenizing, parsing, and transforming regular expressions.
 
@@ -154,31 +157,41 @@ flavor). Syntax classes act as lookup tables, and are layered to create
 flavor variations. Syntax only comes into play in the lexer.
 
 #### Example
-The following instantiates syntax objects for Ruby 2.0, 1.9, 1.8, and
+The following fetches syntax objects for Ruby 2.0, 1.9, 1.8, and
 checks a few of their implementation features.
 
 ```ruby
 require 'regexp_parser'
 
-ruby_20 = Regexp::Syntax.new 'ruby/2.0'
+ruby_20 = Regexp::Syntax.for 'ruby/2.0'
 ruby_20.implements? :quantifier,  :zero_or_one             # => true
 ruby_20.implements? :quantifier,  :zero_or_one_reluctant   # => true
 ruby_20.implements? :quantifier,  :zero_or_one_possessive  # => true
 ruby_20.implements? :conditional, :condition               # => true
 
-ruby_19 = Regexp::Syntax.new 'ruby/1.9'
+ruby_19 = Regexp::Syntax.for 'ruby/1.9'
 ruby_19.implements? :quantifier,  :zero_or_one             # => true
 ruby_19.implements? :quantifier,  :zero_or_one_reluctant   # => true
 ruby_19.implements? :quantifier,  :zero_or_one_possessive  # => true
 ruby_19.implements? :conditional, :condition               # => false
 
-ruby_18 = Regexp::Syntax.new 'ruby/1.8'
+ruby_18 = Regexp::Syntax.for 'ruby/1.8'
 ruby_18.implements? :quantifier,  :zero_or_one             # => true
 ruby_18.implements? :quantifier,  :zero_or_one_reluctant   # => true
 ruby_18.implements? :quantifier,  :zero_or_one_possessive  # => false
 ruby_18.implements? :conditional, :condition               # => false
 ```
 
+Syntax objects can also be queried about their complete and relative feature sets.
+
+```ruby
+require 'regexp_parser'
+
+ruby_20 = Regexp::Syntax.for 'ruby/2.0' # => Regexp::Syntax::V2_0_0
+ruby_20.added_features                  # => { conditional: [...], ... }
+ruby_20.removed_features                # => { property: [:newline], ... }
+ruby_20.features                        # => { anchor: [...], ... }
+```
 
 #### Notes
   * Variations on a token, for example a named group with angle brackets (< and >)
@@ -354,15 +367,15 @@ _Note that not all of these are available in all versions of Ruby_
 | **POSIX Classes**                     | `[:alpha:]`, `[:^digit:]`                               | &#x2713; |
 | **Quantifiers**                       |                                                         | &#x22f1; |
 | &emsp;&nbsp;_**Greedy**_              | `?`, `*`, `+`, `{m,M}`                                  | &#x2713; |
-| &emsp;&nbsp;_**Reluctant** (Lazy)_    | `??`, `*?`, `+?`, `{m,M}?`                              | &#x2713; |
-| &emsp;&nbsp;_**Possessive**_          | `?+`, `*+`, `++`, `{m,M}+`                              | &#x2713; |
+| &emsp;&nbsp;_**Reluctant** (Lazy)_    | `??`, `*?`, `+?` \[1\]                                  | &#x2713; |
+| &emsp;&nbsp;_**Possessive**_          | `?+`, `*+`, `++` \[1\]                                  | &#x2713; |
 | **String Escapes**                    |                                                         | &#x22f1; |
-| &emsp;&nbsp;_**Control**_             | `\C-C`, `\cD`                                           | &#x2713; |
+| &emsp;&nbsp;_**Control** \[2\]_       | `\C-C`, `\cD`                                           | &#x2713; |
 | &emsp;&nbsp;_**Hex**_                 | `\x20`, `\x{701230}`                                    | &#x2713; |
-| &emsp;&nbsp;_**Meta**_                | `\M-c`, `\M-\C-C`, `\M-\cC`, `\C-\M-C`, `\c\M-C`        | &#x2713; |
+| &emsp;&nbsp;_**Meta** \[2\]_          | `\M-c`, `\M-\C-C`, `\M-\cC`, `\C-\M-C`, `\c\M-C`        | &#x2713; |
 | &emsp;&nbsp;_**Octal**_               | `\0`, `\01`, `\012`                                     | &#x2713; |
 | &emsp;&nbsp;_**Unicode**_             | `\uHHHH`, `\u{H+ H+}`                                   | &#x2713; |
-| **Unicode Properties**                | _<sub>([Unicode 11.0.0](http://www.unicode.org/versions/Unicode11.0.0/))</sub>_ | &#x22f1; |
+| **Unicode Properties**                | _<sub>([Unicode 13.0.0](https://www.unicode.org/versions/Unicode13.0.0/))</sub>_ | &#x22f1; |
 | &emsp;&nbsp;_**Age**_                 | `\p{Age=5.2}`, `\P{age=7.0}`, `\p{^age=8.0}`            | &#x2713; |
 | &emsp;&nbsp;_**Blocks**_              | `\p{InArmenian}`, `\P{InKhmer}`, `\p{^InThai}`          | &#x2713; |
 | &emsp;&nbsp;_**Classes**_             | `\p{Alpha}`, `\P{Space}`, `\p{^Alnum}`                  | &#x2713; |
@@ -371,6 +384,14 @@ _Note that not all of these are available in all versions of Ruby_
 | &emsp;&nbsp;_**Scripts**_             | `\p{Arabic}`, `\P{Hiragana}`, `\p{^Greek}`              | &#x2713; |
 | &emsp;&nbsp;_**Simple**_              | `\p{Dash}`, `\p{Extender}`, `\p{^Hyphen}`               | &#x2713; |
 
+**\[1\]**: Ruby does not support lazy or possessive interval quantifiers. Any `+` or `?` that follows an interval
+quantifier will be treated as another, chained quantifier. See also [#3](https://github.com/ammar/regexp_parser/issue/3),
+[#69](https://github.com/ammar/regexp_parser/pull/69).
+
+**\[2\]**: As of Ruby 3.1, meta and control sequences are [pre-processed to hex escapes when used in Regexp literals](
+ https://github.com/ruby/ruby/commit/11ae581a4a7f5d5f5ec6378872eab8f25381b1b9 ), so they will only reach the
+scanner and will only be emitted if a String or a Regexp that has been built with the `::new` constructor is scanned.
+
 ##### Inapplicable Features
 
 Some modifiers, like `o` and `s`, apply to the **Regexp** object itself and do not
@@ -384,7 +405,6 @@ expressions library (Onigmo). They are not supported by the scanner.
   - **Quotes**: `\Q...\E` _[[See]](https://github.com/k-takata/Onigmo/blob/7911409/doc/RE#L499)_
   - **Capture History**: `(?@...)`, `(?@<name>...)` _[[See]](https://github.com/k-takata/Onigmo/blob/7911409/doc/RE#L550)_
 
-
 See something missing? Please submit an [issue](https://github.com/ammar/regexp_parser/issues)
 
 _**Note**: Attempting to process expressions with unsupported syntax features can raise an error,
@@ -392,26 +412,14 @@ or incorrectly return tokens/objects as literals._
 
 
 ## Testing
-To run the tests simply run rake from the root directory, as 'test' is the default task.
+To run the tests simply run rake from the root directory.
 
-It generates the scanner's code from the Ragel source files and runs all the tests, thus it requires Ragel to be installed.
+The default task generates the scanner's code from the Ragel source files and runs all the specs, thus it requires Ragel to be installed.
 
-The tests use RSpec. They can also be run with the test runner that whitelists some warnings:
+Note that changes to Ragel files will not be reflected when running `rspec` on its own, so to run individual tests you might want to run:
 
 ```
-bin/test
-```
-
-You can run a specific test like so:
-
-```
-bin/test spec/scanner/properties_spec.rb
-```
-
-Note that changes to Ragel files will not be reflected when running `rspec` or `bin/test`, so you might want to run:
-
-```
-rake ragel:rb && bin/test spec/scanner/properties_spec.rb
+rake ragel:rb && rspec spec/scanner/properties_spec.rb
 ```
 
 ## Building
@@ -439,11 +447,13 @@ Projects using regexp_parser.
 
 - [capybara](https://github.com/teamcapybara/capybara) is an integration testing tool that uses regexp_parser to convert Regexps to css/xpath selectors.
 
-- [js_regex](https://github.com/janosch-x/js_regex) converts Ruby regular expressions to JavaScript-compatible regular expressions.
+- [js_regex](https://github.com/jaynetics/js_regex) converts Ruby regular expressions to JavaScript-compatible regular expressions.
 
 - [meta_re](https://github.com/ammar/meta_re) is a regular expression preprocessor with alias support.
 
-- [mutant](https://github.com/mbj/mutant) (before v0.9.0) manipulates your regular expressions (amongst others) to see if your tests cover their behavior.
+- [mutant](https://github.com/mbj/mutant) manipulates your regular expressions (amongst others) to see if your tests cover their behavior.
+
+- [repper](https://github.com/jaynetics/repper) is a regular expression pretty-printer for Ruby.
 
 - [rubocop](https://github.com/rubocop-hq/rubocop) is a linter for Ruby that uses regexp_parser to lint Regexps.
 
@@ -476,4 +486,4 @@ Documentation and books used while working on this project.
 
 ---
 ##### Copyright
-_Copyright (c) 2010-2020 Ammar Ali. See LICENSE file for details._
+_Copyright (c) 2010-2022 Ammar Ali. See LICENSE file for details._

data/Rakefile

@@ -1,87 +1,23 @@
+require 'bundler'
 require 'rubygems'
-
+require 'rubygems/package_task'
 require 'rake'
 require 'rake/testtask'
+require 'rspec/core/rake_task'
 
-require 'bundler'
-require 'rubygems/package_task'
-
-
-RAGEL_SOURCE_DIR = File.join(__dir__, 'lib/regexp_parser/scanner')
-RAGEL_OUTPUT_DIR = File.join(__dir__, 'lib/regexp_parser')
-RAGEL_SOURCE_FILES = %w{scanner} # scanner.rl includes property.rl
-
+Dir['tasks/**/*.rake'].each { |file| load(file) }
 
 Bundler::GemHelper.install_tasks
 
+RSpec::Core::RakeTask.new(:spec)
 
 task :default => [:'test:full']
 
 namespace :test do
-  task full: :'ragel:rb' do
-    sh 'bin/test'
-  end
+  task full: [:'ragel:rb', :spec]
 end
 
-namespace :ragel do
-  desc "Process the ragel source files and output ruby code"
-  task :rb do
-    RAGEL_SOURCE_FILES.each do |source_file|
-      output_file = "#{RAGEL_OUTPUT_DIR}/#{source_file}.rb"
-      # using faster flat table driven FSM, about 25% larger code, but about 30% faster
-      sh "ragel -F1 -R #{RAGEL_SOURCE_DIR}/#{source_file}.rl -o #{output_file}"
-
-      contents = File.read(output_file)
-
-      File.open(output_file, 'r+') do |file|
-        contents = "# -*- warn-indent:false;  -*-\n" + contents
-
-        file.write(contents)
-      end
-    end
-  end
-
-  desc "Delete the ragel generated source file(s)"
-  task :clean do
-    RAGEL_SOURCE_FILES.each do |file|
-      sh "rm -f #{RAGEL_OUTPUT_DIR}/#{file}.rb"
-    end
-  end
-end
-
-
 # Add ragel task as a prerequisite for building the gem to ensure that the
 # latest scanner code is generated and included in the build.
 desc "Runs ragel:rb before building the gem"
 task :build => ['ragel:rb']
-
-
-namespace :props do
-  desc 'Write new property value hashes for the properties scanner'
-  task :update do
-    require 'regexp_property_values'
-    RegexpPropertyValues.update
-    dir = File.join(__dir__, 'lib/regexp_parser/scanner/properties')
-
-    require 'psych'
-    write_hash_to_file = ->(hash, path) do
-      File.open(path, 'w') do |f|
-        f.puts '#',
-               "# THIS FILE IS AUTO-GENERATED BY `rake props:update`, DO NOT EDIT",
-               '#',
-               hash.sort.to_h.to_yaml
-      end
-      puts "Wrote #{hash.count} aliases to `#{path}`"
-    end
-
-    long_names_to_tokens = RegexpPropertyValues.all.map do |val|
-      [val.identifier, val.full_name.downcase]
-    end
-    write_hash_to_file.call(long_names_to_tokens, "#{dir}/long.yml")
-
-    short_names_to_tokens = RegexpPropertyValues.alias_hash.map do |k, v|
-      [k.identifier, v.full_name.downcase]
-    end
-    write_hash_to_file.call(short_names_to_tokens, "#{dir}/short.yml")
-  end
-end

data/lib/regexp_parser/error.rb

@@ -1,4 +1,4 @@
 class Regexp::Parser
-  # base class for all gem-specific errors (inherited but never raised itself)
+  # base class for all gem-specific errors
   class Error < StandardError; end
 end

data/lib/regexp_parser/expression/base.rb

@@ -0,0 +1,75 @@
+module Regexp::Expression
+  class Base
+    include Regexp::Expression::Shared
+
+    def initialize(token, options = {})
+      init_from_token_and_options(token, options)
+    end
+
+    def initialize_copy(orig)
+      self.text       = orig.text.dup         if orig.text
+      self.options    = orig.options.dup      if orig.options
+      self.quantifier = orig.quantifier.clone if orig.quantifier
+      super
+    end
+
+    def to_re(format = :full)
+      ::Regexp.new(to_s(format))
+    end
+
+    def quantify(*args)
+      self.quantifier = Quantifier.new(*args)
+    end
+
+    def unquantified_clone
+      clone.tap { |exp| exp.quantifier = nil }
+    end
+
+    # Deprecated. Prefer `#repetitions` which has a more uniform interface.
+    def quantity
+      return [nil,nil] unless quantified?
+      [quantifier.min, quantifier.max]
+    end
+
+    def repetitions
+      return 1..1 unless quantified?
+      min = quantifier.min
+      max = quantifier.max < 0 ? Float::INFINITY : quantifier.max
+      range = min..max
+      # fix Range#minmax on old Rubies - https://bugs.ruby-lang.org/issues/15807
+      if RUBY_VERSION.to_f < 2.7
+        range.define_singleton_method(:minmax) { [min, max] }
+      end
+      range
+    end
+
+    def greedy?
+      quantified? and quantifier.greedy?
+    end
+
+    def reluctant?
+      quantified? and quantifier.reluctant?
+    end
+    alias :lazy? :reluctant?
+
+    def possessive?
+      quantified? and quantifier.possessive?
+    end
+
+    def to_h
+      {
+        type:              type,
+        token:             token,
+        text:              to_s(:base),
+        starts_at:         ts,
+        length:            full_length,
+        level:             level,
+        set_level:         set_level,
+        conditional_level: conditional_level,
+        options:           options,
+        quantifier:        quantified? ? quantifier.to_h : nil,
+      }
+    end
+    alias :attributes :to_h
+  end
+end

data/lib/regexp_parser/expression/classes/anchor.rb

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   module Anchor
     class Base < Regexp::Expression::Base; end
 
@@ -22,5 +21,4 @@ module Regexp::Expression
     EOS      = EndOfString
     EOSobEOL = EndOfStringOrBeforeEndOfLine
   end
-
 end

data/lib/regexp_parser/expression/classes/{backref.rb → backreference.rb}

@@ -1,4 +1,5 @@
 module Regexp::Expression
+  # TODO: unify name with token :backref, one way or the other, in v3.0.0
   module Backreference
     class Base < Regexp::Expression::Base
       attr_accessor :referenced_expression

data/lib/regexp_parser/expression/classes/{set → character_set}/range.rb

@@ -16,8 +16,8 @@ module Regexp::Expression
         count == 2
       end
 
-      def to_s(_format = :full)
-        expressions.join(text)
+      def parts
+        intersperse(expressions, text.dup)
       end
     end
   end

data/lib/regexp_parser/expression/classes/{set.rb → character_set.rb}

@@ -20,8 +20,8 @@ module Regexp::Expression
       self.closed = true
     end
 
-    def to_s(format = :full)
-      "#{text}#{'^' if negated?}#{expressions.join}]#{quantifier_affix(format)}"
+    def parts
+      ["#{text}#{'^' if negated?}", *expressions, ']']
     end
   end
 end # module Regexp::Expression

data/lib/regexp_parser/expression/classes/{type.rb → character_type.rb}

@@ -1,5 +1,4 @@
 module Regexp::Expression
-
   module CharacterType
     class Base < Regexp::Expression::Base; end
 
@@ -15,5 +14,4 @@ module Regexp::Expression
     class Linebreak        < CharacterType::Base; end
     class ExtendedGrapheme < CharacterType::Base; end
   end
-
 end

data/lib/regexp_parser/expression/classes/conditional.rb

@@ -55,8 +55,8 @@ module Regexp::Expression
         condition.reference
       end
 
-      def to_s(format = :full)
-        "#{text}#{condition}#{branches.join('|')})#{quantifier_affix(format)}"
+      def parts
+        [text.dup, condition, *intersperse(branches, '|'), ')']
       end
 
       def initialize_copy(orig)

data/lib/regexp_parser/expression/classes/{escape.rb → escape_sequence.rb}

@@ -1,16 +1,22 @@
 module Regexp::Expression
+  # TODO: unify naming with Token::Escape, on way or the other, in v3.0.0
   module EscapeSequence
     class Base < Regexp::Expression::Base
-      require 'yaml'
-
-      def char
-        # poor man's unescape without using eval
-        YAML.load(%Q(---\n"#{text}"\n))
-      end
-
       def codepoint
         char.ord
       end
+
+      if ''.respond_to?(:undump)
+        def char
+          %("#{text}").undump
+        end
+      else
+        # poor man's unescape without using eval
+        require 'yaml'
+        def char
+          YAML.load(%Q(---\n"#{text}"\n))
+        end
+      end
     end
 
     class Literal < EscapeSequence::Base

data/lib/regexp_parser/expression/classes/free_space.rb

@@ -1,7 +1,6 @@
 module Regexp::Expression
-
   class FreeSpace < Regexp::Expression::Base
-    def quantify(_token, _text, _min = nil, _max = nil, _mode = :greedy)
+    def quantify(*_args)
       raise Regexp::Parser::Error, 'Can not quantify a free space object'
     end
   end
@@ -13,5 +12,4 @@ module Regexp::Expression
       text << exp.text
     end
   end
-
 end

data/lib/regexp_parser/expression/classes/group.rb

@@ -1,8 +1,8 @@
 module Regexp::Expression
   module Group
     class Base < Regexp::Expression::Subexpression
-      def to_s(format = :full)
-        "#{text}#{expressions.join})#{quantifier_affix(format)}"
+      def parts
+        [text.dup, *expressions, ')']
       end
 
       def capturing?; false end
@@ -18,9 +18,9 @@ module Regexp::Expression
         super
       end
 
-      def to_s(format = :full)
+      def parts
         if implicit?
-          "#{expressions.join}#{quantifier_affix(format)}"
+          expressions
         else
           super
         end
@@ -65,8 +65,8 @@ module Regexp::Expression
     end
 
     class Comment < Group::Base
-      def to_s(_format = :full)
-        text.dup
+      def parts
+        [text.dup]
       end
 
       def comment?; true end

data/lib/regexp_parser/expression/classes/keep.rb

@@ -1,5 +1,7 @@
 module Regexp::Expression
   module Keep
+    # TOOD: in regexp_parser v3.0.0 this should possibly be a Subexpression
+    #       that contains all expressions to its left.
     class Mark < Regexp::Expression::Base; end
   end
 end