regexp_parser 2.3.1 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6dedba6b051b22dd917febd957e35e6800b37af958780e331007de1f59b2b466
-  data.tar.gz: 55fa98afa6031a38cac2045f8c592290fad9a4e500597277f3f79be75f1076f3
+  metadata.gz: 8b84a4bb274f31b8608c7dc9d55ff6f1b8d92d0d147976f38079ae7701a6debe
+  data.tar.gz: 41db5f094d0beafade30a1fac2707cbc827831e818c485ad35d7173f18c6a91a
 SHA512:
-  metadata.gz: 6b2b463f3a28450527691d90bcfc8901b815bd4a32e88bcdee1f95db4588599993766687a4c7a1b785b450c0f0025039caf07bb93aaf1013fa365e4ed16fc040
-  data.tar.gz: 8aabccda06bb1f20485610076ad679a0d62e8630f992c55be7158c01f3b8b6dd6eb44a206f71b613c0335db13789d8e21d652973ecfb7c262c25a6d54e35a371
+  metadata.gz: 5dcde6135ac42db609402e47e04ee3be1da8854de286d2baad15dafee04d451814fd7a3bae7adc5440a1fced811e242b69f5fd14bcfc4f3bd5091f86769d56be
+  data.tar.gz: 2660d0fb28a972a1de53b71b16f8591e573d4214724b5eea8a452549598ff5d0fc5b731149e8332f65bce01c812f4d0d72135bba7e3016064d9f05202a8b5580

data/CHANGELOG.md

@@ -1,11 +1,54 @@
 ## [Unreleased]
 
+## [2.4.0] - 2022-05-09 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Fixed
+
+- fixed interpretation of `+` and `?` after interval quantifiers (`{n,n}`)
+  - they used to be treated as reluctant or possessive mode indicators
+  - however, Ruby does not support these modes for interval quantifiers
+  - they are now treated as chained quantifiers instead, as Ruby does it
+  - c.f. [#3](https://github.com/ammar/regexp_parser/issues/3)
+- fixed `Expression::Base#nesting_level` for some tree rewrite cases
+  - e.g. the alternatives in `/a|[b]/` had an inconsistent nesting_level
+- fixed `Scanner` accepting invalid posix classes, e.g. `[[:foo:]]`
+  - they raise a `SyntaxError` when used in a Regexp, so could only be passed as String
+  - they now raise a `Regexp::Scanner::ValidationError` in the `Scanner`
+
+### Added
+
+- added `Expression::Base#==` for (deep) comparison of expressions
+- added `Expression::Base#parts`
+  - returns the text elements and subexpressions of an expression
+  - e.g. `parse(/(a)/)[0].parts # => ["(", #<Literal @text="a"...>, ")"]`
+- added `Expression::Base#te` (a.k.a. token end index)
+  - `Expression::Subexpression` always had `#te`, only terminal nodes lacked it so far
+- made some `Expression::Base` methods available on `Quantifier` instances, too
+  - `#type`, `#type?`, `#is?`, `#one_of?`, `#options`, `#terminal?`
+  - `#base_length`, `#full_length`, `#starts_at`, `#te`, `#ts`, `#offset`
+  - `#conditional_level`, `#level`, `#nesting_level` , `#set_level`
+  - this allows a more unified handling with `Expression::Base` instances
+- allowed `Quantifier#initialize` to take a token and options Hash like other nodes
+- added a deprecation warning for initializing Quantifiers with 4+ arguments:
+
+    Calling `Expression::Base#quantify` or `Quantifier.new` with 4+ arguments
+    is deprecated.
+
+    It will no longer be supported in regexp_parser v3.0.0.
+
+    Please pass a Regexp::Token instead, e.g. replace `type, text, min, max, mode`
+    with `::Regexp::Token.new(:quantifier, type, text)`. min, max, and mode
+    will be derived automatically.
+
+    This is consistent with how Expression::Base instances are created.
+
+
 ## [2.3.1] - 2022-04-24 - [Janosch Müller](mailto:janosch84@gmail.com)
 
 ### Fixed
 
 - removed five inexistent unicode properties from `Syntax#features`
-  - these were never supported by Ruby but incorrectly accepted by the parser
+  - these were never supported by Ruby or the `Regexp::Scanner`
   - thanks to [Markus Schirp](https://github.com/mbj) for the report
 
 ## [2.3.0] - 2022-04-08 - [Janosch Müller](mailto:janosch84@gmail.com)
@@ -188,7 +231,7 @@
 
 ### Added
 
-- `Expression#each_expression` and `#traverse` can now be called without a block
+- `Expression::Base#each_expression` and `#traverse` can now be called without a block
   * this returns an `Enumerator` and allows chaining, e.g. `each_expression.select`
   * thanks to [Masataka Kuwabara](https://github.com/pocke)
 
@@ -214,7 +257,7 @@
 - Fixed `Group#option_changes` not accounting for indirectly disabled (overridden) encoding flags
 - Fixed `Scanner` allowing negative encoding options if there were no positive options, e.g. '(?-u)'
 - Fixed `ScannerError` for some valid meta/control sequences such as '\\C-\\\\'
-- Fixed `Expression#match` and `#=~` not working with a single argument
+- Fixed `Expression::Base#match` and `#=~` not working with a single argument
 
 ### [1.5.0] - 2019-05-14 - [Janosch Müller](mailto:janosch84@gmail.com)
 
@@ -222,15 +265,15 @@
 
 - Added `#referenced_expression` for backrefs, subexp calls and conditionals
   * returns the `Group` expression that is being referenced via name or number
-- Added `Expression#repetitions`
+- Added `Expression::Base#repetitions`
   * returns a `Range` of allowed repetitions (`1..1` if there is no quantifier)
   * like `#quantity` but with a more uniform interface
-- Added `Expression#match_length`
+- Added `Expression::Base#match_length`
   * allows to inspect and iterate over String lengths matched by the Expression
 
 ### Fixed
 
-- Fixed `Expression#clone` "direction"
+- Fixed `Expression::Base#clone` "direction"
   * it used to dup ivars onto the callee, leaving only the clone referencing the original objects
   * this will affect you if you call `#eql?`/`#equal?` on expressions or use them as Hash keys
 - Fixed `#clone` results for `Sequences`, e.g. alternations and conditionals
@@ -392,7 +435,7 @@ This release includes several breaking changes, mostly to character sets, #map a
 - Fixed a thread safety issue (issue #45)
 - Some public class methods that were only reliable for
   internal use are now private instance methods (PR #46)
-- Improved the usefulness of Expression#options (issue #43) -
+- Improved the usefulness of Expression::Base#options (issue #43) -
   #options and derived methods such as #i?, #m? and #x? are now
   defined for all Expressions that are affected by such flags.
 - Fixed scanning of whitespace following (?x) (commit 5c94bd2)

data/README.md

@@ -367,12 +367,12 @@ _Note that not all of these are available in all versions of Ruby_
 | **POSIX Classes**                     | `[:alpha:]`, `[:^digit:]`                               | ✓ |
 | **Quantifiers**                       |                                                         | ⋱ |
 |   _**Greedy**_              | `?`, `*`, `+`, `{m,M}`                                  | ✓ |
-|   _**Reluctant** (Lazy)_    | `??`, `*?`, `+?`, `{m,M}?`                              | ✓ |
-|   _**Possessive**_          | `?+`, `*+`, `++`, `{m,M}+`                              | ✓ |
+|   _**Reluctant** (Lazy)_    | `??`, `*?`, `+?` \[1\]                                  | ✓ |
+|   _**Possessive**_          | `?+`, `*+`, `++` \[1\]                                  | ✓ |
 | **String Escapes**                    |                                                         | ⋱ |
-|   _**Control** \[1\]_       | `\C-C`, `\cD`                                           | ✓ |
+|   _**Control** \[2\]_       | `\C-C`, `\cD`                                           | ✓ |
 |   _**Hex**_                 | `\x20`, `\x{701230}`                                    | ✓ |
-|   _**Meta** \[1\]_          | `\M-c`, `\M-\C-C`, `\M-\cC`, `\C-\M-C`, `\c\M-C`        | ✓ |
+|   _**Meta** \[2\]_          | `\M-c`, `\M-\C-C`, `\M-\cC`, `\C-\M-C`, `\c\M-C`        | ✓ |
 |   _**Octal**_               | `\0`, `\01`, `\012`                                     | ✓ |
 |   _**Unicode**_             | `\uHHHH`, `\u{H+ H+}`                                   | ✓ |
 | **Unicode Properties**                | _<sub>([Unicode 13.0.0](https://www.unicode.org/versions/Unicode13.0.0/))</sub>_ | &#x22f1; |
@@ -384,7 +384,11 @@ _Note that not all of these are available in all versions of Ruby_
 | &emsp;&nbsp;_**Scripts**_             | `\p{Arabic}`, `\P{Hiragana}`, `\p{^Greek}`              | &#x2713; |
 | &emsp;&nbsp;_**Simple**_              | `\p{Dash}`, `\p{Extender}`, `\p{^Hyphen}`               | &#x2713; |
 
-**\[1\]**: As of Ruby 3.1, meta and control sequences are [pre-processed to hex escapes when used in Regexp literals](
+**\[1\]**: Ruby does not support lazy or possessive interval quantifiers. Any `+` or `?` that follows an interval
+quantifier will be treated as another, chained quantifier. See also [#3](https://github.com/ammar/regexp_parser/issue/3),
+[#69](https://github.com/ammar/regexp_parser/pull/69).
+
+**\[2\]**: As of Ruby 3.1, meta and control sequences are [pre-processed to hex escapes when used in Regexp literals](
  https://github.com/ruby/ruby/commit/11ae581a4a7f5d5f5ec6378872eab8f25381b1b9 ), so they will only reach the
 scanner and will only be emitted if a String or a Regexp that has been built with the `::new` constructor is scanned.
 

data/lib/regexp_parser/error.rb

@@ -1,4 +1,4 @@
 class Regexp::Parser
-  # base class for all gem-specific errors (inherited but never raised itself)
+  # base class for all gem-specific errors
   class Error < StandardError; end
 end

data/lib/regexp_parser/expression/base.rb

@@ -1,29 +1,15 @@
 module Regexp::Expression
   class Base
-    attr_accessor :type, :token
-    attr_accessor :text, :ts
-    attr_accessor :level, :set_level, :conditional_level, :nesting_level
-
-    attr_accessor :quantifier
-    attr_accessor :options
+    include Regexp::Expression::Shared
 
     def initialize(token, options = {})
-      self.type              = token.type
-      self.token             = token.token
-      self.text              = token.text
-      self.ts                = token.ts
-      self.level             = token.level
-      self.set_level         = token.set_level
-      self.conditional_level = token.conditional_level
-      self.nesting_level     = 0
-      self.quantifier        = nil
-      self.options           = options
+      init_from_token_and_options(token, options)
     end
 
     def initialize_copy(orig)
-      self.text       = (orig.text       ? orig.text.dup         : nil)
-      self.options    = (orig.options    ? orig.options.dup      : nil)
-      self.quantifier = (orig.quantifier ? orig.quantifier.clone : nil)
+      self.text       = orig.text.dup         if orig.text
+      self.options    = orig.options.dup      if orig.options
+      self.quantifier = orig.quantifier.clone if orig.quantifier
       super
     end
 
@@ -31,48 +17,14 @@ module Regexp::Expression
       ::Regexp.new(to_s(format))
     end
 
-    alias :starts_at :ts
-
-    def base_length
-      to_s(:base).length
-    end
-
-    def full_length
-      to_s.length
-    end
-
-    def offset
-      [starts_at, full_length]
-    end
-
-    def coded_offset
-      '@%d+%d' % offset
-    end
-
-    def to_s(format = :full)
-      "#{text}#{quantifier_affix(format)}"
-    end
-
-    def quantifier_affix(expression_format)
-      quantifier.to_s if quantified? && expression_format != :base
-    end
-
-    def terminal?
-      !respond_to?(:expressions)
-    end
-
-    def quantify(token, text, min = nil, max = nil, mode = :greedy)
-      self.quantifier = Quantifier.new(token, text, min, max, mode)
+    def quantify(*args)
+      self.quantifier = Quantifier.new(*args)
     end
 
     def unquantified_clone
       clone.tap { |exp| exp.quantifier = nil }
     end
 
-    def quantified?
-      !quantifier.nil?
-    end
-
     # Deprecated. Prefer `#repetitions` which has a more uniform interface.
     def quantity
       return [nil,nil] unless quantified?
@@ -104,7 +56,7 @@ module Regexp::Expression
       quantified? and quantifier.possessive?
     end
 
-    def attributes
+    def to_h
       {
         type:              type,
         token:             token,
@@ -118,6 +70,6 @@ module Regexp::Expression
         quantifier:        quantified? ? quantifier.to_h : nil,
       }
     end
-    alias :to_h :attributes
+    alias :attributes :to_h
   end
 end

data/lib/regexp_parser/expression/classes/character_set/range.rb

@@ -16,8 +16,8 @@ module Regexp::Expression
         count == 2
       end
 
-      def to_s(_format = :full)
-        expressions.join(text)
+      def parts
+        intersperse(expressions, text.dup)
       end
     end
   end

data/lib/regexp_parser/expression/classes/character_set.rb

@@ -20,8 +20,8 @@ module Regexp::Expression
       self.closed = true
     end
 
-    def to_s(format = :full)
-      "#{text}#{'^' if negated?}#{expressions.join}]#{quantifier_affix(format)}"
+    def parts
+      ["#{text}#{'^' if negated?}", *expressions, ']']
     end
   end
 end # module Regexp::Expression

data/lib/regexp_parser/expression/classes/conditional.rb

@@ -55,8 +55,8 @@ module Regexp::Expression
         condition.reference
       end
 
-      def to_s(format = :full)
-        "#{text}#{condition}#{branches.join('|')})#{quantifier_affix(format)}"
+      def parts
+        [text.dup, condition, *intersperse(branches, '|'), ')']
       end
 
       def initialize_copy(orig)

data/lib/regexp_parser/expression/classes/free_space.rb

@@ -1,6 +1,6 @@
 module Regexp::Expression
   class FreeSpace < Regexp::Expression::Base
-    def quantify(_token, _text, _min = nil, _max = nil, _mode = :greedy)
+    def quantify(*_args)
       raise Regexp::Parser::Error, 'Can not quantify a free space object'
     end
   end

data/lib/regexp_parser/expression/classes/group.rb

@@ -1,8 +1,8 @@
 module Regexp::Expression
   module Group
     class Base < Regexp::Expression::Subexpression
-      def to_s(format = :full)
-        "#{text}#{expressions.join})#{quantifier_affix(format)}"
+      def parts
+        [text.dup, *expressions, ')']
       end
 
       def capturing?; false end
@@ -18,9 +18,9 @@ module Regexp::Expression
         super
       end
 
-      def to_s(format = :full)
+      def parts
         if implicit?
-          "#{expressions.join}#{quantifier_affix(format)}"
+          expressions
         else
           super
         end
@@ -65,8 +65,8 @@ module Regexp::Expression
     end
 
     class Comment < Group::Base
-      def to_s(_format = :full)
-        text.dup
+      def parts
+        [text.dup]
       end
 
       def comment?; true end

data/lib/regexp_parser/expression/methods/tests.rb

@@ -1,5 +1,5 @@
 module Regexp::Expression
-  class Base
+  module Shared
 
     # Test if this expression has the given test_type, which can be either
     # a symbol or an array of symbols to check against the expression's type.
@@ -93,5 +93,14 @@ module Regexp::Expression
               "Array, Hash, or Symbol expected, #{scope.class.name} given"
       end
     end
+
+    # Deep-compare two expressions for equality.
+    def ==(other)
+      other.class == self.class &&
+        other.to_s == to_s &&
+        other.options == options
+    end
+    alias :=== :==
+    alias :eql? :==
   end
 end

data/lib/regexp_parser/expression/quantifier.rb

@@ -1,26 +1,24 @@
 module Regexp::Expression
+  # TODO: in v3.0.0, maybe put Shared back into Base, and inherit from Base and
+  # call super in #initialize, but raise in #quantifier= and #quantify,
+  # or introduce an Expression::Quantifiable intermediate class.
+  # Or actually allow chaining as a more concise but tricky solution than PR#69.
   class Quantifier
+    include Regexp::Expression::Shared
+
     MODES = %i[greedy possessive reluctant]
 
-    attr_reader :token, :text, :min, :max, :mode
+    attr_reader :min, :max, :mode
 
-    def initialize(token, text, min, max, mode)
-      @token = token
-      @text  = text
-      @mode  = mode
-      @min   = min
-      @max   = max
-    end
-
-    def initialize_copy(orig)
-      @text = orig.text.dup
-      super
-    end
+    def initialize(*args)
+      deprecated_old_init(*args) and return if args.count == 4 || args.count == 5
 
-    def to_s
-      text.dup
+      init_from_token_and_options(*args)
+      @mode = (token[/greedy|reluctant|possessive/] || :greedy).to_sym
+      @min, @max = minmax
+      # TODO: remove in v3.0.0, stop removing parts of #token (?)
+      self.token = token.to_s.sub(/_(greedy|possessive|reluctant)/, '').to_sym
     end
-    alias :to_str :to_s
 
     def to_h
       {
@@ -41,13 +39,32 @@ module Regexp::Expression
     end
     alias :lazy? :reluctant?
 
-    def ==(other)
-      other.class == self.class &&
-        other.token == token &&
-        other.mode == mode &&
-        other.min == min &&
-        other.max == max
+    private
+
+    def deprecated_old_init(token, text, min, max, mode = :greedy)
+      warn "Calling `Expression::Base#quantify` or `#{self.class}.new` with 4+ arguments "\
+           "is deprecated.\nIt will no longer be supported in regexp_parser v3.0.0.\n"\
+           "Please pass a Regexp::Token instead, e.g. replace `type, text, min, max, mode` "\
+           "with `::Regexp::Token.new(:quantifier, type, text)`. min, max, and mode "\
+           "will be derived automatically. \nThis is consistent with how Expression::Base "\
+           "instances are created."
+      @token = token
+      @text  = text
+      @min   = min
+      @max   = max
+      @mode  = mode
+    end
+
+    def minmax
+      case token
+      when /zero_or_one/  then [0, 1]
+      when /zero_or_more/ then [0, -1]
+      when /one_or_more/  then [1, -1]
+      when :interval
+        int_min = text[/\{(\d*)/, 1]
+        int_max = text[/,?(\d*)\}/, 1]
+        [int_min.to_i, (int_max.empty? ? -1 : int_max.to_i)]
+      end
     end
-    alias :eq :==
   end
 end

data/lib/regexp_parser/expression/sequence.rb

@@ -39,12 +39,12 @@ module Regexp::Expression
     end
     alias :ts :starts_at
 
-    def quantify(token, text, min = nil, max = nil, mode = :greedy)
+    def quantify(*args)
       target = expressions.reverse.find { |exp| !exp.is_a?(FreeSpace) }
       target or raise Regexp::Parser::Error,
         "No valid target found for '#{text}' quantifier"
 
-      target.quantify(token, text, min, max, mode)
+      target.quantify(*args)
     end
   end
 end

data/lib/regexp_parser/expression/sequence_operation.rb

@@ -18,8 +18,8 @@ module Regexp::Expression
       self.class::OPERAND.add_to(self, {}, active_opts)
     end
 
-    def to_s(format = :full)
-      sequences.map { |e| e.to_s(format) }.join(text)
+    def parts
+      intersperse(expressions, text.dup)
     end
   end
 end

data/lib/regexp_parser/expression/shared.rb

@@ -0,0 +1,81 @@
+module Regexp::Expression
+  module Shared
+    def self.included(mod)
+      mod.class_eval do
+        attr_accessor :type, :token, :text, :ts, :te,
+                      :level, :set_level, :conditional_level,
+                      :options, :quantifier
+
+        attr_reader   :nesting_level
+      end
+    end
+
+    def init_from_token_and_options(token, options = {})
+      self.type              = token.type
+      self.token             = token.token
+      self.text              = token.text
+      self.ts                = token.ts
+      self.te                = token.te
+      self.level             = token.level
+      self.set_level         = token.set_level
+      self.conditional_level = token.conditional_level
+      self.nesting_level     = 0
+      self.options           = options || {}
+    end
+    private :init_from_token_and_options
+
+    def initialize_copy(orig)
+      self.text       = orig.text.dup         if orig.text
+      self.options    = orig.options.dup      if orig.options
+      self.quantifier = orig.quantifier.clone if orig.quantifier
+      super
+    end
+
+    def starts_at
+      ts
+    end
+
+    def base_length
+      to_s(:base).length
+    end
+
+    def full_length
+      to_s.length
+    end
+
+    def to_s(format = :full)
+      "#{parts.join}#{quantifier_affix(format)}"
+    end
+    alias :to_str :to_s
+
+    def parts
+      [text.dup]
+    end
+
+    def quantifier_affix(expression_format)
+      quantifier.to_s if quantified? && expression_format != :base
+    end
+
+    def quantified?
+      !quantifier.nil?
+    end
+
+    def offset
+      [starts_at, full_length]
+    end
+
+    def coded_offset
+      '@%d+%d' % offset
+    end
+
+    def terminal?
+      !respond_to?(:expressions)
+    end
+
+    def nesting_level=(lvl)
+      @nesting_level = lvl
+      quantifier && quantifier.nesting_level = lvl
+      terminal? || each { |subexp| subexp.nesting_level = lvl + 1 }
+    end
+  end
+end

data/lib/regexp_parser/expression/subexpression.rb

@@ -5,9 +5,8 @@ module Regexp::Expression
     attr_accessor :expressions
 
     def initialize(token, options = {})
-      super
-
       self.expressions = []
+      super
     end
 
     # Override base method to clone the expressions as well.
@@ -43,16 +42,21 @@ module Regexp::Expression
       ts + to_s.length
     end
 
-    def to_s(format = :full)
-      # Note: the format does not get passed down to subexpressions.
-      "#{expressions.join}#{quantifier_affix(format)}"
+    def parts
+      expressions
     end
 
     def to_h
-      attributes.merge({
+      attributes.merge(
         text:        to_s(:base),
         expressions: expressions.map(&:to_h)
-      })
+      )
+    end
+
+    private
+
+    def intersperse(expressions, separator)
+      expressions.flat_map { |exp| [exp, separator] }.slice(0...-1)
     end
   end
 end

data/lib/regexp_parser/expression.rb

@@ -1,5 +1,6 @@
 require 'regexp_parser/error'
 
+require 'regexp_parser/expression/shared'
 require 'regexp_parser/expression/base'
 require 'regexp_parser/expression/quantifier'
 require 'regexp_parser/expression/subexpression'