regexp_parser 1.8.2 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d1c795eb67eee01aa7976a672349b9a0c7f6364262ffb8f53488f6d9949db07
-  data.tar.gz: a38fddc769f16841e54f5dde90535e31ef7145e5bd41560ccf70443968495493
+  metadata.gz: 79c8b7838ef53335c9d0fbd21ffdf6815473ee560380a3687e8fab514d031d53
+  data.tar.gz: 2a91f7c7640fc5f2d304c2cbf240886d8e8642994861a9c092f1d4db2ae6b77a
 SHA512:
-  metadata.gz: 8dfcd3ba9f2a09370142e892184ccb9a1a973e24dea691582ca43cc670589dd7050eb30dbd924ae9e0584d124183dc2f62adca475628a726ecffaa22fcfb8903
-  data.tar.gz: cd23a308e4d62af9a07bbec164c96ca3213de5a4f686c450fc90bca56387faa88febb8a3387e810d0fd8b78e06b191bd76777fe8b3f3cda9da5582b9009d1144
+  metadata.gz: 3559a8c7af9c0087ab7a54862c9913e40a3703ffa23f62e6919eec50042523424c2aa4c99b3de9d28d03fc0edd14af37e0dcd0eab7bf822b9af73113be468b59
+  data.tar.gz: 31ed468565bd41fe2d0bd7b82d53d64e213a15e1ade2108ddf813637c228c18f6f7b456725c7e359a08754188ee19c90d06e013be90775ee6a64723b04fa25f0

data/CHANGELOG.md

@@ -1,5 +1,98 @@
 ## [Unreleased]
 
+## [2.1.0] - 2021-02-22 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- common ancestor for all scanning/parsing/lexing errors
+  * `Regexp::Parser::Error` can now be rescued as a catch-all
+  * the following errors (and their many descendants) now inherit from it:
+    - `Regexp::Expression::Conditional::TooManyBranches`
+    - `Regexp::Parser::ParserError`
+    - `Regexp::Scanner::ScannerError`
+    - `Regexp::Scanner::ValidationError`
+    - `Regexp::Syntax::SyntaxError`
+  * it replaces `ArgumentError` in some rare cases (`Regexp::Parser.parse('?')`)
+  * thanks to [sandstrom](https://github.com/sandstrom) for the cue
+
+### Fixed
+
+- fixed scanning of whole-pattern recursion calls `\g<0>` and `\g'0'`
+  * a regression in v2.0.1 had caused them to be scanned as literals
+- fixed scanning of some backreference and subexpression call edge cases
+  * e.g. `\k<+1>`, `\g<x-1>`
+- fixed tokenization of some escapes in character sets
+  * `.`, `|`, `{`, `}`, `(`, `)`, `^`, `$`, `?`, `+`, `*`
+  * all of these correctly emitted `#type` `:literal` and `#token` `:literal` if *not* escaped
+  * if escaped, they emitted e.g. `#type` `:escape` and `#token` `:group_open` for `[\(]`
+  * the escaped versions now correctly emit `#type` `:escape` and `#token` `:literal`
+- fixed handling of control/metacontrol escapes in character sets
+  * e.g. `[\cX]`, `[\M-\C-X]`
+  * they were misread as bunch of individual literals, escapes, and ranges
+- fixed some cases where calling `#dup`/`#clone` on expressions led to shared state
+
+## [2.0.3] - 2020-12-28 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Fixed
+
+- fixed error when scanning some unlikely and redundant but valid charset patterns
+  * e.g. `/[[.a-b.]]/`, `/[[=e=]]/`,
+- fixed ancestry of some error classes related to syntax version lookup
+  * `NotImplementedError`, `InvalidVersionNameError`, `UnknownSyntaxNameError`
+  * they now correctly inherit from `Regexp::Syntax::SyntaxError` instead of Rubys `::SyntaxError`
+
+## [2.0.2] - 2020-12-25 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Fixed
+
+- fixed `FrozenError` when calling `#to_s` on a frozen `Group::Passive`
+  * thanks to [Daniel Gollahon](https://github.com/dgollahon)
+
+## [2.0.1] - 2020-12-20 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Fixed
+
+- fixed error when scanning some group names
+  * this affected names containing hyphens, digits or multibyte chars, e.g. `/(?<a1>a)/`
+  * thanks to [Daniel Gollahon](https://github.com/dgollahon) for the report
+- fixed error when scanning hex escapes with just one hex digit
+  * e.g. `/\x0A/` was scanned correctly, but the equivalent `/\xA/` was not
+  * thanks to [Daniel Gollahon](https://github.com/dgollahon) for the report
+
+## [2.0.0] - 2020-11-25 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Changed
+
+- some methods that used to return byte-based indices now return char-based indices
+  * the returned values have only changed for Regexps that contain multibyte chars
+  * this is only a breaking change if you used such methods directly AND relied on them pointing to bytes
+  * affected methods:
+  * `Regexp::Token` `#length`, `#offset`, `#te`, `#ts`
+  * `Regexp::Expression::Base` `#full_length`, `#offset`, `#starts_at`, `#te`, `#ts`
+  * thanks to [Akinori MUSHA](https://github.com/knu) for the report
+- removed some deprecated methods/signatures
+  * these are rarely used and have been showing deprecation warnings for a long time
+  * `Regexp::Expression::Subexpression.new` with 3 arguments
+  * `Regexp::Expression::Root.new` without a token argument
+  * `Regexp::Expression.parsed`
+
+### Added
+
+- `Regexp::Expression::Base#base_length`
+  * returns the character count of an expression body, ignoring any quantifier
+- pragmatic, experimental support for chained quantifiers
+  * e.g.: `/^a{10}{4,6}$/` matches exactly 40, 50 or 60 `a`s
+  * successive quantifiers used to be silently dropped by the parser
+  * they are now wrapped with passive groups as if they were written `(?:a{10}){4,6}`
+  * thanks to [calfeld](https://github.com/calfeld) for reporting this a while back
+
+### Fixed
+
+- incorrect encoding output for non-ascii comments
+  * this led to a crash when calling `#to_s` on parse results containing such comments
+  * thanks to [Michael Glass](https://github.com/michaelglass) for the report
+- some crashes when scanning contrived patterns such as `'\😋'`
+
 ### [1.8.2] - 2020-10-11 - [Janosch Müller](mailto:janosch84@gmail.com)
 
 ### Fixed

data/Gemfile

@@ -3,7 +3,12 @@ source 'https://rubygems.org'
 gemspec
 
 group :development, :test do
+  gem 'ice_nine', '~> 0.11.2'
   gem 'rake', '~> 13.0'
   gem 'regexp_property_values', '~> 1.0'
-  gem 'rspec', '~> 3.8'
+  gem 'rspec', '~> 3.10'
+  if RUBY_VERSION.to_f >= 2.7
+    gem 'gouteur'
+    gem 'rubocop', '~> 1.7'
+  end
 end

data/README.md

@@ -1,6 +1,6 @@
 # Regexp::Parser
 
-[![Gem Version](https://badge.fury.io/rb/regexp_parser.svg)](http://badge.fury.io/rb/regexp_parser) [![Build Status](https://secure.travis-ci.org/ammar/regexp_parser.svg?branch=master)](http://travis-ci.org/ammar/regexp_parser) [![Code Climate](https://codeclimate.com/github/ammar/regexp_parser.svg)](https://codeclimate.com/github/ammar/regexp_parser/badges)
+[![Gem Version](https://badge.fury.io/rb/regexp_parser.svg)](http://badge.fury.io/rb/regexp_parser) [![Build Status](https://github.com/ammar/regexp_parser/workflows/tests/badge.svg)](https://github.com/ammar/regexp_parser/actions) [![Build Status](https://github.com/ammar/regexp_parser/workflows/gouteur/badge.svg)](https://github.com/ammar/regexp_parser/actions) [![Code Climate](https://codeclimate.com/github/ammar/regexp_parser.svg)](https://codeclimate.com/github/ammar/regexp_parser/badges)
 
 A Ruby gem for tokenizing, parsing, and transforming regular expressions.
 
@@ -22,9 +22,6 @@ _For examples of regexp_parser in use, see [Example Projects](#example-projects)
 * Ragel >= 6.0, but only if you want to build the gem or work on the scanner.
 
 
-_Note: See the .travis.yml file for covered versions._
-
-
 ---
 ## Install
 

data/Rakefile

@@ -7,8 +7,8 @@ require 'bundler'
 require 'rubygems/package_task'
 
 
-RAGEL_SOURCE_DIR = File.expand_path '../lib/regexp_parser/scanner', __FILE__
-RAGEL_OUTPUT_DIR = File.expand_path '../lib/regexp_parser', __FILE__
+RAGEL_SOURCE_DIR = File.join(__dir__, 'lib/regexp_parser/scanner')
+RAGEL_OUTPUT_DIR = File.join(__dir__, 'lib/regexp_parser')
 RAGEL_SOURCE_FILES = %w{scanner} # scanner.rl includes property.rl
 
 
@@ -25,11 +25,11 @@ end
 
 namespace :ragel do
   desc "Process the ragel source files and output ruby code"
-  task :rb do |t|
-    RAGEL_SOURCE_FILES.each do |file|
-      output_file = "#{RAGEL_OUTPUT_DIR}/#{file}.rb"
+  task :rb do
+    RAGEL_SOURCE_FILES.each do |source_file|
+      output_file = "#{RAGEL_OUTPUT_DIR}/#{source_file}.rb"
       # using faster flat table driven FSM, about 25% larger code, but about 30% faster
-      sh "ragel -F1 -R #{RAGEL_SOURCE_DIR}/#{file}.rl -o #{output_file}"
+      sh "ragel -F1 -R #{RAGEL_SOURCE_DIR}/#{source_file}.rl -o #{output_file}"
 
       contents = File.read(output_file)
 
@@ -42,7 +42,7 @@ namespace :ragel do
   end
 
   desc "Delete the ragel generated source file(s)"
-  task :clean do |t|
+  task :clean do
     RAGEL_SOURCE_FILES.each do |file|
       sh "rm -f #{RAGEL_OUTPUT_DIR}/#{file}.rb"
     end
@@ -61,7 +61,7 @@ namespace :props do
   task :update do
     require 'regexp_property_values'
     RegexpPropertyValues.update
-    dir = File.expand_path('../lib/regexp_parser/scanner/properties', __FILE__)
+    dir = File.join(__dir__, 'lib/regexp_parser/scanner/properties')
 
     require 'psych'
     write_hash_to_file = ->(hash, path) do

data/lib/regexp_parser.rb

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 require 'regexp_parser/version'
+require 'regexp_parser/error'
 require 'regexp_parser/token'
 require 'regexp_parser/scanner'
 require 'regexp_parser/syntax'

data/lib/regexp_parser/error.rb

@@ -0,0 +1,4 @@
+class Regexp::Parser
+  # base class for all gem-specific errors (inherited but never raised itself)
+  class Error < StandardError; end
+end

data/lib/regexp_parser/expression.rb

@@ -21,7 +21,7 @@ module Regexp::Expression
       self.options           = options
     end
 
-    def initialize_clone(orig)
+    def initialize_copy(orig)
       self.text       = (orig.text       ? orig.text.dup         : nil)
       self.options    = (orig.options    ? orig.options.dup      : nil)
       self.quantifier = (orig.quantifier ? orig.quantifier.clone : nil)
@@ -34,6 +34,10 @@ module Regexp::Expression
 
     alias :starts_at :ts
 
+    def base_length
+      to_s(:base).length
+    end
+
     def full_length
       to_s.length
     end
@@ -118,23 +122,6 @@ module Regexp::Expression
     alias :to_h :attributes
   end
 
-  def self.parsed(exp)
-    warn('WARNING: Regexp::Expression::Base.parsed is buggy and '\
-         'will be removed in 2.0.0. Use Regexp::Parser.parse instead.')
-    case exp
-    when String
-      Regexp::Parser.parse(exp)
-    when Regexp
-      Regexp::Parser.parse(exp.source) # <- causes loss of root options
-    when Regexp::Expression            # <- never triggers
-      exp
-    else
-      raise ArgumentError, 'Expression.parsed accepts a String, Regexp, or '\
-                           'a Regexp::Expression as a value for exp, but it '\
-                           "was given #{exp.class.name}."
-    end
-  end
-
 end # module Regexp::Expression
 
 require 'regexp_parser/expression/quantifier'

data/lib/regexp_parser/expression/classes/backref.rb

@@ -2,6 +2,11 @@ module Regexp::Expression
   module Backreference
     class Base < Regexp::Expression::Base
       attr_accessor :referenced_expression
+
+      def initialize_copy(orig)
+        self.referenced_expression = orig.referenced_expression.dup
+        super
+      end
     end
 
     class Number < Backreference::Base

data/lib/regexp_parser/expression/classes/conditional.rb

@@ -1,6 +1,6 @@
 module Regexp::Expression
   module Conditional
-    class TooManyBranches < StandardError
+    class TooManyBranches < Regexp::Parser::Error
       def initialize
         super('The conditional expression has more than 2 branches')
       end
@@ -15,6 +15,11 @@ module Regexp::Expression
         ref = text.tr("'<>()", "")
         ref =~ /\D/ ? ref : Integer(ref)
       end
+
+      def initialize_copy(orig)
+        self.referenced_expression = orig.referenced_expression.dup
+        super
+      end
     end
 
     class Branch < Regexp::Expression::Sequence; end
@@ -53,6 +58,11 @@ module Regexp::Expression
       def to_s(format = :full)
         "#{text}#{condition}#{branches.join('|')})#{quantifier_affix(format)}"
       end
+
+      def initialize_copy(orig)
+        self.referenced_expression = orig.referenced_expression.dup
+        super
+      end
     end
   end
 end

data/lib/regexp_parser/expression/classes/free_space.rb

@@ -1,8 +1,8 @@
 module Regexp::Expression
 
   class FreeSpace < Regexp::Expression::Base
-    def quantify(token, text, min = nil, max = nil, mode = :greedy)
-      raise "Can not quantify a free space object"
+    def quantify(_token, _text, _min = nil, _max = nil, _mode = :greedy)
+      raise Regexp::Parser::Error, 'Can not quantify a free space object'
     end
   end
 

data/lib/regexp_parser/expression/classes/group.rb

@@ -10,11 +10,36 @@ module Regexp::Expression
       def comment?; false end
     end
 
-    class Atomic  < Group::Base; end
-    class Passive < Group::Base; end
+    class Passive < Group::Base
+      attr_writer :implicit
+
+      def initialize(*)
+        @implicit = false
+        super
+      end
+
+      def to_s(format = :full)
+        if implicit?
+          "#{expressions.join}#{quantifier_affix(format)}"
+        else
+          super
+        end
+      end
+
+      def implicit?
+        @implicit
+      end
+    end
+
     class Absence < Group::Base; end
+    class Atomic  < Group::Base; end
     class Options < Group::Base
       attr_accessor :option_changes
+
+      def initialize_copy(orig)
+        self.option_changes = orig.option_changes.dup
+        super
+      end
     end
 
     class Capture < Group::Base
@@ -33,7 +58,7 @@ module Regexp::Expression
         super
       end
 
-      def initialize_clone(orig)
+      def initialize_copy(orig)
         @name = orig.name.dup
         super
       end

data/lib/regexp_parser/expression/classes/property.rb

@@ -7,7 +7,7 @@ module Regexp::Expression
       end
 
       def name
-        text =~ /\A\\[pP]\{([^}]+)\}\z/; $1
+        text[/\A\\[pP]\{([^}]+)\}\z/, 1]
       end
 
       def shortcut

data/lib/regexp_parser/expression/classes/root.rb

@@ -1,24 +1,12 @@
 module Regexp::Expression
 
   class Root < Regexp::Expression::Subexpression
-    # TODO: this override is here for backwards compatibility, remove in 2.0.0
-    def initialize(*args)
-      unless args.first.is_a?(Regexp::Token)
-        warn('WARNING: Root.new without a Token argument is deprecated and '\
-             'will be removed in 2.0.0. Use Root.build for the old behavior.')
-        return super(self.class.build_token, *args)
-      end
-      super
+    def self.build(options = {})
+      new(build_token, options)
     end
 
-    class << self
-      def build(options = {})
-        new(build_token, options)
-      end
-
-      def build_token
-        Regexp::Token.new(:expression, :root, '', 0)
-      end
+    def self.build_token
+      Regexp::Token.new(:expression, :root, '', 0)
     end
   end
 end

data/lib/regexp_parser/expression/classes/set/range.rb

@@ -7,7 +7,8 @@ module Regexp::Expression
       alias :ts :starts_at
 
       def <<(exp)
-        complete? && raise("Can't add more than 2 expressions to a Range")
+        complete? and raise Regexp::Parser::Error,
+          "Can't add more than 2 expressions to a Range"
         super
       end
 

data/lib/regexp_parser/expression/methods/match_length.rb

@@ -10,7 +10,7 @@ class Regexp::MatchLength
     self.exp_class = exp.class
     self.min_rep = exp.repetitions.min
     self.max_rep = exp.repetitions.max
-    if base = opts[:base]
+    if (base = opts[:base])
       self.base_min = base
       self.base_max = base
       self.reify = ->{ '.' * base }
@@ -32,7 +32,7 @@ class Regexp::MatchLength
     end
   end
 
-  def endless_each(&block)
+  def endless_each
     return enum_for(__method__) unless block_given?
     (min..max).each { |num| yield(num) if include?(num) }
   end

data/lib/regexp_parser/expression/methods/traverse.rb

@@ -36,7 +36,7 @@ module Regexp::Expression
 
     # Iterates over the expressions of this expression as an array, passing
     # the expression and its index within its parent to the given block.
-    def each_expression(include_self = false, &block)
+    def each_expression(include_self = false)
       return enum_for(__method__, include_self) unless block_given?
 
       traverse(include_self) do |event, exp, index|
@@ -47,7 +47,7 @@ module Regexp::Expression
     # Returns a new array with the results of calling the given block once
     # for every expression. If a block is not given, returns an array with
     # each expression and its level index as an array.
-    def flat_map(include_self = false, &block)
+    def flat_map(include_self = false)
       result = []
 
       each_expression(include_self) do |exp, index|

data/lib/regexp_parser/expression/quantifier.rb

@@ -12,7 +12,7 @@ module Regexp::Expression
       @max   = max
     end
 
-    def initialize_clone(orig)
+    def initialize_copy(orig)
       @text = orig.text.dup
       super
     end
@@ -40,5 +40,14 @@ module Regexp::Expression
       RUBY
     end
     alias :lazy? :reluctant?
+
+    def ==(other)
+      other.class == self.class &&
+        other.token == token &&
+        other.mode == mode &&
+        other.min == min &&
+        other.max == max
+    end
+    alias :eq :==
   end
 end