regexp_parser 1.3.0 → 1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9af74355b57139b93d0207bfbf6fc6fbb43c1eb6aa531bd11710e39168af107a
-  data.tar.gz: 8d63533aa2206b121fd59959a43cad2b64e3b3c6d0f8617953d2c401033c86d5
+  metadata.gz: dd872b22bf04a288790ef0f73df9041f14fb88a08c2a03852d9dbbc238b452d6
+  data.tar.gz: 4641097a24b5fa0f7b0c8e5aacc152587fe8b15d30f3f78bbec8157887b8b897
 SHA512:
-  metadata.gz: de6afb1888075eaa8d220192882494eb67c54aaba5fa84b991adf753dbb06949b8b0ea6d44054d32c3cba1e8cb6ee04d8f8f2f8643e4663188ddca3b7ec7e07a
-  data.tar.gz: f9e7d91a317bb153b645380a6447faca2656b87fa164afae9dc1cf9bc63463c11a87388258d18a49ed3df7afc3a7b25cd0a2113db412c2e7f504cee9aec8c255
+  metadata.gz: 858570df4a7047a2d8b09555b56de28a66ca4f8022e596c249900f5312f8e7fb9376384ca816bc3c08f3e324930702ad410a28b5be680adea6867e1f8075441e
+  data.tar.gz: 0d70e7b4f18739826bb334fb305e335e44a354ae302214ca3c1884f66ace8680e48a9e4c64b890b220b82056da761084413c8b9b8c5e363382f5cf165b3d3448

data/CHANGELOG.md

@@ -1,5 +1,76 @@
 ## [Unreleased]
 
+### [1.7.1] - 2020-06-07 - [Ammar Ali](mailto:ammarabuali@gmail.com)
+
+### Fixed
+
+- Support for literals that include the unescaped delimiters `{`, `}`, and `]`. These
+  delimiters are informally supported by various regexp engines.
+
+### [1.7.0] - 2020-02-23 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- `Expression#each_expression` and `#traverse` can now be called without a block
+  * this returns an `Enumerator` and allows chaining, e.g. `each_expression.select`
+  * thanks to [Masataka Kuwabara](https://github.com/pocke)
+
+### Fixed
+
+- `MatchLength#each` no longer ignores the given `limit:` when called without a block
+
+### [1.6.0] - 2019-06-16 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- Added support for 16 new unicode properties introduced in Ruby 2.6.2 and 2.6.3
+
+### [1.5.1] - 2019-05-23 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Fixed
+
+- Fixed `#options` (and thus `#i?`, `#u?` etc.) not being set for some expressions:
+  * this affected posix classes as well as alternation, conditional, and intersection branches
+  * `#options` was already correct for all child expressions of such branches
+  * this only made an operational difference for posix classes as they respect encoding flags
+- Fixed `#options` not respecting all negative options in weird cases like '(?u-m-x)'
+- Fixed `Group#option_changes` not accounting for indirectly disabled (overridden) encoding flags
+- Fixed `Scanner` allowing negative encoding options if there were no positive options, e.g. '(?-u)'
+- Fixed `ScannerError` for some valid meta/control sequences such as '\\C-\\\\'
+- Fixed `Expression#match` and `#=~` not working with a single argument
+
+### [1.5.0] - 2019-05-14 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- Added `#referenced_expression` for backrefs, subexp calls and conditionals
+  * returns the `Group` expression that is being referenced via name or number
+- Added `Expression#repetitions`
+  * returns a `Range` of allowed repetitions (`1..1` if there is no quantifier)
+  * like `#quantity` but with a more uniform interface
+- Added `Expression#match_length`
+  * allows to inspect and iterate over String lengths matched by the Expression
+
+### Fixed
+
+- Fixed `Expression#clone` "direction"
+  * it used to dup ivars onto the callee, leaving only the clone referencing the original objects
+  * this will affect you if you call `#eql?`/`#equal?` on expressions or use them as Hash keys
+- Fixed `#clone` results for `Sequences`, e.g. alternations and conditionals
+  * the inner `#text` was cloned onto the `Sequence` and thus duplicated
+  * e.g. `Regexp::Parser.parse(/(a|bc)/).clone.to_s # => (aa|bcbc)`
+- Fixed inconsistent `#to_s` output for `Sequences`
+  * it used to return only the "specific" text, e.g. "|" for an alternation
+  * now it includes nested expressions as it does for all other `Subexpressions`
+- Fixed quantification of codepoint lists with more than one entry (`\u{62 63 64}+`)
+  * quantifiers apply only to the last entry, so this token is now split up if quantified
+
+### [1.4.0] - 2019-04-02 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- Added support for 19 new unicode properties introduced in Ruby 2.6.0
+
 ### [1.3.0] - 2018-11-14 - [Janosch Müller](mailto:janosch84@gmail.com)
 
 ### Added
@@ -69,7 +140,7 @@ This release includes several breaking changes, mostly to character sets, #map a
 - Changed `(?m)` and the likes to emit as `:options_switch` token (@4ade4d1)
   * allows differentiating from group-local `:options`, e.g. `(?m:.)`
 - Changed name of `Backreference::..NestLevel` to `..RecursionLevel` (@4184339)
-- Changed B`ackreference::Number#number` from `String` to `Integer` (@40a2231)
+- Changed `Backreference::Number#number` from `String` to `Integer` (@40a2231)
 
 ### Added
 

data/Gemfile

@@ -3,7 +3,7 @@ source 'https://rubygems.org'
 gemspec
 
 group :development, :test do
-  gem 'rake'
-  gem 'regexp_property_values'
-  gem 'test-unit'
+  gem 'rake', '~> 13.0'
+  gem 'regexp_property_values', '~> 1.0'
+  gem 'rspec', '~> 3.8'
 end

data/README.md

@@ -136,11 +136,8 @@ Regexp::Scanner.scan( /(cat?([bhm]at)){3,5}/ ).map {|token| token[2]}
     to the lexer.
 
   * The MRI implementation may accept expressions that either conflict with
-    the documentation or are undocumented. The scanner does not support such
-    implementation quirks.
-    _(See issues [#3](https://github.com/ammar/regexp_parser/issues/3) and
-    [#15](https://github.com/ammar/regexp_parser/issues/15) for examples)_
-
+    the documentation or are undocumented, like `{}` and `]` _(unescaped)_.
+    The scanner will try to support as many of these cases as possible.
 
 ---
 ### Syntax
@@ -357,7 +354,7 @@ _Note that not all of these are available in all versions of Ruby_
 |   _**Meta**_                | `\M-c`, `\M-\C-C`, `\M-\cC`, `\C-\M-C`, `\c\M-C`        | ✓ |
 |   _**Octal**_               | `\0`, `\01`, `\012`                                     | ✓ |
 |   _**Unicode**_             | `\uHHHH`, `\u{H+ H+}`                                   | ✓ |
-| **Unicode Properties**                | _<sub>([Unicode 10.0.0](http://www.unicode.org/versions/Unicode10.0.0/))</sub>_ | &#x22f1; |
+| **Unicode Properties**                | _<sub>([Unicode 11.0.0](http://www.unicode.org/versions/Unicode11.0.0/))</sub>_ | &#x22f1; |
 | &emsp;&nbsp;_**Age**_                 | `\p{Age=5.2}`, `\P{age=7.0}`, `\p{^age=8.0}`            | &#x2713; |
 | &emsp;&nbsp;_**Blocks**_              | `\p{InArmenian}`, `\P{InKhmer}`, `\p{^InThai}`          | &#x2713; |
 | &emsp;&nbsp;_**Classes**_             | `\p{Alpha}`, `\P{Space}`, `\p{^Alnum}`                  | &#x2713; |
@@ -391,28 +388,22 @@ To run the tests simply run rake from the root directory, as 'test' is the defau
 
 It generates the scanner's code from the Ragel source files and runs all the tests, thus it requires Ragel to be installed.
 
-The tests use Ruby's test/unit. They can also be run with:
+The tests use RSpec. They can also be run with the test runner that whitelists some warnings:
 
 ```
 bin/test
 ```
 
-The test runner accepts all arguments accepted by test/unit.  You can run a specific test like so:
-
-```
-bin/test test/scanner/test_properties.rb
-```
-
-It is sometimes helpful during development to focus on a specific test case, for example:
+You can run a specific test like so:
 
 ```
-bin/test test/expression/test_base.rb -n test_expression_to_re
+bin/test spec/scanner/properties_spec.rb
 ```
 
-Note that changes to Ragel files will not be reflected when using `bin/test`, so you might want to run:
+Note that changes to Ragel files will not be reflected when running `rspec` or `bin/test`, so you might want to run:
 
 ```
-rake ragel:rb && bin/test test/scanner/test_properties.rb
+rake ragel:rb && bin/test spec/scanner/properties_spec.rb
 ```
 
 ## Building
@@ -440,7 +431,9 @@ Projects using regexp_parser.
 
 - [meta_re](https://github.com/ammar/meta_re) is a regular expression preprocessor with alias support.
 
-- [mutant](https://github.com/mbj/mutant) manipulates your regular expressions (amongst others) to see if your tests cover their behavior.
+- [mutant](https://github.com/mbj/mutant) (before v0.9.0) manipulates your regular expressions (amongst others) to see if your tests cover their behavior.
+
+- [twitter-cldr-rb](https://github.com/twitter/twitter-cldr-rb) uses regexp_parser to generate examples of postal codes.
 
 - [js_regex](https://github.com/janosch-x/js_regex) converts Ruby regular expressions to JavaScript-compatible regular expressions.
 
@@ -471,4 +464,4 @@ Documentation and books used while working on this project.
 
 ---
 ##### Copyright
-_Copyright (c) 2010-2016 Ammar Ali. See LICENSE file for details._
+_Copyright (c) 2010-2019 Ammar Ali. See LICENSE file for details._

data/Rakefile

@@ -74,14 +74,13 @@ namespace :props do
       puts "Wrote #{hash.count} aliases to `#{path}`"
     end
 
-    _, long_names = RegexpPropertyValues.short_and_long_names
-    long_names_to_tokens = long_names.map do |name|
-      [name.downcase.gsub(/[^0-9a-z=.]/, ''), name.downcase]
+    long_names_to_tokens = RegexpPropertyValues.all.map do |val|
+      [val.identifier, val.full_name.downcase]
     end
     write_hash_to_file.call(long_names_to_tokens, "#{dir}/long.yml")
 
     short_names_to_tokens = RegexpPropertyValues.alias_hash.map do |k, v|
-      [k.downcase.gsub(/[^0-9a-z=.]/, ''), v.downcase]
+      [k.identifier, v.full_name.downcase]
     end
     write_hash_to_file.call(short_names_to_tokens, "#{dir}/short.yml")
   end

data/lib/regexp_parser/expression.rb

@@ -21,10 +21,10 @@ module Regexp::Expression
       self.options           = options
     end
 
-    def initialize_clone(other)
-      other.text       = (text        ? text.dup         : nil)
-      other.options    = (options     ? options.dup      : nil)
-      other.quantifier = (quantifier  ? quantifier.clone : nil)
+    def initialize_clone(orig)
+      self.text       = (orig.text       ? orig.text.dup         : nil)
+      self.options    = (orig.options    ? orig.options.dup      : nil)
+      self.quantifier = (orig.quantifier ? orig.quantifier.clone : nil)
       super
     end
 
@@ -62,15 +62,28 @@ module Regexp::Expression
       self.quantifier = Quantifier.new(token, text, min, max, mode)
     end
 
+    def unquantified_clone
+      clone.tap { |exp| exp.quantifier = nil }
+    end
+
     def quantified?
       !quantifier.nil?
     end
 
+    # Deprecated. Prefer `#repetitions` which has a more uniform interface.
     def quantity
       return [nil,nil] unless quantified?
       [quantifier.min, quantifier.max]
     end
 
+    def repetitions
+      return 1..1 unless quantified?
+      min = quantifier.min
+      max = quantifier.max < 0 ? Float::INFINITY : quantifier.max
+      # fix Range#minmax - https://bugs.ruby-lang.org/issues/15807
+      (min..max).tap { |r| r.define_singleton_method(:minmax) { [min, max] } }
+    end
+
     def greedy?
       quantified? and quantifier.greedy?
     end
@@ -84,49 +97,6 @@ module Regexp::Expression
       quantified? and quantifier.possessive?
     end
 
-    def multiline?
-      options[:m] == true
-    end
-    alias :m? :multiline?
-
-    def case_insensitive?
-      options[:i] == true
-    end
-    alias :i? :case_insensitive?
-    alias :ignore_case? :case_insensitive?
-
-    def free_spacing?
-      options[:x] == true
-    end
-    alias :x? :free_spacing?
-    alias :extended? :free_spacing?
-
-    if RUBY_VERSION >= '2.0'
-      def default_classes?
-        options[:d] == true
-      end
-      alias :d? :default_classes?
-
-      def ascii_classes?
-        options[:a] == true
-      end
-      alias :a? :ascii_classes?
-
-      def unicode_classes?
-        options[:u] == true
-      end
-      alias :u? :unicode_classes?
-    end
-
-    def matches?(string)
-      Regexp.new(to_s) =~ string ? true : false
-    end
-
-    def match(string, offset)
-      Regexp.new(to_s).match(string, offset)
-    end
-    alias :=~ :match
-
     def attributes
       {
         type:              type,
@@ -145,12 +115,14 @@ module Regexp::Expression
   end
 
   def self.parsed(exp)
+    warn('WARNING: Regexp::Expression::Base.parsed is buggy and '\
+         'will be removed in 2.0.0. Use Regexp::Parser.parse instead.')
     case exp
     when String
       Regexp::Parser.parse(exp)
     when Regexp
-      Regexp::Parser.parse(exp.source)
-    when Regexp::Expression
+      Regexp::Parser.parse(exp.source) # <- causes loss of root options
+    when Regexp::Expression            # <- never triggers
       exp
     else
       raise ArgumentError, 'Expression.parsed accepts a String, Regexp, or '\
@@ -161,10 +133,6 @@ module Regexp::Expression
 
 end # module Regexp::Expression
 
-require 'regexp_parser/expression/methods/tests'
-require 'regexp_parser/expression/methods/traverse'
-require 'regexp_parser/expression/methods/strfregexp'
-
 require 'regexp_parser/expression/quantifier'
 require 'regexp_parser/expression/subexpression'
 require 'regexp_parser/expression/sequence'
@@ -186,3 +154,10 @@ require 'regexp_parser/expression/classes/set'
 require 'regexp_parser/expression/classes/set/intersection'
 require 'regexp_parser/expression/classes/set/range'
 require 'regexp_parser/expression/classes/type'
+
+require 'regexp_parser/expression/methods/match'
+require 'regexp_parser/expression/methods/match_length'
+require 'regexp_parser/expression/methods/options'
+require 'regexp_parser/expression/methods/strfregexp'
+require 'regexp_parser/expression/methods/tests'
+require 'regexp_parser/expression/methods/traverse'

data/lib/regexp_parser/expression/classes/backref.rb

@@ -1,9 +1,12 @@
 module Regexp::Expression
   module Backreference
-    class Base < Regexp::Expression::Base; end
+    class Base < Regexp::Expression::Base
+      attr_accessor :referenced_expression
+    end
 
     class Number < Backreference::Base
       attr_reader :number
+      alias reference number
 
       def initialize(token, options = {})
         @number = token.text[token.token.equal?(:number) ? 1..-1 : 3..-2].to_i
@@ -13,6 +16,7 @@ module Regexp::Expression
 
     class Name < Backreference::Base
       attr_reader :name
+      alias reference name
 
       def initialize(token, options = {})
         @name = token.text[3..-2]
@@ -20,27 +24,31 @@ module Regexp::Expression
       end
     end
 
+    class NumberRelative     < Backreference::Number
+      attr_accessor :effective_number
+      alias reference effective_number
+    end
+
     class NumberCall         < Backreference::Number; end
-    class NumberRelative     < Backreference::Number; end
-    class NumberCallRelative < Backreference::Number; end
-    class NameCall < Backreference::Name; end
+    class NameCall           < Backreference::Name; end
+    class NumberCallRelative < Backreference::NumberRelative; end
 
-    class NumberRecursionLevel < Backreference::Base
-      attr_reader :number, :recursion_level
+    class NumberRecursionLevel < Backreference::Number
+      attr_reader :recursion_level
 
       def initialize(token, options = {})
-        @number, @recursion_level = token.text[3..-2].split(/(?=[+-])/).map(&:to_i)
         super
+        @number, @recursion_level = token.text[3..-2].split(/(?=[+-])/).map(&:to_i)
       end
     end
 
-    class NameRecursionLevel < Backreference::Base
-      attr_reader :name, :recursion_level
+    class NameRecursionLevel < Backreference::Name
+      attr_reader :recursion_level
 
       def initialize(token, options = {})
+        super
         @name, recursion_level = token.text[3..-2].split(/(?=[+-])/)
         @recursion_level = recursion_level.to_i
-        super
       end
     end
   end

data/lib/regexp_parser/expression/classes/conditional.rb

@@ -7,6 +7,8 @@ module Regexp::Expression
     end
 
     class Condition < Regexp::Expression::Base
+      attr_accessor :referenced_expression
+
       # Name or number of the referenced capturing group that determines state.
       # Returns a String if reference is by name, Integer if by number.
       def reference
@@ -18,13 +20,16 @@ module Regexp::Expression
     class Branch < Regexp::Expression::Sequence; end
 
     class Expression < Regexp::Expression::Subexpression
+      attr_accessor :referenced_expression
+
       def <<(exp)
         expressions.last << exp
       end
 
-      def add_sequence
+      def add_sequence(active_opts = {})
         raise TooManyBranches.new if branches.length == 2
-        Branch.add_to(self, { conditional_level: conditional_level + 1 })
+        params = { conditional_level: conditional_level + 1 }
+        Branch.add_to(self, params, active_opts)
       end
       alias :branch :add_sequence
 

data/lib/regexp_parser/expression/classes/escape.rb

@@ -60,10 +60,6 @@ module Regexp::Expression
         codepoint.chr('utf-8')
       end
 
-      def codepoint
-        raise NotImplementedError, 'implement in subclass'
-      end
-
       private
 
       def control_sequence_to_s(control_sequence)

data/lib/regexp_parser/expression/classes/group.rb

@@ -19,20 +19,22 @@ module Regexp::Expression
 
     class Capture < Group::Base
       attr_accessor :number, :number_at_level
+      alias identifier number
 
       def capturing?; true end
     end
 
     class Named < Group::Capture
       attr_reader :name
+      alias identifier name
 
       def initialize(token, options = {})
         @name = token.text[3..-2]
         super
       end
 
-      def initialize_clone(other)
-        other.instance_variable_set(:@name, name.dup)
+      def initialize_clone(orig)
+        @name = orig.name.dup
         super
       end
     end