regexp_parser 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20ba21704667276107a1041b3bb5943bbbec0078f706cf0d7db85110631dfe8d
-  data.tar.gz: 87886f6cad480ebc62f3e1f243d9b61170097e5419fc8b3972cd3348e5d8d7e0
+  metadata.gz: 9af74355b57139b93d0207bfbf6fc6fbb43c1eb6aa531bd11710e39168af107a
+  data.tar.gz: 8d63533aa2206b121fd59959a43cad2b64e3b3c6d0f8617953d2c401033c86d5
 SHA512:
-  metadata.gz: '0678640973741b2ea63053c058809fa075b3b465756bddee9a1914f67f7181a3681d3592662d4eadf5a60e844c550950b371577239924c4d3ce7f07f9fdfefa6'
-  data.tar.gz: 3bf18d0d7989c1f9eef010d1579ac78537c6c083c9b7c7c2f0cda094c0f973e1fdcc17c5992ae35d823720d2cdb10a60424876e08bd4b2b60b125c8b107a62bf
+  metadata.gz: de6afb1888075eaa8d220192882494eb67c54aaba5fa84b991adf753dbb06949b8b0ea6d44054d32c3cba1e8cb6ee04d8f8f2f8643e4663188ddca3b7ec7e07a
+  data.tar.gz: f9e7d91a317bb153b645380a6447faca2656b87fa164afae9dc1cf9bc63463c11a87388258d18a49ed3df7afc3a7b25cd0a2113db412c2e7f504cee9aec8c255

data/CHANGELOG.md

@@ -1,3 +1,16 @@
+## [Unreleased]
+
+### [1.3.0] - 2018-11-14 - [Janosch Müller](mailto:janosch84@gmail.com)
+
+### Added
+
+- `Syntax#features` returns a `Hash` of all types and tokens supported by a given `Syntax`
+
+### Fixed
+
+- Thanks to [Akira Matsuda](https://github.com/amatsuda)
+  * eliminated warning "assigned but unused variable - testEof"
+
 ## [1.2.0] - 2018-09-28 - [Janosch Müller](mailto:janosch84@gmail.com)
 
 ### Added
@@ -147,15 +160,15 @@ This release includes several breaking changes, mostly to character sets, #map a
 
 ## [0.4.5] - 2017-09-17 - [Ammar Ali](mailto:ammarabuali@gmail.com)
 
-- Thanks to Janosch Müller (https://github.com/janosch-x):
+- Thanks to [Janosch Müller](https://github.com/janosch-x):
   * Support ruby 2.2.7 (PR #42)
 - Added ruby version files for 2.2.8, 2.3.5, and 2.4.2
 
 ## [0.4.4] - 2017-07-10 - [Ammar Ali](mailto:ammarabuali@gmail.com)
 
-- Thanks to Janosch Müller (https://github.com/janosch-x):
+- Thanks to [Janosch Müller](https://github.com/janosch-x):
   * Add support for new absence operator (PR #33)
-- Thanks to Bartek Bułat (https://github.com/barthez):
+- Thanks to [Bartek Bułat](https://github.com/barthez):
   * Add support for Ruby 2.3.4 version (PR #40)
 
 ## [0.4.3] - 2017-03-24 - [Ammar Ali](mailto:ammarabuali@gmail.com)
@@ -164,7 +177,7 @@ This release includes several breaking changes, mostly to character sets, #map a
 
 ## [0.4.2] - 2017-01-10 - [Ammar Ali](mailto:ammarabuali@gmail.com)
 
-- Thanks to Janosch Müller (https://github.com/janosch-x):
+- Thanks to [Janosch Müller](https://github.com/janosch-x):
   * Support ruby 2.4 (PR #30)
   * Improve codepoint handling (PR #27)
 
@@ -179,26 +192,26 @@ This release includes several breaking changes, mostly to character sets, #map a
 
 ## [0.3.6] - 2016-06-08 - [Ammar Ali](mailto:ammarabuali@gmail.com)
 
-- Thanks to John Backus (https://github.com/backus):
+- Thanks to [John Backus](https://github.com/backus):
   * Remove warnings (PR #26)
 
 ## [0.3.5] - 2016-05-30 - [Ammar Ali](mailto:ammarabuali@gmail.com)
 
-- Thanks to John Backus (https://github.com/backus):
+- Thanks to [John Backus](https://github.com/backus):
   * Fix parsing of /\xFF/n (hex:escape) (PR #24)
 
 ## [0.3.4] - 2016-05-25 - [Ammar Ali](mailto:ammarabuali@gmail.com)
 
-- Thanks to John Backus (https://github.com/backus):
+- Thanks to [John Backus](https://github.com/backus):
   * Fix warnings (PR #19)
-- Thanks to Dana Scheider (https://github.com/danascheider):
+- Thanks to [Dana Scheider](https://github.com/danascheider):
   * Correct error in README (PR #20)
 - Fixed mistyped \h and \H character types (issue #21)
 - Added ancestry syntax files for latest rubies (issue #22)
 
 ## [0.3.3] - 2016-04-26 - [Ammar Ali](mailto:ammarabuali@gmail.com)
 
-- Thanks to John Backus (https://github.com/backus):
+- Thanks to [John Backus](https://github.com/backus):
   * Fixed scanning of zero length comments (PR #12)
   * Fixed missing escape:codepoint_list syntax token (PR #14)
   * Fixed to_s for modified interval quantifiers (PR #17)

data/lib/regexp_parser/scanner.rb

@@ -2742,6 +2742,9 @@ end
 
 # line 810 "/Users/jannoschmuller/code/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
 
+    # to avoid "warning: assigned but unused variable - testEof"
+    testEof = testEof
+
     if cs == re_scanner_error
       text = ts ? copy(data, ts-1..-1) : data.pack('c*')
       raise ScannerError.new("Scan error at '#{text}'")

data/lib/regexp_parser/scanner/scanner.rl

@@ -808,6 +808,9 @@ class Regexp::Scanner
     %% write init;
     %% write exec;
 
+    # to avoid "warning: assigned but unused variable - testEof"
+    testEof = testEof
+
     if cs == re_scanner_error
       text = ts ? copy(data, ts-1..-1) : data.pack('c*')
       raise ScannerError.new("Scan error at '#{text}'")

data/lib/regexp_parser/syntax/base.rb

@@ -18,6 +18,10 @@ module Regexp::Syntax
       implements Token::FreeSpace::Type, Token::FreeSpace::All
     end
 
+    def features
+      @implements
+    end
+
     def implementations(type)
       @implements[type] ||= Set.new
     end

data/lib/regexp_parser/syntax/tokens/anchor.rb

@@ -1,6 +1,5 @@
 module Regexp::Syntax
   module Token
-
     module Anchor
       Basic       = [:bol, :eol]
       Extended    = Basic + [:word_boundary, :nonword_boundary]
@@ -12,6 +11,5 @@ module Regexp::Syntax
     end
 
     Map[Anchor::Type] = Anchor::All
-
   end
 end

data/lib/regexp_parser/syntax/tokens/assertion.rb

@@ -1,6 +1,5 @@
 module Regexp::Syntax
   module Token
-
     module Assertion
       Lookahead = [:lookahead, :nlookahead]
       Lookbehind = [:lookbehind, :nlookbehind]
@@ -10,6 +9,5 @@ module Regexp::Syntax
     end
 
     Map[Assertion::Type] = Assertion::All
-
   end
 end

data/lib/regexp_parser/syntax/tokens/backref.rb

@@ -1,6 +1,5 @@
 module Regexp::Syntax
   module Token
-
     module Backreference
       Name      = [:name_ref]
       Number    = [:number, :number_ref, :number_rel_ref]
@@ -21,6 +20,5 @@ module Regexp::Syntax
 
     Map[Backreference::Type] = Backreference::All +
                                SubexpressionCall::All
-
   end
 end

data/lib/regexp_parser/syntax/tokens/character_set.rb

@@ -1,13 +1,10 @@
 module Regexp::Syntax
   module Token
-
     module CharacterSet
-      OpenClose = [:open, :close]
-
-      Basic     = [:negate, :range]
-      Extended  = Basic + [:intersection, :backspace]
+      Basic     = [:open, :close, :negate, :range]
+      Extended  = Basic + [:intersection]
 
-      All = Basic + Extended
+      All = Extended
       Type = :set
     end
 

data/lib/regexp_parser/syntax/tokens/character_type.rb

@@ -1,6 +1,5 @@
 module Regexp::Syntax
   module Token
-
     module CharacterType
       Basic     = []
       Extended  = [:digit, :nondigit, :space, :nonspace, :word, :nonword]
@@ -13,6 +12,5 @@ module Regexp::Syntax
     end
 
     Map[CharacterType::Type] = CharacterType::All
-
   end
 end

data/lib/regexp_parser/syntax/tokens/conditional.rb

@@ -1,22 +1,16 @@
 module Regexp::Syntax
   module Token
-
     module Conditional
-      Delimiters  = [:open, :close]
+      Delimiters = [:open, :close]
 
-      Condition   = [:condition_open, :condition, :condition_close]
-      Separator   = [:separator]
+      Condition  = [:condition_open, :condition, :condition_close]
+      Separator  = [:separator]
 
-      All = (
-        Conditional::Delimiters +
-        Conditional::Condition +
-        Conditional::Separator
-      ).flatten
+      All = Conditional::Delimiters + Conditional::Condition + Conditional::Separator
 
       Type = :conditional
     end
 
     Map[Conditional::Type] = Conditional::All
-
   end
 end

data/lib/regexp_parser/syntax/tokens/escape.rb

@@ -1,10 +1,7 @@
 module Regexp::Syntax
   module Token
-
     module Escape
-      Basic     = [:backslash, :literal]
-
-      Backreference = [:digit]
+      Basic = [:backslash, :literal]
 
       Control = [:control, :meta_sequence]
 
@@ -24,11 +21,10 @@ module Regexp::Syntax
 
       Octal = [:octal]
 
-      All   = Basic + Backreference + ASCII + Meta
+      All   = Basic + Control + ASCII + Unicode + Meta + Hex + Octal
       Type  = :escape
     end
 
     Map[Escape::Type] = Escape::All
-
   end
 end

data/lib/regexp_parser/syntax/tokens/group.rb

@@ -1,6 +1,5 @@
 module Regexp::Syntax
   module Token
-
     module Group
       Basic     = [:capture, :close]
       Extended  = Basic + [:options, :options_switch]
@@ -20,6 +19,5 @@ module Regexp::Syntax
     end
 
     Map[Group::Type] = Group::All
-
   end
 end

data/lib/regexp_parser/syntax/tokens/keep.rb

@@ -1,6 +1,5 @@
 module Regexp::Syntax
   module Token
-
     module Keep
       Mark = [:mark]
 
@@ -9,6 +8,5 @@ module Regexp::Syntax
     end
 
     Map[Keep::Type] = Keep::All
-
   end
 end

data/lib/regexp_parser/syntax/tokens/meta.rb

@@ -1,15 +1,13 @@
 module Regexp::Syntax
   module Token
-
     module Meta
-      Basic     = [:dot]
-      Extended  = Basic + [:alternation]
+      Basic    = [:dot]
+      Extended = Basic + [:alternation]
 
       All = Extended
       Type = :meta
     end
 
     Map[Meta::Type] = Meta::All
-
   end
 end

data/lib/regexp_parser/syntax/tokens/posix_class.rb

@@ -1,8 +1,8 @@
 module Regexp::Syntax
   module Token
     module PosixClass
-      Standard  = [:alnum, :alpha, :blank, :cntrl, :digit, :graph,
-                   :lower, :print, :punct, :space, :upper, :xdigit]
+      Standard = [:alnum, :alpha, :blank, :cntrl, :digit, :graph,
+                  :lower, :print, :punct, :space, :upper, :xdigit]
 
       Extensions = [:ascii, :word]
 
@@ -10,6 +10,7 @@ module Regexp::Syntax
       Type = :posixclass
       NonType = :nonposixclass
     end
+
     Map[PosixClass::Type]    = PosixClass::All
     Map[PosixClass::NonType] = PosixClass::All
   end

data/lib/regexp_parser/syntax/tokens/quantifier.rb

@@ -1,6 +1,5 @@
 module Regexp::Syntax
   module Token
-
     module Quantifier
       Greedy    = [
         :zero_or_one,
@@ -32,6 +31,5 @@ module Regexp::Syntax
     end
 
     Map[Quantifier::Type] = Quantifier::All
-
   end
 end

data/lib/regexp_parser/syntax/tokens/unicode_property.rb

@@ -1,13 +1,12 @@
 module Regexp::Syntax
   module Token
-
     module UnicodeProperty
       CharType_V1_9_0 = [:alnum, :alpha, :ascii, :blank, :cntrl, :digit, :graph,
-                       :lower, :print, :punct, :space, :upper, :word, :xdigit]
+                         :lower, :print, :punct, :space, :upper, :word, :xdigit]
 
       CharType_V2_5_0 = [:xposixpunct]
 
-      POSIX  = [:any, :assigned, :newline]
+      POSIX = [:any, :assigned, :newline]
 
       module Category
         Letter        = [:letter, :uppercase_letter, :lowercase_letter,
@@ -36,14 +35,12 @@ module Regexp::Syntax
               Symbol + Separator + Codepoint
       end
 
-      # As of ruby version 1.9.3
       Age_V1_9_3 = [:'age=1.1', :'age=2.0', :'age=2.1', :'age=3.0', :'age=3.1',
                     :'age=3.2', :'age=4.0', :'age=4.1', :'age=5.0', :'age=5.1',
                     :'age=5.2', :'age=6.0']
 
       Age_V2_0_0 = [:'age=6.1']
 
-      # These were merged (from Onigmo) in the branch for 2.2.0
       Age_V2_2_0 = [:'age=6.2', :'age=6.3', :'age=7.0']
 
       Age_V2_3_0 = [:'age=8.0']
@@ -291,8 +288,6 @@ module Regexp::Syntax
 
       UnicodeBlock_V1_9_0 = [
         :in_alphabetic_presentation_forms,
-        :in_arabic_presentation_forms_a,
-        :in_arabic_presentation_forms_b,
         :in_arabic,
         :in_armenian,
         :in_arrows,
@@ -350,17 +345,12 @@ module Regexp::Syntax
         :in_khmer_symbols,
         :in_khmer,
         :in_lao,
-        :in_latin_1_supplement,
-        :in_latin_extended_a,
-        :in_latin_extended_b,
         :in_latin_extended_additional,
         :in_letterlike_symbols,
         :in_limbu,
         :in_low_surrogates,
         :in_malayalam,
         :in_mathematical_operators,
-        :in_miscellaneous_mathematical_symbols_a,
-        :in_miscellaneous_mathematical_symbols_b,
         :in_miscellaneous_symbols_and_arrows,
         :in_miscellaneous_symbols,
         :in_miscellaneous_technical,
@@ -378,8 +368,6 @@ module Regexp::Syntax
         :in_spacing_modifier_letters,
         :in_specials,
         :in_superscripts_and_subscripts,
-        :in_supplemental_arrows_a,
-        :in_supplemental_arrows_b,
         :in_supplemental_mathematical_operators,
         :in_syriac,
         :in_tagalog,
@@ -610,7 +598,8 @@ module Regexp::Syntax
 
       Emoji = Emoji_V2_5_0
 
-      V1_9_0 = CharType_V1_9_0 + POSIX + Category::All + Derived_V1_9_0 + Script_V1_9_0 + UnicodeBlock_V1_9_0
+      V1_9_0 = Category::All + POSIX + CharType_V1_9_0 + Derived_V1_9_0 + Script_V1_9_0 + UnicodeBlock_V1_9_0
+
       V1_9_3 = Age_V1_9_3 + Script_V1_9_3
 
       V2_0_0 = Age_V2_0_0 + Derived_V2_0_0 + Script_V2_0_0 + UnicodeBlock_V2_0_0
@@ -623,7 +612,7 @@ module Regexp::Syntax
 
       V2_5_0 = Age_V2_5_0 + CharType_V2_5_0 + Derived_V2_5_0 + Emoji_V2_5_0 + Script_V2_5_0 + UnicodeBlock_V2_5_0
 
-      All  = V1_9_0 + V1_9_3 + V2_0_0 + V2_2_0 + V2_3_0 + V2_4_0 + V2_5_0
+      All = V1_9_0 + V1_9_3 + V2_0_0 + V2_2_0 + V2_3_0 + V2_4_0 + V2_5_0
 
       Type = :property
       NonType = :nonproperty
@@ -631,6 +620,5 @@ module Regexp::Syntax
 
     Map[UnicodeProperty::Type] = UnicodeProperty::All
     Map[UnicodeProperty::NonType] = UnicodeProperty::All
-
   end
 end

data/lib/regexp_parser/syntax/versions/1.8.6.rb

@@ -7,22 +7,15 @@ module Regexp::Syntax
       implements :assertion, Assertion::Lookahead
       implements :backref, [:number]
       implements :posixclass, PosixClass::Standard
-      implements :escape,
-        Escape::Basic + Escape::Backreference +
-        Escape::ASCII + Escape::Meta + Escape::Control
-
       implements :group, Group::All
-
       implements :meta, Meta::Extended
-
+      implements :set, CharacterSet::All
+      implements :type, CharacterType::Extended
+      implements :escape,
+        Escape::Basic + Escape::ASCII + Escape::Meta + Escape::Control
       implements :quantifier,
         Quantifier::Greedy + Quantifier::Reluctant +
         Quantifier::Interval + Quantifier::IntervalReluctant
-
-      implements :set, CharacterSet::OpenClose + CharacterSet::Extended
-
-      implements :type,
-        CharacterType::Extended
     end
   end
 end

data/lib/regexp_parser/syntax/versions/1.9.1.rb

@@ -3,26 +3,14 @@ module Regexp::Syntax
     def initialize
       super
 
-      implements :assertion, Assertion::Lookbehind +
-        SubexpressionCall::All
-
-      implements :backref, Backreference::All +
-        SubexpressionCall::All
-
+      implements :assertion, Assertion::Lookbehind
+      implements :backref, Backreference::All + SubexpressionCall::All
       implements :posixclass, PosixClass::Extensions
-
       implements :nonposixclass, PosixClass::All
-
       implements :escape, Escape::Unicode + Escape::Hex + Escape::Octal
-
       implements :type, CharacterType::Hex
-
-      implements :property,
-        UnicodeProperty::V1_9_0
-
-      implements :nonproperty,
-        UnicodeProperty::V1_9_0
-
+      implements :property, UnicodeProperty::V1_9_0
+      implements :nonproperty, UnicodeProperty::V1_9_0
       implements :quantifier,
         Quantifier::Possessive + Quantifier::IntervalPossessive
     end

data/lib/regexp_parser/syntax/versions/2.0.0.rb

@@ -8,7 +8,6 @@ module Regexp::Syntax
       implements :conditional, Conditional::All
       implements :property,    UnicodeProperty::V2_0_0
       implements :nonproperty, UnicodeProperty::V2_0_0
-
       implements :type,        CharacterType::Clustered
 
       excludes   :property,    :newline

data/lib/regexp_parser/version.rb

@@ -1,5 +1,5 @@
 class Regexp
   class Parser
-    VERSION = '1.2.0'
+    VERSION = '1.3.0'
   end
 end

data/test/syntax/test_all.rb

@@ -2,3 +2,5 @@ require File.expand_path("../../helpers", __FILE__)
 
 require File.expand_path("../test_syntax", __FILE__)
 require File.expand_path("../versions/test_all", __FILE__)
+
+require File.expand_path("../test_syntax_token_map", __FILE__)

data/test/syntax/test_syntax_token_map.rb

@@ -0,0 +1,25 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class TestSyntaxTokenMap < Test::Unit::TestCase
+  def test_syntax_token_map_complete
+    map = Regexp::Syntax::Token::Map
+    latest_syntax = Regexp::Syntax.new 'ruby/2.9'
+
+    latest_syntax.features.each do |type, tokens|
+      tokens.each do |token|
+        assert map[type].include?(token),
+               "`#{type} #{token}` implemented but missing from Map"
+      end
+    end
+  end
+
+  def test_syntax_token_map_uniq
+    combinations = Regexp::Syntax::Token::Map.flat_map do |type, tokens|
+      tokens.map { |token| "#{type} #{token}" }
+    end
+
+    non_uniq = combinations.group_by { |str| str }.select { |_, v| v.count > 1 }
+
+    assert_empty non_uniq.keys
+  end
+end

data/test/syntax/versions/test_1.8.rb

@@ -11,7 +11,9 @@ class TestSyntaxV1_8 < Test::Unit::TestCase
     implements: {
       assertion:    [Assertion::Lookahead].flatten,
       backref:      [:number],
-      escape:       [Escape::All].flatten,
+      escape:       [
+        Escape::Basic + Escape::ASCII + Escape::Meta + Escape::Control
+      ].flatten,
       group:        [Group::V1_8_6].flatten,
       quantifier:   [
           Quantifier::Greedy + Quantifier::Reluctant +

data/test/syntax/versions/test_1.9.1.rb

@@ -10,8 +10,7 @@ class TestSyntaxV1_9_1 < Test::Unit::TestCase
   tests = {
     implements: {
       escape: [
-        Escape::Backreference + Escape::ASCII + Escape::Meta + Escape::Unicode +
-        Escape::Hex + Escape::Octal
+        Escape::Hex + Escape::Octal + Escape::Unicode
       ].flatten,
 
       type: [

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: regexp_parser
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Ammar Ali
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-28 00:00:00.000000000 Z
+date: 2018-11-14 00:00:00.000000000 Z
 dependencies: []
 description: A library for tokenizing, lexing, and parsing Ruby regular expressions.
 email:
@@ -144,6 +144,7 @@ files:
 - test/support/warning_extractor.rb
 - test/syntax/test_all.rb
 - test/syntax/test_syntax.rb
+- test/syntax/test_syntax_token_map.rb
 - test/syntax/versions/test_1.8.rb
 - test/syntax/versions/test_1.9.1.rb
 - test/syntax/versions/test_1.9.3.rb
@@ -223,6 +224,7 @@ test_files:
 - test/expression/test_traverse.rb
 - test/expression/test_all.rb
 - test/expression/test_strfregexp.rb
+- test/syntax/test_syntax_token_map.rb
 - test/syntax/versions/test_2.0.0.rb
 - test/syntax/versions/test_aliases.rb
 - test/syntax/versions/test_1.8.rb