regexp_parser 0.1.6 → 0.2.0

data/lib/regexp_parser/syntax.rb

@@ -1,5 +1,6 @@
+require File.expand_path('../syntax/tokens', __FILE__)
+
 module Regexp::Syntax
-  require File.expand_path('../syntax/tokens', __FILE__)
 
   class SyntaxError < StandardError
     def initialize(what)
@@ -63,10 +64,18 @@ module Regexp::Syntax
       when 'ruby/2.1.0';  syntax = Regexp::Syntax::Ruby::V210.new
       when 'ruby/2.1.2';  syntax = Regexp::Syntax::Ruby::V212.new
       when 'ruby/2.1.3';  syntax = Regexp::Syntax::Ruby::V213.new
+      when 'ruby/2.1.4';  syntax = Regexp::Syntax::Ruby::V214.new
+      when 'ruby/2.1.5';  syntax = Regexp::Syntax::Ruby::V215.new
 
       # aliases for the latest 2.1 implementations
       when 'ruby/2.1';    syntax = Regexp::Syntax::Ruby::V21.new
 
+      # Ruby 2.2.x
+      when 'ruby/2.2.0';  syntax = Regexp::Syntax::Ruby::V220.new
+
+      # aliases for the latest 2.2 implementations
+      when 'ruby/2.2';    syntax = Regexp::Syntax::Ruby::V22.new
+
       else
         raise UnknownSyntaxError.new(name)
     end
@@ -87,7 +96,8 @@ module Regexp::Syntax
     def initialize
       @implements = {}
 
-      implements :literal, [:literal]
+      implements Token::Literal::Type,   Token::Literal::All
+      implements Token::FreeSpace::Type, Token::FreeSpace::All
     end
 
     def implementation

data/lib/regexp_parser/syntax/ruby/1.9.1.rb

@@ -18,10 +18,10 @@ module Regexp::Syntax
         implements :escape, CharacterType::Hex
 
         implements :property,
-          UnicodeProperty::All
+          UnicodeProperty::V190
 
         implements :nonproperty,
-          UnicodeProperty::All
+          UnicodeProperty::V190
 
         implements :quantifier,
           Quantifier::Possessive + Quantifier::IntervalPossessive
@@ -30,7 +30,7 @@ module Regexp::Syntax
           CharacterSet::POSIX::StandardNegative +
           CharacterSet::POSIX::Extensions +
           CharacterSet::POSIX::ExtensionsNegative +
-          UnicodeProperty::All
+          UnicodeProperty::V190
 
         implements :subset, CharacterSet::OpenClose +
           CharacterSet::Extended + CharacterSet::Types +

data/lib/regexp_parser/syntax/ruby/1.9.3.rb

@@ -10,13 +10,8 @@ module Regexp::Syntax
         super
 
         # these were added with update of Oniguruma to Unicode 6.0
-        implements :property,
-          [:script_mandaic, :script_batak, :script_brahmi] +
-          UnicodeProperty::Age
-
-        implements :nonproperty,
-          [:script_mandaic, :script_batak, :script_brahmi] +
-          UnicodeProperty::Age
+        implements :property,    UnicodeProperty::V193
+        implements :nonproperty, UnicodeProperty::V193
       end
     end
   end

data/lib/regexp_parser/syntax/ruby/2.0.0.rb

@@ -8,7 +8,10 @@ module Regexp::Syntax
       def initialize
         super
 
-        #implements :escape, CharacterType::Hex
+        implements :keep,        Keep::All
+        implements :conditional, Conditional::All
+        implements :property,    UnicodeProperty::V200
+        implements :nonproperty, UnicodeProperty::V200
       end
     end
 

data/lib/regexp_parser/syntax/ruby/2.1.4.rb

@@ -0,0 +1,13 @@
+require File.expand_path('../2.1.3', __FILE__)
+
+module Regexp::Syntax
+  module Ruby
+
+    class V214 < Regexp::Syntax::Ruby::V213
+      def initialize
+        super
+      end
+    end
+
+  end
+end

data/lib/regexp_parser/syntax/ruby/2.1.5.rb

@@ -0,0 +1,13 @@
+require File.expand_path('../2.1.4', __FILE__)
+
+module Regexp::Syntax
+  module Ruby
+
+    class V215 < Regexp::Syntax::Ruby::V214
+      def initialize
+        super
+      end
+    end
+
+  end
+end

data/lib/regexp_parser/syntax/ruby/2.1.rb

@@ -1,8 +1,8 @@
-require File.expand_path('../2.1.3', __FILE__)
+require File.expand_path('../2.1.5', __FILE__)
 
 module Regexp::Syntax
   module Ruby
     # uses the latest 2.1 release
-    class V21 < Regexp::Syntax::Ruby::V213; end
+    class V21 < Regexp::Syntax::Ruby::V215; end
   end
 end

data/lib/regexp_parser/syntax/ruby/2.2.0.rb

@@ -0,0 +1,16 @@
+require File.expand_path('../2.1', __FILE__)
+
+module Regexp::Syntax
+  module Ruby
+
+    class V220 < Regexp::Syntax::Ruby::V21
+      def initialize
+        super
+
+        implements :property,    UnicodeProperty::V220
+        implements :nonproperty, UnicodeProperty::V220
+      end
+    end
+
+  end
+end

data/lib/regexp_parser/syntax/ruby/2.2.rb

@@ -0,0 +1,8 @@
+require File.expand_path('../2.2.0', __FILE__)
+
+module Regexp::Syntax
+  module Ruby
+    # uses the latest 2.2 release
+    class V22 < Regexp::Syntax::Ruby::V220; end
+  end
+end

data/lib/regexp_parser/syntax/tokens.rb

@@ -8,13 +8,30 @@ module Regexp::Syntax
       Type = :literal
     end
 
-    Map[Literal::Type] = Literal::All
+    module FreeSpace
+      All  = [:comment, :whitespace]
+      Type = :free_space
+    end
+
+    Map[FreeSpace::Type] = FreeSpace::All
+    Map[Literal::Type]   = Literal::All
   end
 end
 
 
 # Load all the token files, they will populate the Map constant.
-Dir[File.dirname(__FILE__) + '/tokens/*.rb'].each {|f| require f }
+require 'regexp_parser/syntax/tokens/anchor'
+require 'regexp_parser/syntax/tokens/assertion'
+require 'regexp_parser/syntax/tokens/backref'
+require 'regexp_parser/syntax/tokens/character_set'
+require 'regexp_parser/syntax/tokens/character_type'
+require 'regexp_parser/syntax/tokens/conditional'
+require 'regexp_parser/syntax/tokens/escape'
+require 'regexp_parser/syntax/tokens/group'
+require 'regexp_parser/syntax/tokens/keep'
+require 'regexp_parser/syntax/tokens/meta'
+require 'regexp_parser/syntax/tokens/quantifier'
+require 'regexp_parser/syntax/tokens/unicode_property'
 
 
 # After loading all the tokens the map is full. Extract all tokens and types

data/lib/regexp_parser/syntax/tokens/conditional.rb

@@ -0,0 +1,22 @@
+module Regexp::Syntax
+  module Token
+
+    module Conditional
+      Delimiters  = [:open, :close]
+
+      Condition   = [:condition_open, :condition, :condition_close]
+      Separator   = [:separator]
+
+      All = (
+        Conditional::Delimiters +
+        Conditional::Condition +
+        Conditional::Separator
+      ).flatten
+
+      Type = :conditional
+    end
+
+    Map[Conditional::Type] = Conditional::All
+
+  end
+end

data/lib/regexp_parser/syntax/tokens/keep.rb

@@ -0,0 +1,14 @@
+module Regexp::Syntax
+  module Token
+
+    module Keep
+      Mark = [:mark]
+
+      All  = Mark
+      Type = :keep
+    end
+
+    Map[Keep::Type] = Keep::All
+
+  end
+end

data/lib/regexp_parser/syntax/tokens/unicode_property.rb

@@ -34,9 +34,17 @@ module Regexp::Syntax
               Symbol + Separator + Codepoint
       end
 
-      Age = [:age_1_1, :age_2_0, :age_2_1, :age_3_0, :age_3_1,
-             :age_3_2, :age_4_0, :age_4_1, :age_5_0, :age_5_1,
-             :age_5_2, :age_6_0]
+      # As of ruby version 1.9.3
+      Age_V193 = [:age_1_1, :age_2_0, :age_2_1, :age_3_0, :age_3_1,
+                  :age_3_2, :age_4_0, :age_4_1, :age_5_0, :age_5_1,
+                  :age_5_2, :age_6_0]
+
+      Age_V200 = [:age_6_1]
+
+      # These were merged (from Onigmo) in the branch for 2.2.0
+      Age_V220 = [:age_6_2, :age_6_3, :age_7_0]
+
+      Age = Age_V193 + Age_V200 + Age_V220
 
       Derived = [
         :ascii_hex,
@@ -191,7 +199,40 @@ module Regexp::Syntax
 
       Script_6_0 = [:script_brahmi, :script_batak, :script_mandaic]
 
-      All = CharType + POSIX + Category::All + Age + Derived + Script
+      Script_7_0 = [
+        :script_caucasian_albanian,
+        :script_bassa_vah,
+        :script_duployan,
+        :script_elbasan,
+        :script_grantha,
+        :script_pahawh_hmong,
+        :script_khojki,
+        :script_linear_a,
+        :script_mahajani,
+        :script_manichaean,
+        :script_mende_kikakui,
+        :script_modi,
+        :script_mro,
+        :script_old_north_arabian,
+        :script_nabataean,
+        :script_palmyrene,
+        :script_pau_cin_hau,
+        :script_old_permic,
+        :script_psalter_pahlavi,
+        :script_siddham,
+        :script_khudawadi,
+        :script_tirhuta,
+        :script_warang_citi
+      ]
+
+      V190 = CharType + POSIX + Category::All + Derived + Script
+      V193 = Age_V193 + Script_6_0
+
+      V200 = Age_V200
+
+      V220 = Age_V220 + Script_7_0
+
+      All  = V190 + V193 + V200 + V220
 
       Type = :property
       NonType = :nonproperty

data/lib/regexp_parser/token.rb

@@ -1,6 +1,16 @@
 class Regexp
 
-  TOKEN_KEYS = [:type, :token, :text, :ts, :te, :level, :set_level].freeze
+  TOKEN_KEYS = [
+    :type,
+    :token,
+    :text,
+    :ts,
+    :te,
+    :level,
+    :set_level,
+    :conditional_level
+  ].freeze
+
   Token = Struct.new(*TOKEN_KEYS) do
     def offset
       [self.ts, self.te]
@@ -10,13 +20,6 @@ class Regexp
       self.te - self.ts
     end
 
-    def to_h
-      hash = {}
-      members.each do |member|
-        hash[member.to_sym] = self.send(member.to_sym)
-      end; hash
-    end
-
     def next(exp = nil)
       if exp
         @next = exp
@@ -32,6 +35,18 @@ class Regexp
         @previous
       end
     end
+
+    if RUBY_VERSION < '2.0.0'
+      def to_h
+        hash = {}
+
+        members.each do |member|
+          hash[member.to_sym] = self.send(member.to_sym)
+        end
+
+        hash
+      end
+    end
   end
 
 end

data/lib/regexp_parser/version.rb

@@ -0,0 +1,5 @@
+class Regexp
+  module Parser
+    VERSION = '0.2.0'
+  end
+end

data/regexp_parser.gemspec

@@ -0,0 +1,35 @@
+$:.unshift File.join(File.dirname(__FILE__), 'lib')
+
+require 'regexp_parser/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = 'regexp_parser'
+  gem.version       = ::Regexp::Parser::VERSION
+
+  gem.summary       = "Scanner, lexer, parser for ruby's regular expressions"
+  gem.description   = 'A library for tokenizing, lexing, and parsing Ruby regular expressions.'
+  gem.homepage      = 'http://github.com/ammar/regexp_parser'
+
+  if gem.respond_to?(:metadata)
+    gem.metadata    = { 'issue_tracker' => 'https://github.com/ammar/regexp_parser/issues' }
+  end
+
+  gem.authors       = ['Ammar Ali']
+  gem.email         = ['ammarabuali@gmail.com']
+
+  gem.license       = 'MIT'
+
+  gem.require_paths = ['lib']
+
+  gem.files         = Dir.glob('{lib,test}/**/*.rb') +
+                      Dir.glob('lib/**/*.rl') +
+                      %w(Gemfile Rakefile LICENSE README.md ChangeLog regexp_parser.gemspec)
+
+  gem.test_files    = Dir.glob('test/**/*.rb')
+
+  gem.rdoc_options  = ["--inline-source", "--charset=UTF-8"]
+
+  gem.platform      = Gem::Platform::RUBY
+
+  gem.required_ruby_version = '>= 1.8.7'
+end

data/test/expression/test_all.rb

@@ -1,7 +1,12 @@
 require File.expand_path("../../helpers", __FILE__)
 
 %w{
-  base to_s clone
+  base to_s to_h clone subexpression free_space tests
+  traverse strfregexp
 }.each do|tc|
   require File.expand_path("../test_#{tc}", __FILE__)
 end
+
+if RUBY_VERSION >= '2.0.0'
+  require File.expand_path("../test_conditionals", __FILE__)
+end

data/test/expression/test_base.rb

@@ -13,6 +13,25 @@ class ExpressionBase < Test::Unit::TestCase
     assert_equal( re.source, re_text )
   end
 
+  def test_expression_level
+    regexp = /^a(b(c(d)))e$/
+    root = RP.parse(regexp)
+
+    %w{^ a (b(c(d))) e $}.each_with_index do |t, i|
+      assert_equal( t, root[i].to_s )
+      assert_equal( 0, root[i].level )
+    end
+
+    assert_equal( 'b', root[2][0].to_s )
+    assert_equal( 1,   root[2][0].level )
+
+    assert_equal( 'c', root[2][1][0].to_s )
+    assert_equal( 2,   root[2][1][0].level )
+
+    assert_equal( 'd', root[2][1][1][0].to_s )
+    assert_equal( 3,   root[2][1][1][0].level )
+  end
+
   def test_expression_terminal?
     root = RP.parse('^a([b]+)c$')
 

data/test/expression/test_conditionals.rb

@@ -0,0 +1,114 @@
+require File.expand_path("../../helpers", __FILE__)
+
+class ExpressionConditionals < Test::Unit::TestCase
+
+  def setup
+    @root = RP.parse(/^a(b(?(1)c|(?(2)d|(?(3)e|f)))g)$/)
+
+    @cond_1 = @root[2][1]
+    @cond_2 = @root[2][1][2][0]
+    @cond_3 = @root[2][1][2][0][2][0]
+  end
+
+  def is_conditional_condition?(exp)
+    exp.is_a?(Conditional::Condition)
+  end
+
+  def is_conditional_branch?(exp)
+    exp.is_a?(Conditional::Branch)
+  end
+
+  def test_expression_conditional_root_level
+    %w{^ a (b(?(1)c|(?(2)d|(?(3)e|f)))g) $}.each_with_index do |t, i|
+      assert_equal( 0, @root[i].conditional_level )
+      assert_equal( t, @root[i].to_s )
+    end
+
+    # First capture group
+    assert_equal( 'b',    @root[2][0].text )
+    assert_equal( 0,      @root[2][0].conditional_level )
+  end
+
+  def test_expression_conditional_level_one
+    condition = @cond_1.condition
+    branch_1  = @cond_1.branches.first
+    branch_2  = @cond_1.branches.last
+
+    # Condition
+    assert_equal( true,   is_conditional_condition?(condition) )
+    assert_equal( '(1)',  condition.text )
+    assert_equal( 1,      condition.conditional_level )
+
+    # Branch 1
+    assert_equal( true,   is_conditional_branch?(branch_1) )
+    assert_equal( 'c',    branch_1.text )
+    assert_equal( 1,      branch_1.conditional_level )
+
+    # Literal
+    assert_equal( 'c',    branch_1.first.text )
+    assert_equal( 1,      branch_1.first.conditional_level )
+
+  end
+
+  def test_expression_conditional_level_two
+    condition = @cond_2.condition
+    branch_1  = @cond_2.branches.first
+    branch_2  = @cond_2.branches.last
+
+    assert_equal( '(?',   @cond_2.text )
+    assert_equal( 1,      @cond_2.conditional_level )
+
+    # Condition
+    assert_equal( true,   is_conditional_condition?(condition) )
+    assert_equal( '(2)',  condition.text )
+    assert_equal( 2,      condition.conditional_level )
+
+    # Branch 1
+    assert_equal( true,   is_conditional_branch?(branch_1) )
+    assert_equal( 'd',    branch_1.text )
+    assert_equal( 2,      branch_1.conditional_level )
+
+    # Literal
+    assert_equal( 'd',    branch_1.first.text )
+    assert_equal( 2,      branch_1.first.conditional_level )
+
+    # Branch 2
+    assert_equal( '(?',   branch_2.first.text )
+    assert_equal( 2,      branch_2.first.conditional_level )
+  end
+
+  def test_expression_conditional_level_three
+    condition = @cond_3.condition
+    branch_1  = @cond_3.branches.first
+    branch_2  = @cond_3.branches.last
+
+    # Condition
+    assert_equal( true,   is_conditional_condition?(condition) )
+    assert_equal( '(3)',  condition.text )
+    assert_equal( 3,      condition.conditional_level )
+
+    # Same as branch 2 in level two
+    assert_equal( '(?',         @cond_3.text )
+    assert_equal( '(?(3)e|f)',  @cond_3.to_s )
+    assert_equal( 2,            @cond_3.conditional_level )
+
+    # Branch 1
+    assert_equal( true,   is_conditional_branch?(branch_1) )
+    assert_equal( 'e',    branch_1.text )
+    assert_equal( 3,      branch_1.conditional_level )
+
+    # Literal
+    assert_equal( 'e',    branch_1.first.text )
+    assert_equal( 3,      branch_1.first.conditional_level )
+
+    # Branch 2
+    assert_equal( true,   is_conditional_branch?(branch_2) )
+    assert_equal( 'f',    branch_2.text )
+    assert_equal( 3,      branch_2.conditional_level )
+
+    # Literal
+    assert_equal( 'f',    branch_2.first.text )
+    assert_equal( 3,      branch_2.first.conditional_level )
+  end
+
+end