regexp_parser 0.1.5 → 0.1.6

data/Rakefile

@@ -1,32 +1,48 @@
+require 'rubygems'
+
 require 'rake'
 require 'rake/testtask'
+
+require 'bundler'
 require 'rubygems/package_task'
-require 'yaml'
 
-task :default => [:test]
 
 RAGEL_SOURCE_DIR = File.expand_path '../lib/regexp_parser/scanner', __FILE__
 RAGEL_OUTPUT_DIR = File.expand_path '../lib/regexp_parser', __FILE__
+RAGEL_SOURCE_FILES = %w{scanner} # scanner.rl includes property.rl
+
 
-RAGEL_SOURCE_FILES = %w{scanner}
+Bundler::GemHelper.install_tasks
+
+
+task :default => [:test]
+
+Rake::TestTask.new('test') do |t|
+  t.description = "Run all unit tests under the test directory"
 
-desc "Find and run all unit tests under test/ directory"
-Rake::TestTask.new("test") do |t|
   t.libs << "test"
-  t.test_files = FileList['test/**/test_*.rb']
+  t.test_files = FileList['test/test_all.rb']
 end
 
-task :test
-
 namespace :test do
   %w{scanner lexer parser expression syntax}.each do |component|
     Rake::TestTask.new(component) do |t|
+      t.description = "Run all #{component} unit tests under the test/#{component} directory"
+
       t.libs << "test"
       t.test_files = ["test/#{component}/test_all.rb"]
     end
   end
+
+  Rake::TestTask.new('full' => 'ragel:rb') do |t|
+    t.description = "Regenerate the scanner and run all unit tests under the test directory"
+
+    t.libs << "test"
+    t.test_files = FileList['test/test_all.rb']
+  end
 end
 
+
 namespace :ragel do
   desc "Process the ragel source files and output ruby code"
   task :rb do |t|
@@ -44,43 +60,9 @@ namespace :ragel do
   end
 end
 
-spec = Gem::Specification.new do |gem|
-  gem.name = 'regexp_parser'
-  gem.version = YAML.load(File.read('VERSION.yml')).values.compact.join('.')
-  gem.date = '2014-01-14'
-
-  gem.license = 'MIT'
-  gem.summary = %q{Scanner, lexer, parser for ruby's regular expressions}
-  gem.description = %q{A library for tokenizing, lexing, and parsing Ruby regular expressions.}
-  gem.homepage = %q{http://github.com/ammar/regexp_parser}
-
-  gem.authors = ["Ammar Ali"]
-  gem.email = 'ammarabuali@gmail.com'
-
-  gem.rdoc_options = ["--inline-source", "--charset=UTF-8"]
-  gem.extra_rdoc_files = ["ChangeLog", "LICENSE", "README.rdoc"]
-
-  gem.require_paths = ["lib"]
-
-  gem.files = Dir.glob("{lib,test}/**/*.rb") + Dir.glob("lib/**/*.rl") +
-              %w(VERSION.yml Rakefile LICENSE README.rdoc ChangeLog)
 
-  gem.test_files = Dir.glob("test/**/*.rb")
+# Add ragel task as a prerequisite for building the gem to ensure that the
+# latest scanner code is generated and included in the build.
+desc "Runs ragel:rb before building the gem"
+task :build => ['ragel:rb']
 
-  gem.required_rubygems_version = Gem::Requirement.new(">= 0") if
-    gem.respond_to? :required_rubygems_version=
-end
-
-Gem::PackageTask.new(spec) do |pkg|
-  pkg.need_zip = true
-  pkg.need_tar = true
-end
-
-namespace :gem do
-  desc "Release the gem to rubygems.org"
-  task :release do |t|
-    Rake::Task['ragel:rb'].execute
-    Rake::Task['repackage'].execute
-    #sh "gem push"
-  end
-end

data/VERSION.yml

@@ -1,5 +1,5 @@
 --- 
 :major: 0
 :minor: 1
-:patch: 5
+:patch: 6
 :build: 

data/lib/regexp_parser/parser.rb

@@ -406,8 +406,10 @@ module Regexp::Parser
   end
 
   def self.close_group
-    last_group = @nesting.pop
+    @nesting.pop
+
     @node = @nesting.last
+    @node = @node.last if @node.last and @node.last.is_a?(Alternation)
   end
 
   def self.open_set(token)

data/lib/regexp_parser/scanner.rb

@@ -5,7 +5,7 @@
 
 
 # THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
-# This file was generated from scanner.rl
+# This file was generated from lib/regexp_parser/scanner/scanner.rl
 
 module Regexp::Scanner
   
@@ -1226,7 +1226,7 @@ self._re_scanner_trans_targs = [
 	154, 155, 156, 145, 145, 145, 145, 45, 
 	47, 50, 145, 145, 51, 145, 145, 52, 
 	55, 103, 106, 107, 109, 161, 162, 157, 
-	162, 162, 157, 157, 163, 157, 164, 157, 
+	162, 157, 157, 157, 163, 157, 164, 157, 
 	112, 166, 157, 157, 167, 167, 168, 168, 
 	167, 115, 116, 167, 167, 119, 129, 167, 
 	169, 167, 167, 167, 167, 117, 167, 167, 
@@ -2292,6 +2292,30 @@ te = p+1
 		next
 	end
 
+     end
+		end
+	when 69 then
+# line 242 "/Users/ammar/src/ruby/projects/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
+		begin
+te = p+1
+ begin 
+      case text = text(data, ts, te, 1).first
+      when '\d'; emit(set_type, :type_digit,     text, ts-1, te)
+      when '\D'; emit(set_type, :type_nondigit,  text, ts-1, te)
+      when '\h'; emit(set_type, :type_hex,       text, ts-1, te)
+      when '\H'; emit(set_type, :type_nonhex,    text, ts-1, te)
+      when '\s'; emit(set_type, :type_space,     text, ts-1, te)
+      when '\S'; emit(set_type, :type_nonspace,  text, ts-1, te)
+      when '\w'; emit(set_type, :type_word,      text, ts-1, te)
+      when '\W'; emit(set_type, :type_nonword,   text, ts-1, te)
+      end
+      	begin
+		top -= 1
+		cs = stack[top]
+		_goto_level = _again
+		next
+	end
+
      end
 		end
 	when 75 then
@@ -2419,27 +2443,6 @@ p = p - 1; begin
 		next
 	end
 end
-	when 15 then
-	begin begin p = ((te))-1; end
-
-      case text = text(data, ts, te, 1).first
-      when '\d'; emit(set_type, :type_digit,     text, ts-1, te)
-      when '\D'; emit(set_type, :type_nondigit,  text, ts-1, te)
-      when '\h'; emit(set_type, :type_hex,       text, ts-1, te)
-      when '\H'; emit(set_type, :type_nonhex,    text, ts-1, te)
-      when '\s'; emit(set_type, :type_space,     text, ts-1, te)
-      when '\S'; emit(set_type, :type_nonspace,  text, ts-1, te)
-      when '\w'; emit(set_type, :type_word,      text, ts-1, te)
-      when '\W'; emit(set_type, :type_nonword,   text, ts-1, te)
-      end
-      	begin
-		top -= 1
-		cs = stack[top]
-		_goto_level = _again
-		next
-	end
-
-    end
 	when 18 then
 	begin begin p = ((te))-1; end
 
@@ -3428,14 +3431,6 @@ te = p+1
 # line 136 "/Users/ammar/src/ruby/projects/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
 		begin
  group_depth += 1; in_group = true 		end
-	when 69 then
-# line 1 "NONE"
-		begin
-te = p+1
-		end
-# line 242 "/Users/ammar/src/ruby/projects/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
-		begin
-act = 15;		end
 	when 68 then
 # line 1 "NONE"
 		begin
@@ -3476,7 +3471,7 @@ te = p+1
 # line 672 "/Users/ammar/src/ruby/projects/regexp_parser/lib/regexp_parser/scanner/scanner.rl"
 		begin
 act = 53;		end
-# line 3480 "/Users/ammar/src/ruby/projects/regexp_parser/lib/regexp_parser/scanner.rb"
+# line 3475 "/Users/ammar/src/ruby/projects/regexp_parser/lib/regexp_parser/scanner.rb"
 	end
 	end
 	end
@@ -3494,7 +3489,7 @@ ts = nil;		end
 		begin
 act = 0
 		end
-# line 3498 "/Users/ammar/src/ruby/projects/regexp_parser/lib/regexp_parser/scanner.rb"
+# line 3493 "/Users/ammar/src/ruby/projects/regexp_parser/lib/regexp_parser/scanner.rb"
 	end
 
 	if cs == 0
@@ -3528,7 +3523,7 @@ act = 0
     text = ts ? copy(data, ts-1..-1) : data.pack('c*')
     raise PrematureEndError.new( text )
   		end
-# line 3532 "/Users/ammar/src/ruby/projects/regexp_parser/lib/regexp_parser/scanner.rb"
+# line 3527 "/Users/ammar/src/ruby/projects/regexp_parser/lib/regexp_parser/scanner.rb"
 	  end
 	end
 

data/lib/regexp_parser/scanner/scanner.rl

@@ -239,7 +239,7 @@
       fret;
     };
 
-    char_type {
+    char_type > (escaped_set_alpha, 4) {
       case text = text(data, ts, te, 1).first
       when '\d'; emit(set_type, :type_digit,     text, ts-1, te)
       when '\D'; emit(set_type, :type_nondigit,  text, ts-1, te)
@@ -677,7 +677,7 @@
 }%%
 
 # THIS IS A GENERATED FILE, DO NOT EDIT DIRECTLY
-# This file was generated from scanner.rl
+# This file was generated from lib/regexp_parser/scanner/scanner.rl
 
 module Regexp::Scanner
   %% write data;

data/lib/regexp_parser/syntax.rb

@@ -37,12 +37,15 @@ module Regexp::Syntax
     self.load(name)
 
     case name
+      # Ruby 1.8.x (NOTE: 1.8.6 is no longer a supported runtime,
+      # but its regex features are still recognized.)
       when 'ruby/1.8.6';  syntax = Regexp::Syntax::Ruby::V186.new
       when 'ruby/1.8.7';  syntax = Regexp::Syntax::Ruby::V187.new
 
       # alias for the latest 1.8 implementation
       when 'ruby/1.8';    syntax = Regexp::Syntax::Ruby::V18.new
 
+      # Ruby 1.9.x
       when 'ruby/1.9.1';  syntax = Regexp::Syntax::Ruby::V191.new
       when 'ruby/1.9.2';  syntax = Regexp::Syntax::Ruby::V192.new
       when 'ruby/1.9.3';  syntax = Regexp::Syntax::Ruby::V193.new
@@ -50,11 +53,18 @@ module Regexp::Syntax
       # alias for the latest 1.9 implementation
       when 'ruby/1.9';    syntax = Regexp::Syntax::Ruby::V19.new
 
-      when 'ruby/2.0.0';  syntax = Regexp::Syntax::Ruby::V20.new
-      when 'ruby/2.1.0';  syntax = Regexp::Syntax::Ruby::V21.new
+      # Ruby 2.0.x
+      when 'ruby/2.0.0';  syntax = Regexp::Syntax::Ruby::V200.new
 
-      # aliases for the latest 2.x implementations
+      # aliases for the latest 2.0 implementations
       when 'ruby/2.0';    syntax = Regexp::Syntax::Ruby::V20.new
+
+      # Ruby 2.1.x
+      when 'ruby/2.1.0';  syntax = Regexp::Syntax::Ruby::V210.new
+      when 'ruby/2.1.2';  syntax = Regexp::Syntax::Ruby::V212.new
+      when 'ruby/2.1.3';  syntax = Regexp::Syntax::Ruby::V213.new
+
+      # aliases for the latest 2.1 implementations
       when 'ruby/2.1';    syntax = Regexp::Syntax::Ruby::V21.new
 
       else

data/lib/regexp_parser/syntax/ruby/2.0.0.rb

@@ -1,10 +1,10 @@
-require File.expand_path('../1.9.3', __FILE__)
+require File.expand_path('../1.9', __FILE__)
 
 module Regexp::Syntax
   module Ruby
 
     # use the last 1.9 release as the base
-    class V20 < Regexp::Syntax::Ruby::V193
+    class V200 < Regexp::Syntax::Ruby::V19
       def initialize
         super
 

data/lib/regexp_parser/syntax/ruby/2.0.rb

@@ -0,0 +1,8 @@
+require File.expand_path('../2.0.0', __FILE__)
+
+module Regexp::Syntax
+  module Ruby
+    # use the last 2.0 release as the base
+    class V20 < Regexp::Syntax::Ruby::V200; end
+  end
+end

data/lib/regexp_parser/syntax/ruby/2.1.0.rb

@@ -1,9 +1,9 @@
-require File.expand_path('../2.0.0', __FILE__)
+require File.expand_path('../2.0', __FILE__)
 
 module Regexp::Syntax
   module Ruby
 
-    class V21 < Regexp::Syntax::Ruby::V20
+    class V210 < Regexp::Syntax::Ruby::V20
       def initialize
         super
       end

data/lib/regexp_parser/syntax/ruby/2.1.2.rb

@@ -0,0 +1,13 @@
+require File.expand_path('../2.1.0', __FILE__)
+
+module Regexp::Syntax
+  module Ruby
+
+    class V212 < Regexp::Syntax::Ruby::V210
+      def initialize
+        super
+      end
+    end
+
+  end
+end

data/lib/regexp_parser/syntax/ruby/2.1.3.rb

@@ -0,0 +1,13 @@
+require File.expand_path('../2.1.2', __FILE__)
+
+module Regexp::Syntax
+  module Ruby
+
+    class V213 < Regexp::Syntax::Ruby::V212
+      def initialize
+        super
+      end
+    end
+
+  end
+end

data/lib/regexp_parser/syntax/ruby/2.1.rb

@@ -0,0 +1,8 @@
+require File.expand_path('../2.1.3', __FILE__)
+
+module Regexp::Syntax
+  module Ruby
+    # uses the latest 2.1 release
+    class V21 < Regexp::Syntax::Ruby::V213; end
+  end
+end

data/test/parser/test_alternation.rb

@@ -78,4 +78,15 @@ class ParserAlternation < Test::Unit::TestCase
     assert_equal( true,   subalts[1].is_a?(Sequence) )
   end
 
+  def test_parse_alternation_continues_after_nesting
+    root = RP.parse(/a|(b)c/)
+
+    seq = root.expressions[0][1].expressions
+
+    assert_equal( 2,  seq.length )
+
+    assert_equal( true,   seq[0].is_a?(Group::Capture) )
+    assert_equal( true,   seq[1].is_a?(Literal) )
+  end
+
 end

data/test/parser/test_sets.rb

@@ -36,6 +36,13 @@ class TestParserSets < Test::Unit::TestCase
     assert_equal( false, exp.include?(']') )
   end
 
+  def test_parse_chat_type_set_members
+    exp = RP.parse('[\da-z]', :any)[0]
+
+    assert_equal( true,  exp.include?('\d') )
+    assert_equal( true,  exp.include?('a-z') )
+  end
+
   def test_parse_set_collating_sequence
     exp = RP.parse('[a[.span-ll.]h]', :any)[0]
 

data/test/scanner/test_sets.rb

@@ -28,6 +28,7 @@ class ScannerSets < Test::Unit::TestCase
     '[a\-c]'                => [2, :set,    :escape,          '\-',         2, 4],
 
     '[\d]'                  => [1, :set,    :type_digit,      '\d',         1, 3],
+    '[\da-z]'               => [1, :set,    :type_digit,      '\d',         1, 3],
     '[\D]'                  => [1, :set,    :type_nondigit,   '\D',         1, 3],
 
     '[\h]'                  => [1, :set,    :type_hex,        '\h',         1, 3],

data/test/syntax/ruby/test_2.x.rb

@@ -0,0 +1,46 @@
+require File.expand_path("../../../helpers", __FILE__)
+
+class TestSyntaxRuby_V2x < Test::Unit::TestCase
+
+  # Since the only Regular expression change that was introduced in ruby 2.x
+  # (the conditional expressions) is not supportted yet, these tests only
+  # check that the syntax files have the correct inheritance (based off the
+  # last release) and load without error.
+
+  def test_syntax_file_2_0_0
+    syntax = Regexp::Syntax.new 'ruby/2.0.0'
+
+    assert_equal( true, syntax.kind_of?(Regexp::Syntax::Ruby::V19) )
+  end
+
+  def test_syntax_file_2_0_alias
+    syntax = Regexp::Syntax.new 'ruby/2.0'
+
+    assert_equal( true, syntax.kind_of?(Regexp::Syntax::Ruby::V200) )
+  end
+
+  def test_syntax_file_2_1_0
+    syntax = Regexp::Syntax.new 'ruby/2.1.0'
+
+    assert_equal( true, syntax.kind_of?(Regexp::Syntax::Ruby::V20) )
+  end
+
+  def test_syntax_file_2_1_2
+    syntax = Regexp::Syntax.new 'ruby/2.1.2'
+
+    assert_equal( true, syntax.kind_of?(Regexp::Syntax::Ruby::V210) )
+  end
+
+  def test_syntax_file_2_1_3
+    syntax = Regexp::Syntax.new 'ruby/2.1.3'
+
+    assert_equal( true, syntax.kind_of?(Regexp::Syntax::Ruby::V212) )
+  end
+
+  def test_syntax_file_2_1_alias
+    syntax = Regexp::Syntax.new 'ruby/2.1'
+
+    assert_equal( true, syntax.kind_of?(Regexp::Syntax::Ruby::V213) )
+  end
+
+end