ruby_parser 2.0.3 → 2.0.4

data/.autotest

@@ -1,16 +1,18 @@
 # -*- ruby -*-
 
 require 'autotest/restart'
-require 'autotest/rcov'
+require 'autotest/rcov' if ENV['RCOV']
 
 Autotest.add_hook :initialize do |at|
   at.extra_files << "../../ParseTree/dev/test/pt_testcase.rb"
-  at.libs << ":../../ParseTree/dev/lib:../../ParseTree/dev/test:../../sexp_processor/dev/lib"
+  at.libs << ":../../ParseTree/dev/test:../../sexp_processor/dev/lib"
   at.add_exception 'unit'
   at.add_exception 'coverage'
   at.add_exception 'coverage.info'
   at.add_exception '.diff'
 
+  at.libs << ':../../minitest/dev/lib'
+  at.testlib = "minitest/autorun"
   at.unit_diff = "unit_diff -u -b"
 
   at.add_mapping(/^lib\/.*\.y$/) do |f, _|

data/History.txt

@@ -1,3 +1,16 @@
+=== 2.0.4 / 2009-08-18
+
+* 1 minor enhancement:
+
+  * Changed requires around to be more accurate.
+
+* 4 bug fixes:
+
+  * Fixed .autotest for minitest
+  * Fixed emacs escape lexing bug: "\C-\\" (maglev/gemstone)
+  * Fixed octal lexing edgecases. (maglev/gemstone)
+  * Fixed regexp lexing edgecases. (maglev/gemstone)
+
 === 2.0.3 / 2009-06-23
 
 * 4 minor enhancements:

data/Rakefile

@@ -14,8 +14,8 @@ hoe = Hoe.spec 'ruby_parser' do
 
   self.rubyforge_name = 'parsetree'
 
-  extra_dev_deps << 'ParseTree'
-  extra_deps     << ['sexp_processor', '>= 3.0.1']
+  extra_dev_deps << ['ParseTree',      '~> 3.0']
+  extra_deps     << ['sexp_processor', '~> 3.0']
 end
 
 hoe.spec.files += ['lib/ruby_parser.rb'] # jim.... cmon man

data/lib/ruby_lexer.rb

@@ -1,14 +1,10 @@
-$: << File.expand_path("~/Work/p4/zss/src/ParseTree/dev/lib") # for me, not you.
-require 'sexp'
-require 'ruby_parser_extras'
-
 class RubyLexer
   attr_accessor :command_start
   attr_accessor :cmdarg
   attr_accessor :cond
   attr_accessor :nest
 
-  ESC_RE = /\\([0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-.|(C-|c)\?|(C-|c).|[^0-7xMCc])/
+  ESC_RE = /\\([0-7]{1,3}|x[0-9a-fA-F]{1,2}|M-[^\\]|(C-|c)[^\\]|[^0-7xMCc])/
 
   # Additional context surrounding tokens that both the lexer and
   # grammar use.
@@ -248,9 +244,9 @@ class RubyLexer
       int_with_base(2)
     when src.scan(/[+-]?0d[0-9_]+/) then
       int_with_base(10)
-    when src.scan(/[+-]?0o?[0-7_]*[89]/) then
+    when src.scan(/[+-]?0[Oo]?[0-7_]*[89]/) then
       rb_compile_error "Illegal octal digit."
-    when src.scan(/[+-]?0o?[0-7_]+|0o/) then
+    when src.scan(/[+-]?0[Oo]?[0-7_]+|0[Oo]/) then
       int_with_base(8)
     when src.scan(/[+-]?[\d_]+_(e|\.)/) then
       rb_compile_error "Trailing '_' in number."
@@ -411,7 +407,8 @@ class RubyLexer
       src.matched.to_i(8).chr
     when src.scan(/x([0-9a-fA-F]{1,2})/) then # hex constant
       src[1].to_i(16).chr
-    when src.scan(/M-\\/) then
+    when src.check(/M-\\[\\MCc]/) then
+      src.scan(/M-\\/) # eat it
       c = self.read_escape
       c[0] = (c[0].ord | 0x80).chr
       c
@@ -419,12 +416,13 @@ class RubyLexer
       c = src[1]
       c[0] = (c[0].ord | 0x80).chr
       c
-    when src.scan(/C-\\|c\\/) then
+    when src.check(/(C-|c)\\[\\MCc]/) then
+      src.scan(/(C-|c)\\/) # eat it
       c = self.read_escape
       c[0] = (c[0].ord & 0x9f).chr
       c
     when src.scan(/C-\?|c\?/) then
-      0177.chr
+      127.chr
     when src.scan(/(C-|c)(.)/) then
       c = src[2]
       c[0] = (c[0].ord & 0x9f).chr
@@ -586,8 +584,8 @@ class RubyLexer
       "v"    => "\13",
       "\\"   => '\\',
       "\n"   => "",
-      "C-\?" => 0177.chr,
-      "c\?"  => 0177.chr,
+      "C-\?" => 127.chr,
+      "c\?"  => 127.chr,
     }[s]
 
     return r if r

data/lib/ruby_parser.rb

@@ -9,6 +9,7 @@ class RubyParser < Racc::Parser
 
 
 require "ruby_lexer"
+require "ruby_parser_extras"
 
 # Local Variables: **
 # racc-token-length-max:14 **

data/lib/ruby_parser.y

@@ -1783,6 +1783,7 @@ end
 ---- inner
 
 require "ruby_lexer"
+require "ruby_parser_extras"
 
 # Local Variables: **
 # racc-token-length-max:14 **

data/lib/ruby_parser_extras.rb

@@ -115,7 +115,7 @@ class RPStringScanner < StringScanner
 end
 
 class RubyParser < Racc::Parser
-  VERSION = '2.0.3' unless constants.include? "VERSION" # SIGH
+  VERSION = '2.0.4' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
@@ -591,7 +591,7 @@ class RubyParser < Racc::Parser
     options = val[2]
 
     o, k = 0, nil
-    options.split(//).each do |c| # FIX: this has a better home
+    options.split(//).uniq.each do |c| # FIX: this has a better home
       v = {
         'x' => Regexp::EXTENDED,
         'i' => Regexp::IGNORECASE,

data/test/test_ruby_lexer.rb

@@ -1,7 +1,9 @@
 #!/usr/local/bin/ruby
 
-require "minitest/autorun"
-require "ruby_lexer"
+require 'rubygems'
+require 'minitest/autorun'
+require 'ruby_lexer'
+require 'ruby_parser'
 
 class TestRubyLexer < MiniTest::Unit::TestCase
   alias :deny :refute
@@ -830,6 +832,22 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     util_bad_token "01__23"
   end
 
+  def test_yylex_integer_oct_O
+    util_lex_token "0O52", :tINTEGER, 42
+  end
+
+  def test_yylex_integer_oct_O_bad_range
+    util_bad_token "0O8"
+  end
+
+  def test_yylex_integer_oct_O_bad_underscores
+    util_bad_token "0O1__23"
+  end
+
+  def test_yylex_integer_oct_O_not_bad_none
+    util_lex_token "0O ", :tINTEGER, 0
+  end
+
   def test_yylex_integer_oct_o
     util_lex_token "0o52", :tINTEGER, 42
   end
@@ -1280,6 +1298,10 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tREGEXP_END,     "")
   end
 
+  def test_yylex_regexp_escape_hex_bad
+    util_bad_token '/regex\\xzxp/', :tREGEXP_BEG, "/"
+  end
+
   def test_yylex_regexp_escape_hex_one
     util_lex_token('/^[\\xd\\xa]{2}/on',
                    :tREGEXP_BEG,     '/',
@@ -1287,10 +1309,6 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tREGEXP_END,     'on')
   end
 
-  def test_yylex_regexp_escape_hex_bad
-    util_bad_token '/regex\\xzxp/', :tREGEXP_BEG, "/"
-  end
-
   def test_yylex_regexp_escape_oct1
     util_lex_token('/regex\\0xp/',
                    :tREGEXP_BEG,     "/",
@@ -1421,24 +1439,47 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tSTRING, "string")
   end
 
-  def test_yylex_string_double_escape_M
-    util_lex_token('"\\M-g"',
-                   :tSTRING, "\347")
+  def test_yylex_string_double_escape_C
+    util_lex_token('"\\C-a"',
+                   :tSTRING, "\001")
   end
 
-  def test_yylex_string_escape_x_single
-    util_lex_token('"\\x0"',
-                   :tSTRING, "\000")
+  def test_yylex_string_double_escape_C_backslash
+    util_lex_token('"\\C-\\\\"',
+                   :tSTRING_BEG, "\"",
+                   :tSTRING_CONTENT, "\034",
+                   :tSTRING_END, "\"")
   end
 
-  def test_yylex_string_double_escape_chars
-    util_lex_token('"s\\tri\\ng"',
-                   :tSTRING, "s\tri\ng")
+  def test_yylex_string_double_escape_C_escape
+    util_lex_token('"\\C-\\M-a"',
+                   :tSTRING_BEG, "\"",
+                   :tSTRING_CONTENT, "\201",
+                   :tSTRING_END, "\"")
   end
 
-  def test_yylex_string_double_escape_hex
-    util_lex_token('"n = \\x61\\x62\\x63"',
-                   :tSTRING, "n = abc")
+  def test_yylex_string_double_escape_C_question
+    util_lex_token('"\\C-?"',
+                   :tSTRING, "\177")
+  end
+
+  def test_yylex_string_double_escape_M
+    util_lex_token('"\\M-a"',
+                   :tSTRING, "\341")
+  end
+
+  def test_yylex_string_double_escape_M_backslash
+    util_lex_token('"\\M-\\\\"',
+                   :tSTRING_BEG, "\"",
+                   :tSTRING_CONTENT, "\334",
+                   :tSTRING_END, "\"")
+  end
+
+  def test_yylex_string_double_escape_M_escape
+    util_lex_token('"\\M-\\C-a"',
+                   :tSTRING_BEG, "\"",
+                   :tSTRING_CONTENT, "\201",
+                   :tSTRING_END, "\"")
   end
 
   def test_yylex_string_double_escape_bs1
@@ -1451,6 +1492,40 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tSTRING, "a\\a")
   end
 
+  def test_yylex_string_double_escape_c
+    util_lex_token('"\\ca"',
+                   :tSTRING, "\001")
+  end
+
+  def test_yylex_string_double_escape_c_backslash
+    util_lex_token('"\\c\\"',
+                   :tSTRING_BEG, "\"",
+                   :tSTRING_CONTENT, "\034",
+                   :tSTRING_END, "\"")
+  end
+
+  def test_yylex_string_double_escape_c_escape
+    util_lex_token('"\\c\\M-a"',
+                   :tSTRING_BEG, "\"",
+                   :tSTRING_CONTENT, "\201",
+                   :tSTRING_END, "\"")
+  end
+
+  def test_yylex_string_double_escape_c_question
+    util_lex_token('"\\c?"',
+                   :tSTRING, "\177")
+  end
+
+  def test_yylex_string_double_escape_chars
+    util_lex_token('"s\\tri\\ng"',
+                   :tSTRING, "s\tri\ng")
+  end
+
+  def test_yylex_string_double_escape_hex
+    util_lex_token('"n = \\x61\\x62\\x63"',
+                   :tSTRING, "n = abc")
+  end
+
   def test_yylex_string_double_escape_octal
     util_lex_token('"n = \\101\\102\\103"',
                    :tSTRING, "n = ABC")
@@ -1484,6 +1559,11 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tSTRING, "blah # blah")
   end
 
+  def test_yylex_string_escape_x_single
+    util_lex_token('"\\x0"',
+                   :tSTRING, "\000")
+  end
+
   def test_yylex_string_pct_Q
     util_lex_token("%Q[s1 s2]",
                    :tSTRING_BEG,     "%Q[",
@@ -1556,6 +1636,14 @@ class TestRubyLexer < MiniTest::Unit::TestCase
                    :tSTRING_END,     nil)
   end
 
+  def test_yylex_string_pct_w_tab
+    util_lex_token("%w[abc\tdef]",
+                   :tAWORDS_BEG,      "%w[",
+                   :tSTRING_CONTENT, "abc\tdef",
+                   :tSPACE,              nil,
+                   :tSTRING_END,     nil)
+  end
+
   def test_yylex_string_single
     util_lex_token("'string'",
                    :tSTRING, "string")

data/test/test_ruby_parser.rb

@@ -1,5 +1,6 @@
 #!/usr/local/bin/ruby
 
+require 'rubygems'
 require 'minitest/autorun'
 require 'ruby_parser'
 
@@ -364,6 +365,22 @@ class TestRubyParser < RubyParserTestCase
     assert_equal pt, @processor.parse(rb)
   end
 
+  def test_regexp
+    regexps = {
+      "/wtf/" => /wtf/,
+      "/wtf/n" => /wtf/n,
+      "/wtf/m" => /wtf/m,
+      "/wtf/nm" => /wtf/nm,
+      "/wtf/nmnmnmnm" => /wtf/nm,
+    }
+
+    regexps.each do |rb, lit|
+      assert_equal s(:lit, lit), @processor.parse(rb)
+    end
+
+    # TODO: add more including interpolation etc
+  end
+
   def test_str_pct_Q_nested
     rb = "%Q[before [#\{nest}] after]"
     pt = s(:dstr, "before [", s(:evstr, s(:call, nil, :nest, s(:arglist))), s(:str, "] after"))

data/test/test_ruby_parser_extras.rb

@@ -1,3 +1,4 @@
+require 'rubygems'
 require 'minitest/autorun'
 require 'ruby_parser_extras'
 

metadata

@@ -1,36 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: ruby_parser
 version: !ruby/object:Gem::Version 
-  version: 2.0.3
+  version: 2.0.4
 platform: ruby
 authors: 
 - Ryan Davis
 autorequire: 
 bindir: bin
-cert_chain: 
-- |
-  -----BEGIN CERTIFICATE-----
-  MIIDPjCCAiagAwIBAgIBADANBgkqhkiG9w0BAQUFADBFMRMwEQYDVQQDDApyeWFu
-  ZC1ydWJ5MRkwFwYKCZImiZPyLGQBGRYJemVuc3BpZGVyMRMwEQYKCZImiZPyLGQB
-  GRYDY29tMB4XDTA5MDMwNjE4NTMxNVoXDTEwMDMwNjE4NTMxNVowRTETMBEGA1UE
-  AwwKcnlhbmQtcnVieTEZMBcGCgmSJomT8ixkARkWCXplbnNwaWRlcjETMBEGCgmS
-  JomT8ixkARkWA2NvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALda
-  b9DCgK+627gPJkB6XfjZ1itoOQvpqH1EXScSaba9/S2VF22VYQbXU1xQXL/WzCkx
-  taCPaLmfYIaFcHHCSY4hYDJijRQkLxPeB3xbOfzfLoBDbjvx5JxgJxUjmGa7xhcT
-  oOvjtt5P8+GSK9zLzxQP0gVLS/D0FmoE44XuDr3iQkVS2ujU5zZL84mMNqNB1znh
-  GiadM9GHRaDiaxuX0cIUBj19T01mVE2iymf9I6bEsiayK/n6QujtyCbTWsAS9Rqt
-  qhtV7HJxNKuPj/JFH0D2cswvzznE/a5FOYO68g+YCuFi5L8wZuuM8zzdwjrWHqSV
-  gBEfoTEGr7Zii72cx+sCAwEAAaM5MDcwCQYDVR0TBAIwADALBgNVHQ8EBAMCBLAw
-  HQYDVR0OBBYEFEfFe9md/r/tj/Wmwpy+MI8d9k/hMA0GCSqGSIb3DQEBBQUAA4IB
-  AQAY59gYvDxqSqgC92nAP9P8dnGgfZgLxP237xS6XxFGJSghdz/nI6pusfCWKM8m
-  vzjjH2wUMSSf3tNudQ3rCGLf2epkcU13/rguI88wO6MrE0wi4ZqLQX+eZQFskJb/
-  w6x9W1ur8eR01s397LSMexySDBrJOh34cm2AlfKr/jokKCTwcM0OvVZnAutaovC0
-  l1SVZ0ecg88bsWHA0Yhh7NFxK1utWoIhtB6AFC/+trM0FQEB/jZkIS8SaNzn96Rl
-  n0sZEf77FLf5peR8TP/PtmIg7Cyqz23sLM4mCOoTGIy5OcZ8TdyiyINUHtb5ej/T
-  FBHgymkyj/AOSqKRIpXPhjC6
-  -----END CERTIFICATE-----
+cert_chain: []
 
-date: 2009-06-23 00:00:00 -07:00
+date: 2009-08-18 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -39,9 +18,9 @@ dependencies:
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: 3.0.1
+        version: "3.0"
     version: 
 - !ruby/object:Gem::Dependency 
   name: ParseTree
@@ -49,9 +28,9 @@ dependencies:
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        version: "0"
+        version: "3.0"
     version: 
 - !ruby/object:Gem::Dependency 
   name: hoe
@@ -61,7 +40,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 2.3.0
+        version: 2.3.3
     version: 
 description: |-
   ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
@@ -139,7 +118,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: parsetree
-rubygems_version: 1.3.4
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
 summary: ruby_parser (RP) is a ruby parser written in pure ruby (utilizing racc--which does by default use a C extension)

data.tar.gz.sig

@@ -1,2 +0,0 @@
-�FW�s"a�zM���h���/'��y������"���u��[6֮��
-��oE�y±#Q�+Tm���f��}́�,m��|��l���r�1��Wo�S�R%F�b�yD��x#8xÄC��x��w��ɛ�L�ڀ��9��g��J����q�Dm�������>D��_���