ruby_parser 2.0.5 → 2.1.0

data/.autotest

@@ -1,11 +1,12 @@
 # -*- ruby -*-
 
 require 'autotest/restart'
+require 'autotest/isolate'
 require 'autotest/rcov' if ENV['RCOV']
 
 Autotest.add_hook :initialize do |at|
-  at.extra_files << "../../ParseTree/dev/test/pt_testcase.rb"
-  at.libs << ":../../ParseTree/dev/test:../../sexp_processor/dev/lib"
+  at.extra_files << "../../sexp_processor/dev/lib/pt_testcase.rb"
+  at.libs << ":../../sexp_processor/dev/lib"
   at.add_exception 'unit'
   at.add_exception 'coverage'
   at.add_exception 'coverage.info'
@@ -13,7 +14,6 @@ Autotest.add_hook :initialize do |at|
 
   at.libs << ':../../minitest/dev/lib'
   at.testlib = "minitest/autorun"
-  at.unit_diff = "unit_diff -u -b"
 
   at.add_mapping(/^lib\/.*\.y$/) do |f, _|
     at.files_matching %r%^test/.*#{File.basename(f, '.y').gsub '_', '_?'}.rb$%

data/History.txt

@@ -1,3 +1,25 @@
+=== 2.1.0 / 2011-08-15
+
+* 2 minor enhancements:
+
+  * Added new accessor canonicalize_conditions to toggle conditional canonicalization (on by default). (confused)
+  * Awesome cleanup: Replaced call to append_block by block_append. (Confusion)
+
+* 2 bug fixes:
+
+  * Fixed handling last line of =begin/=end. (raybaxter)
+  * Fixed source line numbers after heredocs. (jbarreneche)
+
+=== 2.0.6 / 2011-02-18
+
+* 1 minor enhancement:
+
+  * Switched to hoe's racc plugin to clean up rakefile and builds
+
+* 1 bug fix:
+
+  * Fixed empty =begin/end.
+
 === 2.0.5 / 2010-09-01
 
 * 1 minor enhancement:

data/Manifest.txt

@@ -7,6 +7,7 @@ bin/ruby_parse
 lib/gauntlet_rubyparser.rb
 lib/ruby_lexer.rb
 lib/ruby_parser.y
+lib/ruby_parser.rb
 lib/ruby_parser_extras.rb
 test/test_ruby_lexer.rb
 test/test_ruby_parser.rb

data/README.txt

@@ -1,6 +1,7 @@
 = ruby_parser
 
-* http://parsetree.rubyforge.org/
+home :: https://github.com/seattlerb/ruby_parser
+rdoc :: http://parsetree.rubyforge.org/ruby_parser
 
 == DESCRIPTION:
 

data/Rakefile

@@ -4,41 +4,19 @@ require 'rubygems'
 require 'hoe'
 
 Hoe.plugin :seattlerb
+Hoe.plugin :racc
+Hoe.plugin :isolate
 
-Hoe.add_include_dirs("../../ParseTree/dev/test",
-                     "../../RubyInline/dev/lib",
-                     "../../sexp_processor/dev/lib")
+Hoe.add_include_dirs "../../sexp_processor/dev/lib"
 
-hoe = Hoe.spec 'ruby_parser' do
+Hoe.spec 'ruby_parser' do
   developer 'Ryan Davis', 'ryand-ruby@zenspider.com'
 
   self.rubyforge_name = 'parsetree'
 
-  extra_dev_deps << ['ParseTree',      '~> 3.0']
-  extra_deps     << ['sexp_processor', '~> 3.0']
-end
-
-hoe.spec.files += ['lib/ruby_parser.rb'] # jim.... cmon man
-
-[:default, :multi, :test].each do |t|
-  task t => :parser
-end
-
-path = "pkg/ruby_parser-#{hoe.version}"
-task path => :parser do
-  Dir.chdir path do
-    sh "rake parser"
-  end
-end
-
-desc "build the parser"
-task :parser => ["lib/ruby_parser.rb"]
+  dependency 'sexp_processor', '~> 3.0'
 
-rule '.rb' => '.y' do |t|
-  # -v = verbose
-  # -t = debugging parser ~4% reduction in speed -- keep for now
-  # -l = no-line-convert
-  sh "racc -v -t -l -o #{t.name} #{t.source}"
+  self.perforce_ignore << "lib/ruby_parser.rb" if plugin? :perforce
 end
 
 task :clean do
@@ -46,7 +24,6 @@ task :clean do
         Dir["**/*.diff"] +
         Dir["coverage.info"] +
         Dir["coverage"] +
-        Dir["lib/ruby_parser.rb"] +
         Dir["lib/*.output"])
 end
 

data/lib/ruby_lexer.rb

@@ -199,6 +199,7 @@ class RubyLexer
       # TODO: think about storing off the char range instead
       line = src.string[src.pos, src.matched_size]
       src.string[src.pos, src.matched_size] = "\n"
+      src.extra_lines_added += 1
       src.pos += 1
     else
       line = nil
@@ -231,7 +232,7 @@ class RubyLexer
   end
 
   def lex_state= o
-    raise "wtf?" unless Symbol === o
+    raise "wtf\?" unless Symbol === o
     @lex_state = o
   end
 
@@ -655,7 +656,7 @@ class RubyLexer
           self.lineno = nil
           c = src.matched
           if c == '#' then
-            src.unread c # ok
+            src.pos -= 1
 
             while src.scan(/\s*#.*(\n+|\z)/) do
               @comments << src.matched.gsub(/^ +#/, '#').gsub(/^ +$/, '')
@@ -723,11 +724,11 @@ class RubyLexer
             self.fix_arg_lex_state
             tok = self.yacc_value = src.matched
             return TOKENS[tok]
-          elsif src.was_begin_of_line and src.scan(/\=begin(?=\s)/) then
+          elsif src.scan(/\=begin(?=\s)/) then
             # @comments << '=' << src.matched
             @comments << src.matched
 
-            unless src.scan(/.*?\n=end\s*(\n|\z)/m) then
+            unless src.scan(/.*?\n=end( |\t|\f)*[^(\n|\z)]*(\n|\z)/m) then
               @comments.clear
               rb_compile_error("embedded document meets end of file")
             end

data/lib/ruby_parser.rb

@@ -1,6 +1,6 @@
 #
 # DO NOT MODIFY!!!!
-# This file is automatically generated by Racc 1.4.7
+# This file is automatically generated by Racc 1.4.6
 # from Racc grammer file "".
 #
 
@@ -773,7 +773,7 @@ clist = [
 '299,300,276,279,282,285,288,290,292,294,296,,301,278,,281,284,287,,',
 ',,,,,,,,,,,295,,298,,277,280,283,286,289,291,293,,297,,,,,,,,,,,,,,',
 ',281' ]
-        racc_action_table = arr = ::Array.new(23872, nil)
+        racc_action_table = arr = Array.new(23872, nil)
         idx = 0
         clist.each do |str|
           str.split(',', -1).each do |i|
@@ -1617,7 +1617,7 @@ clist = [
 '747,747,747,747,747,747,747,,747,640,640,640,640,640,640,640,640,640',
 '640,640,,640,640,,747,640,640,,,,,,,,,,,,,,640,,640,,640,640,640,640',
 '640,640,640,,640,,,,,,,,,,,,,,,,640' ]
-        racc_action_check = arr = ::Array.new(23872, nil)
+        racc_action_check = arr = Array.new(23872, nil)
         idx = 0
         clist.each do |str|
           str.split(',', -1).each do |i|
@@ -1881,7 +1881,7 @@ clist = [
 ',,,,,,,,,,,,,,,,,,820,,,,823,,,,,,,,,,,38,,849,,832,,,835,,,,,38,,,',
 ',38,,,,,,,,,,,38,,,38,,,,,,,,,,,,38,661,38,,,,,,,584,,,589,,,,,,,,,',
 ',,,,,,,,,,,,,,880,,,,,882,,,885,,,,,,,,,,,,,,,,,,589,,,,,,,,,,,,,,900' ]
-        racc_goto_table = arr = ::Array.new(2455, nil)
+        racc_goto_table = arr = Array.new(2455, nil)
         idx = 0
         clist.each do |str|
           str.split(',', -1).each do |i|
@@ -1952,7 +1952,7 @@ clist = [
 '45,,,,,45,,,,,,,,,,,,,,,,,,,24,,,,24,,,,,,,,,,,45,,45,,24,,,24,,,,,45',
 ',,,,45,,,,,,,,,,,45,,,45,,,,,,,,,,,,45,24,45,,,,,,,24,,,24,,,,,,,,,',
 ',,,,,,,,,,,,,,24,,,,,24,,,24,,,,,,,,,,,,,,,,,,24,,,,,,,,,,,,,,24' ]
-        racc_goto_check = arr = ::Array.new(2455, nil)
+        racc_goto_check = arr = Array.new(2455, nil)
         idx = 0
         clist.each do |str|
           str.split(',', -1).each do |i|
@@ -2941,7 +2941,7 @@ Racc_token_to_s_table = [
   "@39",
   "assoc" ]
 
-Racc_debug_parser = true
+Racc_debug_parser = false
 
 ##### State transition tables end #####
 
@@ -5370,7 +5370,7 @@ def _reduce_460(val, _values, result)
 end
 
 def _reduce_461(val, _values, result)
-                      result = self.append_to_block val[0], val[2]
+                      result = self.block_append val[0], val[2]
                     
     result
 end

data/lib/ruby_parser.y

@@ -1679,7 +1679,7 @@ xstring_contents: none
                     }
                 | f_optarg tCOMMA f_opt
                     {
-                      result = self.append_to_block val[0], val[2]
+                      result = self.block_append val[0], val[2]
                     }
 
     restarg_mark: tSTAR2 | tSTAR

data/lib/ruby_parser_extras.rb

@@ -29,13 +29,20 @@ class RPStringScanner < StringScanner
 #       old_getch
 #     end
 #   end
-
   def current_line # HAHA fuck you (HACK)
     string[0..pos][/\A.*__LINE__/m].split(/\n/).size
   end
 
+  def extra_lines_added
+    @extra_lines_added ||= 0
+  end
+
+  def extra_lines_added= val
+    @extra_lines_added = val
+  end
+
   def lineno
-    string[0...pos].count("\n") + 1
+    string[0...pos].count("\n") + 1 - extra_lines_added
   end
 
   # TODO: once we get rid of these, we can make things like
@@ -48,6 +55,7 @@ class RPStringScanner < StringScanner
 
   def unread_many str # TODO: remove this entirely - we should not need it
     warn({:unread_many => caller[0]}.inspect) if ENV['TALLY']
+    self.extra_lines_added += str.count("\n")
     string[pos, 0] = str
   end
 
@@ -115,20 +123,11 @@ class RPStringScanner < StringScanner
 end
 
 class RubyParser < Racc::Parser
-  VERSION = '2.0.5' unless constants.include? "VERSION" # SIGH
+  VERSION = '2.1.0' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
 
-  def append_to_block head, tail # FIX: wtf is this?!? switch to block_append
-    return head if tail.nil?
-    return tail if head.nil?
-
-    head = s(:block, head) unless head.node_type == :block
-    head << tail
-    head
-  end
-
   def arg_add(node1, node2) # TODO: nuke
     return s(:arglist, node2) unless node1
 
@@ -217,9 +216,9 @@ class RubyParser < Racc::Parser
     return result
   end
 
-  def block_append(head, tail, strip_tail_block=false)
-    return head unless tail
-    return tail unless head
+  def block_append(head, tail)
+    return head if tail.nil?
+    return tail if head.nil?
 
     case head[0]
     when :lit, :str then
@@ -229,16 +228,10 @@ class RubyParser < Racc::Parser
     line = [head.line, tail.line].compact.min
 
     head = remove_begin(head)
-    head = s(:block, head) unless head[0] == :block
-
-    if strip_tail_block and Sexp === tail and tail[0] == :block then
-      head.push(*tail.values)
-    else
-      head << tail
-    end
+    head = s(:block, head) unless head.node_type == :block
 
     head.line = line
-    head
+    head << tail
   end
 
   def cond node
@@ -335,13 +328,26 @@ class RubyParser < Racc::Parser
     raise "identifier #{id.inspect} is not valid"
   end
 
-  def initialize
-    super
+  ##
+  # Canonicalize conditionals. Eg:
+  #
+  #   not x ? a : b
+  #
+  # becomes:
+  #
+  #   x ? b : a
+
+  attr_accessor :canonicalize_conditions
+
+  def initialize(options = {})
+    super()
     self.lexer = RubyLexer.new
     self.lexer.parser = self
     @env = Environment.new
     @comments = []
 
+    @canonicalize_conditions = true
+
     self.reset
   end
 
@@ -534,7 +540,7 @@ class RubyParser < Racc::Parser
   def new_if c, t, f
     l = [c.line, t && t.line, f && f.line].compact.min
     c = cond c
-    c, t, f = c.last, f, t if c[0] == :not
+    c, t, f = c.last, f, t if c[0] == :not and canonicalize_conditions
     s(:if, c, t, f).line(l)
   end
 
@@ -658,26 +664,31 @@ class RubyParser < Racc::Parser
     end
   end
 
-  def new_until block, expr, pre
-    expr = (expr.first == :not ? expr.last : s(:not, expr)).line(expr.line)
-    new_while block, expr, pre
-  end
-
-  def new_while block, expr, pre
+  def new_until_or_while type, block, expr, pre
+    other = type == :until ? :while : :until
     line = [block && block.line, expr.line].compact.min
     block, pre = block.last, false if block && block[0] == :begin
 
     expr = cond expr
-    result = if expr.first == :not then
-               s(:until, expr.last, block, pre)
+
+    result = unless expr.first == :not and canonicalize_conditions then
+               s(type,  expr,      block, pre)
              else
-               s(:while, expr, block, pre)
+               s(other, expr.last, block, pre)
              end
 
     result.line = line
     result
   end
 
+  def new_until block, expr, pre
+    new_until_or_while :until, block, expr, pre
+  end
+
+  def new_while block, expr, pre
+    new_until_or_while :while, block, expr, pre
+  end
+
   def new_xstring str
     if str then
       case str[0]

data/test/test_ruby_lexer.rb

@@ -255,6 +255,11 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     assert_equal "=begin blah\nblah\n=end\n", @lex.comments
   end
 
+  def test_yylex_comment_end_space_and_text
+    util_lex_token("=begin blah\nblah\n=end blab\n")
+    assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
+  end
+
   def test_yylex_comment_eos
     util_lex_token("# comment")
   end

data/test/test_ruby_parser.rb

@@ -33,8 +33,6 @@ class TestRubyParser < RubyParserTestCase
   def setup
     super
 
-    # puts self.name
-
     @processor = RubyParser.new
   end
 
@@ -175,6 +173,15 @@ class TestRubyParser < RubyParserTestCase
     assert_equal pt, @processor.parse(rb)
   end
 
+  def test_bug_comment_eq_begin
+    rb = "\n\n#\n=begin\nblah\n=end\n\n"
+    pt = nil
+    exp = rb.strip + "\n"
+
+    assert_equal pt, @processor.parse(rb)
+    assert_equal exp, @processor.lexer.comments
+  end
+
   def test_dstr_evstr
     rb = "\"#\{'a'}#\{b}\""
     pt = s(:dstr, "a", s(:evstr, s(:call, nil, :b, s(:arglist))))
@@ -413,13 +420,13 @@ class TestRubyParser < RubyParserTestCase
     "case_nested_inner_no_expr"          => 2,
     "case_no_expr"                       => 2,
     "case_splat"                         => 2,
-    "dstr_heredoc_expand"                => 2,
-    "dstr_heredoc_windoze_sucks"         => 2,
-    "dstr_heredoc_yet_again"             => 2,
-    "str_heredoc"                        => 2,
-    "str_heredoc_call"                   => 2,
-    "str_heredoc_empty"                  => 2,
-    "str_heredoc_indent"                 => 2,
+    "dstr_heredoc_expand"                => 1,
+    "dstr_heredoc_windoze_sucks"         => 1,
+    "dstr_heredoc_yet_again"             => 1,
+    "str_heredoc"                        => 1,
+    "str_heredoc_call"                   => 1,
+    "str_heredoc_empty"                  => 1,
+    "str_heredoc_indent"                 => 1,
     "structure_unused_literal_wwtt"      => 3, # yes, 3... odd test
     "undef_block_1"                      => 2,
     "undef_block_2"                      => 2,
@@ -432,7 +439,7 @@ class TestRubyParser < RubyParserTestCase
     assert_equal expected, @result.line, "should have proper line number"
   end
 
-  def test_position_info
+  def test_position_info_block
     rb = "a = 42\np a"
     pt = s(:block,
            s(:lasgn, :a, s(:lit, 42)),
@@ -456,7 +463,7 @@ class TestRubyParser < RubyParserTestCase
     assert_same result.file, result.call.file
   end
 
-  def test_position_info2
+  def test_position_info_defn
     rb = "def x(y)\n  p(y)\n  y *= 2\n  return y;\nend" # TODO: remove () & ;
     pt = s(:defn, :x, s(:args, :y),
            s(:scope,
@@ -477,4 +484,83 @@ class TestRubyParser < RubyParserTestCase
     assert_equal 3, body.lasgn.line,  "lasgn should have line number"
     assert_equal 4, body.return.line, "return should have line number"
   end
+
+  def test_position_info_heredoc
+    rb = <<-CODE
+      string = <<-HEREDOC
+        very long string
+      HEREDOC
+      puts string
+    CODE
+
+    result = @processor.parse rb
+    assert_equal 1, result.lasgn.line
+    assert_equal 4, result.call.line
+  end
+
+  def test_parse_if_not_canonical
+    rb = "if not var.nil? then 'foo' else 'bar'\nend"
+    pt = s(:if,
+           s(:call, s(:call, nil, :var, s(:arglist)), :nil?, s(:arglist)),
+           s(:str, "bar"),
+           s(:str, "foo"))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_parse_if_not_noncanonical
+    rb = "if not var.nil? then 'foo' else 'bar'\nend"
+    pt = s(:if,
+           s(:not,
+             s(:call, s(:call, nil, :var, s(:arglist)), :nil?, s(:arglist))),
+           s(:str, "foo"),
+           s(:str, "bar"))
+
+    @processor.canonicalize_conditions = false
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_parse_while_not_canonical
+    rb = "while not var.nil?\n  'foo'\nend"
+    pt = s(:until,
+           s(:call, s(:call, nil, :var, s(:arglist)), :nil?, s(:arglist)),
+           s(:str, "foo"), true)
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_parse_while_not_noncanonical
+    rb = "while not var.nil?\n  'foo'\nend"
+    pt = s(:while,
+           s(:not,
+             s(:call, s(:call, nil, :var, s(:arglist)), :nil?, s(:arglist))),
+           s(:str, "foo"), true)
+
+    @processor.canonicalize_conditions = false
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_parse_until_not_canonical
+    rb = "until not var.nil?\n  'foo'\nend"
+
+    pt = s(:while,
+           s(:call, s(:call, nil, :var, s(:arglist)), :nil?, s(:arglist)),
+           s(:str, "foo"), true)
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_parse_until_not_noncanonical
+    rb = "until not var.nil?\n  'foo'\nend"
+    pt = s(:until,
+           s(:not,
+             s(:call, s(:call, nil, :var, s(:arglist)), :nil?, s(:arglist))),
+           s(:str, "foo"), true)
+
+    @processor.canonicalize_conditions = false
+
+    assert_equal pt, @processor.parse(rb)
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ruby_parser
 version: !ruby/object:Gem::Version 
-  hash: 5
-  prerelease: false
+  hash: 11
+  prerelease: 
   segments: 
   - 2
+  - 1
   - 0
-  - 5
-  version: 2.0.5
+  version: 2.1.0
 platform: ruby
 authors: 
 - Ryan Davis
@@ -36,8 +36,7 @@ cert_chain:
   FBHgymkyj/AOSqKRIpXPhjC6
   -----END CERTIFICATE-----
 
-date: 2010-09-01 00:00:00 -07:00
-default_executable: 
+date: 2011-08-15 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: sexp_processor
@@ -55,19 +54,19 @@ dependencies:
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: rubyforge
+  name: racc
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        hash: 7
+        hash: 11
         segments: 
-        - 2
-        - 0
+        - 1
         - 4
-        version: 2.0.4
+        - 6
+        version: 1.4.6
   type: :development
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
@@ -76,47 +75,30 @@ dependencies:
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        hash: 9
+        hash: 11
         segments: 
-        - 1
-        - 7
-        - 1
-        version: 1.7.1
+        - 2
+        - 4
+        version: "2.4"
   type: :development
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: ParseTree
+  name: hoe
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 7
+        hash: 21
         segments: 
-        - 3
-        - 0
-        version: "3.0"
-  type: :development
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: hoe
-  prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 19
-        segments: 
-        - 2
-        - 6
         - 2
-        version: 2.6.2
+        - 11
+        version: "2.11"
   type: :development
-  version_requirements: *id005
+  version_requirements: *id004
 description: |-
   ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
   racc--which does by default use a C extension). RP's output is
@@ -163,13 +145,13 @@ files:
 - lib/gauntlet_rubyparser.rb
 - lib/ruby_lexer.rb
 - lib/ruby_parser.y
+- lib/ruby_parser.rb
 - lib/ruby_parser_extras.rb
 - test/test_ruby_lexer.rb
 - test/test_ruby_parser.rb
 - test/test_ruby_parser_extras.rb
-- lib/ruby_parser.rb
-has_rdoc: true
-homepage: http://parsetree.rubyforge.org/
+- .gemtest
+homepage: https://github.com/seattlerb/ruby_parser
 licenses: []
 
 post_install_message: 
@@ -199,7 +181,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: parsetree
-rubygems_version: 1.3.7
+rubygems_version: 1.8.5
 signing_key: 
 specification_version: 3
 summary: ruby_parser (RP) is a ruby parser written in pure ruby (utilizing racc--which does by default use a C extension)