ruby_parser 2.0.6 → 2.1.0

data/.autotest

@@ -1,11 +1,12 @@
 # -*- ruby -*-
 
 require 'autotest/restart'
+require 'autotest/isolate'
 require 'autotest/rcov' if ENV['RCOV']
 
 Autotest.add_hook :initialize do |at|
-  at.extra_files << "../../ParseTree/dev/test/pt_testcase.rb"
-  at.libs << ":../../ParseTree/dev/test:../../sexp_processor/dev/lib"
+  at.extra_files << "../../sexp_processor/dev/lib/pt_testcase.rb"
+  at.libs << ":../../sexp_processor/dev/lib"
   at.add_exception 'unit'
   at.add_exception 'coverage'
   at.add_exception 'coverage.info'
@@ -13,7 +14,6 @@ Autotest.add_hook :initialize do |at|
 
   at.libs << ':../../minitest/dev/lib'
   at.testlib = "minitest/autorun"
-  at.unit_diff = "unit_diff -u -b"
 
   at.add_mapping(/^lib\/.*\.y$/) do |f, _|
     at.files_matching %r%^test/.*#{File.basename(f, '.y').gsub '_', '_?'}.rb$%

data/History.txt

@@ -1,3 +1,15 @@
+=== 2.1.0 / 2011-08-15
+
+* 2 minor enhancements:
+
+  * Added new accessor canonicalize_conditions to toggle conditional canonicalization (on by default). (confused)
+  * Awesome cleanup: Replaced call to append_block by block_append. (Confusion)
+
+* 2 bug fixes:
+
+  * Fixed handling last line of =begin/=end. (raybaxter)
+  * Fixed source line numbers after heredocs. (jbarreneche)
+
 === 2.0.6 / 2011-02-18
 
 * 1 minor enhancement:

data/README.txt

@@ -1,6 +1,7 @@
 = ruby_parser
 
-* http://parsetree.rubyforge.org/
+home :: https://github.com/seattlerb/ruby_parser
+rdoc :: http://parsetree.rubyforge.org/ruby_parser
 
 == DESCRIPTION:
 

data/Rakefile

@@ -5,18 +5,16 @@ require 'hoe'
 
 Hoe.plugin :seattlerb
 Hoe.plugin :racc
+Hoe.plugin :isolate
 
-Hoe.add_include_dirs("../../ParseTree/dev/test",
-                     "../../RubyInline/dev/lib",
-                     "../../sexp_processor/dev/lib")
+Hoe.add_include_dirs "../../sexp_processor/dev/lib"
 
 Hoe.spec 'ruby_parser' do
   developer 'Ryan Davis', 'ryand-ruby@zenspider.com'
 
   self.rubyforge_name = 'parsetree'
 
-  extra_dev_deps << ['ParseTree',      '~> 3.0']
-  extra_deps     << ['sexp_processor', '~> 3.0']
+  dependency 'sexp_processor', '~> 3.0'
 
   self.perforce_ignore << "lib/ruby_parser.rb" if plugin? :perforce
 end

data/lib/ruby_lexer.rb

@@ -199,6 +199,7 @@ class RubyLexer
       # TODO: think about storing off the char range instead
       line = src.string[src.pos, src.matched_size]
       src.string[src.pos, src.matched_size] = "\n"
+      src.extra_lines_added += 1
       src.pos += 1
     else
       line = nil
@@ -727,7 +728,7 @@ class RubyLexer
             # @comments << '=' << src.matched
             @comments << src.matched
 
-            unless src.scan(/.*?\n=end\s*(\n|\z)/m) then
+            unless src.scan(/.*?\n=end( |\t|\f)*[^(\n|\z)]*(\n|\z)/m) then
               @comments.clear
               rb_compile_error("embedded document meets end of file")
             end

data/lib/ruby_parser.rb

@@ -5370,7 +5370,7 @@ def _reduce_460(val, _values, result)
 end
 
 def _reduce_461(val, _values, result)
-                      result = self.append_to_block val[0], val[2]
+                      result = self.block_append val[0], val[2]
                     
     result
 end

data/lib/ruby_parser.y

@@ -1679,7 +1679,7 @@ xstring_contents: none
                     }
                 | f_optarg tCOMMA f_opt
                     {
-                      result = self.append_to_block val[0], val[2]
+                      result = self.block_append val[0], val[2]
                     }
 
     restarg_mark: tSTAR2 | tSTAR

data/lib/ruby_parser_extras.rb

@@ -29,13 +29,20 @@ class RPStringScanner < StringScanner
 #       old_getch
 #     end
 #   end
-
   def current_line # HAHA fuck you (HACK)
     string[0..pos][/\A.*__LINE__/m].split(/\n/).size
   end
 
+  def extra_lines_added
+    @extra_lines_added ||= 0
+  end
+
+  def extra_lines_added= val
+    @extra_lines_added = val
+  end
+
   def lineno
-    string[0...pos].count("\n") + 1
+    string[0...pos].count("\n") + 1 - extra_lines_added
   end
 
   # TODO: once we get rid of these, we can make things like
@@ -48,6 +55,7 @@ class RPStringScanner < StringScanner
 
   def unread_many str # TODO: remove this entirely - we should not need it
     warn({:unread_many => caller[0]}.inspect) if ENV['TALLY']
+    self.extra_lines_added += str.count("\n")
     string[pos, 0] = str
   end
 
@@ -115,20 +123,11 @@ class RPStringScanner < StringScanner
 end
 
 class RubyParser < Racc::Parser
-  VERSION = '2.0.6' unless constants.include? "VERSION" # SIGH
+  VERSION = '2.1.0' unless constants.include? "VERSION" # SIGH
 
   attr_accessor :lexer, :in_def, :in_single, :file
   attr_reader :env, :comments
 
-  def append_to_block head, tail # FIX: wtf is this?!? switch to block_append
-    return head if tail.nil?
-    return tail if head.nil?
-
-    head = s(:block, head) unless head.node_type == :block
-    head << tail
-    head
-  end
-
   def arg_add(node1, node2) # TODO: nuke
     return s(:arglist, node2) unless node1
 
@@ -217,9 +216,9 @@ class RubyParser < Racc::Parser
     return result
   end
 
-  def block_append(head, tail, strip_tail_block=false)
-    return head unless tail
-    return tail unless head
+  def block_append(head, tail)
+    return head if tail.nil?
+    return tail if head.nil?
 
     case head[0]
     when :lit, :str then
@@ -229,16 +228,10 @@ class RubyParser < Racc::Parser
     line = [head.line, tail.line].compact.min
 
     head = remove_begin(head)
-    head = s(:block, head) unless head[0] == :block
-
-    if strip_tail_block and Sexp === tail and tail[0] == :block then
-      head.push(*tail.values)
-    else
-      head << tail
-    end
+    head = s(:block, head) unless head.node_type == :block
 
     head.line = line
-    head
+    head << tail
   end
 
   def cond node
@@ -335,13 +328,26 @@ class RubyParser < Racc::Parser
     raise "identifier #{id.inspect} is not valid"
   end
 
-  def initialize
-    super
+  ##
+  # Canonicalize conditionals. Eg:
+  #
+  #   not x ? a : b
+  #
+  # becomes:
+  #
+  #   x ? b : a
+
+  attr_accessor :canonicalize_conditions
+
+  def initialize(options = {})
+    super()
     self.lexer = RubyLexer.new
     self.lexer.parser = self
     @env = Environment.new
     @comments = []
 
+    @canonicalize_conditions = true
+
     self.reset
   end
 
@@ -534,7 +540,7 @@ class RubyParser < Racc::Parser
   def new_if c, t, f
     l = [c.line, t && t.line, f && f.line].compact.min
     c = cond c
-    c, t, f = c.last, f, t if c[0] == :not
+    c, t, f = c.last, f, t if c[0] == :not and canonicalize_conditions
     s(:if, c, t, f).line(l)
   end
 
@@ -658,26 +664,31 @@ class RubyParser < Racc::Parser
     end
   end
 
-  def new_until block, expr, pre
-    expr = (expr.first == :not ? expr.last : s(:not, expr)).line(expr.line)
-    new_while block, expr, pre
-  end
-
-  def new_while block, expr, pre
+  def new_until_or_while type, block, expr, pre
+    other = type == :until ? :while : :until
     line = [block && block.line, expr.line].compact.min
     block, pre = block.last, false if block && block[0] == :begin
 
     expr = cond expr
-    result = if expr.first == :not then
-               s(:until, expr.last, block, pre)
+
+    result = unless expr.first == :not and canonicalize_conditions then
+               s(type,  expr,      block, pre)
              else
-               s(:while, expr, block, pre)
+               s(other, expr.last, block, pre)
              end
 
     result.line = line
     result
   end
 
+  def new_until block, expr, pre
+    new_until_or_while :until, block, expr, pre
+  end
+
+  def new_while block, expr, pre
+    new_until_or_while :while, block, expr, pre
+  end
+
   def new_xstring str
     if str then
       case str[0]

data/test/test_ruby_lexer.rb

@@ -255,6 +255,11 @@ class TestRubyLexer < MiniTest::Unit::TestCase
     assert_equal "=begin blah\nblah\n=end\n", @lex.comments
   end
 
+  def test_yylex_comment_end_space_and_text
+    util_lex_token("=begin blah\nblah\n=end blab\n")
+    assert_equal "=begin blah\nblah\n=end blab\n", @lex.comments
+  end
+
   def test_yylex_comment_eos
     util_lex_token("# comment")
   end

data/test/test_ruby_parser.rb

@@ -33,8 +33,6 @@ class TestRubyParser < RubyParserTestCase
   def setup
     super
 
-    # puts self.name
-
     @processor = RubyParser.new
   end
 
@@ -178,7 +176,7 @@ class TestRubyParser < RubyParserTestCase
   def test_bug_comment_eq_begin
     rb = "\n\n#\n=begin\nblah\n=end\n\n"
     pt = nil
-    exp = rb[2..-1]
+    exp = rb.strip + "\n"
 
     assert_equal pt, @processor.parse(rb)
     assert_equal exp, @processor.lexer.comments
@@ -422,13 +420,13 @@ class TestRubyParser < RubyParserTestCase
     "case_nested_inner_no_expr"          => 2,
     "case_no_expr"                       => 2,
     "case_splat"                         => 2,
-    "dstr_heredoc_expand"                => 2,
-    "dstr_heredoc_windoze_sucks"         => 2,
-    "dstr_heredoc_yet_again"             => 2,
-    "str_heredoc"                        => 2,
-    "str_heredoc_call"                   => 2,
-    "str_heredoc_empty"                  => 2,
-    "str_heredoc_indent"                 => 2,
+    "dstr_heredoc_expand"                => 1,
+    "dstr_heredoc_windoze_sucks"         => 1,
+    "dstr_heredoc_yet_again"             => 1,
+    "str_heredoc"                        => 1,
+    "str_heredoc_call"                   => 1,
+    "str_heredoc_empty"                  => 1,
+    "str_heredoc_indent"                 => 1,
     "structure_unused_literal_wwtt"      => 3, # yes, 3... odd test
     "undef_block_1"                      => 2,
     "undef_block_2"                      => 2,
@@ -441,7 +439,7 @@ class TestRubyParser < RubyParserTestCase
     assert_equal expected, @result.line, "should have proper line number"
   end
 
-  def test_position_info
+  def test_position_info_block
     rb = "a = 42\np a"
     pt = s(:block,
            s(:lasgn, :a, s(:lit, 42)),
@@ -465,7 +463,7 @@ class TestRubyParser < RubyParserTestCase
     assert_same result.file, result.call.file
   end
 
-  def test_position_info2
+  def test_position_info_defn
     rb = "def x(y)\n  p(y)\n  y *= 2\n  return y;\nend" # TODO: remove () & ;
     pt = s(:defn, :x, s(:args, :y),
            s(:scope,
@@ -486,4 +484,83 @@ class TestRubyParser < RubyParserTestCase
     assert_equal 3, body.lasgn.line,  "lasgn should have line number"
     assert_equal 4, body.return.line, "return should have line number"
   end
+
+  def test_position_info_heredoc
+    rb = <<-CODE
+      string = <<-HEREDOC
+        very long string
+      HEREDOC
+      puts string
+    CODE
+
+    result = @processor.parse rb
+    assert_equal 1, result.lasgn.line
+    assert_equal 4, result.call.line
+  end
+
+  def test_parse_if_not_canonical
+    rb = "if not var.nil? then 'foo' else 'bar'\nend"
+    pt = s(:if,
+           s(:call, s(:call, nil, :var, s(:arglist)), :nil?, s(:arglist)),
+           s(:str, "bar"),
+           s(:str, "foo"))
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_parse_if_not_noncanonical
+    rb = "if not var.nil? then 'foo' else 'bar'\nend"
+    pt = s(:if,
+           s(:not,
+             s(:call, s(:call, nil, :var, s(:arglist)), :nil?, s(:arglist))),
+           s(:str, "foo"),
+           s(:str, "bar"))
+
+    @processor.canonicalize_conditions = false
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_parse_while_not_canonical
+    rb = "while not var.nil?\n  'foo'\nend"
+    pt = s(:until,
+           s(:call, s(:call, nil, :var, s(:arglist)), :nil?, s(:arglist)),
+           s(:str, "foo"), true)
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_parse_while_not_noncanonical
+    rb = "while not var.nil?\n  'foo'\nend"
+    pt = s(:while,
+           s(:not,
+             s(:call, s(:call, nil, :var, s(:arglist)), :nil?, s(:arglist))),
+           s(:str, "foo"), true)
+
+    @processor.canonicalize_conditions = false
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_parse_until_not_canonical
+    rb = "until not var.nil?\n  'foo'\nend"
+
+    pt = s(:while,
+           s(:call, s(:call, nil, :var, s(:arglist)), :nil?, s(:arglist)),
+           s(:str, "foo"), true)
+
+    assert_equal pt, @processor.parse(rb)
+  end
+
+  def test_parse_until_not_noncanonical
+    rb = "until not var.nil?\n  'foo'\nend"
+    pt = s(:until,
+           s(:not,
+             s(:call, s(:call, nil, :var, s(:arglist)), :nil?, s(:arglist))),
+           s(:str, "foo"), true)
+
+    @processor.canonicalize_conditions = false
+
+    assert_equal pt, @processor.parse(rb)
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: ruby_parser
 version: !ruby/object:Gem::Version 
-  hash: 3
+  hash: 11
   prerelease: 
   segments: 
   - 2
+  - 1
   - 0
-  - 6
-  version: 2.0.6
+  version: 2.1.0
 platform: ruby
 authors: 
 - Ryan Davis
@@ -36,8 +36,7 @@ cert_chain:
   FBHgymkyj/AOSqKRIpXPhjC6
   -----END CERTIFICATE-----
 
-date: 2011-02-18 00:00:00 -08:00
-default_executable: 
+date: 2011-08-15 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: sexp_processor
@@ -62,12 +61,12 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 9
+        hash: 11
         segments: 
         - 1
         - 4
-        - 7
-        version: 1.4.7
+        - 6
+        version: 1.4.6
   type: :development
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
@@ -76,47 +75,30 @@ dependencies:
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
         hash: 11
         segments: 
         - 2
-        - 0
-        - 2
-        version: 2.0.2
+        - 4
+        version: "2.4"
   type: :development
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: ParseTree
+  name: hoe
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 7
-        segments: 
-        - 3
-        - 0
-        version: "3.0"
-  type: :development
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: hoe
-  prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 41
+        hash: 21
         segments: 
         - 2
-        - 9
-        - 1
-        version: 2.9.1
+        - 11
+        version: "2.11"
   type: :development
-  version_requirements: *id005
+  version_requirements: *id004
 description: |-
   ruby_parser (RP) is a ruby parser written in pure ruby (utilizing
   racc--which does by default use a C extension). RP's output is
@@ -169,8 +151,7 @@ files:
 - test/test_ruby_parser.rb
 - test/test_ruby_parser_extras.rb
 - .gemtest
-has_rdoc: true
-homepage: http://parsetree.rubyforge.org/
+homepage: https://github.com/seattlerb/ruby_parser
 licenses: []
 
 post_install_message: 
@@ -200,7 +181,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: parsetree
-rubygems_version: 1.4.2
+rubygems_version: 1.8.5
 signing_key: 
 specification_version: 3
 summary: ruby_parser (RP) is a ruby parser written in pure ruby (utilizing racc--which does by default use a C extension)