parser 2.1.0.pre2 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ada647b43495037cf17844d58832f74d63348d88
-  data.tar.gz: 4c4a750b4e696fb5f31f8b6c46718f2d71731651
+  metadata.gz: 6c8816d86f0b6658877e3cf52a953145a3cfb6a8
+  data.tar.gz: 2190740a21e07f44df7171d0f6128cf215db2d9e
 SHA512:
-  metadata.gz: 69ae3a6286559fa7942af81b631cc1389cb0075f5c683cc460a46f1d3a29a1527f174604dd220c687c4b660e1e4bb409b49ce401f8113d4b5be5481486a3fe12
-  data.tar.gz: 94b9c811b9d55d468ecc7de739228fa4c6622eb46bbfb7bdf3b4ef637f566fd2804e459354059b92ae6fb96fa0d2b2d9ffe9980bbf2a35236ac1d86687da985e
+  metadata.gz: 75057aa78a15de4a464146b0d979f6c039e9ca8407cb140b293168b23d1d87829da79762402683cf8909e696e6d9b85b216e09e004d9eff568cdec819797abcb
+  data.tar.gz: 4508a8d6b23ecb1bbe786c1c3bdea5621ab4ef45b83eff998f2a27726c7fb13d8c6a58d4bf0492742bc45e62e3f4c11a5c10baf8cc5f63d44c42246d0e2db0fc

data/.travis.yml

@@ -4,7 +4,7 @@ rvm:
  - 1.9.2
  - 1.9.3
  - 2.0.0
- - rbx-18mode
- - rbx-19mode
  - jruby-18mode
  - jruby-19mode
+ - rbx
+cache: bundler

data/CHANGELOG.md

@@ -1,18 +1,16 @@
 Changelog
 =========
 
-v2.1.0.pre2 (2013-11-25)
-------------------------
+v2.1.0 (2013-12-25)
+-------------------
 
 API modifications:
  * Parser::Diagnostic: expose reason symbolically (closes #115, #116). (Ian MacLeod)
-
-vir (2013-11-13)
-----------------
-
-API modifications:
  * lexer.rl: coerce literals to UTF-8 in ASCII-encoded files if they contain \uXXXX (Peter Zotov)
 
+Bugs fixed:
+ * builders/default: represent heredocs with dedicated map (fixes #100). (Peter Zotov)
+
 v2.1.0.pre1 (2013-11-12)
 ------------------------
 

data/Gemfile

@@ -2,3 +2,8 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in parser.gemspec
 gemspec
+
+platforms :rbx do
+  gem 'rubysl', '~> 2.0'
+  gem 'rubinius', '~> 2.0'
+end

data/doc/AST_FORMAT.md

@@ -76,6 +76,18 @@ Format:
  ~~~~~~~~~~~~~~ expression
 ~~~
 
+#### Here document
+
+Format:
+
+~~~
+(str "foo\nbar\n")
+'<<HERE␊foo␊bar␊HERE'
+ ~~~~~~ expression
+        ~~~~~~~~ heredoc_body
+                ~~~~ heredoc_end
+~~~
+
 ### Symbol
 
 #### Plain
@@ -106,6 +118,8 @@ Format:
 
 ### Execute-string
 
+#### Plain
+
 Format:
 
 ~~~
@@ -115,6 +129,18 @@ Format:
  ~~~~~~~~~~~ expression
 ~~~
 
+#### Here document
+
+Format:
+
+~~~
+(xstr (str "foo\nbar\n"))
+"<<`HERE`␊foo␊bar␊HERE"
+ ~~~~~~~~ expression
+          ~~~~~~~~ heredoc_body
+                  ~~~~ heredoc_end
+~~~
+
 ### Regexp
 
 #### Options

data/lib/parser.rb

@@ -45,6 +45,7 @@ module Parser
     require 'parser/source/map/ternary'
     require 'parser/source/map/for'
     require 'parser/source/map/rescue_body'
+    require 'parser/source/map/heredoc'
   end
 
   require 'parser/syntax_error'

data/lib/parser/builders/default.rb

@@ -109,11 +109,11 @@ module Parser
           parts.first
         else
           n(:str, parts.first.children,
-            collection_map(begin_t, parts, end_t))
+            string_map(begin_t, parts, end_t))
         end
       else
         n(:dstr, [ *parts ],
-          collection_map(begin_t, parts, end_t))
+          string_map(begin_t, parts, end_t))
       end
     end
 
@@ -157,7 +157,7 @@ module Parser
 
     def xstring_compose(begin_t, parts, end_t)
       n(:xstr, [ *parts ],
-        collection_map(begin_t, parts, end_t))
+        string_map(begin_t, parts, end_t))
     end
 
     # Regular expressions
@@ -1092,6 +1092,20 @@ module Parser
       Source::Map::Collection.new(loc(begin_t), loc(end_t), expr_l)
     end
 
+    def string_map(begin_t, parts, end_t)
+      if begin_t && value(begin_t).start_with?('<<')
+        if parts.any?
+          expr_l = join_exprs(parts.first, parts.last)
+        else
+          expr_l = loc(end_t).begin
+        end
+
+        Source::Map::Heredoc.new(loc(begin_t), expr_l, loc(end_t))
+      else
+        collection_map(begin_t, parts, end_t)
+      end
+    end
+
     def regexp_map(begin_t, end_t, options_e)
       Source::Map::Collection.new(loc(begin_t), loc(end_t),
                                   loc(begin_t).join(options_e.loc.expression))

data/lib/parser/lexer.rl

@@ -81,6 +81,12 @@ class Parser::Lexer
   %% write data nofinal;
   # %
 
+  ESCAPES = {
+    'a' => "\a", 'b'  => "\b", 'e'  => "\e", 'f' => "\f",
+    'n' => "\n", 'r'  => "\r", 's'  => "\s", 't' => "\t",
+    'v' => "\v", '\\' => "\\"
+  }
+
   attr_reader   :source_buffer
   attr_reader   :encoding
 
@@ -635,11 +641,8 @@ class Parser::Lexer
   }
 
   action unescape_char {
-    @escape = {
-      'a' => "\a", 'b'  => "\b", 'e'  => "\e", 'f' => "\f",
-      'n' => "\n", 'r'  => "\r", 's'  => "\s", 't' => "\t",
-      'v' => "\v", '\\' => "\\"
-    }.fetch(@source[p - 1].chr, @source[p - 1].chr)
+    char = @source[p - 1].chr
+    @escape = ESCAPES.fetch(char, char)
   }
 
   action invalid_complex_escape {
@@ -1591,7 +1594,7 @@ class Parser::Lexer
         tok(@ts, @heredoc_e) =~ /^<<(-?)(["'`]?)(.*)\2$/
 
         indent    = !$1.empty?
-        type      =  $2.empty? ? '"' : $2
+        type      =  '<<' + ($2.empty? ? '"' : $2)
         delimiter =  $3
 
         fnext *push_literal(type, delimiter, @ts, @heredoc_e, indent);

data/lib/parser/lexer/literal.rb

@@ -8,27 +8,30 @@ module Parser
 
     TYPES = {
     # type      start token     interpolate?
-      "'"  => [ :tSTRING_BEG,   false ],
-      '%q' => [ :tSTRING_BEG,   false ],
-      '"'  => [ :tSTRING_BEG,   true  ],
-      '%'  => [ :tSTRING_BEG,   true  ],
-      '%Q' => [ :tSTRING_BEG,   true  ],
-
-      '%w' => [ :tQWORDS_BEG,   false ],
-      '%W' => [ :tWORDS_BEG,    true  ],
-
-      '%i' => [ :tQSYMBOLS_BEG, false ],
-      '%I' => [ :tSYMBOLS_BEG,  true  ],
-
-      ":'" => [ :tSYMBEG,       false ],
-      '%s' => [ :tSYMBEG,       false ],
-      ':"' => [ :tSYMBEG,       true  ],
-
-      '/'  => [ :tREGEXP_BEG,   true  ],
-      '%r' => [ :tREGEXP_BEG,   true  ],
-
-      '%x' => [ :tXSTRING_BEG,  true  ],
-      '`'  => [ :tXSTRING_BEG,  true  ],
+      "'"   => [ :tSTRING_BEG,   false ],
+      "<<'" => [ :tSTRING_BEG,   false ],
+      '%q'  => [ :tSTRING_BEG,   false ],
+      '"'   => [ :tSTRING_BEG,   true  ],
+      '<<"' => [ :tSTRING_BEG,   true  ],
+      '%'   => [ :tSTRING_BEG,   true  ],
+      '%Q'  => [ :tSTRING_BEG,   true  ],
+
+      '%w'  => [ :tQWORDS_BEG,   false ],
+      '%W'  => [ :tWORDS_BEG,    true  ],
+
+      '%i'  => [ :tQSYMBOLS_BEG, false ],
+      '%I'  => [ :tSYMBOLS_BEG,  true  ],
+
+      ":'"  => [ :tSYMBEG,       false ],
+      '%s'  => [ :tSYMBEG,       false ],
+      ':"'  => [ :tSYMBEG,       true  ],
+
+      '/'   => [ :tREGEXP_BEG,   true  ],
+      '%r'  => [ :tREGEXP_BEG,   true  ],
+
+      '%x'  => [ :tXSTRING_BEG,  true  ],
+      '`'   => [ :tXSTRING_BEG,  true  ],
+      '<<`' => [ :tXSTRING_BEG,  true  ],
     }
 
     attr_reader   :heredoc_e, :str_s

data/lib/parser/source/map.rb

@@ -10,6 +10,10 @@ module Parser
     #  * other ranges (`begin`, `end`, `operator`, ...): node-specific ranges
     #    pointing to various interesting tokens corresponding to the node.
     #
+    # Note that the {Map::Heredoc} map is the only one whose `expression` does
+    # not include other ranges. It only covers the heredoc marker (`<<HERE`),
+    # not the here document itself.
+    #
     # All ranges except `expression` are defined by {Map} subclasses.
     #
     # Ranges (except `expression`) can be `nil` if the corresponding token is

data/lib/parser/source/map/heredoc.rb

@@ -0,0 +1,17 @@
+module Parser
+  module Source
+
+    class Map::Heredoc < Map
+      attr_reader :heredoc_body
+      attr_reader :heredoc_end
+
+      def initialize(begin_l, body_l, end_l)
+        @heredoc_body = body_l
+        @heredoc_end  = end_l
+
+        super(begin_l)
+      end
+    end
+
+  end
+end

data/lib/parser/version.rb

@@ -1,3 +1,3 @@
 module Parser
-  VERSION = '2.1.0.pre2'
+  VERSION = '2.1.0'
 end

data/test/test_lexer.rb

@@ -745,8 +745,8 @@ class TestLexer < Minitest::Test
   def test_heredoc_backtick
     assert_scanned("a = <<`EOF`\n  blah blah\nEOF\n",
                    :tIDENTIFIER,     "a",
-                   :tEQL,              "=",
-                   :tXSTRING_BEG,    "`",
+                   :tEQL,            "=",
+                   :tXSTRING_BEG,    "<<`",
                    :tSTRING_CONTENT, "  blah blah\n",
                    :tSTRING_END,     "EOF",
                    :tNL,             nil)
@@ -755,8 +755,8 @@ class TestLexer < Minitest::Test
   def test_heredoc_double
     assert_scanned("a = <<\"EOF\"\n  blah blah\nEOF\n",
                    :tIDENTIFIER,     "a",
-                   :tEQL,              "=",
-                   :tSTRING_BEG,     "\"",
+                   :tEQL,            "=",
+                   :tSTRING_BEG,     "<<\"",
                    :tSTRING_CONTENT, "  blah blah\n",
                    :tSTRING_END,     "EOF",
                    :tNL,             nil)
@@ -765,8 +765,8 @@ class TestLexer < Minitest::Test
   def test_heredoc_double_dash
     assert_scanned("a = <<-\"EOF\"\n  blah blah\n  EOF\n",
                    :tIDENTIFIER,     "a",
-                   :tEQL,              "=",
-                   :tSTRING_BEG,     "\"",
+                   :tEQL,            "=",
+                   :tSTRING_BEG,     "<<\"",
                    :tSTRING_CONTENT, "  blah blah\n",
                    :tSTRING_END,     "EOF",
                    :tNL,             nil)
@@ -775,22 +775,22 @@ class TestLexer < Minitest::Test
   def test_heredoc_double_eos
     refute_scanned("a = <<\"EOF\"\nblah",
                    :tIDENTIFIER,     "a",
-                   :tEQL,              "=",
-                   :tSTRING_BEG,     "\"")
+                   :tEQL,            "=",
+                   :tSTRING_BEG,     "<<\"")
   end
 
   def test_heredoc_double_eos_nl
     refute_scanned("a = <<\"EOF\"\nblah\n",
                    :tIDENTIFIER,     "a",
-                   :tEQL,              "=",
-                   :tSTRING_BEG,     "\"")
+                   :tEQL,            "=",
+                   :tSTRING_BEG,     "<<\"")
   end
 
   def test_heredoc_double_interp
     assert_scanned("a = <<\"EOF\"\n#x a \#@a b \#$b c \#{3} \nEOF\n",
                    :tIDENTIFIER,     "a",
-                   :tEQL,              "=",
-                   :tSTRING_BEG,     "\"",
+                   :tEQL,            "=",
+                   :tSTRING_BEG,     "<<\"",
                    :tSTRING_CONTENT, "#x a ",
                    :tSTRING_DVAR,    nil,
                    :tIVAR,           "@a",
@@ -808,7 +808,7 @@ class TestLexer < Minitest::Test
 
   def test_heredoc_empty
     assert_scanned("<<\"\"\n\#{x}\nblah2\n\n",
-                   :tSTRING_BEG,     "\"",
+                   :tSTRING_BEG,     "<<\"",
                    :tSTRING_DBEG,    "\#{",
                    :tIDENTIFIER,     "x",
                    :tRCURLY,         "}",
@@ -821,8 +821,8 @@ class TestLexer < Minitest::Test
   def test_heredoc_none
     assert_scanned("a = <<EOF\nblah\nblah\nEOF",
                    :tIDENTIFIER,     "a",
-                   :tEQL,              "=",
-                   :tSTRING_BEG,     "\"",
+                   :tEQL,            "=",
+                   :tSTRING_BEG,     "<<\"",
                    :tSTRING_CONTENT, "blah\n",
                    :tSTRING_CONTENT, "blah\n",
                    :tSTRING_END,     "EOF",
@@ -832,8 +832,8 @@ class TestLexer < Minitest::Test
   def test_heredoc_none_dash
     assert_scanned("a = <<-EOF\nblah\nblah\n  EOF",
                    :tIDENTIFIER,     "a",
-                   :tEQL,              "=",
-                   :tSTRING_BEG,     "\"",
+                   :tEQL,            "=",
+                   :tSTRING_BEG,     "<<\"",
                    :tSTRING_CONTENT, "blah\n",
                    :tSTRING_CONTENT, "blah\n",
                    :tSTRING_END,     "EOF",
@@ -843,8 +843,8 @@ class TestLexer < Minitest::Test
   def test_heredoc_single
     assert_scanned("a = <<'EOF'\n  blah blah\nEOF\n",
                    :tIDENTIFIER,     "a",
-                   :tEQL,              "=",
-                   :tSTRING_BEG,     "'",
+                   :tEQL,            "=",
+                   :tSTRING_BEG,     "<<'",
                    :tSTRING_CONTENT, "  blah blah\n",
                    :tSTRING_END,     "EOF",
                    :tNL,             nil)
@@ -860,8 +860,8 @@ class TestLexer < Minitest::Test
   def test_heredoc_single_dash
     assert_scanned("a = <<-'EOF'\n  blah blah\n  EOF\n",
                    :tIDENTIFIER,     "a",
-                   :tEQL,              "=",
-                   :tSTRING_BEG,     "'",
+                   :tEQL,            "=",
+                   :tSTRING_BEG,     "<<'",
                    :tSTRING_CONTENT, "  blah blah\n",
                    :tSTRING_END,     "EOF",
                    :tNL,             nil)
@@ -870,8 +870,8 @@ class TestLexer < Minitest::Test
   def test_heredoc_one_character
     assert_scanned("a = <<E\nABCDEF\nE\n",
                    :tIDENTIFIER,     "a",
-                   :tEQL,              "=",
-                   :tSTRING_BEG,     "\"",
+                   :tEQL,            "=",
+                   :tSTRING_BEG,     "<<\"",
                    :tSTRING_CONTENT, "ABCDEF\n",
                    :tSTRING_END,     "E",
                    :tNL,             nil)
@@ -880,8 +880,8 @@ class TestLexer < Minitest::Test
   def test_heredoc_cr
     assert_scanned("a = <<E\r\r\nABCDEF\r\r\nE\r\r\r\n",
                    :tIDENTIFIER,     "a",
-                   :tEQL,              "=",
-                   :tSTRING_BEG,     "\"",
+                   :tEQL,            "=",
+                   :tSTRING_BEG,     "<<\"",
                    :tSTRING_CONTENT, "ABCDEF\r\n",
                    :tSTRING_END,     "E",
                    :tNL,             nil)
@@ -2495,14 +2495,14 @@ class TestLexer < Minitest::Test
   def test_whitespace_cr
     setup_lexer(20)
     assert_scanned("<<E\nfoo\nE\rO",
-                   :tSTRING_BEG,     '"',
+                   :tSTRING_BEG,     '<<"',
                    :tSTRING_CONTENT, "foo\n",
                    :tSTRING_END,     'E',
                    :tNL,             nil)
 
     setup_lexer(21)
     refute_scanned("<<E\nfoo\nE\rO",
-                   :tSTRING_BEG,     '"',
+                   :tSTRING_BEG,     '<<"',
                    :tSTRING_CONTENT, "foo\n")
   end
 
@@ -2581,7 +2581,7 @@ class TestLexer < Minitest::Test
 
   def test_bug_expr_beg_heredoc
     assert_scanned("<<EOL % [\nfoo\nEOL\n]",
-                   :tSTRING_BEG,      '"',
+                   :tSTRING_BEG,      '<<"',
                    :tSTRING_CONTENT,  "foo\n",
                    :tSTRING_END,      'EOL',
                    :tPERCENT,         '%',
@@ -2638,9 +2638,9 @@ class TestLexer < Minitest::Test
 
     @lex.state = :expr_arg
     assert_scanned(" <<EOS\nEOS",
-                   :tSTRING_BEG,     "\"",
-                   :tSTRING_END,     "EOS",
-                   :tNL,             nil)
+                   :tSTRING_BEG, "<<\"",
+                   :tSTRING_END, "EOS",
+                   :tNL,         nil)
   end
 
   def test_bug_expr_arg_slash
@@ -2682,7 +2682,7 @@ class TestLexer < Minitest::Test
   def test_bug_heredoc_continuation
     @lex.state = :expr_arg
     assert_scanned(" <<EOS\nEOS\nend",
-                   :tSTRING_BEG,     "\"",
+                   :tSTRING_BEG,     "<<\"",
                    :tSTRING_END,     "EOS",
                    :tNL,             nil,
                    :kEND,            "end")
@@ -2690,7 +2690,7 @@ class TestLexer < Minitest::Test
 
   def test_bug_heredoc_cr_lf
     assert_scanned("<<FIN\r\nfoo\r\nFIN\r\n",
-                   :tSTRING_BEG,     "\"",
+                   :tSTRING_BEG,     "<<\"",
                    :tSTRING_CONTENT, "foo\n",
                    :tSTRING_END,     "FIN",
                    :tNL,             nil)
@@ -2735,7 +2735,7 @@ class TestLexer < Minitest::Test
 
   def test_bug_heredoc_backspace_nl
     assert_scanned(" <<'XXX'\nf \\\nXXX\n",
-                   :tSTRING_BEG,     "'",
+                   :tSTRING_BEG,     "<<'",
                    :tSTRING_CONTENT, "f \\\n",
                    :tSTRING_END,     "XXX",
                    :tNL,             nil)
@@ -2743,7 +2743,7 @@ class TestLexer < Minitest::Test
 
   def test_bug_heredoc_lshft
     assert_scanned("<<RULES << CLEANINGS\nRULES",
-                   :tSTRING_BEG, '"',
+                   :tSTRING_BEG, '<<"',
                    :tSTRING_END, 'RULES',
                    :tLSHFT,      '<<',
                    :tCONSTANT,   'CLEANINGS')
@@ -2871,7 +2871,7 @@ class TestLexer < Minitest::Test
 
   def test_bug_interleaved_heredoc
     assert_scanned(%Q{<<w; "\nfoo\nw\n"},
-                   :tSTRING_BEG,     '"',
+                   :tSTRING_BEG,     '<<"',
                    :tSTRING_CONTENT, "foo\n",
                    :tSTRING_END,     'w',
                    :tSEMI,           ';',
@@ -2881,7 +2881,7 @@ class TestLexer < Minitest::Test
 
     @lex.state = :expr_beg
     assert_scanned(%Q{<<w; %w[\nfoo\nw\n1]},
-                   :tSTRING_BEG,     '"',
+                   :tSTRING_BEG,     '<<"',
                    :tSTRING_CONTENT, "foo\n",
                    :tSTRING_END,     'w',
                    :tSEMI,           ';',
@@ -2892,7 +2892,7 @@ class TestLexer < Minitest::Test
 
                    @lex.state = :expr_beg
     assert_scanned(%Q{<<w; "\#{\nfoo\nw\n}"},
-                   :tSTRING_BEG,     '"',
+                   :tSTRING_BEG,     '<<"',
                    :tSTRING_CONTENT, "foo\n",
                    :tSTRING_END,     'w',
                    :tSEMI,           ';',

data/test/test_parser.rb

@@ -237,6 +237,29 @@ class TestParser < Minitest::Test
       %w(1.8))
   end
 
+  def test_heredoc
+    assert_parses(
+      s(:dstr, s(:str, "foo\n"), s(:str, "bar\n")),
+      %Q{<<HERE!foo!bar!HERE}.gsub('!', "\n"),
+      %q{~~~~~~ expression
+        |       ~~~~~~~~ heredoc_body
+        |               ~~~~ heredoc_end})
+
+    assert_parses(
+      s(:dstr, s(:str, "foo\n"), s(:str, "bar\n")),
+      %Q{<<'HERE'!foo!bar!HERE}.gsub('!', "\n"),
+      %q{~~~~~~~~ expression
+        |         ~~~~~~~~ heredoc_body
+        |                 ~~~~ heredoc_end})
+
+    assert_parses(
+      s(:xstr, s(:str, "foo\n"), s(:str, "bar\n")),
+      %Q{<<`HERE`!foo!bar!HERE}.gsub('!', "\n"),
+      %q{~~~~~~~~ expression
+        |         ~~~~~~~~ heredoc_body
+        |                 ~~~~ heredoc_end})
+  end
+
   # Symbols
 
   def test_symbol_plain

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parser
 version: !ruby/object:Gem::Version
-  version: 2.1.0.pre2
+  version: 2.1.0
 platform: ruby
 authors:
 - Peter Zotov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-11-25 00:00:00.000000000 Z
+date: 2013-12-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ast
@@ -286,6 +286,7 @@ files:
 - lib/parser/source/map/constant.rb
 - lib/parser/source/map/definition.rb
 - lib/parser/source/map/for.rb
+- lib/parser/source/map/heredoc.rb
 - lib/parser/source/map/keyword.rb
 - lib/parser/source/map/operator.rb
 - lib/parser/source/map/rescue_body.rb
@@ -337,12 +338,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>'
+  - - '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.0
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: A Ruby parser written in pure Ruby.