parser 2.0.0.beta3 → 2.0.0.beta4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d3c90abfdc392d6c6d0c73d73138f073428117e6
-  data.tar.gz: d0c947aa1ada143e47b3558ae215a4f245f2ccff
+  metadata.gz: b376a16dbd765d49118c18432b8134011d732f09
+  data.tar.gz: 12cad6ee3c5f47b267b4b93d442e3e5ae80be478
 SHA512:
-  metadata.gz: 76e1c1063a7ab852b19cc2b6a6eb1121cd7f9aa13ec40b3712dc568dfd24b8ac7b72610386d4a33932e22331eac5fca2575efdf302b3daf9cfe398edac1d13be
-  data.tar.gz: 4647419022b86a8d8a53f9a126d0c47fa6d9f23d5b2ed2b5ce60277b6803a0202f7701e24c8905ce41d072954aa60408198fd98087584f9fdf44f688b176420b
+  metadata.gz: 5be76612d68a0692c0c3d2691bb83816b35aeda66d496cfebbb04c65a5dd0cacf8be7c6d739fbc0cbefff05f5f20b270fd3bd426c602f1ad32bcbf216102e6be
+  data.tar.gz: b1fa6025f4b3311870f5403a7fab57cd9da3e851f89ebff271a5853e783e7c4790a153440c753f681054edf0f94ca2821e5f793427cbc2ed7ed1bef0d4a2dbc9

data/CHANGELOG.md

@@ -1,6 +1,13 @@
 Changelog
 =========
 
+v2.0.0.beta4 (2013-06-05)
+-------------------------
+
+Bugs fixed:
+ * lexer.rl: fix heredoc parsing with CRLF line endings (closes #61). (Peter Zotov)
+ * lexer.rl: fix premature ending of heredoc "\<\<D\nABCDEF\nD" (fixes #59). (Peter Zotov)
+
 v2.0.0.beta3 (2013-05-29)
 -------------------------
 

data/README.md

@@ -114,6 +114,10 @@ $ ruby-parse -E -e "2+2"
   [slop]: http://rubygems.org/gems/slop
   [insane-lexer]: http://whitequark.org/blog/2013/04/01/ruby-hacking-guide-ch-11-finite-state-lexer/
 
+## Documentation
+
+Documentation for parser is available online on [rdoc.info](http://rdoc.info/github/whitequark/parser).
+
 ## Contributors
 
  * Peter Zotov ([whitequark][])

data/lib/parser/lexer.rl

@@ -673,7 +673,7 @@ class Parser::Lexer
   # @heredoc_s to literal.saved_herebody_s, and after an interpolation (possibly
   # containing another heredocs) is closed, the previous value is restored.
 
-  e_heredoc_nl = c_nl $ {
+  e_heredoc_nl = c_nl % {
     # After every heredoc was parsed, @herebody_s contains the
     # position of next token after all heredocs.
     if @herebody_s
@@ -683,10 +683,10 @@ class Parser::Lexer
   };
 
   action extend_string {
-    if literal.nest_and_try_closing tok, @ts, @te
+    if !literal.heredoc? && literal.nest_and_try_closing(tok, @ts, @te)
       fnext *pop_literal; fbreak;
     else
-      literal.extend_string tok, @ts, @te
+      literal.extend_string(tok, @ts, @te)
     end
   }
 
@@ -744,8 +744,8 @@ class Parser::Lexer
     if literal.heredoc?
       # Try ending the heredoc with the complete most recently
       # scanned line. @herebody_s always refers to the start of such line.
-      if literal.nest_and_try_closing(tok(@herebody_s, @te - 1),
-                                      @herebody_s, @te - 1)
+      if literal.nest_and_try_closing(tok(@herebody_s, @ts),
+                                      @herebody_s, @ts)
         # Adjust @herebody_s to point to the next line.
         @herebody_s = @te
 
@@ -1362,11 +1362,11 @@ class Parser::Lexer
       # Heredoc start.
       # <<EOF | <<-END | <<"FOOBAR" | <<-`SMTH`
       '<<' '-'?
-        ( '"' ( c_any - c_nl - '"' )* '"'
-        | "'" ( c_any - c_nl - "'" )* "'"
-        | "`" ( c_any - c_nl - "`" )* "`"
-        | bareword )           % { @heredoc_e     = p }
-        ( c_any - c_nl )* c_nl % { new_herebody_s = p }
+        ( '"' ( c_line - '"' )* '"'
+        | "'" ( c_line - "'" )* "'"
+        | "`" ( c_line - "`" )* "`"
+        | bareword ) % { @heredoc_e     = p }
+        c_line* c_nl % { new_herebody_s = p }
       => {
         tok(@ts, @heredoc_e) =~ /^<<(-?)(["'`]?)(.*)\2$/
 

data/lib/parser/version.rb

@@ -1,3 +1,3 @@
 module Parser
-  VERSION = '2.0.0.beta3'
+  VERSION = '2.0.0.beta4'
 end

data/test/test_lexer.rb

@@ -811,6 +811,16 @@ class TestLexer < Minitest::Test
                    :tNL,             nil)
   end
 
+  def test_heredoc_one_character
+    util_lex_token("a = <<E\nABCDEF\nE\n",
+                   :tIDENTIFIER,     "a",
+                   :tEQL,              "=",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_CONTENT, "ABCDEF\n",
+                   :tSTRING_END,     "E",
+                   :tNL,             nil)
+  end
+
   def test_identifier
     util_lex_token("identifier", :tIDENTIFIER, "identifier")
   end
@@ -2499,6 +2509,14 @@ class TestLexer < Minitest::Test
                    :kEND,            "end")
   end
 
+  def test_bug_heredoc_cr_lf
+    util_lex_token("<<FIN\r\nfoo\r\nFIN\r\n",
+                   :tSTRING_BEG,     "\"",
+                   :tSTRING_CONTENT, "foo\r\n",
+                   :tSTRING_END,     "EOS",
+                   :tNL,             nil)
+  end
+
   def test_bug_eh_symbol_no_newline
     util_lex_token("?\"\nfoo",
                    :tINTEGER,     34,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: parser
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta3
+  version: 2.0.0.beta4
 platform: ruby
 authors:
 - Peter Zotov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-05-28 00:00:00.000000000 Z
+date: 2013-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ast