yap-shell-parser 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 56c11c5e48960b86e262bf7b4c6dddf59cbbce51
-  data.tar.gz: 0120fd57b88b3ec458a5a88f9d6187e37ca33190
+  metadata.gz: 59dc1f16d41d54f7c14d7ff68abc1ad404d7bd39
+  data.tar.gz: 95d1cfdfbe7ad7e35b6cc2e4be77a3cba9326a26
 SHA512:
-  metadata.gz: 00370bf66d7d030a3aee98a670d5697b63f1a6ee2be3717db6f20968cf4ba7453c4f1ab2c6fc5bd864e7dfc9a265fb79462dd787c411dc24b389ccecadac9c71
-  data.tar.gz: 7bd7d734dabbdeafad2d892d23a5d2387218c7505be61b0431d1c639ac8c911fb4c5a97cae05653d69a5ba1e673fa3f62d2ac4347d0818698cbb094d3a979fd4
+  metadata.gz: 0f324a8808805a0f4d64c0f63248040639e4dd06c48e6e818341fc6d1a895eee6330a71f4db3d1d3def9f12f6cf00605bc863b1c75d8003aa45f023678db1ee4
+  data.tar.gz: b7a672e1cfabd50b07871b2f67f8042349cbd367cd53e1d8d3eafd23eaad7655c254f10484c5824893152e3be1711759e67cd07efe8a47aa3e7381ba0b8f3e5c

data/lib/yap/shell/parser/grammar.y

@@ -29,18 +29,18 @@ stmts : stmts Separator stmt
   | range_stmt
 
 stmt : stmt Conditional pipeline
-    { result = ConditionalNode.new(val[1].value, val[0], val[2]) }
+    { result = ConditionalNode.new(val[1], val[0], val[2]) }
   | pipeline
   | block_stmt
 
 block_stmt : range_stmt BlockBegin stmts BlockEnd
     { result = val[0].tap { |range_node| range_node.tail = BlockNode.new(nil, val[2]) } }
   | range_stmt BlockBegin BlockParams stmts BlockEnd
-    { result = val[0].tap { |range_node| range_node.tail = BlockNode.new(nil, val[3], params: val[2].value) } }
+    { result = val[0].tap { |range_node| range_node.tail = BlockNode.new(nil, val[3], params: val[2]) } }
   | stmt BlockBegin stmts BlockEnd
     { result = BlockNode.new(val[0], val[2]) }
   | stmt BlockBegin BlockParams stmts BlockEnd
-    { result = BlockNode.new(val[0], val[3], params: val[2].value) }
+    { result = BlockNode.new(val[0], val[3], params: val[2]) }
 
 range_stmt : Range
     { result = RangeNode.new(val[0]) }
@@ -74,10 +74,10 @@ command_w_heredoc : command_w_redirects Heredoc
     { val[0].heredoc = val[1] ; result = val[0] }
   | command_w_redirects
   | Redirection
-    { result = RedirectionNode.new(val[0].value, val[0].attrs[:target]) }
+    { result = RedirectionNode.new(val[0]) }
 
 command_w_redirects : command_w_redirects Redirection
-    { val[0].redirects << RedirectionNode.new(val[1].value, val[1].attrs[:target]) ; result = val[0] }
+    { val[0].redirects << RedirectionNode.new(val[1]) ; result = val[0] }
   | command_w_vars
   | command
   | vars
@@ -86,28 +86,28 @@ command_w_vars : vars command
   { result = EnvWrapperNode.new(val[0], val[1]) }
 
 vars : vars LValue RValue
-    { val[0].add_var(val[1].value, val[2].value) ; result = val[0] }
+    { val[0].add_var(val[1], ArgumentNode.new(val[2])) ; result = val[0] }
   | LValue RValue
-    { result = EnvNode.new(val[0].value, val[1].value) }
+    { result = EnvNode.new(val[0], ArgumentNode.new(val[1])) }
 
 command : command2
 
 command2: Command
-    { result = CommandNode.new(val[0].value) }
+    { result = CommandNode.new(val[0]) }
   | Command args
-    { result = CommandNode.new(val[0].value, val[1].flatten) }
+    { result = CommandNode.new(val[0], val[1].flatten) }
   | LiteralCommand
-    { result = CommandNode.new(val[0].value, literal:true) }
+    { result = CommandNode.new(val[0], literal:true) }
   | LiteralCommand args
-    { result = CommandNode.new(val[0].value, val[1].flatten, literal:true) }
+    { result = CommandNode.new(val[0], val[1].flatten, literal:true) }
 
 args : Argument
-    { result = [ArgumentNode.new(val[0].value)] }
+    { result = [ArgumentNode.new(val[0])] }
   | args Argument
-    { result = [val[0], ArgumentNode.new(val[1].value)].flatten }
+    { result = [val[0], ArgumentNode.new(val[1])].flatten }
 
 internal_eval : InternalEval
-    { result = InternalEvalNode.new(val[0].value) }
+    { result = InternalEvalNode.new(val[0]) }
 
 ---- header
 if $0 ==__FILE__

data/lib/yap/shell/parser/lexer.rb

@@ -4,6 +4,8 @@ module Yap::Shell
   class Parser::Lexer
     class Error < ::StandardError ; end
     class HeredocMissingEndDelimiter < Error ; end
+    class LineContinuationFound < Error ; end
+    class NonterminatedString < Error ; end
 
     class Token
       include Comparable
@@ -54,7 +56,7 @@ module Yap::Shell
     REDIRECTION2           = /\A((&>|<)\s*(#{ARG}))/
 
     NUMERIC_RANGE              = /\A\(((\d+)\.\.(\d+))\)(\.each)?/
-    NUMERIC_RANGE_W_CALL      = /\A\(((\d+)\.\.(\d+))\)(\.each)?\s*:\s*/
+    NUMERIC_RANGE_W_CALL       = /\A\(((\d+)\.\.(\d+))\)(\.each)?\s*:\s*/
     NUMERIC_RANGE_W_PARAM      = /\A(\((\d+)\.\.(\d+))\)\s+as\s+([A-z0-9,\s]+)\s*:\s*/
     NUMERIC_REPETITION         = /\A((\d+)(\.times))/
     NUMERIC_REPETITION_2       = /\A((\d+)(\.times))\s*:\s*/
@@ -63,6 +65,8 @@ module Yap::Shell
     BLOCK_BEGIN = /\A\s+(\{)\s*(?:\|\s*([A-z0-9,\s]+)\s*\|)?/
     BLOCK_END = /\A\s+(\})\s*/
 
+    LINE_CONTINUATION = /.*\\\Z/
+
     SPLIT_BLOCK_PARAMS_RGX = /\s*,\s*|\s*/
 
     # Loop over the given input and yield command substitutions. This yields
@@ -99,7 +103,7 @@ module Yap::Shell
     end
 
     def tokenize(str)
-      @str = str
+      @chunk = str
       @tokens = []
       @lineno = 0
       @looking_for_args = false
@@ -111,6 +115,9 @@ module Yap::Shell
       last_position = -1
       process_next_chunk = -> { @chunk = str.slice(@current_position..-1) ; @chunk != "" }
 
+      strip_line_continutations_before_newlines
+      line_continuation_token
+
       while process_next_chunk.call
         result =
           comment_token ||
@@ -120,12 +127,12 @@ module Yap::Shell
           subgroup_token ||
           assignment_token ||
           literal_command_token ||
+          string_token ||
           command_token ||
           whitespace_token ||
           terminator_token ||
           redirection_token ||
           heredoc_token ||
-          string_argument_token ||
           argument_token ||
           internal_eval_token
 
@@ -325,15 +332,16 @@ module Yap::Shell
         prev_char = ''
         loop do
           ch = @chunk[characters_read]
-          # binding.pry
+
           if %w(' ").include?(ch)
+            @quoted_by = ch
             result = process_string @chunk[characters_read..-1], ch
             str << result.str
             characters_read += result.consumed_length
           elsif ch == '\\'
             # no-op
             characters_read += 1
-          elsif prev_char != '\\' && ch =~ /[\s\|;&\)\}]/
+          elsif prev_char != '\\' && ch =~ /[\s\|;&\(\)]/
             break
           else
             str << ch
@@ -346,7 +354,11 @@ module Yap::Shell
         end
 
         if characters_read > 0
-          token :Argument, str
+          attrs = {}
+          if @quoted_by
+            attrs.merge!(quoted_by: @quoted_by)
+          end
+          token :Argument, str, attrs: attrs
           characters_read
         else
           nil
@@ -363,7 +375,7 @@ module Yap::Shell
         @chunk = @chunk[i..-1]
         if %w(' ").include?(@chunk[0])
           result = process_string @chunk[0..-1], @chunk[0]
-          token :RValue, result.str
+          token :RValue, result.str, attrs: { quoted_by: @chunk[0] }
           consumed_length += result.consumed_length
         elsif md=@chunk.match(RH_VALUE)
           token :RValue, md[1]
@@ -373,6 +385,19 @@ module Yap::Shell
       end
     end
 
+    def line_continuation_token
+      if @chunk.match(LINE_CONTINUATION)
+        raise(
+          LineContinuationFound,
+          "Expected more input, line continutation found"
+        )
+      end
+    end
+
+    def strip_line_continutations_before_newlines
+      @chunk.gsub!(/\\\n/, '\\')
+    end
+
     def terminator_token
       if md=@chunk.match(CONDITIONAL_TERMINATOR)
         @looking_for_args = false
@@ -390,10 +415,14 @@ module Yap::Shell
     end
 
     # Matches single and double quoted strings
-    def string_argument_token
+    def string_token
       if %w(' ").include?(@chunk[0])
         result = process_string @chunk[0..-1], @chunk[0]
-        token :Argument, result.str
+        if @looking_for_args
+          token :Argument, result.str, attrs: { quoted_by: @chunk[0] }
+        else
+          token :Command, result.str
+        end
         return result.consumed_length
       end
     end
@@ -490,6 +519,7 @@ module Yap::Shell
 
         if i >= input_str.length
           puts "#{' '*indent}C-yah: result:#{result_str.inspect}  length: #{input_str.length}"  if ENV["DEBUG"]
+          raise NonterminatedString, "Expected to find #{delimiter} in:\n  #{input_str}"
           return OpenStruct.new(str:result_str, consumed_length: input_str.length)
         end
 

data/lib/yap/shell/parser/nodes.rb

@@ -19,26 +19,21 @@ module Yap::Shell
 
       attr_reader :lvalue
 
-      def initialize(lvalue)
-        @lvalue = lvalue
+      def initialize(token)
+        @lvalue = token.value
+        @attrs = token.attrs
       end
 
-      def inspect
-        to_s
+      def quoted?
+        double_quoted? || single_quoted?
       end
 
-      def to_s
-        "ArgumentNode(#{lvalue.inspect})"
+      def double_quoted?
+        @attrs[:quoted_by] == '"'
       end
-    end
 
-    class AssignmentNode
-      include Visitor
-
-      attr_reader :lvalue, :rvalue
-
-      def initialize(lvalue, rvalue)
-        @lvalue, @rvalue = lvalue, rvalue
+      def single_quoted?
+        @attrs[:quoted_by] == "'"
       end
 
       def inspect
@@ -46,7 +41,7 @@ module Yap::Shell
       end
 
       def to_s
-        "A(#{lvalue.inspect}=#{rvalue.inspect})"
+        "ArgumentNode(#{lvalue.inspect})"
       end
     end
 
@@ -56,8 +51,8 @@ module Yap::Shell
       attr_reader :command, :args
       attr_accessor :heredoc, :redirects
 
-      def initialize(command, *args, literal:false, heredoc:nil)
-        @command = command
+      def initialize(token, *args, literal:false, heredoc:nil)
+        @command = token.value
         @args = args.flatten
         @literal = literal
         @heredoc = nil
@@ -108,13 +103,13 @@ module Yap::Shell
 
       attr_reader :env
 
-      def initialize(lvalue, rvalue)
+      def initialize(token, argument_node)
         @env = {}
-        @env[lvalue] = rvalue
+        @env[token.value] = argument_node
       end
 
-      def add_var(lvalue, rvalue)
-        @env[lvalue] = rvalue
+      def add_var(token, argument_node)
+        @env[token.value] = argument_node
       end
     end
 
@@ -165,8 +160,8 @@ module Yap::Shell
 
       attr_reader :operator, :expr1, :expr2
 
-      def initialize(operator, expr1, expr2)
-        @operator = operator
+      def initialize(token, expr1, expr2)
+        @operator = token.value
         @expr1    = expr1
         @expr2    = expr2
       end
@@ -182,8 +177,8 @@ module Yap::Shell
       attr_reader :src
       alias_method :command, :src
 
-      def initialize(src)
-        @src = src
+      def initialize(token)
+        @src = token.value
       end
 
       def args
@@ -231,9 +226,9 @@ module Yap::Shell
 
       attr_reader :kind, :target
 
-      def initialize(kind, target)
-        @kind = kind
-        @target = target
+      def initialize(token)
+        @kind = token.value
+        @target = token.attrs[:target]
       end
 
       def to_s(indent:0)
@@ -289,10 +284,10 @@ module Yap::Shell
 
       attr_accessor :head, :tail, :params
 
-      def initialize(head, tail, params: [])
+      def initialize(head, tail, params: nil)
         @head = head
         @tail = tail
-        @params = params
+        @params = params ? params.value : []
       end
 
       def to_s(indent:0)

data/lib/yap/shell/parser/version.rb

@@ -3,7 +3,7 @@ require 'yap/shell/parser'
 module Yap
   module Shell
     module Parser
-      VERSION = "0.4.0"
+      VERSION = "0.5.0"
     end
   end
 end

data/lib/yap/shell/parser_impl.rb

@@ -293,7 +293,7 @@ module_eval(<<'.,.,', 'grammar.y', 27)
 
 module_eval(<<'.,.,', 'grammar.y', 31)
   def _reduce_6(val, _values, result)
-     result = ConditionalNode.new(val[1].value, val[0], val[2]) 
+     result = ConditionalNode.new(val[1], val[0], val[2]) 
     result
   end
 .,.,
@@ -311,7 +311,7 @@ module_eval(<<'.,.,', 'grammar.y', 36)
 
 module_eval(<<'.,.,', 'grammar.y', 38)
   def _reduce_10(val, _values, result)
-     result = val[0].tap { |range_node| range_node.tail = BlockNode.new(nil, val[3], params: val[2].value) } 
+     result = val[0].tap { |range_node| range_node.tail = BlockNode.new(nil, val[3], params: val[2]) } 
     result
   end
 .,.,
@@ -325,7 +325,7 @@ module_eval(<<'.,.,', 'grammar.y', 40)
 
 module_eval(<<'.,.,', 'grammar.y', 42)
   def _reduce_12(val, _values, result)
-     result = BlockNode.new(val[0], val[3], params: val[2].value) 
+     result = BlockNode.new(val[0], val[3], params: val[2]) 
     result
   end
 .,.,
@@ -409,14 +409,14 @@ module_eval(<<'.,.,', 'grammar.y', 73)
 
 module_eval(<<'.,.,', 'grammar.y', 76)
   def _reduce_29(val, _values, result)
-     result = RedirectionNode.new(val[0].value, val[0].attrs[:target]) 
+     result = RedirectionNode.new(val[0]) 
     result
   end
 .,.,
 
 module_eval(<<'.,.,', 'grammar.y', 79)
   def _reduce_30(val, _values, result)
-     val[0].redirects << RedirectionNode.new(val[1].value, val[1].attrs[:target]) ; result = val[0] 
+     val[0].redirects << RedirectionNode.new(val[1]) ; result = val[0] 
     result
   end
 .,.,
@@ -436,14 +436,14 @@ module_eval(<<'.,.,', 'grammar.y', 85)
 
 module_eval(<<'.,.,', 'grammar.y', 88)
   def _reduce_35(val, _values, result)
-     val[0].add_var(val[1].value, val[2].value) ; result = val[0] 
+     val[0].add_var(val[1], ArgumentNode.new(val[2])) ; result = val[0] 
     result
   end
 .,.,
 
 module_eval(<<'.,.,', 'grammar.y', 90)
   def _reduce_36(val, _values, result)
-     result = EnvNode.new(val[0].value, val[1].value) 
+     result = EnvNode.new(val[0], ArgumentNode.new(val[1])) 
     result
   end
 .,.,
@@ -452,49 +452,49 @@ module_eval(<<'.,.,', 'grammar.y', 90)
 
 module_eval(<<'.,.,', 'grammar.y', 95)
   def _reduce_38(val, _values, result)
-     result = CommandNode.new(val[0].value) 
+     result = CommandNode.new(val[0]) 
     result
   end
 .,.,
 
 module_eval(<<'.,.,', 'grammar.y', 97)
   def _reduce_39(val, _values, result)
-     result = CommandNode.new(val[0].value, val[1].flatten) 
+     result = CommandNode.new(val[0], val[1].flatten) 
     result
   end
 .,.,
 
 module_eval(<<'.,.,', 'grammar.y', 99)
   def _reduce_40(val, _values, result)
-     result = CommandNode.new(val[0].value, literal:true) 
+     result = CommandNode.new(val[0], literal:true) 
     result
   end
 .,.,
 
 module_eval(<<'.,.,', 'grammar.y', 101)
   def _reduce_41(val, _values, result)
-     result = CommandNode.new(val[0].value, val[1].flatten, literal:true) 
+     result = CommandNode.new(val[0], val[1].flatten, literal:true) 
     result
   end
 .,.,
 
 module_eval(<<'.,.,', 'grammar.y', 104)
   def _reduce_42(val, _values, result)
-     result = [ArgumentNode.new(val[0].value)] 
+     result = [ArgumentNode.new(val[0])] 
     result
   end
 .,.,
 
 module_eval(<<'.,.,', 'grammar.y', 106)
   def _reduce_43(val, _values, result)
-     result = [val[0], ArgumentNode.new(val[1].value)].flatten 
+     result = [val[0], ArgumentNode.new(val[1])].flatten 
     result
   end
 .,.,
 
 module_eval(<<'.,.,', 'grammar.y', 109)
   def _reduce_44(val, _values, result)
-     result = InternalEvalNode.new(val[0].value) 
+     result = InternalEvalNode.new(val[0]) 
     result
   end
 .,.,

data/spec/yap/shell/lexer_spec.rb

@@ -72,18 +72,25 @@ describe Yap::Shell::Parser::Lexer do
         end
 
         describe "ls {echo n }" do
-          it { should eq [
-            t(:Command, "ls", lineno:0),
-            t(:BlockBegin, '{', lineno: 0),
-            t(:Command, "echo", lineno:0),
-            t(:Argument, "n", lineno:0),
-            t(:BlockEnd, '}', lineno: 0)
-          ]}
+          it "parses as a block" do
+            should eq [
+              t(:Command, "ls", lineno:0),
+              t(:BlockBegin, '{', lineno: 0),
+              t(:Command, "echo", lineno:0),
+              t(:Argument, "n", lineno:0),
+              t(:BlockEnd, '}', lineno: 0)
+            ]
+          end
         end
 
         describe "ls s{ echo n}" do
-          it "fails to lex" do
-            expect { subject }.to raise_error
+          it "parses as arguments" do
+            should eq [
+              t(:Command, "ls", lineno:0),
+              t(:Argument, "s{", lineno:0),
+              t(:Argument, "echo", lineno:0),
+              t(:Argument, "n}", lineno:0)
+            ]
           end
         end
       end
@@ -468,7 +475,7 @@ describe Yap::Shell::Parser::Lexer do
         let(:str){ "ls 'hello there'" }
         it { should eq [
           t(:Command, "ls", lineno:0),
-          t(:Argument, "hello there", lineno:0)
+          t(:Argument, "hello there", lineno:0, attrs: { quoted_by: "'" })
         ]}
       end
 
@@ -476,7 +483,7 @@ describe Yap::Shell::Parser::Lexer do
         let(:str){ "ls 'hello \\'there\\''" }
         it { should eq [
           t(:Command, "ls", lineno:0),
-          t(:Argument, "hello 'there'", lineno:0)
+          t(:Argument, "hello 'there'", lineno:0, attrs: { quoted_by: "'" })
         ]}
       end
 
@@ -484,7 +491,7 @@ describe Yap::Shell::Parser::Lexer do
         let(:str){ %|ls 'hello "there"'| }
         it { should eq [
           t(:Command, "ls", lineno:0),
-          t(:Argument, 'hello "there"', lineno:0)
+          t(:Argument, 'hello "there"', lineno:0, attrs: { quoted_by: "'" })
         ]}
       end
 
@@ -492,7 +499,7 @@ describe Yap::Shell::Parser::Lexer do
         let(:str){ "ls 'hello \\'there \\\\'guy\\\\' \\''" }
         it { should eq [
           t(:Command, "ls", lineno:0),
-          t(:Argument, "hello 'there \\'guy\\' '", lineno:0)
+          t(:Argument, "hello 'there \\'guy\\' '", lineno:0, attrs: { quoted_by: "'" })
         ]}
       end
 
@@ -500,8 +507,8 @@ describe Yap::Shell::Parser::Lexer do
         let(:str){ "ls 'hello \\'there\\'' 'how are \\'you\\'?'" }
         it { should eq [
           t(:Command, "ls", lineno:0),
-          t(:Argument, "hello 'there'", lineno:0),
-          t(:Argument, "how are 'you'?", lineno:0)
+          t(:Argument, "hello 'there'", lineno:0, attrs: { quoted_by: "'" }),
+          t(:Argument, "how are 'you'?", lineno:0, attrs: { quoted_by: "'" })
         ]}
       end
 
@@ -509,7 +516,7 @@ describe Yap::Shell::Parser::Lexer do
         let(:str){ "alias z='echo hi'" }
         it { should eq [
           t(:Command, "alias", lineno:0),
-          t(:Argument, "z=echo hi", lineno:0),
+          t(:Argument, "z=echo hi", lineno:0, attrs: { quoted_by: "'" }),
         ]}
       end
     end
@@ -519,7 +526,7 @@ describe Yap::Shell::Parser::Lexer do
         let(:str){ 'ls "hello there"' }
         it { should eq [
           t(:Command, 'ls', lineno:0),
-          t(:Argument, 'hello there', lineno:0)
+          t(:Argument, 'hello there', lineno:0, attrs: { quoted_by: '"' })
         ]}
       end
 
@@ -527,7 +534,7 @@ describe Yap::Shell::Parser::Lexer do
         let(:str){ %|ls "hello 'there'"| }
         it { should eq [
           t(:Command, 'ls', lineno:0),
-          t(:Argument, "hello 'there'", lineno:0)
+          t(:Argument, "hello 'there'", lineno:0, attrs: { quoted_by: '"' })
         ]}
       end
 
@@ -535,7 +542,7 @@ describe Yap::Shell::Parser::Lexer do
         let(:str){ 'ls "hello \\"there\\""' }
         it { should eq [
           t(:Command, 'ls', lineno:0),
-          t(:Argument, 'hello "there"', lineno:0)
+          t(:Argument, 'hello "there"', lineno:0, attrs: { quoted_by: '"' })
         ]}
       end
 
@@ -543,7 +550,7 @@ describe Yap::Shell::Parser::Lexer do
         let(:str){ 'ls "hello \\"there \\\\"guy\\\\" \\""' }
         it { should eq [
           t(:Command, 'ls', lineno:0),
-          t(:Argument, 'hello "there \\"guy\\" "', lineno:0)
+          t(:Argument, 'hello "there \\"guy\\" "', lineno:0, attrs: { quoted_by: '"' })
         ]}
       end
 
@@ -551,8 +558,8 @@ describe Yap::Shell::Parser::Lexer do
         let(:str){ 'ls "hello \\"there\\"" "how are \\"you\\"?"' }
         it { should eq [
           t(:Command, 'ls', lineno:0),
-          t(:Argument, 'hello "there"', lineno:0),
-          t(:Argument, 'how are "you"?', lineno:0)
+          t(:Argument, 'hello "there"', lineno:0, attrs: { quoted_by: '"' }),
+          t(:Argument, 'how are "you"?', lineno:0, attrs: { quoted_by: '"' })
         ]}
       end
 
@@ -560,10 +567,28 @@ describe Yap::Shell::Parser::Lexer do
         let(:str){ "alias z=\"echo hi\"" }
         it { should eq [
           t(:Command, "alias", lineno:0),
-          t(:Argument, "z=echo hi", lineno:0),
+          t(:Argument, "z=echo hi", lineno:0, attrs: { quoted_by: '"' })
         ]}
       end
     end
+
+    context 'args with special characters' do
+      describe 'echo @{u}' do
+        it { should eq [
+          t(:Command, "echo", lineno:0),
+          t(:Argument, "@{u}", lineno:0)
+        ]}
+      end
+
+      describe 'echo @\(u\)' do
+        it "parentheses must be escaped to be treated as arguments" do
+          should eq [
+            t(:Command, "echo", lineno:0),
+            t(:Argument, "@(u)", lineno:0)
+          ]
+        end
+      end
+    end
   end
 
   context "statements" do
@@ -1063,7 +1088,7 @@ describe Yap::Shell::Parser::Lexer do
         t(:LValue, "FOO", lineno: 0),
         t(:RValue, "abc", lineno: 0),
         t(:LValue, "BAR", lineno: 0),
-        t(:RValue, "hello world", lineno: 0),
+        t(:RValue, "hello world", lineno: 0, attrs: { quoted_by: "'"}),
         t(:Command, "ls", lineno: 0),
         t(:Argument, "-l", lineno: 0)
       ]}
@@ -1332,4 +1357,79 @@ describe Yap::Shell::Parser::Lexer do
       end
     end
   end
+
+  describe 'strings' do
+    describe 'non-terminated strings' do
+      it 'raises an error on non-terminated double quotes' do
+        expect do
+          described_class.new.tokenize('"nonterminated')
+        end.to raise_error(
+          Yap::Shell::Parser::Lexer::NonterminatedString,
+          %|Expected to find " in:\n  "nonterminated|
+        )
+      end
+
+      it 'raises an error on non-terminated single quotes' do
+        expect do
+          described_class.new.tokenize("'nonterminated")
+        end.to raise_error(
+          Yap::Shell::Parser::Lexer::NonterminatedString,
+          %|Expected to find ' in:\n  'nonterminated|
+        )
+      end
+    end
+  end
+
+  describe 'line continutations' do
+    describe 'backward slashes' do
+      it 'raises an error when found at the end of the single line string' do
+        expect do
+          described_class.new.tokenize('echo hello \\')
+        end.to raise_error(
+          Yap::Shell::Parser::Lexer::LineContinuationFound,
+          %|Expected more input, line continutation found|
+        )
+      end
+
+      it 'raises an error when found at the end of a multiline string' do
+        expect do
+          described_class.new.tokenize("echo hello \n world\\")
+        end.to raise_error(
+          Yap::Shell::Parser::Lexer::LineContinuationFound,
+          %|Expected more input, line continutation found|
+        )
+      end
+
+      it 'does not raise when found before the end of a single line string' do
+        expect do
+          described_class.new.tokenize("echo hello \\ world")
+        end.to_not raise_error
+      end
+
+      it 'does not raise when found before the end of a multiline string' do
+        expect do
+          described_class.new.tokenize("echo hello \n world\\ word")
+        end.to_not raise_error
+      end
+
+      context 'newlines that follow line continuations inside a string are stripped' do
+        let(:str){ "echo foo\\\nbar" }
+        it { should eq [
+          t(:Command, "echo", lineno:0),
+          t(:Argument, "foobar", lineno:0)
+        ]}
+      end
+
+      context 'newlines that follow line continuations at the end of a string are not stripped' do
+        it 'raises an error when found at the end of a multiline string' do
+          expect do
+            described_class.new.tokenize("echo foo\\\nbar\\\n" )
+          end.to raise_error(
+            Yap::Shell::Parser::Lexer::LineContinuationFound,
+            %|Expected more input, line continutation found|
+          )
+        end
+      end
+    end
+  end
 end

data/spec/yap/shell/parser_spec.rb

@@ -61,6 +61,9 @@ describe Yap::Shell::Parser do
   it { is_expected.to parse "echo bar && > foo.txt" }
   it { is_expected.to parse "#this is a comment" }
   it { is_expected.to parse "echo foo #this last part is a comment" }
+  it { is_expected.to parse "git reset --hard @{u}" }
 
   it { is_expected.to fail_parsing "ls ()" }
+  it { is_expected.to parse("echo 'hi'")}
+  it { is_expected.to parse("ls > foo.txt")}
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yap-shell-parser
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Zach Dennis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-03-25 00:00:00.000000000 Z
+date: 2016-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler