yap-shell-parser 0.0.4 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e0f5112fefe27736d9fe13a853af8d68fe0ef467
-  data.tar.gz: b878cc0fab5a610f2d9c301bc298ce57ef446b29
+  metadata.gz: f0e319c332283c42adde5fcba98b7ee4c8f8e8bb
+  data.tar.gz: 9d1e3d51c8df81d6e3a50cca393bff86f3edbaeb
 SHA512:
-  metadata.gz: 4ece41ddbb734944259ea26c50c8b3aa1d5ccdfa9592dc348c210ad75af0ac9918bbe872a13dbe2f369a3a5fd7a7ac124f189226c3d369b0cd35588804bc46a5
-  data.tar.gz: 15e8978edc514cd0d81a1521b5872d6766e59b710057bfd5e42699b151e742c39d9d38500335658dfbfdc676b5fa84c241b4c5f282d7cef4d1572181294e5258
+  metadata.gz: eaaa6d7f6082dbb8915253dc41eb62295b98b7170398c6e4918396f19c3c714a2a5c968cc9e34f865fbe13c082d2e5ee688212952107a10fc6d41b1d0c6dbe97
+  data.tar.gz: 2a8f4c7efa8af4a3b14c5b275da68cd8071231afb68a653f374af6f888e9bbce6b2690e053ddc3735be0810a8f86dd4966f66a91ecf5aadf832d35228bc01f92

data/lib/tasks/gem.rake

@@ -15,11 +15,13 @@ namespace :bump do
           _major = major.call($1) if major
           _minor = minor.call($2) if minor
           _patch = patch.call($3) if patch
-          version = %|VERSION = "#{_major}.#{_minor}.#{_patch}"|
+          version = "#{_major}.#{_minor}.#{_patch}"
+          results = %|VERSION = "#{version}"|
         end
         File.write(@file, contents)
         system "bundle"
         system "git add #{ProjectVersion::FILE} && git commit -m 'Bumping version to #{version}'"
+        system "git tag v#{version}"
       end
 
       private

data/lib/yap/shell/parser/grammar.y

@@ -3,7 +3,7 @@
 # convert Array-like string into Ruby's Array.
 
 class Yap::Shell::ParserImpl
-  token Command LiteralCommand Argument Heredoc InternalEval Separator Conditional Pipe Redirection LValue RValue
+  token Command LiteralCommand Argument Heredoc InternalEval Separator Conditional Pipe Redirection LValue RValue BeginCommandSubstitution EndCommandSubstitution
   #
   # prechigh
   # #   left '**' '*' '/' '%'
@@ -35,9 +35,19 @@ pipeline : pipeline Pipe stmts2
 
 stmts2 : '(' stmts ')'
     { result = val[1] }
+  | stmts2 stmt_w_substitutions
+    { result = ConcatenationNode.new(val[0], val[1]) }
+  | stmt_w_substitutions
   | command_w_heredoc
   | internal_eval
 
+stmt_w_substitutions : stmt_w_substitutions2 args
+    { result = val[0] ; val[0].tail = val[1] }
+  | stmt_w_substitutions2
+
+stmt_w_substitutions2 : BeginCommandSubstitution stmts EndCommandSubstitution
+  { result = CommandSubstitutionNode.new(val[1]) }
+
 command_w_heredoc : command_w_redirects Heredoc
     { val[0].heredoc = val[1] ; result = val[0] }
   | command_w_redirects
@@ -68,26 +78,42 @@ command2: Command
     { result = CommandNode.new(val[0].value, val[1].flatten, literal:true) }
 
 args : Argument
-    { result = [val[0].value] }
+    { result = [ArgumentNode.new(val[0].value)] }
   | args Argument
-    { result = [val[0], val[1].value] }
+    { result = [val[0], ArgumentNode.new(val[1].value)].flatten }
 
 internal_eval : InternalEval
     { result = InternalEvalNode.new(val[0].value) }
 
+---- header
+if $0 ==__FILE__
+  $LOAD_PATH.unshift File.dirname(__FILE__) + "/../../"
+  module Yap
+    module Shell
+      module Parser
+      end
+    end
+  end
+  require 'yap/shell/parser/nodes'
+end
 
 ---- inner
   include Yap::Shell::Parser::Nodes
+
+  def each_command_substitution_for(input, &blk)
+    Yap::Shell::Parser::Lexer.new.each_command_substitution_for(input, &blk)
+  end
+
 #=end
   def parse(str)
-    # @yydebug = true
+    @yydebug = true
 
     @q = Yap::Shell::Parser::Lexer.new.tokenize(str)
     # @q.push [false, '$']   # is optional from Racc 1.3.7
 # puts @q.inspect
 # puts "---- parse tree follows ----"
     __send__(Racc_Main_Parsing_Routine, _racc_setup(), false)
-    #do_parse
+    # do_parse
   end
 
   def next_token
@@ -101,41 +127,20 @@ if $0 == __FILE__
   require 'yap/shell/parser/lexer'
   require 'yap/shell/parser/nodes'
     [
-    # "echo foo",
-    # "echo foo ; echo bar baz yep",
-    # "echo foo && echo bar baz yep",
-    # "echo foo && echo bar && ls foo && ls bar",
-    # "echo foo ; echo bar baz yep ; ls foo",
-    # "echo foo && echo bar ; ls baz",
-    # "echo foo && echo bar ; ls baz ; echo zach || echo gretchen",
-    # "echo foo | bar",
-    # "echo foo | bar && foo | bar",
-    # "foo && bar ; word || baz ; yep | grep -v foo",
-    # "( foo )",
-    # "( foo a b && bar c d )",
-    # "( foo a b && (bar c d | baz e f))",
-    # "((((foo))))",
-    # "foo -b -c ; (this ;that ;the; other  ;thing) && yep",
-    # "foo -b -c ; (this ;that && other  ;thing) && yep",
-    # "4 + 5",
-    # "!'hello' ; 4 - 4 && 10 + 3",
-    # "\\foo <<-EOT\nbar\nEOT",
-    # "ls | grep md | grep WISH",
-    # "(!upcase)",
-    # "echo foo > bar.txt",
-    # "ls -l > a.txt ; echo f 2> b.txt ; cat b &> c.txt ; du -sh 1>&2 1>hey.txt",
-    # "!Dir.chdir('..')",
-    # "FOO=123",
-    # "FOO=123 BAR=345",
-    # "FOO=abc bar=2314 car=14ab ls -l",
-    "FOO=abc BAR='hello world' ls -l ; CAR=f echo foo && say hi"
+    # "echo `echo hi`",
+    # "`git cbranch`",
+    # "`git cbranch`.bak",
+    # "echo `echo hi`",
+    "echo `echo hi` foo bar baz",
+    # "`hi``bye` `what`",
+    # "echo && `what` && where is `that`thing | `you know`",
     ].each do |src|
       puts 'parsing:'
       print src
       puts
       puts 'result:'
       require 'pp'
-      ast = Yap::Shell::Parser.new.parse(src)
+      ast = Yap::Shell::ParserImpl.new.parse(src)
       pp ast
     end
 

data/lib/yap/shell/parser/lexer.rb

@@ -49,6 +49,40 @@ module Yap::Shell
     REDIRECTION            = /\A(([12]?>&?[12]?)\s*(?![12]>)(#{ARG})?)/
     REDIRECTION2           = /\A((&>|<)\s*(#{ARG}))/
 
+
+    # Loop over the given input and yield command substitutions. This yields
+    # an object that responds to #str, and #position.
+    #
+    # * The #str will be the contents of the command substitution, e.g. foo in `foo` or $(foo)
+    # * The #position will be range denoting where the command substitution started and stops in the string
+    #
+    # This will yield a result for every command substitution found.
+    #
+    # == Note
+    #
+    # This will not yield nested command substitutions. The caller is responsible
+    # for that.
+    def each_command_substitution_for(input, &blk)
+      return unless input
+
+      i = 0
+      loop do
+        break if i >= input.length
+
+        @chunk = input[i..-1]
+        if md=@chunk.match(COMMAND_SUBSTITUTION)
+          start = i
+          delimiter = md[1] == "$(" ? ")" : md[1]
+          result = process_string @chunk[md[0].length-1..-1], delimiter
+          consumed_length_so_far = result.consumed_length + (md[0].length - 1)
+          i += consumed_length_so_far
+          yield OpenStruct.new(str:result.str, position:(start..i))
+        else
+          i += 1
+        end
+      end
+    end
+
     def tokenize(str)
       @str = str
       @tokens = []
@@ -172,7 +206,7 @@ module Yap::Shell
             result = process_string @chunk[i..-1], ch
             str << result.str
             i += result.consumed_length
-          elsif ch =~ /[\s\|;&]/
+          elsif ch =~ /[\s\|;&\)]/
             break
           else
             str << ch
@@ -233,14 +267,41 @@ module Yap::Shell
 
     def command_substitution_token
       if md=@chunk.match(COMMAND_SUBSTITUTION)
+        @looking_for_args = true
+
         delimiter = md[1] == "$(" ? ")" : md[1]
         result = process_string @chunk[md[0].length-1..-1], delimiter
-        token :BeginSubcommand, md[1]
+
+        consumed_length_so_far = result.consumed_length + (md[0].length - 1)
+        append_result = process_until_separator(@chunk[consumed_length_so_far..-1])
+
+        token :BeginCommandSubstitution, md[1]
         @tokens.push *self.class.new.tokenize(result.str)
-        token :EndSubcommand, delimiter
 
-        result.consumed_length + (md[0].length - 1)
+        if append_result.consumed_length > 0
+          token :EndCommandSubstitution, delimiter, attrs:{concat_with: append_result.str}
+        else
+          token :EndCommandSubstitution, delimiter
+        end
+
+        return consumed_length_so_far + append_result.consumed_length
+      end
+    end
+
+    def process_until_separator(input_str)
+      str = ""
+      i = 0
+      loop do
+        ch = input_str[i]
+
+        if ch && ch !~ /[\s;\|&>\$<`]/
+          str << ch
+          i+=1
+        else
+          break
+        end
       end
+      OpenStruct.new(str:str, consumed_length: str.length)
     end
 
     def process_internal_eval(input_str, consumed:0)

data/lib/yap/shell/parser/nodes.rb

@@ -6,6 +6,24 @@ module Yap::Shell
       end
     end
 
+    class ArgumentNode
+      include Visitor
+
+      attr_reader :lvalue
+
+      def initialize(lvalue)
+        @lvalue = lvalue
+      end
+
+      def inspect
+        to_s
+      end
+
+      def to_s
+        "ArgumentNode(#{lvalue.inspect})"
+      end
+    end
+
     class AssignmentNode
       include Visitor
 
@@ -201,5 +219,44 @@ module Yap::Shell
       end
     end
 
+    class ConcatenationNode
+      include Visitor
+
+      attr_reader :left, :right
+
+      def initialize(left, right)
+        @left = left
+        @right = right
+      end
+
+      def to_s(indent:0)
+        "ConcatenationNode(left: #{left.to_s}, right: #{right.to_s})"
+      end
+
+      def inspect
+        to_s
+      end
+
+    end
+
+    class CommandSubstitutionNode
+      include Visitor
+
+      attr_accessor :tail
+      attr_reader :node
+
+      def initialize(node)
+        @node = node
+      end
+
+      def to_s(indent:0)
+        "CommandSubstitutionNode(#{@node.to_s}, tail: #{tail.inspect})"
+      end
+
+      def inspect
+        to_s
+      end
+    end
+
   end
 end

data/lib/yap/shell/parser/version.rb

@@ -3,7 +3,7 @@ require 'yap/shell/parser'
 module Yap
   module Shell
     module Parser
-      VERSION = "0.0.4"
+      VERSION = "0.1.0"
     end
   end
 end

data/lib/yap/shell/parser_impl.rb

@@ -5,22 +5,39 @@
 #
 
 require 'racc/parser.rb'
+
+if $0 ==__FILE__
+  $LOAD_PATH.unshift File.dirname(__FILE__) + "/../../"
+  module Yap
+    module Shell
+      module Parser
+      end
+    end
+  end
+  require 'yap/shell/parser/nodes'
+end
+
 module Yap
   module Shell
     class ParserImpl < Racc::Parser
 
-module_eval(<<'...end grammar.y/module_eval...', 'grammar.y', 80)
+module_eval(<<'...end grammar.y/module_eval...', 'grammar.y', 101)
   include Yap::Shell::Parser::Nodes
+
+  def each_command_substitution_for(input, &blk)
+    Yap::Shell::Parser::Lexer.new.each_command_substitution_for(input, &blk)
+  end
+
 #=end
   def parse(str)
-    # @yydebug = true
+    @yydebug = true
 
     @q = Yap::Shell::Parser::Lexer.new.tokenize(str)
     # @q.push [false, '$']   # is optional from Racc 1.3.7
 # puts @q.inspect
 # puts "---- parse tree follows ----"
     __send__(Racc_Main_Parsing_Routine, _racc_setup(), false)
-    #do_parse
+    # do_parse
   end
 
   def next_token
@@ -31,78 +48,95 @@ module_eval(<<'...end grammar.y/module_eval...', 'grammar.y', 80)
 ##### State transition tables begin ###
 
 racc_action_table = [
-    15,    16,    29,    23,    17,    29,    15,    16,    24,    13,
-    17,     6,    15,    16,    27,    13,    17,     6,    15,    16,
-    31,    13,    17,     6,    15,    16,    21,    13,    17,     6,
-    15,    16,    20,    13,    19,     6,    19,    18,    36,    26,
-    37,    35,    37,    20,    21 ]
+    18,    19,    28,    22,    20,    34,    18,    19,    28,    16,
+    20,    11,    41,     6,    28,    16,    22,    11,    37,     6,
+    18,    19,    11,    43,    20,    24,    18,    19,    23,    16,
+    20,    11,    22,     6,    42,    16,    21,    11,    30,     6,
+    18,    19,    44,    31,    20,    42,    18,    19,    42,    16,
+    20,    11,    23,     6,    24,    16,    11,    11,   nil,     6,
+    18,    19,   nil,   nil,   nil,   nil,   nil,   nil,   nil,    33 ]
 
 racc_action_check = [
-     0,     0,    16,     9,     0,    15,     6,     6,     9,     0,
-     6,     0,    21,    21,    13,     6,    21,     6,    20,    20,
-    18,    21,    20,    21,    19,    19,     4,    20,    19,    20,
-    12,    12,     3,    19,    22,    19,     2,     1,    26,    12,
-    28,    22,    30,    32,    33 ]
+     0,     0,    19,    26,     0,    16,    11,    11,    18,     0,
+    11,     0,    26,     0,    10,    11,    29,    11,    21,    11,
+     6,     6,     5,    29,     6,     4,    24,    24,     3,     6,
+    24,     6,     2,     6,    27,    24,     1,    24,    12,    24,
+    23,    23,    33,    12,    23,    35,    22,    22,    36,    23,
+    22,    23,    38,    23,    39,    22,    40,    22,   nil,    22,
+    15,    15,   nil,   nil,   nil,   nil,   nil,   nil,   nil,    15 ]
 
 racc_action_pointer = [
-    -2,    37,    29,    24,    17,   nil,     4,   nil,   nil,    -2,
-   nil,   nil,    28,     2,   nil,     1,    -2,   nil,    20,    22,
-    16,    10,    27,   nil,   nil,   nil,    26,   nil,    36,   nil,
-    38,   nil,    35,    35,   nil,   nil,   nil,   nil ]
+    -2,    36,    25,    20,    16,     9,    18,   nil,   nil,   nil,
+    10,     4,    33,   nil,   nil,    58,    -7,   nil,     4,    -2,
+   nil,    18,    44,    38,    24,   nil,    -4,    30,   nil,     9,
+   nil,   nil,   nil,    30,   nil,    41,    44,   nil,    44,    45,
+    43,   nil,   nil,   nil,   nil ]
 
 racc_action_default = [
-   -28,   -28,    -1,    -3,    -5,    -7,   -28,    -9,   -10,   -12,
-   -14,   -15,   -16,   -28,   -20,   -21,   -23,   -27,   -28,   -28,
-   -28,   -28,   -28,   -11,   -13,   -17,   -28,   -19,   -22,   -25,
-   -24,    38,    -2,    -4,    -6,    -8,   -18,   -26 ]
+   -33,   -33,    -1,    -3,    -5,    -7,   -33,   -10,   -11,   -12,
+   -14,   -33,   -17,   -19,   -20,   -21,   -33,   -25,   -26,   -28,
+   -32,   -33,   -33,   -33,   -33,    -9,   -33,   -13,   -30,   -33,
+   -16,   -18,   -22,   -33,   -24,   -27,   -29,    45,    -2,    -4,
+    -6,    -8,   -31,   -15,   -23 ]
 
 racc_goto_table = [
-     2,    28,    30,    33,    32,    34,    22,    25,     1 ]
+    25,     2,    27,    40,    39,    38,     1,    26,    32,   nil,
+    35,    36,    29,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,   nil,   nil,   nil,   nil,    25 ]
 
 racc_goto_check = [
-     2,    13,    13,     4,     3,     5,     2,    10,     1 ]
+     6,     2,    10,     5,     4,     3,     1,     2,    13,   nil,
+    10,    10,     2,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,   nil,   nil,   nil,   nil,     6 ]
 
 racc_goto_pointer = [
-   nil,     8,     0,   -15,   -17,   -16,   nil,   nil,   nil,   nil,
-    -5,   nil,   nil,   -14 ]
+   nil,     6,     1,   -17,   -19,   -21,    -5,   nil,   nil,   nil,
+    -8,   nil,   nil,    -7,   nil,   nil ]
 
 racc_goto_default = [
    nil,   nil,   nil,     3,     4,     5,     7,     8,     9,    10,
-    11,    12,    14,   nil ]
+   nil,    12,    13,    14,    15,    17 ]
 
 racc_reduce_table = [
   0, 0, :racc_error,
-  1, 16, :_reduce_none,
-  3, 17, :_reduce_2,
-  1, 17, :_reduce_3,
-  3, 18, :_reduce_4,
   1, 18, :_reduce_none,
-  3, 19, :_reduce_6,
-  1, 19, :_reduce_none,
-  3, 20, :_reduce_8,
-  1, 20, :_reduce_none,
+  3, 19, :_reduce_2,
+  1, 19, :_reduce_3,
+  3, 20, :_reduce_4,
   1, 20, :_reduce_none,
-  2, 21, :_reduce_11,
+  3, 21, :_reduce_6,
   1, 21, :_reduce_none,
+  3, 22, :_reduce_8,
+  2, 22, :_reduce_9,
+  1, 22, :_reduce_none,
+  1, 22, :_reduce_none,
+  1, 22, :_reduce_none,
   2, 23, :_reduce_13,
   1, 23, :_reduce_none,
-  1, 23, :_reduce_none,
-  1, 23, :_reduce_none,
-  2, 24, :_reduce_17,
-  3, 26, :_reduce_18,
-  2, 26, :_reduce_19,
-  1, 25, :_reduce_none,
-  1, 27, :_reduce_21,
-  2, 27, :_reduce_22,
-  1, 27, :_reduce_23,
-  2, 27, :_reduce_24,
-  1, 28, :_reduce_25,
-  2, 28, :_reduce_26,
-  1, 22, :_reduce_27 ]
-
-racc_reduce_n = 28
-
-racc_shift_n = 38
+  3, 26, :_reduce_15,
+  2, 24, :_reduce_16,
+  1, 24, :_reduce_none,
+  2, 28, :_reduce_18,
+  1, 28, :_reduce_none,
+  1, 28, :_reduce_none,
+  1, 28, :_reduce_none,
+  2, 29, :_reduce_22,
+  3, 31, :_reduce_23,
+  2, 31, :_reduce_24,
+  1, 30, :_reduce_none,
+  1, 32, :_reduce_26,
+  2, 32, :_reduce_27,
+  1, 32, :_reduce_28,
+  2, 32, :_reduce_29,
+  1, 27, :_reduce_30,
+  2, 27, :_reduce_31,
+  1, 25, :_reduce_32 ]
+
+racc_reduce_n = 33
+
+racc_shift_n = 45
 
 racc_token_table = {
   false => 0,
@@ -118,10 +152,12 @@ racc_token_table = {
   :Redirection => 10,
   :LValue => 11,
   :RValue => 12,
-  "(" => 13,
-  ")" => 14 }
+  :BeginCommandSubstitution => 13,
+  :EndCommandSubstitution => 14,
+  "(" => 15,
+  ")" => 16 }
 
-racc_nt_base = 15
+racc_nt_base = 17
 
 racc_use_result_var = true
 
@@ -155,6 +191,8 @@ Racc_token_to_s_table = [
   "Redirection",
   "LValue",
   "RValue",
+  "BeginCommandSubstitution",
+  "EndCommandSubstitution",
   "\"(\"",
   "\")\"",
   "$start",
@@ -163,14 +201,16 @@ Racc_token_to_s_table = [
   "stmt",
   "pipeline",
   "stmts2",
+  "stmt_w_substitutions",
   "command_w_heredoc",
   "internal_eval",
+  "stmt_w_substitutions2",
+  "args",
   "command_w_redirects",
   "command_w_vars",
   "command",
   "vars",
-  "command2",
-  "args" ]
+  "command2" ]
 
 Racc_debug_parser = false
 
@@ -219,99 +259,124 @@ module_eval(<<'.,.,', 'grammar.y', 36)
   end
 .,.,
 
-# reduce 9 omitted
+module_eval(<<'.,.,', 'grammar.y', 38)
+  def _reduce_9(val, _values, result)
+     result = ConcatenationNode.new(val[0], val[1]) 
+    result
+  end
+.,.,
 
 # reduce 10 omitted
 
-module_eval(<<'.,.,', 'grammar.y', 41)
-  def _reduce_11(val, _values, result)
+# reduce 11 omitted
+
+# reduce 12 omitted
+
+module_eval(<<'.,.,', 'grammar.y', 44)
+  def _reduce_13(val, _values, result)
+     result = val[0] ; val[0].tail = val[1] 
+    result
+  end
+.,.,
+
+# reduce 14 omitted
+
+module_eval(<<'.,.,', 'grammar.y', 48)
+  def _reduce_15(val, _values, result)
+     result = CommandSubstitutionNode.new(val[1]) 
+    result
+  end
+.,.,
+
+module_eval(<<'.,.,', 'grammar.y', 51)
+  def _reduce_16(val, _values, result)
      val[0].heredoc = val[1] ; result = val[0] 
     result
   end
 .,.,
 
-# reduce 12 omitted
+# reduce 17 omitted
 
-module_eval(<<'.,.,', 'grammar.y', 45)
-  def _reduce_13(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 55)
+  def _reduce_18(val, _values, result)
      val[0].redirects << RedirectionNode.new(val[1].value, val[1].attrs[:target]) ; result = val[0] 
     result
   end
 .,.,
 
-# reduce 14 omitted
+# reduce 19 omitted
 
-# reduce 15 omitted
+# reduce 20 omitted
 
-# reduce 16 omitted
+# reduce 21 omitted
 
-module_eval(<<'.,.,', 'grammar.y', 51)
-  def _reduce_17(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 61)
+  def _reduce_22(val, _values, result)
      result = EnvWrapperNode.new(val[0], val[1]) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 54)
-  def _reduce_18(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 64)
+  def _reduce_23(val, _values, result)
      val[0].add_var(val[1].value, val[2].value) ; result = val[0] 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 56)
-  def _reduce_19(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 66)
+  def _reduce_24(val, _values, result)
      result = EnvNode.new(val[0].value, val[1].value) 
     result
   end
 .,.,
 
-# reduce 20 omitted
+# reduce 25 omitted
 
-module_eval(<<'.,.,', 'grammar.y', 61)
-  def _reduce_21(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 71)
+  def _reduce_26(val, _values, result)
      result = CommandNode.new(val[0].value) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 63)
-  def _reduce_22(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 73)
+  def _reduce_27(val, _values, result)
      result = CommandNode.new(val[0].value, val[1].flatten) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 65)
-  def _reduce_23(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 75)
+  def _reduce_28(val, _values, result)
      result = CommandNode.new(val[0].value, literal:true) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 67)
-  def _reduce_24(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 77)
+  def _reduce_29(val, _values, result)
      result = CommandNode.new(val[0].value, val[1].flatten, literal:true) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 70)
-  def _reduce_25(val, _values, result)
-     result = [val[0].value] 
+module_eval(<<'.,.,', 'grammar.y', 80)
+  def _reduce_30(val, _values, result)
+     result = [ArgumentNode.new(val[0].value)] 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 72)
-  def _reduce_26(val, _values, result)
-     result = [val[0], val[1].value] 
+module_eval(<<'.,.,', 'grammar.y', 82)
+  def _reduce_31(val, _values, result)
+     result = [val[0], ArgumentNode.new(val[1].value)].flatten 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 75)
-  def _reduce_27(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 85)
+  def _reduce_32(val, _values, result)
      result = InternalEvalNode.new(val[0].value) 
     result
   end
@@ -331,41 +396,20 @@ if $0 == __FILE__
   require 'yap/shell/parser/lexer'
   require 'yap/shell/parser/nodes'
     [
-    # "echo foo",
-    # "echo foo ; echo bar baz yep",
-    # "echo foo && echo bar baz yep",
-    # "echo foo && echo bar && ls foo && ls bar",
-    # "echo foo ; echo bar baz yep ; ls foo",
-    # "echo foo && echo bar ; ls baz",
-    # "echo foo && echo bar ; ls baz ; echo zach || echo gretchen",
-    # "echo foo | bar",
-    # "echo foo | bar && foo | bar",
-    # "foo && bar ; word || baz ; yep | grep -v foo",
-    # "( foo )",
-    # "( foo a b && bar c d )",
-    # "( foo a b && (bar c d | baz e f))",
-    # "((((foo))))",
-    # "foo -b -c ; (this ;that ;the; other  ;thing) && yep",
-    # "foo -b -c ; (this ;that && other  ;thing) && yep",
-    # "4 + 5",
-    # "!'hello' ; 4 - 4 && 10 + 3",
-    # "\\foo <<-EOT\nbar\nEOT",
-    # "ls | grep md | grep WISH",
-    # "(!upcase)",
-    # "echo foo > bar.txt",
-    # "ls -l > a.txt ; echo f 2> b.txt ; cat b &> c.txt ; du -sh 1>&2 1>hey.txt",
-    # "!Dir.chdir('..')",
-    # "FOO=123",
-    # "FOO=123 BAR=345",
-    # "FOO=abc bar=2314 car=14ab ls -l",
-    "FOO=abc BAR='hello world' ls -l ; CAR=f echo foo && say hi"
+    # "echo `echo hi`",
+    # "`git cbranch`",
+    # "`git cbranch`.bak",
+    # "echo `echo hi`",
+    "echo `echo hi` foo bar baz",
+    # "`hi``bye` `what`",
+    # "echo && `what` && where is `that`thing | `you know`",
     ].each do |src|
       puts 'parsing:'
       print src
       puts
       puts 'result:'
       require 'pp'
-      ast = Yap::Shell::Parser.new.parse(src)
+      ast = Yap::Shell::ParserImpl.new.parse(src)
       pp ast
     end
 

data/spec/yap/shell/lexer_spec.rb

@@ -444,6 +444,19 @@ describe Yap::Shell::Parser::Lexer do
       ]}
     end
 
+    describe 'simple command with arguments: (foo a b)' do
+      let(:str){ "(foo a (b))" }
+      it { should eq [
+        t('(', '(', lineno:0),
+        t(:Command, 'foo', lineno:0),
+        t(:Argument, 'a', lineno:0),
+        t('(', '(', lineno:0),
+        t(:Argument, 'b', lineno:0),
+        t(')', ')', lineno:0),
+        t(')', ')', lineno:0)
+      ]}
+    end
+
     describe 'simple interval eval: (!bar)' do
       let(:str){ "(!bar)" }
       it { should eq [
@@ -695,29 +708,121 @@ describe Yap::Shell::Parser::Lexer do
     describe "backticks can wrap simple commands: `pwd`" do
       let(:str){ "`pwd`" }
       it { should eq [
-        t(:BeginSubcommand, "`", lineno: 0),
+        t(:BeginCommandSubstitution, "`", lineno: 0),
         t(:Command, "pwd", lineno: 0),
-        t(:EndSubcommand, "`", lineno: 0)
+        t(:EndCommandSubstitution, "`", lineno: 0)
+      ]}
+    end
+
+    describe "backticks can be used with additional arguments: git branch `git cbranch` bak" do
+      let(:str){ "git branch `git cbranch` bak" }
+      it { should eq [
+        t(:Command, "git", lineno: 0),
+        t(:Argument, "branch", lineno: 0),
+        t(:BeginCommandSubstitution, '`', lineno: 0),
+        t(:Command, 'git', lineno: 0),
+        t(:Argument, 'cbranch', lineno: 0),
+        t(:EndCommandSubstitution, '`', lineno: 0),
+        t(:Argument, 'bak', lineno: 0)
       ]}
     end
 
-    describe "backticks can be used as part of an argument: git branch `git cbranch`.bak" do
+    describe "backticks back to back: `hi``bye`" do
+      let(:str){ "`hi``bye`" }
+      it { should eq [
+        t(:BeginCommandSubstitution, '`', lineno: 0),
+        t(:Command, 'hi', lineno: 0),
+        t(:EndCommandSubstitution, '`', lineno: 0),
+        t(:BeginCommandSubstitution, '`', lineno: 0),
+        t(:Command, 'bye', lineno: 0),
+        t(:EndCommandSubstitution, '`', lineno: 0),
+      ]}
+    end
+
+    describe "backticks and following arguments can be separated: `ls`<SEPERATOR>bak" do
+      context "separated with a space" do
+        let(:str){ "`ls` bak" }
+        it { should eq [
+          t(:BeginCommandSubstitution, '`', lineno: 0),
+          t(:Command, 'ls', lineno: 0),
+          t(:EndCommandSubstitution, '`', lineno: 0),
+          t(:Argument, 'bak', lineno: 0)
+        ]}
+      end
+
+      context "separated with a semi-colon" do
+        let(:str){ "`ls`;bak" }
+        it { should eq [
+          t(:BeginCommandSubstitution, '`', lineno: 0),
+          t(:Command, 'ls', lineno: 0),
+          t(:EndCommandSubstitution, '`', lineno: 0),
+          t(:Separator, ';', lineno: 0),
+          t(:Command, 'bak', lineno: 0)
+        ]}
+      end
+
+      context "separated with a pipe" do
+        let(:str){ "`ls`|bak" }
+        it { should eq [
+          t(:BeginCommandSubstitution, '`', lineno: 0),
+          t(:Command, 'ls', lineno: 0),
+          t(:EndCommandSubstitution, '`', lineno: 0),
+          t(:Pipe, '|', lineno: 0),
+          t(:Command, 'bak', lineno: 0)
+        ]}
+      end
+
+      context "separated with &&" do
+        let(:str){ "`ls`&&bak" }
+        it { should eq [
+          t(:BeginCommandSubstitution, '`', lineno: 0),
+          t(:Command, 'ls', lineno: 0),
+          t(:EndCommandSubstitution, '`', lineno: 0),
+          t(:Conditional, '&&', lineno: 0),
+          t(:Command, 'bak', lineno: 0)
+        ]}
+      end
+
+      context "separated with ||" do
+        let(:str){ "`ls`||bak" }
+        it { should eq [
+          t(:BeginCommandSubstitution, '`', lineno: 0),
+          t(:Command, 'ls', lineno: 0),
+          t(:EndCommandSubstitution, '`', lineno: 0),
+          t(:Conditional, '||', lineno: 0),
+          t(:Command, 'bak', lineno: 0)
+        ]}
+      end
+
+      context "separated with &" do
+        let(:str){ "`ls`&bak" }
+        pending "background syntax not supported yet"
+        # it { should eq [
+        #   t(:BeginCommandSubstitution, '`', lineno: 0),
+        #   t(:Command, 'ls', lineno: 0),
+        #   t(:EndCommandSubstitution, '`', lineno: 0),
+        #   t(:Separator, '&', lineno: 0),
+        #   t(:Command, 'bak', lineno: 0)
+        # ]}
+      end
+    end
+
+    describe "backticks can be instructed to concatenate with arguments that aren't separated: git branch `git cbranch`.bak" do
       let(:str){ "git branch `git cbranch`.bak" }
       it { should eq [
         t(:Command, "git", lineno: 0),
         t(:Argument, "branch", lineno: 0),
-        t(:BeginSubcommand, '`', lineno: 0),
+        t(:BeginCommandSubstitution, '`', lineno: 0),
         t(:Command, 'git', lineno: 0),
         t(:Argument, 'cbranch', lineno: 0),
-        t(:EndSubcommand, '`', lineno: 0),
-        t(:Argument, '.bak', lineno: 0)
+        t(:EndCommandSubstitution, '`', lineno: 0, attrs:{concat_with:'.bak'})
       ]}
     end
 
     describe "backticks can wrap complex statements: `ls -al && foo bar || baz`" do
       let(:str){ "`ls -al && foo bar || baz`" }
       it { should eq [
-        t(:BeginSubcommand, "`", lineno: 0),
+        t(:BeginCommandSubstitution, "`", lineno: 0),
         t(:Command,'ls', lineno: 0),
         t(:Argument, '-al', lineno: 0),
         t(:Conditional, '&&', lineno: 0),
@@ -725,7 +830,7 @@ describe Yap::Shell::Parser::Lexer do
         t(:Argument, 'bar', lineno: 0),
         t(:Conditional, '||', lineno: 0),
         t(:Command, 'baz', lineno: 0),
-        t(:EndSubcommand, "`", lineno: 0)
+        t(:EndCommandSubstitution, "`", lineno: 0)
       ]}
     end
 
@@ -733,9 +838,9 @@ describe Yap::Shell::Parser::Lexer do
       let(:str){ "echo `pwd`" }
       it { should eq [
         t(:Command,'echo', lineno: 0),
-        t(:BeginSubcommand, "`", lineno: 0),
+        t(:BeginCommandSubstitution, "`", lineno: 0),
         t(:Command,'pwd', lineno: 0),
-        t(:EndSubcommand, "`", lineno: 0)
+        t(:EndCommandSubstitution, "`", lineno: 0)
       ]}
     end
 
@@ -743,7 +848,7 @@ describe Yap::Shell::Parser::Lexer do
       let(:str){ "echo `pwd && foo bar || baz ; yep` ; hello" }
       it { should eq [
         t(:Command,'echo', lineno: 0),
-        t(:BeginSubcommand, "`", lineno: 0),
+        t(:BeginCommandSubstitution, "`", lineno: 0),
         t(:Command,'pwd', lineno: 0),
         t(:Conditional, '&&', lineno: 0),
         t(:Command, 'foo', lineno: 0),
@@ -752,7 +857,7 @@ describe Yap::Shell::Parser::Lexer do
         t(:Command, 'baz', lineno: 0),
         t(:Separator, ";", lineno: 0),
         t(:Command, 'yep', lineno: 0),
-        t(:EndSubcommand, "`", lineno: 0),
+        t(:EndCommandSubstitution, "`", lineno: 0),
         t(:Separator, ";", lineno: 0),
         t(:Command, "hello", lineno: 0)
       ]}
@@ -761,16 +866,16 @@ describe Yap::Shell::Parser::Lexer do
     describe "dollar-sign paren can wrap simple commands: $(pwd)" do
       let(:str){ "$(pwd)" }
       it { should eq [
-        t(:BeginSubcommand, "$(", lineno: 0),
+        t(:BeginCommandSubstitution, "$(", lineno: 0),
         t(:Command, "pwd", lineno: 0),
-        t(:EndSubcommand, ")", lineno: 0)
+        t(:EndCommandSubstitution, ")", lineno: 0)
       ]}
     end
 
     describe "dollar-sign paren can wrap complex statements: $(ls -al && foo bar || baz)" do
       let(:str){ "$(ls -al && foo bar || baz)" }
       it { should eq [
-        t(:BeginSubcommand, "$(", lineno: 0),
+        t(:BeginCommandSubstitution, "$(", lineno: 0),
         t(:Command,'ls', lineno: 0),
         t(:Argument, '-al', lineno: 0),
         t(:Conditional, '&&', lineno: 0),
@@ -778,7 +883,7 @@ describe Yap::Shell::Parser::Lexer do
         t(:Argument, 'bar', lineno: 0),
         t(:Conditional, '||', lineno: 0),
         t(:Command, 'baz', lineno: 0),
-        t(:EndSubcommand, ")", lineno: 0)
+        t(:EndCommandSubstitution, ")", lineno: 0)
       ]}
     end
 
@@ -786,9 +891,9 @@ describe Yap::Shell::Parser::Lexer do
       let(:str){ "echo $(pwd)" }
       it { should eq [
         t(:Command,'echo', lineno: 0),
-        t(:BeginSubcommand, "$(", lineno: 0),
+        t(:BeginCommandSubstitution, "$(", lineno: 0),
         t(:Command,'pwd', lineno: 0),
-        t(:EndSubcommand, ")", lineno: 0)
+        t(:EndCommandSubstitution, ")", lineno: 0)
       ]}
     end
 
@@ -796,7 +901,7 @@ describe Yap::Shell::Parser::Lexer do
       let(:str){ "echo $(pwd && foo bar || baz ; yep) ; hello" }
       it { should eq [
         t(:Command,'echo', lineno: 0),
-        t(:BeginSubcommand, "$(", lineno: 0),
+        t(:BeginCommandSubstitution, "$(", lineno: 0),
         t(:Command,'pwd', lineno: 0),
         t(:Conditional, '&&', lineno: 0),
         t(:Command, 'foo', lineno: 0),
@@ -805,7 +910,7 @@ describe Yap::Shell::Parser::Lexer do
         t(:Command, 'baz', lineno: 0),
         t(:Separator, ";", lineno: 0),
         t(:Command, 'yep', lineno: 0),
-        t(:EndSubcommand, ")", lineno: 0),
+        t(:EndCommandSubstitution, ")", lineno: 0),
         t(:Separator, ";", lineno: 0),
         t(:Command, "hello", lineno: 0)
       ]}
@@ -813,4 +918,54 @@ describe Yap::Shell::Parser::Lexer do
 
   end
 
+  describe '#each_command_substitution_for' do
+    context 'when there are no substitutions' do
+      let(:str){ "echo " }
+
+      it "doesn't yield" do
+        expect { |b|
+          described_class.new.each_command_substitution_for(str, &b)
+        }.to_not yield_control
+      end
+    end
+
+    context 'when there is one substitution string' do
+      let(:str){ "echo `echo hi`" }
+
+      it "yields the command substitution string" do
+        expect { |b|
+          described_class.new.each_command_substitution_for(str, &b)
+        }.to yield_with_args OpenStruct.new(str:"echo hi", position:5..14)
+      end
+    end
+
+    context 'when there are multiple substitution strings' do
+      let(:str){ "echo `hi` foo $(world) `bye` $(world) foo" }
+
+      it "yields the command substitution string" do
+        expect { |b|
+          described_class.new.each_command_substitution_for(str, &b)
+        }.to yield_successive_args(
+          OpenStruct.new(str:"hi", position:5..9),
+          OpenStruct.new(str:"world", position:14..22),
+          OpenStruct.new(str:"bye", position:23..28),
+          OpenStruct.new(str:"world", position:29..37)
+        )
+      end
+    end
+
+    context 'when there are nested substitution strings' do
+      let(:str){ "echo `hi $(bye)`" }
+
+      it "only yields the top-level substitutions" do
+        expect { |b|
+          described_class.new.each_command_substitution_for(str, &b)
+        }.to yield_successive_args(
+          OpenStruct.new(str:"hi $(bye)", position:5..16),
+        )
+      end
+    end
+
+  end
+
 end

data/spec/yap/shell/parser_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+require 'yap/shell/parser'
+require 'pry'
+
+describe Yap::Shell::Parser do
+  subject(:parser){ Yap::Shell::Parser.new }
+
+  def self.it_parses(str)
+    context "#{str.inspect}" do
+      it "parses" do
+        expect do
+          begin
+            parser.parse(str)
+          # rescue => ex
+            # raise "Parser error:\n\t#{str}\n\n#{ex.message}\n#{ex.backtrace.join("\n")}"
+          end
+        end.to_not raise_error
+      end
+    end
+  end
+
+  it_parses "ls"
+  it_parses "echo foo"
+  it_parses "echo foo ; echo bar baz yep"
+  it_parses "echo foo && echo bar baz yep"
+  it_parses "echo foo && echo bar && ls foo && ls bar"
+  it_parses "echo foo ; echo bar baz yep ; ls foo"
+  it_parses "echo foo && echo bar ; ls baz"
+  it_parses "echo foo && echo bar ; ls baz ; echo zach || echo gretchen"
+  it_parses "echo foo | bar"
+  it_parses "echo foo | bar && foo | bar"
+  it_parses "foo && bar ; word || baz ; yep | grep -v foo"
+  it_parses "( foo )"
+  it_parses "( foo a b && bar c d )"
+  it_parses "( foo a b && (bar c d | baz e f))"
+  it_parses "((((foo))))"
+  it_parses "foo -b -c ; (this ;that ;the; other  ;thing) && yep"
+  it_parses "foo -b -c ; (this ;that && other  ;thing) && yep"
+  it_parses "4 + 5"
+  it_parses "!'hello' ; 4 - 4 && 10 + 3"
+  it_parses "\\foo <<-EOT\nbar\nEOT"
+  it_parses "ls | grep md | grep WISH"
+  it_parses "(!upcase)"
+  it_parses "echo foo > bar.txt"
+  it_parses "ls -l > a.txt ; echo f 2> b.txt ; cat b &> c.txt ; du -sh 1>&2 1>hey.txt"
+  it_parses "!Dir.chdir('..')"
+  it_parses "FOO=123"
+  it_parses "FOO=123 BAR=345"
+  it_parses "FOO=abc bar=2314 car=14ab ls -l"
+  it_parses "FOO=abc BAR='hello world' ls -l ; CAR=f echo foo && say hi"
+  it_parses "`git cbranch`"
+  it_parses "`git cbranch`.bak"  # TODO THIS NEEDS TO MAYBE NOT WORK?
+  it_parses "echo `echo hi`"
+  it_parses "echo `echo hi` foo"
+  it_parses "`hi``bye` `what`"
+  it_parses "echo && `what` && where is `that`thing | `you know`"
+end

data/yap-shell-parser.gemspec

@@ -22,4 +22,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.2"
+  spec.add_development_dependency "pry"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yap-shell-parser
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors:
 - Zach Dennis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-10 00:00:00.000000000 Z
+date: 2015-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: The parser for the yap shell
 email:
 - zach.dennis@gmail.com
@@ -77,6 +91,7 @@ files:
 - lib/yap/shell/parser_impl.rb
 - spec/spec_helper.rb
 - spec/yap/shell/lexer_spec.rb
+- spec/yap/shell/parser_spec.rb
 - yap-shell-parser.gemspec
 homepage: https://github.com/zdennis/yap-shell-parser
 licenses:
@@ -105,3 +120,5 @@ summary: The parser for the yap shell
 test_files:
 - spec/spec_helper.rb
 - spec/yap/shell/lexer_spec.rb
+- spec/yap/shell/parser_spec.rb
+has_rdoc: