yap-shell-parser 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1f871f25358079c9bb5bfda0d59b77504b36b88
-  data.tar.gz: ae41ab9e03b55318f48c9fd739448600c30e545c
+  metadata.gz: 56c11c5e48960b86e262bf7b4c6dddf59cbbce51
+  data.tar.gz: 0120fd57b88b3ec458a5a88f9d6187e37ca33190
 SHA512:
-  metadata.gz: 2b6dc9b5ebfee853ef445c1f8a60ed9e9ca8d81365497f2ae5fda0e64721ffa215c451ebd1cd95c8485759fa918d0085c7b8dfc75a1cbf7be823d2969e6f8356
-  data.tar.gz: fd2670a3b93ec221a664a71e0280ef79b5cf25f7a968abde4d2655188f89c5bbcf2de784f574f3a8ecd4355ab007ea1fcadc7d7253477b12e02962d0e37a4c34
+  metadata.gz: 00370bf66d7d030a3aee98a670d5697b63f1a6ee2be3717db6f20968cf4ba7453c4f1ab2c6fc5bd864e7dfc9a265fb79462dd787c411dc24b389ccecadac9c71
+  data.tar.gz: 7bd7d734dabbdeafad2d892d23a5d2387218c7505be61b0431d1c639ac8c911fb4c5a97cae05653d69a5ba1e673fa3f62d2ac4347d0818698cbb094d3a979fd4

data/lib/yap/shell/parser/grammar.y

@@ -3,7 +3,7 @@
 # convert Array-like string into Ruby's Array.
 
 class Yap::Shell::ParserImpl
-  token Command LiteralCommand Argument Heredoc InternalEval Separator Conditional Pipe Redirection LValue RValue BeginCommandSubstitution EndCommandSubstitution Range BlockBegin BlockEnd BlockParams
+  token Command LiteralCommand Argument Heredoc InternalEval Separator Conditional Pipe Redirection LValue RValue BeginCommandSubstitution EndCommandSubstitution Range BlockBegin BlockEnd BlockParams BlankLine Comment
   #
   # prechigh
   # #   left '**' '*' '/' '%'
@@ -19,6 +19,8 @@ class Yap::Shell::ParserImpl
 rule
 
 program : stmts
+  | BlankLine
+    { result = NoOpNode }
 
 stmts : stmts Separator stmt
     { result = StatementsNode.new(val[0], val[2]) }
@@ -46,6 +48,12 @@ range_stmt : Range
 pipeline : pipeline Pipe stmts2
     { result = PipelineNode.new(val[0], val[2]) }
   | stmts2
+  | stmts2_w_comment
+
+stmts2_w_comment : stmts2 Comment
+    { result = StatementsNode.new(val[0], CommentNode.new(val[1])) }
+  | Comment
+    { result = CommentNode.new(val[0]) }
 
 stmts2 : '(' stmts ')'
     { result = val[1] }
@@ -65,6 +73,8 @@ stmt_w_substitutions2 : BeginCommandSubstitution stmts EndCommandSubstitution
 command_w_heredoc : command_w_redirects Heredoc
     { val[0].heredoc = val[1] ; result = val[0] }
   | command_w_redirects
+  | Redirection
+    { result = RedirectionNode.new(val[0].value, val[0].attrs[:target]) }
 
 command_w_redirects : command_w_redirects Redirection
     { val[0].redirects << RedirectionNode.new(val[1].value, val[1].attrs[:target]) ; result = val[0] }

data/lib/yap/shell/parser/lexer.rb

@@ -37,9 +37,10 @@ module Yap::Shell
     end
 
     COMMAND_SUBSTITUTION   = /\A(`|\$\()/
-    ARG                    = /[^\s;\|\(\)\{\}\[\]\&\!\\\<`][^\s;\|\(\)\{\}\[\]\&\>\<`]*/
+    ARG                    = /[^\s;\|\(\)\{\}\[\]\&\!\\\<\>`][^\s;\|\(\)\{\}\[\]\&\>\<`]*/
     COMMAND                = /\A(#{ARG})/
     LITERAL_COMMAND        = /\A\\(#{ARG})/
+    COMMENT                = /\A#[^$]+/
     WHITESPACE             = /\A\s+/
     LH_ASSIGNMENT          = /\A(([A-z_][\w]*)=)/
     RH_VALUE               = /\A(\S+)/
@@ -55,12 +56,12 @@ module Yap::Shell
     NUMERIC_RANGE              = /\A\(((\d+)\.\.(\d+))\)(\.each)?/
     NUMERIC_RANGE_W_CALL      = /\A\(((\d+)\.\.(\d+))\)(\.each)?\s*:\s*/
     NUMERIC_RANGE_W_PARAM      = /\A(\((\d+)\.\.(\d+))\)\s+as\s+([A-z0-9,\s]+)\s*:\s*/
-    NUMERIC_REPETITION         = /\A((\d+)(\.times))\s*/
+    NUMERIC_REPETITION         = /\A((\d+)(\.times))/
     NUMERIC_REPETITION_2       = /\A((\d+)(\.times))\s*:\s*/
     NUMERIC_REPETITION_W_PARAM = /\A((\d+)(\.times))\s+as\s+([A-z0-9,\s]+)\s*:\s*/
 
-    BLOCK_BEGIN = /\A\s*(\{)\s*(?:\|\s*([A-z0-9,\s]+)\s*\|)?/
-    BLOCK_END = /\A\s*(\})\s*/
+    BLOCK_BEGIN = /\A\s+(\{)\s*(?:\|\s*([A-z0-9,\s]+)\s*\|)?/
+    BLOCK_END = /\A\s+(\})\s*/
 
     SPLIT_BLOCK_PARAMS_RGX = /\s*,\s*|\s*/
 
@@ -107,10 +108,12 @@ module Yap::Shell
       max = 100
       count = 0
       @current_position = 0
+      last_position = -1
       process_next_chunk = -> { @chunk = str.slice(@current_position..-1) ; @chunk != "" }
 
       while process_next_chunk.call
         result =
+          comment_token ||
           block_token ||
           numerical_range_token ||
           command_substitution_token ||
@@ -127,8 +130,10 @@ module Yap::Shell
           internal_eval_token
 
         count += 1
-        raise "Infinite loop detected on #{@chunk.inspect}" if count == max
+        # raise "Infinite loop detected on #{@chunk.inspect}" if count == max
+        raise "Infinite loop detected in #{str.inspect}\non chunk:\n  #{@chunk.inspect}" if @current_position == last_position
 
+        last_position = @current_position
         @current_position += result.to_i
       end
 
@@ -136,6 +141,8 @@ module Yap::Shell
         token *args
       end
 
+      token :BlankLine, str if @tokens.empty?
+
       @tokens
     end
 
@@ -161,6 +168,13 @@ module Yap::Shell
       end
     end
 
+    def comment_token
+      if md=@chunk.match(COMMENT)
+        token :Comment, md[0]
+        md[0].length
+      end
+    end
+
     def numerical_range_token
       return if @looking_for_args
 
@@ -311,10 +325,14 @@ module Yap::Shell
         prev_char = ''
         loop do
           ch = @chunk[characters_read]
+          # binding.pry
           if %w(' ").include?(ch)
             result = process_string @chunk[characters_read..-1], ch
             str << result.str
             characters_read += result.consumed_length
+          elsif ch == '\\'
+            # no-op
+            characters_read += 1
           elsif prev_char != '\\' && ch =~ /[\s\|;&\)\}]/
             break
           else

data/lib/yap/shell/parser/nodes.rb

@@ -6,6 +6,14 @@ module Yap::Shell
       end
     end
 
+    module NoOpNode
+      extend Visitor
+
+      def self.accept(*args)
+        # no-op
+      end
+    end
+
     class ArgumentNode
       include Visitor
 
@@ -77,6 +85,24 @@ module Yap::Shell
       end
     end
 
+    class CommentNode
+      include Visitor
+
+      attr_reader :text
+
+      def initialize(text)
+        @text = text
+      end
+
+      def inspect
+        to_s
+      end
+
+      def to_s
+        "CommentNode(#{@text})"
+      end
+    end
+
     class EnvNode
       include Visitor
 

data/lib/yap/shell/parser/version.rb

@@ -3,7 +3,7 @@ require 'yap/shell/parser'
 module Yap
   module Shell
     module Parser
-      VERSION = "0.3.1"
+      VERSION = "0.4.0"
     end
   end
 end

data/lib/yap/shell/parser_impl.rb

@@ -21,7 +21,7 @@ module Yap
   module Shell
     class ParserImpl < Racc::Parser
 
-module_eval(<<'...end grammar.y/module_eval...', 'grammar.y', 115)
+module_eval(<<'...end grammar.y/module_eval...', 'grammar.y', 125)
   include Yap::Shell::Parser::Nodes
 
 #=end
@@ -45,122 +45,131 @@ module_eval(<<'...end grammar.y/module_eval...', 'grammar.y', 115)
 ##### State transition tables begin ###
 
 racc_action_table = [
-    21,    22,    25,    25,    23,    28,    14,    52,    33,    19,
-    25,    14,    55,     7,    21,    22,    51,     9,    23,    54,
-    59,    21,    22,    19,    25,    14,    25,     7,    21,    22,
-    38,     9,    23,    35,    57,    52,    60,    19,    36,    14,
-    26,     7,    21,    22,    26,     9,    23,    52,    27,    25,
-    42,    19,    27,    14,    21,    22,    53,    29,    23,     9,
-    29,    33,    25,    19,    33,    14,    24,     7,    21,    22,
-    49,     9,    23,    14,    39,    28,   nil,    19,   nil,    14,
-   nil,     7,    21,    22,   nil,     9,    23,   nil,   nil,   nil,
-   nil,    19,   nil,    14,    21,    22,   nil,   nil,    23,     9,
-   nil,   nil,   nil,    19,   nil,    14,   nil,     7,    21,    22,
-   nil,     9,    23,   nil,   nil,   nil,   nil,    19,   nil,    14,
-   nil,     7,    21,    22,   nil,     9,    23,   nil,   nil,   nil,
-   nil,    19,   nil,    14,   nil,     7,   nil,   nil,    47,     9 ]
+    25,    26,    29,    17,    27,    29,    29,    29,    19,    23,
+    34,    17,    60,     8,    32,    64,    62,     3,    11,    12,
+    25,    26,    56,    29,    27,    29,    30,    40,    19,    23,
+    58,    17,    41,     8,    31,    65,    25,    26,    11,    12,
+    27,    57,    25,    26,    19,    23,    27,    17,    38,     8,
+    19,    23,    59,    17,    11,    12,    25,    26,    30,    57,
+    27,    12,    25,    26,    19,    23,    31,    17,    57,     8,
+    47,    43,    54,    33,    11,    12,    25,    26,    33,    38,
+    27,    29,    25,    26,    19,    23,    27,    17,    38,     8,
+    19,    23,    28,    17,    11,    12,    17,    44,    25,    26,
+    11,    12,    27,    32,   nil,   nil,    19,    23,   nil,    17,
+   nil,     8,   nil,   nil,    25,    26,    11,    12,    27,   nil,
+   nil,   nil,    19,    23,   nil,    17,   nil,     8,   nil,   nil,
+    25,    26,    11,    12,    27,   nil,   nil,   nil,    19,    23,
+   nil,    17,   nil,     8,   nil,   nil,    52,   nil,    11,    12 ]
 
 racc_action_check = [
-     0,     0,    46,    31,     0,    44,     8,    32,    13,     0,
-    56,     0,    46,     0,    49,    49,    31,     0,    49,    38,
-    56,    18,    18,    49,    48,    49,    58,    49,    47,    47,
-    18,    49,    47,    15,    48,    40,    58,    47,    15,    47,
-     3,    47,    29,    29,    43,    47,    29,    41,     3,    34,
-    24,    29,    43,    29,    28,    28,    34,     5,    28,    29,
-    45,    22,     2,    28,    21,    28,     1,    28,    14,    14,
-    28,    28,    14,    50,    19,     4,   nil,    14,   nil,    14,
-   nil,    14,    26,    26,   nil,    14,    26,   nil,   nil,   nil,
-   nil,    26,   nil,    26,     9,     9,   nil,   nil,     9,    26,
-   nil,   nil,   nil,     9,   nil,     9,   nil,     9,    25,    25,
-   nil,     9,    25,   nil,   nil,   nil,   nil,    25,   nil,    25,
-   nil,    25,    27,    27,   nil,    25,    27,   nil,   nil,   nil,
-   nil,    27,   nil,    27,   nil,    27,   nil,   nil,    27,    27 ]
+     0,     0,    51,     9,     0,    61,    53,    36,     0,     0,
+     9,     0,    51,     0,    49,    61,    53,     0,     0,     0,
+    54,    54,    36,    39,    54,    63,     4,    18,    54,    54,
+    39,    54,    18,    54,     4,    63,    52,    52,    54,    54,
+    52,    37,    33,    33,    52,    52,    33,    52,    16,    52,
+    33,    33,    43,    33,    52,    52,    32,    32,    48,    45,
+    32,    33,    22,    22,    32,    32,    48,    32,    46,    32,
+    28,    22,    32,     6,    32,    32,    17,    17,    50,    26,
+    17,     2,    30,    30,    17,    17,    30,    17,    25,    17,
+    30,    30,     1,    30,    17,    17,    55,    23,    12,    12,
+    30,    30,    12,     5,   nil,   nil,    12,    12,   nil,    12,
+   nil,    12,   nil,   nil,    29,    29,    12,    12,    29,   nil,
+   nil,   nil,    29,    29,   nil,    29,   nil,    29,   nil,   nil,
+    31,    31,    29,    29,    31,   nil,   nil,   nil,    31,    31,
+   nil,    31,   nil,    31,   nil,   nil,    31,   nil,    31,    31 ]
 
 racc_action_pointer = [
-    -2,    66,    55,    32,    59,    48,   nil,   nil,    -7,    92,
-   nil,   nil,   nil,     4,    66,    28,   nil,   nil,    19,    62,
-   nil,    60,    57,   nil,    50,   106,    80,   120,    52,    40,
-   nil,    -4,     3,   nil,    42,   nil,   nil,   nil,     7,   nil,
-    31,    43,   nil,    36,   -11,    51,    -5,    26,    17,    12,
-    60,   nil,   nil,   nil,   nil,   nil,     3,   nil,    19,   nil,
-   nil ]
+    -2,    92,    74,   nil,    18,    87,    64,   nil,   nil,   -10,
+   nil,   nil,    96,   nil,   nil,   nil,    44,    74,    22,   nil,
+   nil,   nil,    60,    85,   nil,    84,    75,   nil,    70,   112,
+    80,   128,    54,    40,   nil,   nil,     0,    37,   nil,    16,
+   nil,   nil,   nil,    40,   nil,    55,    64,   nil,    50,    -2,
+    69,    -5,    34,    -1,    18,    83,   nil,   nil,   nil,   nil,
+   nil,    -2,   nil,    18,   nil,   nil ]
 
 racc_action_default = [
-   -40,   -40,    -1,    -3,    -4,    -6,    -7,   -12,   -14,   -40,
-   -17,   -18,   -19,   -21,   -40,   -24,   -26,   -27,   -28,   -40,
-   -32,   -33,   -35,   -39,   -40,   -40,   -40,   -40,   -40,   -40,
-   -16,   -40,   -20,   -37,   -40,   -23,   -25,   -29,   -40,   -31,
-   -34,   -36,    61,    -2,   -40,    -5,   -40,   -40,   -40,   -40,
-   -13,   -15,   -38,   -22,   -30,   -10,   -40,    -8,   -40,   -11,
-    -9 ]
+   -45,   -45,    -1,    -2,    -4,    -5,    -7,    -8,   -13,   -15,
+   -16,   -18,   -45,   -21,   -22,   -23,   -25,   -45,   -28,   -29,
+   -31,   -32,   -33,   -45,   -37,   -38,   -40,   -44,   -45,   -45,
+   -45,   -45,   -45,   -45,   -17,   -20,   -45,   -24,   -42,   -45,
+   -27,   -30,   -34,   -45,   -36,   -39,   -41,    66,    -3,   -45,
+    -6,   -45,   -45,   -45,   -45,   -14,   -19,   -43,   -26,   -35,
+   -11,   -45,    -9,   -45,   -12,   -10 ]
 
 racc_goto_table = [
-     2,    30,    32,    45,    44,    50,    43,     1,    37,    31,
-    40,    41,   nil,   nil,    34,   nil,   nil,   nil,   nil,   nil,
-   nil,   nil,   nil,   nil,   nil,   nil,   nil,    46,    48,   nil,
+     2,    35,    50,    49,    37,    55,    48,     1,    42,   nil,
+   nil,   nil,    36,    45,    46,   nil,   nil,    39,   nil,   nil,
    nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
-   nil,   nil,   nil,    30,   nil,   nil,   nil,    56,   nil,    58 ]
+   nil,    51,    53,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,   nil,   nil,   nil,   nil,   nil,   nil,    35,   nil,   nil,
+   nil,   nil,    61,   nil,    63 ]
 
 racc_goto_check = [
-     2,     8,    12,     5,     4,     7,     3,     1,    15,     2,
-    12,    12,   nil,   nil,     2,   nil,   nil,   nil,   nil,   nil,
-   nil,   nil,   nil,   nil,   nil,   nil,   nil,     2,     2,   nil,
+     2,     9,     5,     4,    13,     7,     3,     1,    16,   nil,
+   nil,   nil,     2,    13,    13,   nil,   nil,     2,   nil,   nil,
    nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
-   nil,   nil,   nil,     8,   nil,   nil,   nil,     2,   nil,     2 ]
+   nil,     2,     2,   nil,   nil,   nil,   nil,   nil,   nil,   nil,
+   nil,   nil,   nil,   nil,   nil,   nil,   nil,     9,   nil,   nil,
+   nil,   nil,     2,   nil,     2 ]
 
 racc_goto_pointer = [
-   nil,     7,     0,   -19,   -21,   -23,   nil,   -24,    -7,   nil,
-   nil,   nil,   -11,   nil,   nil,   -10,   nil,   nil ]
+   nil,     7,     0,   -23,   -26,   -28,   nil,   -28,   nil,    -8,
+   nil,   nil,   nil,   -12,   nil,   nil,   -14,   nil,   nil ]
 
 racc_goto_default = [
-   nil,   nil,   nil,     3,     4,     5,     6,     8,    10,    11,
-    12,    13,   nil,    15,    16,    17,    18,    20 ]
+   nil,   nil,   nil,     4,     5,     6,     7,     9,    10,    13,
+    14,    15,    16,   nil,    18,    20,    21,    22,    24 ]
 
 racc_reduce_table = [
   0, 0, :racc_error,
-  1, 22, :_reduce_none,
-  3, 23, :_reduce_2,
-  1, 23, :_reduce_3,
-  1, 23, :_reduce_none,
-  3, 24, :_reduce_5,
   1, 24, :_reduce_none,
-  1, 24, :_reduce_none,
-  4, 27, :_reduce_8,
-  5, 27, :_reduce_9,
-  4, 27, :_reduce_10,
-  5, 27, :_reduce_11,
-  1, 25, :_reduce_12,
-  3, 26, :_reduce_13,
+  1, 24, :_reduce_2,
+  3, 25, :_reduce_3,
+  1, 25, :_reduce_4,
+  1, 25, :_reduce_none,
+  3, 26, :_reduce_6,
   1, 26, :_reduce_none,
-  3, 28, :_reduce_15,
-  2, 28, :_reduce_16,
-  1, 28, :_reduce_none,
+  1, 26, :_reduce_none,
+  4, 29, :_reduce_9,
+  5, 29, :_reduce_10,
+  4, 29, :_reduce_11,
+  5, 29, :_reduce_12,
+  1, 27, :_reduce_13,
+  3, 28, :_reduce_14,
   1, 28, :_reduce_none,
   1, 28, :_reduce_none,
-  2, 29, :_reduce_20,
-  1, 29, :_reduce_none,
-  3, 32, :_reduce_22,
-  2, 30, :_reduce_23,
+  2, 31, :_reduce_17,
+  1, 31, :_reduce_18,
+  3, 30, :_reduce_19,
+  2, 30, :_reduce_20,
+  1, 30, :_reduce_none,
+  1, 30, :_reduce_none,
   1, 30, :_reduce_none,
-  2, 34, :_reduce_25,
-  1, 34, :_reduce_none,
-  1, 34, :_reduce_none,
-  1, 34, :_reduce_none,
-  2, 35, :_reduce_29,
-  3, 37, :_reduce_30,
-  2, 37, :_reduce_31,
-  1, 36, :_reduce_none,
-  1, 38, :_reduce_33,
+  2, 32, :_reduce_24,
+  1, 32, :_reduce_none,
+  3, 35, :_reduce_26,
+  2, 33, :_reduce_27,
+  1, 33, :_reduce_none,
+  1, 33, :_reduce_29,
+  2, 37, :_reduce_30,
+  1, 37, :_reduce_none,
+  1, 37, :_reduce_none,
+  1, 37, :_reduce_none,
   2, 38, :_reduce_34,
-  1, 38, :_reduce_35,
-  2, 38, :_reduce_36,
-  1, 33, :_reduce_37,
-  2, 33, :_reduce_38,
-  1, 31, :_reduce_39 ]
+  3, 40, :_reduce_35,
+  2, 40, :_reduce_36,
+  1, 39, :_reduce_none,
+  1, 41, :_reduce_38,
+  2, 41, :_reduce_39,
+  1, 41, :_reduce_40,
+  2, 41, :_reduce_41,
+  1, 36, :_reduce_42,
+  2, 36, :_reduce_43,
+  1, 34, :_reduce_44 ]
 
-racc_reduce_n = 40
+racc_reduce_n = 45
 
-racc_shift_n = 61
+racc_shift_n = 66
 
 racc_token_table = {
   false => 0,
@@ -182,10 +191,12 @@ racc_token_table = {
   :BlockBegin => 16,
   :BlockEnd => 17,
   :BlockParams => 18,
-  "(" => 19,
-  ")" => 20 }
+  :BlankLine => 19,
+  :Comment => 20,
+  "(" => 21,
+  ")" => 22 }
 
-racc_nt_base = 21
+racc_nt_base = 23
 
 racc_use_result_var = true
 
@@ -225,6 +236,8 @@ Racc_token_to_s_table = [
   "BlockBegin",
   "BlockEnd",
   "BlockParams",
+  "BlankLine",
+  "Comment",
   "\"(\"",
   "\")\"",
   "$start",
@@ -235,6 +248,7 @@ Racc_token_to_s_table = [
   "pipeline",
   "block_stmt",
   "stmts2",
+  "stmts2_w_comment",
   "stmt_w_substitutions",
   "command_w_heredoc",
   "internal_eval",
@@ -254,202 +268,232 @@ Racc_debug_parser = false
 
 # reduce 1 omitted
 
-module_eval(<<'.,.,', 'grammar.y', 23)
+module_eval(<<'.,.,', 'grammar.y', 22)
   def _reduce_2(val, _values, result)
-     result = StatementsNode.new(val[0], val[2]) 
+     result = NoOpNode 
     result
   end
 .,.,
 
 module_eval(<<'.,.,', 'grammar.y', 25)
   def _reduce_3(val, _values, result)
+     result = StatementsNode.new(val[0], val[2]) 
+    result
+  end
+.,.,
+
+module_eval(<<'.,.,', 'grammar.y', 27)
+  def _reduce_4(val, _values, result)
      result = StatementsNode.new(val[0]) 
     result
   end
 .,.,
 
-# reduce 4 omitted
+# reduce 5 omitted
 
-module_eval(<<'.,.,', 'grammar.y', 29)
-  def _reduce_5(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 31)
+  def _reduce_6(val, _values, result)
      result = ConditionalNode.new(val[1].value, val[0], val[2]) 
     result
   end
 .,.,
 
-# reduce 6 omitted
-
 # reduce 7 omitted
 
-module_eval(<<'.,.,', 'grammar.y', 34)
-  def _reduce_8(val, _values, result)
-     result = val[0].tap { |range_node| range_node.tail = BlockNode.new(nil, val[2]) } 
-    result
-  end
-.,.,
+# reduce 8 omitted
 
 module_eval(<<'.,.,', 'grammar.y', 36)
   def _reduce_9(val, _values, result)
-     result = val[0].tap { |range_node| range_node.tail = BlockNode.new(nil, val[3], params: val[2].value) } 
+     result = val[0].tap { |range_node| range_node.tail = BlockNode.new(nil, val[2]) } 
     result
   end
 .,.,
 
 module_eval(<<'.,.,', 'grammar.y', 38)
   def _reduce_10(val, _values, result)
-     result = BlockNode.new(val[0], val[2]) 
+     result = val[0].tap { |range_node| range_node.tail = BlockNode.new(nil, val[3], params: val[2].value) } 
     result
   end
 .,.,
 
 module_eval(<<'.,.,', 'grammar.y', 40)
   def _reduce_11(val, _values, result)
-     result = BlockNode.new(val[0], val[3], params: val[2].value) 
+     result = BlockNode.new(val[0], val[2]) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 43)
+module_eval(<<'.,.,', 'grammar.y', 42)
   def _reduce_12(val, _values, result)
-     result = RangeNode.new(val[0]) 
+     result = BlockNode.new(val[0], val[3], params: val[2].value) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 46)
+module_eval(<<'.,.,', 'grammar.y', 45)
   def _reduce_13(val, _values, result)
+     result = RangeNode.new(val[0]) 
+    result
+  end
+.,.,
+
+module_eval(<<'.,.,', 'grammar.y', 48)
+  def _reduce_14(val, _values, result)
      result = PipelineNode.new(val[0], val[2]) 
     result
   end
 .,.,
 
-# reduce 14 omitted
+# reduce 15 omitted
+
+# reduce 16 omitted
+
+module_eval(<<'.,.,', 'grammar.y', 53)
+  def _reduce_17(val, _values, result)
+     result = StatementsNode.new(val[0], CommentNode.new(val[1])) 
+    result
+  end
+.,.,
+
+module_eval(<<'.,.,', 'grammar.y', 55)
+  def _reduce_18(val, _values, result)
+     result = CommentNode.new(val[0]) 
+    result
+  end
+.,.,
 
-module_eval(<<'.,.,', 'grammar.y', 50)
-  def _reduce_15(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 58)
+  def _reduce_19(val, _values, result)
      result = val[1] 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 52)
-  def _reduce_16(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 60)
+  def _reduce_20(val, _values, result)
      result = ConcatenationNode.new(val[0], val[1]) 
     result
   end
 .,.,
 
-# reduce 17 omitted
+# reduce 21 omitted
 
-# reduce 18 omitted
+# reduce 22 omitted
 
-# reduce 19 omitted
+# reduce 23 omitted
 
-module_eval(<<'.,.,', 'grammar.y', 58)
-  def _reduce_20(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 66)
+  def _reduce_24(val, _values, result)
      result = val[0] ; val[0].tail = val[1] 
     result
   end
 .,.,
 
-# reduce 21 omitted
+# reduce 25 omitted
 
-module_eval(<<'.,.,', 'grammar.y', 62)
-  def _reduce_22(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 70)
+  def _reduce_26(val, _values, result)
      result = CommandSubstitutionNode.new(val[1]) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 65)
-  def _reduce_23(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 73)
+  def _reduce_27(val, _values, result)
      val[0].heredoc = val[1] ; result = val[0] 
     result
   end
 .,.,
 
-# reduce 24 omitted
+# reduce 28 omitted
 
-module_eval(<<'.,.,', 'grammar.y', 69)
-  def _reduce_25(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 76)
+  def _reduce_29(val, _values, result)
+     result = RedirectionNode.new(val[0].value, val[0].attrs[:target]) 
+    result
+  end
+.,.,
+
+module_eval(<<'.,.,', 'grammar.y', 79)
+  def _reduce_30(val, _values, result)
      val[0].redirects << RedirectionNode.new(val[1].value, val[1].attrs[:target]) ; result = val[0] 
     result
   end
 .,.,
 
-# reduce 26 omitted
+# reduce 31 omitted
 
-# reduce 27 omitted
+# reduce 32 omitted
 
-# reduce 28 omitted
+# reduce 33 omitted
 
-module_eval(<<'.,.,', 'grammar.y', 75)
-  def _reduce_29(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 85)
+  def _reduce_34(val, _values, result)
      result = EnvWrapperNode.new(val[0], val[1]) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 78)
-  def _reduce_30(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 88)
+  def _reduce_35(val, _values, result)
      val[0].add_var(val[1].value, val[2].value) ; result = val[0] 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 80)
-  def _reduce_31(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 90)
+  def _reduce_36(val, _values, result)
      result = EnvNode.new(val[0].value, val[1].value) 
     result
   end
 .,.,
 
-# reduce 32 omitted
+# reduce 37 omitted
 
-module_eval(<<'.,.,', 'grammar.y', 85)
-  def _reduce_33(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 95)
+  def _reduce_38(val, _values, result)
      result = CommandNode.new(val[0].value) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 87)
-  def _reduce_34(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 97)
+  def _reduce_39(val, _values, result)
      result = CommandNode.new(val[0].value, val[1].flatten) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 89)
-  def _reduce_35(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 99)
+  def _reduce_40(val, _values, result)
      result = CommandNode.new(val[0].value, literal:true) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 91)
-  def _reduce_36(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 101)
+  def _reduce_41(val, _values, result)
      result = CommandNode.new(val[0].value, val[1].flatten, literal:true) 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 94)
-  def _reduce_37(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 104)
+  def _reduce_42(val, _values, result)
      result = [ArgumentNode.new(val[0].value)] 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 96)
-  def _reduce_38(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 106)
+  def _reduce_43(val, _values, result)
      result = [val[0], ArgumentNode.new(val[1].value)].flatten 
     result
   end
 .,.,
 
-module_eval(<<'.,.,', 'grammar.y', 99)
-  def _reduce_39(val, _values, result)
+module_eval(<<'.,.,', 'grammar.y', 109)
+  def _reduce_44(val, _values, result)
      result = InternalEvalNode.new(val[0].value) 
     result
   end

data/spec/yap/shell/lexer_spec.rb

@@ -9,6 +9,37 @@ describe Yap::Shell::Parser::Lexer do
     [tag, Yap::Shell::Parser::Lexer::Token.new(tag, val, lineno:lineno, attrs:attrs)]
   end
 
+  describe "blank lines" do
+    describe "" do
+      it { should eq [
+        t(:BlankLine, "", lineno:0)
+      ]}
+    end
+
+    describe "     " do
+      it { should eq [
+        t(:BlankLine, "     ", lineno:0)
+      ]}
+    end
+  end
+
+  describe "comments" do
+    describe "hash symbol and everything there-after is a comment" do
+      describe "#this is a comment" do
+        it { should eq [
+          t(:Comment, "#this is a comment", lineno:0)
+        ]}
+      end
+
+      describe "ls #this last part is a comment" do
+        it { should eq [
+          t(:Command, "ls", lineno:0),
+          t(:Comment, "#this last part is a comment", lineno:0)
+        ]}
+      end
+    end
+  end
+
   describe "block expressions" do
     describe "can follow any command" do
       describe "ls { this_is_a_block }" do
@@ -20,7 +51,7 @@ describe Yap::Shell::Parser::Lexer do
         ]}
       end
 
-      describe "whitespace before/after the range doesn't matter" do
+      describe "whitespace is required before the start of the block, but not after" do
         describe "   ls        {     this_is_a_block    }   " do
           it { should eq [
             t(:Command, "ls", lineno:0),
@@ -30,7 +61,7 @@ describe Yap::Shell::Parser::Lexer do
           ]}
         end
 
-        describe "ls {echo n}" do
+        describe "ls { echo n }" do
           it { should eq [
             t(:Command, "ls", lineno:0),
             t(:BlockBegin, '{', lineno: 0),
@@ -39,6 +70,22 @@ describe Yap::Shell::Parser::Lexer do
             t(:BlockEnd, '}', lineno: 0)
           ]}
         end
+
+        describe "ls {echo n }" do
+          it { should eq [
+            t(:Command, "ls", lineno:0),
+            t(:BlockBegin, '{', lineno: 0),
+            t(:Command, "echo", lineno:0),
+            t(:Argument, "n", lineno:0),
+            t(:BlockEnd, '}', lineno: 0)
+          ]}
+        end
+
+        describe "ls s{ echo n}" do
+          it "fails to lex" do
+            expect { subject }.to raise_error
+          end
+        end
       end
     end
 
@@ -278,11 +325,6 @@ describe Yap::Shell::Parser::Lexer do
     end
   end
 
-  describe "empty string" do
-    let(:str){ "" }
-    it { should eq [] }
-  end
-
   describe "string with newlines" do
     let(:str){ "ls \n\nlib:\ntasks\nyap\nyap.rb" }
     it { should eq [
@@ -404,11 +446,11 @@ describe Yap::Shell::Parser::Lexer do
       ]}
     end
 
-    describe "commands with a escaped args: ls some\ dir" do
+    describe "commands with escaped spaces: ls some\ dir" do
       let(:str){ 'ls some\ dir' }
       it { should eq [
         t(:Command, "ls", lineno:0),
-        t(:Argument, 'some\ dir', lineno:0)
+        t(:Argument, 'some dir', lineno:0)
       ]}
     end
 
@@ -878,6 +920,20 @@ describe Yap::Shell::Parser::Lexer do
           t(:Redirection, ">", lineno: 0, attrs: { target: "a.txt" }),
         ]}
       end
+
+      describe "can be the start of a command (for clearing files)" do
+        describe "> a.txt" do
+          it { should eq [
+            t(:Redirection, ">", lineno: 0, attrs: { target: "a.txt" })
+          ]}
+        end
+
+        describe ">foo.txt" do
+          it { should eq [
+            t(:Redirection, ">", lineno: 0, attrs: { target: "foo.txt" })
+          ]}
+        end
+      end
     end
 
     describe "stderr" do

data/spec/yap/shell/parser_spec.rb

@@ -14,6 +14,8 @@ describe Yap::Shell::Parser do
     end
   end
 
+  it { is_expected.to parse "" }
+  it { is_expected.to parse " " }
   it { is_expected.to parse "ls" }
   it { is_expected.to parse "echo foo" }
   it { is_expected.to parse "echo foo ; echo bar baz yep" }
@@ -52,9 +54,13 @@ describe Yap::Shell::Parser do
   it { is_expected.to parse "(0..3)" }
   it { is_expected.to parse "(0..3): echo hi" }
   it { is_expected.to parse "(0..3) as n: echo hi $n" }
-  it { is_expected.to parse "echo hi ; (0..3) {echo hi $n}" }
+  it { is_expected.to parse "echo hi ; (0..3) {echo hi $n }" }
   it { is_expected.to parse "echo hi ; (0..3) { echo hi $n } ; echo bye" }
   it { is_expected.to parse "ls *.rb { |f,g,h| echo $f && echo $h && echo $i }" }
+  it { is_expected.to parse ">foo.txt" }
+  it { is_expected.to parse "echo bar && > foo.txt" }
+  it { is_expected.to parse "#this is a comment" }
+  it { is_expected.to parse "echo foo #this last part is a comment" }
 
   it { is_expected.to fail_parsing "ls ()" }
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yap-shell-parser
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Zach Dennis
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-12 00:00:00.000000000 Z
+date: 2016-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,3 +122,4 @@ test_files:
 - spec/spec_helper.rb
 - spec/yap/shell/lexer_spec.rb
 - spec/yap/shell/parser_spec.rb
+has_rdoc: