yap-shell-parser 0.2.2 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8e9067d2b8034ad2ca424fbdb61b5c939be1470e
-  data.tar.gz: 1ae4cf2f162e5c19bd8419619034759e069836d5
+  metadata.gz: b1f871f25358079c9bb5bfda0d59b77504b36b88
+  data.tar.gz: ae41ab9e03b55318f48c9fd739448600c30e545c
 SHA512:
-  metadata.gz: 7bfc31b24ff9925175ded8555a3a8462c4d88fca804f035432c572c2a623069fe370437fa8e14097494fc53e40eccfe97396034763ec7c476a8326d8a99c455d
-  data.tar.gz: 8534df6ecc65ec92307d491e3dd760ac3129cc3808df6e2f07b2d8a733cafc1bfce0ecb56148c6ea4fa40d5233095af6fb28740109d24b7eb9fee43429ebf1ba
+  metadata.gz: 2b6dc9b5ebfee853ef445c1f8a60ed9e9ca8d81365497f2ae5fda0e64721ffa215c451ebd1cd95c8485759fa918d0085c7b8dfc75a1cbf7be823d2969e6f8356
+  data.tar.gz: fd2670a3b93ec221a664a71e0280ef79b5cf25f7a968abde4d2655188f89c5bbcf2de784f574f3a8ecd4355ab007ea1fcadc7d7253477b12e02962d0e37a4c34

data/.ruby-version

@@ -0,0 +1 @@
+2.2.3

data/lib/yap/shell/parser/grammar.y

@@ -3,7 +3,7 @@
 # convert Array-like string into Ruby's Array.
 
 class Yap::Shell::ParserImpl
-  token Command LiteralCommand Argument Heredoc InternalEval Separator Conditional Pipe Redirection LValue RValue BeginCommandSubstitution EndCommandSubstitution
+  token Command LiteralCommand Argument Heredoc InternalEval Separator Conditional Pipe Redirection LValue RValue BeginCommandSubstitution EndCommandSubstitution Range BlockBegin BlockEnd BlockParams
   #
   # prechigh
   # #   left '**' '*' '/' '%'
@@ -24,10 +24,24 @@ stmts : stmts Separator stmt
     { result = StatementsNode.new(val[0], val[2]) }
   | stmt
     { result = StatementsNode.new(val[0]) }
+  | range_stmt
 
 stmt : stmt Conditional pipeline
     { result = ConditionalNode.new(val[1].value, val[0], val[2]) }
   | pipeline
+  | block_stmt
+
+block_stmt : range_stmt BlockBegin stmts BlockEnd
+    { result = val[0].tap { |range_node| range_node.tail = BlockNode.new(nil, val[2]) } }
+  | range_stmt BlockBegin BlockParams stmts BlockEnd
+    { result = val[0].tap { |range_node| range_node.tail = BlockNode.new(nil, val[3], params: val[2].value) } }
+  | stmt BlockBegin stmts BlockEnd
+    { result = BlockNode.new(val[0], val[2]) }
+  | stmt BlockBegin BlockParams stmts BlockEnd
+    { result = BlockNode.new(val[0], val[3], params: val[2].value) }
+
+range_stmt : Range
+    { result = RangeNode.new(val[0]) }
 
 pipeline : pipeline Pipe stmts2
     { result = PipelineNode.new(val[0], val[2]) }
@@ -105,6 +119,7 @@ end
     @yydebug = true
 
     @q = Yap::Shell::Parser::Lexer.new.tokenize(str)
+    pp @q if ENV["DEBUG"]
     # @q.push [false, '$']   # is optional from Racc 1.3.7
 # puts @q.inspect
 # puts "---- parse tree follows ----"
@@ -119,6 +134,7 @@ end
 ---- footer
 
 if $0 == __FILE__
+  require 'pry'
   $LOAD_PATH.unshift File.dirname(__FILE__) + "/lib/"
   require 'yap/shell/parser/lexer'
   require 'yap/shell/parser/nodes'
@@ -127,7 +143,9 @@ if $0 == __FILE__
     # "`git cbranch`",
     # "`git cbranch`.bak",
     # "echo `echo hi`",
-    "echo `echo hi` foo bar baz",
+    # "ls *.rb te* { |f| f }",
+    # "f { |a, b, c| echo foo }"
+    "(0..3) as n : echo hi",
     # "`hi``bye` `what`",
     # "echo && `what` && where is `that`thing | `you know`",
     ].each do |src|

data/lib/yap/shell/parser/lexer.rb

@@ -2,6 +2,9 @@ require 'ostruct'
 
 module Yap::Shell
   class Parser::Lexer
+    class Error < ::StandardError ; end
+    class HeredocMissingEndDelimiter < Error ; end
+
     class Token
       include Comparable
 
@@ -37,18 +40,29 @@ module Yap::Shell
     ARG                    = /[^\s;\|\(\)\{\}\[\]\&\!\\\<`][^\s;\|\(\)\{\}\[\]\&\>\<`]*/
     COMMAND                = /\A(#{ARG})/
     LITERAL_COMMAND        = /\A\\(#{ARG})/
-    WHITESPACE             = /\A[^\n\S]+/
+    WHITESPACE             = /\A\s+/
     LH_ASSIGNMENT          = /\A(([A-z_][\w]*)=)/
     RH_VALUE               = /\A(\S+)/
     STATEMENT_TERMINATOR   = /\A(;)/
     PIPE_TERMINATOR        = /\A(\|)/
     CONDITIONAL_TERMINATOR = /\A(&&|\|\|)/
-    HEREDOC                = /\A<<-?([A-z0-9]+)\s*^(.*)?(^\s*\1\s*$)/m
+    HEREDOC_START          = /\A<<-?([A-z0-9]+)\s*\n/
     INTERNAL_EVAL          = /\A(?:(\!)|([0-9]+))/
     SUBGROUP               = /\A(\(|\))/
     REDIRECTION            = /\A(([12]?>&?[12]?)\s*(?![12]>)(#{ARG})?)/
     REDIRECTION2           = /\A((&>|<)\s*(#{ARG}))/
 
+    NUMERIC_RANGE              = /\A\(((\d+)\.\.(\d+))\)(\.each)?/
+    NUMERIC_RANGE_W_CALL      = /\A\(((\d+)\.\.(\d+))\)(\.each)?\s*:\s*/
+    NUMERIC_RANGE_W_PARAM      = /\A(\((\d+)\.\.(\d+))\)\s+as\s+([A-z0-9,\s]+)\s*:\s*/
+    NUMERIC_REPETITION         = /\A((\d+)(\.times))\s*/
+    NUMERIC_REPETITION_2       = /\A((\d+)(\.times))\s*:\s*/
+    NUMERIC_REPETITION_W_PARAM = /\A((\d+)(\.times))\s+as\s+([A-z0-9,\s]+)\s*:\s*/
+
+    BLOCK_BEGIN = /\A\s*(\{)\s*(?:\|\s*([A-z0-9,\s]+)\s*\|)?/
+    BLOCK_END = /\A\s*(\})\s*/
+
+    SPLIT_BLOCK_PARAMS_RGX = /\s*,\s*|\s*/
 
     # Loop over the given input and yield command substitutions. This yields
     # an object that responds to #str, and #position.
@@ -88,6 +102,7 @@ module Yap::Shell
       @tokens = []
       @lineno = 0
       @looking_for_args = false
+      @tokens_to_add_when_done = []
 
       max = 100
       count = 0
@@ -96,6 +111,8 @@ module Yap::Shell
 
       while process_next_chunk.call
         result =
+          block_token ||
+          numerical_range_token ||
           command_substitution_token ||
           subgroup_token ||
           assignment_token ||
@@ -115,6 +132,10 @@ module Yap::Shell
         @current_position += result.to_i
       end
 
+      @tokens_to_add_when_done.each do |args|
+        token *args
+      end
+
       @tokens
     end
 
@@ -124,6 +145,71 @@ module Yap::Shell
       @tokens.push [tag, Token.new(tag, value, lineno:@lineno, attrs:attrs)]
     end
 
+    def block_token
+      if md=@chunk.match(BLOCK_BEGIN)
+        @looking_for_args = false
+        token :BlockBegin, md[1]
+        if md[2]
+          params = md[2].split(SPLIT_BLOCK_PARAMS_RGX)
+          token :BlockParams, params
+        end
+        md[0].length
+      elsif md=@chunk.match(BLOCK_END)
+        @looking_for_args = false
+        token :BlockEnd, md[1]
+        md[0].length
+      end
+    end
+
+    def numerical_range_token
+      return if @looking_for_args
+
+      if md=@chunk.match(NUMERIC_RANGE_W_CALL)
+        start, stop = md[2].to_i, md[3].to_i
+        token :Range, (start..stop)
+        token :BlockBegin, '{'
+        @tokens_to_add_when_done << [:BlockEnd, '}']
+        md[0].length
+
+      elsif md=@chunk.match(NUMERIC_RANGE_W_PARAM)
+        start, stop = md[2].to_i, md[3].to_i
+        token :Range, (start..stop)
+        token :BlockBegin, '{'
+        params = md[4].split(SPLIT_BLOCK_PARAMS_RGX)
+        token :BlockParams, params
+        @tokens_to_add_when_done << [:BlockEnd, '}']
+        md[0].length
+
+      elsif md=@chunk.match(NUMERIC_REPETITION_2)
+        start, stop = 1, md[2].to_i
+        token :Range, (start..stop)
+        token :BlockBegin, '{'
+        @tokens_to_add_when_done << [:BlockEnd, '}']
+        md[0].length
+
+      elsif md=@chunk.match(NUMERIC_REPETITION_W_PARAM)
+        start, stop = 1, md[2].to_i
+        token :Range, (start..stop)
+        token :BlockBegin, '{'
+        params = md[4].split(SPLIT_BLOCK_PARAMS_RGX)
+        token :BlockParams, params
+        @tokens_to_add_when_done << [:BlockEnd, '}']
+        md[0].length
+
+      elsif md=@chunk.match(NUMERIC_REPETITION)
+        start, stop = 1, md[2].to_i
+        token :Range, (start..stop)
+        md[0].length
+
+      elsif md=@chunk.match(NUMERIC_RANGE)
+        start, stop = md[2].to_i, md[3].to_i
+        token :Range, (start..stop)
+        md[0].length
+
+      end
+
+    end
+
     def command_token
       if !@looking_for_args && md=@chunk.match(COMMAND)
         @looking_for_args = true
@@ -149,9 +235,31 @@ module Yap::Shell
     end
 
     def heredoc_token
-      if md=@chunk.match(HEREDOC)
-        token :Heredoc, md[2]
-        md[0].length
+      if md=@chunk.match(HEREDOC_START)
+        delimiter = md[1]
+        str = @chunk[md[0].length..-1]
+        consumed_length = md[0].length
+
+        delimeter_regex = Regexp.escape(delimiter)
+
+        contents = ""
+        found_ending_delimiter = false
+        str.lines.each do |line|
+          if md=line.match(/^(.*?)\s*#{delimeter_regex}\s*$/m)
+            contents << $1
+            found_ending_delimiter = true
+          else
+            contents << line
+          end
+          consumed_length += line.length
+        end
+
+        unless found_ending_delimiter
+          raise HeredocMissingEndDelimiter, "Missing end delimiter on #{@chunk}"
+        end
+
+        token :Heredoc, contents
+        consumed_length
       end
     end
 
@@ -207,7 +315,7 @@ module Yap::Shell
             result = process_string @chunk[characters_read..-1], ch
             str << result.str
             characters_read += result.consumed_length
-          elsif prev_char != '\\' && ch =~ /[\s\|;&\)]/
+          elsif prev_char != '\\' && ch =~ /[\s\|;&\)\}]/
             break
           else
             str << ch

data/lib/yap/shell/parser/nodes.rb

@@ -258,5 +258,47 @@ module Yap::Shell
       end
     end
 
+    class BlockNode
+      include Visitor
+
+      attr_accessor :head, :tail, :params
+
+      def initialize(head, tail, params: [])
+        @head = head
+        @tail = tail
+        @params = params
+      end
+
+      def to_s(indent:0)
+        if @counter_reference
+          "BlockNode(#{@head.inspect}, tail: #{@tail.inspect} params: #{@params.inspect})"
+        else
+          "BlockNode(#{@head.inspect}, tail: #{@tail.inspect} params: #{@params.inspect})"
+        end
+      end
+
+      def inspect
+        to_s
+      end
+    end
+
+    class RangeNode
+      include Visitor
+
+      attr_accessor :head, :tail
+
+      def initialize(head, tail=nil)
+        @head = head
+        @tail = tail
+      end
+
+      def to_s(indent:0)
+        "(#{@head.inspect}, tail: #{tail.inspect})"
+      end
+
+      def inspect
+        to_s
+      end
+    end
   end
 end

data/lib/yap/shell/parser/version.rb

@@ -3,7 +3,7 @@ require 'yap/shell/parser'
 module Yap
   module Shell
     module Parser
-      VERSION = "0.2.2"
+      VERSION = "0.3.1"
     end
   end
 end