riml 0.1.5 → 0.1.6

data/bin/riml

@@ -22,12 +22,14 @@ module Riml
         opts.separator ""
         opts.separator "Specific options:"
 
-        opts.on("-c", "--compile FILE", "Compile riml file to VimL.") do |file|
-          if File.exists?(file)
-            options.compile_files << file
-          else
-            warn "Couldn't find file #{file.inspect}."
-            exit 1
+        opts.on("-c", "--compile FILES", Array, "Compile riml file(s) to VimL.") do |files|
+          files.each do |f|
+            if File.exists?(f)
+              options.compile_files << f
+            else
+              warn "Couldn't find file #{f.inspect}."
+              exit 1
+            end
           end
         end
 
@@ -65,7 +67,7 @@ module Riml
         options = Options.parse(ARGV)
 
         if options.stdio
-          puts Riml.compile($stdin.gets)
+          puts Riml.compile($stdin.read)
         elsif options.compile_files.any?
           Riml.compile_files(*options.compile_files)
         end

data/lib/ast_rewriter.rb

@@ -7,25 +7,26 @@ module Riml
     include Riml::Constants
 
     attr_accessor :ast
-    def initialize(ast = nil)
+    attr_reader   :classes
+
+    def initialize(ast = nil, classes = nil)
       @ast = ast
+      @classes = classes || ClassMap.new
     end
 
-    # Map of {"ClassName" => ClassDefinitionNode}
-    # Can also query object for superclass of a named class, etc...
-    #
-    # Ex : classes["SomeClass"].superclass_name => "SomeClassBase"
-    def classes
-      @@classes ||= ClassMap.new
-    end
 
     def rewrite
       establish_parents(ast)
-      StrictEqualsComparisonOperator.new(ast).rewrite_on_match
-      VarEqualsComparisonOperator.new(ast).rewrite_on_match
-      ClassDefinitionToFunctions.new(ast).rewrite_on_match
-      ObjectInstantiationToCall.new(ast).rewrite_on_match
-      CallToExplicitCall.new(ast).rewrite_on_match
+      rewriters = [
+        StrictEqualsComparisonOperator.new(ast, classes),
+        VarEqualsComparisonOperator.new(ast, classes),
+        ClassDefinitionToFunctions.new(ast, classes),
+        ObjectInstantiationToCall.new(ast, classes),
+        CallToExplicitCall.new(ast, classes)
+      ]
+      rewriters.each do |rewriter|
+        rewriter.rewrite_on_match
+      end
       ast
     end
 
@@ -100,7 +101,7 @@ module Riml
         classes[node.name] = node
 
         name, expressions = node.name, node.expressions
-        InsertInitializeMethod.new(node).rewrite_on_match
+        InsertInitializeMethod.new(node, classes).rewrite_on_match
         constructor = node.constructor
         constructor.scope_modifier = 'g:' unless constructor.scope_modifier
         constructor.name = node.constructor_name
@@ -110,8 +111,8 @@ module Riml
           AssignNode.new('=', GetVariableNode.new(nil, dict_name), DictionaryNode.new({}))
         )
 
-        SuperToObjectExtension.new(constructor, node).rewrite_on_match
-        MethodToNestedFunction.new(node, constructor, dict_name).rewrite_on_match
+        SuperToObjectExtension.new(constructor, classes, node).rewrite_on_match
+        MethodToNestedFunction.new(node, classes, constructor, dict_name).rewrite_on_match
         SelfToDictName.new(dict_name).rewrite_on_match(constructor)
 
         constructor.expressions.push(
@@ -122,8 +123,8 @@ module Riml
 
       class MethodToNestedFunction < AST_Rewriter
         attr_reader :constructor, :dict_name
-        def initialize(class_node, constructor, dict_name)
-          super(class_node)
+        def initialize(class_node, classes, constructor, dict_name)
+          super(class_node, classes)
           @dict_name, @constructor = dict_name, constructor
         end
 
@@ -190,8 +191,8 @@ module Riml
 
       class SuperToObjectExtension < AST_Rewriter
         attr_reader :class_node
-        def initialize(constructor, class_node)
-          super(constructor)
+        def initialize(constructor, classes, class_node)
+          super(constructor, classes)
           @class_node = class_node
         end
 

data/lib/class_map.rb

@@ -1,6 +1,10 @@
 module Riml
   class ClassNotFound < NameError; end
 
+  # Map of {"ClassName" => ClassDefinitionNode}
+  # Can also query object for superclass of a named class, etc...
+  #
+  # Ex : classes["SomeClass"].superclass_name => "SomeClassBase"
   class ClassMap
     def initialize
       @map = {}

data/lib/compiler.rb

@@ -6,7 +6,7 @@ module Riml
 
     # Base abstract visitor
     class Visitor
-      attr_accessor :propagate_up_tree
+      attr_writer :propagate_up_tree
       attr_reader :value
 
       def initialize(options={})
@@ -136,7 +136,7 @@ module Riml
         when Numeric
           node.value
         when String
-          StringNode === node ? string_surround(node) : node.value.dup
+          StringNode === node ? string_surround(node) : node.value
         when Array
           node.value.each {|n| n.parent_node = node}
           '[' <<
@@ -145,7 +145,7 @@ module Riml
             n.compiled_output
           end.join(', ') << ']'
         when Hash
-          node.value.each {|k_n, v_n| k_n.parent_node, v_n.parent_node = [node, node]}
+          node.value.each {|k_n, v_n| k_n.parent_node, v_n.parent_node = node, node}
           '{' <<
           node.value.map do |k,v|
             k.accept(visitor_for_node(k))
@@ -185,11 +185,9 @@ module Riml
     ContinueNodeVisitor = LiteralNodeVisitor
     BreakNodeVisitor = LiteralNodeVisitor
 
-    class HeredocNodeVisitor < Visitor
+    class ListUnpackNodeVisitor < ListNodeVisitor
       def compile(node)
-        node.string_node.parent_node = node
-        node.string_node.accept(StringNodeVisitor.new)
-        node.compiled_output
+        node.compiled_output = super.reverse.sub(',', ';').reverse
       end
     end
 
@@ -223,15 +221,16 @@ module Riml
         #       myVariable = "override riml default scoping"
         node.scope_modifier = "" if node.scope_modifier == "n:"
         return node.scope_modifier if node.scope_modifier
-        node.scope_modifier = scope_modifier_for_variable_name(node)
+        node.scope_modifier = scope_modifier_for_node(node)
       end
 
       private
-      def scope_modifier_for_variable_name(node)
+      def scope_modifier_for_node(node)
         if node.scope
           return "a:" if node.scope.argument_variable_names.include?(node.name)
-          return ""
+          return "" unless node.is_a?(CallNode)
         end
+        return "" if (node.is_a?(CallNode) || node.is_a?(DefNode)) && node.autoload?
         "s:"
       end
     end
@@ -265,7 +264,9 @@ module Riml
 
       private
       def check_for_splat_match!(node, splat)
-        if node.name == splat[1..-1]
+        # if `function doIt(*options)`, then:
+        # *options OR options in function body becomes `a:000`
+        if [ splat, splat[1..-1] ].include?(node.name)
           node.scope_modifier = "a:"
           node.name = '000'
         end
@@ -333,17 +334,17 @@ module Riml
       end
     end
 
-    class DefNodeVisitor < Visitor
+    class DefNodeVisitor < ScopedVisitor
       def visit(node)
         setup_local_scope_for_descendants(node)
         super
       end
 
       def compile(node)
-        modifier = node.scope ? nil : node.scope_modifier || 's:'
+        set_modifier(node)
         bang = node.bang
         params = process_parameters!(node)
-        declaration = "function#{bang} #{modifier}"
+        declaration = "function#{bang} #{node.scope_modifier}"
         declaration <<
         if node.name.respond_to?(:variable)
           node.name.accept(visitor_for_node(node.name))
@@ -474,7 +475,9 @@ module Riml
           end
           node.compiled_output << 'source'
           compile_arguments(node)
-          node.compiled_output.gsub(/['"]/, '').gsub('.riml', '.vim')
+          node.compiled_output.gsub!(/['"]/, '')
+          node.compiled_output.sub!('.riml', '.vim')
+          node.compiled_output
         end
       end
 
@@ -514,9 +517,9 @@ module Riml
     end
 
     class TryNodeVisitor < Visitor
-      # try_block, catch_nodes, ensure_block
+      # try_block, catch_nodes, finally_block
       def compile(node)
-        try, catches, _ensure = node.try_block, node.catch_nodes, node.ensure_block
+        try, catches, finally = node.try_block, node.catch_nodes, node.finally_block
         node.compiled_output = "try\n"
         try.accept(visitor_for_node(try, :propagate_up_tree => false))
         try.compiled_output.each_line do |line|
@@ -530,10 +533,10 @@ module Riml
           end
         end if catches
 
-        if _ensure
+        if finally
           node.compiled_output << "finally\n"
-          _ensure.accept(visitor_for_node(_ensure, :propagate_up_tree => false))
-          _ensure.compiled_output.each_line do |line|
+          finally.accept(visitor_for_node(finally, :propagate_up_tree => false))
+          finally.compiled_output.each_line do |line|
             node.compiled_output << node.indent + line
           end
         end
@@ -603,6 +606,9 @@ module Riml
       end
     end
 
+    # root node has access to compiler instance in order to append to
+    # the compiler's `compile_queue`. This happens when another file is
+    # sourced using `riml_source`.
     module CompilerAccessible
       attr_accessor :current_compiler
     end
@@ -613,8 +619,6 @@ module Riml
 
     # compiles nodes into output code
     def compile(root_node)
-      # so we can compile concurrently, as some nodes have access to the
-      # compiler instance itself
       root_node.extend CompilerAccessible
       root_node.current_compiler = self
       root_node.accept(NodesVisitor.new)

data/lib/constants.rb

@@ -7,8 +7,7 @@ module Riml
     VIML_END_KEYWORDS =
       %w(endfunction endif endwhile endfor endtry)
     RIML_KEYWORDS =
-      %w(def defm super end then unless until true false nil ensure
-         class new)
+      %w(def defm super end then unless until true false nil class new)
     DEFINE_KEYWORDS = %w(def def! defm defm! function function!)
     KEYWORDS = VIML_KEYWORDS + VIML_END_KEYWORDS + RIML_KEYWORDS
 

data/lib/grammar.y

@@ -2,7 +2,7 @@ class Riml::Parser
 
 token IF ELSE ELSEIF THEN UNLESS END
 token WHILE UNTIL BREAK CONTINUE
-token TRY CATCH ENSURE
+token TRY CATCH FINALLY
 token FOR IN
 token DEF DEF_BANG SPLAT CALL BUILTIN_COMMAND # such as echo "hi"
 token CLASS NEW DEFM DEFM_BANG SUPER RIML_COMMAND
@@ -10,7 +10,7 @@ token RETURN
 token NEWLINE
 token NUMBER
 token STRING_D STRING_S # single- and double-quoted
-token HEREDOC EX_LITERAL
+token EX_LITERAL
 token REGEXP
 token TRUE FALSE NIL
 token LET UNLET UNLET_BANG IDENTIFIER
@@ -79,7 +79,6 @@ rule
   | VariableRetrieval                     { result = val[0] }
   | Literal                               { result = val[0] }
   | Call                                  { result = val[0] }
-  | Heredoc                               { result = val[0] }
   | Ternary                               { result = val[0] }
   | ObjectInstantiation                   { result = val[0] }
   | BinaryOperator                        { result = val[0] }
@@ -114,10 +113,6 @@ rule
   | STRING_D                              { result = StringNode.new(val[0], :d) }
   ;
 
-  Heredoc:
-    HEREDOC String                        { result = HeredocNode.new(val[0], val[1]) }
-  ;
-
   Regexp:
     REGEXP                                { result = RegexpNode.new(val[0]) }
   ;
@@ -130,6 +125,10 @@ rule
     ListLiteral                           { result = ListNode.new(val[0]) }
   ;
 
+  ListUnpack:
+    '[' ListItems ';' ValueExpression ']' { result = ListUnpackNode.new(val[1] << val[3]) }
+  ;
+
   ListLiteral:
     '[' ListItems ']'                     { result = val[1] }
   | '[' ListItems ',' ']'                 { result = val[1] }
@@ -299,6 +298,7 @@ rule
     VariableRetrieval                          { result = val[0] }
   | DictGet                                    { result = val[0] }
   | List                                       { result = val[0] }
+  | ListUnpack                                 { result = val[0] }
   | ListOrDictGet                              { result = val[0] }
   ;
 
@@ -401,7 +401,7 @@ rule
   Try:
     TRY Block END                              { result = TryNode.new(val[1], nil, nil) }
   | TRY Block Catch END                        { result = TryNode.new(val[1], val[2], nil) }
-  | TRY Block Catch ENSURE Block END           { result = TryNode.new(val[1], val[2], val[4]) }
+  | TRY Block Catch FINALLY Block END          { result = TryNode.new(val[1], val[2], val[4]) }
   ;
 
   Catch:
@@ -480,7 +480,7 @@ end
       raise Riml::ParseError,  "line #{@lexer.lineno}: #{e.message}"
     end
 
-    @ast_rewriter = ast_rewriter
+    @ast_rewriter ||= ast_rewriter
     return ast unless @ast_rewriter
     @ast_rewriter.ast = ast
     @ast_rewriter.rewrite

data/lib/lexer.rb

@@ -7,12 +7,18 @@ module Riml
 
     SINGLE_LINE_COMMENT_REGEX = /\A\s*"(.*)$/
     OPERATOR_REGEX = /\A#{Regexp.union(['||', '&&', '===', '+=', '-=', '.='] + COMPARISON_OPERATORS)}/
+    INTERPOLATION_REGEX = /\A"(.*?)(\#\{(.*?)\})(.*?)"/m
+    INTERPOLATION_SPLIT_REGEX = /(\#{.*?})/m
 
     attr_reader :tokens, :prev_token, :lineno, :chunk
 
     def initialize(code)
       @code = code
       @code.chomp!
+      set_start_state!
+    end
+
+    def set_start_state!
       @i = 0 # number of characters consumed
       @token_buf = []
       @tokens = []
@@ -21,15 +27,15 @@ module Riml
       @current_indent = 0
       @indent_pending = false
       @dedent_pending = false
-      @one_line_conditional_END_pending = false
+      @one_line_conditional_end_pending = false
       @splat_allowed = false
+      @in_function_declaration = false
     end
 
     def tokenize
-      @i, @current_indent = 0, 0
-      while more_code_to_tokenize?
-        new_token = next_token
-        @tokens << new_token unless new_token.nil?
+      set_start_state!
+      while (token = next_token) != nil
+        @tokens << token
       end
       @tokens
     end
@@ -38,7 +44,7 @@ module Riml
       while @token_buf.empty? && more_code_to_tokenize?
         tokenize_chunk(get_new_chunk)
       end
-      if @token_buf.any?
+      if !@token_buf.empty?
         return @prev_token = @token_buf.shift
       end
       check_indentation
@@ -65,7 +71,7 @@ module Riml
         @i += splat_var.size
         @token_buf << [:SCOPE_MODIFIER, 'a:'] << [:IDENTIFIER, splat_var[2..-1]]
       # the 'n' scope modifier is added by riml
-      elsif scope_modifier = chunk[/\A([bwtglsavn]:)[\w_]/]
+      elsif scope_modifier = chunk[/\A([bwtglsavn]:)[\w]/]
         @i += 2
         @token_buf << [:SCOPE_MODIFIER, $1]
       elsif scope_modifier_literal = chunk[/\A([bwtglsavn]:)/]
@@ -85,9 +91,6 @@ module Riml
             old_identifier = identifier.dup
             identifier.sub!(/function/, "def")
             @i += (old_identifier.size - identifier.size)
-          elsif identifier == 'finally'
-            identifier = 'ensure'
-            @i += 1 # diff b/t the two string lengths
           elsif VIML_END_KEYWORDS.include? identifier
             old_identifier = identifier.dup
             identifier = 'end'
@@ -137,17 +140,10 @@ module Riml
       elsif decimal = chunk[/\A[0-9]+(\.[0-9]+)?/]
         @token_buf << [:NUMBER, decimal.to_s]
         @i += decimal.size
-      elsif interpolation = chunk[/\A"(.*?)(\#\{(.*?)\})(.*?)"/]
-        # "#{hey} guys" = "hey" . " guys"
-        unless $1.empty?
-          @token_buf << [:STRING_D, $1]
-          @token_buf << ['.', '.']
-        end
-        @token_buf << [:IDENTIFIER, $3]
-        unless $4.empty?
-          @token_buf << ['.', '.']
-          @token_buf << [ :STRING_D, " #{$4[1..-1]}" ]
-        end
+      elsif interpolation = chunk[INTERPOLATION_REGEX]
+        # "hey there, #{name}" = "hey there, " . name
+        parts = interpolation[1...-1].split(INTERPOLATION_SPLIT_REGEX)
+        handle_interpolation(*parts)
         @i += interpolation.size
       elsif single_line_comment = chunk[SINGLE_LINE_COMMENT_REGEX] && (prev_token.nil? || prev_token[0] == :NEWLINE)
         comment = chunk[SINGLE_LINE_COMMENT_REGEX]
@@ -166,8 +162,8 @@ module Riml
         @token_buf << [:NEWLINE, "\n"] unless prev_token && prev_token[0] == :NEWLINE
 
         # pending indents/dedents
-        if @one_line_conditional_END_pending
-          @one_line_conditional_END_pending = false
+        if @one_line_conditional_end_pending
+          @one_line_conditional_end_pending = false
         elsif @indent_pending
           @indent_pending = false
         elsif @dedent_pending
@@ -179,11 +175,15 @@ module Riml
       elsif heredoc_pattern = chunk[%r{\A<<(.+?)\r?\n}]
         pattern = $1
         @i += heredoc_pattern.size
-        @token_buf << [:HEREDOC, pattern]
         new_chunk = get_new_chunk
-        heredoc_string = new_chunk[%r|(.+?\r?\n)(#{Regexp.escape(pattern)})|]
-        @i += heredoc_string.size + $2.size
-        @token_buf << [:STRING_D, $1]
+        heredoc_string = new_chunk[%r|(.+?\r?\n)(#{Regexp.escape(pattern)})|, 1]
+        @i += heredoc_string.size + pattern.size
+        if ('"' + heredoc_string + '"') =~ INTERPOLATION_REGEX
+          parts = heredoc_string.split(INTERPOLATION_SPLIT_REGEX)
+          handle_interpolation(*parts)
+        else
+          @token_buf << [:STRING_D, heredoc_string]
+        end
         @lineno += (1 + heredoc_string.each_line.to_a.size)
       # operators of more than 1 char
       elsif operator = chunk[OPERATOR_REGEX]
@@ -220,13 +220,13 @@ module Riml
         @indent_pending = true
       when :if, :unless
         if one_line_conditional?(chunk)
-          @one_line_conditional_END_pending = true
+          @one_line_conditional_end_pending = true
         elsif !statement_modifier?(chunk)
           @current_indent += 2
           @indent_pending = true
         end
       when :end
-        unless @one_line_conditional_END_pending
+        unless @one_line_conditional_end_pending
           @current_indent -= 2
           @dedent_pending = true
         end
@@ -258,6 +258,22 @@ module Riml
       chunk[/^(if|unless).+?(else)?.+?end$/]
     end
 
+    def handle_interpolation(*parts)
+      parts.delete_if {|p| p.empty?}.each_with_index do |part, i|
+        if part[0..1] == '#{' && part[-1] == '}'
+          @token_buf.concat tokenize_without_moving_pos(part[2...-1])
+        else
+          @token_buf << [:STRING_D, part]
+        end
+        # string-concatenate all the parts unless this is the last part
+        @token_buf << ['.', '.'] unless parts[i + 1].nil?
+      end
+    end
+
+    def tokenize_without_moving_pos(code)
+      Lexer.new(code).tokenize
+    end
+
     def statement_modifier?(chunk)
       old_i = @i
       # backtrack until the beginning of the line