riml 0.1.7 → 0.1.8

data/CONTRIBUTING

@@ -0,0 +1,44 @@
+Thanks for reading this!
+
+If you're interested in contributing to Riml, thanks! Here's a brief overview
+of how it works.
+
+Brief Overview
+==============
+
+1. The lexer lexes riml source into Array of tokens.
+
+2. The parser, generated by Racc from the 'grammar.y' file, creates an
+Abstract Syntax Tree from the nodes in the 'nodes.rb' file.
+
+3. The AST_Rewriter rewrites portions of the tree. These portions are
+Riml-only constructs, and mainly it's rewriting Riml-only syntax nodes into
+VimL-compatible syntax-nodes. For example, one of the ways a ClassNode is
+rewritten here is by adding an 'initialize' function to it if it doesn't have
+one already. Once this is done, onto the compiler!
+
+4. The compiler, implemented using the Visitor pattern, visits the nodes in a
+top-down fashion starting with the root, and these nodes append their compiled
+output (VimL) to their parent node's output. After all is done, the root node (only
+node without a parent) is left with the full output.
+
+Pull Requests
+=============
+
+When appropriate, please add unit tests for features or bugs. Also, please
+update all documentation related to the changes that you've made, including
+comments in the source code. If you don't know where all the affected areas
+might be, don't hesitate to ask.
+
+For pure documentation-related changes, please put '[ci skip]' somewhere in the
+commit message so that Travis-ci skips the build.
+
+Note
+====
+
+If you're working on the grammar file, make sure to re-create the parser after
+each change before testing. This is done by going to the 'lib' directory and
+typing:
+
+`racc -o parser.rb grammar.y`
+

data/Gemfile

@@ -0,0 +1,13 @@
+source :rubygems
+
+group :development do
+  # create the parser by going to the lib directory
+  # and executing the following:
+  #
+  # $ racc -o parser.rb grammar.y
+  gem 'racc'
+end
+
+group :test do
+  gem 'rake' # for travis-ci
+end

data/Gemfile.lock

@@ -0,0 +1,12 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    racc (1.4.9)
+    rake (10.0.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  racc
+  rake

data/README.md

@@ -94,7 +94,7 @@ Classes
       defm translate                                     let frenchToEnglishTranslationObj = {}
         if (self.input == "Bonjour!")                    let translationObj = g:TranslationConstructor(a:input)
           echo "Hello!"                                  call extend(frenchToEnglishTranslationObj, translationObj)
-        else                                             let translationObj.translate = function('g:FrenchToEnglishTranslation_translate')
+        else                                             let frenchToEnglishTranslationObj.translate = function('g:FrenchToEnglishTranslation_translate')
           echo "Sorry, I don't know that word."          return frenchToEnglishTranslationObj
         end                                            endfunction
       end

data/Rakefile

@@ -0,0 +1,15 @@
+require File.expand_path('../lib/environment', __FILE__)
+require 'rake/testtask'
+
+task :default => :test
+task :test => [:output_test_count]
+
+desc 'Run all *_tests and *_specs (default)'
+test = Rake::TestTask.new(:test) do |t|
+  TEST_LIST = FileList['test/**/*_test.rb'].to_a
+  t.test_files = TEST_LIST
+end
+
+task :output_test_count do
+  puts "#{TEST_LIST.size} test files to run."
+end

data/bin/riml

@@ -12,25 +12,30 @@ module Riml
 
   class Options
     def self.parse(argv)
+      ARGV << '--help' if ARGV.size.zero?
 
       # defaults
       options = OpenStruct.new
       options.compile_files = []
+      options.check_syntax_files = []
+      options.repl = false
+      options.vi_readline = false
 
       OptionParser.new do |opts|
-        opts.banner = "Usage: riml [options]"
+        opts.banner = "Usage: riml [options] [file1][,file2]..."
         opts.separator ""
         opts.separator "Specific options:"
 
-        opts.on("-c", "--compile FILES", Array, "Compile riml file(s) to VimL.") do |files|
-          files.each do |f|
-            if File.exists?(f)
-              options.compile_files << f
-            else
-              warn "Couldn't find file #{f.inspect}."
-              exit 1
-            end
-          end
+        opts.on("-c", "--compile FILES", Array, "Compiles riml file(s) to VimL.") do |filenames|
+          append_filenames_to_list_if_all_exist(options.compile_files, *filenames)
+        end
+
+        opts.on("-s", "--stdio", "Takes riml from stdin and outputs VimL to stdout.") do
+          options.stdio = true
+        end
+
+        opts.on("-k", "--check FILES", Array, "Checks syntax of file(s). Because Riml is a superset of VimL, this can be used to check VimL's syntax as well.") do |filenames|
+          append_filenames_to_list_if_all_exist(options.check_syntax_files, *filenames)
         end
 
         opts.on("-t", "--source-path PATH", "Path riml uses for `riml_source` to find files. Defaults to pwd.") do |path|
@@ -42,8 +47,13 @@ module Riml
           end
         end
 
-        opts.on("-s", "--stdio", "Pipe in riml to STDIN and get back VimL on STDOUT.") do
-          options.stdio = true
+        opts.on("-i", "--interactive", "Start an interactive riml session (REPL).") do
+          require 'repl'
+          options.repl = true
+        end
+
+        opts.on("--vi", "Use vi readline settings during interactive session.") do
+          options.vi_readline = true
         end
 
         opts.on_tail("-v", "--version", "Show riml version.") do
@@ -59,6 +69,19 @@ module Riml
 
       options
     end
+
+    private
+
+    def self.append_filenames_to_list_if_all_exist(list, *filenames)
+      filenames.each do |fname|
+        if File.exists?(fname)
+          list << fname
+        else
+          warn "Couldn't find file #{fname.inspect}."
+          exit 1
+        end
+      end
+    end
   end
 
   class Runner
@@ -70,6 +93,14 @@ module Riml
           puts Riml.compile($stdin.read)
         elsif options.compile_files.any?
           Riml.compile_files(*options.compile_files)
+        elsif options.check_syntax_files.any?
+          files = options.check_syntax_files
+          Riml.check_syntax_files(*files)
+          size = files.size
+          # "ok (1 file)" OR "ok (2 files)"
+          puts "ok (#{size} file#{'s' if size > 1})"
+        elsif options.repl
+          Riml::Repl.new(options.vi_readline).run
         end
       end
     end

data/lib/ast_rewriter.rb

@@ -98,6 +98,9 @@ module Riml
       end
 
       def replace(node)
+        if classes[node.name]
+          raise ClassRedefinitionError, "can't redefine class #{node.name.inspect}"
+        end
         classes[node.name] = node
 
         name, expressions = node.name, node.expressions

data/lib/class_map.rb

@@ -35,10 +35,6 @@ module Riml
       @map.keys
     end
 
-    def clear
-      @map.clear
-    end
-
     protected
     def ensure_key_is_string!(key)
       unless key.is_a?(String)

data/lib/compiler.rb

@@ -111,8 +111,8 @@ module Riml
       def compile(nodes)
         nodes.each_with_index do |node, i|
           visitor = visitor_for_node(node)
-          next_node = nodes.nodes[i+1]
           node.parent_node = nodes
+          next_node = nodes.nodes[i+1]
           if ElseNode === next_node
             node.force_newline = true
           end
@@ -194,11 +194,11 @@ module Riml
     class ReturnNodeVisitor < Visitor
       def compile(node)
         node.compiled_output = "return"
-        return (node.compiled_output << "\n") if node.expression.nil?
+        node.force_newline = true
+        return node.compiled_output if node.expression.nil?
         node.expression.parent_node = node
         node.compiled_output << " "
         node.expression.accept(visitor_for_node(node.expression))
-        node.force_newline = true
         node.compiled_output
       end
     end
@@ -287,9 +287,8 @@ module Riml
       end
 
       def compile_parts(parts)
-        compiled = ""
-        parts.each do |part|
-          output = if CurlyBraceVariable === part
+        parts.map do |part|
+          if CurlyBraceVariable === part
             compile_parts(part)
           elsif part.nested?
             compile_nested_parts(part.value, part)
@@ -298,11 +297,9 @@ module Riml
             part.value.accept(visitor_for_node(part.value))
             "{#{part.value.compiled_output}}"
           else
-            "#{part.value}"
+            part.value
           end
-          compiled << output
-        end
-        compiled
+        end.join
       end
 
       def compile_nested_parts(parts, root_part)
@@ -319,9 +316,8 @@ module Riml
             next
           end
           part.value.accept(visitor_for_node(part.value, :propagate_up_tree => false))
-          root_part.compiled_output << "{#{part.value.compiled_output}}"
+          root_part.compiled_output << "{#{part.value.compiled_output}}#{'}' * nested}"
         end
-        root_part.compiled_output << ('}' * nested)
       end
     end
 
@@ -446,7 +442,7 @@ module Riml
             node.name.accept(visitor_for_node(node.name))
             node.name.compiled_output
           else
-            "#{node.full_name}"
+            node.full_name
           end
         compile_arguments(node)
         node.compiled_output
@@ -494,6 +490,7 @@ module Riml
     class RimlCommandNodeVisitor < CallNodeVisitor
       def compile(node)
         if node.name == 'riml_source'
+          node.name = 'source'
           node.arguments.map(&:value).each do |file|
             unless File.exists?(File.join(Riml.source_path, file))
               raise Riml::FileNotFound, "#{file.inspect} could not be found in " \
@@ -502,12 +499,12 @@ module Riml
 
             root_node(node).current_compiler.compile_queue << file
           end
-          node.compiled_output << 'source'
-          compile_arguments(node)
-          node.compiled_output.gsub!(/['"]/, '')
-          node.compiled_output.sub!('.riml', '.vim')
-          node.compiled_output
         end
+        node.compiled_output << node.name
+        compile_arguments(node)
+        node.compiled_output.gsub!(/['"]/, '')
+        node.compiled_output.sub!('.riml', '.vim')
+        node.compiled_output
       end
 
       def root_node(node)

data/lib/constants.rb

@@ -1,7 +1,7 @@
 module Riml
   module Constants
     VIML_KEYWORDS =
-      %w(function function! if else elseif while for in command command!
+      %w(function function! if else elseif while for in
          return is isnot finish break continue call let unlet unlet! try
          catch finally)
     VIML_END_KEYWORDS =
@@ -17,6 +17,8 @@ module Riml
       %w(echo echon echohl execute sleep)
     RIML_COMMANDS =
       %w(riml_source)
+    VIML_COMMANDS =
+      %w(source source! command! command silent silent!)
 
     IGNORECASE_CAPABLE_OPERATORS =
       %w(== != >= > <= < =~ !~)

data/lib/errors.rb

@@ -6,4 +6,6 @@ module Riml
   CompileError = Class.new(RimlError)
 
   FileNotFound = Class.new(RimlError)
+
+  ClassRedefinitionError = Class.new(RimlError)
 end

data/lib/grammar.y

@@ -72,17 +72,24 @@ rule
 
   # Expressions that evaluate to a value
   ValueExpression:
+    ValueExpressionWithoutDictLiteral     { result = val[0] }
+  | Dictionary                            { result = val[0] }
+  | Dictionary DictGetWithDotLiteral      { result = DictGetDotNode.new(val[0], val[1]) }
+  | BinaryOperator                        { result = val[0] }
+  | Ternary                               { result = val[0] }
+  | '(' ValueExpression ')'               { result = WrapInParensNode.new(val[1]) }
+  ;
+
+  ValueExpressionWithoutDictLiteral:
     UnaryOperator                         { result = val[0] }
   | Assign                                { result = val[0] }
   | DictGet                               { result = val[0] }
   | ListOrDictGet                         { result = val[0] }
   | AllVariableRetrieval                  { result = val[0] }
-  | Literal                               { result = val[0] }
+  | LiteralWithoutDictLiteral             { result = val[0] }
   | Call                                  { result = val[0] }
-  | Ternary                               { result = val[0] }
   | ObjectInstantiation                   { result = val[0] }
-  | BinaryOperator                        { result = val[0] }
-  | '(' ValueExpression ')'               { result = WrapInParensNode.new(val[1]) }
+  | '(' ValueExpressionWithoutDictLiteral ')'               { result = WrapInParensNode.new(val[1]) }
   ;
 
   Terminator:
@@ -93,11 +100,15 @@ rule
 
   # All hard-coded values
   Literal:
+    LiteralWithoutDictLiteral             { result = val[0] }
+  | Dictionary                            { result = val[0] }
+  ;
+
+  LiteralWithoutDictLiteral:
     Number                                { result = val[0] }
   | String                                { result = val[0] }
   | Regexp                                { result = val[0] }
   | List                                  { result = val[0] }
-  | Dictionary                            { result = val[0] }
   | ScopeModifierLiteral                  { result = val[0] }
   | TRUE                                  { result = TrueNode.new }
   | FALSE                                 { result = FalseNode.new }
@@ -163,13 +174,14 @@ rule
   ;
 
   DictGet:
-    Dictionary DictGetWithDotLiteral             { result = DictGetDotNode.new(val[0], val[1]) }
-  | AllVariableRetrieval DictGetWithDot          { result = DictGetDotNode.new(val[0], val[1]) }
+    AllVariableRetrieval DictGetWithDot          { result = DictGetDotNode.new(val[0], val[1]) }
   | ListOrDictGet DictGetWithDot                 { result = DictGetDotNode.new(val[0], val[1]) }
+  | '(' ValueExpression ')' DictGetWithDot       { result = DictGetDotNode.new(WrapInParensNode.new(val[1]), val[3]) }
   ;
 
   ListOrDictGet:
-    ValueExpression ListOrDictGetWithBrackets    { result = ListOrDictGetNode.new(val[0], val[1]) }
+    ValueExpressionWithoutDictLiteral ListOrDictGetWithBrackets  { result = ListOrDictGetNode.new(val[0], val[1]) }
+  | '(' ValueExpression ')' ListOrDictGetWithBrackets            { result = ListOrDictGetNode.new(WrapInParensNode.new(val[1]), val[3]) }
   ;
 
   ListOrDictGetWithBrackets:
@@ -296,9 +308,9 @@ rule
 
   AssignLHS:
     AllVariableRetrieval                       { result = val[0] }
-  | DictGet                                    { result = val[0] }
   | List                                       { result = val[0] }
   | ListUnpack                                 { result = val[0] }
+  | DictGet                                    { result = val[0] }
   | ListOrDictGet                              { result = val[0] }
   ;
 

data/lib/lexer.rb

@@ -10,7 +10,9 @@ module Riml
     INTERPOLATION_REGEX = /\A"(.*?)(\#\{(.*?)\})(.*?)"/m
     INTERPOLATION_SPLIT_REGEX = /(\#{.*?})/m
 
-    attr_reader :tokens, :prev_token, :lineno, :chunk
+    attr_reader :tokens, :prev_token, :lineno, :chunk, :current_indent
+    # for REPL
+    attr_accessor :ignore_indentation_check
 
     def initialize(code)
       @code = code
@@ -19,7 +21,10 @@ module Riml
     end
 
     def set_start_state!
-      @i = 0 # number of characters consumed
+      # number of characters consumed
+      @i = 0
+      # array of doubles and triples: [tokenname, tokenval, lineno_to_add(optional)]
+      # ex: [[:NEWLINE, "\n"]] OR [[:NEWLINE, "\n", 1]]
       @token_buf = []
       @tokens = []
       @prev_token = nil
@@ -45,18 +50,22 @@ module Riml
         tokenize_chunk(get_new_chunk)
       end
       if !@token_buf.empty?
-        return @prev_token = @token_buf.shift
+        token = @token_buf.shift
+        if token.size == 3
+          @lineno += token.pop
+        end
+        return @prev_token = token
       end
-      check_indentation
+      check_indentation unless ignore_indentation_check
       nil
     end
 
     def tokenize_chunk(chunk)
       @chunk = chunk
       # deal with line continuations
-      if cont = chunk[/\A(\n*)\s*\\/]
+      if cont = chunk[/\A\n*\s*\\/m]
         @i += cont.size
-        @lineno += $1.size
+        @lineno += cont.each_line.to_a.size - 1
         return
       end
 
@@ -111,6 +120,13 @@ module Riml
           @token_buf << [:BUILTIN_COMMAND, identifier]
         elsif RIML_COMMANDS.include? identifier
           @token_buf << [:RIML_COMMAND, identifier]
+        elsif VIML_COMMANDS.include?(identifier) && (prev_token.nil? || prev_token[0] == :NEWLINE)
+          @i += identifier.size
+          new_chunk = get_new_chunk
+          until_eol = new_chunk[/.*$/].to_s
+          @token_buf << [:EX_LITERAL, identifier << until_eol]
+          @i += until_eol.size
+          return
         # method names and variable names
         else
           @token_buf << [:IDENTIFIER, identifier]
@@ -145,12 +161,12 @@ module Riml
         parts = interpolation[1...-1].split(INTERPOLATION_SPLIT_REGEX)
         handle_interpolation(*parts)
         @i += interpolation.size
-      elsif single_line_comment = chunk[SINGLE_LINE_COMMENT_REGEX] && (prev_token.nil? || prev_token[0] == :NEWLINE)
-        comment = chunk[SINGLE_LINE_COMMENT_REGEX]
-        @i += comment.size + 1 # consume next newline character
-        @lineno += 1
+      elsif (single_line_comment = chunk[SINGLE_LINE_COMMENT_REGEX]) && (prev_token.nil? || prev_token[0] == :NEWLINE)
+        @i += single_line_comment.size + 1 # consume next newline character
+        @lineno += single_line_comment.each_line.to_a.size
       elsif inline_comment = chunk[/\A\s*"[^"]*?$/]
         @i += inline_comment.size # inline comment, don't consume newline character
+        @lineno += inline_comment.each_line.to_a.size - 1
       elsif string_double = chunk[/\A"(.*?)(?<!\\)"/, 1]
         @token_buf << [:STRING_D, string_double]
         @i += string_double.size + 2
@@ -184,12 +200,11 @@ module Riml
         else
           @token_buf << [:STRING_D, heredoc_string]
         end
-        @lineno += (1 + heredoc_string.each_line.to_a.size)
+        @lineno += heredoc_string.each_line.to_a.size
       # operators of more than 1 char
       elsif operator = chunk[OPERATOR_REGEX]
         @token_buf << [operator, operator]
         @i += operator.size
-      # FIXME: this doesn't work well enough
       elsif regexp = chunk[%r{\A/.*?[^\\]/}]
         @token_buf << [:REGEXP, regexp]
         @i += regexp.size
@@ -236,11 +251,11 @@ module Riml
     def parse_dict_vals!
       # dict.key OR dict.key.other_key
       new_chunk = get_new_chunk
-      if new_chunk[/\A\.([\w.]+)/]
-        parts = $1.split('.')
-        @i += $1.size + 1
+      if vals = new_chunk[/\A\.([\w.]+)/, 1]
+        parts = vals.split('.')
+        @i += vals.size + 1
         if @in_function_declaration
-          @token_buf.last[1] << ".#{$1}"
+          @token_buf.last[1] << ".#{vals}"
         else
           while key = parts.shift
             @token_buf << [:DICT_VAL, key]