riml 0.1.3

data/lib/grammar.y

@@ -0,0 +1,485 @@
+class Riml::Parser
+
+token IF ELSE ELSEIF THEN UNLESS END
+token WHILE UNTIL BREAK CONTINUE
+token TRY CATCH ENSURE
+token FOR IN
+token DEF DEF_BANG SPLAT CALL BUILTIN_COMMAND # such as echo "hi"
+token CLASS NEW DEFM DEFM_BANG SUPER
+token RETURN
+token NEWLINE
+token NUMBER
+token STRING_D STRING_S # single- and double-quoted
+token HEREDOC EX_LITERAL
+token REGEXP
+token TRUE FALSE NIL
+token LET UNLET UNLET_BANG IDENTIFIER
+token DICT_VAL # like dict.key, 'key' is a DICT_VAL
+token SCOPE_MODIFIER SCOPE_MODIFIER_LITERAL SPECIAL_VAR_PREFIX
+token FINISH
+
+prechigh
+  right '!'
+  left '*' '/' '%'
+  left '+' '+=' '-' '-=' '.'
+  left '>' '>#' '>?' '<' '<#' '<?' '>=' '>=#' '>=?'  '<=' '<=#' '<=?'
+  left '==' '==?' '==#' '=~' '=~?' '=~#' '!~' '!~?' '!~#' '!=' '!=?' '!=#'
+  left IS ISNOT
+  left '&&'
+  left '||'
+  right '?'
+  right '='
+  left ','
+  left IF UNLESS
+preclow
+
+# All rules
+rule
+
+  Root:
+    /* nothing */                         { result = Nodes.new([]) }
+  | Expressions                           { result = val[0] }
+  ;
+
+  # any list of expressions
+  Expressions:
+    Expression                            { result = Nodes.new([ val[0] ]) }
+  | Expressions Terminator Expression     { result = val[0] << val[2] }
+  | Expressions Terminator                { result = val[0] }
+  | Terminator                            { result = Nodes.new([]) }
+  ;
+
+  # All types of expressions in Riml
+  Expression:
+    BinaryOperator                        { result = val[0] }
+  | UnaryOperator                         { result = val[0] }
+  | Call                                  { result = val[0] }
+  | Assign                                { result = val[0] }
+  | DictGet                               { result = val[0] }
+  | ListOrDictGet                         { result = val[0] }
+  | Def                                   { result = val[0] }
+  | Return                                { result = val[0] }
+  | VariableRetrieval                     { result = val[0] }
+  | UnletVariable                         { result = val[0] }
+  | Literal                               { result = val[0] }
+  | ExLiteral                             { result = val[0] }
+  | Heredoc                               { result = val[0] }
+  | If                                    { result = val[0] }
+  | Unless                                { result = val[0] }
+  | Ternary                               { result = val[0] }
+  | While                                 { result = val[0] }
+  | Until                                 { result = val[0] }
+  | For                                   { result = val[0] }
+  | Try                                   { result = val[0] }
+  | ClassDefinition                       { result = val[0] }
+  | ObjectInstantiation                   { result = val[0] }
+  | Super                                 { result = val[0] }
+  | LoopKeyword                           { result = val[0] }
+  | EndScript                             { result = val[0] }
+  | '(' Expression ')'                    { result = val[1] }
+  ;
+
+  Terminator:
+    NEWLINE                               { result = nil }
+  | ';'                                   { result = nil }
+  | '|'                                   { result = nil }
+  ;
+
+  # All hard-coded values
+  Literal:
+    Number                                { result = val[0] }
+  | String                                { result = val[0] }
+  | Regexp                                { result = val[0] }
+  | List                                  { result = val[0] }
+  | Dictionary                            { result = val[0] }
+  | ScopeModifierLiteral                  { result = val[0] }
+  | TRUE                                  { result = TrueNode.new }
+  | FALSE                                 { result = FalseNode.new }
+  | NIL                                   { result = NilNode.new }
+  ;
+
+  Number:
+    NUMBER                                { result = NumberNode.new(val[0]) }
+  | '-' NUMBER                            { result = NumberNode.new(val[1].insert(0, val[0])) }
+  | '+' NUMBER                            { result = NumberNode.new(val[1].insert(0, val[0])) }
+  ;
+
+  String:
+    STRING_S                              { result = StringNode.new(val[0], :s) }
+  | STRING_D                              { result = StringNode.new(val[0], :d) }
+  ;
+
+  Heredoc:
+    HEREDOC String                        { result = HeredocNode.new(val[0], val[1]) }
+  ;
+
+  Regexp:
+    REGEXP                                { result = RegexpNode.new(val[0]) }
+  ;
+
+  ScopeModifierLiteral:
+    SCOPE_MODIFIER_LITERAL                { result = ScopeModifierLiteralNode.new(val[0]) }
+  ;
+
+  List:
+    ListLiteral                           { result = ListNode.new(val[0]) }
+  ;
+
+  ListLiteral:
+    '[' ListItems ']'                     { result = val[1] }
+  | '[' ListItems ',' ']'                 { result = val[1] }
+  ;
+
+  ListItems:
+    /* nothing */                         { result = [] }
+  | Expression                            { result = [val[0]] }
+  | ListItems ',' Expression              { result = val[0] << val[2] }
+  ;
+
+  Dictionary:
+    DictionaryLiteral                     { result = DictionaryNode.new(val[0]) }
+  ;
+
+  # {'key': 'value', 'key': 'value'}
+  DictionaryLiteral:
+    '{' DictItems '}'                     { result = Hash[val[1]] }
+  | '{' DictItems ',' '}'                 { result = Hash[val[1]] }
+  ;
+
+  # [[key, value], [key, value]]
+  DictItems:
+    /* nothing */                         { result = [] }
+  | DictItem                              { result = val }
+  | DictItems ',' DictItem                { result = val[0] << val[2] }
+  ;
+
+  # [key, value]
+  DictItem:
+    Literal ':' Literal                   { result = [val[0], val[2]] }
+  ;
+
+  DictGet:
+    Dictionary ListOrDictGetWithBrackets         { result = DictGetBracketNode.new(val[0], val[1]) }
+  | Dictionary DictGetWithDotLiteral             { result = DictGetDotNode.new(val[0], val[1]) }
+  | VariableRetrieval DictGetWithDot             { result = DictGetDotNode.new(val[0], val[1]) }
+  | ListOrDictGet DictGetWithDot                 { result = DictGetDotNode.new(val[0], val[1]) }
+  ;
+
+  ListOrDictGet:
+    VariableRetrieval ListOrDictGetWithBrackets    { result = ListOrDictGetNode.new(val[0], val[1]) }
+  | DictGet ListOrDictGetWithBrackets              { result = ListOrDictGetNode.new(val[0], val[1]) }
+  | Call ListOrDictGetWithBrackets                 { result = ListOrDictGetNode.new(val[0], val[1]) }
+  ;
+
+  ListOrDictGetWithBrackets:
+    '['  Expression ']'                          { result = [val[1]] }
+  | '['  SubList    ']'                          { result = [val[1]] }
+  | ListOrDictGetWithBrackets '[' Expression ']' { result = val[0] << val[2] }
+  | ListOrDictGetWithBrackets '[' SubList    ']' { result = val[0] << val[2] }
+  ;
+
+  SubList:
+    Expression ':' Expression                    { result = Nodes.new([val[0], LiteralNode.new(' : '), val[2]]) }
+  | Expression ':'                               { result = Nodes.new([val[0], LiteralNode.new(' :')]) }
+  | ':' Expression                               { result = Nodes.new([LiteralNode.new(': '), val[1]]) }
+  | ':'                                          { result = Nodes.new([LiteralNode.new(':')]) }
+  ;
+
+  DictGetWithDot:
+    DICT_VAL                        { result = [val[0]]}
+  | DictGetWithDot DICT_VAL         { result = val[0] << val[1] }
+  ;
+
+  DictGetWithDotLiteral:
+    '.' IDENTIFIER                  { result = [val[1]] }
+  | DictGetWithDotLiteral DICT_VAL  { result = val[0] << val[1] }
+  ;
+
+  Call:
+    Scope DefCallIdentifier '(' ArgList ')'       { result = CallNode.new(val[0], val[1], val[3]) }
+  | DictGet '(' ArgList ')'                       { result = CallNode.new(nil, val[0], val[2]) }
+  | CALL Scope DefCallIdentifier '(' ArgList ')'  { result = ExplicitCallNode.new(val[1], val[2], val[4]) }
+  | CALL DictGet '(' ArgList ')'                  { result = ExplicitCallNode.new(nil, val[1], val[3]) }
+  | BUILTIN_COMMAND '(' ArgList ')'               { result = CallNode.new(nil, val[0], val[2]) }
+  | BUILTIN_COMMAND ArgList                       { result = CallNode.new(nil, val[0], val[1]) }
+  ;
+
+  Scope:
+    SCOPE_MODIFIER         { result = val[0] }
+  | /* nothing */          { result = nil }
+  ;
+
+  ArgList:
+    /* nothing */                         { result = [] }
+  | Expression                            { result = val }
+  | ArgList "," Expression                { result = val[0] << val[2] }
+  ;
+
+  BinaryOperator:
+    Expression '||' Expression            { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '&&' Expression            { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+
+  | Expression '==' Expression            { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '==#' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '==?' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+
+  # added by riml
+  | Expression '===' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+
+  | Expression '!=' Expression            { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '!=#' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '!=?' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+
+  | Expression '=~' Expression            { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '=~#' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '=~?' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+
+  | Expression '!~' Expression            { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '!~#' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '!~?' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+
+  | Expression '>' Expression             { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '>#' Expression            { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '>?' Expression            { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+
+  | Expression '>=' Expression            { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '>=#' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '>=?' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+
+  | Expression '<' Expression             { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '<#' Expression            { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '<?' Expression            { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+
+  | Expression '<=' Expression            { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '<=#' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '<=?' Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+
+  | Expression '+' Expression             { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '-' Expression             { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '*' Expression             { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '/' Expression             { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression '.' Expression             { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+
+  | Expression IS    Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  | Expression ISNOT Expression           { result = BinaryOperatorNode.new(val[1], [val[0], val[2]]) }
+  ;
+
+  UnaryOperator:
+    '!' Expression                        { result = UnaryOperatorNode.new(val[0], val[1]) }
+  ;
+
+  # ['=', LHS, RHS]
+  Assign:
+    LET AssignExpression                  { result = AssignNode.new(val[1][0], val[1][1], val[1][2]) }
+  | AssignExpression                      { result = AssignNode.new(val[0][0], val[0][1], val[0][2]) }
+  ;
+
+  # ['=', AssignLHS, Expression]
+  AssignExpression:
+    AssignLHS '='  Expression             { result = [val[1], val[0], val[2]] }
+  | AssignLHS '+=' Expression             { result = [val[1], val[0], val[2]] }
+  | AssignLHS '-=' Expression             { result = [val[1], val[0], val[2]] }
+  | AssignLHS '.=' Expression             { result = [val[1], val[0], val[2]] }
+  ;
+
+  AssignLHS:
+    VariableRetrieval                     { result = val[0] }
+  | DictGet                               { result = val[0] }
+  | List                                  { result = val[0] }
+  | ListOrDictGet                         { result = val[0] }
+  ;
+
+  # retrieving the value of a variable
+  VariableRetrieval:
+    Scope IDENTIFIER                            { result = GetVariableNode.new(val[0], val[1]) }
+  | SPECIAL_VAR_PREFIX IDENTIFIER               { result = GetSpecialVariableNode.new(val[0], val[1]) }
+  | Scope CurlyBraceName                        { result = GetCurlyBraceNameNode.new(val[0], val[1])}
+  ;
+
+  UnletVariable:
+    UNLET VariableRetrieval                               { result = UnletVariableNode.new('!', [ val[1] ]) }
+  | UNLET_BANG VariableRetrieval                          { result = UnletVariableNode.new('!', [ val[1] ]) }
+  | UnletVariable VariableRetrieval                       { result = val[0] << val[1] }
+  ;
+
+  CurlyBraceName:
+    IDENTIFIER '{' VariableRetrieval '}'               { result = CurlyBraceVariable.new([ CurlyBracePart.new(val[0]), CurlyBracePart.new(val[2]) ]) }
+  | '{' VariableRetrieval '}' IDENTIFIER               { result = CurlyBraceVariable.new([ CurlyBracePart.new(val[1]), CurlyBracePart.new(val[3]) ]) }
+  | CurlyBraceName IDENTIFIER                          { result = val[0] << CurlyBracePart.new(val[1]) }
+  ;
+
+  # Method definition
+  # [scope_modifier, name, parameters, keyword, expressions]
+  Def:
+    FunctionType Scope DefCallIdentifier Keyword Block END                               { result = Object.const_get(val[0]).new('!', val[1], val[2], [], val[3], val[4]) }
+  | FunctionType Scope DefCallIdentifier '(' ParamList ')' Keyword Block END             { result = Object.const_get(val[0]).new('!', val[1], val[2], val[4], val[6], val[7]) }
+  | FunctionType Scope DefCallIdentifier '(' SPLAT     ')' Keyword Block END             { result = Object.const_get(val[0]).new('!', val[1], val[2], [val[4]], val[6], val[7]) }
+  | FunctionType Scope DefCallIdentifier '(' ParamList ',' SPLAT ')' Keyword Block END   { result = Object.const_get(val[0]).new('!', val[1], val[2], val[4] << val[6], val[8], val[9]) }
+  ;
+
+  FunctionType:
+    DEF           { result = "DefNode" }
+  | DEF_BANG      { result = "DefNode" }
+  | DEFM          { result = "DefMethodNode" }
+  ;
+
+  DefCallIdentifier:
+    # use '' for first argument instead of nil in order to avoid a double scope-modifier
+    CurlyBraceName          { result = GetCurlyBraceNameNode.new('', val[0])}
+  | IDENTIFIER              { result = val[0] }
+  ;
+
+  # Example: 'range' or 'dict' after function definition
+  Keyword:
+    IDENTIFIER            { result = val[0] }
+  | /* nothing */         { result = nil }
+  ;
+
+  ParamList:
+    /* nothing */                         { result = [] }
+  | IDENTIFIER                            { result = val }
+  | ParamList ',' IDENTIFIER              { result = val[0] << val[2] }
+  ;
+
+  Return:
+    RETURN Expression                     { result = ReturnNode.new(val[1]) }
+  | RETURN Terminator                     { result = ReturnNode.new(nil)    }
+  ;
+
+  EndScript:
+    FINISH                                { result = FinishNode.new }
+  ;
+
+  # [expression, expressions]
+  If:
+    IF Expression IfBlock END               { result = IfNode.new(val[1], val[2]) }
+  | IF Expression THEN Expression END       { result = IfNode.new(val[1], Nodes.new([val[3]])) }
+  | Expression IF Expression                { result = IfNode.new(val[2], Nodes.new([val[0]])) }
+  ;
+
+  Unless:
+    UNLESS Expression IfBlock END           { result = UnlessNode.new(val[1], val[2]) }
+  | UNLESS Expression THEN Expression END   { result = UnlessNode.new(val[1], Nodes.new([val[3]])) }
+  | Expression UNLESS Expression            { result = UnlessNode.new(val[2], Nodes.new([val[0]])) }
+  ;
+
+  Ternary:
+    Expression '?' Expression ':' Expression   { result = TernaryOperatorNode.new([val[0], val[2], val[4]]) }
+  ;
+
+  While:
+    WHILE Expression Block END                 { result = WhileNode.new(val[1], val[2]) }
+  ;
+
+  LoopKeyword:
+    BREAK                                      { result = BreakNode.new }
+  | CONTINUE                                   { result = ContinueNode.new }
+  ;
+
+  Until:
+    UNTIL Expression Block END                 { result = UntilNode.new(val[1], val[2]) }
+  ;
+
+  For:
+    FOR IDENTIFIER IN Expression Block END     { result = ForNode.new(val[1], val[3], val[4]) }
+  | FOR List IN Expression Block END           { result = ForNode.new(val[1], val[3], val[4]) }
+  ;
+
+  Try:
+    TRY Block END                              { result = TryNode.new(val[1], nil, nil) }
+  | TRY Block Catch END                        { result = TryNode.new(val[1], val[2], nil) }
+  | TRY Block Catch ENSURE Block END           { result = TryNode.new(val[1], val[2], val[4]) }
+  ;
+
+  Catch:
+    /* nothing */                              { result = nil }
+  | CATCH Block                                { result = [ CatchNode.new(nil, val[1]) ] }
+  | CATCH Regexp Block                         { result = [ CatchNode.new(val[1], val[2]) ] }
+  | Catch CATCH Block                          { result = val[0] << CatchNode.new(nil, val[2]) }
+  | Catch CATCH Regexp Block                   { result = val[0] << CatchNode.new(val[2], val[3]) }
+  ;
+
+  # [expressions]
+  # expressions list could contain an ElseNode, which contains expressions
+  # itself
+  Block:
+    NEWLINE Expressions                        { result = val[1] }
+  | NEWLINE                                    { result = Nodes.new([]) }
+  ;
+
+  IfBlock:
+    Block                                      { result = val[0] }
+  | NEWLINE Expressions ElseBlock              { result = val[1].concat(val[2]) }
+  ;
+
+  ElseBlock:
+    ElseifBlock ELSE NEWLINE Expressions       { result = [val[0], ElseNode.new(val[3])].compact }
+  ;
+
+  ElseifBlock:
+    ELSEIF Expression NEWLINE Expressions                   { result = Nodes.new([ElseifNode.new(val[1], val[3])]) }
+  | ElseifBlock ELSEIF Expression NEWLINE Expressions       { result = val[0] << ElseifNode.new(val[2], val[4]) }
+  | /* nothing */                                           { result = nil }
+  ;
+
+  ClassDefinition:
+    CLASS IDENTIFIER Block END                   { result = ClassDefinitionNode.new(val[1], nil, val[2]) }
+  | CLASS IDENTIFIER '<' IDENTIFIER Block END    { result = ClassDefinitionNode.new(val[1], val[3], val[4]) }
+  ;
+
+  ObjectInstantiation:
+    NEW Call                  { result = ObjectInstantiationNode.new(val[1]) }
+  ;
+
+  Super:
+    SUPER '(' ArgList ')'     { result = SuperNode.new(val[2], true) }
+  | SUPER                     { result = SuperNode.new([], false) }
+  ;
+
+  ExLiteral:
+    EX_LITERAL                { result = ExLiteralNode.new(val[0])}
+  ;
+end
+
+---- header
+  require File.expand_path("../lexer", __FILE__)
+  require File.expand_path("../nodes", __FILE__)
+  require File.expand_path("../ast_rewriter", __FILE__)
+  require File.expand_path("../errors", __FILE__)
+---- inner
+  # This code will be put as-is in the parser class
+
+  # parses tokens or code into output nodes
+  def parse(object, rewrite_ast = true)
+    if tokens?(object)
+      @tokens = object
+    elsif code?(object)
+      @lexer = Riml::Lexer.new(object)
+    end
+    ast = do_parse
+    return ast if rewrite_ast == false
+    AST_Rewriter.new(ast).rewrite
+  end
+
+  alias do_parse_without_error_handling do_parse
+  def do_parse_with_error_handling
+    do_parse_without_error_handling
+  rescue Racc::ParseError => e
+    raise unless @lexer
+    raise Riml::ParseError,  "line #{@lexer.lineno}: #{e.message}"
+  end
+  alias do_parse do_parse_with_error_handling
+
+  def next_token
+    return @tokens.shift unless @lexer
+    @lexer.next_token
+  end
+
+  private
+  def tokens?(object)
+    Array === object
+  end
+
+  def code?(object)
+    String === object
+  end

data/lib/helper.rb

@@ -0,0 +1,45 @@
+require File.expand_path('../../config/environment', __FILE__)
+require 'nodes'
+require 'lexer'
+require 'parser'
+require 'compiler'
+
+module Riml
+  # lex code into tokens
+  def self.lex(code)
+    Lexer.new(code).tokenize
+  end
+
+  # parse code (or tokens) into nodes
+  def self.parse(input, rewrite_ast = true)
+    unless input.is_a?(Array) || input.is_a?(String)
+      raise ArgumentError, "input must be tokens or code, is #{input.class}"
+    end
+    Parser.new.parse(input, rewrite_ast)
+  end
+
+  # compile nodes (or tokens or code) into output code
+  def self.compile(input)
+    if input.is_a?(Nodes)
+      nodes = input
+    elsif input.is_a?(String) || input.is_a?(Array)
+      nodes = parse(input)
+    else
+      raise ArgumentError, "input must be nodes, tokens or code, is #{input.class}"
+    end
+    Compiler.new.compile(nodes)
+  end
+
+  # expects `file_name` to be readable file
+  def self.compile_file(file_name)
+    input = File.read(file_name)
+    output = compile(input)
+    file_basename = File.basename(file_name)
+    unless File.extname(file_basename).empty?
+      file_basename = file_basename.split(".").tap {|parts| parts.pop}.join(".")
+    end
+    File.open("#{file_basename}.vim", 'w') do |f|
+      f.write output
+    end
+  end
+end