coffee-script 0.1.0

data/lib/coffee_script/grammar.y

@@ -0,0 +1,403 @@
+class Parser
+
+# Declare tokens produced by the lexer
+token IF ELSE THEN UNLESS
+token NUMBER STRING REGEX
+token TRUE FALSE YES NO ON OFF
+token IDENTIFIER PROPERTY_ACCESS
+token CODE PARAM NEW RETURN
+token TRY CATCH FINALLY THROW
+token BREAK CONTINUE
+token FOR IN WHILE
+token SWITCH WHEN
+token SUPER
+token DELETE
+token NEWLINE
+token COMMENT
+token JS
+
+# Declare order of operations.
+prechigh
+  nonassoc UMINUS NOT '!' '!!' '~' '++' '--'
+  left     '*' '/' '%'
+  left     '+' '-'
+  left     '<<' '>>' '>>>'
+  left     '&' '|' '^'
+  left     '<=' '<' '>' '>='
+  right    '==' '!=' IS AINT
+  left     '&&' '||' AND OR
+  right    '-=' '+=' '/=' '*='
+  right    DELETE
+  left     "."
+  right    THROW FOR IN WHILE NEW
+  left     UNLESS IF ELSE
+  left     ":" '||:' '&&:'
+  right    RETURN
+preclow
+
+# We expect 4 shift/reduce errors for optional syntax.
+# There used to be 252 -- greatly improved.
+expect 4
+
+rule
+
+  # All parsing will end in this rule, being the trunk of the AST.
+  Root:
+    /* nothing */                     { result = Expressions.new([]) }
+  | Terminator                        { result = Expressions.new([]) }
+  | Expressions                       { result = val[0] }
+  ;
+
+  # Any list of expressions or method body, seperated by line breaks or semis.
+  Expressions:
+    Expression                        { result = Expressions.new(val) }
+  | Expressions Terminator Expression { result = val[0] << val[2] }
+  | Expressions Terminator            { result = val[0] }
+  | Terminator Expressions            { result = val[1] }
+  ;
+
+  # All types of expressions in our language.
+  Expression:
+    PureExpression
+  | Statement
+  ;
+
+  # The parts that are natural JavaScript expressions.
+  PureExpression:
+    Literal
+  | Value
+  | Call
+  | Code
+  | Operation
+  ;
+
+  # We have to take extra care to convert these statements into expressions.
+  Statement:
+    Assign
+  | If
+  | Try
+  | Throw
+  | Return
+  | While
+  | For
+  | Switch
+  | Comment
+  ;
+
+  # All tokens that can terminate an expression.
+  Terminator:
+    "\n"
+  | ";"
+  ;
+
+  # All tokens that can serve to begin the second block of a multi-part expression.
+  Then:
+    THEN
+  | Terminator
+  ;
+
+  # All hard-coded values.
+  Literal:
+    NUMBER                            { result = LiteralNode.new(val[0]) }
+  | STRING                            { result = LiteralNode.new(val[0]) }
+  | JS                                { result = LiteralNode.new(val[0]) }
+  | REGEX                             { result = LiteralNode.new(val[0]) }
+  | BREAK                             { result = LiteralNode.new(val[0]) }
+  | CONTINUE                          { result = LiteralNode.new(val[0]) }
+  | TRUE                              { result = LiteralNode.new(true) }
+  | FALSE                             { result = LiteralNode.new(false) }
+  | YES                               { result = LiteralNode.new(true) }
+  | NO                                { result = LiteralNode.new(false) }
+  | ON                                { result = LiteralNode.new(true) }
+  | OFF                               { result = LiteralNode.new(false) }
+  ;
+
+  # Assignment to a variable.
+  Assign:
+    Value ":" Expression              { result = AssignNode.new(val[0], val[2]) }
+  ;
+
+  # Assignment within an object literal.
+  AssignObj:
+    IDENTIFIER ":" Expression         { result = AssignNode.new(val[0], val[2], :object) }
+  | Comment                           { result = val[0] }
+  ;
+
+  # A return statement.
+  Return:
+    RETURN Expression                 { result = ReturnNode.new(val[1]) }
+  ;
+
+  # A comment.
+  Comment:
+    COMMENT                           { result = CommentNode.new(val[0]) }
+  ;
+
+  # Arithmetic and logical operators
+  # For Ruby's Operator precedence, see:
+  # https://www.cs.auckland.ac.nz/references/ruby/ProgrammingRuby/language.html
+  Operation:
+    '!' Expression                    { result = OpNode.new(val[0], val[1]) }
+  | '!!' Expression                   { result = OpNode.new(val[0], val[1]) }
+  | '-' Expression = UMINUS           { result = OpNode.new(val[0], val[1]) }
+  | NOT Expression                    { result = OpNode.new(val[0], val[1]) }
+  | '~' Expression                    { result = OpNode.new(val[0], val[1]) }
+  | '--' Expression                    { result = OpNode.new(val[0], val[1]) }
+  | '++' Expression                    { result = OpNode.new(val[0], val[1]) }
+  | Expression '--'                    { result = OpNode.new(val[1], val[0], nil, true) }
+  | Expression '++'                    { result = OpNode.new(val[1], val[0], nil, true) }
+
+  | Expression '*' Expression         { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '/' Expression         { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '%' Expression         { result = OpNode.new(val[1], val[0], val[2]) }
+
+  | Expression '+' Expression         { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '-' Expression         { result = OpNode.new(val[1], val[0], val[2]) }
+
+  | Expression '<<' Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '>>' Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '>>>' Expression       { result = OpNode.new(val[1], val[0], val[2]) }
+
+  | Expression '&' Expression         { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '|' Expression         { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '^' Expression         { result = OpNode.new(val[1], val[0], val[2]) }
+
+  | Expression '<=' Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '<' Expression         { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '>' Expression         { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '>=' Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+
+  | Expression '==' Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '!=' Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression IS Expression          { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression AINT Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+
+  | Expression '&&' Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '||' Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression AND Expression         { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression OR Expression          { result = OpNode.new(val[1], val[0], val[2]) }
+
+  | Expression '-=' Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '+=' Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '/=' Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '*=' Expression        { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '||:' Expression       { result = OpNode.new(val[1], val[0], val[2]) }
+  | Expression '&&:' Expression       { result = OpNode.new(val[1], val[0], val[2]) }
+
+  | DELETE Expression                 { result = OpNode.new(val[0], val[1]) }
+  ;
+
+  # Function definition.
+  Code:
+    ParamList "=>" CodeBody "."       { result = CodeNode.new(val[0], val[2]) }
+  | "=>" CodeBody "."                 { result = CodeNode.new([], val[1]) }
+  ;
+
+  # The body of a function.
+  CodeBody:
+    /* nothing */                     { result = Expressions.new([]) }
+  | Expressions                       { result = val[0] }
+  ;
+
+  # The parameters to a function definition.
+  ParamList:
+    PARAM                             { result = val }
+  | ParamList "," PARAM               { result = val[0] << val[2] }
+  ;
+
+  # Expressions that can be treated as values.
+  Value:
+    IDENTIFIER                        { result = ValueNode.new(val[0]) }
+  | Array                             { result = ValueNode.new(val[0]) }
+  | Object                            { result = ValueNode.new(val[0]) }
+  | Parenthetical                     { result = ValueNode.new(val[0]) }
+  | Value Accessor                    { result = val[0] << val[1] }
+  | Invocation Accessor               { result = ValueNode.new(val[0], [val[1]]) }
+  ;
+
+  # Accessing into an object or array, through dot or index notation.
+  Accessor:
+    PROPERTY_ACCESS IDENTIFIER        { result = AccessorNode.new(val[1]) }
+  | Index                             { result = val[0] }
+  | Slice                             { result = val[0] }
+  ;
+
+  # Indexing into an object or array.
+  Index:
+    "[" Expression "]"                { result = IndexNode.new(val[1]) }
+  ;
+
+  # Array slice literal.
+  Slice:
+    "[" Expression "," Expression "]" { result = SliceNode.new(val[1], val[3]) }
+  ;
+
+  # An object literal.
+  Object:
+    "{" AssignList "}"                { result = ObjectNode.new(val[1]) }
+  ;
+
+  # Assignment within an object literal (comma or newline separated).
+  AssignList:
+    /* nothing */                     { result = []}
+  | AssignObj                         { result = val }
+  | AssignList "," AssignObj          { result = val[0] << val[2] }
+  | AssignList Terminator AssignObj   { result = val[0] << val[2] }
+  ;
+
+  # All flavors of function call (instantiation, super, and regular).
+  Call:
+    Invocation                        { result = val[0] }
+  | NEW Invocation                    { result = val[1].new_instance }
+  | Super                             { result = val[0] }
+  ;
+
+  # A generic function invocation.
+  Invocation:
+    Value "(" ArgList ")"             { result = CallNode.new(val[0], val[2]) }
+  ;
+
+  # Calling super.
+  Super:
+    SUPER "(" ArgList ")"             { result = CallNode.new(:super, val[2]) }
+  ;
+
+  # The array literal.
+  Array:
+    "[" ArgList "]"                   { result = ArrayNode.new(val[1]) }
+  ;
+
+  # A list of arguments to a method call, or as the contents of an array.
+  ArgList:
+    /* nothing */                     { result = [] }
+  | Expression                        { result = val }
+  | ArgList "," Expression            { result = val[0] << val[2] }
+  | ArgList Terminator Expression     { result = val[0] << val[2] }
+  ;
+
+  # Try/catch/finally exception handling blocks.
+  Try:
+    TRY Expressions Catch "."         { result = TryNode.new(val[1], val[2][0], val[2][1]) }
+  | TRY Expressions Catch
+    FINALLY Expressions "."           { result = TryNode.new(val[1], val[2][0], val[2][1], val[4]) }
+  ;
+
+  # A catch clause.
+  Catch:
+    /* nothing */                     { result = [nil, nil] }
+  | CATCH IDENTIFIER Expressions      { result = [val[1], val[2]] }
+  ;
+
+  # Throw an exception.
+  Throw:
+    THROW Expression                  { result = ThrowNode.new(val[1]) }
+  ;
+
+  # Parenthetical expressions.
+  Parenthetical:
+    "(" Expressions ")"               { result = ParentheticalNode.new(val[1]) }
+  ;
+
+  # The while loop. (there is no do..while).
+  While:
+    WHILE Expression Then
+      Expressions "."                 { result = WhileNode.new(val[1], val[3]) }
+  ;
+
+  # Array comprehensions, including guard and current index.
+  For:
+  Expression FOR IDENTIFIER
+    IN PureExpression "."             { result = ForNode.new(val[0], val[4], val[2], nil) }
+  | Expression FOR
+      IDENTIFIER "," IDENTIFIER
+      IN PureExpression "."           { result = ForNode.new(val[0], val[6], val[2], nil, val[4]) }
+  | Expression FOR IDENTIFIER
+      IN PureExpression
+      IF Expression "."               { result = ForNode.new(val[0], val[4], val[2], val[6]) }
+  | Expression FOR
+      IDENTIFIER "," IDENTIFIER
+      IN PureExpression
+      IF Expression "."               { result = ForNode.new(val[0], val[6], val[2], val[8], val[4]) }
+  ;
+
+  # Switch/When blocks.
+  Switch:
+    SWITCH Expression Then
+      Whens "."                       { result = val[3].rewrite_condition(val[1]) }
+  | SWITCH Expression Then
+      Whens ELSE Expressions "."      { result = val[3].rewrite_condition(val[1]).add_else(val[5]) }
+  ;
+
+  # The inner list of whens.
+  Whens:
+    When                              { result = val[0] }
+  | Whens When                        { result = val[0] << val[1] }
+  ;
+
+  # An individual when.
+  When:
+    WHEN Expression Then Expressions  { result = IfNode.new(val[1], val[3]) }
+  ;
+
+  # All of the following nutso if-else destructuring is to make the
+  # grammar expand unambiguously.
+
+  # An elsif portion of an if-else block.
+  ElsIf:
+    ELSE IF Expression
+      Then Expressions                { result = IfNode.new(val[2], val[4]) }
+  ;
+
+  # Multiple elsifs can be chained together.
+  ElsIfs:
+    ElsIf                             { result = val[0] }
+  | ElsIfs ElsIf                      { result = val[0].add_else(val[1]) }
+  ;
+
+  # Terminating else bodies are strictly optional.
+  ElseBody
+    "."                               { result = nil }
+  | ELSE Expressions "."              { result = val[1] }
+  ;
+
+  # All the alternatives for ending an if-else block.
+  IfEnd:
+    ElseBody                          { result = val[0] }
+  | ElsIfs ElseBody                   { result = val[0].add_else(val[1]) }
+  ;
+
+  # The full complement of if blocks, including postfix one-liner ifs and unlesses.
+  If:
+    IF Expression
+      Then Expressions IfEnd          { result = IfNode.new(val[1], val[3], val[4]) }
+  | Expression IF Expression          { result = IfNode.new(val[2], Expressions.new([val[0]]), nil, {:statement => true}) }
+  | Expression UNLESS Expression      { result = IfNode.new(val[2], Expressions.new([val[0]]), nil, {:statement => true, :invert => true}) }
+  ;
+
+end
+
+---- header
+module CoffeeScript
+
+---- inner
+  # Lex and parse a CoffeeScript.
+  def parse(code)
+    # Uncomment the following line to enable grammar debugging, in combination
+    # with the -g flag in the Rake build task.
+    # @yydebug = true
+    @tokens = Lexer.new.tokenize(code)
+    do_parse
+  end
+
+  # Retrieve the next token from the list.
+  def next_token
+    @tokens.shift
+  end
+
+  # Raise a custom error class that knows about line numbers.
+  def on_error(error_token_id, error_value, value_stack)
+    raise ParseError.new(token_to_str(error_token_id), error_value, value_stack)
+  end
+
+---- footer
+end

data/lib/coffee_script/lexer.rb

@@ -0,0 +1,187 @@
+module CoffeeScript
+
+  # The lexer reads a stream of CoffeeScript and divvys it up into tagged
+  # tokens. A minor bit of the ambiguity in the grammar has been avoided by
+  # pushing some extra smarts into the Lexer.
+  class Lexer
+
+    # The list of keywords passed verbatim to the parser.
+    KEYWORDS   = ["if", "else", "then", "unless",
+                  "true", "false", "yes", "no", "on", "off",
+                  "and", "or", "is", "aint", "not",
+                  "new", "return",
+                  "try", "catch", "finally", "throw",
+                  "break", "continue",
+                  "for", "in", "while",
+                  "switch", "when",
+                  "super",
+                  "delete"]
+
+    # Token matching regexes.
+    IDENTIFIER = /\A([a-zA-Z$_]\w*)/
+    NUMBER     = /\A\b((0(x|X)[0-9a-fA-F]+)|([0-9]+(\.[0-9]+)?(e[+\-]?[0-9]+)?))\b/i
+    STRING     = /\A(""|''|"(.*?)[^\\]"|'(.*?)[^\\]')/m
+    JS         = /\A(``|`(.*?)[^\\]`)/m
+    OPERATOR   = /\A([+\*&|\/\-%=<>:!]+)/
+    WHITESPACE = /\A([ \t\r]+)/
+    NEWLINE    = /\A(\n+)/
+    COMMENT    = /\A((#[^\n]*\s*)+)/m
+    CODE       = /\A(=>)/
+    REGEX      = /\A(\/(.*?)[^\\]\/[imgy]{0,4})/
+
+    # Token cleaning regexes.
+    JS_CLEANER = /(\A`|`\Z)/
+    MULTILINER = /\n/
+    COMMENT_CLEANER = /(^\s*#|\n\s*$)/
+
+    # Tokens that always constitute the start of an expression.
+    EXP_START  = ['{', '(', '[']
+
+    # Tokens that always constitute the end of an expression.
+    EXP_END    = ['}', ')', ']']
+
+    # Scan by attempting to match tokens one character at a time. Slow and steady.
+    def tokenize(code)
+      @code = code.chomp  # Cleanup code by remove extra line breaks
+      @i = 0              # Current character position we're parsing
+      @line = 1           # The current line.
+      @tokens = []        # Collection of all parsed tokens in the form [:TOKEN_TYPE, value]
+      while @i < @code.length
+        @chunk = @code[@i..-1]
+        extract_next_token
+      end
+      @tokens
+    end
+
+    # At every position, run this list of match attempts, short-circuiting if
+    # any of them succeed.
+    def extract_next_token
+      return if identifier_token
+      return if number_token
+      return if string_token
+      return if js_token
+      return if regex_token
+      return if comment_token
+      return if whitespace_token
+      return    literal_token
+    end
+
+    # Matches identifying literals: variables, keywords, method names, etc.
+    def identifier_token
+      return false unless identifier = @chunk[IDENTIFIER, 1]
+      # Keywords are special identifiers tagged with their own name, 'if' will result
+      # in an [:IF, "if"] token
+      tag = KEYWORDS.include?(identifier) ? identifier.upcase.to_sym : :IDENTIFIER
+      @tokens[-1][0] = :PROPERTY_ACCESS if tag == :IDENTIFIER && last_value == '.'
+      token(tag, identifier)
+      @i += identifier.length
+    end
+
+    # Matches numbers, including decimals, hex, and exponential notation.
+    def number_token
+      return false unless number = @chunk[NUMBER, 1]
+      token(:NUMBER, number)
+      @i += number.length
+    end
+
+    # Matches strings, including multi-line strings.
+    def string_token
+      return false unless string = @chunk[STRING, 1]
+      escaped = string.gsub(MULTILINER) do |match|
+        @line += 1
+        " \\\n"
+      end
+      token(:STRING, escaped)
+      @i += string.length
+    end
+
+    # Matches interpolated JavaScript.
+    def js_token
+      return false unless script = @chunk[JS, 1]
+      token(:JS, script.gsub(JS_CLEANER, ''))
+      @i += script.length
+    end
+
+    # Matches regular expression literals.
+    def regex_token
+      return false unless regex = @chunk[REGEX, 1]
+      token(:REGEX, regex)
+      @i += regex.length
+    end
+
+    # Matches and consumes comments.
+    def comment_token
+      return false unless comment = @chunk[COMMENT, 1]
+      @line += comment.scan(MULTILINER).length
+      token(:COMMENT, comment.gsub(COMMENT_CLEANER, '').split(MULTILINER))
+      token("\n", "\n")
+      @i += comment.length
+    end
+
+    # Matches and consumes non-meaningful whitespace.
+    def whitespace_token
+      return false unless whitespace = @chunk[WHITESPACE, 1]
+      @i += whitespace.length
+    end
+
+    # We treat all other single characters as a token. Eg.: ( ) , . !
+    # Multi-character operators are also literal tokens, so that Racc can assign
+    # the proper order of operations. Multiple newlines get merged together.
+    def literal_token
+      value = @chunk[NEWLINE, 1]
+      if value
+        @line += value.length
+        token("\n", "\n") unless last_value == "\n"
+        return @i += value.length
+      end
+      value = @chunk[OPERATOR, 1]
+      tag_parameters if value && value.match(CODE)
+      value ||= @chunk[0,1]
+      skip_following_newlines if EXP_START.include?(value)
+      remove_leading_newlines if EXP_END.include?(value)
+      token(value, value)
+      @i += value.length
+    end
+
+    # Add a token to the results, taking note of the line number, and
+    # immediately-preceding comment.
+    def token(tag, value)
+      @tokens << [tag, Value.new(value, @line)]
+    end
+
+    # Peek at the previous token.
+    def last_value
+      @tokens.last && @tokens.last[1]
+    end
+
+    # A source of ambiguity in our grammar was parameter lists in function
+    # definitions (as opposed to argument lists in function calls). Tag
+    # parameter identifiers in order to avoid this.
+    def tag_parameters
+      index = 0
+      loop do
+        tok = @tokens[index -= 1]
+        return if !tok
+        next if tok[0] == ','
+        return if tok[0] != :IDENTIFIER
+        tok[0] = :PARAM
+      end
+    end
+
+    # Consume and ignore newlines immediately after this point.
+    def skip_following_newlines
+      newlines = @code[(@i+1)..-1][NEWLINE, 1]
+      if newlines
+        @line += newlines.length
+        @i += newlines.length
+      end
+    end
+
+    # Discard newlines immediately before this point.
+    def remove_leading_newlines
+      @tokens.pop if last_value == "\n"
+    end
+
+  end
+
+end