nscript 0.1.0

data/lib/nscript/lexer/lexer.rb

@@ -0,0 +1,230 @@
+module NScript
+
+  class Lexer
+
+    KEYWORDS   = ["if", "else", "then", "unless",
+                  "true", "false", "yes", "no", "on", "off",
+                  "and", "or", "is", "isnt", "not",
+                  "new", "return",
+                  "try", "catch", "finally", "throw",
+                  "break", "continue",
+                  "for", "in", "of", "by", "where", "while",
+                  "delete", "instanceof", "typeof",
+                  "switch", "when",
+                  "super", "extends"]
+
+    IDENTIFIER = /\A([a-zA-Z$_](\w|\$)*)/
+    NUMBER     = /\A(\b((0(x|X)[0-9a-fA-F]+)|([0-9]+(\.[0-9]+)?(e[+\-]?[0-9]+)?)))\b/i
+    STRING     = /\A(""|''|"(.*?)([^\\]|\\\\)"|'(.*?)([^\\]|\\\\)')/m
+    HEREDOC    = /\A("{6}|'{6}|"{3}\n?(.*?)\n?([ \t]*)"{3}|'{3}\n?(.*?)\n?([ \t]*)'{3})/m
+    JS         = /\A(``|`(.*?)([^\\]|\\\\)`)/m
+    OPERATOR   = /\A([+\*&|\/\-%=<>:!?]+)/
+    WHITESPACE = /\A([ \t]+)/
+    COMMENT    = /\A(((\n?[ \t]*)?#.*$)+)/
+    CODE       = /\A((-|=)>)/
+    REGEX      = /\A(\/(.*?)([^\\]|\\\\)\/[imgy]{0,4})/
+    MULTI_DENT = /\A((\n([ \t]*))+)(\.)?/
+    LAST_DENT  = /\n([ \t]*)/
+    ASSIGNMENT = /\A(:|=)\Z/
+
+    JS_CLEANER      = /(\A`|`\Z)/
+    MULTILINER      = /\n/
+    STRING_NEWLINES = /\n[ \t]*/
+    COMMENT_CLEANER = /(^[ \t]*#|\n[ \t]*$)/
+    NO_NEWLINE      = /\A([+\*&|\/\-%=<>:!.\\][<>=&|]*|and|or|is|isnt|not|delete|typeof|instanceof)\Z/
+    HEREDOC_INDENT  = /^[ \t]+/
+
+    NOT_REGEX  = [
+      :IDENTIFIER, :NUMBER, :REGEX, :STRING,
+      ')', '++', '--', ']', '}',
+      :FALSE, :NULL, :TRUE
+    ]
+
+    CALLABLE = [:IDENTIFIER, :SUPER, ')', ']', '}', :STRING]
+
+    def tokenize(code)
+      @code    = code.chomp # Cleanup code by remove extra line breaks
+      @i       = 0          # Current character position we're parsing
+      @line    = 1          # The current line.
+      @indent  = 0          # The current indent level.
+      @indents = []         # The stack of all indent levels we are currently within.
+      @tokens  = []         # Collection of all parsed tokens in the form [:TOKEN_TYPE, value]
+      @spaced  = nil        # The last value that has a space following it.
+      while @i < @code.length
+        @chunk = @code[@i..-1]
+        extract_next_token
+      end
+      puts "original stream: #{@tokens.inspect}" if ENV['VERBOSE']
+      close_indentation
+      Rewriter.new.rewrite(@tokens)
+    end
+
+    def extract_next_token
+      return if identifier_token
+      return if number_token
+      return if heredoc_token
+      return if string_token
+      return if js_token
+      return if regex_token
+      return if indent_token
+      return if comment_token
+      return if whitespace_token
+      return    literal_token
+    end
+
+    def identifier_token
+      return false unless identifier = @chunk[IDENTIFIER, 1]
+      # Keywords are special identifiers tagged with their own name,
+      # 'if' will result in an [:IF, "if"] token.
+      tag = KEYWORDS.include?(identifier) ? identifier.upcase.to_sym : :IDENTIFIER
+      tag = :LEADING_WHEN if tag == :WHEN && [:OUTDENT, :INDENT, "\n"].include?(last_tag)
+      @tokens[-1][0] = :PROTOTYPE_ACCESS if tag == :IDENTIFIER && last_value == '::'
+      if tag == :IDENTIFIER && last_value == '.' && !(@tokens[-2] && @tokens[-2][1] == '.')
+        if @tokens[-2][0] == "?"
+          @tokens[-1][0] = :SOAK_ACCESS
+          @tokens.delete_at(-2)
+        else
+          @tokens[-1][0] = :PROPERTY_ACCESS
+        end
+      end
+      token(tag, identifier)
+      @i += identifier.length
+    end
+
+    def number_token
+      return false unless number = @chunk[NUMBER, 1]
+      token(:NUMBER, number)
+      @i += number.length
+    end
+
+    def string_token
+      return false unless string = @chunk[STRING, 1]
+      escaped = string.gsub(STRING_NEWLINES, " \\\n")
+      token(:STRING, escaped)
+      @line += string.count("\n")
+      @i += string.length
+    end
+
+    def heredoc_token
+      return false unless match = @chunk.match(HEREDOC)
+      doc = match[2] || match[4]
+      indent = doc.scan(HEREDOC_INDENT).min
+      doc.gsub!(/^#{indent}/, "")
+      doc.gsub!("\n", "\\n")
+      doc.gsub!('"', '\\"')
+      token(:STRING, "\"#{doc}\"")
+      @line += match[1].count("\n")
+      @i += match[1].length
+    end
+
+    def js_token
+      return false unless script = @chunk[JS, 1]
+      token(:JS, script.gsub(JS_CLEANER, ''))
+      @i += script.length
+    end
+
+    def regex_token
+      return false unless regex = @chunk[REGEX, 1]
+      return false if NOT_REGEX.include?(last_tag)
+      token(:REGEX, regex)
+      @i += regex.length
+    end
+
+    def comment_token
+      return false unless comment = @chunk[COMMENT, 1]
+      @line += comment.scan(MULTILINER).length
+      token(:COMMENT, comment.gsub(COMMENT_CLEANER, '').split(MULTILINER))
+      token("\n", "\n")
+      @i += comment.length
+    end
+
+    def indent_token
+      return false unless indent = @chunk[MULTI_DENT, 1]
+      @line += indent.scan(MULTILINER).size
+      @i += indent.size
+      next_character = @chunk[MULTI_DENT, 4]
+      no_newlines = next_character == '.' || (last_value.to_s.match(NO_NEWLINE) && @tokens[-2][0] != '.'  && !last_value.match(CODE))
+      return suppress_newlines(indent) if no_newlines
+      size = indent.scan(LAST_DENT).last.last.length
+      return newline_token(indent) if size == @indent
+      if size > @indent
+        token(:INDENT, size - @indent)
+        @indents << (size - @indent)
+      else
+        outdent_token(@indent - size)
+      end
+      @indent = size
+    end
+
+    def outdent_token(move_out)
+      while move_out > 0 && !@indents.empty?
+        last_indent = @indents.pop
+        token(:OUTDENT, last_indent)
+        move_out -= last_indent
+      end
+      token("\n", "\n")
+    end
+
+    def whitespace_token
+      return false unless whitespace = @chunk[WHITESPACE, 1]
+      @spaced = last_value
+      @i += whitespace.length
+    end
+
+    def newline_token(newlines)
+      token("\n", "\n") unless last_value == "\n"
+      true
+    end
+
+    def suppress_newlines(newlines)
+      @tokens.pop if last_value == "\\"
+      true
+    end
+
+    def literal_token
+      value = @chunk[OPERATOR, 1]
+      tag_parameters if value && value.match(CODE)
+      value ||= @chunk[0,1]
+      tag = value.match(ASSIGNMENT) ? :ASSIGN : value
+      if !@spaced.equal?(last_value) && CALLABLE.include?(last_tag)
+        tag = :CALL_START  if value == '('
+        tag = :INDEX_START if value == '['
+      end
+      token(tag, value)
+      @i += value.length
+    end
+
+    def token(tag, value)
+      @tokens << [tag, Value.new(value, @line)]
+    end
+
+    def last_value
+      @tokens.last && @tokens.last[1]
+    end
+
+    def last_tag
+      @tokens.last && @tokens.last[0]
+    end
+
+    def tag_parameters
+      return if last_tag != ')'
+      i = 0
+      loop do
+        i -= 1
+        tok = @tokens[i]
+        return if !tok
+        case tok[0]
+        when :IDENTIFIER  then tok[0] = :PARAM
+        when ')'          then tok[0] = :PARAM_END
+        when '('          then return tok[0] = :PARAM_START
+        end
+      end
+    end
+
+    def close_indentation
+      outdent_token(@indent)
+    end
+
+  end
+
+end

data/lib/nscript/lexer/rewriter.rb

@@ -0,0 +1,238 @@
+module NScript
+
+  class Rewriter
+
+    BALANCED_PAIRS = [['(', ')'], ['[', ']'], ['{', '}'], [:INDENT, :OUTDENT],
+      [:PARAM_START, :PARAM_END], [:CALL_START, :CALL_END], [:INDEX_START, :INDEX_END]]
+
+    EXPRESSION_START = BALANCED_PAIRS.map {|pair| pair.first }
+    EXPRESSION_TAIL  = BALANCED_PAIRS.map {|pair| pair.last }
+    EXPRESSION_CLOSE = [:CATCH, :WHEN, :ELSE, :FINALLY] + EXPRESSION_TAIL
+
+    IMPLICIT_FUNC = [:IDENTIFIER, :SUPER, ')', :CALL_END, ']', :INDEX_END]
+    IMPLICIT_END  = [:IF, :UNLESS, :FOR, :WHILE, "\n", :OUTDENT]
+    IMPLICIT_CALL = [:IDENTIFIER, :NUMBER, :STRING, :JS, :REGEX, :NEW, :PARAM_START,
+                     :TRY, :DELETE, :TYPEOF, :SWITCH,
+                     :TRUE, :FALSE, :YES, :NO, :ON, :OFF, '!', '!!', :NOT,
+                     '@', '->', '=>', '[', '(', '{']
+
+     INVERSES = BALANCED_PAIRS.inject({}) do |memo, pair|
+       memo[pair.first] = pair.last
+       memo[pair.last]  = pair.first
+       memo
+     end
+
+     SINGLE_LINERS  = [:ELSE, "->", "=>", :TRY, :FINALLY, :THEN]
+     SINGLE_CLOSERS = ["\n", :CATCH, :FINALLY, :ELSE, :OUTDENT, :LEADING_WHEN, :PARAM_START]
+
+     def rewrite(tokens)
+       @tokens = tokens
+       adjust_comments
+       remove_leading_newlines
+       remove_mid_expression_newlines
+       move_commas_outside_outdents
+       close_open_calls_and_indexes
+       add_implicit_parentheses
+       add_implicit_indentation
+       ensure_balance(*BALANCED_PAIRS)
+       rewrite_closing_parens
+       @tokens
+     end
+
+     def scan_tokens
+       i = 0
+       loop do
+         break unless @tokens[i]
+         move = yield(@tokens[i - 1], @tokens[i], @tokens[i + 1], i)
+         i += move
+       end
+     end
+
+     def adjust_comments
+       scan_tokens do |prev, token, post, i|
+         next 1 unless token[0] == :COMMENT
+         before, after = @tokens[i - 2], @tokens[i + 2]
+         if before && after &&
+             ((before[0] == :INDENT && after[0] == :OUTDENT) ||
+             (before[0] == :OUTDENT && after[0] == :INDENT)) &&
+             before[1] == after[1]
+           @tokens.delete_at(i + 2)
+           @tokens.delete_at(i - 2)
+           next 0
+         elsif prev[0] == "\n" && [:INDENT].include?(after[0])
+           @tokens.delete_at(i + 2)
+           @tokens[i - 1] = after
+           next 1
+         elsif !["\n", :INDENT, :OUTDENT].include?(prev[0])
+           @tokens.insert(i, ["\n", Value.new("\n", token[1].line)])
+           next 2
+         else
+           next 1
+         end
+       end
+     end
+
+     def remove_leading_newlines
+       @tokens.shift if @tokens[0][0] == "\n"
+     end
+
+     def remove_mid_expression_newlines
+       scan_tokens do |prev, token, post, i|
+         next 1 unless post && EXPRESSION_CLOSE.include?(post[0]) && token[0] == "\n"
+         @tokens.delete_at(i)
+         next 0
+       end
+     end
+
+     def move_commas_outside_outdents
+       scan_tokens do |prev, token, post, i|
+         if token[0] == :OUTDENT && prev[0] == ','
+           @tokens.delete_at(i)
+           @tokens.insert(i - 1, token)
+         end
+         next 1
+       end
+     end
+
+     def close_open_calls_and_indexes
+       parens, brackets = [0], [0]
+       scan_tokens do |prev, token, post, i|
+         case token[0]
+         when :CALL_START  then parens.push(0)
+         when :INDEX_START then brackets.push(0)
+         when '('          then parens[-1] += 1
+         when '['          then brackets[-1] += 1
+         when ')'
+           if parens.last == 0
+             parens.pop
+             token[0] = :CALL_END
+           else
+             parens[-1] -= 1
+           end
+         when ']'
+           if brackets.last == 0
+             brackets.pop
+             token[0] = :INDEX_END
+           else
+             brackets[-1] -= 1
+           end
+         end
+         next 1
+       end
+     end
+
+     def add_implicit_parentheses
+       stack = [0]
+       scan_tokens do |prev, token, post, i|
+         stack.push(0) if token[0] == :INDENT
+         if token[0] == :OUTDENT
+           last = stack.pop
+           stack[-1] += last
+         end
+         if stack.last > 0 && (IMPLICIT_END.include?(token[0]) || post.nil?)
+           idx = token[0] == :OUTDENT ? i + 1 : i
+           stack.last.times { @tokens.insert(idx, [:CALL_END, Value.new(')', token[1].line)]) }
+           size, stack[-1] = stack[-1] + 1, 0
+           next size
+         end
+         next 1 unless IMPLICIT_FUNC.include?(prev[0]) && IMPLICIT_CALL.include?(token[0])
+         @tokens.insert(i, [:CALL_START, Value.new('(', token[1].line)])
+         stack[-1] += 1
+         next 2
+       end
+     end
+
+     def add_implicit_indentation
+       scan_tokens do |prev, token, post, i|
+         next 1 unless SINGLE_LINERS.include?(token[0]) && post[0] != :INDENT &&
+           !(token[0] == :ELSE && post[0] == :IF) # Elsifs shouldn't get blocks.
+         starter = token[0]
+         line = token[1].line
+         @tokens.insert(i + 1, [:INDENT, Value.new(2, line)])
+         idx = i + 1
+         parens = 0
+         loop do
+           idx += 1
+           tok = @tokens[idx]
+           if (!tok || SINGLE_CLOSERS.include?(tok[0]) ||
+               (tok[0] == ')' && parens == 0)) &&
+               !(starter == :ELSE && tok[0] == :ELSE)
+             insertion = @tokens[idx - 1][0] == "," ? idx - 1 : idx
+             @tokens.insert(insertion, [:OUTDENT, Value.new(2, line)])
+             break
+           end
+           parens += 1 if tok[0] == '('
+           parens -= 1 if tok[0] == ')'
+         end
+         next 1 unless token[0] == :THEN
+         @tokens.delete_at(i)
+         next 0
+       end
+     end
+
+     def ensure_balance(*pairs)
+       puts "\nbefore ensure_balance: #{@tokens.inspect}" if ENV['VERBOSE']
+       levels, lines = Hash.new(0), Hash.new
+       scan_tokens do |prev, token, post, i|
+         pairs.each do |pair|
+           open, close = *pair
+           levels[open] += 1 if token[0] == open
+           levels[open] -= 1 if token[0] == close
+           lines[token[0]] = token[1].line
+           raise ParseError.new(token[0], token[1], nil) if levels[open] < 0
+         end
+         next 1
+       end
+       unclosed = levels.detect {|k, v| v > 0 }
+       sym = unclosed && unclosed[0]
+       raise ParseError.new(sym, Value.new(sym, lines[sym]), nil, "unclosed '#{sym}'") if unclosed
+     end
+
+     def rewrite_closing_parens
+       verbose = ENV['VERBOSE']
+       stack, debt = [], Hash.new(0)
+       stack_stats = lambda { "stack: #{stack.inspect} debt: #{debt.inspect}\n\n" }
+       puts "rewrite_closing_original: #{@tokens.inspect}" if verbose
+       scan_tokens do |prev, token, post, i|
+         tag, inv = token[0], INVERSES[token[0]]
+         # Push openers onto the stack.
+         if EXPRESSION_START.include?(tag)
+           stack.push(token)
+           puts "pushing #{tag} #{stack_stats[]}" if verbose
+           next 1
+         # The end of an expression, check stack and debt for a pair.
+         elsif EXPRESSION_TAIL.include?(tag)
+           puts @tokens[i..-1].inspect if verbose
+           # If the tag is already in our debt, swallow it.
+           if debt[inv] > 0
+             debt[inv] -= 1
+             @tokens.delete_at(i)
+             puts "tag in debt #{tag} #{stack_stats[]}" if verbose
+             next 0
+           else
+             # Pop the stack of open delimiters.
+             match = stack.pop
+             mtag  = match[0]
+             # Continue onwards if it's the expected tag.
+             if tag == INVERSES[mtag]
+               puts "expected tag #{tag} #{stack_stats[]}" if verbose
+               next 1
+             else
+               # Unexpected close, insert correct close, adding to the debt.
+               debt[mtag] += 1
+               puts "unexpected #{tag}, replacing with #{INVERSES[mtag]} #{stack_stats[]}" if verbose
+               val = mtag == :INDENT ? match[1] : INVERSES[mtag]
+               @tokens.insert(i, [INVERSES[mtag], Value.new(val, token[1].line)])
+               next 1
+             end
+           end
+         else
+           # Uninteresting token:
+           next 1
+         end
+       end
+     end
+
+  end
+
+end