mix-language 1.0.0 → 1.0.1

data/lib/ruby/compiler.rb

@@ -1,528 +1,65 @@
 # encoding: UTF-8
 # 
-# Copyright © 2011 Jesse Sielaff
+# Copyright © 2011-2012 Jesse Sielaff
 # 
+require_relative './precompiler.rb'
+require_relative './tokenizer.rb'
 require_relative './mix.tab.rb'
+require_relative './parser.rb'
+require_relative './translator.rb'
 
 class MixCompiler
-  def initialize (filename)
-    @primary_filename = filename
+  def initialize
+    @precompiler = MixPrecompiler.new
+    @tokenizer = MixTokenizer.new
+    @parser = MixParser.new
+    @translator = MixTranslator.new
   end
   
-  NODE_TYPES = {}
-  %w:
-    access_brackets access_colon
-    access_hash and app array
-    break call false function if
-    mix mixin newline not null
-    number object op or return
-    self set statement string
-    switch true variable while
-  :
-  .each_with_index {|t,i| NODE_TYPES[t.to_sym] = i + 1 }
-  
-  OPERATORS =
-  {
-    '+' => :plus, '-' => :minus,
-    '*' => :star, '/' => :slash, '%' => :percent,
-    '>' => :gt, '>>' => :gtgt, '>=' => :gteq,
-    '<' => :lt, '<<' => :ltlt, '<=' => :lteq,
-    '==' => :eq, '!=' => :neq
-  }
-  
-  def append_statement (outermost_statement, final_statement)
-    current = outermost_statement
-    
-    current = current[:values][1] while current[:values][1]
-    current[:values][1] = final_statement
-    
-    outermost_statement
-  end
-  
-  def build_script
-    script = File.open(@primary_filename) {|f| "#{f.read}\n" }
-    files = {'mix' => [], 'js' => %w[base/env.js base/eval.js base/object.js base/array.js base/false.js base/function.js base/null.js base/number.js base/string.js base/true.js]}
-    
-    @unordered_mix_filenames = [@primary_filename]
-    
-    scan_includes(script, files)
-    files['mix'].push [@primary_filename, script]
-    
-    @js_filenames = files['js']
-    @filenames = files['mix'].map {|name,*| name }
-    @script = files['mix'].map {|*,script| script }.join("\034")
+  def basename (filename)
+    filename.match(/(.*?)(\.mix$|$)/)[1]
   end
   
-  def precompile (js_file_string)
-    NODE_TYPES.each {|k,v| js_file_string.gsub!("case #{k}:","case #{v}:") }
-    
-    %w:ARRAY ELEMENT EVENT FALSE FUNCTION NULL NUMBER OBJECT STRING TEXT TRUE:.each_with_index do |t,i|
-      js_file_string.gsub!("#{t}_TYPE","#{i}")
-    end
-    
-    js_file_string.gsub!(/\s*\/\*.*?\*\/\s*/m,"\n").gsub!(/\s*\n\s*/,?\n).gsub!(/\s*([{;:,=])\s*/,'\1').gsub!("\n}",'}')
+  def combine_strings (js_string, ast_string, symbol_string)
+    js_string
+      .sub(%%'...Replace with AST nodes...'%, ast_string)
+      .sub(%%'...Replace with symbol table...'%, symbol_string)
   end
   
-  def compile
-    build_script
-    node = parse
-    node_string = js(node)
-    symbol_string = @symbols.keys.map {|s| "'#{s}'" }.join ?,
-    
-    js_file_string = @js_filenames.map do |filename|
-      File.read(File.expand_path('../../javascript/' + filename, __FILE__))
-    end.join ?\n
-    
-    precompile(js_file_string)
-    
-    <<-JAVASCRIPT
-mix_result_should_be_undefined = function ()
-{
-
-var HTML_TYPES = {};
-var SYMBOLS = [#{symbol_string.force_encoding('US-ASCII')}];
-#{js_file_string}
-
-window.onload = function ()
-{
-  startup();
-  evaluate(#{node_string});
-}
-
-}();
-    JAVASCRIPT
+  def compile (filename)
+    mix_files, js_string = precompile_js(filename)
+    mix_tokens = tokenize_mix_files(mix_files)
+    mix_node = parse_token_stream(mix_tokens)
+    ast_string, symbol_string = translate_to_js(mix_node)
+    js_output_string = combine_strings(js_string, ast_string, symbol_string)
+    write_js_file(filename, js_output_string)
     
   rescue SyntaxError => e
-    "window.onload = function () { if (console && console.log) console.log(\"SyntaxError: #{e.message}\"); };"
-  end
-  
-  def getch
-    @script[@pointer += 1]
-  end
-  
-  def js (node)
-    return "0" unless node
-    
-    t = node[:type]
-    a, b, c = node[:values]
-    args = [NODE_TYPES[t]]
-    
-    case (t = node[:type])
-      when :access_brackets, :and, :or, :set, :statement, :while
-        args << js(a) << js(b)
-      
-      when :access_colon, :access_hash, :mix
-        args << js(a) << @symbols[b.to_sym]
-      
-      when :app, :false, :null, :self, :true
-        nil
-      
-      when :array
-        items = a.map {|x| js(x) }.join ?,
-        args << "[#{items}]"
-      
-      when :break, :not, :return
-        args << js(a)
-      
-      when :call
-        function = js(a)
-        arguments = b.map {|x| js(x) }.join ?,
-        
-        args << function << "[#{arguments}]"
-      
-      when :function
-        parameters = a[0].map {|x| @symbols[x.to_sym] }.join ?,
-        variables = a[1].map {|x| @symbols[x.to_sym] }.join ?,
-        body = js(b)
-        
-        args << "[[#{parameters}],[#{variables}]]" << body
-      
-      when :if
-        args << js(a) << js(b) << js(c)
-      
-      when :mixin
-        name = @symbols[a.to_sym]
-        init_statement = js(b[0])
-        definitions = b[1].map {|s| js(s) }.join ?,
-        
-        args << name << init_statement << "[#{definitions}]"
-      
-      when :newline
-        file, line = @symbols[a[0].to_sym], a[1]
-        statement = js(b)
-        
-        args << file << line << statement
-      
-      when :number
-        args << a
-      
-      when :object
-        entries = a.map {|kv| "[#{@symbols[kv[0].to_sym]},#{js(kv[1])}]" }.join ?,
-        mixins = b.map {|x| @symbols[x.to_sym] }.join ?,
-        
-        args << "[#{entries}]" << "[#{mixins}]"
-      
-      when :op
-        args << @symbols[OPERATORS[a]] << js(b) << js(c)
-      
-      when :string, :variable
-        args << @symbols[a.to_sym]
-      
-      when :switch
-        condition = js(a)
-        cases = b.map {|x| "[#{js(x[0])},#{js(x[1])}]" }.join ?,
-        else_case = js(c)
-        
-        args << condition << cases << else_case
-      
-      else
-        raise "BUG: node type :#{t} not implemented in MixCompiler#js"
-    end
-    
-    return "[#{args.join ?,}]"
-  end
-  
-  def newline (statement_node)
-    statement_node[:values][0] = node(:newline, @location, statement_node[:values][0])
-    statement_node
+    write_js_file("window.onload = function () { if (console && console.log) console.log(\"SyntaxError: #{e.message}\"); };")
   end
   
-  def next_token
-    @tokens.shift
+  def parse_token_stream (mix_tokens)
+    @parser.parse(mix_tokens)
   end
   
-  def node (type, *values)
-    { type: type, values: values }
+  def precompile_js (filename)
+    mix_filename = basename(filename) + '.mix'
+    @precompiler.precompile(mix_filename)
   end
   
-  def on_error (token, value, stack)
-    if value
-      value, file, line = value[0].inspect, *value[1]
-    else
-      value, file, line = 'EOF', @primary_filename, @line
+  def tokenize_mix_files (mix_files)
+    mix_files.each_with_object([]) do |(filename, script), tokens|
+      @tokenizer.tokenize(filename, script, tokens)
     end
-    
-    raise SyntaxError, "Unexpected token #{token_to_str(token)} (#{value}): #{file}:#{line}".gsub(?",'\"')
-  end
-  
-  def parameter (name)
-    @variables[:parameters] << name
-    @variables[:local] << name
-  end
-  
-  def parse
-    tokenize
-    symbol_index = -1
-    @symbols = Hash.new {|h,k| h[k] = (symbol_index += 1) }
-    @variables = { parameters: [], local: [], embedded: [], prev: nil }
-    
-    return do_parse
-  end
-  
-  def pop_variables
-    vars = [@variables[:parameters], @variables[:embedded]]
-    @variables = @variables[:prev]
-    
-    return vars
-  end
-  
-  def push_variables
-    embedded = @variables[:local] + @variables[:embedded]
-    @variables = { parameters: [], local: [], embedded: embedded, prev: @variables }
   end
   
-  def variable (name)
-    unless (@variables[:local] + @variables[:embedded]).include?(name)
-      @variables[:local] << name
-    end
-    
-    node :variable, name
-  end
-  
-  def scan_includes (script, files)
-    require 'strscan'
-    
-    ss = StringScanner.new(script)
-    js_filenames = []
-    
-    while ss.scan(/~~\s+(.*\.(js|mix))\s*\n/)
-      name = ss[1]
-      type = ss[2]
-      
-      if type == 'mix'
-        unless @unordered_mix_filenames.include?(name)
-          script = File.open(name) {|f| "#{f.read}\n" }
-          @unordered_mix_filenames << name
-          scan_includes(script, files)
-          files['mix'].push [name,script]
-        end
-      else
-        js_filenames << name
-      end
-    end
-    
-    files['js'] |= js_filenames
-  end
-  
-  def token (token, value = token)
-    @tokens << [token, [value, [@filename, @line]]]
-  end
-  
-  def tokenize
-    @filename = @filenames[file_index = 0]
-    @line = 1
-    @pointer = -1
-    @indent_stack = [0]
-    @state = :newline
-    @tokens = []
-    
-    empty_lines = 0
-    
-    while c = getch
-      case c
-        when ?\034
-          @filename = @filenames[file_index += 1]
-          @line = 1
-        
-        when ?~
-          if getch == ?~
-            :skip until getch =~ /\n/
-            unscan
-            next
-          end
-          
-          unscan
-          :skip while getch =~ /\s/
-          unscan
-        
-        when ' ', ?\t
-          @state = :space if @state == :reference
-        
-        when ?\n
-          if @state == :newline
-            empty_lines += 1
-            next
-          end
-          
-          current_indent = 0
-          previous_indent = @indent_stack.last
-          
-          while (c = getch) == ' '
-            current_indent += 1
-          end
-          
-          unscan
-          
-          if c == ?\n
-            empty_lines += 1
-            next
-          end
-          
-          @state = :newline
-          token :NEWLINE
-          
-          @line += (empty_lines + 1)
-          empty_lines = 0
-          
-          if current_indent > previous_indent
-            @indent_stack << current_indent
-            token :INDENT
-          elsif current_indent < previous_indent
-            unless @indent_stack.include?(current_indent)
-              raise(SyntaxError, "Indent error: #{@filename}:#{@line}")
-            end
-            
-            until @indent_stack.last == current_indent
-              @indent_stack.pop
-              token :OUTDENT
-            end
-          end
-        
-        when ?!, ?=
-          @state = :begin
-          
-          if getch == ?=
-            token c + '='
-          else
-            unscan
-            token c
-          end
-        
-        when ?+, ?*, ?/, ?%
-          @state = :begin
-          
-          if getch == ?=
-            token '·=', c
-          else
-            unscan
-            token c
-          end
-        
-        when ?-
-          if getch == ?=
-            token '·=', c
-          elsif c =~ /[0-9]/
-            token '-1'
-            unscan
-          else
-            unscan
-            
-            if @state == :begin || @state == :newline
-              token '-·'
-            else
-              token '-'
-            end
-          end
-          
-          @state = :begin
-        
-        when ??, ?., ?;, ?,, ?(, ?{
-          @state = :begin
-          token c
-        
-        when ?'
-          @state = :reference
-          
-          s = ""
-          
-          until (c = getch) == ?'
-            s << c
-          end
-          
-          token '\''
-          token :STRING, s
-          token '\''
-        
-        when ?>, ?<
-          @state = :begin
-          
-          if getch == c
-            if getch == ?=
-              token '·=', c * 2
-            else
-              unscan
-              token c * 2
-            end
-          else
-            unscan
-            
-            if getch == ?=
-              token c + '='
-            else
-              unscan
-              token c
-            end
-          end
-        
-        when ?&, ?|
-          @state = :begin
-          
-          if getch == c
-            if getch == ?=
-              token c + c + '='
-            else
-              unscan
-              token c + c
-            end
-          else
-            unscan
-            token c
-          end
-        
-        when /[0-9]/
-          @state = :reference
-          value = c
-          
-          while (c = getch) =~ /[0-9]/
-            value << c
-          end
-          
-          unscan
-          
-          token :NUMBER, value.to_i
-        
-        when ?], ?}, ?)
-          @state = :reference
-          token c
-        
-        when ?:, ?[, ?#
-          if @state == :reference
-            token '·' + c
-          else
-            token c
-          end
-          
-          @state = :begin
-        
-        when /[A-Z]/
-          @state = :begin
-          value = c
-          
-          while (c = getch) =~ /[_a-zA-Z0-9]/
-            value << c
-          end
-          
-          unscan
-          
-          token :MIXIN, value.to_sym
-        
-        when /[_a-z]/
-          @state = :reference
-          value = c
-          
-          while (c = getch) =~ /[_a-zA-Z0-9]/
-            value << c
-          end
-          
-          if c == ?:
-            if getch =~ /[a-z]/
-              unscan
-            else
-              @state = :begin
-              token :KEY, value.to_sym
-              unscan
-              next
-            end
-            
-            unscan
-          elsif c == ?? || c == ?!
-            value << c
-          else
-            unscan
-          end
-          
-          if %w:
-            app break case
-            elsif else false if
-            null return self
-            switch true unless
-            until while
-          :
-          .include?(value)
-            token value.upcase.to_sym
-          else
-            token :IDENTIFIER, value.to_sym
-          end
-          
-          case c = getch
-            when ?:, ?#
-              token '·' + c
-            when ?[
-              token '·['
-              @state = :begin
-            else
-              unscan
-          end
-        
-        else
-          raise SyntaxError, "Invalid token `#{c}': #{@filename}:#{@line}"
-      end
-    end
-    
-    @tokens << [false, false]
+  def translate_to_js (node)
+    @translator.translate(node)
   end
   
-  def unscan
-    @pointer -= 1
+  def write_js_file (filename, js_output_string)
+    js_filename = basename(filename) + '.js'
+    File.open(js_filename,'w') {|f| f.write(js_output_string) }
+    puts "Writing to #{js_filename}"
   end
 end

data/lib/ruby/mix.grammar

@@ -1,14 +1,14 @@
 # encoding: UTF-8
 # 
-# Copyright © 2011 Jesse Sielaff
+# Copyright © 2011-2012 Jesse Sielaff
 # 
 
-class MixCompiler
+class MixParser
   options no_result_var
   
   prechigh
     right '!'
-    right '-·' '-1'
+    right '-·' NEGATIVE
     left '*' '/' '%'
     left '+' '-'
     left '<<' '>>'
@@ -43,7 +43,7 @@ class MixCompiler
       | function_statement
     
     mixin_statement
-      : MIXIN ':' newline INDENT mixin_body OUTDENT { node :mixin, val[0][0], val[4] }
+      : MIXIN ':' newline INDENT mixin_body OUTDENT { node :mixin, val[0][0], val[4][0], val[4][1] }
     
     mixin_body
       : statements { [val[0], []] }
@@ -56,10 +56,10 @@ class MixCompiler
     
     definition
       : IDENTIFIER function_block { node :set, node(:access_hash, node(:self), val[0][0]), val[1] }
-      | IDENTIFIER inline_function newline { node :newline, @location, node(:set, node(:access_hash, node(:self), val[0][0]), val[1]) }
+      | IDENTIFIER inline_function newline { node :newline, *@location, node(:set, node(:access_hash, node(:self), val[0][0]), val[1]) }
     
     function_block
-      : parameter_list indent_block { node :function, pop_variables, val[1] }
+      : parameter_list indent_block { node :function, *pop_variables, val[1] }
     
     parameter_list
       : '#' { push_variables; nil }
@@ -92,8 +92,8 @@ class MixCompiler
       | cases case { val[0] << val[1] }
     
     case
-      : CASE exp indent_block { [node(:newline, @location, val[1]), val[2]] }
-      | CASE exp ':' inline_statements newline { [node(:newline, @location, val[1]), val[3]] }
+      : CASE exp indent_block { [node(:newline, *@location, val[1]), val[2]] }
+      | CASE exp ':' inline_statements newline { [node(:newline, *@location, val[1]), val[3]] }
     
     inline_statements
       : inline_statement { node :statement, val[0], nil }
@@ -112,8 +112,8 @@ class MixCompiler
       | inline_statement UNTIL exp { node :while, node(:not, val[2]), val[0] }
     
     mix_statement
-      : '+' MIXIN object { node :mix, val[2], val[1][0] }
-      | '+' MIXIN { node :mix, node(:self), val[1][0] }
+      : '+' MIXIN object { node :mix, val[1][0], val[2] }
+      | '+' MIXIN { node :mix, val[1][0], node(:self) }
     
     keyword_statement
       : BREAK exp { node :break, val[1] }
@@ -184,10 +184,10 @@ class MixCompiler
     
     literal_number
       : NUMBER { node :number, val[0][0] }
-      | '-1' NUMBER { node :number, -val[1][0] }
+      | NEGATIVE NUMBER { node :number, -val[1][0] }
     
     literal_string
-      : '\'' STRING '\'' { node :string, val[1][0] }
+      : STRING { node :string, val[0][0] }
     
     literal_constant
       : APP { node :app }
@@ -223,7 +223,7 @@ class MixCompiler
       : '(' inline_statements ')' { val[1] }
     
     inline_function
-      : parameter_list curly_brace_block { node :function, pop_variables, val[1] }
+      : parameter_list curly_brace_block { node :function, *pop_variables, val[1] }
     
     curly_brace_block
       : '{' inline_statements '}' { val[1] }