mix-language 1.0.0 → 1.0.1

data/lib/ruby/new_tokenizer.rb

@@ -0,0 +1,290 @@
+# encoding: UTF-8
+# 
+# Copyright © 2011-2012 Jesse Sielaff
+# 
+
+class MixTokenizer
+  def access?
+    @state == :access
+  end
+  
+  def access!
+    @state = :access
+  end
+  
+  def access_token
+    add_token '·' + @token
+  end
+  
+  def add_token (token, value = token)
+    @token_stream << [token, [value, [@filename, @line]]]
+  end
+  
+  def adjust_indent
+    current_indent = @token.length
+    previous_indent = @indent_stack.last
+    
+    if current_indent > previous_indent
+      indent_token
+    elsif current_indent < previous_indent
+      unless @indent_stack.include?(current_indent)
+        raise SyntaxError, "Indent error: #{@filename}:#{@line}"
+      end
+      
+      outdent_token until @indent_stack.last == current_indent
+    end
+  end
+  
+  def comparison_token
+    add_token @token + '='
+  end
+  
+  def double_token?
+    lookahead?(@token) do
+      @token *= 2
+      yield
+    end
+  end
+  
+  def getch
+    @script[@pointer += 1]
+  end
+  
+  def handle_token
+    case @token
+      when ?~
+        lookahead?(?~) { skip_to_end_of_line } || skip_whitespace
+      
+      when ' ', ?\t
+        no_access!
+      
+      when ?\n
+        return if line_empty?
+        
+        @token = ''
+        scan_token(/ /)
+        return if lookahead?(?\n) { @empty_lines += 1; unscan }
+        
+        newline_token
+        adjust_indent
+      
+      when ??, ?., ?;, ?,, ?(, ?{
+        no_access!
+        plain_token
+      
+      when ?], ?}, ?)
+        access!
+        plain_token
+      
+      when ?[, ?:, ?#
+        (access?) ? access_token : plain_token
+        no_access!
+      
+      when ?!, ?=
+        no_access!
+        lookahead?(?=) { comparison_token } || plain_token
+      
+      when ?+, ?*, ?/, ?%
+        no_access!
+        lookahead?(?=) { operator_assignment_token } || plain_token
+      
+      when ?-
+        return if lookahead?(?=) { operator_assignment_token }
+        return if lookahead?(/[0-9]/) { negative_token; unscan }
+        
+        (unary_minus_possible?) ? unary_minus_token : plain_token
+        no_access!
+      
+      when ?>, ?<
+        no_access!
+        return if double_token? { lookahead?(?=) { operator_assignment_token } || plain_token }
+        lookahead?(?=) { comparison_token } || plain_token
+      
+      when ?&, ?|
+        no_access!
+        return if double_token? { lookahead?(?=) { operator_assignment_token } || plain_token }
+        plain_token
+      
+      when /[0-9]/
+        access!
+        scan_token(/\./)
+        scan_token(/[0-9]/)
+        number_token
+      
+      when /[A-Z]/
+        no_access!
+        scan_token(/[_a-zA-Z0-9]/)
+        mixin_token
+      
+      when ?'
+        access!
+        @token = ""
+        scan_token(/[^']/)
+        getch
+        
+        lookahead?(?:) do
+          return if lookahead?(/[^a-z]/) do
+            no_access!
+            key_token
+          end
+        end
+        
+        string_token
+      
+      when /[_a-z]/
+        access!
+        scan_token(/[_a-zA-Z0-9]/)
+        scan_token(/[?!]/)
+        
+        lookahead?(?:) do
+          return if lookahead?(/[^a-z]/) do
+            no_access!
+            key_token
+          end
+        end
+        
+        (keyword?) ? keyword_token : identifier_token
+        
+        case @token = getch
+          when ?:, ?#
+            access_token
+          when ?[
+            access_token
+            no_access!
+          else
+            unscan
+        end
+      
+      else
+        raise SyntaxError, "Invalid token `#{@token}': #{@filename}:#{@line}"
+    end
+  end
+  
+  def identifier_token
+    add_token :IDENTIFIER, @token.to_sym
+  end
+  
+  def indent_token
+    @indent_stack << @token.length
+    add_token :INDENT
+  end
+  
+  def key_token
+    add_token :KEY, @token.to_sym
+  end
+  
+  def keyword?
+    %w|app break case
+      elsif else false
+      if null return
+      self switch true
+      unless until while|.include? @token
+  end
+  
+  def keyword_token
+    add_token @token.upcase.to_sym
+  end
+  
+  def line_empty?
+    if @state == :newline
+      @empty_lines += 1
+      return true
+    end
+  end
+  
+  def mixin_token
+    add_token :MIXIN, @token.to_sym
+  end
+  
+  def negative_token
+    add_token :NEGATIVE
+  end
+  
+  def newline_token
+    add_token :NEWLINE
+    
+    @line += (@empty_lines + 1)
+    @empty_lines = 0
+    @state = :newline
+  end
+  
+  def lookahead? (pattern)
+    if getch =~ Regexp.new(pattern)
+      yield
+      true
+    else
+      unscan
+      false
+    end
+  end
+  
+  def no_access!
+    @state = :no_access
+  end
+  
+  def number_token
+    add_token :NUMBER, @token.to_i
+  end
+  
+  def operator_assignment_token
+    add_token '·=', @token
+  end
+  
+  def outdent_token
+    @indent_stack.pop
+    add_token :OUTDENT
+  end
+  
+  def plain_token
+    add_token @token
+  end
+  
+  def scan_token (pattern)
+    while (c = getch) =~ pattern
+      @token << c
+    end
+    
+    unscan
+  end
+  
+  def skip_to_end_of_line
+    :skip until getch =~ /\n/
+    unscan
+  end
+  
+  def skip_whitespace
+    :skip while getch =~ /\s/
+    unscan
+  end
+  
+  def string_token
+    add_token :STRING, @token
+  end
+  
+  def tokenize (filename, script, token_stream)
+    @filename = filename
+    @script = script
+    @token_stream = token_stream
+    @line = 1
+    @pointer = -1
+    @empty_lines = 0
+    @indent_stack = [0]
+    @state = :newline
+    
+    handle_token while @token = getch
+    
+    puts @token_stream.inspect
+  end
+  
+  def unary_minus_possible?
+    @state == :begin || @state == :newline
+  end
+  
+  def unary_minus_token
+    add_token '-·'
+  end
+  
+  def unscan
+    @pointer -= 1
+  end
+end

data/lib/ruby/parser.rb

@@ -0,0 +1,71 @@
+# encoding: UTF-8
+# 
+# Copyright © 2011-2012 Jesse Sielaff
+# 
+
+class MixParser
+  def append_statement (outermost_statement, final_statement)
+    current = outermost_statement
+    
+    current = current[:values][1] while current[:values][1]
+    current[:values][1] = final_statement
+    
+    outermost_statement
+  end
+  
+  def newline (statement_node)
+    statement_node[:values][0] = node(:newline, *@location, statement_node[:values][0])
+    statement_node
+  end
+  
+  def next_token
+    @token_stream.shift || [false, false]
+  end
+  
+  def node (type, *values)
+    { type: type, values: values }
+  end
+  
+  def on_error (token, value, stack)
+    if value
+      value, file, line = value[0].inspect, *value[1]
+    else
+      value, file, line = 'EOF', *@location
+    end
+    
+    raise SyntaxError, "Unexpected token #{token_to_str(token)} (#{value}): #{file}:#{line}".gsub(?",'\"')
+  end
+  
+  def parameter (name)
+    @variables[:parameters] << name
+    @variables[:local] << name
+  end
+  
+  def parse (tokens)
+    @location = ['',0]
+    @token_stream = tokens
+    @variables = { parameters: [], local: [], embedded: [], prev: nil }
+    
+    return do_parse
+  end
+  
+  def pop_variables
+    vars = [@variables[:parameters], @variables[:embedded]]
+    @variables = @variables[:prev]
+    
+    return vars
+  end
+  
+  def push_variables
+    embedded = @variables[:local] + @variables[:embedded]
+    @variables = { parameters: [], local: [], embedded: embedded, prev: @variables }
+  end
+  
+  def variable (name)
+    unless (@variables[:local] + @variables[:embedded]).include?(name)
+      @variables[:local] << name
+    end
+    
+    node :variable, name
+  end
+end

data/lib/ruby/precompiler.rb

@@ -0,0 +1,106 @@
+# encoding: UTF-8
+# 
+# Copyright © 2011-2012 Jesse Sielaff
+# 
+require 'strscan'
+
+class MixPrecompiler
+  JS_BASE_FILES = %w{
+    env eval object array
+    false function null
+    number string true }
+  .each_with_object({}) do |x,h|
+    filename = File.expand_path("../../javascript/base/#{x}.js", __FILE__)
+    h[filename] = File.read(filename)
+  end
+  
+  JS_OBJECT_TYPES = %w=
+    ARRAY ELEMENT EVENT
+    FALSE FUNCTION NULL
+    NUMBER OBJECT STRING
+    TEXT TRUE =
+  
+  def build_js_string
+    @js_string = @js_files.map {|*,script| script }.join ?\n
+    
+    replace_type_variables
+    replace_case_markers
+    minify
+    
+    @js_string = <<-JAVASCRIPT
+mix_result_should_be_undefined=function(){var SYMBOLS='...Replace with symbol table...';#{@js_string};
+window.onload=function(){startup();evaluate('...Replace with AST nodes...');}}();
+JAVASCRIPT
+  end
+  
+  def collect_scripts
+    @mix_files = {}
+    @js_files = JS_BASE_FILES.dup
+    
+    scan_mix_file(@primary_filename)
+  end
+  
+  def minify
+    @js_string.gsub!(/\s*\/\*.*?\*\/\s*/m, ?\n)
+    @js_string.gsub!(/\s*\n\s*/, ?\n)
+    @js_string.gsub!(/\s*([{;:,=])\s*/, '\1')
+    @js_string.gsub!("\n}", ?})
+  end
+  
+  def precompile (filename)
+    @primary_filename = filename
+    collect_scripts
+    build_js_string
+    
+    return @mix_files, @js_string
+  end
+  
+  def replace_case_markers
+    MixTranslator::NODE_TYPES.each do |k,v|
+      @js_string.gsub!("case #{k}:","case #{v}:")
+    end
+  end
+  
+  def replace_type_variables
+    JS_OBJECT_TYPES.each_with_index do |t,i|
+      @js_string.gsub!("#{t}_TYPE", i.to_s)
+    end
+  end
+  
+  def scan_includes (script)
+    ss = StringScanner.new(script)
+    js_filenames = []
+    
+    while ss.scan(/~~\s*(.*)\.(js|mix)\s*\n/)
+      basename = ss[1]
+      type = ss[2]
+      filename = "#{basename}.#{type}"
+      
+      if type == 'mix'
+        unless [@primary_filename, *@mix_files.keys].include?(filename)
+          scan_mix_file(filename)
+        end
+      else
+        js_filenames << filename
+      end
+    end
+    
+    js_filenames.each do |name|
+      filename = File.expand_path("../../javascript/#{name}", __FILE__)
+      next if @js_files.has_key?(filename)
+      @js_files[filename] = File.read(filename)
+    end
+  end
+  
+  def scan_mix_file (filename)
+    unless File.exists? filename
+      puts "no such file #{filename}"
+      exit
+    end
+    
+    script = File.read(filename) + ?\n
+    scan_includes(script)
+    
+    @mix_files[filename] = script
+  end
+end

data/lib/ruby/tokenizer.rb

@@ -0,0 +1,268 @@
+# encoding: UTF-8
+# 
+# Copyright © 2011-2012 Jesse Sielaff
+# 
+
+class MixTokenizer
+  def getch
+    @script[@pointer += 1]
+  end
+  
+  def token (token, value = token)
+    @tokens << [token, [value, [@filename, @line]]]
+  end
+  
+  def tokenize (filename, script, tokens)
+    @filename = filename
+    @script = script
+    @tokens = tokens
+    
+    @line = 1
+    @pointer = -1
+    @indent_stack = [0]
+    @state = :newline
+    
+    empty_lines = 0
+    
+    while c = getch
+      case c
+        when ?~
+          if getch == ?~
+            :skip until getch =~ /\n/
+            unscan
+            next
+          end
+          
+          unscan
+          :skip while getch =~ /\s/
+          unscan
+        
+        when ' ', ?\t
+          @state = :space if @state == :reference
+        
+        when ?\n
+          if @state == :newline
+            empty_lines += 1
+            next
+          end
+          
+          current_indent = 0
+          previous_indent = @indent_stack.last
+          
+          while (c = getch) == ' '
+            current_indent += 1
+          end
+          
+          unscan
+          
+          if c == ?\n
+            empty_lines += 1
+            next
+          end
+          
+          @state = :newline
+          token :NEWLINE
+          
+          @line += (empty_lines + 1)
+          empty_lines = 0
+          
+          if current_indent > previous_indent
+            @indent_stack << current_indent
+            token :INDENT
+          elsif current_indent < previous_indent
+            unless @indent_stack.include?(current_indent)
+              raise(SyntaxError, "Indent error: #{@filename}:#{@line}")
+            end
+            
+            until @indent_stack.last == current_indent
+              @indent_stack.pop
+              token :OUTDENT
+            end
+          end
+        
+        when ?!, ?=
+          @state = :begin
+          
+          if getch == ?=
+            token c + '='
+          else
+            unscan
+            token c
+          end
+        
+        when ?+, ?*, ?/, ?%
+          @state = :begin
+          
+          if getch == ?=
+            token '·=', c
+          else
+            unscan
+            token c
+          end
+        
+        when ?-
+          if getch == ?=
+            token '·=', c
+          elsif c =~ /[0-9]/
+            token :NEGATIVE
+            unscan
+          else
+            unscan
+            
+            if @state == :begin || @state == :newline
+              token '-·'
+            else
+              token '-'
+            end
+          end
+          
+          @state = :begin
+        
+        when ??, ?., ?;, ?,, ?(, ?{
+          @state = :begin
+          token c
+        
+        when ?'
+          @state = :reference
+          
+          s = ""
+          
+          until (c = getch) == ?'
+            s << c
+          end
+          
+          token :STRING, s
+        
+        when ?>, ?<
+          @state = :begin
+          
+          if getch == c
+            if getch == ?=
+              token '·=', c * 2
+            else
+              unscan
+              token c * 2
+            end
+          else
+            unscan
+            
+            if getch == ?=
+              token c + '='
+            else
+              unscan
+              token c
+            end
+          end
+        
+        when ?&, ?|
+          @state = :begin
+          
+          if getch == c
+            if getch == ?=
+              token c + c + '='
+            else
+              unscan
+              token c + c
+            end
+          else
+            unscan
+            token c
+          end
+        
+        when /[0-9]/
+          @state = :reference
+          value = c
+          
+          while (c = getch) =~ /[0-9]/
+            value << c
+          end
+          
+          unscan
+          
+          token :NUMBER, value.to_i
+        
+        when ?], ?}, ?)
+          @state = :reference
+          token c
+        
+        when ?:, ?[, ?#
+          if @state == :reference
+            token '·' + c
+          else
+            token c
+          end
+          
+          @state = :begin
+        
+        when /[A-Z]/
+          @state = :begin
+          value = c
+          
+          while (c = getch) =~ /[_a-zA-Z0-9]/
+            value << c
+          end
+          
+          unscan
+          
+          token :MIXIN, value.to_sym
+        
+        when /[_a-z]/
+          @state = :reference
+          value = c
+          
+          while (c = getch) =~ /[_a-zA-Z0-9]/
+            value << c
+          end
+          
+          if c == ?:
+            if getch =~ /[a-z]/
+              unscan
+            else
+              @state = :begin
+              token :KEY, value.to_sym
+              unscan
+              next
+            end
+            
+            unscan
+          elsif c == ?? || c == ?!
+            value << c
+          else
+            unscan
+          end
+          
+          if %w:
+            app break case
+            elsif else false if
+            null return self
+            switch true unless
+            until while
+          :
+          .include?(value)
+            token value.upcase.to_sym
+          else
+            token :IDENTIFIER, value.to_sym
+          end
+          
+          case c = getch
+            when ?:, ?#
+              token '·' + c
+            when ?[
+              token '·['
+              @state = :begin
+            else
+              unscan
+          end
+        
+        else
+          raise SyntaxError, "Invalid token `#{c}': #{@filename}:#{@line}"
+      end
+    end
+    
+    @tokens
+  end
+  
+  def unscan
+    @pointer -= 1
+  end
+end