rubex 0.0.1 → 0.1

data/lib/rubex/error.rb

@@ -0,0 +1,15 @@
+module Rubex
+  class SymbolNotFoundError < StandardError;  end
+
+  class ArrayLengthMismatchError < StandardError; end
+
+  class TypeMismatchError < StandardError; end
+
+  class SyntaxError < StandardError; end
+
+  class NoMethodError < StandardError; end
+
+  class TypeError < StandardError; end
+
+  class LibraryNotFoundError < StandardError; end
+end

data/lib/rubex/helpers.rb

@@ -0,0 +1,154 @@
+module Rubex
+  module Helpers
+    class << self
+      def to_lhs_type lhs, rhs
+        if lhs.type.object?
+          return rhs.to_ruby_object
+        elsif !lhs.type.object? && rhs.type.object?
+          return rhs.from_ruby_object(lhs)
+        else
+          return rhs
+        end  
+      end
+
+      def result_type_for left, right
+        begin
+          return left.dup if left == right
+          return (left < right ? right.dup : left.dup)
+        rescue ArgumentError => e
+          raise Rubex::TypeError, e.to_s
+        end
+      end
+
+      def determine_dtype data, ptr_level
+        if ptr_level && ptr_level[-1] == "*"
+          ptr_level = ptr_level.dup
+          base_type = Rubex::DataType::CPtr.new simple_dtype(data)
+          ptr_level.chop!
+
+          ptr_level.each_char do |star|
+            base_type = Rubex::DataType::CPtr.new base_type
+          end
+
+          return base_type
+        else
+          return simple_dtype(data)
+        end
+      end
+
+      def simple_dtype dtype
+        if dtype.is_a?(Rubex::DataType::CFunction)
+          dtype
+        else
+          begin
+            Rubex::CUSTOM_TYPES[dtype] || Rubex::TYPE_MAPPINGS[dtype].new  
+          rescue
+            raise Rubex::TypeError, "Type #{dtype} not previously declared."
+          end          
+        end
+      end
+
+      def create_arg_arrays arg_list
+        arg_list.inject([]) do |array, arg|
+          entry = arg.entry
+          c_name = entry.type.base_type.c_function? ? "" : entry.c_name
+          array << [entry.type.to_s, c_name]
+          array
+        end
+      end
+    end
+
+    module Writers
+      def declare_temps code, scope
+        scope.temp_entries.each do |var|
+          code.declare_variable type: var.type.to_s, c_name: var.c_name
+        end  
+      end
+
+      def declare_vars code, scope
+        scope.var_entries.each do |var|
+          if var.type.base_type.c_function?
+            code.declare_func_ptr var: var
+          else
+            code.declare_variable type: var.type.to_s, c_name: var.c_name
+          end
+        end
+      end
+
+      def declare_carrays code, scope
+        scope.carray_entries.select { |s|
+          s.type.dimension.is_a? Rubex::AST::Expression::Literal::Base
+        }. each do |arr|
+          type = arr.type.type.to_s
+          c_name = arr.c_name
+          dimension = arr.type.dimension.c_code(@scope)
+          value = arr.value.map { |a| a.c_code(@scope) } if arr.value
+          code.declare_carray(type: type, c_name: c_name, dimension: dimension,
+            value: value)
+        end
+      end
+
+      def declare_types code, scope
+        scope.type_entries.each do |entry|
+          # if !entry.extern
+            type = entry.type
+
+            if type.alias_type?
+              base = type.old_type
+              if base.respond_to?(:base_type) && base.base_type.c_function?
+                func = base.base_type
+                str = "typedef #{func.type} (#{type.old_type.ptr_level} #{type.new_type})"
+                str << "(" + func.arg_list.map { |e| e.type.to_s }.join(',') + ")"
+                str << ";"
+                code << str
+              else
+                code << "typedef #{type.old_type} #{type.new_type};"
+              end
+            elsif type.struct_or_union? && !entry.extern?
+              code << sue_header(entry)
+              code.block(sue_footer(entry)) do
+                declare_vars code, type.scope
+                declare_carrays code, type.scope
+                declare_ruby_objects code, type.scope
+              end
+            end
+            code.nl
+          # end
+        end
+      end
+
+      def sue_header entry
+        type = entry.type
+        str = "#{type.kind} #{type.name}"
+        if !entry.extern
+          str.prepend "typedef "
+        end
+
+        str
+      end
+
+      def sue_footer entry
+        str =
+        if entry.extern
+          ";"
+        else
+          " #{entry.type.c_name};"
+        end
+
+        str
+      end
+
+      def declare_ruby_objects code, scope
+        scope.ruby_obj_entries.each do |var|
+          code.declare_variable type: var.type.to_s, c_name: var.c_name
+        end
+      end
+    end
+
+    module NodeTypeMethods
+      [:expression?, :statement?, :literal?, :ruby_method?].each do |meth|
+        define_method(meth) { false }
+      end
+    end
+  end
+end

data/lib/rubex/lexer.rex

@@ -1,50 +1,214 @@
 class Rubex::Lexer
+options
+  lineno
 macros
+  # reserved words
+
+  SELF            /self/
   DEF             /def/
+  CFUNC           /cfunc/
   RETURN          /return/
+  PRINT           /print/
+  IF              /if/
+  ELSE            /else/
+  ELSIF           /elsif/
+  THEN            /then/
+  STATIC_ARRAY    /StaticArray/
+  FOR             /for/
+  WHILE           /while/
+  DO              /do/
+  EACH            /each/
+  TRUE            /true/
+  FALSE           /false/
+  NIL             /nil/
+  STRUCT          /struct/
+  UNION           /union/
+  ALIAS           /alias/
+  LIB             /lib/
+  CLASS           /class/
+  NULL            /NULL/
+  ATTACH          /attach/
+  BLOCK_GIVEN     /block_given\?/
 
-  IDENTIFIER      /[a-z_][a-zA-Z_0-9]*/
+  IDENTIFIER      /[a-zA-Z_][a-zA-Z_0-9]*/
   LPAREN          /\(/
   RPAREN          /\)/
+  LSQUARE         /\[/
+  RSQUARE         /\]/
+  LBRACE          /{/
+  RBRACE          /}/
   NL              /\n/
   COMMA           /,/
+  SQUOTE          /'/
+  DQUOTE          /"/
+  SCOLON          /;/
+  COLON           /:/
+  INTEGER         /-?\d+/
+  FLOAT           /-?\d+\.\d+/
+  DOT             /\./
+  QMARK           /\?/
+  OCTOTHORP       /#/
 
   # operators
+
   EXPO            /\*\*/
-  MULTIPLY        /\*/
+  EXPOASSIGN      /\*\*=/
+  STAR            /\*/
+  STARASSIGN      /\*=/
   DIVIDE          /\//
+  DIVIDEASSIGN    /\/=/
   PLUS            /\+/
+  PLUSASSIGN      /\+=/
   MINUS           /\-/
+  MINUSASSIGN     /\-=/
   MODULUS         /%/
-  EQUAL           /=/
+  MODULUSASSIGN   /%=/
+  ASSIGN          /=/
+  NEQ             /!=/
+  EQ              /==/
+  LT              /</
+  LTEQ            /<=/
+  GT              />/
+  GTEQ            />=/
+  ANDOP           /&&/
+  OROP            /\|\|/
+  BANG            /!/
+  BIT_AND         /&/
+  BIT_AND_ASSIGN  /&=/
+  BIT_OR          /\|/
+  BIT_OR_ASSIGN   /\|=/
+  BIT_XOR         /\^/
+  BIT_XOR_ASSIGN  /\^=/
+  BIT_LSHIFT      /<</
+  BIT_LSHIFT_ASSIGN /<<=/
+  BIT_RSHIFT        />>/
+  BIT_RSHIFT_ASSIGN />>=/
+  BIT_NOT           /~/
+
 rules
-  /#{DEF}/  { [:kDEF, text] }
-  /end/  { [:kEND, text]  }
-  /#{RETURN}/ { [:kRETURN, text] }
 
-  /i32/           { [:kDTYPE_I32, text] }
+  # literals
+
+  /'.\'/          { [:tSINGLE_CHAR, text] }
+  /#{FLOAT}/      { [:tFLOAT, text] }
+  /#{INTEGER}/    { [:tINTEGER, text] }
+
+  # String literal
+
+                  /#{DQUOTE}/  { @state = :STRING_LITERAL; @string_text = ""; nil }
+  :STRING_LITERAL /#{DQUOTE}/  { @state = nil; return [:tSTRING, @string_text] }
+  :STRING_LITERAL /[^"\\]/     { @string_text << text; nil }
+  :STRING_LITERAL /\\/         { @state = :STRING_LITERAL_BSLASH; @string_text << text; nil }
+  :STRING_LITERAL_BSLASH /./   { @state = :STRING_LITERAL; @string_text << text; nil }
+
+  # Comments
+
+             /#{OCTOTHORP}/ { @state = :COMMENT; nil }
+  :COMMENT   /./           { @state = :COMMENT; nil }
+  # return a tNL since it is a way for the parser to figure that a line of code has end.
+  :COMMENT   /\n/       { @state = nil; [:tNL, "\n"] }
+
+  # Reserved words
+
+  /#{STATIC_ARRAY}/ { [:kSTATIC_ARRAY, text] }
+  /#{FOR}/    { [:kFOR, text]    }
+  /#{WHILE}/  { [:kWHILE, text]  }
+  /#{TRUE}/   { [:kTRUE, text]   }
+  /#{FALSE}/  { [:kFALSE, text]  }
+  /#{NIL}/    { [:kNIL, text]    }
+  /#{LIB}/    { [:kLIB, text]    }
+  /#{CLASS}/  { [:kCLASS, text]  }
+  /#{NULL}/   { [:kNULL, text] }
+  /fwd/       { [:kFWD, text] }
+  /#{ATTACH}/ { [:kATTACH, text] }
+  /#{BLOCK_GIVEN}/ { [:kBLOCK_GIVEN, text] }
+
+  # Method hacks
 
-  /#{IDENTIFIER}/ { [:tIDENTIFIER, text] }
-  /#{LPAREN}/     { [:tLPAREN, text] }
-  /#{RPAREN}/     { [:tRPAREN, text] }
-  /#{NL}/         { [:tNL, text] }
-  /#{COMMA}/      { [:tCOMMA, text] }
+  /data\$/                { [:kDATA_VAR, text] }
+  /#{DOT}#{EACH}/ { [:kDOT_EACH, text] }
+
+  # Data Types
+
+  /unsigned long long int/ { [:kDTYPE_ULLINT, text] }
+  /unsigned long int/      { [:kDTYPE_ULINT, text] }
+  /unsigned int/           { [:kDTYPE_UINT, text] }
+  /unsigned char/          { [:kDTYPE_UCHAR, text] }
+  /long int/               { [:kDTYPE_LINT, text] }
+  /long long int/          { [:kDTYPE_LLINT, text] }
+  # /long double/            { [:kDTYPE_LF64, text] }
+  # /long f64/               { [:kDTYPE_LF64, text] }
+
+  # Keywords
+
+  /#{STRUCT}\ / { [:kSTRUCT, text] }
+  /#{UNION}\ /  { [:kUNION, text]  }
+  /#{ALIAS}\ /  { [:kALIAS, text]  }
+
+  /:#{IDENTIFIER}/        { [:tSYMBOL, text] }
+  /#{IDENTIFIER}/         { [:tIDENTIFIER, text] }
+  /#{LBRACE}/             { [:tLBRACE, text] }
+  /#{RBRACE}/             { [:tRBRACE, text] }
+  /#{LPAREN}/             { [:tLPAREN, text] }
+  /#{RPAREN}/             { [:tRPAREN, text] }
+  /#{LSQUARE}/            { [:tLSQUARE, text] }
+  /#{RSQUARE}/            { [:tRSQUARE, text] }
+  /#{COMMA}/              { [:tCOMMA, text] }
+  /#{SCOLON}/             { [:tSCOLON, text] }
+  /#{NL}/                 { [:tNL, text] }
+  /#{QMARK}/              { [:tQMARK, text]}
+  /#{DOT}/                { [:tDOT, text]    }
+  /#{COLON}/              { [:tCOLON, text]  }
 
   # operators
 
-  /#{PLUS}/       { [:tPLUS, text]}
-  /#{MINUS}/      { [:tMINUS, text]}
-  /#{MULTIPLY}/   { [:tMULTIPLY, text]}
-  /#{DIVIDE}/     { [:tDIVIDE, text]}
-  /#{EXPO}/       { [:tEXPO, text]}
-  /#{MODULUS}/    { [:tMODULUS, text]}
-  /#{EXPO}/       { [:tEXPO, text]}
+  /#{PLUSASSIGN}/       { [:tOP_ASSIGN, text]}
+  /#{MINUSASSIGN}/      { [:tOP_ASSIGN, text]}
+  /#{STARASSIGN}/       { [:tOP_ASSIGN, text]}
+  /#{DIVIDEASSIGN}/     { [:tOP_ASSIGN, text]}
+  /#{EXPOASSIGN}/       { [:tOP_ASSIGN, text]}
+  /#{MODULUSASSIGN}/    { [:tOP_ASSIGN, text]}
+  /#{PLUS}/             { [:tPLUS, text]}
+  /#{MINUS}/            { [:tMINUS, text]}
+  /#{STAR}/             { [:tSTAR, text]}
+  /#{DIVIDE}/           { [:tDIVIDE, text]}
+  /#{EXPO}/             { [:tEXPO, text]}
+  /#{MODULUS}/          { [:tMODULUS, text]}
+  /#{EXPO}/             { [:tEXPO, text]}
+  /#{EQ}/               { [:tEQ, text]  }
+  /#{NEQ}/              { [:tNEQ, text]  }
+  /#{ASSIGN}/           { [:tASSIGN, text] }
+  /#{BANG}/             { [:tBANG, text] }
+
+  /#{ANDOP}/      { [:tANDOP, text] }
+  /#{OROP}/       { [:tOROP, text] }
+
+  /#{BIT_AND_ASSIGN}/     { [:tOP_ASSIGN, text] }  
+  /#{BIT_OR_ASSIGN}/      { [:tOP_ASSIGN  , text] }
+  /#{BIT_XOR_ASSIGN}/     { [:tOP_ASSIGN , text] }
+  /#{BIT_LSHIFT_ASSIGN}/  { [:tOP_ASSIGN, text] }
+  /#{BIT_RSHIFT_ASSIGN}/  { [:tOP_ASSIGN, text] }
+  /#{BIT_AND}/            { [:tBIT_AND      , text] } 
+  /#{BIT_OR}/             { [:tBIT_OR        , text] }  
+  /#{BIT_XOR}/            { [:tBIT_XOR       , text] }  
+  /#{BIT_LSHIFT}/         { [:tBIT_LSHIFT    , text] }  
+  /#{BIT_RSHIFT}/         { [:tBIT_RSHIFT      , text] }
+  /#{BIT_NOT}/            { [:tBIT_NOT, text] }
+
+  /#{LTEQ}/       { [:tLTEQ, text] }
+  /#{LT}/         { [:tLT, text] }
+
+  /#{GTEQ}/       { [:tGTEQ, text] }
+  /#{GT}/         { [:tGT, text] }
 
   # whitespace
 
-  / /             {}
+  /^\n\s*$/
+  /\s+/
+
 inner
   def do_parse
-    # this is a stub since oedipus lex uses this internally.
+    self.ss.string = self.ss.string.split("\n").map! { |e| e.rstrip }.join("\n")
+    self.ss << "\n"
   end
-end # Rubex::Lexer
+end # Rubex::Lexer

data/lib/rubex/lexer.rex.rb

@@ -1,30 +1,94 @@
 # encoding: UTF-8
 #--
 # This file is automatically generated. Do not modify it.
-# Generated by: oedipus_lex version 2.4.1.
+# Generated by: oedipus_lex version 2.5.0.
 # Source: lib/rubex/lexer.rex
 #++
 
 class Rubex::Lexer
   require 'strscan'
 
-  DEF        = /def/
-  RETURN     = /return/
-  IDENTIFIER = /[a-z_][a-zA-Z_0-9]*/
-  LPAREN     = /\(/
-  RPAREN     = /\)/
-  NL         = /\n/
-  COMMA      = /,/
-  EXPO       = /\*\*/
-  MULTIPLY   = /\*/
-  DIVIDE     = /\//
-  PLUS       = /\+/
-  MINUS      = /\-/
-  MODULUS    = /%/
-  EQUAL      = /=/
-
-  class ScanError < StandardError ; end
+  SELF              = /self/
+  DEF               = /def/
+  CFUNC             = /cfunc/
+  RETURN            = /return/
+  PRINT             = /print/
+  IF                = /if/
+  ELSE              = /else/
+  ELSIF             = /elsif/
+  THEN              = /then/
+  STATIC_ARRAY      = /StaticArray/
+  FOR               = /for/
+  WHILE             = /while/
+  DO                = /do/
+  EACH              = /each/
+  TRUE              = /true/
+  FALSE             = /false/
+  NIL               = /nil/
+  STRUCT            = /struct/
+  UNION             = /union/
+  ALIAS             = /alias/
+  LIB               = /lib/
+  CLASS             = /class/
+  NULL              = /NULL/
+  ATTACH            = /attach/
+  BLOCK_GIVEN       = /block_given\?/
+  IDENTIFIER        = /[a-zA-Z_][a-zA-Z_0-9]*/
+  LPAREN            = /\(/
+  RPAREN            = /\)/
+  LSQUARE           = /\[/
+  RSQUARE           = /\]/
+  LBRACE            = /{/
+  RBRACE            = /}/
+  NL                = /\n/
+  COMMA             = /,/
+  SQUOTE            = /'/
+  DQUOTE            = /"/
+  SCOLON            = /;/
+  COLON             = /:/
+  INTEGER           = /-?\d+/
+  FLOAT             = /-?\d+\.\d+/
+  DOT               = /\./
+  QMARK             = /\?/
+  OCTOTHORP         = /#/
+  EXPO              = /\*\*/
+  EXPOASSIGN        = /\*\*=/
+  STAR              = /\*/
+  STARASSIGN        = /\*=/
+  DIVIDE            = /\//
+  DIVIDEASSIGN      = /\/=/
+  PLUS              = /\+/
+  PLUSASSIGN        = /\+=/
+  MINUS             = /\-/
+  MINUSASSIGN       = /\-=/
+  MODULUS           = /%/
+  MODULUSASSIGN     = /%=/
+  ASSIGN            = /=/
+  NEQ               = /!=/
+  EQ                = /==/
+  LT                = /</
+  LTEQ              = /<=/
+  GT                = />/
+  GTEQ              = />=/
+  ANDOP             = /&&/
+  OROP              = /\|\|/
+  BANG              = /!/
+  BIT_AND           = /&/
+  BIT_AND_ASSIGN    = /&=/
+  BIT_OR            = /\|/
+  BIT_OR_ASSIGN     = /\|=/
+  BIT_XOR           = /\^/
+  BIT_XOR_ASSIGN    = /\^=/
+  BIT_LSHIFT        = /<</
+  BIT_LSHIFT_ASSIGN = /<<=/
+  BIT_RSHIFT        = />>/
+  BIT_RSHIFT_ASSIGN = />>=/
+  BIT_NOT           = /~/
 
+  class LexerError < StandardError ; end
+  class ScanError < LexerError ; end
+
+  attr_accessor :lineno
   attr_accessor :filename
   attr_accessor :ss
   attr_accessor :state
@@ -41,13 +105,13 @@ class Rubex::Lexer
     yield
   end
 
-
   def scanner_class
     StringScanner
   end unless instance_methods(false).map(&:to_s).include?("scanner_class")
 
   def parse str
     self.ss     = scanner_class.new str
+    self.lineno = 1
     self.state  ||= nil
 
     do_parse
@@ -60,39 +124,125 @@ class Rubex::Lexer
     end
   end
 
+  def location
+    [
+      (filename || "<input>"),
+      lineno,
+    ].compact.join(":")
+  end
+
   def next_token
 
     token = nil
 
     until ss.eos? or token do
+      self.lineno += 1 if ss.peek(1) == "\n"
       token =
         case state
         when nil then
           case
-          when text = ss.scan(/#{DEF}/) then
-            action { [:kDEF, text] }
-          when text = ss.scan(/end/) then
-            action { [:kEND, text]  }
-          when text = ss.scan(/#{RETURN}/) then
-            action { [:kRETURN, text] }
-          when text = ss.scan(/i32/) then
-            action { [:kDTYPE_I32, text] }
+          when text = ss.scan(/'.\'/) then
+            action { [:tSINGLE_CHAR, text] }
+          when text = ss.scan(/#{FLOAT}/) then
+            action { [:tFLOAT, text] }
+          when text = ss.scan(/#{INTEGER}/) then
+            action { [:tINTEGER, text] }
+          when ss.skip(/#{DQUOTE}/) then
+            action { @state = :STRING_LITERAL; @string_text = ""; nil }
+          when ss.skip(/#{OCTOTHORP}/) then
+            action { @state = :COMMENT; nil }
+          when text = ss.scan(/#{STATIC_ARRAY}/) then
+            action { [:kSTATIC_ARRAY, text] }
+          when text = ss.scan(/#{FOR}/) then
+            action { [:kFOR, text]    }
+          when text = ss.scan(/#{WHILE}/) then
+            action { [:kWHILE, text]  }
+          when text = ss.scan(/#{TRUE}/) then
+            action { [:kTRUE, text]   }
+          when text = ss.scan(/#{FALSE}/) then
+            action { [:kFALSE, text]  }
+          when text = ss.scan(/#{NIL}/) then
+            action { [:kNIL, text]    }
+          when text = ss.scan(/#{LIB}/) then
+            action { [:kLIB, text]    }
+          when text = ss.scan(/#{CLASS}/) then
+            action { [:kCLASS, text]  }
+          when text = ss.scan(/#{NULL}/) then
+            action { [:kNULL, text] }
+          when text = ss.scan(/fwd/) then
+            action { [:kFWD, text] }
+          when text = ss.scan(/#{ATTACH}/) then
+            action { [:kATTACH, text] }
+          when text = ss.scan(/#{BLOCK_GIVEN}/) then
+            action { [:kBLOCK_GIVEN, text] }
+          when text = ss.scan(/data\$/) then
+            action { [:kDATA_VAR, text] }
+          when text = ss.scan(/#{DOT}#{EACH}/) then
+            action { [:kDOT_EACH, text] }
+          when text = ss.scan(/unsigned long long int/) then
+            action { [:kDTYPE_ULLINT, text] }
+          when text = ss.scan(/unsigned long int/) then
+            action { [:kDTYPE_ULINT, text] }
+          when text = ss.scan(/unsigned int/) then
+            action { [:kDTYPE_UINT, text] }
+          when text = ss.scan(/unsigned char/) then
+            action { [:kDTYPE_UCHAR, text] }
+          when text = ss.scan(/long int/) then
+            action { [:kDTYPE_LINT, text] }
+          when text = ss.scan(/long long int/) then
+            action { [:kDTYPE_LLINT, text] }
+          when text = ss.scan(/#{STRUCT}\ /) then
+            action { [:kSTRUCT, text] }
+          when text = ss.scan(/#{UNION}\ /) then
+            action { [:kUNION, text]  }
+          when text = ss.scan(/#{ALIAS}\ /) then
+            action { [:kALIAS, text]  }
+          when text = ss.scan(/:#{IDENTIFIER}/) then
+            action { [:tSYMBOL, text] }
           when text = ss.scan(/#{IDENTIFIER}/) then
             action { [:tIDENTIFIER, text] }
+          when text = ss.scan(/#{LBRACE}/) then
+            action { [:tLBRACE, text] }
+          when text = ss.scan(/#{RBRACE}/) then
+            action { [:tRBRACE, text] }
           when text = ss.scan(/#{LPAREN}/) then
             action { [:tLPAREN, text] }
           when text = ss.scan(/#{RPAREN}/) then
             action { [:tRPAREN, text] }
-          when text = ss.scan(/#{NL}/) then
-            action { [:tNL, text] }
+          when text = ss.scan(/#{LSQUARE}/) then
+            action { [:tLSQUARE, text] }
+          when text = ss.scan(/#{RSQUARE}/) then
+            action { [:tRSQUARE, text] }
           when text = ss.scan(/#{COMMA}/) then
             action { [:tCOMMA, text] }
+          when text = ss.scan(/#{SCOLON}/) then
+            action { [:tSCOLON, text] }
+          when text = ss.scan(/#{NL}/) then
+            action { [:tNL, text] }
+          when text = ss.scan(/#{QMARK}/) then
+            action { [:tQMARK, text]}
+          when text = ss.scan(/#{DOT}/) then
+            action { [:tDOT, text]    }
+          when text = ss.scan(/#{COLON}/) then
+            action { [:tCOLON, text]  }
+          when text = ss.scan(/#{PLUSASSIGN}/) then
+            action { [:tOP_ASSIGN, text]}
+          when text = ss.scan(/#{MINUSASSIGN}/) then
+            action { [:tOP_ASSIGN, text]}
+          when text = ss.scan(/#{STARASSIGN}/) then
+            action { [:tOP_ASSIGN, text]}
+          when text = ss.scan(/#{DIVIDEASSIGN}/) then
+            action { [:tOP_ASSIGN, text]}
+          when text = ss.scan(/#{EXPOASSIGN}/) then
+            action { [:tOP_ASSIGN, text]}
+          when text = ss.scan(/#{MODULUSASSIGN}/) then
+            action { [:tOP_ASSIGN, text]}
           when text = ss.scan(/#{PLUS}/) then
             action { [:tPLUS, text]}
           when text = ss.scan(/#{MINUS}/) then
             action { [:tMINUS, text]}
-          when text = ss.scan(/#{MULTIPLY}/) then
-            action { [:tMULTIPLY, text]}
+          when text = ss.scan(/#{STAR}/) then
+            action { [:tSTAR, text]}
           when text = ss.scan(/#{DIVIDE}/) then
             action { [:tDIVIDE, text]}
           when text = ss.scan(/#{EXPO}/) then
@@ -101,20 +251,94 @@ class Rubex::Lexer
             action { [:tMODULUS, text]}
           when text = ss.scan(/#{EXPO}/) then
             action { [:tEXPO, text]}
-          when ss.skip(/ /) then
-            action {}
+          when text = ss.scan(/#{EQ}/) then
+            action { [:tEQ, text]  }
+          when text = ss.scan(/#{NEQ}/) then
+            action { [:tNEQ, text]  }
+          when text = ss.scan(/#{ASSIGN}/) then
+            action { [:tASSIGN, text] }
+          when text = ss.scan(/#{BANG}/) then
+            action { [:tBANG, text] }
+          when text = ss.scan(/#{ANDOP}/) then
+            action { [:tANDOP, text] }
+          when text = ss.scan(/#{OROP}/) then
+            action { [:tOROP, text] }
+          when text = ss.scan(/#{BIT_AND_ASSIGN}/) then
+            action { [:tOP_ASSIGN, text] }  
+          when text = ss.scan(/#{BIT_OR_ASSIGN}/) then
+            action { [:tOP_ASSIGN  , text] }
+          when text = ss.scan(/#{BIT_XOR_ASSIGN}/) then
+            action { [:tOP_ASSIGN , text] }
+          when text = ss.scan(/#{BIT_LSHIFT_ASSIGN}/) then
+            action { [:tOP_ASSIGN, text] }
+          when text = ss.scan(/#{BIT_RSHIFT_ASSIGN}/) then
+            action { [:tOP_ASSIGN, text] }
+          when text = ss.scan(/#{BIT_AND}/) then
+            action { [:tBIT_AND      , text] } 
+          when text = ss.scan(/#{BIT_OR}/) then
+            action { [:tBIT_OR        , text] }  
+          when text = ss.scan(/#{BIT_XOR}/) then
+            action { [:tBIT_XOR       , text] }  
+          when text = ss.scan(/#{BIT_LSHIFT}/) then
+            action { [:tBIT_LSHIFT    , text] }  
+          when text = ss.scan(/#{BIT_RSHIFT}/) then
+            action { [:tBIT_RSHIFT      , text] }
+          when text = ss.scan(/#{BIT_NOT}/) then
+            action { [:tBIT_NOT, text] }
+          when text = ss.scan(/#{LTEQ}/) then
+            action { [:tLTEQ, text] }
+          when text = ss.scan(/#{LT}/) then
+            action { [:tLT, text] }
+          when text = ss.scan(/#{GTEQ}/) then
+            action { [:tGTEQ, text] }
+          when text = ss.scan(/#{GT}/) then
+            action { [:tGT, text] }
+          when ss.skip(/^\n\s*$/) then
+            # do nothing
+          when ss.skip(/\s+/) then
+            # do nothing
+          else
+            text = ss.string[ss.pos .. -1]
+            raise ScanError, "can not match (#{state.inspect}) at #{location}: '#{text}'"
+          end
+        when :STRING_LITERAL then
+          case
+          when ss.skip(/#{DQUOTE}/) then
+            action { @state = nil; return [:tSTRING, @string_text] }
+          when text = ss.scan(/[^"\\]/) then
+            action { @string_text << text; nil }
+          when text = ss.scan(/\\/) then
+            action { @state = :STRING_LITERAL_BSLASH; @string_text << text; nil }
+          else
+            text = ss.string[ss.pos .. -1]
+            raise ScanError, "can not match (#{state.inspect}) at #{location}: '#{text}'"
+          end
+        when :STRING_LITERAL_BSLASH then
+          case
+          when text = ss.scan(/./) then
+            action { @state = :STRING_LITERAL; @string_text << text; nil }
+          else
+            text = ss.string[ss.pos .. -1]
+            raise ScanError, "can not match (#{state.inspect}) at #{location}: '#{text}'"
+          end
+        when :COMMENT then
+          case
+          when ss.skip(/./) then
+            action { @state = :COMMENT; nil }
+          when ss.skip(/\n/) then
+            action { @state = nil; [:tNL, "\n"] }
           else
             text = ss.string[ss.pos .. -1]
-            raise ScanError, "can not match (#{state.inspect}): '#{text}'"
+            raise ScanError, "can not match (#{state.inspect}) at #{location}: '#{text}'"
           end
         else
-          raise ScanError, "undefined state: '#{state}'"
+          raise ScanError, "undefined state at #{location}: '#{state}'"
         end # token = case state
 
       next unless token # allow functions to trigger redo w/ nil
     end # while
 
-    raise "bad lexical result: #{token.inspect}" unless
+    raise LexerError, "bad lexical result at #{location}: #{token.inspect}" unless
       token.nil? || (Array === token && token.size >= 2)
 
     # auto-switch state
@@ -123,6 +347,8 @@ class Rubex::Lexer
     token
   end # def next_token
     def do_parse
+      self.ss.string = self.ss.string.split("\n").map! { |e| e.rstrip }.join("\n")
+      self.ss << "\n"
     end
 end # class