opal 0.3.2 → 0.3.6

data/gems/dev/lib/dev/parser.js

@@ -1,979 +0,0 @@
-// if (typeof require !== 'undefined') {
-//   var RubyParser    = require('./ruby_parser').RubyParser,
-//       // Lexer         = require('./lexer').Lexer,
-//       StringScanner = require('./string_scanner').StringScanner;
-// } else {
-//   // var RubyParser = parser;
-// }
-
-var RubyParser = require('opal_dev/ruby_parser').RubyParser;
-var StringScanner = require('opal_dev/string_scanner').StringScanner;
-var RubyGenerator = require('opal_dev/generator');
-
-for (var prop in RubyGenerator) {
-  if (RubyGenerator.hasOwnProperty(prop)) {
-    // console.log("adding prop " + prop);
-    RubyParser.prototype[prop] = RubyGenerator[prop];
-  }
-}
-
-exports.compile = function(source) {
-  var nodes = exports.RubyParser.parse(source);
-  // print('nodes are:');
-  // print(nodes);
-  // print('generating:');
-  // var g = new RubyGenerator(nodes, {});
-  // var res = g.generate_main_context();
-  // var res = RubyGenerator.generate_main_context(nodes, {});
-  var res = nodes.generateTop();
-  // print("GOT RESULT");
-  // print(res[0]);
-  // print('WOWOWOWOWOWOWOW');
-  return res;
-};
-
-var EXPR_BEG    = 0,    EXPR_END    = 1,    EXPR_ENDARG = 2,    EXPR_ARG   = 3,
-    EXPR_CMDARG = 4,    EXPR_MID    = 5,    EXPR_FNAME  = 6,    EXPR_DOT   = 7,
-    EXPR_CLASS  = 8,    EXPR_VALUE  = 9;
-
-RubyParser.prototype.parse = function(source) {
-  // print('1. Lexing...');
-  var token;
-  
-  this._string = source;
-  this._scanner = new StringScanner(source);
-  this.lex_state = EXPR_BEG;
-  this._tokens = [];
-  this._string_parse_stack = [];
-  this._line_number = 1;
-  
-  // cond stack
-  this._cond = 0;
-  // cmd arg stack
-  this._cmdarg = 0;
-  
-  // while ((token = this.next_token()) && token[0] !== false) {
-    // token[2] = this._line_number;
-    // this._tokens.push(token);
-  // } 
-  
-  // console.log("## Pre Optimize:\n");
-  // for (var i = 0; i < this._tokens.length; i++) {
-  //   console.log(this._tokens[i]);
-  // }
-  // 
-  // this._tokens = new Optimizer(this._tokens).optimize();
-  // console.log("\n## Post Optimize:\n")
-  // for (var i = 0; i < this._tokens.length; i++) {
-    // console.log(this._tokens[i]);
-  // }
-  // console.log("\n");
-  // return this._tokens;
-  // print('parsing some codeeeeez');
-  
-  // print('1. lexing');
-  // this._lexer = new Lexer();
-  // this._tokens = this._lexer.tokenize(source + '\n');
-  // add [false, false] to singnify EOF
-  // this._tokens.push([false, false]);
-  // print(this._tokens);
-  
-  // console.log('2. parsing');
-  try {
-    var result = this.do_parse();
-    // console.log(result);
-    return result;
-  } catch (e) {
-      e.message += " on line: " + this._line_number;
-      throw e;
-  }
-};
-
-// Tokens are fed into the parser as an array. The 0 index is the token type,
-// and the index 1 is an object, with the line number the token was found on,
-// and the value. e.g.
-//
-//    ['IDENTIFIER', { value: 'wow', line: 1 }]
-//
-RubyParser.prototype.next_token = function() {
-  var token = this.get_next_token();
-  // console.log('[' + token.join(', ') + '] - ' + this._line_number);
-  token[1] = { value: token[1], line: this._line_number };
-  return token;
-};
-
-RubyParser.prototype.cond_push = function(n) {
-  // print('cond_push ' + n);
-  return this._cond = (this._cond << 1) | ((n) & 1);
-};
-
-RubyParser.prototype.cond_pop = function() {
-  // print('cond_pop');
-  return this._cond = this._cond >> 1;
-};
-
-RubyParser.prototype.cond_lexpop = function() {
-  // print('cond_lexpop');
-  return this._cond = (this._cond >> 1) | (this._cond & 1);
-};
-
-RubyParser.prototype.cond_p = function() {
-  // print('cond_p');
-  return this._cond & 1;
-};
-
-RubyParser.prototype.cmdarg_push = function(n) {
-  // print('cmdarg_push ' + n);
-  return this._cmdarg = (this._cmdarg << 1) | ((n) & 1);
-};
-
-RubyParser.prototype.cmdarg_pop = function() {
-  // print('cmdarg_pop');
-  return this._cmdarg = this._cmdarg >> 1;
-};
-
-RubyParser.prototype.cmdarg_lexpop = function() {
-  // print('cmdarg_lexpop');
-  return this._cmdarg = (this._cmdarg >> 1) | (this._cmdarg & 1);
-};
-
-RubyParser.prototype.cmdarg_p = function() {
-  // print('cmdarg_p');
-  return this._cmdarg & 1;
-};
-
-RubyParser.prototype.current_string_parse = function() {
-  if (this._string_parse_stack.length == 0) return null;
-  return this._string_parse_stack[this._string_parse_stack.length - 1];
-};
-
-RubyParser.prototype.push_string_parse = function(o) {
-  this._string_parse_stack.push(o);
-};
-
-RubyParser.prototype.pop_string_parse = function() {
-  this._string_parse_stack.pop();
-};
-
-RubyParser.prototype.next_string_token = function() {
-  var str_parse = this.current_string_parse(), scanner = this._scanner;
-  
-  // everything bar single quote and lower case barewords can interpolate
-  var interpolate = (str_parse.beg !== "'" && str_parse.beg !== '%w');
-  
-  // see if we can read end of string/xstring/regexp markers
-  if (scanner.scan( new RegExp('^\\' + str_parse.end))) {
-    this.pop_string_parse();
-    if (str_parse.beg == '"' || str_parse.beg == "'") {
-      this.lex_state = EXPR_END;
-      return ['STRING_END', scanner.matched];
-    }
-    else if (str_parse.beg == '`') {
-      // assume to be xstring
-      this.lex_state = EXPR_END;
-      return ['STRING_END', scanner.matched];
-    }
-    else if (str_parse.beg == '/') {
-      var result = "";
-      if (scanner.scan(/^\w+/)) {
-        result = scanner.matched;
-      }
-      this.lex_state = EXPR_END;
-      return ['REGEXP_END', result];
-    }
-    else { // words?
-      this.lex_state = EXPR_END;
-      return ['STRING_END', scanner.matched];
-    }
-  }
-  
-  // not end of string, so we must be parsing contents
-  var str_buffer = [];
-  
-  if (scanner.scan(/^#(\$|\@)\w+/)) {
-    if (interpolate) {
-      return ['STRING_DVAR', scanner.matched.substr(2)];
-    }
-    else {
-      str_buffer.push(scanner.matched);
-    }
-  }
-  else if (scanner.scan(/^#\{/)) {
-    if (interpolate) {
-      // we are into ruby code, so stop parsing content (for the moment)
-      str_parse.content = false;
-      return ['STRING_DBEG', scanner.matched];
-    }
-    else {
-      str_buffer.push(scanner.matched);
-    }
-  }
-  // causes error, so we will just collect it later on with other text
-  else if (scanner.scan(/^#/)) {
-    str_buffer.push('#');
-  }
-  
-  this.add_string_content(str_buffer, str_parse);
-  var complete_str = str_buffer.join('');
-
-  return ['STRING_CONTENT', complete_str];
-};
-
-RubyParser.prototype.add_string_content = function(str_buffer, str_parse) {
-  var scanner = this._scanner;
-  // regexp for end of string/regexp 
-  var end_str_re = new RegExp('^' + str_parse.end);
-  // can we interpolate?
-  var interpolate = ['"', '%W', '/', '`'].indexOf(str_parse.beg) != -1;
-
-  while (!scanner.eos()) {
-    var c = null, handled = true;
-    if (scanner.check(end_str_re)) {
-      //throw new Error("EOS");
-      break;
-
-    } else if (interpolate && scanner.check(/^#(?=[\@\{])/)) {
-      break;
-   
-    } else if (scanner.scan(/^\\\\/)) {
-      c = scanner.matched; 
-
-    } else if (scanner.scan(/^\\/)) {
-      c = scanner.matched;
-      if (scanner.scan(end_str_re)) {
-        c += scanner.matched;
-      }
-    
-    } else {
-      handled = false;
-    }
-    
-    if (!handled) {
-      var reg = new RegExp("^[^\\" + str_parse.end + "\#\0\\\\]+|.");
-      scanner.scan(reg);
-      c = scanner.matched;
-    }
-
-    c = c || scanner.matched;
-    str_buffer.push(c);
-  }
-
-  if (scanner.eos()) {
-    throw new Error("reached end of file when in string")
-  }
-};
-
-
-
-RubyParser.prototype.get_next_token = function() {
-  var string_scanner;
-  if ((string_scanner = this.current_string_parse()) && string_scanner.content){
-    return this.next_string_token();
-  }
-  
-  var scanner    = this._scanner,
-      space_seen = false,
-      c          = '',
-      cmd_start  = false;
-      
-  
-  while (true) {
-    if (scanner.scan(/^(\ |\t|\r)/)) {
-      space_seen = true;
-      continue;
-    }
-    else if (scanner.scan(/^(\n|#)/)) {
-      c = scanner.matched;
-      if (c == '#') {
-        scanner.scan(/^(.*)/);
-      }
-      else {
-        this._line_number++;
-      }
-      
-      scanner.scan(/^(\n+)/);
-      this._line_number += scanner.matched.length;
-      
-      if ([EXPR_BEG, EXPR_DOT].indexOf(this.lex_state) !== -1) {
-        continue;
-      }
-      cmd_start = true;
-      this.lex_state = EXPR_BEG;
-      return ["\\n", "\\n"];
-    }
-    
-    else if (scanner.scan(/^\;/)) {
-      this.lex_state = EXPR_BEG;
-      return [";", ";"];
-    }
-    
-    else if (scanner.scan(/^\"/)) {
-      this.push_string_parse({ beg: '"', content: true, end:'"' });
-      return ['STRING_BEG', scanner.matched];
-    }
-    else if (scanner.scan(/^\'/)) {
-      this.push_string_parse({ beg: "'", content: true, end:"'" });
-      return ['STRING_BEG', scanner.matched];
-    }
-    else if (scanner.scan(/^\`/)) {
-      this.push_string_parse({ beg: "`", content: true, end: "`" });
-      return ["XSTRING_BEG", scanner.matched];
-    }
-    else if (scanner.scan(/^\%[Ww]/)) {
-      var start_word = scanner.scan(/^./),
-          end_word   = { '(': ')', '[': ']', '{': '}'}[start_word],
-          end_word   = end_word || start_word;
-      
-      this.push_string_parse({ beg: start_word, content: true, end: end_word });
-      return ["WORDS_BEG", scanner.matched]; 
-    }
-    else if (scanner.scan(/^\%[Qq]/)) {
-      var start_word = scanner.scan(/^./),
-          end_word   = { '(': ')', '[': ']', '{': '}'}[start_word],
-          end_word   = end_word || start_word;
-      
-      this.push_string_parse({ beg: start_word, content: true, end: end_word });
-      return ["STRING_BEG", scanner.matched];
-    }
-    else if (scanner.scan(/^\%[Rr]/)) {
-      var start_word = scanner.scan(/^./),
-          end_word   = { '(': ')', '{': '}', '[': ']' }[start_word],
-          end_word   = end_word || start_word;
-      
-      this.push_string_parse({ beg: '/', content: true, end: end_word });
-      return ['REGEXP_BEG', scanner.matched];
-    }
-    
-    else if (scanner.scan(/^\//)) {
-      if (this.lex_state == EXPR_BEG || this.lex_state == EXPR_MID) {
-        this.push_string_parse({ beg: "/", content: true, end: "/" });
-        return ["REGEXP_BEG", scanner.matched];
-      }
-      else if (scanner.scan(/^\=/)) {
-        this.lex_state = EXPR_BEG;
-        return ["OP_ASGN", "/"];
-      }
-      else if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-      }
-      return ["/", '/'];
-    }
-    
-    else if (scanner.scan(/^\%/)) {
-      // if (scanner.scan(/^\=/)) {
-        // this.lex_state = EXPR_BEG;
-        // return ["OP_ASGN", "%"];
-      // }
-      if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-      }
-      else {
-        this.lex_state = EXPR_BEG;
-      }
-      return ["%", '%'];
-    }
-    
-    else if (scanner.scan(/^\(/)) {
-      var result = '(';
-      if (this.lex_state == EXPR_BEG || this.lex_state == EXPR_MID) {
-        result = 'PAREN_BEG';
-      }
-      else if (space_seen) {
-        result = '(';
-      }
-      this.lex_state = EXPR_BEG;
-      this.cond_push(0);
-      this.cmdarg_push(0);
-      return [result, scanner.matched];
-    }
-    
-    else if (scanner.scan(/^\)/)) {
-      this.cond_lexpop();
-      this.cmdarg_lexpop();
-      this.lex_state = EXPR_END;
-      return [")", scanner.matched];
-    }
-    
-    else if (scanner.scan(/^\[/)) {
-      var result = scanner.matched;
-      
-      if (this.lex_state == EXPR_FNAME || this.lex_state == EXPR_DOT) {
-        this.lex_state = EXPR_ARG;
-        if (scanner.scan(/^\]\=/)) {
-          return ["[]=", "[]="];
-        }
-        else if (scanner.scan(/^\]/)) {
-          return ["[]", "[]"];
-        }
-        else {
-          throw "error - unexpected '[' token"
-        }
-      }
-      else if (this.lex_state == EXPR_BEG || this.lex_state == EXPR_MID || space_seen) {
-        this.lex_state = EXPR_BEG;
-        this.cond_push(0);
-        this.cmdarg_push(0);
-        return ["[", scanner.matched];
-      }
-      else {
-        this.lex_state = EXPR_BEG;
-        this.cond_push(0);
-        this.cmdarg_push(0);
-        return ["[@", scanner.matched];
-      }
-    }
-    
-    else if (scanner.scan(/^\]/)) {
-      this.cond_lexpop();
-      this.cmdarg_lexpop();
-      this.lex_state = EXPR_END;
-      return ["]", scanner.matched];
-    }
-    else if (scanner.scan(/^\}/)) {
-      this.cond_lexpop();
-      this.cmdarg_lexpop();
-      this.lex_state = EXPR_END;
-      
-      if (this.current_string_parse()) {
-        this.current_string_parse().content = true
-      }
-      // if (string_parse) string_parse.content = true;
-      return ["}", scanner.matched];
-    }
-    
-    else if (scanner.scan(/^\.\.\./)) {
-      this.lex_state = EXPR_BEG;
-      return ["...", scanner.matched];
-    }
-    else if (scanner.scan(/^\.\./)) {
-      this.lex_state = EXPR_BEG;
-      return ["..", scanner.matched];
-    }
-    else if (scanner.scan(/^\./)) {
-      if (this.lex_state !== EXPR_FNAME) this.lex_state = EXPR_DOT;
-      return [".", scanner.matched];
-    }
-    
-    else if (scanner.scan(/^\*\*\=/)) {
-      this.lex_state = EXPR_BEG;
-      return ["OP_ASGN", "**"];
-    }
-    else if (scanner.scan(/^\*\*/)) {
-      return ["**", "**"];
-    }
-    else if (scanner.scan(/^\*\=/)) {
-      this.lex_state = EXPR_BEG;
-      return ["OP_ASGN", "*"];
-    }
-    else if (scanner.scan(/^\*/)) {
-      if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-        return ["*", scanner.matched];
-      }
-      else if (space_seen && scanner.check(/^\S/)) {
-        this.lex_state = EXPR_BEG;
-        return ["SPLAT", scanner.matched];
-      }
-      else if (this.lex_state == EXPR_BEG || this.lex_state == EXPR_MID) {
-        this.lex_state = EXPR_BEG;
-        return ["SPLAT", scanner.matched];
-      }
-      else {
-        this.lex_state = EXPR_BEG;
-        return ["*", scanner.matched];
-      }
-    }
-    
-    else if (scanner.scan(/^\:\:/)) {
-      if ([EXPR_BEG, EXPR_MID, EXPR_CLASS].indexOf(this.lex_state) !== -1) {
-        this.lex_state = EXPR_BEG;
-        return ["::@", scanner.matched];
-      }
-      this.lex_state = EXPR_DOT;
-      return ["::", scanner.matched];
-    }
-    else if (scanner.scan(/^\:/)) {
-      if (this.lex_state == EXPR_END || this.lex_state == EXPR_ENDARG || scanner.check(/^\s/)) {
-        if (!scanner.check(/^\w/)) {
-          this.lex_state = EXPR_BEG;
-          return [":", scanner.matched];
-        }
-        
-        this.lex_state = EXPR_FNAME;
-        return ["SYMBOL_BEG", scanner.matched];
-      }
-      
-      if (scanner.scan(/^\'/)) {
-        this.push_string_parse({ beg: "'", content: true, end: "'" });
-      }
-      else if (scanner.scan(/^\"/)) {
-        this.push_string_parse({ beg: '"', content: true, end: '"' });
-      }
-      
-      this.lex_state = EXPR_FNAME;
-      return ["SYMBOL_BEG", scanner.matched];
-    }
-    
-    else if (scanner.check(/^\|/)) {
-      if (scanner.scan(/^\|\|\=/)) {
-        this.lex_state = EXPR_BEG;
-        return ["OP_ASGN", "||"];
-      }
-      else if (scanner.scan(/^\|\|/)) {
-        this.lex_state = EXPR_BEG;
-        return ["||", scanner.matched];
-      }
-      else if (scanner.scan(/^\|\=/)) {
-        this.lex_state = EXPR_BEG;
-        return ["OP_ASGN", "|"];
-      }
-      else if (scanner.scan(/^\|/)) {
-        if (this.lex_state == EXPR_FNAME) {
-          this.lex_state = EXPR_END;
-          return ["|", scanner.matched];
-        }
-        this.lex_state = EXPR_BEG;
-        return ["|", scanner.matched];
-      }
-    }
-    
-    else if (scanner.scan(/^\^/)) {
-      if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-        return ["^", scanner.matched];
-      }
-      this.lex_state = EXPR_BEG;
-      return ["^", scanner.matched];
-    }
-    
-    else if (scanner.scan(/^\&\&\=/)) {
-      this.lex_state = EXPR_BEG;
-      return ["OP_ASGN", "&&"];
-    }
-    else if (scanner.scan(/^\&\&/)) {
-      this.lex_state = EXPR_BEG;
-      return ["&&", scanner.matched];
-    }
-    else if (scanner.scan(/^\&\=/)) {
-      this.lex_state = EXPR_BEG;
-      return ["OP_ASGN", "&"];
-    }
-    else if (scanner.scan(/^\&/)) {
-      // print(this.lex_state);
-      if (space_seen && !scanner.check(/^\s/) && this.lex_state == EXPR_CMDARG){
-        return ["&@", scanner.matched];
-      }
-      else if (this.lex_state == EXPR_BEG || this.lex_state == EXPR_MID) {
-        return ["&@", scanner.matched];
-      }
-      else {
-        return ["&", scanner.matched];
-      }
-    }
-    
-    else if (scanner.scan(/^\<\<\=/)) {
-      this.lex_state = EXPR_BEG;
-      return ["OP_ASGN", "<<"];
-    }
-    else if (scanner.scan(/^\<\</)) {
-      if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-        return ["<<", "<<"];
-      }
-      if ([EXPR_END, EXPR_DOT, EXPR_ENDARG, EXPR_CLASS].indexOf(this.lex_state) == -1 && space_seen) {
-        this.lex_state = EXPR_BEG;
-        return ["<<", "<<"];
-      }
-      this.lex_state = EXPR_BEG;
-      return ["<<", "<<"];
-    }
-    else if (scanner.scan(/^\<\=\>/)) {
-      if (this.lex_state == EXPR_FNAME) this.lex_state = EXPR_END
-      else this.lex_state = EXPR_BEG;
-      return ["<=>", "<=>"];
-    }
-    else if (scanner.scan(/^\<\=/)) {
-      if (this.lex_state == EXPR_FNAME) this.lex_state = EXPR_END
-      else this.lex_state = EXPR_BEG;
-      return ["<=", "<="];
-    }
-    else if (scanner.scan(/^\</)) {
-      if (this.lex_state == EXPR_FNAME) this.lex_state = EXPR_END
-      else this.lex_state = EXPR_BEG;
-      return ["<", "<"];
-    }
-    
-    else if (scanner.scan(/^\>\=/)) {
-      if (this.lex_state == EXPR_FNAME) this.lex_state = EXPR_END
-      else this.lex_state = EXPR_BEG;
-      return [">=", scanner.matched];
-    }
-    else if (scanner.scan(/^\>\>\=/)) {
-      return ["OP_ASGN", ">>"];
-    }
-    else if (scanner.scan(/^\>\>/)) {
-      return [">>", scanner.matched];
-    }
-    else if (scanner.scan(/^\>/)) {
-      if (this.lex_state == EXPR_FNAME) this.lex_state = EXPR_END
-      else this.lex_state = EXPR_BEG;
-      return [">", ">"];
-    }
-
-    else if (scanner.scan(/^[+-]/)) {
-      var result = scanner.matched;
-      // var sign = (result == '+') ? 'UPLUS' : 'UMINUS';
-      var sign = result + '@';
-      
-      if (this.lex_state == EXPR_BEG || this.lex_state == EXPR_MID) {
-        this.lex_state = EXPR_BEG;
-        return [sign, sign];
-      }
-      else if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-        if (scanner.scan(/^@/)) {
-          return ['IDENTIFIER', result + scanner.matched];
-        }
-        return [result, result];
-      }
-      
-      if (scanner.scan(/^\=/)) {
-        this.lex_state = EXPR_BEG;
-        return ["OP_ASGN", result];
-      }
-      this.lex_state = EXPR_BEG;
-      return [result, result];
-    }
-    
-    else if (scanner.scan(/^\?/)) {
-      if (this.lex_state = EXPR_END || this.lex_state == EXPR_ENDARG) {
-        this.lex_state = EXPR_BEG;
-      }
-      return ["?", scanner.matched];
-    }
-    
-    else if (scanner.scan(/^\=\=\=/)) {
-      if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-        return ["===", "==="];
-      }
-      this.lex_state = EXPR_BEG;
-      return ["===", "==="];
-    }
-    else if (scanner.scan(/^\=\=/)) {
-      if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-        return ["==", "=="];
-      }
-      this.lex_state = EXPR_BEG;
-      return ["==", "=="];
-    }
-    else if (scanner.scan(/^\=\~/)) {
-      if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-        return ["=~", "=~"];
-      }
-      this.lex_state = EXPR_BEG;
-      return ["=~", "=~"];
-    }
-    else if (scanner.scan(/^\=\>/)) {
-      this.lex_state = EXPR_BEG;
-      return ["=>", scanner.matched];
-    }
-    else if (scanner.scan(/^\=/)) {
-      this.lex_state = EXPR_BEG;
-      return ["=", "="];
-    }
-    
-    else if (scanner.scan(/^\!\=/)) {
-      if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-        return ["!=", "!="];
-      }
-      this.lex_state = EXPR_BEG;
-      return ["!=", scanner.matched];
-    }
-    else if (scanner.scan(/^\!\~/)) {
-      if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-        return ["!~", "!~"];
-      }
-      this.lex_state = EXPR_BEG;
-      return ["!~", "!~"];
-    }
-    else if (scanner.scan(/^\!/)) {
-      if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-        return ["!", "!"];
-      }
-      this.lex_state = EXPR_BEG;
-      return ["!", "!"];
-    }
-    
-    else if (scanner.scan(/^\~/)) {
-      if (this.lex_state == EXPR_FNAME) {
-        this.lex_state = EXPR_END;
-        return ["~", "~"];
-      }
-      this.lex_state = EXPR_BEG;
-      return ["~", "~"];
-    }
-    
-    // FIXME: do we really need to differentiate between these. generates the
-    // same code. our checks will be in the gvar getters (for the relative 
-    // parts..)
-    // 
-    // else if (scanner.scan(/^\$([1-9]\d*)/)) {
-    //   this.lex_state = EXPR_END;
-    //   return ["NTH_REF", scanner.matched];
-    // }
-    // else if (scanner.scan(/^\$([\+\'\&\`])/)) {
-    //   this.lex_state = EXPR_END;
-    //   return ["BACK_REF", scanner.matched];
-    // }
-    // else if (scanner.scan(/^\$[!@\"~*$?\/\\:;=.,<>_]/)) {
-    //   this.lex_state = EXPR_END;
-    //   return ["GVAR", scanner.matched];
-    // }
-    else if (scanner.scan(/^\$[\+\'\`\&!@\"~*$?\/\\:;=.,<>_]/)) {
-      this.lex_state = EXPR_END;
-      return ["GVAR", scanner.matched];
-    }
-    else if (scanner.scan(/^\$\w+/)) {
-      this.lex_state = EXPR_END;
-      return ["GVAR", scanner.matched];
-    }
-    else if (scanner.scan(/^\@\@\w*/)) {
-      this.lex_state = EXPR_END;
-      return ["CVAR", scanner.matched];
-    }
-    else if (scanner.scan(/^\@\w*/)) {
-      this.lex_state = EXPR_END;
-      return ["IVAR", scanner.matched];
-    }
-    
-    else if (scanner.scan(/^\,/)) {
-      this.lex_state = EXPR_BEG;
-      return [",", scanner.matched];
-    }
-    
-    else if (scanner.scan(/^\{/)) {
-      var result;
-      // print(this.lex_state);
-      if (this.lex_state == EXPR_END || this.lex_state == EXPR_CMDARG) {
-        result = '{@';
-      }
-      else if (this.lex_state == EXPR_ENDARG) {
-        result = 'LBRACE_ARG';
-      }
-      else {
-        result = '{';
-      }
-      
-      this.lex_state = EXPR_BEG;
-      this.cond_push(0);
-      this.cmdarg_push(0);
-      
-      return [result, scanner.matched];
-    }
-    
-    else if (scanner.check(/^[0-9]/)) {
-      this.lex_state = EXPR_END;
-      if (scanner.scan(/^[\d_]+\.[\d_]+\b/)) {
-        return ['FLOAT', scanner.matched.replace(/_/g, '')];
-      }
-      else if (scanner.scan(/^[\d_]+\b/)) {
-        return ['INTEGER', scanner.matched.replace(/_/g, '')];
-      }
-      else if (scanner.scan(/^0(x|X)(\d|[a-f]|[A-F])+/)) {
-        return ['INTEGER', scanner.matched.replace(/_/g, '')];
-      }
-      else {
-        // console.log('unexpected number type');
-        return [false, false];
-      }
-    }
-    
-    else if (scanner.scan(/^(\w)+[\?\!]?/)) {
-      switch (scanner.matched) {
-        case 'class':
-          if (this.lex_state == EXPR_DOT) {
-            this.lex_state = EXPR_END;
-            return ["IDENTIFIER", scanner.matched];
-          }
-          this.lex_state = EXPR_CLASS;
-          return ["CLASS", scanner.matched];
-        case 'module':
-          if (this.lex_state == EXPR_DOT) return ["IDENITFIER", scanner.matched];
-          this.lex_state = EXPR_CLASS;
-          return ["MODULE", scanner.matched];
-        case 'def':
-          this.lex_state = EXPR_FNAME;
-          return ["DEF", scanner.matched];
-        case 'end':
-          this.lex_state = EXPR_END;
-          return ["END", scanner.matched];
-        
-        case 'do':
-          if (this.cond_p()) {
-            this.lex_state = EXPR_BEG;
-            return ["DO_COND", scanner.matched];
-          }
-          else if (this.cmdarg_p() && this.lex_state != EXPR_CMDARG) {
-            this.lex_state = EXPR_BEG;
-            return ["DO_BLOCK", scanner.matched];
-          }
-
-          else if (this.lex_state == EXPR_ENDARG) {
-            return ["DO_BLOCK", scanner.matched];
-          }
-          else {
-            this.lex_state = EXPR_BEG;
-            return ["DO", scanner.matched];
-          }
-            
-            // this.lex_state = EXPR_BEG;
-            // return ["DO", scanner.matched];
-          // }
-          // this.lex_state = EXPR_BEG;
-          // return ["DO_BLOCK", scanner.matched];
-        case 'if':
-          if (this.lex_state == EXPR_BEG) return ["IF", scanner.matched];
-          this.lex_state = EXPR_BEG;
-          return ["IF_MOD", scanner.matched];
-        case 'unless':
-          if (this.lex_state == EXPR_BEG) return ["UNLESS", scanner.matched];
-          this.lex_state = EXPR_BEG;
-          return ["UNLESS_MOD", scanner.matched];
-        case 'else':
-          return ["ELSE", scanner.matched];
-        case 'elsif':
-          return ["ELSIF", scanner.matched];
-        case 'self':
-          if (this.lex_state !== EXPR_FNAME) this.lex_state = EXPR_END;
-          return ["SELF", scanner.matched];
-        case 'true':
-          this.lex_state = EXPR_END;
-          return ["TRUE", scanner.matched];
-        case 'false':
-          this.lex_state = EXPR_END;
-          return ["FALSE", scanner.matched];
-        case 'nil':
-          this.lex_state = EXPR_END;
-          return ["NIL", scanner.matched];
-        case '__LINE__':
-          this.lex_state = EXPR_END;
-          return ["LINE", this._line_number.toString()];
-        case '__FILE__':
-          this.lex_state = EXPR_END;
-          return ["FILE", scanner.matched];
-        case 'begin':
-          this.lex_state = EXPR_BEG;
-          return ["BEGIN", scanner.matched];
-        case 'rescue':
-        if (this.lex_state == EXPR_DOT || this.lex_state == EXPR_FNAME) return ["IDENTIFIER", scanner.matched];
-          if (this.lex_state == EXPR_BEG) return ["RESCUE", scanner.matched];
-          this.lex_state = EXPR_BEG;
-          return ["RESCUE_MOD", scanner.matched];
-        case 'ensure':
-          this.lex_state = EXPR_BEG;
-          return ["ENSURE", scanner.matched];
-        case 'case':
-          this.lex_state = EXPR_BEG;
-          return ["CASE", scanner.matched];
-        case 'when':
-          this.lex_state = EXPR_BEG;
-          return ["WHEN", scanner.matched];
-        case 'or':
-          this.lex_state = EXPR_BEG;
-          return ["OR", scanner.matched];
-        case 'and':
-          this.lex_state = EXPR_BEG;
-          return ["AND", scanner.matched];
-        case 'not':
-          this.lex_state = EXPR_BEG;
-          return ["NOT", scanner.matched];
-        case 'return':
-          this.lex_state = EXPR_MID;
-          return ["RETURN", scanner.matched];
-        case 'next':
-          if (this.lex_state == EXPR_DOT) return ["IDENTIFIER", scanner.matched];
-          this.lex_state = EXPR_MID;
-          return ["NEXT", scanner.matched];
-        case 'break':
-          this.lex_state = EXPR_MID;
-          return ["BREAK", scanner.matched];
-        case 'super':
-          this.lex_state = EXPR_ARG;
-          return ["SUPER", scanner.matched];
-        case 'then':
-          return ["THEN", scanner.matched];
-        case 'while':
-          if (this.lex_state == EXPR_BEG) return ["WHILE", scanner.matched];
-          this.lex_state = EXPR_BEG;
-          return ["WHILE_MOD", scanner.matched];
-        case 'until':
-          // generator determines between while and until (mod)
-          if (this.lex_state == EXPR_BEG) return ["WHILE", scanner.matched];
-          this.lex_state = EXPR_BEG;
-          return ["WHILE_MOD", scanner.matched];
-        case 'block_given?':
-          this.lex_state = EXPR_END;
-          return ["BLOCK_GIVEN", scanner.matched];
-        case 'yield':
-          this.lex_state = EXPR_ARG;
-          return ["YIELD", scanner.matched];
-        // case 'require':
-          // if (this.lex_state == EXPR_DOT || this.lex_state == EXPR_FNAME) {
-            // return ["IDENTIFIER", scanner.matched];
-          // }
-          // this.lex_state = EXPR_MID;
-          // return ['REQUIRE', scanner.matched];
-      }
-      
-      var matched = scanner.matched;
-      
-      if (scanner.peek(2) !== '::' && scanner.scan(/^\:/)) {
-        return["LABEL", matched];
-      }
-      
-      if (this.lex_state == EXPR_FNAME) {
-        if (scanner.scan(/^=/)) {
-          this.lex_state = EXPR_END;
-          return ["IDENTIFIER", matched + scanner.matched];
-        }
-        
-        // this.lex_state = EXPR_END;
-        // return ["IDENTIFIER", matched];
-      }
-      
-      // IDENTIFIER2, when we have identifer() .. when we dont preceed identifier
-      // with :: or .
-      // this makes our parser easier and removes conflicts
-      // if (this.lex_state !== EXPR_DOT && scanner.peek(1) == '(') {
-        // this.lex_state = EXPR_CMDARG;
-        // return ["IDENTIFIER2", matched];
-      // }
-      
-      if ([EXPR_BEG, EXPR_DOT, EXPR_MID, EXPR_ARG, EXPR_CMDARG].indexOf(this.lex_state) !== -1) {
-        this.lex_state = EXPR_CMDARG;
-      }
-      else {
-        this.lex_state = EXPR_END;
-      }
-      
-      return [/^[A-Z]/.exec(matched) ? "CONSTANT" : "IDENTIFIER", matched];
-    }
-    
-    else {
-      
-      return [false, false];
-    }
-    
-    return [false, false];
-  }
-};
-
-
-
-exports.RubyParser = new RubyParser();