js2 0.1.8 → 0.3.0.pre5

data/lib/js2/ragel/tokenizer.rl.erb

@@ -1,347 +0,0 @@
-# Somewhat based on http://www.mozilla.org/js/language/js20-2000-07/formal/lexer-grammar.html
-# Regular Expression Literals determined with these rules:
-#  http://www.mozilla.org/js/language/js20-1999-03-25/tokens.html
-
- %%{
-  machine dude;
-  alphtype char;
-
-  action actionStringBegin {
-  }
-
-  action actionStringAcc {
-  }
-
-  action actionStringAccBsEscape {
-  }
-
-  action actionStringAccUtf16 {
-  }
-
-  unicodecharacter = any;
-  unicodeinitialalphabetic = alpha;
-  unicodealphanumeric = alnum;
-  whitespacecharacter = [\t\v\f ];
-  lineterminator = ('\r' | '\n');
-  asciidigit = digit;
-  ws = (whitespacecharacter | lineterminator);
-
-  literals = (
-      '=='  | '!='  | '===' | '!==' | '<='  | '>='  | '||'  | '&&'  | '++'  | '--'  | '<<'  | '<<=' | '>>'  | '>>=' | '>>>' | '>>>='| '&='  | '%='  | '^='  | '|='  | '+='  | '-='  | '*='  | '/='  
-  );
-
-  # put in class
-  # put in accessor
-  keywords = (
-    'break' | 'case' | 'catch' | 'continue' | 'default' | 'delete' | 'do' |
-    'else' | 'finally' | 'for' | 'function' | 'if' | 'in' | 'instanceof' |
-    'new' | 'return' | 'switch' | 'this' | 'throw' | 'try' | 'typeof' | 'var' | 'accessor' |
-    'void' | 'while' | 'with' | 'const' | 'true' | 'false' | 'null' | 'debugger' | 
-    'class' | 'static' | 'foreach' | 'module' | 'include' | 'property'
-  );
-
-  # took out class
-  reserved = (
-    'abstract' | 'boolean' | 'byte' | 'char' | 'double' | 'enum' | 'export' |
-    'extends' | 'final' | 'float' | 'goto' | 'implements' | 'import' | 'int' | 'interface' |
-    'long' | 'native' | 'package' | 'private' | 'protected' | 'public' | 'short' |
-    'super' | 'synchronized' | 'throws' | 'transient' | 'volatile' 
-  );
-
-  nonterminator = any - lineterminator;
-  linecommentcharacters = nonterminator*;
-  nonterminatororslash = nonterminator - '/';
-  nonterminatororastreisk = nonterminator - '*';
-  nonterminatororasteriskorslash = nonterminator - ('*' | '/');
-  blockcommentcharacters = (nonterminatororslash | (nonterminatororastreisk '/'))*;
-  multilineblockcommentcharacters = (blockcommentcharacters lineterminator)*;
-
-  linecomment = '//' linecommentcharacters;
-  singlelineblockcomment = '/*' blockcommentcharacters '*/';
-  multilineblockcomment = '/*' multilineblockcommentcharacters blockcommentcharacters '*/';
-  comment = linecomment | singlelineblockcomment | multilineblockcomment;
-  
-  string_begin = '\'' @ actionStringBegin;
-  string_end = '\'';
-  stringchar_normal = ^(['\\] | 0..0x1f) @ actionStringAcc;
-  stringchar_bs_esc = '\\'['\\/bfnrt] @ actionStringAccBsEscape;
-  stringchar_utf16 = '\\u'[0-9a-fA-F]{4} @ actionStringAccUtf16;
-  stringchar_bs_other = '\\'^(['\\/bfnrtu]|0..0x1f) @ actionStringAccBsEscape;
-  stringchar = (stringchar_normal | stringchar_bs_esc | stringchar_utf16 | stringchar_bs_other);
-  string = string_begin . stringchar* . string_end;
-
-  ds_string_begin = '"' @ actionStringBegin;
-  ds_string_end = '"';
-  ds_stringchar_normal = ^(["\\] | 0..0x1f) @ actionStringAcc;
-  ds_stringchar_bs_esc = '\\'["\\/bfnrt] @ actionStringAccBsEscape;
-  ds_stringchar_utf16 = '\\u'[0-9a-fA-F]{4} @ actionStringAccUtf16;
-  ds_stringchar_bs_other = '\\'^(["\\/bfnrtu]|0..0x1f) @ actionStringAccBsEscape;
-  ds_stringchar = (ds_stringchar_normal | ds_stringchar_bs_esc | ds_stringchar_utf16 | ds_stringchar_bs_other);
-  ds_string = ds_string_begin . ds_stringchar* . ds_string_end;
-
-  all_string = string | ds_string;
-
-  integer = '-'? . digit+;
-  float = '-'? (
-    (('0' | [1-9][0-9]*) '.' [0-9]+ ([Ee] [+\-]?[0-9]+)?)
-    | (('0' | [1-9][0-9]*) ([Ee] [+\-]?[0-9]+))
-  );
-  number = integer | float;
-
-  identifier_char = ([a-zA-Z0-9_]) | '$';
-  identifier = identifier_char+;
-
-  ordinaryregexpchar = nonterminator - ('/' | '\\');
-  regexpchars = (ordinaryregexpchar | '\\' nonterminator)+;
-  regexpbody = '/' @ {regexp_start = p;} regexpchars '/';
-  regexpflags = ('g' | 'i' | 'm')*;
-  regexpliteral = regexpbody regexpflags;
-
-  specialcasedivide = (
-    identifier '/'
-  );
-  single_char = any;
-
-  var_list   =  (identifier . ws* . ',' . ws*)* . identifier?;
-  arg_list   = '(' . ws* . var_list . ws* .')';
-
-  curry_with = 'with' . ws* . arg_list;
-  curry      = 'curry' . ws* . arg_list? . ws* . curry_with? . ws*;
-
-  foreach    = 'foreach' . ws* . '(' . ws* . 'var' . ws* . identifier . (ws* . ':' . ws* . identifier)? . ws*;
-
-  js2_include    = 'include' . ws+  . identifier;
-  js2_class      = 'class' . ws+  . identifier;
-  js2_module     = 'module' . ws+  . identifier;
-
-  property   = ws . 'property' . ws . var_list;
-  member     = ws . ('static' . ws+)? . 'var' . ws+  . identifier;
-  method     = ws . ('static' . ws+)? . 'function' . ws* . identifier;
-  private    = ws . 'private' . ws;
-  accessor   = ws . 'accessor' . ws;
-
-
-
-
-  main := |*
-
-    lineterminator => { 
-      //line_number++;
-    };
-
-    js2_class => {
-      if (in_class == 0) {
-        <%= replacer.start_node :CLASS %>
-        in_class = curly_idx;
-      }
-    };
-
-    private => {
-      if (in_class && in_class == curly_idx) {
-        <%= replacer.start_node :PRIVATE %>
-        <%= replacer.stop_node  %>
-      }
-    };
-
-    js2_module => {
-      if (in_class == 0) {
-        <%= replacer.start_node :MODULE %>
-        in_class = curly_idx;
-      }
-    };
-
-    property => {
-      if (in_class && in_class == curly_idx) {
-        <%= replacer.start_member :PROPERTY %>
-      }
-    };
-
-    accessor => {
-      if (in_class && in_class == curly_idx) {
-        <%= replacer.start_member :ACCESSOR %>
-      }
-    };
-
-
-    member => {
-      if (in_class && in_class == curly_idx) {
-        <%= replacer.start_member :MEMBER %>
-      }
-    };
-
-    method => {
-      if (in_class && in_class == curly_idx) {
-        <%= replacer.start_node :METHOD %>
-      }
-    };
-
-
-    js2_include => {
-      if (in_class && in_class == curly_idx) {
-        <%= replacer.start_member :INCLUDE %>
-      }
-    };
-
-    foreach => {
-      <%= replacer.start_node :FOREACH %>
-      mark_on_br = br_count;
-      br_count++;
-    };
-
-    curry => {
-      <%= replacer.start_node :CURRY %>
-    };
-
-    whitespacecharacter => {};
-    comment => { 
-      <%= replacer.comment %>  
-    };
-
-    all_string => {};
-    number => {};
-    keywords => { };
-
-    literals => { };
-    reserved => { };
-    identifier => { };
-
-    <%= replacer.def_literals %>
-
-    single_char | (('(' | '{' | ';' | '}') ws* regexpliteral) => {
-      char single = data[ts-data];
-
-      if (single == '{') {
-        in_foreach = 0;
-        cb_count++;
-      } else if (single == '}') {
-        cb_count--;
-        if (curly_idx && curlies[curly_idx] == cb_count) {
-          in_foreach = 0;
-          <%= replacer.stop_node %>
-          if (curlies[in_class] == cb_count) {
-            in_class = 0;
-          }
-        }
-      } else if (single == ';' && close_on_semi == 1) {
-        <%= replacer.stop_member %>
-      }
-
-      if (single == ';' || single == '}') {
-        classable = 1;
-      }
-
-      if (single == '(') {
-        br_count++;
-      } else if (single == ')') {
-        br_count--;
-        if (mark_on_br == br_count) {
-          <%= replacer.stop_node %>
-          mark_on_br = -1;
-        }
-      }
-    };
-
-  *|;
-}%%
-
-require 'rubygems'
-require 'inline'
-
-class JS2::Parser::Tokenizer
-  attr_accessor :data
-
-  inline do |builder|
-    builder.c_raw <<-END
-
-    int tokenize (int argc, VALUE *argv, VALUE self) {
-      // convert ruby string to char*
-      VALUE r_str     = argv[0];
-      int data_length = #{RUBY_VERSION.match(/^1\.8/) ? "RSTRING(r_str)->len" : "RSTRING_LEN(r_str)"};
-      char* data      = StringValuePtr(r_str);
-
-      int in_class  = 0;
-      int in_module = 0;
-      int close_on_semi = 0;
-      int classable     = 1;
-      int in_foreach = 0;
-
-      // start vars
-      VALUE start_argv[3];
-      ID start_sym = rb_intern("start_node");
-
-      // stop vars
-      VALUE stop_argv[1];
-      ID stop_sym = rb_intern("stop_node");
-
-      // mark vars
-      VALUE mark_argv[1];
-      ID mark_sym = rb_intern("mark_node");
-      int mark_on_br = -1;
-
-
-      VALUE warn_intv[1];
-      ID warn_sym = rb_intern("warn_int");
-
-
-      // state vars
-      int i = 0; // iterator
-      int j = 0; // iterator
-
-      char keyword[100]; // keyword
-      char single;
-      int is_static = 0;
-      int is_private = 0;
-
-      // curly handling
-      int curlies[1000];
-      curlies[0]    = 0;
-      int curly_idx = 0; // on purpose!
-      int cb_count  = 0;
-      int br_count  = 0;
-      int close_on_br = 0;
-
-      <%= replacer.declare_vars [ :CLASS, :METHOD, :MEMBER, :STUFF, :ACCESSOR, :COMMENT, :FOREACH, :MODULE, :INCLUDE, :CURRY, :PROPERTY, :PRIVATE, :STATIC ] %>
-      // ragel variables
-      int cs, act;
-      char *ts  = NULL;
-      char *te  = NULL;
-      char *p   = data;
-      char *pe  = p + data_length;
-      char *eof = pe;
-      char* regexp_start = NULL;
-
-      <%= replacer.start_node :STUFF, 0, true %>
-
-      %% write data;
-      %% write init;
-      %% write exec;
-
-      <%= replacer.stop_node('data_length-1') %>
-    }
-
-    END
-  end
-
-  def warn_int (int)
-    puts int.to_s
-  end
-
-  def start_node (type, idx, is_static)
-    @lexer.start_node(type, idx, is_static == 1)
-  end
-
-  def stop_node (idx)
-    @lexer.stop_node(idx)
-  end
-
-  def mark_node (idx)
-    @lexer.mark_node(idx)
-  end
-
-  def tokenize! (data, lexer)
-    @data  = data
-    @lexer = lexer
-    self.tokenize(data)
-  end
-
-end

data/lib/js2/standard/factory.rb

@@ -1,289 +0,0 @@
-# TODO: break this up into multiple files
-class JS2::Standard::PageNode < JS2::Standard::Node
-  attr_accessor :klasses, :file
-
-  def initialize (idx, string, factory)
-    @klasses = []
-    super(idx, string, factory)
-  end
-
-  def stop (idx)
-    JS2::Standard::ForeachNode.reset!
-    super(idx)
-  end
-end
-
-class JS2::Standard::CommentNode < JS2::Standard::Node
-  def clean
-    return to_s.gsub(%r|^\s*\/?\*+\s?\/?|, '')
-  end
-end
-
-class JS2::Standard::ClassNode < JS2::Standard::Node
-  REGEX = /^(\s*class|module)\s+([\w\.]+)\s+(extends\s+([\w\.]+))?\s*\{(.*)/m
-
-  def handle_first_string (str)
-    if m = str.match(REGEX)
-      @name    = m[2]
-      @extends = m[4]
-      @ending  = m[5]
-
-      if @extends
-        @extends = "#{@name}.oo('extends', #{@extends});"
-      end
-
-      return %|JS2.OO.#{create_name}("#{@name}"); #{@extends} (function (K,Package) {var self=K; var _super=JS2.OO['super']; #{@ending}|
-    end
-  end
-
-  def methods
-    ret = []
-
-    @children.each do |c|
-      if c.is_a?(JS2::Standard::MethodNode)
-        ret << c
-      end
-    end
-
-    return ret
-  end
-
-  def name
-    if m = @string[@start_idx .. @stop_idx].match(REGEX)
-      return m[2]
-    else
-      return ''
-    end
-  end
-
-  def create_name
-    return 'createClass'
-  end
-
-  def handle_ending (str)
-    pkg = @name.split(/\./)
-    pkg.pop()
-
-    if pkg.empty?
-      pkg = 'null'
-    else
-      pkg = pkg.join('.')
-    end
-
-    return str.sub(/\s*\z/, ")(#{@name}, #{pkg});")
-  end
-end
-
-class JS2::Standard::ModuleNode < JS2::Standard::ClassNode
-  def createName
-    return 'createModule'
-  end
-end
-
-class JS2::Standard::PrivateNode < JS2::Standard::Node
-  def handle_first_string (str)
-    return str.sub(/private/m, '// private')
-  end
-end
-
-class JS2::Standard::IncludeNode < JS2::Standard::Node
-  def handle_first_string (s)
-    return s.sub(/include\s*/, "K.oo('include', ").sub(/;$/, ');');
-  end
-end
-
-
-class JS2::Standard::MemberNode < JS2::Standard::Node
-  def handle_first_string (s)
-    method = 'member'
-
-    if s.match(/^(\s*)(static)\s*/)
-      method = 'staticMember'
-      s = s.sub(/static\s*/, '')
-    end
-
-    return s.sub(/var\s*([\$\w]+)/) { "K.oo('#{method}', '" + $1 + "'" }.sub(/;(\s*)$/) {  ");#{$1}" }.sub(/\s*=/, ', ');
-  end
-
-end
-
-class JS2::Standard::CurryNode < JS2::Standard::Node
-  REGEX        = %r|^curry\s*([^\{]*)?\{(.*)$|m
-  REGEX_WITH   = %r|with\s+\(([^)]*)\)|
-  REGEX_ARGS   = %r|^\s*\(([^)]*)\)|
-  REGEX_ENDING = %r|\}([^\}]*)\z|
-
-  def handle_first_string (str)
-    m = str.match(REGEX)
-    @decl = m[1].strip
-    @stop = m[2]
-
-    @args      = ''
-    @in_scoped = ''
-    @scoped    = ''
-
-    if m = @decl.match(REGEX_WITH)
-      in_scoped = m[1].split(',').collect { |v| v.strip }
-      scoped    = in_scoped.collect { |v| v == 'this' ? 'self' : v }
-      if in_scoped.length
-        @scoped =  scoped.join(', ')
-        @in_scoped = in_scoped.join(', ')
-      end
-    end
-
-    if m = @decl.match(%r|^\s*\(([^)]*)\)|)
-      @args = m[1].strip
-    end
-
-    return %|(function (#{@scoped}) { return function (#{@args}) {#{@stop}|
-  end
-
-  def handle_ending (str)
-    if m = str.match(REGEX_ENDING)
-      return str.sub(REGEX_ENDING) { |str| "}})(#{@in_scoped})#{m[1]}" }
-    end
-
-    return str
-  end
-end
-
-
-class JS2::Standard::MethodNode < JS2::Standard::Node
-  attr_accessor :name, :args, :static
-
-  REGEX = /^(\s*)(static\s+)?function\s+([\$\w\.]+)\s*\(([^)]*)\)\s*\{(.*)/m
-
-  def handle_first_string (s)
-    m = s.match(REGEX)
-    space  = m[1]
-    @static = m[2]
-    @name   = m[3]
-    @args   = m[4]
-    @start  = m[5]
-
-    m = static ? 'staticMember' : 'method'
-    return %|#{space}K.oo('#{m}', "#{@name}", function (#{@args}) {#{@start}|
-  end
-
-  def handle_ending (str)
-    return str.sub!(/(\s*)\z/) { |m| ");#{m}" }
-  end
-
-end
-
-class JS2::Standard::AccessorNode < JS2::Standard::Node
-  REGEX = /(\s*)accessor(\s+)([\w+,\s]+\w)(\s*);(\s*)/
-
-  def handle_first_string (str)
-    m = str.match(REGEX)
-    space     = m[1]
-    mid_space = m[2]
-    list      = m[3].split(/,/).collect { |i| i.strip }
-    end_space = m[4]
-    trailing  = m[5]
-
-    return %|#{space}K.oo('accessor',#{mid_space}[ #{list.collect { |i| "'#{i}'" }.join(',')} ])#{end_space};#{trailing}|
-  end
-end
-
-class JS2::Standard::ForeachNode < JS2::Standard::Node
-  #                   start     var           iterator           array
-  REGEX = /(\s*)foreach\s*\(\s*var\s*([\$\w]+)(\s*:\s*([\$\w]))?\s*in\s*([^\s]+)\s*\)/
-  attr_accessor :iterator, :item, :array
-  @@inc = 0
-
-  def handle_first_string (s)
-    m = s.match(REGEX)
-    start    = m[1]
-    item     = m[2]
-    iterator = m[4]
-    array    = m[5]
-
-    it = iterator
-
-    unless iterator
-      it = 'it' + @@inc.to_s
-      @@inc += 1
-    end
-
-    len = it + '__len'
-    arr = it + '__arr'
-
-    return %(#{start}for (var #{it}=0,#{item},#{arr}=#{array},#{len}=#{arr}.length; (#{item}=#{arr}[#{it}]) || #{it}<#{len}; #{it}++))
-  end
-
-  def self.reset!
-    @@inc = 0
-  end
-end
-
-class JS2::Standard::PropertyNode < JS2::Standard::Node
-  REGEX = /(\s*)property(\s+)([\w+,\s]+\w)(\s*);(\s*)/
-  def handle_first_string (s)
-    m = s.match(REGEX)
-    space     = m[1]
-    mid_space = m[2]
-    list      = m[3].split(/,/).collect { |i| i.strip }
-    end_space = m[4]
-    trailing  = m[5]
-
-    return %|#{space}K.oo('property',#{mid_space}[ #{list.collect { |i| "'#{i}'" }.join(',')} ])#{end_space};#{trailing}|
-  end
-end
-
-
-class JS2::Standard::StuffNode < JS2::Standard::Node
-end
-
-
-class JS2::Standard::Factory
-  attr_accessor :decorators
-
-  @@supports = [ :CLASS, :MEMBER, :METHOD, :ACCESSOR, :FOREACH, :PROPERTY, :INCLUDE, :CURRY, :PAGE, :COMMENT, :STUFF, :MODULE, :PRIVATE ]
-  @@lookup = Hash.new
-
-  @@supports.each do |v|
-    name = v.to_s.downcase.sub(/(\w)/) { |m| m.upcase }
-    @@lookup[v] = eval "JS2::Standard::#{name}Node"
-  end
-
-  def initialize 
-    @decorators = []
-  end
-
-  def get_class (type)
-    klass = @@lookup[type] || JS2::Standard::Node
-  end
-
-  def page_node (string, file = nil)
-    @page = new_node(:PAGE, 0, string) 
-    @page.file = file
-    @comment = nil
-    return @page
-  end
-
-  def new_node (type, idx, string)
-    klass = get_class(type)
-    node = klass.new(idx, string, self)
-
-    if type == :CLASS || type == :MODULE
-      @page.klasses << node
-    end
-
-    if type == :COMMENT
-      @comment = node
-    elsif @comment
-      node.comment = @comment
-      @comment = nil
-    end
-
-    setup_node(node)
-
-    return node
-  end
-
-  def setup_node (node)
-    # virtual
-  end
-end
-

data/lib/js2/standard/node.rb

@@ -1,75 +0,0 @@
-class JS2::Standard::Node
-  attr_accessor :type, :start_idx, :stop_idx, :children, :starting, :factory, :comment
-
-  def initialize (start_idx, str, factory)
-    @start_idx = start_idx
-    @string    = str
-    @children  = []
-    @factory   = factory
-    @output    = ''
-  end
-
-  def add_child (type, start_idx)
-    child = factory.new_node(type, start_idx, @string)
-    @children.push(child)
-    return child
-  end
-
-  def stop (idx)
-    @stop_idx = idx
-    process!
-  end
-
-  def to_s
-    return @output
-  end
-
-  def process! ()
-    last_idx = @start_idx
-    str = ''
-
-    first = true
-    @children.each do |c|
-      if c.start_idx > last_idx
-        str << handle_string(@string[last_idx .. c.start_idx-1], first)
-        first = false
-      end
-
-      result = c.to_s()
-
-      @factory.decorators.each do |d|
-        d.decorate(result, c) 
-      end
-
-      str << result
-      last_idx = c.stop_idx + 1
-    end
-
-    if last_idx < @stop_idx
-      str << handle_string(@string[last_idx .. @stop_idx], first)
-    end
-
-    str = handle_ending(str)
-
-    @output = str 
-  end
-
-  private
-
-  def handle_string (str, first) 
-    if first
-      return handle_first_string(str)
-    else
-      return str 
-    end
-  end
-
-  def handle_ending (str)
-    return str
-  end
-
-  def handle_first_string (str)
-    return str
-  end
-
-end