js2 0.0.8 → 0.0.9

data/lib/js2/processor.rb

@@ -1,7 +1,7 @@
 require 'yaml'
 
 class JS2::Processor
-  attr_accessor :fh, :haml_engine, :decorator
+  attr_accessor :fh, :haml_engine
 
   # ----------------------------------------------------------------------------
   # Rails support
@@ -20,13 +20,16 @@ class JS2::Processor
   # ----------------------------------------------------------------------------
   # yaml support
   # ----------------------------------------------------------------------------
-  def self.from_yaml(file)
-    config = JS2::Config.new(YAML.load_file(file))
+  def self.from_yaml(file, env = nil)
+    hash = YAML.load_file(file)
+    hash = hash[env] if env
+
+    config = JS2::Config.new(hash)
     return self.get(config)
   end
 
-  def self.daemon_from_yaml(file)
-    p = self.from_yaml(file)
+  def self.daemon_from_yaml(file, env = nil)
+    p = self.from_yaml(file, env)
     return JS2::Daemon.new(p)
   end
 
@@ -67,7 +70,7 @@ class JS2::Processor
     # haml engine
     haml_engine_class = config.haml_engine_class ? 
                         config.haml_engine_class.constantize : 
-                        JS2::Process::HamlEngine
+                        JS2::HamlEngine
 
     haml_engine = haml_engine_class.new
 
@@ -77,87 +80,33 @@ class JS2::Processor
     js2_haml_dir = config.js2_haml_dir
     haml_dir     = config.haml_dir
 
-    file_handler = JS2::Process::FileHandler.new(
+    file_handler = JS2::FileHandler.new(
       :read_dir     => js2_dir,
       :write_dir    => write_dir,
       :haml_dir     => js2_haml_dir,
       :view_dir     => haml_dir
     )
 
-    decorator_klass = config.test_mode ? 
-                JS2::Decorator::Test :
-                JS2::Decorator::Standard
-
     return self.new(
       :test_mode       => config.test_mode,
       :reference_dir   => config.reference_dir,
       :haml_engine     => haml_engine,
       :file_handler    => file_handler,
-      :decorator_klass => decorator_klass
+      :vars            => config.vars
     )
   end
 
   def initialize (params)
     @haml_engine     = params[:haml_engine] 
     @fh              = params[:file_handler] 
-    @decorator_klass = params[:decorator_klass]   
     @reference_dir   = params[:reference_dir]
     @test_mode       = params[:test_mode]
+    @vars            = params[:vars]
   end
 
   def write_files
-    parser       ||= JS2::Parser::Fast.new()
-    haml_parser  ||= JS2::Parser::Haml.new(@haml_engine)
-    universe       = JS2::Process::Universe.new
-
-    decorator = @decorator_klass.new
-
-    bootstrap_file = File.dirname(__FILE__) + "/../js2bootstrap.js2"
-    bootstrap_nodes = parser.parse(bootstrap_file)
-    universe.add_nodes(bootstrap_nodes)
-
-    @fh.get_files.each do |filename|
-      nodes = parser.parse(filename)
-      universe.add_nodes(nodes)
-    end
-
-    @fh.get_yml_files.each do |filename|
-      begin
-        yml = YAML.load_file(filename)
-        universe.add_yml(yml, filename)
-      rescue Exception => e
-        puts "YAML Error in #{filename}:"
-        puts e.to_s
-      end
-    end
-
-    @fh.get_haml_files.each do |filename|
-      hash = haml_parser.parse(filename)
-      universe.add_haml_hash(hash)
-    end
-
-    universe.finalize!
-
-    universe.node_containers.each do |nc|
-      str = decorator.draw_nodes(nc)
-      @fh.write_file(nc.filename, str)
-    end
-
-    if @test_mode
-      decorator.write_references(@reference_dir)
-    end
-
-    @fh.get_view_files.each do |filename|
-      js        = haml_parser.parse_out_js(filename)
-      nodes     = parser.parse_string(js) 
-      decorator = @decorator_klass.new
-      decorator.draw_nodes(nodes)
-      decorator.write_references(@reference_dir)
-    end
-
-    universe.compilations.each_pair do |filename, hash|
-      @fh.write_compilation(filename, hash)
-    end
+    universe = JS2::Universe.new(@fh, @haml_engine)
+    universe.write
   end
 
 end

data/lib/js2/replace.rb

@@ -0,0 +1,106 @@
+require 'erb'
+
+LITERALS = [ '!=', '!==', '#', '%', '%=', '&&', '&&=', '&=', '*', '*=', '+', '+=', ',', '-', '-=', '->', '.', '/', '/=', ':', '::', '<', '<<', '<<=', '<=', '=', '==', '===', '>', '>=', '>>', '>>=', '>>>', '>>>=', '?', '@', '[', '^', '^=', '^^', '^^=', '|', '|=', '||', '||=', 'abstract', 'break', 'case', 'catch', 'class', 'const', 'continue', 'debugger', 'default', 'delete', 'do', 'else', 'enum', 'export', 'extends', 'field', 'final', 'finally', 'for', 'function', 'goto', 'if', 'implements', 'import', 'in', 'instanceof', 'native', 'new', 'package', 'private', 'protected', 'public', 'return', 'static', 'switch', 'synchronized', 'throw', 'throws', 'transient', 'try', 'typeof', 'var', 'volatile', 'while', 'with', 'foreach', 'module', 'include' ]
+
+class Replacer
+  def initialize
+    @types = []
+  end
+
+  def def_literals
+    ret = []
+    LITERALS.each do |literal|
+      ret << "'#{literal}' ws* regexpliteral => { };"
+    end
+    ret.join("\n") + "\n"
+  end
+
+  def warn_int (int)
+    return <<-END
+      rb_funcall2(self, warn_sym, 1, warn_intv);
+    END
+  end
+
+  def mark_node ()
+    return <<-END
+      mark_argv[0] = INT2FIX(ts-data+1);
+      rb_funcall2(self, mark_sym, 1, mark_argv);
+    END
+  end
+
+  def start_node (type, idx = nil, ignore_curly = false) 
+    return <<-END
+      #{ignore_curly ? '' : 'curlies[++curly_idx] = cb_count;'};
+
+      start_argv[0] = sym_#{type};
+      start_argv[1] = INT2FIX(#{idx || 'ts-data'});
+      start_argv[2] = INT2FIX(is_static);
+      rb_funcall2(self, start_sym, 3, start_argv);
+      is_static = 0;
+    END
+  end
+
+  def start_member (type, idx = nil) 
+    return <<-END
+      close_on_semi = 1;
+      start_argv[0] = sym_#{type};
+      start_argv[1] = INT2FIX(#{idx || 'ts-data'});
+      start_argv[2] = INT2FIX(is_static);
+      rb_funcall2(self, start_sym, 3, start_argv);
+      is_static = 0;
+    END
+  end
+
+
+  def stop_node  (idx = nil)
+    return <<-END
+      if (curly_idx) curly_idx--;
+      stop_argv[0] = INT2FIX(#{idx || 'te-data'});
+      rb_funcall2(self, stop_sym, 1, stop_argv);
+    END
+  end
+
+  def stop_member (idx = nil)
+    return <<-END
+      close_on_semi = 0;
+      stop_argv[0] = INT2FIX(#{idx || 'te-data'});
+      rb_funcall2(self, stop_sym, 1, stop_argv);
+    END
+  end
+
+
+  def in_class 
+    return "in_class == (cb_count + 1)"
+  end
+
+  def in_module
+    return "in_class == curly_idx || in_module == curly_idx"
+  end
+
+  def declare_vars (types)
+    ret = ''
+    types.each do |type|
+      ret += self.declare(type)
+    end
+
+    return ret
+  end
+
+  def declare (type)
+    return <<-END
+      // #{type}
+      ID  sym_#{type} = ID2SYM(rb_intern("#{type}"));
+    END
+  end
+
+end
+
+module JS2
+end
+
+replacer = Replacer.new
+template = ERB.new(File.read('tokenizer.rl.erb'), 0, "%<>")
+
+File.open('tokenizer.rl', 'w') do |f|
+  f << template.result(binding)
+end

data/lib/js2/{decorator/test.rb → sel_decorator.rb}

@@ -1,27 +1,24 @@
 require 'json'
 
-class JS2::Decorator::Test < JS2::Decorator::Standard
+class JS2::SelDecorator
 
   def initialize ()
-    super()
     @references = Hash.new
     @in_class = false
   end
 
-
-  def draw_class (node) 
-    str = super(node)
-    @scope = node.name
-    @in_class = true
-    str = process_str(str)
+  def reset!
+    @references = Hash.new
     @in_class = false
-    return str
   end
 
-  def draw_stuff (node)
-    str = super(node)
-    @scope = nil
-    return process_str(str)
+  def translate (str, scope = nil, in_class = false)
+    @scope    = scope
+    @in_class = in_class
+    ret = process_str(str)
+    @in_class = false
+    @scope    = nil
+    return ret
   end
 
   def write_references  (dir)
@@ -120,7 +117,7 @@ class JS2::Decorator::Test < JS2::Decorator::Standard
           @scope.to_json
         end
 
-      return padding + %{TMP_SEL_MARKER = this.SEL_MARKER || JS2.SEL_MARKER; TMP_SEL_MARKER.#{insert}(#{scope}, #{key.to_json}, #{lval}, #{finder});}
+      return padding + %{var TMP_SEL_MARKER = this.SEL_MARKER || JS2.SEL_MARKER; TMP_SEL_MARKER.#{insert}(#{scope}, #{key.to_json}, #{lval}, #{finder});}
     else
       return false
     end

data/{meta/c_tokenizer.rl.erb → lib/js2/tokenizer.rl.erb}

@@ -24,6 +24,7 @@
   whitespacecharacter = [\t\v\f ];
   lineterminator = ('\r' | '\n');
   asciidigit = digit;
+  ws = (whitespacecharacter | lineterminator);
 
   literals = (
       '=='  | '!='  | '===' | '!==' | '<='  | '>='  | '||'  | '&&'  | '++'  | '--'  | '<<'  | '<<=' | '>>'  | '>>=' | '>>>' | '>>>='| '&='  | '%='  | '^='  | '|='  | '+='  | '-='  | '*='  | '/='  
@@ -101,83 +102,102 @@
   );
   single_char = any;
 
-  arg_list   = '(' . whitespacecharacter* . (identifier . whitespacecharacter* . ',' . whitespacecharacter*)* . identifier? . whitespacecharacter* . ')';
-  curry_with = 'with' . whitespacecharacter* . arg_list;
-  curry      = 'curry' . whitespacecharacter* . arg_list? . whitespacecharacter* . curry_with? . whitespacecharacter* . '{';
+  var_list   =  (identifier . ws* . ',' . ws*)* . identifier?;
+  arg_list   = '(' . ws* . var_list . ws* .')';
+
+  curry_with = 'with' . ws* . arg_list;
+  curry      = 'curry' . ws* . arg_list? . ws* . curry_with? . ws*;
+
+  foreach    = 'foreach' . ws* . '(' . ws* . 'var' . ws* . identifier . (ws* . ':' . ws* . identifier)? . ws*;
+
+  js2_include    = 'include' . ws+  . identifier;
+  js2_class      = 'class' . ws+  . identifier;
+  js2_module     = 'module' . ws+  . identifier;
+
+  property   = ws . 'property' . ws . var_list;
+  member     = ws . ('static' . ws+)? . 'var' . ws+  . identifier;
+  method     = ws . ('static' . ws+)? . 'function' . ws* . identifier;
+  private    = ws . 'private' . ws;
+  accessor   = ws . 'accessor' . ws;
+
+
+
 
   main := |*
 
     lineterminator => { 
-      line_number++;
+      //line_number++;
     };
 
-    curry => {
-      curr = ts-data;
-      <%= replacer.start :CURRY %>
-      cb_count++;  
+    js2_class => {
+      if (in_class == 0) {
+        <%= replacer.start_node :CLASS %>
+        in_class = curly_idx;
+      }
     };
 
-    whitespacecharacter => {};
-    comment => {
-      if (<%= replacer.directly_in? :CLASS %>) {
-        <%= replacer.do_next(:COMMENT, 'ts-data', 'te-data-1') %> 
+    private => {
+      if (in_class && in_class == curly_idx) {
+        <%= replacer.start_node :PRIVATE %>
+        <%= replacer.stop_node  %>
       }
     };
 
-    all_string => {};
-    number => {};
-    keywords => { 
-      curr = ts-data;
-
-      <%= replacer.set_keyword %>
-
-      if (<%= replacer.is? :CLASS %> || <%= replacer.is? :MODULE %>) {
-        if (<%= replacer.is? :CLASS %>) {
-          <%= replacer.start :CLASS %>
-        } else {
-          <%= replacer.start :MODULE %>
-        }
+    js2_module => {
+      if (in_class == 0) {
+        <%= replacer.start_node :MODULE %>
+        in_class = curly_idx;
+      }
+    };
 
-        // if "class" wasn't the first thing in the file
-        // then submit the :STUFF
-        if (curr > 0) {
-          <%= replacer.stop :STUFF, 'curr-1' %>
-        }
+    property => {
+      if (in_class && in_class == curly_idx) {
+        <%= replacer.start_member :PROPERTY %>
+      }
+    };
 
-      } else if (<%= replacer.directly_in? :CLASS %> || <%= replacer.directly_in? :MODULE %>) { 
-        if (<%= replacer.is? :STATIC %>) {
-          <%= replacer.set_static %>
-        }
+    accessor => {
+      if (in_class && in_class == curly_idx) {
+        <%= replacer.start_member :ACCESSOR %>
+      }
+    };
 
-        if (<%= replacer.is? :FUNCTION %>) {
-          <%= replacer.start :METHOD %>
-        }
 
-        if (<%= replacer.is? :VAR %>) {
-          <%= replacer.start :MEMBER %>
-        }
+    member => {
+      if (in_class && in_class == curly_idx) {
+        <%= replacer.start_member :MEMBER %>
+      }
+    };
 
+    method => {
+      if (in_class && in_class == curly_idx) {
+        <%= replacer.start_node :METHOD %>
+      }
+    };
 
-        if (<%= replacer.is? :ACCESSOR %>) {
-          <%= replacer.start :MEMBER %>
-        }
 
-        if (<%= replacer.is? :PROPERTY %>) {
-          <%= replacer.start :PROPERTY %>
-        }
+    js2_include => {
+      if (in_class && in_class == curly_idx) {
+        <%= replacer.start_member :INCLUDE %>
+      }
+    };
 
+    foreach => {
+      <%= replacer.start_node :FOREACH %>
+      mark_on_br = br_count;
+      br_count++;
+    };
 
-        if (<%= replacer.is? :INCLUDE %>) {
-          <%= replacer.start :INCLUDE %>
-        }
+    curry => {
+      <%= replacer.start_node :CURRY %>
+    };
 
-      } 
-        
-      if (<%= replacer.is? :FOREACH %>) {
-        <%= replacer.start :FOREACH %>
-      }
+    whitespacecharacter => {};
+    comment => { };
 
-    };
+    all_string => {};
+    number => {};
+    keywords => { };
 
     literals => { };
     reserved => { };
@@ -185,38 +205,39 @@
 
     <%= replacer.def_literals %>
 
-    single_char | (('(' | '{' | ';') whitespacecharacter* regexpliteral) => {
-      curr   = ts-data;
-      single = data[ts-data];
-
-      if (single == '(') {
-        br_count++;
-      } else if (single == ')') {
-        br_count--;
-
-        if (in_FOREACH && br_count == br_lvl_FOREACH) {
-          <%= replacer.stop :FOREACH %>
-        }
-      }
+    single_char | (('(' | '{' | ';' | '}') ws* regexpliteral) => {
+      char single = data[ts-data];
 
       if (single == '{') {
+        in_foreach = 0;
         cb_count++;
       } else if (single == '}') {
-        if (<%= replacer.directly_in? :CLASS %> || <%= replacer.directly_in? :MODULE %>) { 
-          <%= replacer.start :STUFF, 'curr+1' %>
-        }
-
         cb_count--;
-
-        // stuff that stops on }'s
-        <%= replacer.handle_stops [ :METHOD, :CLASS, :FOREACH, :MODULE, :CURRY ] %>
+        if (curly_idx && curlies[curly_idx] == cb_count) {
+          in_foreach = 0;
+          <%= replacer.stop_node %>
+          if (curlies[in_class] == cb_count) {
+            in_class = 0;
+          }
+        }
+      } else if (single == ';' && close_on_semi == 1) {
+        <%= replacer.stop_member %>
       }
 
-      if (single == ';') {
-        <%= replacer.handle_semicolon_stops [ :MEMBER, :INCLUDE, :PROPERTY ] %>
+      if (single == ';' || single == '}') {
+        classable = 1;
       }
 
-   };
+      if (single == '(') {
+        br_count++;
+      } else if (single == ')') {
+        br_count--;
+        if (mark_on_br == br_count) {
+          <%= replacer.stop_node %>
+          mark_on_br = -1;
+        }
+      }
+    };
 
   *|;
 }%%
@@ -224,7 +245,7 @@
 require 'rubygems'
 require 'inline'
 
-class JS2::Parser::Fast
+class JS2::Parser
   attr_accessor :data
 
   inline do |builder|
@@ -236,33 +257,48 @@ class JS2::Parser::Fast
       int data_length = RSTRING(r_str)->len;
       char* data      = STR2CSTR(r_str);
 
-      VALUE ln_argv[1];
-      int ln_argc     = 1;
-      int line_number = 1;
-      ID  ln_sym      = rb_intern("set_line_number");
+      int in_class  = 0;
+      int in_module = 0;
+      int close_on_semi = 0;
+      int classable     = 1;
+      int in_foreach = 0;
+
+      // start vars
+      VALUE start_argv[3];
+      ID start_sym = rb_intern("start_node");
+
+      // stop vars
+      VALUE stop_argv[1];
+      ID stop_sym = rb_intern("stop_node");
 
-      // do_next vars
-      VALUE do_next_argv[4];
-      int   do_next_argc = 4;
-      ID    do_next_sym  = rb_intern("do_next");
+      // mark vars
+      VALUE mark_argv[1];
+      ID mark_sym = rb_intern("mark_node");
+      int mark_on_br = -1;
+
+
+      VALUE warn_intv[1];
+      ID warn_sym = rb_intern("warn_int");
 
-      VALUE set_static_argv[0];
-      int   set_static_argc = 0;
-      ID    set_static_sym  = rb_intern("set_static"); 
 
       // state vars
       int i = 0; // iterator
       int j = 0; // iterator
-      char word[100]; // keyword
-      char single;
-
-      int curr     = 0;
-      int prev     = -1;
-      int cb_count = 0;
-      int br_count = 0;
 
-      <%= replacer.declare_vars [ :CLASS, :METHOD, :MEMBER, :STUFF, :ACCESSOR, :COMMENT, :FOREACH, :MODULE, :INCLUDE, :CURRY, :PROPERTY ] %>
- 
+      char keyword[100]; // keyword
+      char single;
+      int is_static = 0;
+      int is_private = 0;
+
+      // curly handling
+      int curlies[1000];
+      curlies[0]    = 0;
+      int curly_idx = 0; // on purpose!
+      int cb_count  = 0;
+      int br_count  = 0;
+      int close_on_br = 0;
+
+      <%= replacer.declare_vars [ :CLASS, :METHOD, :MEMBER, :STUFF, :ACCESSOR, :COMMENT, :FOREACH, :MODULE, :INCLUDE, :CURRY, :PROPERTY, :PRIVATE, :STATIC ] %>
       // ragel variables
       int cs, act;
       char *ts  = NULL;
@@ -272,51 +308,47 @@ class JS2::Parser::Fast
       char *eof = pe;
       char* regexp_start = NULL;
 
-      <%= replacer.start :STUFF %>
+      <%= replacer.start_node :STUFF, 0, true %>
 
       %% write data;
       %% write init;
       %% write exec;
 
-      if (in_STUFF == 1) {
-        <%= replacer.stop :STUFF %>
-      }
+      <%= replacer.stop_node('data_length-1') %>
     }
 
     END
   end
 
-  def set_line_number (l)
-    @line_number = l
+  def warn_int (int)
+    puts int.to_s
   end
 
-  def set_static ()
-    @nodes.set_static
+  def start_node (type, idx, is_static)
+    @ph.start_node(type, idx, is_static == 1)
   end
 
-  def do_next (symbol, start_idx, end_idx, line)
-    if end_idx == -1 then end_idx = start_idx end
-    @nodes.do_next(symbol, start_idx, end_idx, line)
+  def stop_node (idx)
+    @ph.stop_node(idx)
   end
 
-  def parse_string (str)
-    self.data = str
-    @nodes = JS2::AST::Nodes.new("string", str)
-    self.do_parse(str)
-    return @nodes
+  def mark_node (idx)
+    @ph.mark_node(idx)
   end
 
-  def parse(filename)
-    self.data = File.read(filename)
-    @nodes    = JS2::AST::Nodes.new(filename, data)
 
-    #@cleanser ||= JS2::DecoCleanser.new
-    #@cleanser.cleanse(data)
-
-    eof       = data.size
+  def parse (data)
+    @data = data
+    @ph = JS2::ParserHelper.new(data)
     self.do_parse(data)
+    return @ph
+  end
 
-    return @nodes
+  def parse_file (filename)
+    data = File.read(filename)
+    ph = self.parse(data)
+    ph.filename = filename
+    return ph
   end
 
 end