opal 0.8.1 → 0.9.0.beta1

data/stdlib/pp.rb

@@ -19,10 +19,8 @@ class PP
         p(*args)
       end
     else
-      def pp(obj, out=`console`, width=79)
-        if `#{out} === console`
-          `console.log(obj)`
-        elsif String === out
+      def pp(obj, out=$stdout, width=79)
+        if String === out
           out + obj.inspect + "\n"
         else
           out << obj.inspect + "\n"

data/stdlib/rbconfig.rb

@@ -15,6 +15,4 @@ module RbConfig
 end
 
 # required for mspec it would appear
-RUBY_NAME = 'opal'
-RUBY_EXE = 'opal'
-RUBY_PATCHLEVEL = "327"
+RUBY_EXE = 'bundle exec bin/opal'

data/stdlib/strscan.rb

@@ -7,7 +7,7 @@ class StringScanner
     @pos     = 0
     @matched = nil
     @working = string
-    @match = []
+    @match   = []
   end
 
   attr_reader :string
@@ -15,15 +15,17 @@ class StringScanner
   def beginning_of_line?
     `#@pos === 0 || #@string.charAt(#@pos - 1) === "\n"`
   end
+
   alias bol? beginning_of_line?
 
-  def scan(regex)
+  def scan(pattern)
+    pattern = anchor(pattern)
+
     %x{
-      var regex  = new RegExp('^' + regex.toString().substring(1, regex.toString().length - 1)),
-          result = regex.exec(#@working);
+      var result = pattern.exec(#@working);
 
       if (result == null) {
-        return #{self}.matched = nil;
+        return #@matched = nil;
       }
       else if (typeof(result) === 'object') {
         #@prev_pos = #@pos;
@@ -46,6 +48,38 @@ class StringScanner
     }
   end
 
+  def scan_until(pattern)
+    pattern = anchor(pattern)
+
+    %x{
+      var pos     = #@pos,
+          working = #@working,
+          result;
+
+      while (true) {
+        result   = pattern.exec(working);
+        pos     += 1;
+        working  = working.substr(1);
+
+        if (result == null) {
+          if (working.length === 0) {
+            return #@matched = nil;
+          }
+
+          continue;
+        }
+
+        #@matched  = #@string.substr(#@pos, pos - #@pos - 1 + result[0].length);
+        #@prev_pos = pos - 1;
+        #@pos      = pos;
+        #@working  = working.substr(result[0].length);
+
+        return #@matched;
+      }
+    }
+  end
+
+
   def [](idx)
     %x{
       var match = #@match;
@@ -66,16 +100,36 @@ class StringScanner
     }
   end
 
-  def check(regex)
+  def check(pattern)
+    pattern = anchor(pattern)
+
     %x{
-      var regexp = new RegExp('^' + regex.toString().substring(1, regex.toString().length - 1)),
-          result = regexp.exec(#@working);
+      var result = pattern.exec(#@working);
 
       if (result == null) {
-        return #{self}.matched = nil;
+        return #@matched = nil;
+      }
+
+      return #@matched = result[0];
+    }
+  end
+
+  def check_until(pattern)
+    %x{
+      var prev_pos = #@prev_pos,
+          pos      = #@pos;
+
+      var result = #{scan_until(pattern)};
+
+      if (result !== nil) {
+        #@matched = result.substr(-1);
+        #@working = #@string.substr(pos);
       }
 
-      return #{self}.matched = result[0];
+      #@prev_pos = prev_pos;
+      #@pos      = pos;
+
+      return result;
     }
   end
 
@@ -87,37 +141,72 @@ class StringScanner
     `#@working.length === 0`
   end
 
-  def skip(re)
+  def exist?(pattern)
     %x{
-      re = new RegExp('^' + re.source)
-      var result = re.exec(#@working);
+      var result = pattern.exec(#@working);
 
       if (result == null) {
-        return #{self}.matched = nil;
+        return nil;
+      }
+      else if (result.index == 0) {
+        return 0;
+      }
+      else {
+        return result.index + 1;
+      }
+    }
+  end
+
+  def skip(pattern)
+    pattern = anchor(pattern)
+
+    %x{
+      var result = pattern.exec(#@working);
+
+      if (result == null) {
+        return #@matched = nil;
       }
       else {
         var match_str = result[0];
         var match_len = match_str.length;
-        #{self}.matched = match_str;
-        self.prev_pos = self.pos;
-        #{self}.pos += match_len;
-        #{self}.working = #{self}.working.substring(match_len);
+
+        #@matched   = match_str;
+        #@prev_pos  = #@pos;
+        #@pos      += match_len;
+        #@working   = #@working.substring(match_len);
+
         return match_len;
       }
     }
   end
 
-  def get_byte()
+  def skip_until(pattern)
+    %x{
+      var result = #{scan_until(pattern)};
+
+      if (result === nil) {
+        return nil;
+      }
+      else {
+        #@matched = result.substr(-1);
+
+        return result.length;
+      }
+    }
+  end
+
+  def get_byte
     %x{
       var result = nil;
-      if (#{self}.pos < #{self}.string.length) {
-        self.prev_pos = self.pos;
-        #{self}.pos += 1;
-        result = #{self}.matched = #{self}.working.substring(0, 1);
-        #{self}.working = #{self}.working.substring(1);
+
+      if (#@pos < #@string.length) {
+        #@prev_pos  = #@pos;
+        #@pos      += 1;
+        result      = #@matched = #@working.substring(0, 1);
+        #@working   = #@working.substring(1);
       }
       else {
-        #{self}.matched = nil;
+        #@matched = nil;
       }
 
       return result;
@@ -127,6 +216,23 @@ class StringScanner
   # not exactly, but for now...
   alias getch get_byte
 
+  def match?(pattern)
+    pattern = anchor(pattern)
+
+    %x{
+      var result = pattern.exec(#@working);
+
+      if (result == null) {
+        return nil;
+      }
+      else {
+        #@prev_pos = #@pos;
+
+        return result[0].length;
+      }
+    }
+  end
+
   def pos=(pos)
     %x{
       if (pos < 0) {
@@ -134,10 +240,30 @@ class StringScanner
       }
     }
 
-    @pos = pos
+    @pos     = pos
     @working = `#{@string}.slice(pos)`
   end
 
+  def post_match
+    %x{
+      if (#@matched === nil) {
+        return nil;
+      }
+
+      return #@string.substr(#@pos);
+    }
+  end
+
+  def pre_match
+    %x{
+      if (#@matched === nil) {
+        return nil;
+      }
+
+      return #@string.substr(0, #@prev_pos);
+    }
+  end
+
   def reset
     @working = @string
     @matched = nil
@@ -152,15 +278,25 @@ class StringScanner
     `#@working.length !== 0`
   end
 
+  def rest_size
+    rest.size
+  end
+
   def terminate
     @match = nil
     self.pos = @string.length
   end
 
   def unscan
-    @pos = @prev_pos
+    @pos      = @prev_pos
     @prev_pos = nil
-    @match = nil
+    @match    = nil
+
     self
   end
+
+private
+  def anchor(pattern)
+    `new RegExp('^' + pattern.toString().substr(1, pattern.toString().length - 2))`
+  end
 end

data/tasks/benchmarking.rake

@@ -0,0 +1,88 @@
+namespace :bench do
+  directory "tmp/bench"
+
+  desc "Benchmark Opal"
+  task :opal => "tmp/bench" do |t, args|
+    files = Array(args[:files]) + args.extras
+    index = 0
+    begin
+      index += 1
+      report = "tmp/bench/Opal#{index}"
+    end while File.exist?(report)
+    sh "bundle exec opal benchmark/run.rb #{files.join(" ")} | tee #{report}"
+  end
+
+  desc "Benchmark Ruby"
+  task :ruby => "tmp/bench" do |t, args|
+    files = Array(args[:files]) + args.extras
+    index = 0
+    begin
+      index += 1
+      report = "tmp/bench/Ruby#{index}"
+    end while File.exist?(report)
+    sh "bundle exec ruby benchmark/run.rb #{files.join(" ")} | tee #{report}"
+  end
+
+  desc "Delete all benchmark results"
+  task :clear do
+    sh "rm tmp/bench/*"
+  end
+
+  desc "Combined report of all benchmark results"
+  task :report do |t, args|
+    files = Array(args[:files]) + args.extras
+    if files.empty?
+      files = Dir["tmp/bench/*"]
+    else
+      files = files.map{|file| "tmp/bench/#{ file }"}
+    end
+    reports = []
+    benchmark_names = []
+
+    files.each do |file|
+      report_name = File.basename(file, File.extname(file))
+      report_results = {}
+      File.read(file).each_line do |line|
+        benchmark_name, benchmark_result = line.split(" ")
+        report_results[benchmark_name] = benchmark_result
+        benchmark_names << benchmark_name
+      end
+      reports << [report_name, report_results]
+    end
+
+    benchmark_names.uniq!
+
+    header = ["Benchmark"]
+    reports.each do |report_name, _|
+      header << report_name
+    end
+
+    table = [header]
+    benchmark_names.each do |benchmark_name|
+      row = [benchmark_name]
+      reports.each do |_, report_results|
+        if report_results[benchmark_name]
+          row << format("%0.3f", report_results[benchmark_name])
+        else
+          row << ""
+        end
+      end
+      table << row
+    end
+
+    fmt = ""
+    table.transpose.each_with_index do |column, index|
+      column_width = column.max_by{|string| string.length}.length
+      if index.zero?
+        fmt << "%-#{ column_width }s"
+      else
+        fmt << "  %#{ column_width }s"
+      end
+    end
+    fmt << "\n"
+
+    table.each do |row|
+      printf fmt, *row
+    end
+  end
+end

data/tasks/testing.rake

@@ -1,70 +1,181 @@
+require_relative './testing/mspec_special_calls'
+
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new(:rspec) do |t|
   t.pattern = 'spec/lib/**/*_spec.rb'
 end
 
-require 'mspec/opal/rake_task'
-MSpec::Opal::RakeTask.new(:mspec_phantom) do |config|
-  config.pattern = ENV['MSPEC_PATTERN'] if ENV['MSPEC_PATTERN']
-  config.basedir = ENV['MSPEC_BASEDIR'] if ENV['MSPEC_BASEDIR']
-end
+module Testing
+  extend self
+
+  def stubs
+    %w[
+      mspec/helpers/tmp
+      mspec/helpers/environment
+      mspec/guards/block_device
+      mspec/guards/endian
+    ]
+  end
+
+  def specs(env = ENV)
+    suite = env['SUITE']
+    pattern = env['PATTERN']
+    whitelist_pattern = !!env['RUBYSPECS']
+
+    excepting = []
+    rubyspecs = File.read('spec/rubyspecs').lines.reject do |l|
+      l.strip!
+      l.start_with?('#') || l.empty? || (l.start_with?('!') && excepting.push(l.sub('!', 'spec/') + '.rb'))
+    end.flat_map do |path|
+      path = "spec/#{path}"
+      File.directory?(path) ? Dir[path+'/*.rb'] : "#{path}.rb"
+    end - excepting
+
+    opalspecs = Dir['spec/{opal,lib/parser}/**/*_spec.rb'] + ['spec/lib/lexer_spec.rb']
+    userspecs = Dir[pattern] if pattern
+    userspecs &= rubyspecs if whitelist_pattern
+
+    opalspec_filters = Dir['spec/filters/**/*_opal.rb']
+    rubyspec_filters = Dir['spec/filters/**/*.rb'] - opalspec_filters
+
+    specs = []
+    add_specs = ->(name, new_specs) do
+      puts "Adding #{new_specs.size.to_s.rjust(3)} files (#{name})"
+      specs += new_specs
+    end
+
+    # Filters must be added first
+    suite_filters = suite == 'opal' ? opalspec_filters : rubyspec_filters
+    add_specs["#{suite} filters", suite_filters]
+
+    if pattern
+      add_specs["PATTERN=#{pattern}", userspecs]
+    elsif suite == 'opal'
+      add_specs['spec/opal', opalspecs]
+    elsif suite == 'rubyspec'
+      add_specs['spec/rubyspec', rubyspecs]
+    else
+      warn 'Please provide at lease one of the following ENV vars:'
+      warn 'PATTERN # e.g. env PATTERN="spec/rubyspec/core/numeric/**_spec.rb"'
+      warn 'SUITE   # can be either SUITE=opal or SUITE=rubyspec'
+      exit 1
+    end
+
+    specs
+  end
 
-desc <<-DESC
-Run the MSpec test suite on node
+  def write_file(filename, specs, bm_filepath = nil)
+    requires = specs.map{|s| "require '#{s.sub(/^spec\//,'')}'"}
 
+    if bm_filepath
+      enter_benchmarking_mode = "OSpecRunner.main.bm!(#{Integer(ENV['BM'])}, '#{bm_filepath}')"
+    end
+
+    File.write filename, <<-RUBY
+      require 'spec_helper'
+      #{enter_benchmarking_mode}
+      #{requires.join("\n    ")}
+      OSpecFilter.main.unused_filters_message(list: #{!!ENV['LIST_UNUSED_FILTERS']})
+      OSpecRunner.main.did_finish
+    RUBY
+  end
+
+  def bm_filepath
+    mkdir_p 'tmp/bench'
+    index = 0
+    begin
+      index += 1
+      bm_filepath = "tmp/bench/Spec#{index}"
+    end while File.exist?(bm_filepath)
+  end
+end
+
+pattern_usage = <<-DESC
 Use PATTERN and env var to manually set the glob for specs:
 
   # Will run all specs matching the specified pattern.
   # (Note: the rubyspecs filters will still apply)
-  rake mspec_node PATTERN=spec/corelib/core/module/class_variable*
+  env PATTERN="spec/rubyspec/core/module/class_variable*_spec.rb" rake mspec_node
+  env PATTERN="spec/rubyspec/core/numeric/**_spec.rb" rake mspec_node
 DESC
-task :mspec_node do
-  excepting = []
-  rubyspecs = File.read('spec/rubyspecs').lines.reject do |l|
-    l.strip!; l.start_with?('#') || l.empty? || (l.start_with?('!') && excepting.push(l.sub('!', 'spec/') + '.rb'))
-  end.flat_map do |path|
-    path = "spec/#{path}"
-    File.directory?(path) ? Dir[path+'/*.rb'] : "#{path}.rb"
-  end - excepting
-
-  filters = Dir['spec/filters/**/*.rb']
-  shared = Dir['spec/{opal,lib/parser}/**/*_spec.rb'] + ['spec/lib/lexer_spec.rb']
-
-  specs = []
-  add_specs = ->(name, new_specs) { p [new_specs.size, name]; specs + new_specs}
-
-  specs = add_specs.(:filters, filters)
-  pattern = ENV['PATTERN']
-  whitelist_pattern = !!ENV['RUBYSPECS']
-
-  if pattern
-    custom = Dir[pattern]
-    custom &= rubyspecs if whitelist_pattern
-    specs = add_specs.(:custom, custom)
-  else
-    specs = add_specs.(:shared, shared)
-    specs = add_specs.(:rubyspecs, rubyspecs)
+
+%w[rubyspec opal].each do |suite|
+  desc "Run the MSpec/#{suite} test suite on Phantom.js" + pattern_usage
+  task :"mspec_#{suite}_phantom" do
+    filename = File.expand_path('tmp/mspec_phantom.rb')
+    runner   = "#{__dir__}/testing/phantomjs1-sprockets.js"
+    port     = 9999
+    url      = "http://localhost:#{port}/"
+
+    mkdir_p File.dirname(filename)
+    Testing.write_file filename, Testing.specs(ENV.to_hash.merge 'SUITE' => suite)
+
+    Testing.stubs.each {|s| ::Opal::Processor.stub_file s }
+
+    Opal::Config.arity_check_enabled = true
+    Opal::Config.freezing_stubs_enabled = false
+    Opal::Config.tainting_stubs_enabled = false
+    Opal::Config.dynamic_require_severity = :error
+
+    Opal.use_gem 'mspec'
+    Opal.append_path 'spec'
+    Opal.append_path 'lib'
+    Opal.append_path File.dirname(filename)
+
+    app = Opal::Server.new { |s| s.main = File.basename(filename) }
+    server = Thread.new { Rack::Server.start(app: app, Port: port) }
+    sleep 1
+
+    begin
+      sh 'phantomjs', runner, url
+    ensure
+      server.kill if server.alive?
+    end
   end
 
-  requires = specs.map{|s| "require '#{s.sub(/^spec\//,'')}'"}
-  include_paths = '-Ispec -Ilib'
+  desc "Run the MSpec test suite on Node.js" + pattern_usage
+  task :"mspec_#{suite}_node" do
+    include_paths = '-Ispec -Ilib'
 
-  filename = 'tmp/mspec_node.rb'
-  js_filename = 'tmp/mspec_node.js'
-  mkdir_p File.dirname(filename)
-  enter_benchmarking_mode = ENV['BM'] && "OSpecRunner.main.bm!(#{Integer(ENV['BM'])})"
-  File.write filename, <<-RUBY
-    require 'spec_helper'
-    #{enter_benchmarking_mode}
-    #{requires.join("\n    ")}
-    OSpecRunner.main.did_finish
-  RUBY
-
-  stubs = '-smspec/helpers/tmp -smspec/helpers/environment -smspec/guards/block_device -smspec/guards/endian'
-
-  sh "ruby -rbundler/setup -rmspec/opal/special_calls "\
-     "bin/opal -gmspec #{include_paths} #{stubs} -rnodejs -Dwarning -A #{filename} -c > #{js_filename}"
-  sh "NODE_PATH=stdlib/nodejs/node_modules node #{js_filename}"
+    filename = 'tmp/mspec_node.rb'
+    js_filename = 'tmp/mspec_node.js'
+    mkdir_p File.dirname(filename)
+    bm_filepath = Testing.bm_filepath if ENV['BM']
+    Testing.write_file filename, Testing.specs(ENV.to_hash.merge 'SUITE' => suite), bm_filepath
+
+    stubs = Testing.stubs.map{|s| "-s#{s}"}.join(' ')
+
+    sh "ruby -rbundler/setup -r#{__dir__}/testing/mspec_special_calls "\
+       "bin/opal -gmspec #{include_paths} #{stubs} -rnodejs/io -rnodejs/kernel -Dwarning -A #{filename} -c > #{js_filename}"
+    sh "NODE_PATH=stdlib/nodejs/node_modules node #{js_filename}"
+
+    if bm_filepath
+      puts "Benchmark results have been written to #{bm_filepath}"
+      puts "To view the results, run bundle exec rake bench:report"
+    end
+  end
+end
+
+task :jshint do
+  js_filename = 'tmp/jshint.js'
+  mkdir_p 'tmp'
+
+  if ENV['SUITE'] == 'core'
+    sh "ruby -rbundler/setup bin/opal -ce '23' > #{js_filename}"
+    sh "jshint --verbose #{js_filename}"
+  elsif ENV['SUITE'] == 'stdlib'
+    sh "rake dist"
+
+    Dir["build/*.js"].each {|path|
+      unless path =~ /(opal.*js)|.min.js/
+        sh "jshint --verbose #{path}"
+      end
+    }
+  else
+    warn 'Please provide at lease one of the following ENV vars:'
+    warn 'SUITE   # can be either SUITE=core or SUITE=stdlib'
+    exit 1
+  end
 end
 
 task :cruby_tests do
@@ -72,10 +183,12 @@ task :cruby_tests do
     files = Dir[ENV['FILES'] || 'test/test_*.rb']
     include_paths = '-Itest -I. -Itmp -Ilib'
   else
-    include_paths = '-Itest/cruby/test'
+    include_paths = '-Itest/cruby/test -Itest'
     test_dir = Pathname("#{__dir__}/../test/cruby/test")
     files = %w[
+      benchmark/test_benchmark.rb
       ruby/test_call.rb
+      opal/test_keyword.rb
     ].flat_map do |path|
       if path.end_with?('.rb')
         path
@@ -99,10 +212,23 @@ task :cruby_tests do
   puts "== Running: #{files.join ", "}"
 
   sh "ruby -rbundler/setup "\
-     "bin/opal #{include_paths} #{stubs} -rnodejs -Dwarning -A #{filename} -c > #{js_filename}"
+     "bin/opal #{include_paths} #{stubs} -rnodejs -ropal-parser -Dwarning -A #{filename} -c > #{js_filename}"
   sh "NODE_PATH=stdlib/nodejs/node_modules node #{js_filename}"
 end
 
-task :mspec    => [:mspec_node, :mspec_phantom]
+task :mspec    => [:mspec_rubyspec_node, :mspec_rubyspec_phantom, :mspec_opal_node, :mspec_opal_phantom]
 task :minitest => [:cruby_tests]
 task :test_all => [:rspec, :mspec, :minitest]
+
+
+if (current_suite = ENV['SUITE'])
+  # Legacy tasks, only if ENV['SUITE'] is set
+  desc "Deprecated: use mspec_rubyspec_phantom or mspec_opal_phantom instead"
+  task :mspec_phantom => :"mspec_#{current_suite}_phantom"
+
+  desc "Deprecated: use mspec_rubyspec_node or mspec_opal_node instead"
+  task :mspec_node    => :"mspec_#{current_suite}_node"
+else
+  task :mspec_phantom => [:mspec_opal_phantom, :mspec_rubyspec_phantom]
+  task :mspec_node => [:mspec_opal_node, :mspec_rubyspec_node]
+end