opal 1.5.0 → 1.6.0.alpha1

data/lib/opal/builder_scheduler/prefork.rb

@@ -0,0 +1,259 @@
+# frozen_string_literal: true
+
+require 'etc'
+
+module Opal
+  class BuilderScheduler
+    class Prefork < BuilderScheduler
+      # We hook into the process_requires method
+      def process_requires(rel_path, requires, autoloads, options)
+        return if requires.empty?
+
+        if @in_fork
+          io = @in_fork
+          io.send(:new_requires, rel_path, requires, autoloads, options)
+        else
+          prefork_reactor(rel_path, requires, autoloads, options)
+        end
+      end
+
+      private
+
+      class ForkSet < Array
+        def initialize(count, &block)
+          super([])
+
+          @count, @block = count, block
+
+          create_fork
+        end
+
+        def get_events(queue_length)
+          # Wait for anything to happen:
+          # - Either any of our workers return some data
+          # - Or any workers become ready to receive data
+          #   - But only if we have enough work for them
+          ios = IO.select(
+            map(&:read_io),
+            sample(queue_length).map(&:write_io),
+            []
+          )
+          return [[], []] unless ios
+
+          events = ios[0].map do |io|
+            io = from_io(io, :read_io)
+            [io, *io.recv]
+          end
+
+          idles = ios[1].map do |io|
+            from_io(io, :write_io)
+          end
+
+          # Progressively create forks, because we may not need all
+          # the workers at the time. The number 6 was picked due to
+          # some trial and error on a Ryzen machine.
+          #
+          # Do note that prefork may happen more than once.
+          create_fork if length < @count && rand(6) == 1
+
+          [events, idles]
+        end
+
+        def create_fork
+          self << Fork.new(self, &@block)
+        end
+
+        def from_io(io, type)
+          find { |i| i.__send__(type) == io }
+        end
+
+        def close
+          each(&:close)
+        end
+
+        def wait
+          each(&:wait)
+        end
+      end
+
+      class Fork
+        def initialize(forkset)
+          @parent_read, @child_write = IO.pipe
+          @child_read, @parent_write = IO.pipe
+          @forkset = forkset
+          @in_fork = false
+
+          @pid = fork do
+            @in_fork = true
+
+            begin
+              @parent_read.close
+              @parent_write.close
+
+              yield(self)
+            rescue => error
+              send(:exception, error)
+            ensure
+              send(:close) unless write_io.closed?
+              @child_write.close
+            end
+          end
+
+          @child_read.close
+          @child_write.close
+        end
+
+        def close
+          send(:close)
+          @parent_write.close
+        end
+
+        def goodbye
+          read_io.close unless read_io.closed?
+        end
+
+        def send_message(io, msg)
+          msg = Marshal.dump(msg)
+          io.write([msg.length].pack('Q') + msg)
+        end
+
+        def recv_message(io)
+          length, = *io.read(8).unpack('Q')
+          Marshal.load(io.read(length)) # rubocop:disable Security/MarshalLoad
+        end
+
+        def fork?
+          @in_fork
+        end
+
+        def read_io
+          fork? ? @child_read : @parent_read
+        end
+
+        def write_io
+          fork? ? @child_write : @parent_write
+        end
+
+        def eof?
+          write_io.closed?
+        end
+
+        def send(*msg)
+          send_message(write_io, msg)
+        end
+
+        def recv
+          recv_message(read_io)
+        end
+
+        def wait
+          Process.waitpid(@pid, Process::WNOHANG)
+        end
+      end
+
+      # By default we use 3/4 of CPU threads detected.
+      def fork_count
+        ENV['OPAL_PREFORK_THREADS']&.to_i || (Etc.nprocessors * 3 / 4.0).ceil
+      end
+
+      def prefork
+        @forks = ForkSet.new(fork_count, &method(:fork_entrypoint))
+      end
+
+      def fork_entrypoint(io)
+        @in_fork = io
+
+        until io.eof?
+          $0 = 'opal/builder: idle'
+
+          type, *args = *io.recv
+          case type
+          when :compile
+            rel_path, req, autoloads, options = *args
+            $0 = "opal/builder: #{req}"
+            begin
+              asset = builder.process_require_threadsafely(req, autoloads, options)
+              io.send(:new_asset, asset)
+            rescue Builder::MissingRequire => error
+              io.send(:missing_require_exception, rel_path, error)
+            end
+          when :close
+            io.goodbye
+            break
+          end
+        end
+      rescue Errno::EPIPE
+        exit!
+      end
+
+      def prefork_reactor(rel_path, requires, autoloads, options)
+        prefork
+
+        first = rel_path
+        queue = requires.map { |i| [rel_path, i, autoloads, options] }
+
+        awaiting = 0
+        built = 0
+
+        $stderr.print "\r\e[K" if $stderr.tty?
+
+        loop do
+          events, idles = @forks.get_events(queue.length)
+
+          idles.each do |io|
+            break if queue.empty?
+
+            rel_path, req, autoloads, options = *queue.pop
+
+            next if builder.already_processed.include?(req)
+            awaiting += 1
+            builder.already_processed << req
+            io.send(:compile, rel_path, req, autoloads, options)
+          end
+
+          events.each do |io, type, *args|
+            case type
+            when :new_requires
+              rel_path, requires, autoloads, options = *args
+              requires.each do |i|
+                queue << [rel_path, i, autoloads, options]
+              end
+            when :new_asset
+              asset, = *args
+              if !asset
+                # Do nothing, we received a nil which is expected.
+              elsif asset.filename == 'corelib/runtime.js'
+                # Opal runtime should go first... the rest can go their way.
+                builder.processed.unshift(asset)
+              else
+                builder.processed << asset
+              end
+              built += 1
+              awaiting -= 1
+            when :missing_require_exception
+              rel_path, error = *args
+              raise error, "A file required by #{rel_path.inspect} wasn't found.\n#{error.message}", error.backtrace
+            when :exception
+              error, = *args
+              raise error
+            when :close
+              io.goodbye
+            end
+          end
+
+          if $stderr.tty?
+            percent = (100.0 * built / (awaiting + built)).round(1)
+            str = format("[opal/builder] Building %<first>s... (%<percent>4.3g%%)\r", first: first, percent: percent)
+            $stderr.print str
+          end
+
+          break if awaiting == 0 && queue.empty?
+        end
+      ensure
+        $stderr.print "\r\e[K\r" if $stderr.tty?
+        @forks.close
+        @forks.wait
+      end
+    end
+  end
+end

data/lib/opal/builder_scheduler/sequential.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Opal
+  class BuilderScheduler
+    class Sequential < BuilderScheduler
+      def process_requires(rel_path, requires, autoloads, options)
+        requires.map { |r| builder.process_require(r, autoloads, options) }
+      rescue Builder::MissingRequire => error
+        raise error, "A file required by #{rel_path.inspect} wasn't found.\n#{error.message}", error.backtrace
+      end
+    end
+  end
+end

data/lib/opal/builder_scheduler.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'opal/os' unless RUBY_ENGINE == 'opal'
+
+module Opal
+  class BuilderScheduler
+    def initialize(builder)
+      @builder = builder
+    end
+
+    attr_reader :builder
+  end
+
+  singleton_class.attr_accessor :builder_scheduler
+
+  if RUBY_ENGINE != 'opal'
+    # Windows has a faulty `fork`.
+    if OS.windows? || ENV['OPAL_PREFORK_DISABLE']
+      require 'opal/builder_scheduler/sequential'
+      Opal.builder_scheduler = BuilderScheduler::Sequential
+    else
+      require 'opal/builder_scheduler/prefork'
+      Opal.builder_scheduler = BuilderScheduler::Prefork
+    end
+  else
+    require 'opal/builder_scheduler/sequential'
+    Opal.builder_scheduler = BuilderScheduler::Sequential
+  end
+end

data/lib/opal/cache/file_cache.rb

@@ -23,6 +23,11 @@ module Opal
         out = Marshal.dump(data)
         out = Zlib.gzip(out, level: 9)
         File.binwrite(file, out)
+      rescue Zlib::BufError
+        # This sometimes happens, unsure why, makes no sense, possibly
+        # some race condition
+        warn '[Opal]: Zlib::BufError; retrying'
+        retry
       end
 
       def get(key)

data/lib/opal/cli.rb

@@ -48,14 +48,12 @@ module Opal
 
       @requires.unshift('opal') unless options.delete(:skip_opal_require)
 
-      @compiler_options = Hash[
-        *compiler_option_names.map do |option|
-          key = option.to_sym
-          next unless options.key? key
-          value = options.delete(key)
-          [key, value]
-        end.compact.flatten
-      ]
+      @compiler_options = compiler_option_names.map do |option|
+        key = option.to_sym
+        next unless options.key? key
+        value = options.delete(key)
+        [key, value]
+      end.compact.to_h
 
       raise ArgumentError, 'no libraries to compile' if @lib_only && @requires.empty?
       raise ArgumentError, 'no runnable code provided (evals or file)' if @evals.empty? && @file.nil? && !@lib_only
@@ -68,6 +66,16 @@ module Opal
       return debug_source_map if @debug_source_map
       return run_repl if @repl
 
+      rbrequires.each { |file| require file }
+
+      runner = self.runner
+
+      # Some runners may need to use a dynamic builder, that is,
+      # a builder that will try to build the entire package every time
+      # a page is loaded - for example a Server runner that needs to
+      # rerun if files are changed.
+      builder = proc { create_builder }
+
       @exit_status = runner.call(
         options: runner_options,
         output: output,
@@ -90,13 +98,7 @@ module Opal
 
     attr_reader :exit_status
 
-    def builder
-      @builder ||= create_builder
-    end
-
     def create_builder
-      rbrequires.each(&Kernel.method(:require))
-
       builder = Opal::Builder.new(
         stubs: stubs,
         compiler_options: compiler_options,
@@ -113,22 +115,22 @@ module Opal
       gems.each { |gem_name| builder.use_gem gem_name }
 
       # --require
-      requires.each { |required| builder.build(required) }
+      requires.each { |required| builder.build(required, requirable: true, load: true) }
 
       # --preload
       preload.each { |path| builder.build_require(path) }
 
       # --verbose
-      builder.build_str '$VERBOSE = true', '(flags)' if verbose
+      builder.build_str '$VERBOSE = true', '(flags)', no_export: true if verbose
 
       # --debug
-      builder.build_str '$DEBUG = true', '(flags)' if debug
+      builder.build_str '$DEBUG = true', '(flags)', no_export: true if debug
 
       # --eval / stdin / file
       evals_or_file { |source, filename| builder.build_str(source, filename) }
 
       # --no-exit
-      builder.build_str '::Kernel.exit', '(exit)' unless no_exit
+      builder.build_str '::Kernel.exit', '(exit)', no_export: true unless no_exit
 
       builder
     end
@@ -166,6 +168,7 @@ module Opal
         irb_enabled
         inline_operators
         enable_source_location
+        enable_file_source_embed
         use_strict
         parse_comments
         esm
@@ -181,6 +184,7 @@ module Opal
       if evals.any?
         yield evals.join("\n"), '-e'
       elsif file && (filename != '-' || evals.empty?)
+        file.rewind
         yield file.read, filename
       end
     end

data/lib/opal/cli_options.rb

@@ -181,6 +181,10 @@ module Opal
         options[:enable_source_location] = true
       end
 
+      on('--enable-file-source-embed', 'Embeds file sources to be accessed by applications.') do
+        options[:enable_file_source_embed] = true
+      end
+
       on('--use-strict', 'Enables JavaScript\'s strict mode (i.e., adds \'use strict\'; statement)') do
         options[:use_strict] = true
       end

data/lib/opal/cli_runners/chrome.rb

@@ -5,6 +5,7 @@ require 'socket'
 require 'timeout'
 require 'tmpdir'
 require 'rbconfig'
+require 'opal/os'
 
 module Opal
   module CliRunners
@@ -20,7 +21,7 @@ module Opal
       end
 
       def initialize(data)
-        builder = data[:builder]
+        builder = data[:builder].call
         options = data[:options]
         argv    = data[:argv]
 
@@ -67,26 +68,30 @@ module Opal
         map = builder.source_map.to_json
         stack = File.read("#{__dir__}/source-map-support-browser.js")
 
+        ext = builder.output_extension
+        module_type = ' type="module"' if builder.esm?
+
         # Chrome can't handle huge data passed to `addScriptToEvaluateOnLoad`
         # https://groups.google.com/a/chromium.org/forum/#!topic/chromium-discuss/U5qyeX_ydBo
         # The only way is to create temporary files and pass them to chrome.
-        File.write("#{dir}/index.js", js)
+        File.write("#{dir}/index.#{ext}", js)
         File.write("#{dir}/source-map-support.js", stack)
         File.write("#{dir}/index.html", <<~HTML)
           <html><head>
             <meta charset='utf-8'>
             <script src='./source-map-support.js'></script>
             <script>
+            window.opalheadlesschrome = true;
             sourceMapSupport.install({
               retrieveSourceMap: function(path) {
-                return path.endsWith('/index.js') ? {
+                return path.endsWith('/index.#{ext}') ? {
                   url: './index.map', map: #{map.to_json}
                 } : null;
               }
             });
             </script>
           </head><body>
-            <script src='./index.js'></script>
+            <script src='./index.#{ext}'#{module_type}></script>
           </body></html>
         HTML
       end
@@ -111,7 +116,7 @@ module Opal
         raise 'Chrome server can be started only on localhost' if chrome_host != DEFAULT_CHROME_HOST
 
         # Disable web security with "--disable-web-security" flag to be able to do XMLHttpRequest (see test_openuri.rb)
-        chrome_server_cmd = %{#{chrome_executable.shellescape} \
+        chrome_server_cmd = %{#{OS.shellescape(chrome_executable)} \
           --headless \
           --disable-web-security \
           --remote-debugging-port=#{chrome_port} \
@@ -132,7 +137,7 @@ module Opal
         puts 'Make sure that you have it installed and that its version is > 59'
         exit(1)
       ensure
-        if Gem.win_platform? && chrome_pid
+        if windows? && chrome_pid
           Process.kill('KILL', chrome_pid) unless system("taskkill /f /t /pid #{chrome_pid} >NUL 2>NUL")
         elsif chrome_pid
           Process.kill('HUP', chrome_pid)
@@ -149,8 +154,7 @@ module Opal
 
       def chrome_executable
         ENV['GOOGLE_CHROME_BINARY'] ||
-          case RbConfig::CONFIG['host_os']
-          when /bccwin|cygwin|djgpp|mingw|mswin|wince/
+          if OS.windows?
             [
               'C:/Program Files/Google/Chrome Dev/Application/chrome.exe',
               'C:/Program Files/Google/Chrome/Application/chrome.exe'
@@ -158,7 +162,7 @@ module Opal
               next unless File.exist? path
               return path
             end
-          when /darwin|mac os/
+          elsif OS.macos?
             '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
           else
             %w[

data/lib/opal/cli_runners/chrome_cdp_interface.rb

@@ -106,17 +106,34 @@ CDP(options, function(client) {
           else
             `Page.handleJavaScriptDialog({accept: false})`
           end
+        elsif `dialog.type` == 'alert' && `dialog.message` == 'opalheadlesschromeexit'
+          # A special case of an alert with a magic string "opalheadlesschromeexit".
+          # This denotes that `Kernel#exit` has been called. We would have rather used
+          # an exception here, but they don't bubble sometimes.
+          %x{
+            Page.handleJavaScriptDialog({accept: true});
+            Runtime.evaluate({ expression: "window.OPAL_EXIT_CODE" }).then(function(output) {
+              client.close();
+              if (typeof(output.result) !== "undefined" && output.result.type === "number") {
+                process.exit(output.result.value);
+              } else {
+                process.exit(0);
+              }
+            });
+          }
         end
       }
     });
 
     Page.loadEventFired(() => {
       Runtime.evaluate({ expression: "window.OPAL_EXIT_CODE" }).then(function(output) {
-        client.close();
-
         if (typeof(output.result) !== "undefined" && output.result.type === "number") {
+          client.close();
           process.exit(output.result.value);
+        } else if (typeof(output.result) !== "undefined" && output.result.type === "string" && output.result.value === "noexit") {
+          // do nothing, we have headless chrome support enabled and there are most probably async events awaiting
         } else {
+          client.close();
           process.exit(0);
         }
       })

data/lib/opal/cli_runners/compiler.rb

@@ -5,7 +5,7 @@ require 'opal/paths'
 # The compiler runner will just output the compiled JavaScript
 Opal::CliRunners::Compiler = ->(data) {
   options  = data[:options] || {}
-  builder  = data.fetch(:builder)
+  builder  = data.fetch(:builder).call
   map_file = options[:map_file]
   output   = data.fetch(:output)
 

data/lib/opal/cli_runners/gjs.rb

@@ -10,10 +10,12 @@ module Opal
     class Gjs
       def self.call(data)
         exe = ENV['GJS_PATH'] || 'gjs'
+        builder = data[:builder].call
 
         opts = Shellwords.shellwords(ENV['GJS_OPTS'] || '')
+        opts.unshift('-m') if builder.esm?
 
-        SystemRunner.call(data) do |tempfile|
+        SystemRunner.call(data.merge(builder: -> { builder })) do |tempfile|
           [exe, *opts, tempfile.path, *data[:argv]]
         end
       rescue Errno::ENOENT

data/lib/opal/cli_runners/mini_racer.rb

@@ -9,11 +9,15 @@ module Opal
       def self.call(data)
         ::MiniRacer::Platform.set_flags! :harmony
 
-        builder = data.fetch(:builder)
+        builder = data.fetch(:builder).call
         output = data.fetch(:output)
         # TODO: pass it
         argv = data.fetch(:argv)
 
+        # MiniRacer doesn't like to fork. Let's build Opal first
+        # in a forked environment.
+        code = builder.to_s + "\n" + builder.source_map.to_data_uri_comment
+
         v8 = ::MiniRacer::Context.new
         v8.attach('prompt', ->(_msg = '') { $stdin.gets&.chomp })
         v8.attach('console.log', ->(i) { output.print(i); output.flush })
@@ -22,8 +26,6 @@ module Opal
         v8.attach('opalminiracer.exit', ->(status) { Kernel.exit(status) })
         v8.attach('opalminiracer.argv', argv)
 
-        code = builder.to_s + "\n" + builder.source_map.to_data_uri_comment
-
         v8.eval(code)
       end
 

data/lib/opal/cli_runners/nodejs.rb

@@ -3,6 +3,7 @@
 require 'shellwords'
 require 'opal/paths'
 require 'opal/cli_runners/system_runner'
+require 'opal/os'
 
 module Opal
   module CliRunners
@@ -39,10 +40,9 @@ module Opal
 
       # Ensure stdlib node_modules is among NODE_PATHs
       def self.node_modules
-        npsep = Gem.win_platform? ? ';' : ':'
-        ENV['NODE_PATH'].to_s.split(npsep).tap do |paths|
+        ENV['NODE_PATH'].to_s.split(OS.env_sep).tap do |paths|
           paths << NODE_PATH unless paths.include? NODE_PATH
-        end.join(npsep)
+        end.join(OS.env_sep)
       end
 
       class MissingNodeJS < RunnerError