opal-webpack-loader 0.10.3 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bf29108044f8d61b4587c76a1d92e2e0a9722d69e854dd4fe469f47244ea7b24
-  data.tar.gz: e14eba64f9741db698fa3fef0b19ec830e7a4463bbb50898b569fce3d34ca30a
+  metadata.gz: 75cfc2e9527b252335ad359e1eb5c7aa476cec879ed76c4501b686a50f39b1f2
+  data.tar.gz: b6d13eac5c5eaa0e3c46dfd15b8f59d01d59124f92608c741e93952c87bb04f6
 SHA512:
-  metadata.gz: 033b4afb0191e0b9c69fc6f21a0c5419b0e729bce1ca3b50d4a39272a51111a01c9eaaebda566dc9d587725c0708bcf7717ccc716d670c9fc9828038ec725f89
-  data.tar.gz: 884d3f0491fd4c81bd434ef37cf541329af99e0e6a9e6b38171fb8d7e1611cd605732536dfa72babe4620d5ac327e3a183c17d4506813067e93324521ed95083
+  metadata.gz: c9edddd5f0a21625c5376dbf609e594a35bfe36ab84ca2fb804fa7f011093e8a45aa516530bb0dceceec4dcc069619dc0e7c31ceecfd3f68a5ad0c6caf39699f
+  data.tar.gz: f21627293c524a6db7f7c53f89e83f13f77519597133c05dd992173b7d2cff05f0ab915cfe7eb8e52f036ce448f3905e572d75a2d345cefe93990ece36ad342b

data/README.md

@@ -23,7 +23,7 @@ At the [Isomorfeus Framework Project](http://isomorfeus.com)
 opal-webpack-loader-0.7.1 compiles all of opal, a bunch of gems and over 19000SLC on a
 Intel® Core™ i7-7700HQ CPU @ 2.80GHz × 8, with 8 workers in around 1850ms
 - support for memcached or redis as compiler cache
-- parallel builds working on Windows too, but not as fast as on Linux
+- fast builds on windows too, parallel when building multiple assets with parallel webpack
 - opal modules are packaged as es6 modules
 - support for rails with webpacker
 - other webpack features become available, like:

data/bin/opal-webpack-windows-compile-server

@@ -0,0 +1,125 @@
+#!/usr/bin/env ruby
+require 'digest'
+require 'oj'
+require 'c_lexer'
+require 'optparse'
+require 'opal/paths'
+require 'opal/source_map'
+require 'opal/compiler'
+require 'opal-webpack-loader/load_path_manager'
+require 'opal-webpack-loader/pipe_server'
+
+modules_to_require = []
+compiler_options = {}
+compile_server_options = {}
+cache = false
+
+OptionParser.new do |opts|
+  opts.on('-r', '--require MODULE', 'Require the module before starting the compile server.') do |m|
+    modules_to_require << m
+  end
+
+  opts.on('-I', '--include DIR', 'Append a load path (may be used more than once)') do |i|
+    $:.unshift(File.expand_path(i))
+  end
+
+  opts.on('-d', '--dynamic-require-severity SEVERITY', 'Compiler option, one of: error, warning, ignore.') do |s|
+    if %w[error warning ignore].include?(s)
+      compiler_options[:dynamic_require_severity] = s.to_sym
+    end
+  end
+
+  opts.on('-t', '--true FLAG', 'Set compiler flag to true.' ) do |f|
+    compiler_options[f.to_sym] = true
+  end
+
+  opts.on('-f', '--false FLAG', 'Set compiler flag to false.' ) do |f|
+    compiler_options[f.to_sym] = false
+  end
+
+  opts.on('-l', '--load-paths-cache PATH', 'Path to load path cache json') do |l|
+    compile_server_options[:load_paths_cache] = l
+  end
+
+  opts.on('-p', '--pipe-name PATH', 'Name of the named pipe.') do |p|
+    compile_server_options[:pipe_name] = p
+  end
+
+  opts.on('-m', '--memcached URL', 'URL of memcached server. Will enable use of memcached for caching compiler results.') do |m|
+    compile_server_options[:memcached] = m
+    require 'dalli'
+    cache = Dalli::Client.new(compile_server_options[:memchached])
+  end
+
+  opts.on('-e', '--redis URL', 'URL of redis server. Will enable use of redis for caching compiler results.') do |e|
+    compile_server_options[:redis] = e
+    require 'redis'
+    cache = Redis.new(url: compile_server_options[:redis])
+  end
+end.parse!
+
+compiler_options.merge!(es6_modules: true)
+
+modules_to_require.each do |mod|
+  require mod
+end
+
+def compile(request, cache, compiler_options)
+  begin
+    request_json = Oj.load(request.chop!, mode: :strict)
+
+    compile_source_map = request_json["source_map"]
+    filename = request_json["filename"]
+    source = File.read(filename)
+    if cache
+      source_digest = Digest::SHA1.hexdigest(source)
+      key = "owl_#{compiler_options_digest}_#{source_digest}_#{compile_source_map}"
+      result_json = cache.get(key)
+      return result_json if result_json
+    end
+    c = Opal::Compiler.new(source, compiler_options.merge(file: filename))
+    result = { 'javascript' => c.compile }
+    if compile_source_map
+      result['source_map'] = c.source_map.as_json
+      result['source_map']['file'] = filename
+    end
+    result['required_trees'] = c.required_trees
+    result_json = Oj.dump(result, mode: :strict)
+    cache.set(key, result_json) if cache
+    result_json
+  rescue Exception => e
+    Oj.dump({ 'error' => { 'name' => e.class.to_s, 'message' => e.message, 'backtrace' => e.backtrace.join("\n") } }, mode: :strict)
+  end
+end
+
+if ARGV[0] == 'start'
+  number_of_instances = ARGV[1].to_i
+  number_of_instances == 4 if number_of_instances == 0
+else
+  raise 'arguments must be either "stop" or "start number_of_instances"'
+  exit(1)
+end
+
+load_paths = OpalWebpackLoader::LoadPathManager.read_load_paths_cache(compile_server_options[:load_paths_cache])
+if load_paths
+  Opal.append_paths(*load_paths)
+end
+
+begin
+  OpalWebpackLoader::PipeServer.new(compile_server_options[:pipe_name], number_of_instances) do |request|
+    if request
+      if request.start_with?('command:stop')
+        STDERR.puts "Exiting with drama."
+        exit(0)
+      else
+        result = compile(request, cache, compiler_options)
+        result
+      end
+    end
+  end.run
+rescue Exception => e
+  STDERR.puts "opal-webpack-windows-compile-server couldn't start:"
+  STDERR.puts e.backtrace.join("\n")
+  STDERR.puts e.message
+  exit 1
+end

data/bin/owl-compiler

@@ -80,7 +80,10 @@ begin
     source_digest = Digest::SHA1.hexdigest(source)
     key = "owl_#{compiler_options_digest}_#{source_digest}_#{compile_source_map}"
     result_json = compile_server_options[:cache].get(key)
-    STDOUT.puts result_json if result_json
+    if result_json
+      STDOUT.puts result_json
+      exit 0
+    end
   end
   c = Opal::Compiler.new(source, compiler_options)
   result = { 'javascript' => c.compile }

data/lib/opal-webpack-loader/pipe_server.rb

@@ -0,0 +1,219 @@
+require 'ffi'
+
+module OpalWebpackLoader
+  module WindowsyThings
+    extend FFI::Library
+
+    ffi_lib :kernel32, :user32
+
+    ERROR_IO_PENDING      = 997
+    ERROR_PIPE_CONNECTED  = 535
+    ERROR_SUCCESS         = 0
+    
+    FILE_FLAG_OVERLAPPED  = 0x40000000
+    
+    INFINITE              = 0xFFFFFFFF
+    INVALID_HANDLE_VALUE  = FFI::Pointer.new(-1).address
+
+    PIPE_ACCESS_DUPLEX    = 0x00000003
+    PIPE_READMODE_BYTE    = 0x00000000
+    PIPE_READMODE_MESSAGE = 0x00000002
+    PIPE_TYPE_BYTE        = 0x00000000
+    PIPE_TYPE_MESSAGE     = 0x00000004
+    PIPE_WAIT             = 0x00000000
+
+    QS_ALLINPUT           = 0x04FF
+
+    typedef :uintptr_t, :handle
+
+    attach_function :ConnectNamedPipe, [:handle, :pointer], :ulong
+    attach_function :CreateEvent, :CreateEventA, [:pointer, :ulong, :ulong, :string], :handle
+    attach_function :CreateNamedPipe, :CreateNamedPipeA, [:string, :ulong, :ulong, :ulong, :ulong, :ulong, :ulong, :pointer], :handle
+    attach_function :DisconnectNamedPipe, [:handle], :bool
+    attach_function :FlushFileBuffers, [:handle], :bool
+    attach_function :GetLastError, [], :ulong
+    attach_function :GetOverlappedResult, [:handle, :pointer, :pointer, :bool], :bool
+    attach_function :MsgWaitForMultipleObjects, [:ulong, :pointer, :ulong, :ulong, :ulong], :ulong
+    attach_function :ReadFile, [:handle, :buffer_out, :ulong, :pointer, :pointer], :bool
+    attach_function :SetEvent, [:handle], :bool
+    attach_function :WaitForMultipleObjects, [:ulong, :pointer, :ulong, :ulong], :ulong
+    attach_function :WriteFile, [:handle, :buffer_in, :ulong, :pointer, :pointer], :bool
+  end
+
+  class PipeServer
+    include OpalWebpackLoader::WindowsyThings
+
+    CONNECTING_STATE = 0
+    READING_STATE    = 1
+    WRITING_STATE    = 2
+    INSTANCES        = 4
+    PIPE_TIMEOUT     = 5000
+    BUFFER_SIZE      = 65536
+
+    class Overlapped < FFI::Struct
+      layout(
+        :Internal, :uintptr_t,
+        :InternalHigh, :uintptr_t,
+        :Offset, :ulong,
+        :OffsetHigh, :ulong,
+        :hEvent, :uintptr_t
+      )
+    end
+
+    def initialize(pipe_name, instances = 4, &block)
+      @run_block = block
+      @full_pipe_name = "\\\\.\\pipe\\#{pipe_name}"
+      @instances = instances
+      @events = []
+      @events_pointer = FFI::MemoryPointer.new(:uintptr_t, @instances)
+      @pipes = []
+    end
+
+    def run
+      create_instances
+      while_loop
+    end
+
+    private
+
+    def create_instances
+      (0...@instances).each do |i|
+        @events[i] = CreateEvent(nil, 1, 1, nil)
+        raise "CreateEvent failed with #{GetLastError()}" unless @events[i]
+
+        overlap = Overlapped.new
+        overlap[:hEvent] = @events[i]
+
+        @pipes[i] = { overlap: overlap, instance: nil, request: FFI::Buffer.new(1, BUFFER_SIZE), bytes_read: 0, reply: FFI::Buffer.new(1, BUFFER_SIZE), bytes_to_write: 0, state: nil, pending_io: false }
+        @pipes[i][:instance] = CreateNamedPipe(@full_pipe_name, 
+                                              PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED,
+                                              PIPE_TYPE_MESSAGE | PIPE_READMODE_MESSAGE | PIPE_WAIT,
+                                              @instances,
+                                              BUFFER_SIZE,
+                                              BUFFER_SIZE,
+                                              PIPE_TIMEOUT,
+                                              nil)
+
+        raise "CreateNamedPipe failed with #{GetLastError()}" if @pipes[i][:instance] == INVALID_HANDLE_VALUE
+        @pipes[i][:pending_io] = connect_to_new_client(i)
+        @pipes[i][:state] = @pipes[i][:pending_io] ? CONNECTING_STATE : READING_STATE
+      end
+      @events_pointer.write_array_of_ulong_long(@events)
+      nil
+    end
+
+    def while_loop
+      while true
+        i = MsgWaitForMultipleObjects(@instances, @events_pointer, 0, INFINITE, QS_ALLINPUT)
+        # Having this STDOUT.putc is essential, otherwise there is a tendency to block within MsgWaitForMultipleObjects ...
+        STDOUT.putc "."
+        # ... because the ruby interpreter is waiting for objects too on Windows. Thats why we wait for QS_ALLINPUT and
+        # with STDOUT.putc give back control to the ruby interpreter that it can handle its things.
+        if i < 0 || i > (@instances - 1)
+          STDERR.puts "Pipe index out of range. Maybe a error occured."
+          next
+        end
+
+        if @pipes[i][:pending_io]
+          bytes_transferred = FFI::MemoryPointer.new(:ulong)
+          success = GetOverlappedResult(@pipes[i][:instance], @pipes[i][:overlap], bytes_transferred, false)
+
+          case @pipes[i][:state]
+          when CONNECTING_STATE
+            raise "Error #{GetLastError()}" unless success
+            @pipes[i][:state] = READING_STATE
+          when READING_STATE
+            if !success || bytes_transferred.read_ulong == 0
+              disconnect_and_reconnect(i)
+              next
+            else
+              @pipes[i][:bytes_read] = bytes_transferred.read_ulong
+              @pipes[i][:state] = WRITING_STATE
+            end
+          when WRITING_STATE
+            if !success || bytes_transferred.read_ulong != @pipes[i][:bytes_to_write]
+              disconnect_and_reconnect(i)
+              next
+            else
+              @pipes[i][:state] = READING_STATE
+            end
+          else
+            raise "Invalid pipe state."
+          end
+        end
+
+        case @pipes[i][:state]
+        when READING_STATE
+          bytes_read = FFI::MemoryPointer.new(:ulong)
+          success = ReadFile(@pipes[i][:instance], @pipes[i][:request], BUFFER_SIZE, bytes_read, @pipes[i][:overlap].to_ptr)
+          if success && bytes_read.read_ulong != 0
+            @pipes[i][:pending_io] = false
+            @pipes[i][:state] = WRITING_STATE
+            next
+          end
+
+          err = GetLastError()
+          if !success && err == ERROR_IO_PENDING
+            @pipes[i][:pending_io] = true
+            next
+          end
+
+          disconnect_and_reconnect(i)
+        when WRITING_STATE
+          @pipes[i][:reply] = @run_block.call(@pipes[i][:request].get_string(0))
+          @pipes[i][:bytes_to_write] = @pipes[i][:reply].bytesize
+          bytes_written = FFI::MemoryPointer.new(:ulong)
+          success = WriteFile(@pipes[i][:instance], @pipes[i][:reply], @pipes[i][:bytes_to_write], bytes_written, @pipes[i][:overlap].to_ptr)
+
+          if success && bytes_written.read_ulong == @pipes[i][:bytes_to_write]
+            @pipes[i][:pending_io] = false
+            @pipes[i][:state] = READING_STATE
+            next
+          end
+
+          err = GetLastError()
+
+          if !success && err == ERROR_IO_PENDING
+            @pipes[i][:pending_io] = true
+            next
+          end
+
+          disconnect_and_reconnect(i)
+        else
+          raise "Invalid pipe state."
+        end
+      end
+    end
+
+    def disconnect_and_reconnect(i)
+      FlushFileBuffers(@pipes[i][:instance])
+      STDERR.puts("DisconnectNamedPipe failed with #{GetLastError()}") if !DisconnectNamedPipe(@pipes[i][:instance])
+  
+      @pipes[i][:pending_io] = connect_to_new_client(i)
+      
+      @pipes[i][:state] = @pipes[i][:pending_io] ? CONNECTING_STATE : READING_STATE
+    end
+
+    def connect_to_new_client(i)
+      pending_io = false
+      @pipes[i][:request].clear
+      @pipes[i][:reply].clear
+      connected = ConnectNamedPipe(@pipes[i][:instance], @pipes[i][:overlap].to_ptr)
+      last_error = GetLastError()
+      raise "ConnectNamedPipe failed with #{last_error} - #{connected}" if connected != 0
+      
+      case last_error
+      when ERROR_IO_PENDING
+        pending_io = true
+      when ERROR_PIPE_CONNECTED
+        SetEvent(@pipes[i][:overlap][:hEvent])
+      when ERROR_SUCCESS
+        pending_io = true
+      else
+        raise "ConnectNamedPipe failed with error #{last_error}"
+      end
+
+      pending_io
+    end
+  end
+end

data/lib/opal-webpack-loader/version.rb

@@ -1,3 +1,3 @@
 module OpalWebpackLoader
-  VERSION="0.10.3"
+  VERSION="0.11.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opal-webpack-loader
 version: !ruby/object:Gem::Version
-  version: 0.10.3
+  version: 0.11.0
 platform: ruby
 authors:
 - Jan Biedermann
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-17 00:00:00.000000000 Z
+date: 2021-05-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opal
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.7.0
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.0
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
@@ -201,12 +215,14 @@ executables:
 - owl-install
 - owl-gen-loadpath-cache
 - owl-compiler
+- opal-webpack-windows-compile-server
 extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
 - README.md
 - bin/opal-webpack-compile-server
+- bin/opal-webpack-windows-compile-server
 - bin/owl-compiler
 - bin/owl-gen-loadpath-cache
 - bin/owl-install
@@ -216,6 +232,7 @@ files:
 - lib/opal-webpack-loader/installer_cli.rb
 - lib/opal-webpack-loader/load_path_manager.rb
 - lib/opal-webpack-loader/manifest.rb
+- lib/opal-webpack-loader/pipe_server.rb
 - lib/opal-webpack-loader/rails_view_helper.rb
 - lib/opal-webpack-loader/templates/Procfile.erb
 - lib/opal-webpack-loader/templates/app_loader.rb.erb