opal-webpack-loader 0.10.0 → 0.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47e73212d646dde8e4fbad0d525e3146564bde7ca2e5518d644f972a5a908860
-  data.tar.gz: cb1243d2b4bf3158ec87c9815d07333ddeae470c1f838e6f51c5ee4df082f5d9
+  metadata.gz: f1a1b24301ed05a439abf847ae77a460919a227d250432d0c769d3080d100dec
+  data.tar.gz: 48f48641c277ecbf2fd50a7c274bdcf94b960dc28bf5103da70555815b70112f
 SHA512:
-  metadata.gz: 22b303793fb6845b9277530f06d0f194975d2ca7475e7e6434efaebf9a990af904e2f7b1887d42a3fe7967ecfc615271b128a374e6d2b9043db7d2ed7e2dedee
-  data.tar.gz: 7232d4066bbd4ef90fdd852dfef6f08ad7fdbd8e139051a7a62e16b4b9345f06ecb38435ef19220e0657f4eb815bf232e530fe3b67ebcc72acf19d9bda3c441b
+  metadata.gz: 7ef00f2c1e57f71eadaae2a3b2edc6c47745c980a1f85921b0ca9e6e0b092e960c9c3971826ffec3ca22b42e56d45a9b36bd6d692b794b97f23c2343f043e25d
+  data.tar.gz: a5ebdd0518490d0eea2e78de43f1527f333b32ab778603fd24cc8052c3870aa4e5408a631c6a0a676185d7355f488c4cc605f5b8a793418a2072ed3c09174b85

data/README.md

@@ -23,6 +23,7 @@ At the [Isomorfeus Framework Project](http://isomorfeus.com)
 opal-webpack-loader-0.7.1 compiles all of opal, a bunch of gems and over 19000SLC on a
 Intel® Core™ i7-7700HQ CPU @ 2.80GHz × 8, with 8 workers in around 1850ms
 - support for memcached or redis as compiler cache
+- fast builds on windows too, parallel when building multiple assets with parallel-webpack
 - opal modules are packaged as es6 modules
 - support for rails with webpacker
 - other webpack features become available, like:
@@ -35,9 +36,10 @@ Intel® Core™ i7-7700HQ CPU @ 2.80GHz × 8, with 8 workers in around 1850ms
   - everything else webpack can do, like loading stylesheets, etc.
 
 ### Requirements
+- npm or yarn
 - opal-webpack-loader consists of 2 parts, the npm package and the gem, both are required and must be the same version.
-- webpack ^4.41
-- webpack-dev-server ^3.9.0
+- webpack ^4.46
+- webpack-dev-server ^3.11.0
 - one of the ES6 modules branches of opal
   - [PR#1973](https://github.com/opal/opal/pull/1973), (experimental) implementing ES6 modules and changes for 'strict' mode,
     based on Opal master 1.0.0 using javascript string objects "mutable strings" by default for all strings

data/bin/opal-webpack-windows-compile-server

@@ -0,0 +1,125 @@
+#!/usr/bin/env ruby
+require 'digest'
+require 'oj'
+require 'c_lexer'
+require 'optparse'
+require 'opal/paths'
+require 'opal/source_map'
+require 'opal/compiler'
+require 'opal-webpack-loader/load_path_manager'
+require 'opal-webpack-loader/pipe_server'
+
+modules_to_require = []
+compiler_options = {}
+compile_server_options = {}
+cache = false
+
+OptionParser.new do |opts|
+  opts.on('-r', '--require MODULE', 'Require the module before starting the compile server.') do |m|
+    modules_to_require << m
+  end
+
+  opts.on('-I', '--include DIR', 'Append a load path (may be used more than once)') do |i|
+    $:.unshift(File.expand_path(i))
+  end
+
+  opts.on('-d', '--dynamic-require-severity SEVERITY', 'Compiler option, one of: error, warning, ignore.') do |s|
+    if %w[error warning ignore].include?(s)
+      compiler_options[:dynamic_require_severity] = s.to_sym
+    end
+  end
+
+  opts.on('-t', '--true FLAG', 'Set compiler flag to true.' ) do |f|
+    compiler_options[f.to_sym] = true
+  end
+
+  opts.on('-f', '--false FLAG', 'Set compiler flag to false.' ) do |f|
+    compiler_options[f.to_sym] = false
+  end
+
+  opts.on('-l', '--load-paths-cache PATH', 'Path to load path cache json') do |l|
+    compile_server_options[:load_paths_cache] = l
+  end
+
+  opts.on('-p', '--pipe-name PATH', 'Name of the named pipe.') do |p|
+    compile_server_options[:pipe_name] = p
+  end
+
+  opts.on('-m', '--memcached URL', 'URL of memcached server. Will enable use of memcached for caching compiler results.') do |m|
+    compile_server_options[:memcached] = m
+    require 'dalli'
+    cache = Dalli::Client.new(compile_server_options[:memchached])
+  end
+
+  opts.on('-e', '--redis URL', 'URL of redis server. Will enable use of redis for caching compiler results.') do |e|
+    compile_server_options[:redis] = e
+    require 'redis'
+    cache = Redis.new(url: compile_server_options[:redis])
+  end
+end.parse!
+
+compiler_options.merge!(es6_modules: true)
+
+modules_to_require.each do |mod|
+  require mod
+end
+
+def compile(request, cache, compiler_options)
+  begin
+    request_json = Oj.load(request.chop!, mode: :strict)
+
+    compile_source_map = request_json["source_map"]
+    filename = request_json["filename"]
+    source = File.read(filename)
+    if cache
+      source_digest = Digest::SHA1.hexdigest(source)
+      key = "owl_#{compiler_options_digest}_#{source_digest}_#{compile_source_map}"
+      result_json = cache.get(key)
+      return result_json if result_json
+    end
+    c = Opal::Compiler.new(source, compiler_options.merge(file: filename))
+    result = { 'javascript' => c.compile }
+    if compile_source_map
+      result['source_map'] = c.source_map.as_json
+      result['source_map']['file'] = filename
+    end
+    result['required_trees'] = c.required_trees
+    result_json = Oj.dump(result, mode: :strict)
+    cache.set(key, result_json) if cache
+    result_json
+  rescue Exception => e
+    Oj.dump({ 'error' => { 'name' => e.class.to_s, 'message' => e.message, 'backtrace' => e.backtrace.join("\n") } }, mode: :strict)
+  end
+end
+
+if ARGV[0] == 'start'
+  number_of_instances = ARGV[1].to_i
+  number_of_instances == 4 if number_of_instances == 0
+else
+  raise 'arguments must be either "stop" or "start number_of_instances"'
+  exit(1)
+end
+
+load_paths = OpalWebpackLoader::LoadPathManager.read_load_paths_cache(compile_server_options[:load_paths_cache])
+if load_paths
+  Opal.append_paths(*load_paths)
+end
+
+begin
+  OpalWebpackLoader::PipeServer.new(compile_server_options[:pipe_name], number_of_instances) do |request|
+    if request
+      if request.start_with?('command:stop')
+        STDERR.puts "Exiting with drama."
+        exit(0)
+      else
+        result = compile(request, cache, compiler_options)
+        result
+      end
+    end
+  end.run
+rescue Exception => e
+  STDERR.puts "opal-webpack-windows-compile-server couldn't start:"
+  STDERR.puts e.backtrace.join("\n")
+  STDERR.puts e.message
+  exit 1
+end

data/bin/owl-compiler

@@ -80,7 +80,10 @@ begin
     source_digest = Digest::SHA1.hexdigest(source)
     key = "owl_#{compiler_options_digest}_#{source_digest}_#{compile_source_map}"
     result_json = compile_server_options[:cache].get(key)
-    STDOUT.puts result_json if result_json
+    if result_json
+      STDOUT.puts result_json
+      exit 0
+    end
   end
   c = Opal::Compiler.new(source, compiler_options)
   result = { 'javascript' => c.compile }
@@ -95,6 +98,3 @@ begin
 rescue Exception => e
   STDOUT.puts Oj.dump({ 'error' => { 'name' => e.class.to_s, 'message' => e.message, 'backtrace' => e.backtrace.join("\n") } }, mode: :strict)
 end
-
-
-

data/lib/opal-webpack-loader.rb

@@ -38,27 +38,31 @@ OpalWebpackLoader.manifest_path = File.join(Dir.getwd, 'public', 'assets', 'mani
 OpalWebpackLoader.client_asset_path = 'http://localhost:3035/assets/'
 OpalWebpackLoader.use_manifest = false
 
-# TODO require yarn instead of npm
-npm = if Gem.win_platform?
-        `where npm`.chop.lines.last
-      else
-        `which npm`.chop
-      end
+owl_npm_version = nil
+begin
+  yarn = if Gem.win_platform?
+          `where yarn`.chop.lines.last.include?('yarn')
+        else
+          `which yarn`.chop.include?('yarn')
+        end
+  owl_npm_version = `yarn run -s opal-webpack-loader-npm-version`.chop if yarn
 
-if !npm.nil? && !npm.empty?
-  bin_dir = `npm bin`.chop
-  begin
-    owl_npm_version = `#{File.join(bin_dir, 'opal-webpack-loader-npm-version')}`.chop
-  rescue
-    owl_npm_version = nil
+  unless owl_npm_version
+    npm = if Gem.win_platform?
+            `where npm`.chop.lines.last.include?('npm')
+          else
+            `which npm`.chop.include?('npm')
+          end
+    owl_npm_version = `npm exec opal-webpack-loader-npm-version`.chop if npm
   end
+rescue
+  owl_npm_version = nil
+end
 
-  if owl_npm_version != OpalWebpackLoader::VERSION
-    STDERR.puts "opal-webpack-loader: Incorrect version of npm package found or npm package not installed.\n" +
-      "Please install the npm package for opal-webpack-loader:\n" +
-      "\twith npm:\tnpm install opal-webpack-loader@#{OpalWebpackLoader::VERSION}\n" +
-      "\tor with yarn:\tyarn add opal-webpack-loader@#{OpalWebpackLoader::VERSION}\n"
-  end
-else
-  STDERR.puts "opal-webpack-loader: Unable to check npm package version. Please check your npm installation."
+if owl_npm_version != OpalWebpackLoader::VERSION
+  STDERR.puts "opal-webpack-loader: Incorrect version of npm package found or npm package not installed.\n" +
+    "Please install the npm package for opal-webpack-loader:\n" +
+    "\twith npm:\tnpm install opal-webpack-loader@#{OpalWebpackLoader::VERSION}\n" +
+    "\tor with yarn:\tyarn add opal-webpack-loader@#{OpalWebpackLoader::VERSION}\n"
 end
+

data/lib/opal-webpack-loader/pipe_server.rb

@@ -0,0 +1,219 @@
+require 'ffi'
+
+module OpalWebpackLoader
+  module WindowsyThings
+    extend FFI::Library
+
+    ffi_lib :kernel32, :user32
+
+    ERROR_IO_PENDING      = 997
+    ERROR_PIPE_CONNECTED  = 535
+    ERROR_SUCCESS         = 0
+    
+    FILE_FLAG_OVERLAPPED  = 0x40000000
+    
+    INFINITE              = 0xFFFFFFFF
+    INVALID_HANDLE_VALUE  = FFI::Pointer.new(-1).address
+
+    PIPE_ACCESS_DUPLEX    = 0x00000003
+    PIPE_READMODE_BYTE    = 0x00000000
+    PIPE_READMODE_MESSAGE = 0x00000002
+    PIPE_TYPE_BYTE        = 0x00000000
+    PIPE_TYPE_MESSAGE     = 0x00000004
+    PIPE_WAIT             = 0x00000000
+
+    QS_ALLINPUT           = 0x04FF
+
+    typedef :uintptr_t, :handle
+
+    attach_function :ConnectNamedPipe, [:handle, :pointer], :ulong
+    attach_function :CreateEvent, :CreateEventA, [:pointer, :ulong, :ulong, :string], :handle
+    attach_function :CreateNamedPipe, :CreateNamedPipeA, [:string, :ulong, :ulong, :ulong, :ulong, :ulong, :ulong, :pointer], :handle
+    attach_function :DisconnectNamedPipe, [:handle], :bool
+    attach_function :FlushFileBuffers, [:handle], :bool
+    attach_function :GetLastError, [], :ulong
+    attach_function :GetOverlappedResult, [:handle, :pointer, :pointer, :bool], :bool
+    attach_function :MsgWaitForMultipleObjects, [:ulong, :pointer, :ulong, :ulong, :ulong], :ulong
+    attach_function :ReadFile, [:handle, :buffer_out, :ulong, :pointer, :pointer], :bool
+    attach_function :SetEvent, [:handle], :bool
+    attach_function :WaitForMultipleObjects, [:ulong, :pointer, :ulong, :ulong], :ulong
+    attach_function :WriteFile, [:handle, :buffer_in, :ulong, :pointer, :pointer], :bool
+  end
+
+  class PipeServer
+    include OpalWebpackLoader::WindowsyThings
+
+    CONNECTING_STATE = 0
+    READING_STATE    = 1
+    WRITING_STATE    = 2
+    INSTANCES        = 4
+    PIPE_TIMEOUT     = 5000
+    BUFFER_SIZE      = 65536
+
+    class Overlapped < FFI::Struct
+      layout(
+        :Internal, :uintptr_t,
+        :InternalHigh, :uintptr_t,
+        :Offset, :ulong,
+        :OffsetHigh, :ulong,
+        :hEvent, :uintptr_t
+      )
+    end
+
+    def initialize(pipe_name, instances = 4, &block)
+      @run_block = block
+      @full_pipe_name = "\\\\.\\pipe\\#{pipe_name}"
+      @instances = instances
+      @events = []
+      @events_pointer = FFI::MemoryPointer.new(:uintptr_t, @instances)
+      @pipes = []
+    end
+
+    def run
+      create_instances
+      while_loop
+    end
+
+    private
+
+    def create_instances
+      (0...@instances).each do |i|
+        @events[i] = CreateEvent(nil, 1, 1, nil)
+        raise "CreateEvent failed with #{GetLastError()}" unless @events[i]
+
+        overlap = Overlapped.new
+        overlap[:hEvent] = @events[i]
+
+        @pipes[i] = { overlap: overlap, instance: nil, request: FFI::Buffer.new(1, BUFFER_SIZE), bytes_read: 0, reply: FFI::Buffer.new(1, BUFFER_SIZE), bytes_to_write: 0, state: nil, pending_io: false }
+        @pipes[i][:instance] = CreateNamedPipe(@full_pipe_name, 
+                                              PIPE_ACCESS_DUPLEX | FILE_FLAG_OVERLAPPED,
+                                              PIPE_TYPE_MESSAGE | PIPE_READMODE_MESSAGE | PIPE_WAIT,
+                                              @instances,
+                                              BUFFER_SIZE,
+                                              BUFFER_SIZE,
+                                              PIPE_TIMEOUT,
+                                              nil)
+
+        raise "CreateNamedPipe failed with #{GetLastError()}" if @pipes[i][:instance] == INVALID_HANDLE_VALUE
+        @pipes[i][:pending_io] = connect_to_new_client(i)
+        @pipes[i][:state] = @pipes[i][:pending_io] ? CONNECTING_STATE : READING_STATE
+      end
+      @events_pointer.write_array_of_ulong_long(@events)
+      nil
+    end
+
+    def while_loop
+      while true
+        i = MsgWaitForMultipleObjects(@instances, @events_pointer, 0, INFINITE, QS_ALLINPUT)
+        # Having this STDOUT.putc is essential, otherwise there is a tendency to block within MsgWaitForMultipleObjects ...
+        STDOUT.putc "."
+        # ... because the ruby interpreter is waiting for objects too on Windows. Thats why we wait for QS_ALLINPUT and
+        # with STDOUT.putc give back control to the ruby interpreter that it can handle its things.
+        if i < 0 || i > (@instances - 1)
+          STDERR.puts "Pipe index out of range. Maybe a error occured."
+          next
+        end
+
+        if @pipes[i][:pending_io]
+          bytes_transferred = FFI::MemoryPointer.new(:ulong)
+          success = GetOverlappedResult(@pipes[i][:instance], @pipes[i][:overlap], bytes_transferred, false)
+
+          case @pipes[i][:state]
+          when CONNECTING_STATE
+            raise "Error #{GetLastError()}" unless success
+            @pipes[i][:state] = READING_STATE
+          when READING_STATE
+            if !success || bytes_transferred.read_ulong == 0
+              disconnect_and_reconnect(i)
+              next
+            else
+              @pipes[i][:bytes_read] = bytes_transferred.read_ulong
+              @pipes[i][:state] = WRITING_STATE
+            end
+          when WRITING_STATE
+            if !success || bytes_transferred.read_ulong != @pipes[i][:bytes_to_write]
+              disconnect_and_reconnect(i)
+              next
+            else
+              @pipes[i][:state] = READING_STATE
+            end
+          else
+            raise "Invalid pipe state."
+          end
+        end
+
+        case @pipes[i][:state]
+        when READING_STATE
+          bytes_read = FFI::MemoryPointer.new(:ulong)
+          success = ReadFile(@pipes[i][:instance], @pipes[i][:request], BUFFER_SIZE, bytes_read, @pipes[i][:overlap].to_ptr)
+          if success && bytes_read.read_ulong != 0
+            @pipes[i][:pending_io] = false
+            @pipes[i][:state] = WRITING_STATE
+            next
+          end
+
+          err = GetLastError()
+          if !success && err == ERROR_IO_PENDING
+            @pipes[i][:pending_io] = true
+            next
+          end
+
+          disconnect_and_reconnect(i)
+        when WRITING_STATE
+          @pipes[i][:reply] = @run_block.call(@pipes[i][:request].get_string(0))
+          @pipes[i][:bytes_to_write] = @pipes[i][:reply].bytesize
+          bytes_written = FFI::MemoryPointer.new(:ulong)
+          success = WriteFile(@pipes[i][:instance], @pipes[i][:reply], @pipes[i][:bytes_to_write], bytes_written, @pipes[i][:overlap].to_ptr)
+
+          if success && bytes_written.read_ulong == @pipes[i][:bytes_to_write]
+            @pipes[i][:pending_io] = false
+            @pipes[i][:state] = READING_STATE
+            next
+          end
+
+          err = GetLastError()
+
+          if !success && err == ERROR_IO_PENDING
+            @pipes[i][:pending_io] = true
+            next
+          end
+
+          disconnect_and_reconnect(i)
+        else
+          raise "Invalid pipe state."
+        end
+      end
+    end
+
+    def disconnect_and_reconnect(i)
+      FlushFileBuffers(@pipes[i][:instance])
+      STDERR.puts("DisconnectNamedPipe failed with #{GetLastError()}") if !DisconnectNamedPipe(@pipes[i][:instance])
+  
+      @pipes[i][:pending_io] = connect_to_new_client(i)
+      
+      @pipes[i][:state] = @pipes[i][:pending_io] ? CONNECTING_STATE : READING_STATE
+    end
+
+    def connect_to_new_client(i)
+      pending_io = false
+      @pipes[i][:request].clear
+      @pipes[i][:reply].clear
+      connected = ConnectNamedPipe(@pipes[i][:instance], @pipes[i][:overlap].to_ptr)
+      last_error = GetLastError()
+      raise "ConnectNamedPipe failed with #{last_error} - #{connected}" if connected != 0
+      
+      case last_error
+      when ERROR_IO_PENDING
+        pending_io = true
+      when ERROR_PIPE_CONNECTED
+        SetEvent(@pipes[i][:overlap][:hEvent])
+      when ERROR_SUCCESS
+        pending_io = true
+      else
+        raise "ConnectNamedPipe failed with error #{last_error}"
+      end
+
+      pending_io
+    end
+  end
+end

data/lib/opal-webpack-loader/version.rb

@@ -1,3 +1,3 @@
-module OpalWebpackLoader
-  VERSION="0.10.0"
-end
+module OpalWebpackLoader
+  VERSION="0.11.1"
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: opal-webpack-loader
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.1
 platform: ruby
 authors:
 - Jan Biedermann
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-15 00:00:00.000000000 Z
+date: 2021-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: opal
@@ -52,6 +52,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.7.0
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.0
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
@@ -201,12 +215,14 @@ executables:
 - owl-install
 - owl-gen-loadpath-cache
 - owl-compiler
+- opal-webpack-windows-compile-server
 extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
 - README.md
 - bin/opal-webpack-compile-server
+- bin/opal-webpack-windows-compile-server
 - bin/owl-compiler
 - bin/owl-gen-loadpath-cache
 - bin/owl-install
@@ -216,6 +232,7 @@ files:
 - lib/opal-webpack-loader/installer_cli.rb
 - lib/opal-webpack-loader/load_path_manager.rb
 - lib/opal-webpack-loader/manifest.rb
+- lib/opal-webpack-loader/pipe_server.rb
 - lib/opal-webpack-loader/rails_view_helper.rb
 - lib/opal-webpack-loader/templates/Procfile.erb
 - lib/opal-webpack-loader/templates/app_loader.rb.erb
@@ -240,7 +257,7 @@ licenses:
 - MIT
 metadata:
   github_repo: ssh://github.com/isomorfeus/gems
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -256,7 +273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.2.15
-signing_key:
+signing_key: 
 specification_version: 4
 summary: Bundle assets with webpack, resolve and compile opal ruby files and import
   them in the bundle.