rake-compiler-dock 0.7.2 → 1.0.0

data/{lib/rake_compiler_dock → build}/gem_helper.rb

@@ -1,18 +1,3 @@
-# This file is part of Libusb for Ruby.
-#
-# Libusb for Ruby is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Libusb for Ruby is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Libusb for Ruby.  If not, see <http://www.gnu.org/licenses/>.
-
 require "bundler/gem_helper"
 
 module RakeCompilerDock

data/build/parallel_docker_build.rb

@@ -0,0 +1,102 @@
+require "fileutils"
+require "rake"
+
+module RakeCompilerDock
+  # Run docker builds in parallel, but ensure that common docker layers are reused
+  class ParallelDockerBuild
+    include Rake::DSL
+
+    def initialize(dockerfiles, workdir: "tmp/docker", inputdir: ".", task_prefix: "common-")
+      FileUtils.mkdir_p(workdir)
+
+      files = parse_dockerfiles(dockerfiles, inputdir)
+      # pp files
+
+      vcs = find_commons(files)
+      # pp vcs
+
+      define_common_tasks(vcs, workdir, task_prefix)
+    end
+
+    # Read given dockerfiles from inputdir and split into a list of commands.
+    #
+    # Returns:
+    #   {"File0"=>["      FROM a\n", "      RUN a\n", "      RUN d\n"],
+    #    "File1"=>["      FROM a\n",
+    #    ...
+    def parse_dockerfiles(dockerfiles, inputdir)
+      files = dockerfiles.map do |fn|
+        [fn, File.read(File.join(inputdir, fn))]
+      end.map do |fn, f|
+        # Split file contant in lines unless line ends with backslash
+        fc = f.each_line.with_object([]) do |line, lines|
+          if lines.last=~/\\\n\z/
+            lines.last << line
+          else
+            lines << line
+          end
+        end
+        [fn, fc]
+      end.to_h
+    end
+
+    # Build a tree of common parts of given files.
+    #
+    # Returns:
+    #   {["File0", "File1", "File2", "File3"]=>
+    #     [["  FROM a\n"],
+    #      {["File0", "File1"]=>
+    #        [["  RUN a\n", "  RUN d\n"], {["File1"]=>[["  RUN f\n"], {}]}],
+    #       ["File2", "File3"]=>[["  RUN b\n", "  RUN c\n", "  RUN d\n"], {}]}]}
+    def find_commons(files, vmask=nil, li=0)
+      vmask ||= files.keys
+      vcs = Hash.new { [] }
+      files.each do |fn, lines|
+        next unless vmask.include?(fn)
+        vcs[lines[li]] += [fn]
+      end
+
+      vcs.map do |line, vc|
+        next unless line
+        nvcs = find_commons(files, vc, li+1)
+        if nvcs.first && nvcs.first[0] == vc
+          # Append lines that are equal between file(s)
+          nl = [[line] + nvcs.first[1][0], nvcs.first[1][1]]
+        else
+          nl = [[line], nvcs]
+        end
+        [vc, nl]
+      end.compact.to_h
+    end
+
+    # Write intermediate dockerfiles to workdir and define rake tasks
+    #
+    # The rake tasks are named after the dockerfiles given to #new .
+    # This also adds dependant intermediate tasks as prerequisites.
+    def define_common_tasks(vcs, workdir, task_prefix, plines=[])
+      vcs.map do |files, (lines, nvcs)|
+        fn = "#{task_prefix}#{files.join}"
+        File.write(File.join(workdir, fn), (plines + lines).join)
+        task fn do
+          docker_build(fn, workdir)
+        end
+
+        nfn = define_common_tasks(nvcs, workdir, task_prefix, plines + lines)
+        nfn.each do |file|
+          task file => fn
+        end
+        files.each do |file|
+          task file => fn
+        end
+        fn
+      end
+    end
+
+    # Run an intermediate dockerfile without tag
+    #
+    # The layers will be reused in subsequent builds, even if they run in parallel.
+    def docker_build(filename, workdir)
+      sh "docker", "build", "-f", File.join(workdir, filename), "."
+    end
+  end
+end

data/build/patches/{ruby-2.5.3 → ruby-2.5.7}/no_sendfile.patch

@@ -17,7 +17,7 @@ index 82c5940538..a8d3661ec8 100644
 @@ -10721,7 +10721,6 @@ nogvl_copy_stream_wait_write(struct copy_stream_struct *stp)
  }
 
- #if defined __linux__ && defined __NR_copy_file_range
+ #if defined HAVE_COPY_FILE_RANGE || (defined __linux__ && defined __NR_copy_file_range)
 -#  define USE_COPY_FILE_RANGE
  #endif
 

data/build/patches/ruby-2.7.0/no_sendfile.patch

@@ -0,0 +1,24 @@
+diff --git a/configure b/configure
+index ebe3d8c..a336b73 100755
+--- a/configure
++++ b/configure
+@@ -18943,7 +18943,6 @@ do :
+   ac_fn_c_check_func "$LINENO" "sendfile" "ac_cv_func_sendfile"
+ if test "x$ac_cv_func_sendfile" = xyes; then :
+   cat >>confdefs.h <<_ACEOF
+-#define HAVE_SENDFILE 1
+ _ACEOF
+
+ fi
+diff --git a/io.c b/io.c
+index 82c5940538..a8d3661ec8 100644
+--- a/io.c
++++ b/io.c
+@@ -10721,7 +10721,6 @@ nogvl_copy_stream_wait_write(struct copy_stream_struct *stp)
+ }
+
+ #if defined HAVE_COPY_FILE_RANGE || (defined __linux__ && defined __NR_copy_file_range)
+-#  define USE_COPY_FILE_RANGE
+ #endif
+
+ #ifdef USE_COPY_FILE_RANGE

data/build/patches2/hoe-3.20.0/0001-Load-encrypted-private-key-using-ENV-GEM_PRIVATE_KEY.patch

@@ -0,0 +1,32 @@
+From ecc5669223457ceaba6bd94c2947ae99ddfa8f10 Mon Sep 17 00:00:00 2001
+From: Lars Kanis <lars@greiz-reinsdorf.de>
+Date: Fri, 27 Dec 2019 15:57:07 +0100
+Subject: [PATCH] Load encrypted private key using
+ ENV['GEM_PRIVATE_KEY_PASSPHRASE'] as passphrase.
+
+This way the password can be avoided to be entered several times when building multiple gems.
+It also allows parallel builds which are incompatible to interactive password prompts.
+
+The same feature has been implemented in rubygems:
+  https://github.com/rubygems/rubygems/commit/c1a114396fcec124d3feabf74708b4bdec02eac3
+---
+ lib/hoe/signing.rb | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/lib/hoe/signing.rb b/lib/hoe/signing.rb
+index 4833c55..11c8c2f 100644
+--- a/lib/hoe/signing.rb
++++ b/lib/hoe/signing.rb
+@@ -70,7 +70,8 @@ module Hoe::Signing
+     end
+
+     if signing_key and cert_chain then
+-      spec.signing_key = OpenSSL::PKey::RSA.new File.read signing_key
++      passphrase = ENV['GEM_PRIVATE_KEY_PASSPHRASE']
++      spec.signing_key = OpenSSL::PKey::RSA.new(File.read(signing_key), passphrase)
+       spec.cert_chain = cert_chain
+     end
+   end
+--
+2.20.1
+

data/build/patches2/rake-compiler-1.1.0/0001-Allow-parallel-builds-of-cross-rubies.patch

@@ -0,0 +1,119 @@
+From e0be66410b32a41591d27c34e2163ed587ae427b Mon Sep 17 00:00:00 2001
+From: Lars Kanis <lars@greiz-reinsdorf.de>
+Date: Sun, 22 Dec 2019 21:37:07 +0100
+Subject: [PATCH] Allow parallel builds of cross rubies
+
+Download and extraction of sources can not run in parallel, since the sources are used for multiple targets.
+Therefore this part can now be separately executed by
+  rake-compiler prepare-sources RUBY_CC_VERSION=...
+
+Afterwards multiple cross rubies can be built in parallel by
+  rake-compiler cross-ruby RUBY_CC_VERSION=... HOST=...
+
+Access to config.yml must be synchronized, when running in parallel, which is done per File#flock .
+---
+ lib/rake/extensiontask.rb |  5 +++-
+ tasks/bin/cross-ruby.rake | 56 +++++++++++++++++++++------------------
+ 2 files changed, 34 insertions(+), 27 deletions(-)
+
+diff --git a/lib/rake/extensiontask.rb b/lib/rake/extensiontask.rb
+index d85b1a3..89f2198 100644
+--- a/lib/rake/extensiontask.rb
++++ b/lib/rake/extensiontask.rb
+@@ -376,7 +376,10 @@ Java extension should be preferred.
+         return
+       end
+ 
+-      config_file = YAML.load_file(config_path)
++      config_file = File.open(config_path) do |fd|
++        fd.flock(File::LOCK_SH)
++        YAML.load(fd.read)
++      end
+ 
+       # tmp_path
+       tmp_path = "#{@tmp_dir}/#{for_platform}/#{@name}/#{ruby_ver}"
+diff --git a/tasks/bin/cross-ruby.rake b/tasks/bin/cross-ruby.rake
+index 37dd220..3fcfe1d 100644
+--- a/tasks/bin/cross-ruby.rake
++++ b/tasks/bin/cross-ruby.rake
+@@ -126,6 +126,8 @@ file makefile_in => [makefile_in_bak] do |t|
+   }
+ end
+ 
++task "prepare-sources" => makefile_in
++
+ task :mingw32 do
+   unless MINGW_HOST then
+     warn "You need to install mingw32 cross compile functionality to be able to continue."
+@@ -174,40 +176,42 @@ task :install => ["#{USER_HOME}/ruby/#{MINGW_HOST}/#{RUBY_CC_VERSION}/bin/ruby.e
+ desc "Update rake-compiler list of installed Ruby versions"
+ task 'update-config' do
+   config_file = "#{USER_HOME}/config.yml"
+-  if File.exist?(config_file) then
+-    puts "Updating #{config_file}"
+-    config = YAML.load_file(config_file)
+-  else
+-    puts "Generating #{config_file}"
+-    config = {}
+-  end
++  File.open(config_file, File::RDWR|File::CREAT) do |config_fd|
++    config_fd.flock(File::LOCK_EX)
++    config = YAML.load(config_fd.read)
++    if config
++      puts "Updating #{config_file}"
++    else
++      puts "Generating #{config_file}"
++      config = {}
++    end
+ 
+-  files = Dir.glob("#{USER_HOME}/ruby/*/*/**/rbconfig.rb").sort
++    files = Dir.glob("#{USER_HOME}/ruby/*/*/**/rbconfig.rb").sort
+ 
+-  files.each do |rbconfig|
+-    version, platform = rbconfig.match(/.*-(\d.\d.\d).*\/([-\w]+)\/rbconfig/)[1,2]
+-    platforms = [platform]
++    files.each do |rbconfig|
++      version, platform = rbconfig.match(/.*-(\d.\d.\d).*\/([-\w]+)\/rbconfig/)[1,2]
++      platforms = [platform]
+ 
+-    # fake alternate (binary compatible) i386-mswin32-60 platform
+-    platform == "i386-mingw32" and
+-      platforms.push "i386-mswin32-60"
++      # fake alternate (binary compatible) i386-mswin32-60 platform
++      platform == "i386-mingw32" and
++        platforms.push "i386-mswin32-60"
+ 
+-    platforms.each do |plat|
+-      config["rbconfig-#{plat}-#{version}"] = rbconfig
++      platforms.each do |plat|
++        config["rbconfig-#{plat}-#{version}"] = rbconfig
+ 
+-      # also store RubyGems-compatible version
+-      gem_platform = Gem::Platform.new(plat)
+-      config["rbconfig-#{gem_platform}-#{version}"] = rbconfig
++        # also store RubyGems-compatible version
++        gem_platform = Gem::Platform.new(plat)
++        config["rbconfig-#{gem_platform}-#{version}"] = rbconfig
++      end
++
++      puts "Found Ruby version #{version} for platform #{platform} (#{rbconfig})"
+     end
+ 
+-    puts "Found Ruby version #{version} for platform #{platform} (#{rbconfig})"
++    when_writing("Saving changes into #{config_file}") {
++      config_fd.rewind
++      config_fd.puts config.to_yaml
++    }
+   end
+-
+-  when_writing("Saving changes into #{config_file}") {
+-    File.open(config_file, 'w') do |f|
+-      f.puts config.to_yaml
+-    end
+-  }
+ end
+ 
+ task :default do
+-- 
+2.20.1
+

data/build/patches2/rake-compiler-1.1.0/0001-Enable-build-of-static-libruby.patch

@@ -0,0 +1,28 @@
+From 0b929b7027eb592c0971a400184f1dfa5a0bce7c Mon Sep 17 00:00:00 2001
+From: Lars Kanis <lars@greiz-reinsdorf.de>
+Date: Thu, 26 Dec 2019 19:13:48 +0100
+Subject: [PATCH 1/2] Enable build of static libruby
+
+This is used in rake-compiler-dock for linux targets.
+The static libruby is postprocessed in a special way.
+See rake-compiler-dock's Dockerfile.
+---
+ tasks/bin/cross-ruby.rake | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/tasks/bin/cross-ruby.rake b/tasks/bin/cross-ruby.rake
+index c9b0bbb..2a73be7 100644
+--- a/tasks/bin/cross-ruby.rake
++++ b/tasks/bin/cross-ruby.rake
+@@ -108,6 +108,8 @@ end
+ file "#{build_dir}/Makefile" => [build_dir, makefile_in] do |t|
+
+   options = [
++    '--enable-static',
++    '--enable-install-static-library',
+     "--host=#{MINGW_HOST}",
+     "--target=#{MINGW_TARGET}",
+     "--build=#{RUBY_BUILD}",
+--
+2.20.1
+

data/build/patches2/rake-compiler-1.1.0/0001-Fix-determining-of-ruby-versions-in-rake-native-gem.patch

@@ -0,0 +1,44 @@
+From 365843bfe0bfc3f1c1b11e81e24ac7b6c7942bdf Mon Sep 17 00:00:00 2001
+From: Lars Kanis <lars@greiz-reinsdorf.de>
+Date: Fri, 27 Dec 2019 18:18:59 +0100
+Subject: [PATCH] Fix determining of ruby versions in "rake native gem"
+
+"rake native gem" without "cross" didn't set the ruby version constraint.
+Instead it failed with NoMethodError like so:
+
+/ffi $ rake native gem
+no configuration section for specified version of Ruby (rbconfig-i386-mingw32-2.6.3)
+no configuration section for specified version of Ruby (rbconfig-x64-mingw32-2.6.3)
+install -c build/x86_64-linux/ffi_c/2.6.3/ffi_c.so lib/ffi_c.so
+cp build/x86_64-linux/ffi_c/2.6.3/ffi_c.so build/x86_64-linux/stage/lib/ffi_c.so
+rake aborted!
+NoMethodError: undefined method `split' for nil:NilClass
+/home/lars/.rvm/gems/ruby-2.6.3/gems/rake-compiler-1.0.9/lib/rake/extensiontask.rb:515:in `ruby_api_version'
+/home/lars/.rvm/gems/ruby-2.6.3/gems/rake-compiler-1.0.9/lib/rake/extensiontask.rb:262:in `block in define_native_tasks'
+/home/lars/.rvm/gems/ruby-2.6.3/gems/rake-12.3.3/exe/rake:27:in `<top (required)>'
+/home/lars/.rvm/gems/ruby-2.6.3/bin/ruby_executable_hooks:24:in `eval'
+/home/lars/.rvm/gems/ruby-2.6.3/bin/ruby_executable_hooks:24:in `<main>'
+Tasks: TOP => native => native:x86_64-linux => native:ffi:x86_64-linux
+(See full trace by running task with --trace)
+---
+ lib/rake/extensiontask.rb | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/lib/rake/extensiontask.rb b/lib/rake/extensiontask.rb
+index d85b1a3..cb1ea12 100644
+--- a/lib/rake/extensiontask.rb
++++ b/lib/rake/extensiontask.rb
+@@ -242,6 +242,10 @@ Java extension should be preferred.
+       # lib_path
+       lib_path = lib_dir
+
++      # Update compiled platform/version combinations
++      ruby_versions = (@ruby_versions_per_platform[platf] ||= [])
++      ruby_versions << ruby_ver
++
+       # create 'native:gem_name' and chain it to 'native' task
+       unless Rake::Task.task_defined?("native:#{@gem_spec.name}:#{platf}")
+         task "native:#{@gem_spec.name}:#{platf}" do |t|
+--
+2.20.1
+

data/build/patches2/rake-compiler-1.1.0/0002-Extend-mingw-search-pattern-to-find-x86_64-gcc.patch

@@ -0,0 +1,26 @@
+From 07b1a4909fa59324d309bb8a46867cacba97cafd Mon Sep 17 00:00:00 2001
+From: Lars Kanis <lars@greiz-reinsdorf.de>
+Date: Thu, 26 Dec 2019 18:31:25 +0100
+Subject: [PATCH] Extend mingw search pattern to find x86_64 gcc
+
+The previous pattern only recognized 32 bit compiler versions.
+---
+ lib/rake/extensioncompiler.rb | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/lib/rake/extensioncompiler.rb b/lib/rake/extensioncompiler.rb
+index b3d336b..75f4694 100644
+--- a/lib/rake/extensioncompiler.rb
++++ b/lib/rake/extensioncompiler.rb
+@@ -37,7 +37,7 @@ module Rake
+       paths = ENV['PATH'].split(File::PATH_SEPARATOR)
+ 
+       # the pattern to look into (captures *nix and windows executables)
+-      pattern = "{mingw32-,i?86*mingw*}gcc{,.*}"
++      pattern = "{mingw32-,i?86*mingw*,x86*mingw*}gcc{,.*}"
+ 
+       @mingw_gcc_executable = paths.find do |path|
+         # cleanup paths before globbing
+-- 
+2.20.1
+

data/build/patches2/rake-compiler-1.1.0/0002-Strip-cross-built-shared-library-files-while-linking.patch

@@ -0,0 +1,29 @@
+From 13e33405d4e51a52402e1e37e7e2df7f1f32892d Mon Sep 17 00:00:00 2001
+From: Lars Kanis <lars@greiz-reinsdorf.de>
+Date: Thu, 26 Dec 2019 19:18:02 +0100
+Subject: [PATCH 2/2] Strip cross built shared library files while linking
+
+The .so files of extensions are often stripped after compilation per task in a Rakefile.
+However this can be omitted, if the cross ruby version is built with stripping enabled.
+Stripping the files a second time doesn't make a difference then.
+---
+ tasks/bin/cross-ruby.rake | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/tasks/bin/cross-ruby.rake b/tasks/bin/cross-ruby.rake
+index 2a73be7..a9eb16b 100644
+--- a/tasks/bin/cross-ruby.rake
++++ b/tasks/bin/cross-ruby.rake
+@@ -115,7 +115,8 @@ file "#{build_dir}/Makefile" => [build_dir, source_dir] do |t|
+     "--build=#{RUBY_BUILD}",
+     '--enable-shared',
+     '--disable-install-doc',
+-    '--with-ext='
++    '--with-ext=',
++    'LDFLAGS=-pipe -s',
+   ]
+ 
+   # Force Winsock2 for Ruby 1.8, 1.9 defaults to it
+-- 
+2.20.1
+

data/build/patches2/ruby-2.7.0/ruby2_keywords.patch

@@ -0,0 +1,15 @@
+diff --git a/template/fake.rb.in b/template/fake.rb.in
+index f1da719ecb..e0b649fb28 100644
+--- a/template/fake.rb.in
++++ b/template/fake.rb.in
+@@ -27,6 +27,10 @@ case "$0" in /*) r=-r"$0";; *) r=-r"./$0";; esac
+ exec $ruby "$r" "$@"
+ =end
+ =baseruby
++<% if RUBY_VERSION < "2.7" %>
++def ruby2_keywords(*)
++end
++<% end %>
+ class Object
+   remove_const :CROSS_COMPILING if defined?(CROSS_COMPILING)
+   CROSS_COMPILING = RUBY_PLATFORM