rake-compiler-dock 0.7.0 → 1.1.0

data/Rakefile

@@ -1,13 +1,53 @@
+require 'erb'
+require "rake/clean"
 require "rake_compiler_dock"
-require "rake_compiler_dock/gem_helper"
+require_relative "build/gem_helper"
+require_relative "build/parallel_docker_build"
 
 RakeCompilerDock::GemHelper.install_tasks
 
-task :build do
-  sh "docker build -t larskanis/rake-compiler-dock-mri:#{RakeCompilerDock::IMAGE_VERSION} -f Dockerfile.mri ."
-  sh "docker build -t larskanis/rake-compiler-dock-jruby:#{RakeCompilerDock::IMAGE_VERSION} -f Dockerfile.jruby ."
+DOCKERHUB_USER = ENV['DOCKERHUB_USER'] || "larskanis"
+
+namespace :build do
+  platforms = [
+    ["x86-mingw32", "i686-w64-mingw32"],
+    ["x64-mingw32", "x86_64-w64-mingw32"],
+    ["x86-linux", "i686-redhat-linux"],
+    ["x86_64-linux", "x86_64-redhat-linux"],
+    ["x86_64-darwin", "x86_64-apple-darwin"],
+    ["arm64-darwin", "aarch64-apple-darwin"],
+  ]
+  platforms.each do |platform, target|
+    sdf = "Dockerfile.mri.#{platform}"
+
+    desc "Build image for platform #{platform}"
+    task platform => sdf
+    task sdf do
+      sh "docker build -t #{DOCKERHUB_USER}/rake-compiler-dock-mri-#{platform}:#{RakeCompilerDock::IMAGE_VERSION} -f Dockerfile.mri.#{platform} ."
+    end
+
+    df = ERB.new(File.read("Dockerfile.mri.erb")).result(binding)
+    File.write(sdf, df)
+    CLEAN.include(sdf)
+  end
+
+  desc "Build image for JRuby"
+  task :jruby => "Dockerfile.jruby"
+  task "Dockerfile.jruby" do
+    sh "docker build -t #{DOCKERHUB_USER}/rake-compiler-dock-jruby:#{RakeCompilerDock::IMAGE_VERSION} -f Dockerfile.jruby ."
+  end
+
+  RakeCompilerDock::ParallelDockerBuild.new(platforms.map{|pl, _| "Dockerfile.mri.#{pl}" } + ["Dockerfile.jruby"], workdir: "tmp/docker")
+
+  desc "Build images for all MRI platforms in parallel"
+  multitask :mri => platforms.map(&:first)
+
+  desc "Build images for all platforms in parallel"
+  multitask :all => platforms.map(&:first) + ["jruby"]
 end
 
+task :build => "build:all"
+
 desc "Run tests"
 task :test do
   sh "ruby -w -W2 -I. -Ilib -e \"#{Dir["test/test_*.rb"].map{|f| "require '#{f}';"}.join}\" -- -v"

data/{lib/rake_compiler_dock → build}/gem_helper.rb

@@ -1,18 +1,3 @@
-# This file is part of Libusb for Ruby.
-#
-# Libusb for Ruby is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# Libusb for Ruby is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with Libusb for Ruby.  If not, see <http://www.gnu.org/licenses/>.
-
 require "bundler/gem_helper"
 
 module RakeCompilerDock

data/build/parallel_docker_build.rb

@@ -0,0 +1,102 @@
+require "fileutils"
+require "rake"
+
+module RakeCompilerDock
+  # Run docker builds in parallel, but ensure that common docker layers are reused
+  class ParallelDockerBuild
+    include Rake::DSL
+
+    def initialize(dockerfiles, workdir: "tmp/docker", inputdir: ".", task_prefix: "common-")
+      FileUtils.mkdir_p(workdir)
+
+      files = parse_dockerfiles(dockerfiles, inputdir)
+      # pp files
+
+      vcs = find_commons(files)
+      # pp vcs
+
+      define_common_tasks(vcs, workdir, task_prefix)
+    end
+
+    # Read given dockerfiles from inputdir and split into a list of commands.
+    #
+    # Returns:
+    #   {"File0"=>["      FROM a\n", "      RUN a\n", "      RUN d\n"],
+    #    "File1"=>["      FROM a\n",
+    #    ...
+    def parse_dockerfiles(dockerfiles, inputdir)
+      files = dockerfiles.map do |fn|
+        [fn, File.read(File.join(inputdir, fn))]
+      end.map do |fn, f|
+        # Split file contant in lines unless line ends with backslash
+        fc = f.each_line.with_object([]) do |line, lines|
+          if lines.last=~/\\\n\z/
+            lines.last << line
+          else
+            lines << line
+          end
+        end
+        [fn, fc]
+      end.to_h
+    end
+
+    # Build a tree of common parts of given files.
+    #
+    # Returns:
+    #   {["File0", "File1", "File2", "File3"]=>
+    #     [["  FROM a\n"],
+    #      {["File0", "File1"]=>
+    #        [["  RUN a\n", "  RUN d\n"], {["File1"]=>[["  RUN f\n"], {}]}],
+    #       ["File2", "File3"]=>[["  RUN b\n", "  RUN c\n", "  RUN d\n"], {}]}]}
+    def find_commons(files, vmask=nil, li=0)
+      vmask ||= files.keys
+      vcs = Hash.new { [] }
+      files.each do |fn, lines|
+        next unless vmask.include?(fn)
+        vcs[lines[li]] += [fn]
+      end
+
+      vcs.map do |line, vc|
+        next unless line
+        nvcs = find_commons(files, vc, li+1)
+        if nvcs.first && nvcs.first[0] == vc
+          # Append lines that are equal between file(s)
+          nl = [[line] + nvcs.first[1][0], nvcs.first[1][1]]
+        else
+          nl = [[line], nvcs]
+        end
+        [vc, nl]
+      end.compact.to_h
+    end
+
+    # Write intermediate dockerfiles to workdir and define rake tasks
+    #
+    # The rake tasks are named after the dockerfiles given to #new .
+    # This also adds dependant intermediate tasks as prerequisites.
+    def define_common_tasks(vcs, workdir, task_prefix, plines=[])
+      vcs.map do |files, (lines, nvcs)|
+        fn = "#{task_prefix}#{files.join}"
+        File.write(File.join(workdir, fn), (plines + lines).join)
+        task fn do
+          docker_build(fn, workdir)
+        end
+
+        nfn = define_common_tasks(nvcs, workdir, task_prefix, plines + lines)
+        nfn.each do |file|
+          task file => fn
+        end
+        files.each do |file|
+          task file => fn
+        end
+        fn
+      end
+    end
+
+    # Run an intermediate dockerfile without tag
+    #
+    # The layers will be reused in subsequent builds, even if they run in parallel.
+    def docker_build(filename, workdir)
+      sh "docker", "build", "-f", File.join(workdir, filename), "."
+    end
+  end
+end

data/build/patches/{ruby-2.5.3 → ruby-2.5.8}/no_sendfile.patch

@@ -17,7 +17,7 @@ index 82c5940538..a8d3661ec8 100644
 @@ -10721,7 +10721,6 @@ nogvl_copy_stream_wait_write(struct copy_stream_struct *stp)
  }
 
- #if defined __linux__ && defined __NR_copy_file_range
+ #if defined HAVE_COPY_FILE_RANGE || (defined __linux__ && defined __NR_copy_file_range)
 -#  define USE_COPY_FILE_RANGE
  #endif
 

data/build/patches/ruby-3.0.0/no_sendfile.patch

@@ -0,0 +1,24 @@
+diff --git a/configure b/configure
+index ebe3d8c..a336b73 100755
+--- a/configure
++++ b/configure
+@@ -18943,7 +18943,6 @@ do :
+   ac_fn_c_check_func "$LINENO" "sendfile" "ac_cv_func_sendfile"
+ if test "x$ac_cv_func_sendfile" = xyes; then :
+   cat >>confdefs.h <<_ACEOF
+-#define HAVE_SENDFILE 1
+ _ACEOF
+
+ fi
+diff --git a/io.c b/io.c
+index 48592ac51a..2635ac8ec6 100644
+--- a/io.c
++++ b/io.c
+@@ -11111,6 +11111,7 @@ nogvl_copy_stream_wait_write(struct copy_stream_struct *stp)
+     return 0;
+ }
+
++#undef USE_COPY_FILE_RANGE
+ #ifdef USE_COPY_FILE_RANGE
+
+ static ssize_t

data/build/patches2/hoe-3.20.0/0001-Load-encrypted-private-key-using-ENV-GEM_PRIVATE_KEY.patch

@@ -0,0 +1,32 @@
+From ecc5669223457ceaba6bd94c2947ae99ddfa8f10 Mon Sep 17 00:00:00 2001
+From: Lars Kanis <lars@greiz-reinsdorf.de>
+Date: Fri, 27 Dec 2019 15:57:07 +0100
+Subject: [PATCH] Load encrypted private key using
+ ENV['GEM_PRIVATE_KEY_PASSPHRASE'] as passphrase.
+
+This way the password can be avoided to be entered several times when building multiple gems.
+It also allows parallel builds which are incompatible to interactive password prompts.
+
+The same feature has been implemented in rubygems:
+  https://github.com/rubygems/rubygems/commit/c1a114396fcec124d3feabf74708b4bdec02eac3
+---
+ lib/hoe/signing.rb | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/lib/hoe/signing.rb b/lib/hoe/signing.rb
+index 4833c55..11c8c2f 100644
+--- a/lib/hoe/signing.rb
++++ b/lib/hoe/signing.rb
+@@ -70,7 +70,8 @@ module Hoe::Signing
+     end
+
+     if signing_key and cert_chain then
+-      spec.signing_key = OpenSSL::PKey::RSA.new File.read signing_key
++      passphrase = ENV['GEM_PRIVATE_KEY_PASSPHRASE']
++      spec.signing_key = OpenSSL::PKey::RSA.new(File.read(signing_key), passphrase)
+       spec.cert_chain = cert_chain
+     end
+   end
+--
+2.20.1
+

data/build/patches2/rake-compiler-1.1.1/0001-Fix-determining-of-ruby-versions-in-rake-native-gem.patch

@@ -0,0 +1,44 @@
+From 365843bfe0bfc3f1c1b11e81e24ac7b6c7942bdf Mon Sep 17 00:00:00 2001
+From: Lars Kanis <lars@greiz-reinsdorf.de>
+Date: Fri, 27 Dec 2019 18:18:59 +0100
+Subject: [PATCH] Fix determining of ruby versions in "rake native gem"
+
+"rake native gem" without "cross" didn't set the ruby version constraint.
+Instead it failed with NoMethodError like so:
+
+/ffi $ rake native gem
+no configuration section for specified version of Ruby (rbconfig-i386-mingw32-2.6.3)
+no configuration section for specified version of Ruby (rbconfig-x64-mingw32-2.6.3)
+install -c build/x86_64-linux/ffi_c/2.6.3/ffi_c.so lib/ffi_c.so
+cp build/x86_64-linux/ffi_c/2.6.3/ffi_c.so build/x86_64-linux/stage/lib/ffi_c.so
+rake aborted!
+NoMethodError: undefined method `split' for nil:NilClass
+/home/lars/.rvm/gems/ruby-2.6.3/gems/rake-compiler-1.0.9/lib/rake/extensiontask.rb:515:in `ruby_api_version'
+/home/lars/.rvm/gems/ruby-2.6.3/gems/rake-compiler-1.0.9/lib/rake/extensiontask.rb:262:in `block in define_native_tasks'
+/home/lars/.rvm/gems/ruby-2.6.3/gems/rake-12.3.3/exe/rake:27:in `<top (required)>'
+/home/lars/.rvm/gems/ruby-2.6.3/bin/ruby_executable_hooks:24:in `eval'
+/home/lars/.rvm/gems/ruby-2.6.3/bin/ruby_executable_hooks:24:in `<main>'
+Tasks: TOP => native => native:x86_64-linux => native:ffi:x86_64-linux
+(See full trace by running task with --trace)
+---
+ lib/rake/extensiontask.rb | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/lib/rake/extensiontask.rb b/lib/rake/extensiontask.rb
+index d85b1a3..cb1ea12 100644
+--- a/lib/rake/extensiontask.rb
++++ b/lib/rake/extensiontask.rb
+@@ -242,6 +242,10 @@ Java extension should be preferred.
+       # lib_path
+       lib_path = lib_dir
+
++      # Update compiled platform/version combinations
++      ruby_versions = (@ruby_versions_per_platform[platf] ||= [])
++      ruby_versions << ruby_ver
++
+       # create 'native:gem_name' and chain it to 'native' task
+       unless Rake::Task.task_defined?("native:#{@gem_spec.name}:#{platf}")
+         task "native:#{@gem_spec.name}:#{platf}" do |t|
+--
+2.20.1
+

data/build/patches2/rake-compiler-1.1.1/0003-Allow-building-of-cross-rubies-in-parallel.patch

@@ -0,0 +1,214 @@
+From 01fd7364bdcb37ac8709ffa48388b5cbe6478211 Mon Sep 17 00:00:00 2001
+From: Lars Kanis <lars@greiz-reinsdorf.de>
+Date: Fri, 10 Jan 2020 23:57:18 +0100
+Subject: [PATCH] Allow building of cross rubies in parallel
+
+Rubies can be build like so:
+  rake-compiler cross-ruby VERSION=2.7.0:2.6.0 HOST=x86_64-w64-mingw32:i686-w64-mingw32
+
+This builds the cross product of all ":" separated ruby and host versions.
+To force sequential builds add option "-j1".
+---
+ tasks/bin/cross-ruby.rake | 161 +++++++++++++++++++-------------------
+ 1 file changed, 80 insertions(+), 81 deletions(-)
+
+diff --git a/tasks/bin/cross-ruby.rake b/tasks/bin/cross-ruby.rake
+index 278541c..8b88025 100644
+--- a/tasks/bin/cross-ruby.rake
++++ b/tasks/bin/cross-ruby.rake
+@@ -41,106 +41,105 @@ end
+ require 'rake/extensioncompiler'
+ 
+ MAKE = ENV['MAKE'] || %w[gmake make].find { |c| system("#{c} -v > /dev/null 2>&1") }
+-USER_HOME = File.expand_path("~/.rake-compiler")
+-RUBY_CC_VERSION = "ruby-" << ENV.fetch("VERSION", "1.8.7-p371")
++USER_HOME = File.realpath(File.expand_path("~/.rake-compiler"))
+ RUBY_SOURCE = ENV['SOURCE']
+ RUBY_BUILD = RbConfig::CONFIG["host"]
+ 
+-# grab the major "1.8" or "1.9" part of the version number
+-MAJOR = RUBY_CC_VERSION.match(/.*-(\d.\d).\d/)[1]
+-
+-# Use Rake::ExtensionCompiler helpers to find the proper host
+-MINGW_HOST = ENV['HOST'] || Rake::ExtensionCompiler.mingw_host
+-MINGW_TARGET = MINGW_HOST.gsub('msvc', '')
+-
+ # Unset any possible variable that might affect compilation
+ ["CC", "CXX", "CPPFLAGS", "LDFLAGS", "RUBYOPT"].each do |var|
+   ENV.delete(var)
+ end
+ 
+-source_dir = "#{USER_HOME}/sources/#{RUBY_CC_VERSION}"
+-build_dir = "#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}"
+-
+-# define a location where sources will be stored
+-directory source_dir
+-directory build_dir
+-
+-# clean intermediate files and folders
+-CLEAN.include(source_dir)
+-CLEAN.include(build_dir)
+-
+-# remove the final products and sources
+-CLOBBER.include("#{USER_HOME}/sources")
+-CLOBBER.include("#{USER_HOME}/builds")
+-CLOBBER.include("#{USER_HOME}/ruby/#{MINGW_HOST}/#{RUBY_CC_VERSION}")
+-CLOBBER.include("#{USER_HOME}/config.yml")
++RUBY_CC_VERSIONS = ENV.fetch("VERSION", "1.8.7-p371")
++RUBY_CC_VERSIONS.split(":").each do |ruby_cc_version|
++  ruby_cc_version = "ruby-" + ruby_cc_version
++  # grab the major "1.8" or "1.9" part of the version number
++  major = ruby_cc_version.match(/.*-(\d.\d).\d/)[1]
++
++  # define a location where sources will be stored
++  source_dir = "#{USER_HOME}/sources/#{ruby_cc_version}"
++  directory source_dir
++  # clean intermediate files and folders
++  CLEAN.include(source_dir)
++
++  # remove the final products and sources
++  CLOBBER.include("#{USER_HOME}/sources")
++  CLOBBER.include("#{USER_HOME}/builds")
++  CLOBBER.include("#{USER_HOME}/config.yml")
++
++  # Extract the sources
++  source_file = RUBY_SOURCE ? RUBY_SOURCE.split('/').last : "#{ruby_cc_version}.tar.bz2"
++  file source_dir => ["#{USER_HOME}/sources/#{source_file}"] do |t|
++    t.prerequisites.each { |f| sh "tar xf #{File.basename(f)}", chdir: File.dirname(t.name) }
++  end
+ 
+-# ruby source file should be stored there
+-file "#{USER_HOME}/sources/#{RUBY_CC_VERSION}.tar.bz2" => ["#{USER_HOME}/sources"] do |t|
+-  # download the source file using wget or curl
+-  chdir File.dirname(t.name) do
++  # ruby source file should be stored there
++  file "#{USER_HOME}/sources/#{ruby_cc_version}.tar.bz2" => ["#{USER_HOME}/sources"] do |t|
++    # download the source file using wget or curl
+     if RUBY_SOURCE
+       url = RUBY_SOURCE
+     else
+-      url = "http://cache.ruby-lang.org/pub/ruby/#{MAJOR}/#{File.basename(t.name)}"
++      url = "http://cache.ruby-lang.org/pub/ruby/#{major}/#{File.basename(t.name)}"
+     end
+-    sh "wget #{url} || curl -O #{url}"
++    sh "wget #{url} || curl -O #{url}", chdir: File.dirname(t.name)
+   end
+-end
+-
+-# Extract the sources
+-source_file = RUBY_SOURCE ? RUBY_SOURCE.split('/').last : "#{RUBY_CC_VERSION}.tar.bz2"
+-file source_dir => ["#{USER_HOME}/sources/#{source_file}"] do |t|
+-  chdir File.dirname(t.name) do
+-    t.prerequisites.each { |f| sh "tar xf #{File.basename(f)}" }
+-  end
+-end
+ 
+-task :mingw32 do
+-  unless MINGW_HOST then
+-    warn "You need to install mingw32 cross compile functionality to be able to continue."
+-    warn "Please refer to your distribution/package manager documentation about installation."
+-    fail
+-  end
+-end
++  # Create tasks for each host out of the ":" separated hosts list in the HOST variable.
++  # These tasks are processed in parallel as dependencies to the "install" task.
++  mingw_hosts = ENV['HOST'] || Rake::ExtensionCompiler.mingw_host
++  mingw_hosts.split(":").each do |mingw_host|
++
++    # Use Rake::ExtensionCompiler helpers to find the proper host
++    mingw_target = mingw_host.gsub('msvc', '')
++
++    # define a location where built files for each host will be stored
++    build_dir = "#{USER_HOME}/builds/#{mingw_host}/#{ruby_cc_version}"
++    directory build_dir
++    install_dir = "#{USER_HOME}/ruby/#{mingw_host}/#{ruby_cc_version}"
++
++    # clean intermediate files and folders
++    CLEAN.include(build_dir)
++    CLOBBER.include(install_dir)
++
++    task :mingw32 do
++      unless mingw_host then
++        warn "You need to install mingw32 cross compile functionality to be able to continue."
++        warn "Please refer to your distribution/package manager documentation about installation."
++        fail
++      end
++    end
+ 
+-# generate the makefile in a clean build location
+-file "#{build_dir}/Makefile" => [build_dir, source_dir] do |t|
+-
+-  options = [
+-    "--host=#{MINGW_HOST}",
+-    "--target=#{MINGW_TARGET}",
+-    "--build=#{RUBY_BUILD}",
+-    '--enable-shared',
+-    '--disable-install-doc',
+-    '--with-ext=',
+-    'LDFLAGS=-pipe -s',
+-  ]
+-
+-  # Force Winsock2 for Ruby 1.8, 1.9 defaults to it
+-  options << "--with-winsock2" if MAJOR == "1.8"
+-
+-  chdir File.dirname(t.name) do
+-    prefix = File.expand_path("../../../ruby/#{MINGW_HOST}/#{RUBY_CC_VERSION}")
+-    options << "--prefix=#{prefix}"
+-    sh File.expand_path("../../../sources/#{RUBY_CC_VERSION}/configure"), *options
+-  end
+-end
++    # generate the makefile in a clean build location
++    file "#{build_dir}/Makefile" => [build_dir, source_dir] do |t|
++
++      options = [
++        "--host=#{mingw_host}",
++        "--target=#{mingw_target}",
++        "--build=#{RUBY_BUILD}",
++        '--enable-shared',
++        '--disable-install-doc',
++        '--with-ext=',
++        'LDFLAGS=-pipe -s',
++      ]
++
++      # Force Winsock2 for Ruby 1.8, 1.9 defaults to it
++      options << "--with-winsock2" if major == "1.8"
++      options << "--prefix=#{install_dir}"
++      sh File.expand_path("#{USER_HOME}/sources/#{ruby_cc_version}/configure"), *options, chdir: File.dirname(t.name)
++    end
+ 
+-# make
+-file "#{build_dir}/ruby.exe" => ["#{build_dir}/Makefile"] do |t|
+-  chdir File.dirname(t.prerequisites.first) do
+-    sh MAKE
+-  end
+-end
++    # make
++    file "#{build_dir}/ruby.exe" => ["#{build_dir}/Makefile"] do |t|
++      sh MAKE, chdir: File.dirname(t.prerequisites.first)
++    end
+ 
+-# make install
+-file "#{USER_HOME}/ruby/#{MINGW_HOST}/#{RUBY_CC_VERSION}/bin/ruby.exe" => ["#{build_dir}/ruby.exe"] do |t|
+-  chdir File.dirname(t.prerequisites.first) do
+-    sh "#{MAKE} install"
++    # make install
++    file "#{USER_HOME}/ruby/#{mingw_host}/#{ruby_cc_version}/bin/ruby.exe" => ["#{build_dir}/ruby.exe"] do |t|
++      sh "#{MAKE} install", chdir: File.dirname(t.prerequisites.first)
++    end
++    multitask :install => ["#{USER_HOME}/ruby/#{mingw_host}/#{ruby_cc_version}/bin/ruby.exe"]
+   end
+ end
+-task :install => ["#{USER_HOME}/ruby/#{MINGW_HOST}/#{RUBY_CC_VERSION}/bin/ruby.exe"]
+ 
+ desc "Update rake-compiler list of installed Ruby versions"
+ task 'update-config' do
+@@ -187,5 +186,5 @@ task :default do
+   Rake.application.display_tasks_and_comments
+ end
+ 
+-desc "Build #{RUBY_CC_VERSION} suitable for cross-platform development."
++desc "Build rubies suitable for cross-platform development."
+ task 'cross-ruby' => [:mingw32, :install, 'update-config']
+-- 
+2.20.1
+