rcee_precompiled 0.4.0-x86-linux

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a2db5b17deb67ff771b0b171e94b3f031811d73753648649fcc185c7e5d7b4b1
+  data.tar.gz: d8dc00757d9853baecacf8dee928b45e859ced406fd76f57217392d82c48f31d
+SHA512:
+  metadata.gz: 949862cf2477c6241730793cde083ad7505ece30714721d5b5f138f5d1e37ac6d8d52f40aa4587a88633181cef63978591bce0d04c983dfbcea9e8169a8c0642
+  data.tar.gz: 4c4ebbb8f4a6cee4980c2cd0343cf5b995e63af0e81375884af42e7d04fc25be3b3e97ec843e3dd59bb3076b51762cc48e3baa0f81aa6f4cd4b84ec43c5e004a

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+ports

data/Gemfile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in precompiled.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "rake-compiler"
+gem "rake-compiler-dock", "~> 1.2.1"
+
+gem "minitest", "~> 5.0"

data/README.md

@@ -0,0 +1,191 @@
+# RCEE::Precompiled
+
+This gem is part of the Ruby C Extensions Explained project at https://github.com/flavorjones/ruby-c-extensions-explained
+
+## Summary
+
+Installation time is an important aspect of developer happiness, and the Packaged strategies can make your users very unhappy.
+
+One way to speed up installation time is to precompile the extension, so that during `gem install` files are simply being unpacked, and the entire configure/compile/link process is avoided completely. This can be tricky to set up, and requires that you test across all the target platforms; but makes for much happier users.
+
+Note, however, that it's necessary to _also_ ship the vanilla (platform=ruby) gem as a fallback for platforms you haven't pre-compiled for.
+
+In the nine months since Nokogiri v1.11 started shipping precompiled native gems, we've seen over 45 million gem installations, and essentially zero support issues have been opened. Twitter complaints dropped to near zero. Page views of the Nokogiri installation tutorial dropped by half.
+
+Precompiled Nokogiri gems have been an unmitigated success and have made both users and maintainers happy.
+
+
+## Details
+
+An important tool we rely on for precompiling is [`rake-compiler/rake-compiler-dock`](https://github.com/rake-compiler/rake-compiler-dock), maintained by Lars Kanis. `rake-compiler-dock` is a Docker-based build environment that uses `rake-compiler` to _cross compile_ for all of these major platforms.
+
+What this means is, I can run my normal gem build process in a docker container on my Linux machine, and it will build gems that I run on Windows and MacOS.
+
+This is really powerful stuff, and once we assume that we can cross-compile reliably, the remaining problems boil down to modifying how we build the gemfile and making sure we test gems adequately on the target platforms.
+
+First, we need to add some features to our `Rake::ExtensionTask` in `Rakefile`:
+
+``` ruby
+cross_rubies = ["3.1.0", "3.0.0", "2.7.0", "2.6.0"]
+cross_platforms = [
+  "x64-mingw32",
+  "x64-mingw-ucrt",
+  "x86-linux",
+  "x86_64-linux",
+  "aarch64-linux",
+  "x86_64-darwin",
+  "arm64-darwin",
+]
+ENV["RUBY_CC_VERSION"] = cross_rubies.join(":")
+
+Rake::ExtensionTask.new("precompiled", rcee_precompiled_spec) do |ext|
+  ext.lib_dir = "lib/rcee/precompiled"
+  ext.cross_compile = true
+  ext.cross_platform = cross_platforms
+  ext.cross_config_options << "--enable-cross-build" # so extconf.rb knows we're cross-compiling
+  ext.cross_compiling do |spec|
+    # remove things not needed for precompiled gems
+    spec.dependencies.reject! { |dep| dep.name == "mini_portile2" }
+    spec.files.reject! { |file| File.fnmatch?("*.tar.gz", file) }
+  end
+end
+```
+
+This does the following:
+
+- set up some local variables to indicate what ruby versions and which platforms we will build for
+- set an environment variable to let rake-compiler know which ruby versions we'll build
+- tell the extension task to turn on cross-compiling features, including additional rake tasks
+- signal to our `extconf.rb` when we're cross-compiling (in case its behavior needs to change)
+- finally, in a block that is only run when cross-compiling, we modify the gemspec to remove things we don't need in native gems:
+  - the tarball
+  - the dependency on `mini_portile`
+
+Next we need some new rake tasks:
+
+``` ruby
+namespace "gem" do
+  cross_platforms.each do |platform|
+    desc "build native gem for #{platform}"
+    task platform do
+      RakeCompilerDock.sh(<<~EOF, platform: platform)
+        gem install bundler --no-document &&
+        bundle &&
+        bundle exec rake gem:#{platform}:buildit
+      EOF
+    end
+
+    namespace platform do
+      # this runs in the rake-compiler-dock docker container
+      task "buildit" do
+        # use Task#invoke because the pkg/*gem task is defined at runtime
+        Rake::Task["native:#{platform}"].invoke
+        Rake::Task["pkg/#{rcee_precompiled_spec.full_name}-#{Gem::Platform.new(platform)}.gem"].invoke
+      end
+    end
+  end
+end
+```
+
+The top task (or, really, _set_ of tasks) runs on the host system, and invokes the lower task within the appropriate docker container to cross-compile for that platform. So the bottom task is doing most of the work, and it's doing it inside a guest container. Please note that the worker is both building the extension *and* packaging the gem according to the gemspec that was just modified by our extensiontask cross-compiling block.
+
+Changes to the `extconf.rb`:
+
+``` ruby
+ENV["CC"] = RbConfig::CONFIG["CC"]
+```
+
+This makes sure that the cross-compiler is the compiler used within the guest container (and not the native linux compiler).
+
+``` ruby
+cross_build_p = enable_config("cross-build")
+```
+
+The cross-compile rake task signals to `extconf.rb` that it's cross-compiling by using a commandline flag that we can inspect. We'll need this for `libyaml` to make sure that set the appropriate flags during precompilation (flags which shouldn't be set when compiling natively).
+
+``` ruby
+MiniPortile.new("yaml", "0.2.5").tap do |recipe|
+  # ...
+  # configure the environment that MiniPortile will use for subshells
+  ENV.to_h.dup.tap do |env|
+    # -fPIC is necessary for linking into a shared library
+    env["CFLAGS"] = [env["CFLAGS"], "-fPIC"].join(" ")
+    env["SUBDIRS"] = "include src" # libyaml: skip tests
+
+    recipe.configure_options += env.map { |key, value| "#{key}=#{value.strip}" }
+  end
+  # ...
+end
+```
+
+The rest of the extconf changes are related to configuring libyaml at build time. We need to set the -fPIC option so we can mix static and shared libraries together. (This should probably always be set.)
+
+The "SUBDIRS" environment variable is something that's very specific to libyaml, though: it tells libyaml's autoconf build system to skip running the tests. We have to do this because although we can *generate* binaries for other platforms, we can't actually *run* them.
+
+We have one more small change we'll need to make to how the extension is required. Let's take a look at the directory structure in the packaged gem:
+
+``` text
+lib
+└── rcee
+    ├── precompiled
+    │   ├── 2.6
+    │   │   └── precompiled.so
+    │   ├── 2.7
+    │   │   └── precompiled.so
+    │   ├── 3.0
+    │   │   └── precompiled.so
+    │   ├── 3.1
+    │   │   └── precompiled.so
+    │   └── version.rb
+    └── precompiled.rb
+```
+
+You can see that we have FOUR c extensions in this gem, one for each minor version of Ruby that we support. Remember that a C extension is specific to an architecture and a version of Ruby. For example, if we're running Ruby 3.0.1, then we need to load the extension in the 3.0 directory. Let's make sure we do that.
+
+In `lib/rcee/precompiled.rb`, we'll replace the normal `require` with:
+
+``` ruby
+begin
+  # load the precompiled extension file
+  ruby_version = /(\d+\.\d+)/.match(::RUBY_VERSION)
+  require_relative "precompiled/#{ruby_version}/precompiled"
+rescue LoadError
+  # fall back to the extension compiled upon installation.
+  require "rcee/precompiled/precompiled"
+end
+```
+
+Go ahead and try it! `gem install rcee_precompiled`. If you're on windows, linux, or macos you should get a precompiled version that installs in under a second. Everyone else (hello FreeBSD people!) it'll take a few more seconds to build the vanilla gem's packaged tarball.
+
+
+## Testing
+
+See [.github/workflows/precompiled.yml](../.github/workflows/precompiled.yml)
+
+Key things to note:
+
+- matrix across all supported Rubies and platforms (for compile-from-source installation testing)
+- test native gems for a variety of platforms
+  - use rake-compiler-dock images to build the gems
+  - then install on native platforms and verify that it passes tests
+
+Note that there's additional complexity because of how we test:
+
+- see new script bin/test-gem-build which artificially bumps the VERSION string to double-check we're testing the packaged version of the gem (which the tests output)
+- see new script bin/test-gem-install which installs the gem, deletes the local source code, and runs the tests against the installed gem
+- the gemspec handles a missing version file (because we delete the local source code during testing)
+
+
+## What Can Go Wrong
+
+This strategy isn't perfect. Remember what I said earlier, that a compiled C extension is specific to
+
+- the minor version of ruby (e.g., 3.0)
+- the machine architecture (e.g., x86_64)
+- the system libraries
+
+The precompiled strategy mostly takes care of the first two, but there are still edge cases for system libraries. The big gotcha is that linux libc is not the same as linux musl, and we've had to work around this a few times in Nokogiri.
+
+I'm positive that there are more edge cases that will be found as users add more platforms and as more gems start precompiling. I'm willing to bet money that you can break this by setting some Ruby compile-time flags on your system. I'm honestly surprised it works as well as it has. (Worth noting: the `sassc` gem stopped shipping native gems for linux because of the musl incompatibilities.)
+
+So the lesson here is: make sure you have an automated test pipeline that will build a gem and test it on the target platform! This takes time to set up, but it will save you time and effort in the long run.

data/Rakefile

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rubygems/package_task"
+require "rake/testtask"
+require "rake/extensiontask"
+require "rake_compiler_dock"
+
+cross_rubies = ["3.1.0", "3.0.0", "2.7.0", "2.6.0"]
+cross_platforms = [
+  "aarch64-linux",
+  "arm-linux",
+  "arm64-darwin",
+  "x64-mingw-ucrt",
+  "x64-mingw32",
+  "x86-linux",
+  "x86_64-darwin",
+  "x86_64-linux",
+]
+ENV["RUBY_CC_VERSION"] = cross_rubies.join(":")
+
+rcee_precompiled_spec = Bundler.load_gemspec("rcee_precompiled.gemspec")
+Gem::PackageTask.new(rcee_precompiled_spec).define #packaged_tarball version of the gem for platform=ruby
+task "package" => cross_platforms.map { |p| "gem:#{p}" } # "package" task for all the native platforms
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*_test.rb"]
+end
+
+Rake::ExtensionTask.new("precompiled", rcee_precompiled_spec) do |ext|
+  ext.lib_dir = "lib/rcee/precompiled"
+  ext.cross_compile = true
+  ext.cross_platform = cross_platforms
+  ext.cross_config_options << "--enable-cross-build" # so extconf.rb knows we're cross-compiling
+  ext.cross_compiling do |spec|
+    # remove things not needed for precompiled gems
+    spec.dependencies.reject! { |dep| dep.name == "mini_portile2" }
+    spec.files.reject! { |file| File.fnmatch?("*.tar.gz", file) }
+  end
+end
+
+namespace "gem" do
+  cross_platforms.each do |platform|
+    desc "build native gem for #{platform}"
+    task platform do
+      RakeCompilerDock.sh(<<~EOF, platform: platform)
+        gem install bundler --no-document &&
+        bundle &&
+        bundle exec rake gem:#{platform}:buildit
+      EOF
+    end
+
+    namespace platform do
+      # this runs in the rake-compiler-dock docker container
+      task "buildit" do
+        # use Task#invoke because the pkg/*gem task is defined at runtime
+        Rake::Task["native:#{platform}"].invoke
+        Rake::Task["pkg/#{rcee_precompiled_spec.full_name}-#{Gem::Platform.new(platform)}.gem"].invoke
+      end
+    end
+  end
+
+  desc "build native gem for all platforms"
+  multitask "all" => [cross_platforms, "gem"].flatten
+end
+
+desc "Temporarily set VERSION to a unique timestamp"
+task "set-version-to-timestamp" do
+  # this task is used by bin/test-gem-build
+  # to test building, packaging, and installing a precompiled gem
+  version_constant_re = /^\s*VERSION\s*=\s*["'](.*)["']$/
+
+  version_file_path = File.join(__dir__, "lib/rcee/precompiled/version.rb")
+  version_file_contents = File.read(version_file_path)
+
+  current_version_string = version_constant_re.match(version_file_contents)[1]
+  current_version = Gem::Version.new(current_version_string)
+
+  fake_version = Gem::Version.new(format("%s.test.%s", current_version.bump, Time.now.strftime("%Y.%m%d.%H%M")))
+
+  unless version_file_contents.gsub!(version_constant_re, "    VERSION = \"#{fake_version}\"")
+    raise("Could not hack the VERSION constant")
+  end
+
+  File.open(version_file_path, "w") { |f| f.write(version_file_contents) }
+
+  puts "NOTE: wrote version as \"#{fake_version}\""
+end
+
+task default: [:clobber, :compile, :test]
+
+CLEAN.add("{ext,lib}/**/*.{o,so}", "pkg")
+CLOBBER.add("ports")
+
+# when packaging the gem, if the tarball isn't cached, we need to fetch it. the easiest thing to do
+# is to run the compile phase to invoke the extconf and have mini_portile download the file for us.
+# this is wasteful and in the future I would prefer to separate mini_portile from the extconf to
+# allow us to download without compiling.
+Rake::Task["package"].prerequisites.prepend("compile")

data/ext/precompiled/extconf.rb

@@ -0,0 +1,41 @@
+require "mkmf"
+require "mini_portile2"
+
+package_root_dir = File.expand_path(File.join(File.dirname(__FILE__), "..", ".."))
+
+RbConfig::CONFIG["CC"] = RbConfig::MAKEFILE_CONFIG["CC"] = ENV["CC"] if ENV["CC"]
+ENV["CC"] = RbConfig::CONFIG["CC"]
+
+cross_build_p = enable_config("cross-build")
+
+MiniPortile.new("yaml", "0.2.5").tap do |recipe|
+  recipe.files = [{
+    url: "https://github.com/yaml/libyaml/releases/download/0.2.5/yaml-0.2.5.tar.gz",
+    sha256: "c642ae9b75fee120b2d96c712538bd2cf283228d2337df2cf2988e3c02678ef4",
+  }]
+  recipe.target = File.join(package_root_dir, "ports")
+
+  # configure the environment that MiniPortile will use for subshells
+  if cross_build_p
+    ENV.to_h.tap do |env|
+      # -fPIC is necessary for linking into a shared library
+      env["CFLAGS"] = [env["CFLAGS"], "-fPIC"].join(" ")
+      env["SUBDIRS"] = "include src" # libyaml: skip tests
+
+      recipe.configure_options += env.map { |key, value| "#{key}=#{value.strip}" }
+    end
+  end
+
+  unless File.exist?(File.join(recipe.target, recipe.host, recipe.name, recipe.version))
+    recipe.cook
+  end
+
+  recipe.activate
+  pkg_config(File.join(recipe.path, "lib", "pkgconfig", "yaml-0.1.pc"))
+end
+
+unless have_library("yaml", "yaml_get_version", "yaml.h")
+  abort("\nERROR: *** could not find libyaml development environment ***\n\n")
+end
+
+create_makefile("rcee/precompiled/precompiled")

data/ext/precompiled/precompiled.c

@@ -0,0 +1,25 @@
+#include "precompiled.h"
+
+VALUE rb_mRCEE;
+VALUE rb_mPrecompiled;
+VALUE rb_cPrecompiledExtension;
+
+static VALUE
+rb_precompiled_extension_class_do_something(VALUE self)
+{
+  int major, minor, patch;
+
+  yaml_get_version(&major, &minor, &patch);
+
+  return rb_sprintf("libyaml version %d.%d.%d", major, minor, patch);
+}
+
+void
+Init_precompiled(void)
+{
+  rb_mRCEE = rb_define_module("RCEE");
+  rb_mPrecompiled = rb_define_module_under(rb_mRCEE, "Precompiled");
+  rb_cPrecompiledExtension = rb_define_class_under(rb_mPrecompiled, "Extension", rb_cObject);
+  rb_define_singleton_method(rb_cPrecompiledExtension, "do_something",
+                             rb_precompiled_extension_class_do_something, 0);
+}

data/ext/precompiled/precompiled.h

@@ -0,0 +1,7 @@
+#ifndef PRECOMPILED_H
+#define PRECOMPILED_H 1
+
+#include "ruby.h"
+#include "yaml.h"
+
+#endif /* PRECOMPILED_H */

data/lib/rcee/precompiled/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module RCEE
+  module Precompiled
+    VERSION = "0.4.0"
+  end
+end

data/lib/rcee/precompiled.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative "precompiled/version"
+
+begin
+  # load the precompiled extension file
+  ruby_version = /(\d+\.\d+)/.match(::RUBY_VERSION)
+  require_relative "precompiled/#{ruby_version}/precompiled"
+rescue LoadError
+  # fall back to the extension compiled upon installation.
+  # use "require" instead of "require_relative" because non-native gems will place C extension files
+  # in Gem::BasicSpecification#extension_dir after compilation (during normal installation), which
+  # is in $LOAD_PATH but not necessarily relative to this file
+  # (see https://github.com/sparklemotion/nokogiri/issues/2300 for more)
+  require "rcee/precompiled/precompiled"
+end
+
+module RCEE
+  module Precompiled
+    class Error < StandardError; end
+    # Your code goes here...
+  end
+end

data/rcee_precompiled.gemspec

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+begin
+  require_relative "lib/rcee/precompiled/version"
+rescue LoadError
+  puts "WARNING: Could not load RCEE::Precompiled::VERSION"
+end
+
+Gem::Specification.new do |spec|
+  spec.name          = "rcee_precompiled"
+  spec.version       = defined?(RCEE::Precompiled::VERSION) ? RCEE::Precompiled::VERSION : "0.0.0"
+  spec.authors       = ["Mike Dalessio"]
+  spec.email         = ["mike.dalessio@gmail.com"]
+
+  spec.summary       = "Example gem demonstrating a basic C extension."
+  spec.description   = "Part of a project to explain how Ruby C extensions work."
+  spec.homepage      = "https://github.com/flavorjones/ruby-c-extensions-explained"
+  spec.required_ruby_version = ">= 2.6.0"
+  spec.license = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = [
+    ".gitignore",
+    "Gemfile",
+    "README.md",
+    "Rakefile",
+    "ext/precompiled/extconf.rb",
+    "ext/precompiled/precompiled.c",
+    "ext/precompiled/precompiled.h",
+    "lib/rcee/precompiled.rb",
+    "lib/rcee/precompiled/version.rb",
+    "ports/archives/yaml-0.2.5.tar.gz",
+    "rcee_precompiled.gemspec",
+  ]
+
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+  spec.extensions    = ["ext/precompiled/extconf.rb"]
+
+  spec.add_dependency "mini_portile2"
+
+  # For more information and examples about making a new gem, checkout our
+  # guide at: https://bundler.io/guides/creating_gem.html
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: rcee_precompiled
+version: !ruby/object:Gem::Version
+  version: 0.4.0
+platform: x86-linux
+authors:
+- Mike Dalessio
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2022-05-19 00:00:00.000000000 Z
+dependencies: []
+description: Part of a project to explain how Ruby C extensions work.
+email:
+- mike.dalessio@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- ext/precompiled/extconf.rb
+- ext/precompiled/precompiled.c
+- ext/precompiled/precompiled.h
+- lib/rcee/precompiled.rb
+- lib/rcee/precompiled/2.6/precompiled.so
+- lib/rcee/precompiled/2.7/precompiled.so
+- lib/rcee/precompiled/3.0/precompiled.so
+- lib/rcee/precompiled/3.1/precompiled.so
+- lib/rcee/precompiled/version.rb
+- rcee_precompiled.gemspec
+homepage: https://github.com/flavorjones/ruby-c-extensions-explained
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.6'
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: 3.2.dev
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.4
+signing_key:
+specification_version: 4
+summary: Example gem demonstrating a basic C extension.
+test_files: []