ruby_wasm 2.5.0-x86_64-darwin → 2.5.1-x86_64-darwin

data/lib/ruby_wasm/cli.rb

@@ -42,7 +42,7 @@ module RubyWasm
     end
 
     def build(args)
-      # @type var options: Hash[Symbol, untyped]
+      # @type var options: cli_options
       options = {
         save_temps: false,
         optimize: false,
@@ -50,14 +50,16 @@ module RubyWasm
         reconfigure: false,
         clean: false,
         ruby_version: "3.3",
-        target_triplet: "wasm32-unknown-wasi",
+        target_triplet: "wasm32-unknown-wasip1",
         profile: "full",
         stdlib: true,
-        disable_gems: false
+        disable_gems: false,
+        gemfile: nil,
+        patches: [],
       }
       OptionParser
         .new do |opts|
-          opts.banner = "Usage: rbwasm componentize [options]"
+          opts.banner = "Usage: rbwasm build [options]"
           opts.on("-h", "--help", "Prints this help") do
             @stdout.puts opts
             exit
@@ -106,6 +108,10 @@ module RubyWasm
             options[:disable_gems] = true
           end
 
+          opts.on("-p", "--patch PATCH", "Apply a patch") do |patch|
+            options[:patches] << patch
+          end
+
           opts.on("--format FORMAT", "Output format") do |format|
             options[:format] = format
           end
@@ -116,8 +122,19 @@ module RubyWasm
         end
         .parse!(args)
 
+      __skip__ = if defined?(Bundler)
+        Bundler.settings.temporary(force_ruby_platform: true) do
+          do_build_with_force_ruby_platform(options)
+        end
+      else
+        do_build_with_force_ruby_platform(options)
+      end
+    end
+
+    def do_build_with_force_ruby_platform(options)
       verbose = RubyWasm.logger.level == :debug
       executor = RubyWasm::BuildExecutor.new(verbose: verbose)
+
       packager = self.derive_packager(options)
 
       if options[:print_ruby_cache_key]
@@ -152,10 +169,11 @@ module RubyWasm
     private
 
     def build_config(options)
-      config = { target: options[:target_triplet], src: options[:ruby_version] }
+      # @type var config: Packager::build_config
+      config = { target: options[:target_triplet], src: compute_build_source(options) }
       case options[:profile]
       when "full"
-        config[:default_exts] = RubyWasm::Packager::ALL_DEFAULT_EXTS
+        config[:default_exts] = config[:src][:all_default_exts]
         env_additional_exts = ENV["RUBY_WASM_ADDITIONAL_EXTS"] || ""
         unless env_additional_exts.empty?
           config[:default_exts] += "," + env_additional_exts
@@ -170,12 +188,124 @@ module RubyWasm
       config
     end
 
-    def derive_packager(options)
+    def compute_build_source(options)
+      src_name = options[:ruby_version]
+      aliases = self.class.build_source_aliases(root)
+      source = aliases[src_name]
+      if source.nil?
+        if File.directory?(src_name)
+          # Treat as a local source if the given name is a source directory.
+          RubyWasm.logger.debug "Using local source: #{src_name}"
+          if options[:patches].any?
+            RubyWasm.logger.warn "Patches specified through --patch are ignored for local sources"
+          end
+          # @type var local_source: RubyWasm::Packager::build_source_local
+          local_source = { type: "local", path: src_name }
+          # @type var local_source: RubyWasm::Packager::build_source
+          local_source = local_source.merge(name: "local", patches: [])
+          return local_source
+        end
+        # Otherwise, it's an unknown source.
+        raise(
+          "Unknown Ruby source: #{src_name} (available: #{aliases.keys.join(", ")} or a local directory)"
+        )
+      end
+      # Apply user-specified patches in addition to bundled patches.
+      source[:patches].concat(options[:patches])
+      source
+    end
+
+    # Retrieves the alias definitions for the Ruby sources.
+    def self.build_source_aliases(root)
+      # @type var sources: Hash[string, RubyWasm::Packager::build_source]
+      sources = {
+        "head" => {
+          type: "github",
+          repo: "ruby/ruby",
+          rev: "master",
+          all_default_exts: RubyWasm::Packager::ALL_DEFAULT_EXTS,
+        },
+        "3.3" => {
+          type: "tarball",
+          url: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0.tar.gz",
+          all_default_exts: "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl",
+        },
+        "3.2" => {
+          type: "tarball",
+          url: "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.3.tar.gz",
+          all_default_exts: "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl",
+        }
+      }
+
+      # Apply bundled and user-specified `<root>/patches` directories.
+      sources.each do |name, source|
+        source[:name] = name
+        patches_dirs = [bundled_patches_path, File.join(root, "patches")]
+        source[:patches] = patches_dirs.flat_map do |patches_dir|
+          Dir[File.join(patches_dir, name, "*.patch")]
+            .map { |p| File.expand_path(p) }
+        end
+      end
+
+      build_manifest = File.join(root, "build_manifest.json")
+      if File.exist?(build_manifest)
+        begin
+          manifest = JSON.parse(File.read(build_manifest))
+          manifest["ruby_revisions"].each do |name, rev|
+            source = sources[name]
+            next unless source[:type] == "github"
+            # @type var source: RubyWasm::Packager::build_source_github
+            source[:rev] = rev
+          end
+        rescue StandardError => e
+          RubyWasm.logger.warn "Failed to load build_manifest.json: #{e}"
+        end
+      end
+      sources
+    end
+
+    # Retrieves the root directory of the Ruby project.
+    def root
       __skip__ =
-        if defined?(Bundler) && !options[:disable_gems]
+        @root ||=
+          begin
+            if explicit = ENV["RUBY_WASM_ROOT"]
+              File.expand_path(explicit)
+            elsif defined?(Bundler)
+              Bundler.root
+            else
+              Dir.pwd
+            end
+          rescue Bundler::GemfileNotFound
+            Dir.pwd
+          end
+    end
+
+    # Path to the directory containing the bundled patches, which is shipped
+    # as part of ruby_wasm gem to backport fixes or try experimental features
+    # before landing them to the ruby/ruby repository.
+    def self.bundled_patches_path
+      dir = __dir__
+      raise "Unexpected directory structure, no __dir__!??" unless dir
+      lib_source_root = File.join(dir, "..", "..")
+      File.join(lib_source_root, "patches")
+    end
+
+    def derive_packager(options)
+      __skip__ = definition = nil
+      __skip__ = if defined?(Bundler) && !options[:disable_gems]
+        begin
+          # Silence Bundler UI if --print-ruby-cache-key is specified not to bother the JSON output.
+          level = options[:print_ruby_cache_key] ? :silent : Bundler.ui.level
+          old_level = Bundler.ui.level
+          Bundler.ui.level = level
           definition = Bundler.definition
+        ensure
+          Bundler.ui.level = old_level
         end
-      RubyWasm::Packager.new(build_config(options), definition)
+      end
+      RubyWasm.logger.info "Using Gemfile: #{definition.gemfiles}" if definition
+      RubyWasm::Packager.new(root, build_config(options), definition)
     end
 
     def do_print_ruby_cache_key(packager)
@@ -197,9 +327,9 @@ module RubyWasm
       RubyWasm.logger.info "Size: #{SizeFormatter.format(wasm_bytes.size)}"
       case options[:output]
       when "-"
-        @stdout.write wasm_bytes.pack("C*")
+        @stdout.write wasm_bytes
       else
-        File.binwrite(options[:output], wasm_bytes.pack("C*"))
+        File.binwrite(options[:output], wasm_bytes)
         RubyWasm.logger.debug "Wrote #{options[:output]}"
       end
     end

data/lib/ruby_wasm/packager/component_adapter.rb

@@ -0,0 +1,14 @@
+module RubyWasm::Packager::ComponentAdapter
+  module_function
+
+  # The path to the component adapter for the given WASI execution model.
+  #
+  # @param exec_model [String] "command" or "reactor"
+  def wasi_snapshot_preview1(exec_model)
+    File.join(
+      File.dirname(__FILE__),
+      "component_adapter",
+      "wasi_snapshot_preview1.#{exec_model}.wasm"
+    )
+  end
+end

data/lib/ruby_wasm/packager/core.rb

@@ -12,7 +12,7 @@ class RubyWasm::Packager::Core
 
   extend Forwardable
 
-  def_delegators :build_strategy, :cache_key, :artifact
+  def_delegators :build_strategy, :cache_key, :artifact, :build_and_link_exts
 
   private
 
@@ -37,6 +37,10 @@ class RubyWasm::Packager::Core
       raise NotImplementedError
     end
 
+    def build_and_link_exts(executor)
+      raise NotImplementedError
+    end
+
     # Array of paths to extconf.rb files.
     def specs_with_extensions
       @packager.specs.filter_map do |spec|
@@ -61,6 +65,167 @@ class RubyWasm::Packager::Core
   end
 
   class DynamicLinking < BuildStrategy
+    def build(executor, options)
+      build = derive_build
+      force_rebuild =
+        options[:remake] || options[:clean] || options[:reconfigure]
+      if File.exist?(build.crossruby.artifact) && !force_rebuild
+        # Always build extensions because they are usually not expensive to build
+        return build.crossruby.artifact
+      end
+      build.crossruby.clean(executor) if options[:clean]
+
+      do_build =
+        proc do
+          build.crossruby.build(
+            executor,
+            remake: options[:remake],
+            reconfigure: options[:reconfigure]
+          )
+        end
+
+      __skip__ =
+        if defined?(Bundler)
+          Bundler.with_unbundled_env(&do_build)
+        else
+          do_build.call
+        end
+      build.crossruby.artifact
+    end
+
+    def build_and_link_exts(executor)
+      build = derive_build
+      self.build_exts(executor, build)
+      self.link_exts(executor, build)
+    end
+
+    def link_exts(executor, build)
+      ruby_root = build.crossruby.dest_dir
+
+      libraries = [File.join(ruby_root, "usr", "local", "bin", "ruby")]
+
+      # TODO: Should be computed from dyinfo of ruby binary
+      wasi_libc_shared_libs = [
+        "libc.so",
+        "libwasi-emulated-getpid.so",
+        "libwasi-emulated-mman.so",
+        "libwasi-emulated-process-clocks.so",
+        "libwasi-emulated-signal.so",
+      ]
+
+      wasi_libc_shared_libs.each do |lib|
+        # @type var toolchain: RubyWasm::WASISDK
+        toolchain = build.toolchain
+        wasi_sdk_path = toolchain.wasi_sdk_path
+        libraries << File.join(wasi_sdk_path, "share/wasi-sysroot/lib/wasm32-wasi", lib)
+      end
+      wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1("command")
+      adapters = [wasi_adapter]
+      dl_openable_libs = Dir.glob(File.join(ruby_root, "usr", "local", "lib", "ruby", "**", "*.so"))
+      linker = RubyWasmExt::ComponentLink.new
+      linker.use_built_in_libdl(true)
+      linker.stub_missing_functions(false)
+      linker.validate(true)
+
+      libraries.each do |lib|
+        # Non-DL openable libraries should be referenced as base name
+        lib_name = File.basename(lib)
+        module_bytes = File.binread(lib)
+        RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
+        linker.library(lib_name, module_bytes, false)
+      end
+
+      dl_openable_libs.each do |lib|
+        # DL openable lib_name should be a relative path from ruby_root
+        lib_name = "/" + Pathname.new(lib).relative_path_from(Pathname.new(ruby_root)).to_s
+        module_bytes = File.binread(lib)
+        RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
+        linker.library(lib_name, module_bytes, true)
+      end
+
+      adapters.each do |adapter|
+        adapter_name = File.basename(adapter)
+        # e.g. wasi_snapshot_preview1.command.wasm -> wasi_snapshot_preview1
+        adapter_name = adapter_name.split(".")[0]
+        module_bytes = File.binread(adapter)
+        linker.adapter(adapter_name, module_bytes)
+      end
+      return linker.encode()
+    end
+
+    def build_exts(executor, build)
+      exts = specs_with_extensions.flat_map do |spec, exts|
+        exts.map do |ext|
+          ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape"
+          ext_srcdir = File.join(spec.full_gem_path, ext_feature)
+          ext_relative_path = File.join(spec.full_name, ext_feature)
+          RubyWasm::CrossRubyExtProduct.new(
+            ext_srcdir,
+            build.toolchain,
+            ext_relative_path: ext_relative_path
+          )
+        end
+      end
+
+      exts.each do |prod|
+        executor.begin_section prod.class, prod.name, "Building"
+        prod.build(executor, build.crossruby)
+        executor.end_section prod.class, prod.name
+      end
+    end
+
+    def cache_key(digest)
+      derive_build.cache_key(digest)
+    end
+
+    def artifact
+      derive_build.crossruby.artifact
+    end
+
+    def target
+      RubyWasm::Target.new(@packager.full_build_options[:target], pic: true)
+    end
+
+    def derive_build
+      return @build if @build
+      __skip__ =
+        build ||= RubyWasm::Build.new(
+          name, **@packager.full_build_options,
+          target: target,
+          # NOTE: We don't need linking libwasi_vfs because we use wasi-virt instead.
+          wasi_vfs: nil
+        )
+      build.crossruby.cflags = %w[-fPIC -fvisibility=default]
+      if @packager.full_build_options[:target] != "wasm32-unknown-emscripten"
+        build.crossruby.debugflags = %w[-g]
+        build.crossruby.wasmoptflags = %w[-O3 -g --pass-arg=asyncify-relocatable]
+        build.crossruby.ldflags = %w[
+          -Xlinker
+          --stack-first
+          -Xlinker
+          -z
+          -Xlinker
+          stack-size=16777216
+        ]
+        build.crossruby.xldflags = %w[
+          -Xlinker -shared
+          -Xlinker --export-dynamic
+          -Xlinker --export-all
+          -Xlinker --experimental-pic
+          -Xlinker -export-if-defined=__main_argc_argv
+        ]
+      end
+      @build = build
+      build
+    end
+
+    def name
+      require "digest"
+      options = @packager.full_build_options
+      src_channel = options[:src][:name]
+      target_triplet = options[:target]
+      "ruby-#{src_channel}-#{target_triplet}-pic#{options[:suffix]}"
+    end
   end
 
   class StaticLinking < BuildStrategy
@@ -99,10 +264,14 @@ class RubyWasm::Packager::Core
       derive_build.crossruby.artifact
     end
 
+    def target
+      RubyWasm::Target.new(@packager.full_build_options[:target])
+    end
+
     def derive_build
       return @build if @build
       __skip__ =
-        build ||= RubyWasm::Build.new(name, **@packager.full_build_options)
+        build ||= RubyWasm::Build.new(name, **@packager.full_build_options, target: target)
       build.crossruby.user_exts = user_exts(build)
       # Emscripten uses --global-base=1024 by default, but it conflicts with
       # --stack-first and -z stack-size since global-base 1024 is smaller than
@@ -111,7 +280,9 @@ class RubyWasm::Packager::Core
       # script of Ruby.
       if @packager.full_build_options[:target] != "wasm32-unknown-emscripten"
         build.crossruby.debugflags = %w[-g]
-        build.crossruby.wasmoptflags = %w[-O3 -g]
+        # We assume that imported functions provided through WASI will not change
+        # asyncify state, so we ignore them.
+        build.crossruby.wasmoptflags = %w[-O3 -g --pass-arg=asyncify-ignore-imports]
         build.crossruby.ldflags = %w[
           -Xlinker
           --stack-first
@@ -125,6 +296,12 @@ class RubyWasm::Packager::Core
       build
     end
 
+    def build_and_link_exts(executor)
+      build = derive_build
+      ruby_root = build.crossruby.dest_dir
+      File.binread(File.join(ruby_root, "usr", "local", "bin", "ruby"))
+    end
+
     def user_exts(build)
       @user_exts ||=
         specs_with_extensions.flat_map do |spec, exts|

data/lib/ruby_wasm/packager/file_system.rb

@@ -65,12 +65,14 @@ class RubyWasm::Packager::FileSystem
   end
 
   def remove_non_runtime_files(executor)
-    %w[
-      **/*.so
+    patterns = %w[
       usr/local/lib/libruby-static.a
       usr/local/bin/ruby
       usr/local/include
-    ].each do |pattern|
+    ]
+
+    patterns << "**/*.so" unless @packager.support_dynamic_linking?
+    patterns.each do |pattern|
       Dir
         .glob(File.join(@dest_dir, pattern))
         .each do |entry|

data/lib/ruby_wasm/packager.rb

@@ -2,9 +2,15 @@
 class RubyWasm::Packager
   # Initializes a new instance of the RubyWasm::Packager class.
   #
+  # @param root [String] The root directory of the Ruby project.
+  #   The root directory (will) contain the following files:
+  #    * build_manifest.json
+  #    * rubies
+  #    * build
   # @param config [Hash] The build config used for building Ruby.
   # @param definition [Bundler::Definition] The Bundler definition.
-  def initialize(config = nil, definition = nil)
+  def initialize(root, config = nil, definition = nil)
+    @root = root
     @definition = definition
     @config = config
   end
@@ -21,15 +27,13 @@ class RubyWasm::Packager
 
     fs = RubyWasm::Packager::FileSystem.new(dest_dir, self)
     fs.package_ruby_root(tarball, executor)
-
-    ruby_wasm_bin = File.expand_path("bin/ruby", fs.ruby_root)
-    wasm_bytes = File.binread(ruby_wasm_bin).bytes
+    wasm_bytes = ruby_core.build_and_link_exts(executor)
 
     fs.package_gems
     fs.remove_non_runtime_files(executor)
     fs.remove_stdlib(executor) unless options[:stdlib]
 
-    if full_build_options[:target] == "wasm32-unknown-wasi"
+    if full_build_options[:target] == "wasm32-unknown-wasip1" && !support_dynamic_linking?
       # wasi-vfs supports only WASI target
       wasi_vfs = RubyWasmExt::WasiVfs.new
       wasi_vfs.map_dir("/bundle", fs.bundle_dir)
@@ -52,79 +56,23 @@ class RubyWasm::Packager
   # Retrieves the specs from the Bundler definition, excluding the excluded gems.
   def specs
     return [] unless @definition
-    @definition.specs.reject { |spec| EXCLUDED_GEMS.include?(spec.name) }
+    @specs ||= @definition.resolve.materialize(@definition.requested_dependencies)
+      .reject { |spec| EXCLUDED_GEMS.include?(spec.name) }
+    @specs
   end
 
   # Checks if dynamic linking is supported.
   def support_dynamic_linking?
-    @ruby_channel == "head"
-  end
-
-  # Retrieves the root directory of the Ruby project.
-  # The root directory contains the following stuff:
-  #  * patches/*.patch
-  #  * build_manifest.json
-  #  * rubies
-  #  * build
-  def root
-    __skip__ =
-      @root ||=
-        begin
-          if explicit = ENV["RUBY_WASM_ROOT"]
-            File.expand_path(explicit)
-          elsif defined?(Bundler)
-            Bundler.root
-          else
-            Dir.pwd
-          end
-        rescue Bundler::GemfileNotFound
-          Dir.pwd
-        end
-  end
-
-  # Retrieves the alias definitions for the Ruby sources.
-  def self.build_source_aliases(root)
-    patches = Dir[File.join(root, "patches", "*.patch")]
-    sources = {
-      "head" => {
-        type: "github",
-        repo: "ruby/ruby",
-        rev: "master",
-        patches: patches.map { |p| File.expand_path(p) }
-      },
-      "3.3" => {
-        type: "tarball",
-        url: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0.tar.gz"
-      },
-      "3.2" => {
-        type: "tarball",
-        url: "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.3.tar.gz"
-      }
-    }
-    sources.each { |name, source| source[:name] = name }
-
-    build_manifest = File.join(root, "build_manifest.json")
-    if File.exist?(build_manifest)
-      begin
-        manifest = JSON.parse(File.read(build_manifest))
-        manifest["ruby_revisions"].each do |name, rev|
-          sources[name][:rev] = rev
-        end
-      rescue StandardError => e
-        RubyWasm.logger.warn "Failed to load build_manifest.json: #{e}"
-      end
-    end
-    sources
+    ENV["RUBY_WASM_EXPERIMENTAL_DYNAMIC_LINKING"] == "1"
   end
 
   ALL_DEFAULT_EXTS =
-    "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl"
+    "cgi/escape,continuation,coverage,date,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,json,json/generator,json/parser,objspace,pathname,psych,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl"
 
   # Retrieves the build options used for building Ruby itself.
   def build_options
     default = {
-      target: "wasm32-unknown-wasi",
-      src: "3.3",
+      target: RubyWasm::Target.new("wasm32-unknown-wasip1"),
       default_exts: ALL_DEFAULT_EXTS
     }
     override = @config || {}
@@ -135,25 +83,14 @@ class RubyWasm::Packager
   # Retrieves the resolved build options
   def full_build_options
     options = build_options
-    build_dir = File.join(root, "build")
-    rubies_dir = File.join(root, "rubies")
+    build_dir = File.join(@root, "build")
+    rubies_dir = File.join(@root, "rubies")
     toolchain = RubyWasm::Toolchain.get(options[:target], build_dir)
-    src =
-      if options[:src].is_a?(Hash)
-        options[:src]
-      else
-        src_name = options[:src]
-        aliases = self.class.build_source_aliases(root)
-        aliases[src_name] ||
-          raise(
-            "Unknown Ruby source: #{src_name} (available: #{aliases.keys.join(", ")})"
-          )
-      end
     options.merge(
       toolchain: toolchain,
       build_dir: build_dir,
       rubies_dir: rubies_dir,
-      src: src
+      src: options[:src]
     )
   end
 end

data/lib/ruby_wasm/rake_task.rb

@@ -16,6 +16,7 @@ class RubyWasm::BuildTask < ::Rake::TaskLib
     **options,
     &block
   )
+    target = Target.new(target)
     @build =
       RubyWasm::Build.new(
         name,

data/lib/ruby_wasm/version.rb

@@ -1,3 +1,3 @@
 module RubyWasm
-  VERSION = "2.5.0"
+  VERSION = "2.5.1"
 end

data/lib/ruby_wasm.rb

@@ -4,6 +4,7 @@ require_relative "ruby_wasm/version"
 require_relative "ruby_wasm/util"
 require_relative "ruby_wasm/build"
 require_relative "ruby_wasm/packager"
+require_relative "ruby_wasm/packager/component_adapter"
 require_relative "ruby_wasm/packager/file_system"
 require_relative "ruby_wasm/packager/core"