ruby_wasm 2.5.0 → 2.5.2

data/lib/ruby_wasm/packager/component_adapter.rb

@@ -0,0 +1,14 @@
+module RubyWasm::Packager::ComponentAdapter
+  module_function
+
+  # The path to the component adapter for the given WASI execution model.
+  #
+  # @param exec_model [String] "command" or "reactor"
+  def wasi_snapshot_preview1(exec_model)
+    File.join(
+      File.dirname(__FILE__),
+      "component_adapter",
+      "wasi_snapshot_preview1.#{exec_model}.wasm"
+    )
+  end
+end

data/lib/ruby_wasm/packager/core.rb

@@ -12,7 +12,7 @@ class RubyWasm::Packager::Core
 
   extend Forwardable
 
-  def_delegators :build_strategy, :cache_key, :artifact
+  def_delegators :build_strategy, :cache_key, :artifact, :build_and_link_exts
 
   private
 
@@ -20,7 +20,7 @@ class RubyWasm::Packager::Core
     @build_strategy ||=
       begin
         has_exts = @packager.specs.any? { |spec| spec.extensions.any? }
-        if @packager.support_dynamic_linking?
+        if @packager.features.support_dynamic_linking?
           DynamicLinking.new(@packager)
         else
           StaticLinking.new(@packager)
@@ -37,6 +37,10 @@ class RubyWasm::Packager::Core
       raise NotImplementedError
     end
 
+    def build_and_link_exts(executor, module_bytes)
+      raise NotImplementedError
+    end
+
     # Array of paths to extconf.rb files.
     def specs_with_extensions
       @packager.specs.filter_map do |spec|
@@ -61,6 +65,168 @@ class RubyWasm::Packager::Core
   end
 
   class DynamicLinking < BuildStrategy
+    def build(executor, options)
+      build = derive_build
+      force_rebuild =
+        options[:remake] || options[:clean] || options[:reconfigure]
+      if File.exist?(build.crossruby.artifact) && !force_rebuild
+        # Always build extensions because they are usually not expensive to build
+        return build.crossruby.artifact
+      end
+      build.crossruby.clean(executor) if options[:clean]
+
+      do_build =
+        proc do
+          build.crossruby.build(
+            executor,
+            remake: options[:remake],
+            reconfigure: options[:reconfigure]
+          )
+        end
+
+      __skip__ =
+        if defined?(Bundler)
+          Bundler.with_unbundled_env(&do_build)
+        else
+          do_build.call
+        end
+      build.crossruby.artifact
+    end
+
+    def build_and_link_exts(executor, module_bytes)
+      build = derive_build
+      self.build_exts(executor, build)
+      self.link_exts(executor, build)
+    end
+
+    def link_exts(executor, build)
+      ruby_root = build.crossruby.dest_dir
+
+      libraries = [File.join(ruby_root, "usr", "local", "bin", "ruby")]
+
+      # TODO: Should be computed from dyinfo of ruby binary
+      wasi_libc_shared_libs = [
+        "libc.so",
+        "libwasi-emulated-getpid.so",
+        "libwasi-emulated-mman.so",
+        "libwasi-emulated-process-clocks.so",
+        "libwasi-emulated-signal.so",
+      ]
+
+      wasi_libc_shared_libs.each do |lib|
+        # @type var toolchain: RubyWasm::WASISDK
+        toolchain = build.toolchain
+        wasi_sdk_path = toolchain.wasi_sdk_path
+        libraries << File.join(wasi_sdk_path, "share/wasi-sysroot/lib/wasm32-wasi", lib)
+      end
+      wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1("command")
+      adapters = [wasi_adapter]
+      dl_openable_libs = Dir.glob(File.join(ruby_root, "usr", "local", "lib", "ruby", "**", "*.so"))
+      linker = RubyWasmExt::ComponentLink.new
+      linker.use_built_in_libdl(true)
+      linker.stub_missing_functions(false)
+      linker.validate(true)
+
+      libraries.each do |lib|
+        # Non-DL openable libraries should be referenced as base name
+        lib_name = File.basename(lib)
+        module_bytes = File.binread(lib)
+        RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
+        linker.library(lib_name, module_bytes, false)
+      end
+
+      dl_openable_libs.each do |lib|
+        # DL openable lib_name should be a relative path from ruby_root
+        lib_name = "/" + Pathname.new(lib).relative_path_from(Pathname.new(ruby_root)).to_s
+        module_bytes = File.binread(lib)
+        RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
+        linker.library(lib_name, module_bytes, true)
+      end
+
+      adapters.each do |adapter|
+        adapter_name = File.basename(adapter)
+        # e.g. wasi_snapshot_preview1.command.wasm -> wasi_snapshot_preview1
+        adapter_name = adapter_name.split(".")[0]
+        module_bytes = File.binread(adapter)
+        linker.adapter(adapter_name, module_bytes)
+      end
+      return linker.encode()
+    end
+
+    def build_exts(executor, build)
+      exts = specs_with_extensions.flat_map do |spec, exts|
+        exts.map do |ext|
+          ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape"
+          ext_srcdir = File.join(spec.full_gem_path, ext_feature)
+          ext_relative_path = File.join(spec.full_name, ext_feature)
+          RubyWasm::CrossRubyExtProduct.new(
+            ext_srcdir,
+            build.toolchain,
+            features: @packager.features,
+            ext_relative_path: ext_relative_path
+          )
+        end
+      end
+
+      exts.each do |prod|
+        executor.begin_section prod.class, prod.name, "Building"
+        prod.build(executor, build.crossruby)
+        executor.end_section prod.class, prod.name
+      end
+    end
+
+    def cache_key(digest)
+      derive_build.cache_key(digest)
+    end
+
+    def artifact
+      derive_build.crossruby.artifact
+    end
+
+    def target
+      RubyWasm::Target.new(@packager.full_build_options[:target], pic: true)
+    end
+
+    def derive_build
+      return @build if @build
+      __skip__ =
+        build ||= RubyWasm::Build.new(
+          name, **@packager.full_build_options,
+          target: target,
+          # NOTE: We don't need linking libwasi_vfs because we use wasi-virt instead.
+          wasi_vfs: nil
+        )
+      build.crossruby.cflags = %w[-fPIC -fvisibility=default]
+      if @packager.full_build_options[:target] != "wasm32-unknown-emscripten"
+        build.crossruby.debugflags = %w[-g]
+        build.crossruby.wasmoptflags = %w[-O3 -g --pass-arg=asyncify-relocatable]
+        build.crossruby.ldflags = %w[
+          -Xlinker
+          --stack-first
+          -Xlinker
+          -z
+          -Xlinker
+          stack-size=16777216
+        ]
+        build.crossruby.xldflags = %w[
+          -Xlinker -shared
+          -Xlinker --export-dynamic
+          -Xlinker --export-all
+          -Xlinker --experimental-pic
+          -Xlinker -export-if-defined=__main_argc_argv
+        ]
+      end
+      @build = build
+      build
+    end
+
+    def name
+      require "digest"
+      options = @packager.full_build_options
+      src_channel = options[:src][:name]
+      target_triplet = options[:target]
+      "ruby-#{src_channel}-#{target_triplet}-pic#{options[:suffix]}"
+    end
   end
 
   class StaticLinking < BuildStrategy
@@ -93,16 +259,24 @@ class RubyWasm::Packager::Core
 
     def cache_key(digest)
       derive_build.cache_key(digest)
+      if enabled = @packager.features.support_component_model?
+        digest << enabled.to_s
+      end
     end
 
     def artifact
       derive_build.crossruby.artifact
     end
 
+    def target
+      RubyWasm::Target.new(@packager.full_build_options[:target])
+    end
+
     def derive_build
       return @build if @build
-      __skip__ =
-        build ||= RubyWasm::Build.new(name, **@packager.full_build_options)
+      __skip__ = build ||= RubyWasm::Build.new(
+        name, **@packager.full_build_options, target: target,
+      )
       build.crossruby.user_exts = user_exts(build)
       # Emscripten uses --global-base=1024 by default, but it conflicts with
       # --stack-first and -z stack-size since global-base 1024 is smaller than
@@ -111,7 +285,9 @@ class RubyWasm::Packager::Core
       # script of Ruby.
       if @packager.full_build_options[:target] != "wasm32-unknown-emscripten"
         build.crossruby.debugflags = %w[-g]
-        build.crossruby.wasmoptflags = %w[-O3 -g]
+        # We assume that imported functions provided through WASI will not change
+        # asyncify state, so we ignore them.
+        build.crossruby.wasmoptflags = %w[-O3 -g --pass-arg=asyncify-ignore-imports]
         build.crossruby.ldflags = %w[
           -Xlinker
           --stack-first
@@ -125,6 +301,20 @@ class RubyWasm::Packager::Core
       build
     end
 
+    def build_and_link_exts(executor, module_bytes)
+      return module_bytes unless @packager.features.support_component_model?
+
+      linker = RubyWasmExt::ComponentEncode.new
+      linker.validate(true)
+      linker.module(module_bytes)
+      linker.adapter(
+        "wasi_snapshot_preview1",
+        File.binread(RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1("reactor"))
+      )
+
+      linker.encode()
+    end
+
     def user_exts(build)
       @user_exts ||=
         specs_with_extensions.flat_map do |spec, exts|
@@ -135,6 +325,7 @@ class RubyWasm::Packager::Core
             RubyWasm::CrossRubyExtProduct.new(
               ext_srcdir,
               build.toolchain,
+              features: @packager.features,
               ext_relative_path: ext_relative_path
             )
           end
@@ -150,6 +341,9 @@ class RubyWasm::Packager::Core
       exts = specs_with_extensions.sort
       hash = ::Digest::MD5.new
       specs_with_extensions.each { |spec, _| hash << spec.full_name }
+      if enabled = @packager.features.support_component_model?
+        hash << enabled.to_s
+      end
       exts.empty? ? base : "#{base}-#{hash.hexdigest}"
     end
   end

data/lib/ruby_wasm/packager/file_system.rb

@@ -65,12 +65,14 @@ class RubyWasm::Packager::FileSystem
   end
 
   def remove_non_runtime_files(executor)
-    %w[
-      **/*.so
+    patterns = %w[
       usr/local/lib/libruby-static.a
       usr/local/bin/ruby
       usr/local/include
-    ].each do |pattern|
+    ]
+
+    patterns << "**/*.so" unless @packager.features.support_dynamic_linking?
+    patterns.each do |pattern|
       Dir
         .glob(File.join(@dest_dir, pattern))
         .each do |entry|

data/lib/ruby_wasm/packager.rb

@@ -2,11 +2,19 @@
 class RubyWasm::Packager
   # Initializes a new instance of the RubyWasm::Packager class.
   #
+  # @param root [String] The root directory of the Ruby project.
+  #   The root directory (will) contain the following files:
+  #    * build_manifest.json
+  #    * rubies
+  #    * build
   # @param config [Hash] The build config used for building Ruby.
   # @param definition [Bundler::Definition] The Bundler definition.
-  def initialize(config = nil, definition = nil)
+  # @param features [RubyWasm::FeatureSet] The features used for packaging.
+  def initialize(root, config = nil, definition = nil, features: RubyWasm::FeatureSet.derive_from_env)
+    @root = root
     @definition = definition
     @config = config
+    @features = features
   end
 
   # Packages the Ruby code into a Wasm binary. (including extensions)
@@ -22,14 +30,13 @@ class RubyWasm::Packager
     fs = RubyWasm::Packager::FileSystem.new(dest_dir, self)
     fs.package_ruby_root(tarball, executor)
 
-    ruby_wasm_bin = File.expand_path("bin/ruby", fs.ruby_root)
-    wasm_bytes = File.binread(ruby_wasm_bin).bytes
+    wasm_bytes = File.binread(File.join(fs.ruby_root, "bin", "ruby"))
 
     fs.package_gems
     fs.remove_non_runtime_files(executor)
     fs.remove_stdlib(executor) unless options[:stdlib]
 
-    if full_build_options[:target] == "wasm32-unknown-wasi"
+    if full_build_options[:target] == "wasm32-unknown-wasip1"
       # wasi-vfs supports only WASI target
       wasi_vfs = RubyWasmExt::WasiVfs.new
       wasi_vfs.map_dir("/bundle", fs.bundle_dir)
@@ -37,6 +44,7 @@ class RubyWasm::Packager
 
       wasm_bytes = wasi_vfs.pack(wasm_bytes)
     end
+    wasm_bytes = ruby_core.build_and_link_exts(executor, wasm_bytes)
 
     wasm_bytes = RubyWasmExt.preinitialize(wasm_bytes) if options[:optimize]
     wasm_bytes
@@ -52,79 +60,22 @@ class RubyWasm::Packager
   # Retrieves the specs from the Bundler definition, excluding the excluded gems.
   def specs
     return [] unless @definition
-    @definition.specs.reject { |spec| EXCLUDED_GEMS.include?(spec.name) }
+    @specs ||= @definition.resolve.materialize(@definition.requested_dependencies)
+      .reject { |spec| EXCLUDED_GEMS.include?(spec.name) }
+    @specs
   end
 
-  # Checks if dynamic linking is supported.
-  def support_dynamic_linking?
-    @ruby_channel == "head"
-  end
-
-  # Retrieves the root directory of the Ruby project.
-  # The root directory contains the following stuff:
-  #  * patches/*.patch
-  #  * build_manifest.json
-  #  * rubies
-  #  * build
-  def root
-    __skip__ =
-      @root ||=
-        begin
-          if explicit = ENV["RUBY_WASM_ROOT"]
-            File.expand_path(explicit)
-          elsif defined?(Bundler)
-            Bundler.root
-          else
-            Dir.pwd
-          end
-        rescue Bundler::GemfileNotFound
-          Dir.pwd
-        end
-  end
-
-  # Retrieves the alias definitions for the Ruby sources.
-  def self.build_source_aliases(root)
-    patches = Dir[File.join(root, "patches", "*.patch")]
-    sources = {
-      "head" => {
-        type: "github",
-        repo: "ruby/ruby",
-        rev: "master",
-        patches: patches.map { |p| File.expand_path(p) }
-      },
-      "3.3" => {
-        type: "tarball",
-        url: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0.tar.gz"
-      },
-      "3.2" => {
-        type: "tarball",
-        url: "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.3.tar.gz"
-      }
-    }
-    sources.each { |name, source| source[:name] = name }
-
-    build_manifest = File.join(root, "build_manifest.json")
-    if File.exist?(build_manifest)
-      begin
-        manifest = JSON.parse(File.read(build_manifest))
-        manifest["ruby_revisions"].each do |name, rev|
-          sources[name][:rev] = rev
-        end
-      rescue StandardError => e
-        RubyWasm.logger.warn "Failed to load build_manifest.json: #{e}"
-      end
-    end
-    sources
+  def features
+    @features
   end
 
   ALL_DEFAULT_EXTS =
-    "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl"
+    "cgi/escape,continuation,coverage,date,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,json,json/generator,json/parser,objspace,pathname,psych,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl"
 
   # Retrieves the build options used for building Ruby itself.
   def build_options
     default = {
-      target: "wasm32-unknown-wasi",
-      src: "3.3",
+      target: RubyWasm::Target.new("wasm32-unknown-wasip1"),
       default_exts: ALL_DEFAULT_EXTS
     }
     override = @config || {}
@@ -135,25 +86,14 @@ class RubyWasm::Packager
   # Retrieves the resolved build options
   def full_build_options
     options = build_options
-    build_dir = File.join(root, "build")
-    rubies_dir = File.join(root, "rubies")
+    build_dir = File.join(@root, "build")
+    rubies_dir = File.join(@root, "rubies")
     toolchain = RubyWasm::Toolchain.get(options[:target], build_dir)
-    src =
-      if options[:src].is_a?(Hash)
-        options[:src]
-      else
-        src_name = options[:src]
-        aliases = self.class.build_source_aliases(root)
-        aliases[src_name] ||
-          raise(
-            "Unknown Ruby source: #{src_name} (available: #{aliases.keys.join(", ")})"
-          )
-      end
     options.merge(
       toolchain: toolchain,
       build_dir: build_dir,
       rubies_dir: rubies_dir,
-      src: src
+      src: options[:src]
     )
   end
 end

data/lib/ruby_wasm/rake_task.rb

@@ -16,6 +16,7 @@ class RubyWasm::BuildTask < ::Rake::TaskLib
     **options,
     &block
   )
+    target = Target.new(target)
     @build =
       RubyWasm::Build.new(
         name,

data/lib/ruby_wasm/version.rb

@@ -1,3 +1,3 @@
 module RubyWasm
-  VERSION = "2.5.0"
+  VERSION = "2.5.2"
 end

data/lib/ruby_wasm.rb

@@ -3,7 +3,9 @@ require "logger"
 require_relative "ruby_wasm/version"
 require_relative "ruby_wasm/util"
 require_relative "ruby_wasm/build"
+require_relative "ruby_wasm/feature_set"
 require_relative "ruby_wasm/packager"
+require_relative "ruby_wasm/packager/component_adapter"
 require_relative "ruby_wasm/packager/file_system"
 require_relative "ruby_wasm/packager/core"