ruby_wasm 2.5.0.pre.1 → 2.5.1

data/lib/ruby_wasm/cli.rb

@@ -9,7 +9,7 @@ module RubyWasm
     end
 
     def run(args)
-      available_commands = %w[build]
+      available_commands = %w[build pack]
       parser =
         OptionParser.new do |opts|
           opts.banner = <<~USAGE
@@ -32,14 +32,17 @@ module RubyWasm
       case command
       when "build"
         build(args)
+      when "pack"
+        pack(args)
       else
+        @stderr.puts "Unknown command: #{command}"
         @stderr.puts parser
         exit
       end
     end
 
     def build(args)
-      # @type var options: Hash[Symbol, untyped]
+      # @type var options: cli_options
       options = {
         save_temps: false,
         optimize: false,
@@ -47,14 +50,16 @@ module RubyWasm
         reconfigure: false,
         clean: false,
         ruby_version: "3.3",
-        target_triplet: "wasm32-unknown-wasi",
+        target_triplet: "wasm32-unknown-wasip1",
         profile: "full",
         stdlib: true,
-        disable_gems: false
+        disable_gems: false,
+        gemfile: nil,
+        patches: [],
       }
       OptionParser
         .new do |opts|
-          opts.banner = "Usage: rbwasm componentize [options]"
+          opts.banner = "Usage: rbwasm build [options]"
           opts.on("-h", "--help", "Prints this help") do
             @stdout.puts opts
             exit
@@ -103,6 +108,10 @@ module RubyWasm
             options[:disable_gems] = true
           end
 
+          opts.on("-p", "--patch PATCH", "Apply a patch") do |patch|
+            options[:patches] << patch
+          end
+
           opts.on("--format FORMAT", "Output format") do |format|
             options[:format] = format
           end
@@ -113,8 +122,19 @@ module RubyWasm
         end
         .parse!(args)
 
+      __skip__ = if defined?(Bundler)
+        Bundler.settings.temporary(force_ruby_platform: true) do
+          do_build_with_force_ruby_platform(options)
+        end
+      else
+        do_build_with_force_ruby_platform(options)
+      end
+    end
+
+    def do_build_with_force_ruby_platform(options)
       verbose = RubyWasm.logger.level == :debug
       executor = RubyWasm::BuildExecutor.new(verbose: verbose)
+
       packager = self.derive_packager(options)
 
       if options[:print_ruby_cache_key]
@@ -141,13 +161,24 @@ module RubyWasm
       end
     end
 
+    def pack(args)
+      self.require_extension
+      RubyWasmExt::WasiVfs.run_cli([$0, "pack", *args])
+    end
+
     private
 
     def build_config(options)
-      config = { target: options[:target_triplet], src: options[:ruby_version] }
+      build_source, all_default_exts = compute_build_source(options)
+      # @type var config: Packager::build_config
+      config = { target: options[:target_triplet], src: build_source }
       case options[:profile]
       when "full"
-        config[:default_exts] = RubyWasm::Packager::ALL_DEFAULT_EXTS
+        config[:default_exts] = all_default_exts || ""
+        env_additional_exts = ENV["RUBY_WASM_ADDITIONAL_EXTS"] || ""
+        unless env_additional_exts.empty?
+          config[:default_exts] += "," + env_additional_exts
+        end
       when "minimal"
         config[:default_exts] = ""
       else
@@ -158,12 +189,129 @@ module RubyWasm
       config
     end
 
-    def derive_packager(options)
+    def compute_build_source(options)
+      src_name = options[:ruby_version]
+      aliases = self.class.build_source_aliases(root)
+      source = aliases[src_name]
+      if source.nil?
+        if File.directory?(src_name)
+          # Treat as a local source if the given name is a source directory.
+          RubyWasm.logger.debug "Using local source: #{src_name}"
+          if options[:patches].any?
+            RubyWasm.logger.warn "Patches specified through --patch are ignored for local sources"
+          end
+          # @type var local_source: RubyWasm::Packager::build_source_local
+          local_source = { type: "local", path: src_name }
+          # @type var local_source: RubyWasm::Packager::build_source
+          local_source = local_source.merge(name: "local", patches: [])
+          return [local_source, nil]
+        end
+        # Otherwise, it's an unknown source.
+        raise(
+          "Unknown Ruby source: #{src_name} (available: #{aliases.keys.join(", ")} or a local directory)"
+        )
+      end
+      # Apply user-specified patches in addition to bundled patches.
+      source[:patches].concat(options[:patches])
+      # @type var all_default_exts: String
+      __skip__ = all_default_exts = source[:all_default_exts]
+      [source, all_default_exts]
+    end
+
+    # Retrieves the alias definitions for the Ruby sources.
+    def self.build_source_aliases(root)
+      # @type var sources: Hash[string, RubyWasm::Packager::build_source]
+      sources = {
+        "head" => {
+          type: "github",
+          repo: "ruby/ruby",
+          rev: "master",
+          all_default_exts: RubyWasm::Packager::ALL_DEFAULT_EXTS,
+        },
+        "3.3" => {
+          type: "tarball",
+          url: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0.tar.gz",
+          all_default_exts: "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl",
+        },
+        "3.2" => {
+          type: "tarball",
+          url: "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.3.tar.gz",
+          all_default_exts: "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl",
+        }
+      }
+
+      # Apply bundled and user-specified `<root>/patches` directories.
+      sources.each do |name, source|
+        source[:name] = name
+        patches_dirs = [bundled_patches_path, File.join(root, "patches")]
+        source[:patches] = patches_dirs.flat_map do |patches_dir|
+          Dir[File.join(patches_dir, name, "*.patch")]
+            .map { |p| File.expand_path(p) }
+        end
+      end
+
+      build_manifest = File.join(root, "build_manifest.json")
+      if File.exist?(build_manifest)
+        begin
+          manifest = JSON.parse(File.read(build_manifest))
+          manifest["ruby_revisions"].each do |name, rev|
+            source = sources[name]
+            next unless source[:type] == "github"
+            # @type var source: RubyWasm::Packager::build_source_github
+            source[:rev] = rev
+          end
+        rescue StandardError => e
+          RubyWasm.logger.warn "Failed to load build_manifest.json: #{e}"
+        end
+      end
+      sources
+    end
+
+    # Retrieves the root directory of the Ruby project.
+    def root
       __skip__ =
-        if defined?(Bundler) && !options[:disable_gems]
+        @root ||=
+          begin
+            if explicit = ENV["RUBY_WASM_ROOT"]
+              File.expand_path(explicit)
+            elsif defined?(Bundler)
+              Bundler.root
+            else
+              Dir.pwd
+            end
+          rescue Bundler::GemfileNotFound
+            Dir.pwd
+          end
+    end
+
+    # Path to the directory containing the bundled patches, which is shipped
+    # as part of ruby_wasm gem to backport fixes or try experimental features
+    # before landing them to the ruby/ruby repository.
+    def self.bundled_patches_path
+      dir = __dir__
+      raise "Unexpected directory structure, no __dir__!??" unless dir
+      lib_source_root = File.join(dir, "..", "..")
+      File.join(lib_source_root, "patches")
+    end
+
+    def derive_packager(options)
+      __skip__ = definition = nil
+      __skip__ = if defined?(Bundler) && !options[:disable_gems]
+        begin
+          # Silence Bundler UI if --print-ruby-cache-key is specified not to bother the JSON output.
+          level = options[:print_ruby_cache_key] ? :silent : Bundler.ui.level
+          old_level = Bundler.ui.level
+          Bundler.ui.level = level
           definition = Bundler.definition
+        ensure
+          Bundler.ui.level = old_level
         end
-      RubyWasm::Packager.new(build_config(options), definition)
+      end
+      RubyWasm.logger.info "Using Gemfile: #{definition.gemfiles}" if definition
+      RubyWasm::Packager.new(
+        root, build_config(options), definition,
+        features: RubyWasm::FeatureSet.derive_from_env
+      )
     end
 
     def do_print_ruby_cache_key(packager)
@@ -180,16 +328,26 @@ module RubyWasm
     end
 
     def do_build(executor, tmpdir, packager, options)
-      require_relative "ruby_wasm.so"
+      self.require_extension
       wasm_bytes = packager.package(executor, tmpdir, options)
       RubyWasm.logger.info "Size: #{SizeFormatter.format(wasm_bytes.size)}"
       case options[:output]
       when "-"
-        @stdout.write wasm_bytes.pack("C*")
+        @stdout.write wasm_bytes
       else
-        File.binwrite(options[:output], wasm_bytes.pack("C*"))
+        File.binwrite(options[:output], wasm_bytes)
         RubyWasm.logger.debug "Wrote #{options[:output]}"
       end
     end
+
+    def require_extension
+      # Tries to require the extension for the given Ruby version first
+      begin
+        RUBY_VERSION =~ /(\d+\.\d+)/
+        require_relative "#{Regexp.last_match(1)}/ruby_wasm.so"
+      rescue LoadError
+        require_relative "ruby_wasm.so"
+      end
+    end
   end
 end

data/lib/ruby_wasm/feature_set.rb

@@ -0,0 +1,30 @@
+##
+# A set of feature flags that can be used to enable or disable experimental features.
+class RubyWasm::FeatureSet
+  def initialize(features)
+    @features = features
+  end
+
+  # Maps the feature to the environment variable.
+  FEATURES = {
+    dynamic_linking: "RUBY_WASM_EXPERIMENTAL_DYNAMIC_LINKING",
+    component_model: "RUBY_WASM_EXPERIMENTAL_COMPONENT_MODEL",
+  }.freeze
+  private_constant :FEATURES
+
+  # Derives the feature set from the environment variables. A feature
+  # is enabled if the corresponding environment variable is set to "1",
+  # otherwise it is disabled.
+  def self.derive_from_env
+    values = FEATURES.transform_values { |key| ENV[key] == "1" }
+    new(values)
+  end
+
+  def support_dynamic_linking?
+    @features[:dynamic_linking]
+  end
+
+  def support_component_model?
+    @features[:component_model] || @features[:dynamic_linking]
+  end
+end

data/lib/ruby_wasm/packager/component_adapter.rb

@@ -0,0 +1,14 @@
+module RubyWasm::Packager::ComponentAdapter
+  module_function
+
+  # The path to the component adapter for the given WASI execution model.
+  #
+  # @param exec_model [String] "command" or "reactor"
+  def wasi_snapshot_preview1(exec_model)
+    File.join(
+      File.dirname(__FILE__),
+      "component_adapter",
+      "wasi_snapshot_preview1.#{exec_model}.wasm"
+    )
+  end
+end

data/lib/ruby_wasm/packager/core.rb

@@ -12,7 +12,7 @@ class RubyWasm::Packager::Core
 
   extend Forwardable
 
-  def_delegators :build_strategy, :cache_key, :artifact
+  def_delegators :build_strategy, :cache_key, :artifact, :build_and_link_exts
 
   private
 
@@ -20,7 +20,7 @@ class RubyWasm::Packager::Core
     @build_strategy ||=
       begin
         has_exts = @packager.specs.any? { |spec| spec.extensions.any? }
-        if @packager.support_dynamic_linking?
+        if @packager.features.support_dynamic_linking?
           DynamicLinking.new(@packager)
         else
           StaticLinking.new(@packager)
@@ -37,6 +37,10 @@ class RubyWasm::Packager::Core
       raise NotImplementedError
     end
 
+    def build_and_link_exts(executor)
+      raise NotImplementedError
+    end
+
     # Array of paths to extconf.rb files.
     def specs_with_extensions
       @packager.specs.filter_map do |spec|
@@ -61,6 +65,167 @@ class RubyWasm::Packager::Core
   end
 
   class DynamicLinking < BuildStrategy
+    def build(executor, options)
+      build = derive_build
+      force_rebuild =
+        options[:remake] || options[:clean] || options[:reconfigure]
+      if File.exist?(build.crossruby.artifact) && !force_rebuild
+        # Always build extensions because they are usually not expensive to build
+        return build.crossruby.artifact
+      end
+      build.crossruby.clean(executor) if options[:clean]
+
+      do_build =
+        proc do
+          build.crossruby.build(
+            executor,
+            remake: options[:remake],
+            reconfigure: options[:reconfigure]
+          )
+        end
+
+      __skip__ =
+        if defined?(Bundler)
+          Bundler.with_unbundled_env(&do_build)
+        else
+          do_build.call
+        end
+      build.crossruby.artifact
+    end
+
+    def build_and_link_exts(executor)
+      build = derive_build
+      self.build_exts(executor, build)
+      self.link_exts(executor, build)
+    end
+
+    def link_exts(executor, build)
+      ruby_root = build.crossruby.dest_dir
+
+      libraries = [File.join(ruby_root, "usr", "local", "bin", "ruby")]
+
+      # TODO: Should be computed from dyinfo of ruby binary
+      wasi_libc_shared_libs = [
+        "libc.so",
+        "libwasi-emulated-getpid.so",
+        "libwasi-emulated-mman.so",
+        "libwasi-emulated-process-clocks.so",
+        "libwasi-emulated-signal.so",
+      ]
+
+      wasi_libc_shared_libs.each do |lib|
+        # @type var toolchain: RubyWasm::WASISDK
+        toolchain = build.toolchain
+        wasi_sdk_path = toolchain.wasi_sdk_path
+        libraries << File.join(wasi_sdk_path, "share/wasi-sysroot/lib/wasm32-wasi", lib)
+      end
+      wasi_adapter = RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1("command")
+      adapters = [wasi_adapter]
+      dl_openable_libs = Dir.glob(File.join(ruby_root, "usr", "local", "lib", "ruby", "**", "*.so"))
+      linker = RubyWasmExt::ComponentLink.new
+      linker.use_built_in_libdl(true)
+      linker.stub_missing_functions(false)
+      linker.validate(true)
+
+      libraries.each do |lib|
+        # Non-DL openable libraries should be referenced as base name
+        lib_name = File.basename(lib)
+        module_bytes = File.binread(lib)
+        RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
+        linker.library(lib_name, module_bytes, false)
+      end
+
+      dl_openable_libs.each do |lib|
+        # DL openable lib_name should be a relative path from ruby_root
+        lib_name = "/" + Pathname.new(lib).relative_path_from(Pathname.new(ruby_root)).to_s
+        module_bytes = File.binread(lib)
+        RubyWasm.logger.info "Linking #{lib_name} (#{module_bytes.size} bytes)"
+        linker.library(lib_name, module_bytes, true)
+      end
+
+      adapters.each do |adapter|
+        adapter_name = File.basename(adapter)
+        # e.g. wasi_snapshot_preview1.command.wasm -> wasi_snapshot_preview1
+        adapter_name = adapter_name.split(".")[0]
+        module_bytes = File.binread(adapter)
+        linker.adapter(adapter_name, module_bytes)
+      end
+      return linker.encode()
+    end
+
+    def build_exts(executor, build)
+      exts = specs_with_extensions.flat_map do |spec, exts|
+        exts.map do |ext|
+          ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape"
+          ext_srcdir = File.join(spec.full_gem_path, ext_feature)
+          ext_relative_path = File.join(spec.full_name, ext_feature)
+          RubyWasm::CrossRubyExtProduct.new(
+            ext_srcdir,
+            build.toolchain,
+            ext_relative_path: ext_relative_path
+          )
+        end
+      end
+
+      exts.each do |prod|
+        executor.begin_section prod.class, prod.name, "Building"
+        prod.build(executor, build.crossruby)
+        executor.end_section prod.class, prod.name
+      end
+    end
+
+    def cache_key(digest)
+      derive_build.cache_key(digest)
+    end
+
+    def artifact
+      derive_build.crossruby.artifact
+    end
+
+    def target
+      RubyWasm::Target.new(@packager.full_build_options[:target], pic: true)
+    end
+
+    def derive_build
+      return @build if @build
+      __skip__ =
+        build ||= RubyWasm::Build.new(
+          name, **@packager.full_build_options,
+          target: target,
+          # NOTE: We don't need linking libwasi_vfs because we use wasi-virt instead.
+          wasi_vfs: nil
+        )
+      build.crossruby.cflags = %w[-fPIC -fvisibility=default]
+      if @packager.full_build_options[:target] != "wasm32-unknown-emscripten"
+        build.crossruby.debugflags = %w[-g]
+        build.crossruby.wasmoptflags = %w[-O3 -g --pass-arg=asyncify-relocatable]
+        build.crossruby.ldflags = %w[
+          -Xlinker
+          --stack-first
+          -Xlinker
+          -z
+          -Xlinker
+          stack-size=16777216
+        ]
+        build.crossruby.xldflags = %w[
+          -Xlinker -shared
+          -Xlinker --export-dynamic
+          -Xlinker --export-all
+          -Xlinker --experimental-pic
+          -Xlinker -export-if-defined=__main_argc_argv
+        ]
+      end
+      @build = build
+      build
+    end
+
+    def name
+      require "digest"
+      options = @packager.full_build_options
+      src_channel = options[:src][:name]
+      target_triplet = options[:target]
+      "ruby-#{src_channel}-#{target_triplet}-pic#{options[:suffix]}"
+    end
   end
 
   class StaticLinking < BuildStrategy
@@ -99,10 +264,14 @@ class RubyWasm::Packager::Core
       derive_build.crossruby.artifact
     end
 
+    def target
+      RubyWasm::Target.new(@packager.full_build_options[:target])
+    end
+
     def derive_build
       return @build if @build
       __skip__ =
-        build ||= RubyWasm::Build.new(name, **@packager.full_build_options)
+        build ||= RubyWasm::Build.new(name, **@packager.full_build_options, target: target)
       build.crossruby.user_exts = user_exts(build)
       # Emscripten uses --global-base=1024 by default, but it conflicts with
       # --stack-first and -z stack-size since global-base 1024 is smaller than
@@ -111,7 +280,9 @@ class RubyWasm::Packager::Core
       # script of Ruby.
       if @packager.full_build_options[:target] != "wasm32-unknown-emscripten"
         build.crossruby.debugflags = %w[-g]
-        build.crossruby.wasmoptflags = %w[-O3 -g]
+        # We assume that imported functions provided through WASI will not change
+        # asyncify state, so we ignore them.
+        build.crossruby.wasmoptflags = %w[-O3 -g --pass-arg=asyncify-ignore-imports]
         build.crossruby.ldflags = %w[
           -Xlinker
           --stack-first
@@ -125,6 +296,23 @@ class RubyWasm::Packager::Core
       build
     end
 
+    def build_and_link_exts(executor)
+      build = derive_build
+      ruby_root = build.crossruby.dest_dir
+      module_bytes = File.binread(File.join(ruby_root, "usr", "local", "bin", "ruby"))
+      return module_bytes unless @packager.features.support_component_model?
+
+      linker = RubyWasmExt::ComponentEncode.new
+      linker.validate(true)
+      linker.module(module_bytes)
+      linker.adapter(
+        "wasi_snapshot_preview1",
+        File.binread(RubyWasm::Packager::ComponentAdapter.wasi_snapshot_preview1("reactor"))
+      )
+
+      linker.encode()
+    end
+
     def user_exts(build)
       @user_exts ||=
         specs_with_extensions.flat_map do |spec, exts|

data/lib/ruby_wasm/packager/file_system.rb

@@ -65,12 +65,14 @@ class RubyWasm::Packager::FileSystem
   end
 
   def remove_non_runtime_files(executor)
-    %w[
-      **/*.so
+    patterns = %w[
       usr/local/lib/libruby-static.a
       usr/local/bin/ruby
       usr/local/include
-    ].each do |pattern|
+    ]
+
+    patterns << "**/*.so" unless @packager.features.support_dynamic_linking?
+    patterns.each do |pattern|
       Dir
         .glob(File.join(@dest_dir, pattern))
         .each do |entry|
@@ -98,6 +100,7 @@ class RubyWasm::Packager::FileSystem
   def each_gem_require_path(&block)
     each_gem_extension_path(&block)
     @packager.specs.each do |spec|
+      # Use raw_require_paths to exclude extensions
       spec.raw_require_paths.each do |require_path|
         source = File.expand_path(require_path, spec.full_gem_path)
         next unless File.exist?(source)
@@ -114,20 +117,20 @@ class RubyWasm::Packager::FileSystem
     @packager.specs.each do |spec|
       next unless File.exist?(spec.full_gem_path)
 
-      Dir
-        .children(spec.full_gem_path)
-        .each do |require_path|
-          source = File.expand_path(require_path, spec.full_gem_path)
-          next unless File.exist?(source)
-          relative =
-            File.join(
-              bundle_relative_path,
-              "gems",
-              spec.full_name,
-              require_path
-            )
-          yield relative, source
-        end
+      # spec.files is only valid before the gem is packaged.
+      if spec.source.path?
+        relative_paths = spec.files
+      else
+        # All files in .gem are required.
+        relative_paths = Dir.children(spec.full_gem_path)
+      end
+      relative_paths.each do |require_path|
+        source = File.expand_path(require_path, spec.full_gem_path)
+        next unless File.exist?(source)
+        relative =
+          File.join(bundle_relative_path, "gems", spec.full_name, require_path)
+        yield relative, source
+      end
     end
   end