ruby_wasm 2.5.0.pre.1

data/lib/ruby_wasm/packager/core.rb

@@ -0,0 +1,156 @@
+require "forwardable"
+
+class RubyWasm::Packager::Core
+  def initialize(packager)
+    @packager = packager
+  end
+
+  def build(executor, options)
+    strategy = build_strategy
+    strategy.build(executor, options)
+  end
+
+  extend Forwardable
+
+  def_delegators :build_strategy, :cache_key, :artifact
+
+  private
+
+  def build_strategy
+    @build_strategy ||=
+      begin
+        has_exts = @packager.specs.any? { |spec| spec.extensions.any? }
+        if @packager.support_dynamic_linking?
+          DynamicLinking.new(@packager)
+        else
+          StaticLinking.new(@packager)
+        end
+      end
+  end
+
+  class BuildStrategy
+    def initialize(packager)
+      @packager = packager
+    end
+
+    def build(executor, options)
+      raise NotImplementedError
+    end
+
+    # Array of paths to extconf.rb files.
+    def specs_with_extensions
+      @packager.specs.filter_map do |spec|
+        exts =
+          spec.extensions.select do |ext|
+            # Filter out extensions of default gems (e.g. json, openssl)
+            # for the exactly same gem version.
+            File.exist?(File.join(spec.full_gem_path, ext))
+          end
+        next nil if exts.empty?
+        [spec, exts]
+      end
+    end
+
+    def cache_key(digest)
+      raise NotImplementedError
+    end
+
+    def artifact
+      raise NotImplementedError
+    end
+  end
+
+  class DynamicLinking < BuildStrategy
+  end
+
+  class StaticLinking < BuildStrategy
+    def build(executor, options)
+      build = derive_build
+      force_rebuild =
+        options[:remake] || options[:clean] || options[:reconfigure]
+      if File.exist?(build.crossruby.artifact) && !force_rebuild
+        return build.crossruby.artifact
+      end
+      build.crossruby.clean(executor) if options[:clean]
+
+      do_build =
+        proc do
+          build.crossruby.build(
+            executor,
+            remake: options[:remake],
+            reconfigure: options[:reconfigure]
+          )
+        end
+
+      __skip__ =
+        if defined?(Bundler)
+          Bundler.with_unbundled_env(&do_build)
+        else
+          do_build.call
+        end
+      build.crossruby.artifact
+    end
+
+    def cache_key(digest)
+      derive_build.cache_key(digest)
+    end
+
+    def artifact
+      derive_build.crossruby.artifact
+    end
+
+    def derive_build
+      return @build if @build
+      __skip__ =
+        build ||= RubyWasm::Build.new(name, **@packager.full_build_options)
+      build.crossruby.user_exts = user_exts(build)
+      # Emscripten uses --global-base=1024 by default, but it conflicts with
+      # --stack-first and -z stack-size since global-base 1024 is smaller than
+      # the large stack size.
+      # Also -g produces some warnings on Emscripten and it confuses the configure
+      # script of Ruby.
+      if @packager.full_build_options[:target] != "wasm32-unknown-emscripten"
+        build.crossruby.debugflags = %w[-g]
+        build.crossruby.wasmoptflags = %w[-O3 -g]
+        build.crossruby.ldflags = %w[
+          -Xlinker
+          --stack-first
+          -Xlinker
+          -z
+          -Xlinker
+          stack-size=16777216
+        ]
+      end
+      @build = build
+      build
+    end
+
+    def user_exts(build)
+      @user_exts ||=
+        specs_with_extensions.flat_map do |spec, exts|
+          exts.map do |ext|
+            ext_feature = File.dirname(ext) # e.g. "ext/cgi/escape"
+            ext_srcdir = File.join(spec.full_gem_path, ext_feature)
+            ext_relative_path = File.join(spec.full_name, ext_feature)
+            RubyWasm::CrossRubyExtProduct.new(
+              ext_srcdir,
+              build.toolchain,
+              ext_relative_path: ext_relative_path
+            )
+          end
+        end
+    end
+
+    def name
+      require "digest"
+      options = @packager.full_build_options
+      src_channel = options[:src][:name]
+      target_triplet = options[:target]
+      base = "ruby-#{src_channel}-#{target_triplet}#{options[:suffix]}"
+      exts = specs_with_extensions.sort
+      hash = ::Digest::MD5.new
+      specs_with_extensions.each { |spec, _| hash << spec.full_name }
+      exts.empty? ? base : "#{base}-#{hash.hexdigest}"
+    end
+  end
+end

data/lib/ruby_wasm/packager/file_system.rb

@@ -0,0 +1,157 @@
+# Package Ruby code into a mountable directory.
+class RubyWasm::Packager::FileSystem
+  def initialize(dest_dir, packager)
+    @dest_dir = dest_dir
+    @packager = packager
+    @ruby_root = File.join(@dest_dir, "usr", "local")
+  end
+
+  def package_ruby_root(tarball, executor)
+    usr_dir = File.dirname(@ruby_root)
+    executor.mkdir_p usr_dir
+    executor.system(
+      "tar",
+      "-C",
+      usr_dir,
+      "-xzf",
+      tarball,
+      "--strip-components=2"
+    )
+  end
+
+  def remove_stdlib(executor)
+    # Include only rbconfig.rb
+    rbconfig =
+      File.join(
+        @ruby_root,
+        "lib",
+        "ruby",
+        ruby_version,
+        "wasm32-wasi",
+        "rbconfig.rb"
+      )
+    # Remove all files except rbconfig.rb
+    RubyWasm.logger.info "Removing stdlib (except rbconfig.rb: #{rbconfig})"
+    rbconfig_contents = File.read(rbconfig)
+    executor.rm_rf @ruby_root
+    executor.mkdir_p File.dirname(rbconfig)
+    File.write(rbconfig, rbconfig_contents)
+  end
+
+  def package_gems
+    @packager.specs.each do |spec|
+      RubyWasm.logger.info "Packaging gem: #{spec.full_name}"
+    end
+    self.each_gem_content_path do |relative, source|
+      RubyWasm.logger.debug "Packaging gem file: #{relative}"
+      dest = File.join(@dest_dir, relative)
+      FileUtils.mkdir_p File.dirname(dest)
+      FileUtils.cp_r source, dest
+    end
+
+    setup_rb_path = File.join(bundle_relative_path, "setup.rb")
+    RubyWasm.logger.info "Packaging setup.rb: #{setup_rb_path}"
+    full_setup_rb_path = File.join(@dest_dir, setup_rb_path)
+    FileUtils.mkdir_p File.dirname(full_setup_rb_path)
+    File.write(full_setup_rb_path, setup_rb_content)
+  end
+
+  def setup_rb_content
+    content = ""
+    self.each_gem_require_path do |relative, _|
+      content << %Q[$:.unshift File.expand_path("#{File.join("/", relative)}")\n]
+    end
+    content
+  end
+
+  def remove_non_runtime_files(executor)
+    %w[
+      **/*.so
+      usr/local/lib/libruby-static.a
+      usr/local/bin/ruby
+      usr/local/include
+    ].each do |pattern|
+      Dir
+        .glob(File.join(@dest_dir, pattern))
+        .each do |entry|
+          RubyWasm.logger.debug do
+            relative_entry = Pathname.new(entry).relative_path_from(@dest_dir)
+            "Removing non-runtime file: #{relative_entry}"
+          end
+          executor.rm_rf entry
+        end
+    end
+  end
+
+  def bundle_dir
+    File.join(@dest_dir, bundle_relative_path)
+  end
+
+  def ruby_root
+    @ruby_root
+  end
+
+  private
+
+  # Iterates over each gem's require path and extension path.
+  # Yields the installation relative path and the source path.
+  def each_gem_require_path(&block)
+    each_gem_extension_path(&block)
+    @packager.specs.each do |spec|
+      spec.raw_require_paths.each do |require_path|
+        source = File.expand_path(require_path, spec.full_gem_path)
+        next unless File.exist?(source)
+        relative =
+          File.join(bundle_relative_path, "gems", spec.full_name, require_path)
+        yield relative, source
+      end
+    end
+  end
+
+  def each_gem_content_path(&block)
+    each_gem_extension_path(&block)
+
+    @packager.specs.each do |spec|
+      next unless File.exist?(spec.full_gem_path)
+
+      Dir
+        .children(spec.full_gem_path)
+        .each do |require_path|
+          source = File.expand_path(require_path, spec.full_gem_path)
+          next unless File.exist?(source)
+          relative =
+            File.join(
+              bundle_relative_path,
+              "gems",
+              spec.full_name,
+              require_path
+            )
+          yield relative, source
+        end
+    end
+  end
+
+  def each_gem_extension_path
+    @packager.specs.each do |spec|
+      if !spec.extensions.empty? && File.exist?(spec.extension_dir)
+        relative = File.join(bundle_relative_path, "extensions", spec.full_name)
+        yield relative, spec.extension_dir
+      end
+    end
+  end
+
+  def bundle_relative_path
+    "bundle"
+  end
+
+  def ruby_version
+    rubyarchdir = self.rubyarchdir
+    File.basename(File.dirname(rubyarchdir))
+  end
+
+  def rubyarchdir
+    maybe =
+      Dir.glob(File.join(@ruby_root, "lib", "ruby", "*", "wasm32-wasi")).first
+    maybe || raise("Cannot find rubyarchdir")
+  end
+end

data/lib/ruby_wasm/packager.rb

@@ -0,0 +1,159 @@
+# A class responsible for packaging whole Ruby project
+class RubyWasm::Packager
+  # Initializes a new instance of the RubyWasm::Packager class.
+  #
+  # @param config [Hash] The build config used for building Ruby.
+  # @param definition [Bundler::Definition] The Bundler definition.
+  def initialize(config = nil, definition = nil)
+    @definition = definition
+    @config = config
+  end
+
+  # Packages the Ruby code into a Wasm binary. (including extensions)
+  #
+  # @param executor [RubyWasm::BuildExecutor] The executor for building the Wasm binary.
+  # @param dest_dir [String] The destination used to construct the filesystem.
+  # @param options [Hash] The packaging options.
+  # @return [Array<Integer>] The bytes of the packaged Wasm binary.
+  def package(executor, dest_dir, options)
+    ruby_core = self.ruby_core_build()
+    tarball = ruby_core.build(executor, options)
+
+    fs = RubyWasm::Packager::FileSystem.new(dest_dir, self)
+    fs.package_ruby_root(tarball, executor)
+
+    ruby_wasm_bin = File.expand_path("bin/ruby", fs.ruby_root)
+    wasm_bytes = File.binread(ruby_wasm_bin).bytes
+
+    fs.package_gems
+    fs.remove_non_runtime_files(executor)
+    fs.remove_stdlib(executor) unless options[:stdlib]
+
+    if full_build_options[:target] == "wasm32-unknown-wasi"
+      # wasi-vfs supports only WASI target
+      wasi_vfs = RubyWasmExt::WasiVfs.new
+      wasi_vfs.map_dir("/bundle", fs.bundle_dir)
+      wasi_vfs.map_dir("/usr", File.dirname(fs.ruby_root))
+
+      wasm_bytes = wasi_vfs.pack(wasm_bytes)
+    end
+
+    wasm_bytes = RubyWasmExt.preinitialize(wasm_bytes) if options[:optimize]
+    wasm_bytes
+  end
+
+  def ruby_core_build
+    @ruby_core_build ||= RubyWasm::Packager::Core.new(self)
+  end
+
+  # The list of excluded gems from the Bundler definition.
+  EXCLUDED_GEMS = %w[ruby_wasm bundler]
+
+  # Retrieves the specs from the Bundler definition, excluding the excluded gems.
+  def specs
+    return [] unless @definition
+    @definition.specs.reject { |spec| EXCLUDED_GEMS.include?(spec.name) }
+  end
+
+  # Checks if dynamic linking is supported.
+  def support_dynamic_linking?
+    @ruby_channel == "head"
+  end
+
+  # Retrieves the root directory of the Ruby project.
+  # The root directory contains the following stuff:
+  #  * patches/*.patch
+  #  * build_manifest.json
+  #  * rubies
+  #  * build
+  def root
+    __skip__ =
+      @root ||=
+        begin
+          if explicit = ENV["RUBY_WASM_ROOT"]
+            File.expand_path(explicit)
+          elsif defined?(Bundler)
+            Bundler.root
+          else
+            Dir.pwd
+          end
+        rescue Bundler::GemfileNotFound
+          Dir.pwd
+        end
+  end
+
+  # Retrieves the alias definitions for the Ruby sources.
+  def self.build_source_aliases(root)
+    patches = Dir[File.join(root, "patches", "*.patch")]
+    sources = {
+      "head" => {
+        type: "github",
+        repo: "ruby/ruby",
+        rev: "997124fc0b7697bb9383e8feb8e1d88017c4bcb8",
+        patches: patches.map { |p| File.expand_path(p) }
+      },
+      "3.3" => {
+        type: "tarball",
+        url: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0.tar.gz"
+      },
+      "3.2" => {
+        type: "tarball",
+        url: "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.2.tar.gz"
+      }
+    }
+    sources.each { |name, source| source[:name] = name }
+
+    build_manifest = File.join(root, "build_manifest.json")
+    if File.exist?(build_manifest)
+      begin
+        manifest = JSON.parse(File.read(build_manifest))
+        manifest["ruby_revisions"].each do |name, rev|
+          sources[name][:rev] = rev
+        end
+      rescue StandardError => e
+        RubyWasm.logger.warn "Failed to load build_manifest.json: #{e}"
+      end
+    end
+    sources
+  end
+
+  ALL_DEFAULT_EXTS =
+    "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl"
+
+  # Retrieves the build options used for building Ruby itself.
+  def build_options
+    default = {
+      target: "wasm32-unknown-wasi",
+      src: "3.3",
+      default_exts: ALL_DEFAULT_EXTS
+    }
+    override = @config || {}
+    # Merge the default options with the config options
+    default.merge(override)
+  end
+
+  # Retrieves the resolved build options
+  def full_build_options
+    options = build_options
+    build_dir = File.join(root, "build")
+    rubies_dir = File.join(root, "rubies")
+    toolchain = RubyWasm::Toolchain.get(options[:target], build_dir)
+    src =
+      if options[:src].is_a?(Hash)
+        options[:src]
+      else
+        src_name = options[:src]
+        aliases = self.class.build_source_aliases(root)
+        aliases[src_name] ||
+          raise(
+            "Unknown Ruby source: #{src_name} (available: #{aliases.keys.join(", ")})"
+          )
+      end
+    options.merge(
+      toolchain: toolchain,
+      build_dir: build_dir,
+      rubies_dir: rubies_dir,
+      src: src
+    )
+  end
+end

data/lib/ruby_wasm/rake_task.rb

@@ -0,0 +1,58 @@
+require "rake/tasklib"
+require_relative "./build"
+
+class RubyWasm::BuildTask < ::Rake::TaskLib
+  # Name of the task.
+  attr_accessor :name
+
+  def initialize(
+    name,
+    target:,
+    src:,
+    toolchain: nil,
+    build_dir: nil,
+    rubies_dir: nil,
+    **options,
+    &block
+  )
+    @build =
+      RubyWasm::Build.new(
+        name,
+        target: target,
+        src: src,
+        toolchain: toolchain,
+        build_dir: build_dir || File.join(Dir.pwd, "build"),
+        rubies_dir: rubies_dir || File.join(Dir.pwd, "rubies"),
+        **options,
+        &block
+      )
+    yield @build if block_given?
+    @crossruby = @build.crossruby
+    # Rake.verbose can be Object.new by default, so compare with true explicitly.
+    executor = RubyWasm::BuildExecutor.new(verbose: Rake.verbose == true)
+
+    desc "Cross-build Ruby for #{@target}"
+    task name do
+      next if File.exist? @crossruby.artifact
+      @crossruby.build(executor)
+    end
+    namespace name do
+      task :remake do
+        @crossruby.build(executor, remake: true)
+      end
+      task :reconfigure do
+        @crossruby.build(executor, reconfigure: true)
+      end
+      task :clean do
+        @crossruby.clean(executor)
+      end
+    end
+  end
+
+  def hexdigest
+    require "digest"
+    digest = Digest::SHA256.new
+    @build.cache_key(digest)
+    digest.hexdigest
+  end
+end

data/lib/ruby_wasm/util.rb

@@ -0,0 +1,15 @@
+module RubyWasm
+  module SizeFormatter
+    def format(size)
+      units = %w[B KB MB GB TB]
+      unit = 0
+      while size > 1024 and unit < units.size - 1
+        size /= 1024.0
+        unit += 1
+      end
+      "%s #{units[unit]}" % size.round(2)
+    end
+
+    module_function :format
+  end
+end

data/lib/ruby_wasm/version.rb

@@ -0,0 +1,3 @@
+module RubyWasm
+  VERSION = "2.5.0.pre.1"
+end

data/lib/ruby_wasm.rb

@@ -0,0 +1,33 @@
+require "logger"
+
+require_relative "ruby_wasm/version"
+require_relative "ruby_wasm/util"
+require_relative "ruby_wasm/build"
+require_relative "ruby_wasm/packager"
+require_relative "ruby_wasm/packager/file_system"
+require_relative "ruby_wasm/packager/core"
+
+module RubyWasm
+  class << self
+    attr_accessor :log_level
+
+    def logger
+      @logger ||=
+        begin
+          logger =
+            Logger.new(
+              $stderr,
+              level: @log_level || Logger::INFO,
+              progname: "rbwasm"
+            )
+          logger.formatter =
+            proc { |severity, datetime, progname, msg| "#{severity}: #{msg}\n" }
+          logger
+        end
+    end
+
+    def logger=(logger)
+      @logger = logger
+    end
+  end
+end