ruby_wasm 2.5.0-arm64-darwin

data/lib/ruby_wasm/build/product/crossruby.rb

@@ -0,0 +1,330 @@
+require_relative "./product"
+require "json"
+
+module RubyWasm
+  class CrossRubyExtProduct < BuildProduct
+    attr_reader :name
+
+    def initialize(srcdir, toolchain, ext_relative_path: nil)
+      @srcdir, @toolchain = srcdir, toolchain
+      # ext_relative_path is relative path from build dir
+      # e.g. cgi-0.3.6/ext/cgi/escape
+      @ext_relative_path = ext_relative_path || File.basename(srcdir)
+      @name = @ext_relative_path
+    end
+
+    def product_build_dir(crossruby)
+      File.join(crossruby.ext_build_dir, @ext_relative_path)
+    end
+
+    def linklist(crossruby)
+      File.join(product_build_dir(crossruby), "link.filelist")
+    end
+
+    def metadata_json(crossruby)
+      File.join(product_build_dir(crossruby), "rbwasm.metadata.json")
+    end
+
+    def feature_name(crossruby)
+      metadata = JSON.parse(File.read(metadata_json(crossruby)))
+      metadata["target"]
+    end
+
+    def make_args(crossruby)
+      make_args = []
+      make_args << "CC=#{@toolchain.cc}"
+      make_args << "LD=#{@toolchain.ld}"
+      make_args << "AR=#{@toolchain.ar}"
+      make_args << "RANLIB=#{@toolchain.ranlib}"
+
+      make_args << "DESTDIR=#{crossruby.dest_dir}"
+      make_args
+    end
+
+    def build(executor, crossruby)
+      objdir = product_build_dir crossruby
+      executor.mkdir_p objdir
+      do_extconf executor, crossruby
+      executor.system "make",
+                      "-j#{executor.process_count}",
+                      "-C",
+                      "#{objdir}",
+                      *make_args(crossruby),
+                      "static"
+      # A ext can provide link args by link.filelist. It contains only built archive file by default.
+      unless File.exist?(linklist(crossruby))
+        executor.write(
+          linklist(crossruby),
+          Dir.glob("#{objdir}/*.a").join("\n")
+        )
+      end
+    end
+
+    def do_extconf(executor, crossruby)
+      objdir = product_build_dir crossruby
+      source = crossruby.source
+      extconf_args = [
+        "--disable=gems",
+        # HACK: top_srcdir is required to find ruby headers
+        "-e",
+        %Q($top_srcdir="#{source.src_dir}"),
+        # HACK: extout is required to find config.h
+        "-e",
+        %Q($extout="#{crossruby.build_dir}/.ext"),
+        # HACK: skip have_devel check since ruby is not installed yet
+        "-e",
+        "$have_devel = true",
+        # HACK: force static ext build by imitating extmk
+        "-e",
+        "$static = true; trace_var(:$static) {|v| $static = true }",
+        # HACK: $0 should be extconf.rb path due to mkmf source file detection
+        # and we want to insert some hacks before it. But -e and $0 cannot be
+        # used together, so we rewrite $0 in -e.
+        "-e",
+        %Q($0="#{@srcdir}/extconf.rb"),
+        "-e",
+        %Q(require_relative "#{@srcdir}/extconf.rb"),
+        # HACK: extract "$target" from extconf.rb to get a full target name
+        # like "cgi/escape" instead of "escape"
+        "-e",
+        %Q(require "json"; File.write("#{metadata_json(crossruby)}", JSON.dump({target: $target}))),
+        "-I#{crossruby.build_dir}"
+      ]
+      # Clear RUBYOPT to avoid loading unrelated bundle setup
+      executor.system crossruby.baseruby_path,
+                      *extconf_args,
+                      chdir: objdir,
+                      env: {
+                        "RUBYOPT" => ""
+                      }
+    end
+
+    def do_install_rb(executor, crossruby)
+      objdir = product_build_dir crossruby
+      executor.system "make", "-C", objdir, *make_args(crossruby), "install-rb"
+    end
+
+    def cache_key(digest)
+      digest << @name
+      # Compute hash value of files under srcdir
+      Dir
+        .glob("#{@srcdir}/**/*", File::FNM_DOTMATCH)
+        .each do |f|
+          next if File.directory?(f)
+          digest << f
+          digest << File.read(f)
+        end
+    end
+  end
+
+  class CrossRubyProduct < AutoconfProduct
+    attr_reader :source, :toolchain
+    attr_accessor :user_exts,
+                  :wasmoptflags,
+                  :cppflags,
+                  :cflags,
+                  :ldflags,
+                  :debugflags,
+                  :xcflags,
+                  :xldflags
+
+    def initialize(params, build_dir, rubies_dir, baseruby, source, toolchain)
+      @params = params
+      @rubies_dir = rubies_dir
+      @build_dir = build_dir
+      @baseruby = baseruby
+      @source = source
+      @toolchain = toolchain
+      @user_exts = []
+      @wasmoptflags = []
+      @cppflags = []
+      @cflags = []
+      @ldflags = []
+      @debugflags = []
+      @xcflags = []
+      @xldflags = []
+      super(@params.target, @toolchain)
+    end
+
+    def configure(executor, reconfigure: false)
+      if !File.exist?("#{build_dir}/Makefile") || reconfigure
+        args = configure_args(RbConfig::CONFIG["host"], toolchain)
+        executor.system source.configure_file, *args, chdir: build_dir
+      end
+      # NOTE: we need rbconfig.rb at configuration time to build user given extensions with mkmf
+      executor.system "make", "rbconfig.rb", chdir: build_dir
+    end
+
+    def build_exts(executor)
+      @user_exts.each do |prod|
+        executor.begin_section prod.class, prod.name, "Building"
+        prod.build(executor, self)
+        executor.end_section prod.class, prod.name
+      end
+    end
+
+    def build(executor, remake: false, reconfigure: false)
+      executor.mkdir_p dest_dir
+      executor.mkdir_p build_dir
+      @toolchain.install
+      [@source, @baseruby, @libyaml, @zlib, @openssl, @wasi_vfs].each do |prod|
+        executor.begin_section prod.class, prod.name, "Building"
+        prod.build(executor)
+        executor.end_section prod.class, prod.name
+      end
+      executor.begin_section self.class, name, "Configuring"
+      configure(executor, reconfigure: reconfigure)
+      executor.end_section self.class, name
+
+      build_exts(executor)
+
+      executor.begin_section self.class, name, "Building"
+      executor.mkdir_p File.dirname(extinit_obj)
+      executor.system "ruby",
+                      extinit_c_erb,
+                      *@user_exts.map { |ext| ext.feature_name(self) },
+                      "--cc",
+                      toolchain.cc,
+                      "--output",
+                      extinit_obj
+      install_dir = File.join(build_dir, "install")
+      if !File.exist?(install_dir) || remake || reconfigure
+        executor.system "make",
+                        "-j#{executor.process_count}",
+                        "install",
+                        "DESTDIR=#{install_dir}",
+                        chdir: build_dir
+      end
+
+      executor.rm_rf dest_dir
+      executor.cp_r install_dir, dest_dir
+      @user_exts.each { |ext| ext.do_install_rb(executor, self) }
+      executor.system "tar", "cfz", artifact, "-C", @rubies_dir, name
+
+      executor.end_section self.class, name
+    end
+
+    def clean(executor)
+      executor.rm_rf dest_dir
+      executor.rm_rf build_dir
+      executor.rm_rf ext_build_dir
+      executor.rm_f artifact
+    end
+
+    def name
+      @params.name
+    end
+
+    def cache_key(digest)
+      digest << @params.target
+      digest << @params.default_exts
+      @wasmoptflags.each { |f| digest << f }
+      @cppflags.each { |f| digest << f }
+      @cflags.each { |f| digest << f }
+      @ldflags.each { |f| digest << f }
+      @debugflags.each { |f| digest << f }
+      @xcflags.each { |f| digest << f }
+      @xldflags.each { |f| digest << f }
+      @user_exts.each { |ext| ext.cache_key(digest) }
+    end
+
+    def build_dir
+      File.join(@build_dir, @params.target, name)
+    end
+
+    def ext_build_dir
+      File.join(@build_dir, @params.target, name + "-ext")
+    end
+
+    def with_libyaml(libyaml)
+      @libyaml = libyaml
+    end
+
+    def with_zlib(zlib)
+      @zlib = zlib
+    end
+
+    def with_wasi_vfs(wasi_vfs)
+      @wasi_vfs = wasi_vfs
+    end
+
+    def with_openssl(openssl)
+      @openssl = openssl
+    end
+
+    def dest_dir
+      File.join(@rubies_dir, name)
+    end
+
+    def artifact
+      File.join(@rubies_dir, "#{name}.tar.gz")
+    end
+
+    def extinit_obj
+      "#{ext_build_dir}/extinit.o"
+    end
+
+    def extinit_c_erb
+      lib_root = File.expand_path("../../../../..", __FILE__)
+      File.join(lib_root, "ext", "extinit.c.erb")
+    end
+
+    def baseruby_path
+      File.join(@baseruby.install_dir, "bin/ruby")
+    end
+
+    def configure_args(build_triple, toolchain)
+      target = @params.target
+      default_exts = @params.default_exts
+
+      ldflags = @ldflags.dup
+      xldflags = @xldflags.dup
+
+      args = self.system_triplet_args + ["--build", build_triple]
+      args << "--with-static-linked-ext"
+      args << %Q(--with-ext=#{default_exts})
+      args << %Q(--with-libyaml-dir=#{@libyaml.install_root})
+      args << %Q(--with-zlib-dir=#{@zlib.install_root})
+      args << %Q(--with-openssl-dir=#{@openssl.install_root}) if @openssl
+      args << %Q(--with-baseruby=#{baseruby_path})
+
+      case target
+      when "wasm32-unknown-wasi"
+        xldflags << @wasi_vfs.lib_wasi_vfs_a if @wasi_vfs
+        # TODO: Find a way to force cast or update API
+        # @type var wasi_sdk_path: untyped
+        wasi_sdk_path = @toolchain
+        args << %Q(WASMOPT=#{wasi_sdk_path.wasm_opt})
+        args << %Q(WASI_SDK_PATH=#{wasi_sdk_path.wasi_sdk_path})
+      when "wasm32-unknown-emscripten"
+        ldflags.concat(%w[-s MODULARIZE=1])
+        env_emcc_ldflags = ENV["RUBY_WASM_EMCC_LDFLAGS"] || ""
+        unless env_emcc_ldflags.empty?
+          ldflags << env_emcc_ldflags
+        end
+      else
+        raise "unknown target: #{target}"
+      end
+
+      args.concat(self.tools_args)
+      (@user_exts || []).each { |lib| xldflags << "@#{lib.linklist(self)}" }
+      xldflags << extinit_obj
+
+      xcflags = @xcflags.dup
+      xcflags << "-DWASM_SETJMP_STACK_BUFFER_SIZE=24576"
+      xcflags << "-DWASM_FIBER_STACK_BUFFER_SIZE=24576"
+      xcflags << "-DWASM_SCAN_STACK_BUFFER_SIZE=24576"
+
+      args << %Q(LDFLAGS=#{ldflags.join(" ")})
+      args << %Q(XLDFLAGS=#{xldflags.join(" ")})
+      args << %Q(XCFLAGS=#{xcflags.join(" ")})
+      args << %Q(debugflags=#{@debugflags.join(" ")})
+      args << %Q(cppflags=#{@cppflags.join(" ")})
+      unless wasmoptflags.empty?
+        args << %Q(wasmoptflags=#{@wasmoptflags.join(" ")})
+      end
+      args << "--disable-install-doc"
+      args
+    end
+  end
+end

data/lib/ruby_wasm/build/product/libyaml.rb

@@ -0,0 +1,68 @@
+require_relative "./product"
+
+module RubyWasm
+  class LibYAMLProduct < AutoconfProduct
+    attr_reader :target
+
+    LIBYAML_VERSION = "0.2.5"
+
+    def initialize(build_dir, target, toolchain)
+      @build_dir = build_dir
+      @target = target
+      super(target, toolchain)
+    end
+
+    def product_build_dir
+      File.join(@build_dir, target, "yaml-#{LIBYAML_VERSION}")
+    end
+
+    def destdir
+      File.join(product_build_dir, "opt")
+    end
+
+    def install_root
+      File.join(destdir, "usr", "local")
+    end
+
+    def name
+      "libyaml-#{LIBYAML_VERSION}-#{target}"
+    end
+
+    def build(executor)
+      return if Dir.exist?(install_root)
+
+      executor.mkdir_p File.dirname(product_build_dir)
+      executor.rm_rf product_build_dir
+      executor.mkdir_p product_build_dir
+      tarball_path =
+        File.join(product_build_dir, "libyaml-#{LIBYAML_VERSION}.tar.gz")
+      executor.system "curl",
+                      "-o",
+                      tarball_path,
+                      "-L",
+                      "https://github.com/yaml/libyaml/releases/download/#{LIBYAML_VERSION}/yaml-#{LIBYAML_VERSION}.tar.gz"
+      executor.system "tar",
+                      "xzf",
+                      tarball_path,
+                      "-C",
+                      product_build_dir,
+                      "--strip-components=1"
+
+      # obtain the latest config.guess and config.sub for Emscripten and WASI triple support
+      executor.system "curl",
+                      "-o",
+                      "#{product_build_dir}/config/config.guess",
+                      "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD"
+      executor.system "curl",
+                      "-o",
+                      "#{product_build_dir}/config/config.sub",
+                      "https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD"
+
+      executor.system "./configure", *configure_args, chdir: product_build_dir
+      executor.system "make",
+                      "install",
+                      "DESTDIR=#{destdir}",
+                      chdir: product_build_dir
+    end
+  end
+end

data/lib/ruby_wasm/build/product/openssl.rb

@@ -0,0 +1,88 @@
+require_relative "./product"
+
+module RubyWasm
+  class OpenSSLProduct < AutoconfProduct
+    attr_reader :target
+
+    OPENSSL_VERSION = "3.2.0"
+
+    def initialize(build_dir, target, toolchain)
+      @build_dir = build_dir
+      @target = target
+      super(target, toolchain)
+    end
+
+    def product_build_dir
+      File.join(@build_dir, target, "openssl-#{OPENSSL_VERSION}")
+    end
+
+    def destdir
+      File.join(product_build_dir, "opt")
+    end
+
+    def install_root
+      File.join(destdir, "usr", "local")
+    end
+
+    def name
+      "openssl-#{OPENSSL_VERSION}-#{target}"
+    end
+
+    def configure_args
+      args = %w[
+        gcc
+        -static
+        -no-asm
+        -no-threads
+        -no-afalgeng
+        -no-ui-console
+        -no-tests
+        -no-sock
+        -no-dgram
+        --libdir=lib
+        -Wl,--allow-undefined
+      ]
+      if @target == "wasm32-unknown-wasi"
+        args.concat %w[
+                      -D_WASI_EMULATED_SIGNAL
+                      -D_WASI_EMULATED_PROCESS_CLOCKS
+                      -D_WASI_EMULATED_MMAN
+                      -D_WASI_EMULATED_GETPID
+                      -DNO_CHMOD
+                      -DHAVE_FORK=0
+                    ]
+      end
+      args + tools_args
+    end
+
+    def build(executor)
+      return if File.exist?(File.join(install_root, "lib", "libssl.a"))
+
+      executor.mkdir_p File.dirname(product_build_dir)
+      executor.rm_rf product_build_dir
+      executor.mkdir_p product_build_dir
+      tarball_path =
+        File.join(product_build_dir, "openssl-#{OPENSSL_VERSION}.tar.gz")
+      executor.system "curl",
+                      "-o",
+                      tarball_path,
+                      "-L",
+                      "https://www.openssl.org/source/openssl-#{OPENSSL_VERSION}.tar.gz"
+      executor.system "tar",
+                      "xzf",
+                      tarball_path,
+                      "-C",
+                      product_build_dir,
+                      "--strip-components=1"
+
+      executor.system "./Configure", *configure_args, chdir: product_build_dir
+      # Use "install_sw" instead of "install" because it tries to install docs and it's very slow.
+      # OpenSSL build system doesn't have well support for parallel build, so force -j1.
+      executor.system "make",
+                      "-j1",
+                      "install_dev",
+                      "DESTDIR=#{destdir}",
+                      chdir: product_build_dir
+    end
+  end
+end

data/lib/ruby_wasm/build/product/product.rb

@@ -0,0 +1,39 @@
+module RubyWasm
+  class BuildProduct
+    def name
+      raise NotImplementedError, "identifiable product name must be implemented"
+    end
+  end
+
+  class AutoconfProduct < BuildProduct
+    def initialize(target, toolchain)
+      @target = target
+      @toolchain = toolchain
+    end
+    def system_triplet_args
+      args = []
+      case @target
+      when "wasm32-unknown-wasi"
+        args.concat(%W[--host wasm32-wasi])
+      when "wasm32-unknown-emscripten"
+        args.concat(%W[--host wasm32-emscripten])
+      else
+        raise "unknown target: #{@target}"
+      end
+      args
+    end
+
+    def tools_args
+      args = []
+      args << "CC=#{@toolchain.cc}"
+      args << "LD=#{@toolchain.ld}"
+      args << "AR=#{@toolchain.ar}"
+      args << "RANLIB=#{@toolchain.ranlib}"
+      args
+    end
+
+    def configure_args
+      system_triplet_args + tools_args
+    end
+  end
+end

data/lib/ruby_wasm/build/product/ruby_source.rb

@@ -0,0 +1,103 @@
+require_relative "./product"
+
+module RubyWasm
+  class BuildSource < BuildProduct
+    def initialize(params, build_dir)
+      @params = params
+      @build_dir = build_dir
+    end
+
+    def name
+      @params[:name]
+    end
+
+    def cache_key(digest)
+      digest << @params[:type]
+      case @params[:type]
+      when "github"
+        digest << @params[:rev]
+      when "tarball"
+        digest << @params[:url]
+      when "local"
+        digest << File.mtime(@params[:src]).to_i.to_s
+      else
+        raise "unknown source type: #{@params[:type]}"
+      end
+    end
+
+    def src_dir
+      File.join(@build_dir, "checkouts", @params[:name])
+    end
+
+    def configure_file
+      File.join(src_dir, "configure")
+    end
+
+    def fetch(executor)
+      case @params[:type]
+      when "github"
+        repo_url = "https://github.com/#{@params[:repo]}.git"
+        executor.mkdir_p src_dir
+        executor.system "git", "init", chdir: src_dir
+        executor.system "git",
+                        "remote",
+                        "add",
+                        "origin",
+                        repo_url,
+                        chdir: src_dir
+        executor.system(
+          "git",
+          "fetch",
+          "--depth",
+          "1",
+          "origin",
+          "#{@params[:rev]}:origin/#{@params[:rev]}",
+          chdir: src_dir
+        )
+        executor.system(
+          "git",
+          "checkout",
+          "origin/#{@params[:rev]}",
+          chdir: src_dir
+        )
+      when "tarball"
+        executor.mkdir_p src_dir
+        tarball_path =
+          File.join(File.dirname(src_dir), File.basename(src_dir) + ".tar.gz")
+        executor.system("curl", "-L", "-o", tarball_path, @params[:url])
+        executor.system(
+          "tar",
+          "xf",
+          tarball_path,
+          "-C",
+          src_dir,
+          "--strip-components=1"
+        )
+      when "local"
+        executor.mkdir_p File.dirname(src_dir)
+        executor.cp_r @params[:src], src_dir
+      else
+        raise "unknown source type: #{@params[:type]}"
+      end
+      (@params[:patches] || []).each do |patch_path|
+        executor.system "patch", "-p1", patch_path, chdir: src_dir
+      end
+    end
+
+    def build(executor)
+      fetch(executor) unless File.exist?(src_dir)
+      unless File.exist?(configure_file)
+        executor.system "ruby",
+                        "tool/downloader.rb",
+                        "-d",
+                        "tool",
+                        "-e",
+                        "gnu",
+                        "config.guess",
+                        "config.sub",
+                        chdir: src_dir
+        executor.system "./autogen.sh", chdir: src_dir
+      end
+    end
+  end
+end

data/lib/ruby_wasm/build/product/wasi_vfs.rb

@@ -0,0 +1,45 @@
+require_relative "./product"
+
+module RubyWasm
+  class WasiVfsProduct < BuildProduct
+    WASI_VFS_VERSION = "0.5.0"
+
+    def initialize(build_dir)
+      @build_dir = build_dir
+      @need_fetch_lib = ENV["LIB_WASI_VFS_A"].nil?
+    end
+
+    def lib_product_build_dir
+      File.join(
+        @build_dir,
+        "wasm32-unknown-wasi",
+        "wasi-vfs-#{WASI_VFS_VERSION}"
+      )
+    end
+
+    def lib_wasi_vfs_a
+      ENV["LIB_WASI_VFS_A"] || File.join(lib_product_build_dir, "libwasi_vfs.a")
+    end
+
+    def name
+      "wasi-vfs-#{WASI_VFS_VERSION}-#{RbConfig::CONFIG["host"]}"
+    end
+
+    def build(executor)
+      return if !@need_fetch_lib || File.exist?(lib_wasi_vfs_a)
+      require "tmpdir"
+      lib_wasi_vfs_url =
+        "https://github.com/kateinoigakukun/wasi-vfs/releases/download/v#{WASI_VFS_VERSION}/libwasi_vfs-wasm32-unknown-unknown.zip"
+      Dir.mktmpdir do |tmpdir|
+        executor.system "curl",
+                        "-L",
+                        lib_wasi_vfs_url,
+                        "-o",
+                        "#{tmpdir}/libwasi_vfs.zip"
+        executor.system "unzip", "#{tmpdir}/libwasi_vfs.zip", "-d", tmpdir
+        executor.mkdir_p File.dirname(lib_wasi_vfs_a)
+        executor.mv File.join(tmpdir, "libwasi_vfs.a"), lib_wasi_vfs_a
+      end
+    end
+  end
+end