ruby_wasm 2.5.1-aarch64-linux-musl

data/lib/ruby_wasm/build/build_params.rb

@@ -0,0 +1,3 @@
+module RubyWasm
+  BuildParams = Struct.new(:name, :target, :default_exts, keyword_init: true)
+end

data/lib/ruby_wasm/build/downloader.rb

@@ -0,0 +1,18 @@
+module RubyWasm
+  class Downloader
+    def download(url, dest, message)
+      require "open-uri"
+      content_length = 0
+      uri = URI.parse(url)
+      OpenURI.open_uri(
+        uri,
+        content_length_proc: ->(len) { content_length = len },
+        progress_proc: ->(size) do
+          print "\r#{message} (#{SizeFormatter.format(content_length)}) %.2f%%" %
+                  (size.to_f / content_length * 100)
+        end
+      ) { |f| File.open(dest, "wb") { |out| out.write f.read } }
+      puts "\r"
+    end
+  end
+end

data/lib/ruby_wasm/build/executor.rb

@@ -0,0 +1,191 @@
+module RubyWasm
+  # Build executor to run the actual build commands.
+  class BuildExecutor
+    attr_reader :process_count
+
+    def initialize(verbose: false, process_count: nil)
+      @verbose = verbose
+      @github_actions_markup = ENV["ENABLE_GITHUB_ACTIONS_MARKUP"] != nil
+      __skip__ =
+        begin
+          require "etc"
+          @process_count = process_count || Etc.nprocessors
+        rescue LoadError
+          @process_count = process_count || 1
+        end
+    end
+
+    def system(*args, chdir: nil, env: nil)
+      require "open3"
+
+      _print_command(args, env)
+
+      # @type var kwargs: Hash[Symbol, untyped]
+      kwargs = {}
+      kwargs[:chdir] = chdir if chdir
+
+      args = args.to_a.map(&:to_s)
+      # TODO: Remove __skip__ once we have open3 RBS definitions.
+      __skip__ =
+        if @verbose || !$stdout.tty?
+          kwargs[:exception] = true
+          if env
+            Kernel.system(env, *args, **kwargs)
+          else
+            Kernel.system(*args, **kwargs)
+          end
+        else
+          printer = StatusPrinter.new
+          block =
+            proc do |stdin, stdout, stderr, wait_thr|
+              mux = Mutex.new
+              out = String.new
+              err = String.new
+              readers =
+                [
+                  [stdout, :stdout, out],
+                  [stderr, :stderr, err]
+                ].map do |io, name, str|
+                  reader =
+                    Thread.new do
+                      while (line = io.gets)
+                        mux.synchronize do
+                          printer.send(name, line)
+                          str << line
+                        end
+                      end
+                    end
+                  reader.report_on_exception = false
+                  reader
+                end
+
+              readers.each(&:join)
+
+              [out, err, wait_thr.value]
+            end
+          begin
+            stdout, stderr, status =
+              if env
+                Open3.popen3(env, *args, **kwargs, &block)
+              else
+                Open3.popen3(*args, **kwargs, &block)
+              end
+            unless status.success?
+              $stderr.puts stdout
+              $stderr.puts stderr
+              cmd_to_print = args.map { |a| "'#{a}'" }.join(" ")
+              raise "Command failed with status (#{status.exitstatus}): #{cmd_to_print}"
+            end
+          ensure
+            printer.done
+          end
+        end
+      return
+    rescue => e
+      $stdout.flush
+      $stderr.puts "Try running with `rake --verbose` for more complete output."
+      raise e
+    end
+
+    def begin_section(klass, name, note)
+      message = "\e[1;36m==>\e[0m \e[1m#{klass}(#{name}) -- #{note}\e[0m"
+      if @github_actions_markup
+        puts "::group::#{message}"
+      else
+        puts message
+      end
+
+      # Record the start time
+      @start_times ||= Hash.new
+      @start_times[[klass, name]] = Time.now
+
+      $stdout.flush
+    end
+
+    def end_section(klass, name)
+      took = Time.now - @start_times[[klass, name]]
+      puts "::endgroup::" if @github_actions_markup
+      puts "\e[1;36m==>\e[0m \e[1m#{klass}(#{name}) -- done in #{took.round(2)}s\e[0m"
+    end
+
+    def rm_rf(list)
+      FileUtils.rm_rf(list)
+    end
+
+    def rm_f(list)
+      FileUtils.rm_f(list)
+    end
+
+    def cp_r(src, dest)
+      FileUtils.cp_r(src, dest)
+    end
+
+    def mv(src, dest)
+      FileUtils.mv(src, dest)
+    end
+
+    def mkdir_p(list)
+      FileUtils.mkdir_p(list)
+    end
+
+    def ln_s(src, dest)
+      FileUtils.ln_s(src, dest)
+    end
+
+    def write(path, data)
+      File.write(path, data)
+    end
+
+    private
+
+    def _print_command(args, env)
+      require "shellwords"
+      # Bold cyan
+      print "\e[1;36m  ==>\e[0m "
+      print "env " + env.map { |k, v| "#{k}=#{v}" }.join(" ") + " " if env
+      print args.map { |arg| Shellwords.escape(arg.to_s) }.join(" ") + "\n"
+    end
+  end
+
+  # Human readable status printer for the build.
+  class StatusPrinter
+    def initialize
+      @mutex = Mutex.new
+      @counter = 0
+      @indicators = "|/-\\"
+    end
+
+    def stdout(message)
+      require "io/console"
+      @mutex.synchronize do
+        $stdout.print "\e[K"
+        first_line = message.lines(chomp: true).first || ""
+
+        # Make sure we don't line-wrap the output
+        size =
+          __skip__ =
+            IO.respond_to?(:console_size) ? IO.console_size : IO.console.winsize
+        terminal_width = size[1].to_i.nonzero? || 80
+        width_limit = terminal_width / 2 - 3
+
+        if first_line.length > width_limit
+          first_line = (first_line[0..width_limit - 5] || "") + "..."
+        end
+        indicator = @indicators[@counter] || " "
+        to_print = "  " + indicator + " " + first_line
+        $stdout.print to_print
+        $stdout.print "\e[1A\n"
+        @counter += 1
+        @counter = 0 if @counter >= @indicators.length
+      end
+    end
+
+    def stderr(message)
+      @mutex.synchronize { $stdout.print message }
+    end
+
+    def done
+      @mutex.synchronize { $stdout.print "\e[K" }
+    end
+  end
+end

data/lib/ruby_wasm/build/product/baseruby.rb

@@ -0,0 +1,37 @@
+require_relative "./product"
+
+module RubyWasm
+  class BaseRubyProduct < BuildProduct
+    def initialize(build_dir, source)
+      @build_dir = build_dir
+      @source = source
+      @channel = source.name
+    end
+
+    def product_build_dir
+      File.join(@build_dir, RbConfig::CONFIG["host"], "baseruby-#{@channel}")
+    end
+
+    def install_dir
+      File.join(product_build_dir, "opt")
+    end
+
+    def name
+      "baseruby-#{@channel}"
+    end
+
+    def build(executor)
+      executor.mkdir_p product_build_dir
+      @source.build(executor)
+      return if Dir.exist?(install_dir)
+      executor.system @source.configure_file,
+                      "--prefix=#{install_dir}",
+                      "--disable-install-doc",
+                      chdir: product_build_dir
+      executor.system "make",
+                      "-j#{executor.process_count}",
+                      "install",
+                      chdir: product_build_dir
+    end
+  end
+end

data/lib/ruby_wasm/build/product/crossruby.rb

@@ -0,0 +1,360 @@
+require_relative "./product"
+require "json"
+
+module RubyWasm
+  class CrossRubyExtProduct < BuildProduct
+    attr_reader :name
+
+    def initialize(srcdir, toolchain, ext_relative_path: nil)
+      @srcdir, @toolchain = srcdir, toolchain
+      # ext_relative_path is relative path from build dir
+      # e.g. cgi-0.3.6/ext/cgi/escape
+      @ext_relative_path = ext_relative_path || File.basename(srcdir)
+      @name = @ext_relative_path
+    end
+
+    def product_build_dir(crossruby)
+      File.join(crossruby.ext_build_dir, @ext_relative_path)
+    end
+
+    def linklist(crossruby)
+      File.join(product_build_dir(crossruby), "link.filelist")
+    end
+
+    def metadata_json(crossruby)
+      File.join(product_build_dir(crossruby), "rbwasm.metadata.json")
+    end
+
+    def feature_name(crossruby)
+      metadata = JSON.parse(File.read(metadata_json(crossruby)))
+      metadata["target"]
+    end
+
+    def make_args(crossruby)
+      make_args = []
+      make_args << "CC=#{@toolchain.cc}"
+      make_args << "LD=#{@toolchain.ld}"
+      make_args << "AR=#{@toolchain.ar}"
+      make_args << "RANLIB=#{@toolchain.ranlib}"
+
+      make_args
+    end
+
+    def build(executor, crossruby)
+      objdir = product_build_dir crossruby
+      executor.mkdir_p objdir
+      do_extconf executor, crossruby
+
+      executor.system "make", "-C", objdir, *make_args(crossruby), "clean"
+      build_target = crossruby.target.pic? ? "install-so" : "static"
+      executor.system "make",
+                      "-j#{executor.process_count}",
+                      "-C",
+                      "#{objdir}",
+                      *make_args(crossruby),
+                      build_target
+      # A ext can provide link args by link.filelist. It contains only built archive file by default.
+      unless File.exist?(linklist(crossruby))
+        executor.write(
+          linklist(crossruby),
+          Dir.glob("#{objdir}/*.a").join("\n")
+        )
+      end
+    end
+
+    def do_extconf(executor, crossruby)
+      unless crossruby.target.pic?
+        self.do_legacy_extconf(executor, crossruby)
+        return
+      end
+      objdir = product_build_dir crossruby
+      source = crossruby.source
+      rbconfig_rb = Dir.glob(File.join(crossruby.dest_dir, "usr/local/lib/ruby/*/wasm32-wasi/rbconfig.rb")).first
+      raise "rbconfig.rb not found" unless rbconfig_rb
+      extconf_args = [
+        "-C", objdir,
+        "#{@srcdir}/extconf.rb",
+        "--target-rbconfig=#{rbconfig_rb}",
+      ]
+      executor.system Gem.ruby, *extconf_args
+    end
+
+    def do_legacy_extconf(executor, crossruby)
+      objdir = product_build_dir crossruby
+      source = crossruby.source
+      extconf_args = [
+        "-C", objdir,
+        "--disable=gems",
+        # HACK: top_srcdir is required to find ruby headers
+        "-e",
+        %Q($top_srcdir="#{source.src_dir}"),
+        # HACK: extout is required to find config.h
+        "-e",
+        %Q($extout="#{crossruby.build_dir}/.ext"),
+        # HACK: force static ext build by imitating extmk
+        "-e",
+        "$static = true; trace_var(:$static) {|v| $static = true }",
+        # HACK: $0 should be extconf.rb path due to mkmf source file detection
+        # and we want to insert some hacks before it. But -e and $0 cannot be
+        # used together, so we rewrite $0 in -e.
+        "-e",
+        %Q($0="#{@srcdir}/extconf.rb"),
+        "-e",
+        %Q(require_relative "#{@srcdir}/extconf.rb"),
+        # HACK: extract "$target" from extconf.rb to get a full target name
+        # like "cgi/escape" instead of "escape"
+        "-e",
+        %Q(require "json"; File.write("#{metadata_json(crossruby)}", JSON.dump({target: $target}))),
+        "-I#{crossruby.build_dir}"
+      ]
+      # Clear RUBYOPT to avoid loading unrelated bundle setup
+      executor.system crossruby.baseruby_path,
+                      *extconf_args,
+                      env: {
+                        "RUBYOPT" => ""
+                      }
+    end
+
+    def do_install_rb(executor, crossruby)
+      objdir = product_build_dir crossruby
+      executor.system "make", "-C", objdir, *make_args(crossruby), "install-rb"
+    end
+
+    def cache_key(digest)
+      digest << @name
+      # Compute hash value of files under srcdir
+      Dir
+        .glob("#{@srcdir}/**/*", File::FNM_DOTMATCH)
+        .each do |f|
+          next if File.directory?(f)
+          digest << f
+          digest << File.read(f)
+        end
+    end
+  end
+
+  class CrossRubyProduct < AutoconfProduct
+    attr_reader :target, :source, :toolchain
+    attr_accessor :user_exts,
+                  :wasmoptflags,
+                  :cppflags,
+                  :cflags,
+                  :ldflags,
+                  :debugflags,
+                  :xcflags,
+                  :xldflags
+
+    def initialize(params, build_dir, rubies_dir, baseruby, source, toolchain)
+      @params = params
+      @rubies_dir = rubies_dir
+      @build_dir = build_dir
+      @baseruby = baseruby
+      @source = source
+      @toolchain = toolchain
+      @user_exts = []
+      @wasmoptflags = []
+      @cppflags = []
+      @cflags = []
+      @ldflags = []
+      @debugflags = []
+      @xcflags = []
+      @xldflags = []
+      super(@params.target, @toolchain)
+    end
+
+    def configure(executor, reconfigure: false)
+      if !File.exist?("#{build_dir}/Makefile") || reconfigure
+        args = configure_args(RbConfig::CONFIG["host"], toolchain)
+        executor.system source.configure_file, *args, chdir: build_dir
+      end
+      # NOTE: we need rbconfig.rb at configuration time to build user given extensions with mkmf
+      executor.system "make", "rbconfig.rb", chdir: build_dir
+    end
+
+    def need_exts_build?
+      @user_exts.any?
+    end
+
+    def need_extinit_obj?
+      need_exts_build? && !@target.pic?
+    end
+
+    def build_exts(executor)
+      @user_exts.each do |prod|
+        executor.begin_section prod.class, prod.name, "Building"
+        prod.build(executor, self)
+        executor.end_section prod.class, prod.name
+      end
+    end
+
+    def build(executor, remake: false, reconfigure: false)
+      executor.mkdir_p dest_dir
+      executor.mkdir_p build_dir
+      @toolchain.install
+      [@source, @baseruby, @libyaml, @zlib, @openssl, @wasi_vfs].each do |prod|
+        next unless prod
+        executor.begin_section prod.class, prod.name, "Building"
+        prod.build(executor)
+        executor.end_section prod.class, prod.name
+      end
+      executor.begin_section self.class, name, "Configuring"
+      configure(executor, reconfigure: reconfigure)
+      executor.end_section self.class, name
+
+      build_exts(executor) if need_exts_build?
+
+      executor.begin_section self.class, name, "Building"
+
+      if need_extinit_obj?
+        executor.mkdir_p File.dirname(extinit_obj)
+        executor.system "ruby",
+                        extinit_c_erb,
+                        *@user_exts.map { |ext| ext.feature_name(self) },
+                        "--cc",
+                        toolchain.cc,
+                        "--output",
+                        extinit_obj
+      end
+      install_dir = File.join(build_dir, "install")
+      if !File.exist?(install_dir) || remake || reconfigure
+        executor.system "make",
+                        "-j#{executor.process_count}",
+                        "install",
+                        "DESTDIR=#{install_dir}",
+                        chdir: build_dir
+      end
+
+      executor.rm_rf dest_dir
+      executor.cp_r install_dir, dest_dir
+      @user_exts.each { |ext| ext.do_install_rb(executor, self) }
+      executor.system "tar", "cfz", artifact, "-C", @rubies_dir, name
+
+      executor.end_section self.class, name
+    end
+
+    def clean(executor)
+      executor.rm_rf dest_dir
+      executor.rm_rf build_dir
+      executor.rm_rf ext_build_dir
+      executor.rm_f artifact
+    end
+
+    def name
+      @params.name
+    end
+
+    def cache_key(digest)
+      @params.target.cache_key(digest)
+      digest << @params.default_exts
+      @wasmoptflags.each { |f| digest << f }
+      @cppflags.each { |f| digest << f }
+      @cflags.each { |f| digest << f }
+      @ldflags.each { |f| digest << f }
+      @debugflags.each { |f| digest << f }
+      @xcflags.each { |f| digest << f }
+      @xldflags.each { |f| digest << f }
+      @user_exts.each { |ext| ext.cache_key(digest) }
+    end
+
+    def build_dir
+      File.join(@build_dir, @params.target.to_s, name)
+    end
+
+    def ext_build_dir
+      File.join(@build_dir, @params.target.to_s, name + "-ext")
+    end
+
+    def with_libyaml(libyaml)
+      @libyaml = libyaml
+    end
+
+    def with_zlib(zlib)
+      @zlib = zlib
+    end
+
+    def with_wasi_vfs(wasi_vfs)
+      @wasi_vfs = wasi_vfs
+    end
+
+    def with_openssl(openssl)
+      @openssl = openssl
+    end
+
+    def dest_dir
+      File.join(@rubies_dir, name)
+    end
+
+    def artifact
+      File.join(@rubies_dir, "#{name}.tar.gz")
+    end
+
+    def extinit_obj
+      "#{ext_build_dir}/extinit.o"
+    end
+
+    def extinit_c_erb
+      lib_root = File.expand_path("../../../../..", __FILE__)
+      File.join(lib_root, "ext", "extinit.c.erb")
+    end
+
+    def baseruby_path
+      File.join(@baseruby.install_dir, "bin/ruby")
+    end
+
+    def configure_args(build_triple, toolchain)
+      target = @params.target.triple
+      default_exts = @params.default_exts
+
+      ldflags = @ldflags.dup
+      xldflags = @xldflags.dup
+
+      args = self.system_triplet_args + ["--build", build_triple]
+      args << "--with-static-linked-ext" unless @params.target.pic?
+      args << %Q(--with-ext=#{default_exts})
+      args << %Q(--with-libyaml-dir=#{@libyaml.install_root})
+      args << %Q(--with-zlib-dir=#{@zlib.install_root})
+      args << %Q(--with-openssl-dir=#{@openssl.install_root}) if @openssl
+      args << %Q(--with-baseruby=#{baseruby_path})
+
+      case target
+      when /^wasm32-unknown-wasi/
+        xldflags << @wasi_vfs.lib_wasi_vfs_a if @wasi_vfs
+        # TODO: Find a way to force cast or update API
+        # @type var wasi_sdk_path: untyped
+        wasi_sdk_path = @toolchain
+        args << %Q(WASMOPT=#{wasi_sdk_path.wasm_opt})
+        args << %Q(WASI_SDK_PATH=#{wasi_sdk_path.wasi_sdk_path})
+      when "wasm32-unknown-emscripten"
+        ldflags.concat(%w[-s MODULARIZE=1])
+        env_emcc_ldflags = ENV["RUBY_WASM_EMCC_LDFLAGS"] || ""
+        unless env_emcc_ldflags.empty?
+          ldflags << env_emcc_ldflags
+        end
+      else
+        raise "unknown target: #{target}"
+      end
+
+      args.concat(self.tools_args)
+      (@user_exts || []).each { |lib| xldflags << "@#{lib.linklist(self)}" }
+      xldflags << extinit_obj if need_extinit_obj?
+
+      cflags = @cflags.dup
+      xcflags = @xcflags.dup
+      xcflags << "-DWASM_SETJMP_STACK_BUFFER_SIZE=24576"
+      xcflags << "-DWASM_FIBER_STACK_BUFFER_SIZE=24576"
+      xcflags << "-DWASM_SCAN_STACK_BUFFER_SIZE=24576"
+
+      args << %Q(LDFLAGS=#{ldflags.join(" ")})
+      args << %Q(XLDFLAGS=#{xldflags.join(" ")})
+      args << %Q(CFLAGS=#{cflags.join(" ")})
+      args << %Q(XCFLAGS=#{xcflags.join(" ")})
+      args << %Q(debugflags=#{@debugflags.join(" ")})
+      args << %Q(cppflags=#{@cppflags.join(" ")})
+      unless wasmoptflags.empty?
+        args << %Q(wasmoptflags=#{@wasmoptflags.join(" ")})
+      end
+      args << "--disable-install-doc"
+      args
+    end
+  end
+end

data/lib/ruby_wasm/build/product/libyaml.rb

@@ -0,0 +1,70 @@
+require_relative "./product"
+
+module RubyWasm
+  class LibYAMLProduct < AutoconfProduct
+    attr_reader :target
+
+    LIBYAML_VERSION = "0.2.5"
+
+    def initialize(build_dir, target, toolchain)
+      @build_dir = build_dir
+      @target = target
+      super(target, toolchain)
+    end
+
+    def product_build_dir
+      File.join(@build_dir, target.to_s, "yaml-#{LIBYAML_VERSION}")
+    end
+
+    def destdir
+      File.join(product_build_dir, "opt")
+    end
+
+    def install_root
+      File.join(destdir, "usr", "local")
+    end
+
+    def name
+      "libyaml-#{LIBYAML_VERSION}-#{target}"
+    end
+
+    def build(executor)
+      return if Dir.exist?(install_root)
+
+      executor.mkdir_p File.dirname(product_build_dir)
+      executor.rm_rf product_build_dir
+      executor.mkdir_p product_build_dir
+      tarball_path =
+        File.join(product_build_dir, "libyaml-#{LIBYAML_VERSION}.tar.gz")
+      executor.system "curl",
+                      "-o",
+                      tarball_path,
+                      "-L",
+                      "https://github.com/yaml/libyaml/releases/download/#{LIBYAML_VERSION}/yaml-#{LIBYAML_VERSION}.tar.gz"
+      executor.system "tar",
+                      "xzf",
+                      tarball_path,
+                      "-C",
+                      product_build_dir,
+                      "--strip-components=1"
+
+      # obtain the latest config.guess and config.sub for Emscripten and WASI triple support
+      executor.system "curl",
+                      "-o",
+                      "#{product_build_dir}/config/config.guess",
+                      "https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/config.guess"
+      executor.system "curl",
+                      "-o",
+                      "#{product_build_dir}/config/config.sub",
+                      "https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/config.sub"
+
+      configure_args = self.configure_args.dup
+      configure_args << "CFLAGS=-fPIC" if target.pic?
+      executor.system "./configure", *configure_args, chdir: product_build_dir
+      executor.system "make",
+                      "install",
+                      "DESTDIR=#{destdir}",
+                      chdir: product_build_dir
+    end
+  end
+end