hecks 2.8.0 → 3.0.0

data/lib/hecks/rust_build/wasm.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require_relative "../rust_build"
+require_relative "project_rust"
+
+module Hecks
+  module RustBuild
+    # Regenerates a domain's Rust source and cross-compiles it to `wasm32-wasip1`
+    # (docs/decisions/0012-wasm-via-wasi-stdio.md).
+    #
+    #   Wasm.call(["path/to/domain"])
+    #
+    # Writes `<workspace>/dist/<domain>.wasm`, run with
+    # `wasmtime run rust/dist/<domain>.wasm < script.json`, and the domain's `ir.json` beside it,
+    # which `rust/host` reads at runtime. `HECKS_RUST_DIR` names the workspace to build in and
+    # `CARGO_TARGET_DIR` where build output goes, beside the workspace by default.
+    module Wasm
+      # The Cargo target the module is built for.
+      TARGET = "wasm32-wasip1"
+
+      module_function
+
+      # @param argv [Array<String>] the domain's directory
+      # @return [Integer] the exit status
+      # @raise [Failure] when the target is not installed or a step fails
+      def call(argv)
+        domain = argv.first or raise Failure, "usage: hecks build_wasm <domain>"
+        return build_with_rust(domain) if ENV["HECKS_PARSER"] == "rust" && ENV["HECKS_CODEGEN"] == "rust"
+
+        require_target!
+        rust_dir = RustBuild.rust_dir
+        scratch = scratch_copy(rust_dir)
+        puts "== regenerating #{scratch}/src/generated/ for #{domain} =="
+        status = RustBuild.with_env("HECKS_RUST_DIR" => scratch) { ProjectRust.call([domain]) }
+        raise Failure, "project_rust failed for #{domain}" unless status.zero?
+
+        compile(scratch)
+        publish(scratch, rust_dir, File.basename(domain))
+        0
+      end
+
+      # With the Rust parser and codegen, `hecks-build` runs the same steps and writes the same
+      # outputs.
+      def build_with_rust(domain)
+        build_dir = File.join(ROOT, "rust", "build")
+        puts "== cargo build (hecks-build) =="
+        RustBuild.command!("cargo", "build", chdir: build_dir)
+        puts "== hecks-build #{domain} --wasm =="
+        RustBuild.command!(File.join(build_dir, "target", "debug", "hecks-build"), domain, "--wasm")
+        0
+      end
+
+      def require_target!
+        return if system("rustup target list --installed 2>/dev/null | grep -qx #{TARGET}")
+
+        raise Failure, <<~MSG
+          #{TARGET} isn't installed for this toolchain. Install it once with:
+
+              rustup target add #{TARGET}
+
+          then re-run the build.
+        MSG
+      end
+
+      # Built in a scratch copy because generating rewrites `Cargo.toml` and `src/generated/`,
+      # which would dirty tracked files in a checkout.
+      def scratch_copy(rust_dir, name = "project_wasm")
+        scratch = if ENV.key?("HECKS_RUST_DIR")
+                    File.join(rust_dir, "scratch", name)
+                  else
+                    File.join(ROOT, "tmp", name, "rust")
+                  end
+        FileUtils.rm_rf(scratch)
+        FileUtils.mkdir_p(scratch)
+        %w[Cargo.toml Cargo.lock src].each { |entry| FileUtils.cp_r(File.join(rust_dir, entry), scratch) }
+        scratch
+      end
+
+      # `rustup run`, not bare `cargo`: a `cargo` earlier on PATH may never have seen
+      # `rustup target add` and fail with "can't find crate for `std`".
+      def compile(scratch)
+        puts "== cargo build --release --target #{TARGET} =="
+        target_dir = ENV.fetch("CARGO_TARGET_DIR", File.join(RustBuild.rust_dir, "target"))
+        RustBuild.command!("rustup", "run", "stable", "cargo", "build", "--release", "--target", TARGET,
+                           chdir: scratch, env: { "CARGO_TARGET_DIR" => target_dir })
+      end
+
+      def publish(scratch, rust_dir, name)
+        dist = File.join(rust_dir, "dist")
+        FileUtils.mkdir_p(dist)
+        target_dir = ENV.fetch("CARGO_TARGET_DIR", File.join(rust_dir, "target"))
+        out = File.join(dist, "#{name}.wasm")
+        FileUtils.cp(File.join(target_dir, TARGET, "release", "rust.wasm"), out)
+        puts "wrote #{out}"
+        puts %(run it: wasmtime run #{out} < script.json)
+        sidecar = File.join(scratch, "src", "generated", name, "ir.json")
+        return unless File.exist?(sidecar)
+
+        FileUtils.cp(sidecar, File.join(dist, "#{name}.ir.json"))
+        puts "wrote #{File.join(dist, "#{name}.ir.json")}"
+      end
+    end
+  end
+end

data/lib/hecks/rust_build/wasm_browser.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require_relative "../rust_build"
+require_relative "project_rust"
+require_relative "wasm"
+
+module Hecks
+  module RustBuild
+    # Builds the `rust/web` wasm-bindgen crate over a domain's generated code
+    # (docs/decisions/0015-wasm-bindgen-browser-projection.md).
+    #
+    #   WasmBrowser.call(["path/to/domain"])
+    #
+    # Writes `<workspace>/dist/browser/<domain>/{<domain>.js,<domain>_bg.wasm,*.d.ts}`, an ES
+    # module whose `dispatch(json)` returns the same `{"instances","events","refusals"}` JSON as
+    # the WASI module.
+    module WasmBrowser
+      # The Cargo target the module is built for.
+      TARGET = "wasm32-unknown-unknown"
+
+      module_function
+
+      # @param argv [Array<String>] the domain's directory
+      # @return [Integer] the exit status
+      # @raise [Failure] when the target or the matching wasm-bindgen CLI is missing, or a step
+      #   fails
+      def call(argv)
+        domain = argv.first or raise Failure, "usage: hecks build_browser_wasm <domain>"
+        rust_dir = RustBuild.rust_dir
+        web_dir = File.join(rust_dir, "web")
+        name = File.basename(domain)
+        require_target!
+        require_cli!(web_dir)
+        scratch = scratch_copy(rust_dir)
+        puts "== regenerating #{scratch}/src/generated/ for #{domain} =="
+        status = RustBuild.with_env("HECKS_RUST_DIR" => scratch) { ProjectRust.call([domain]) }
+        raise Failure, "project_rust failed for #{domain}" unless status.zero?
+
+        compile(File.join(scratch, "web"), web_dir)
+        bind(web_dir, name)
+        0
+      end
+
+      # Generated in a scratch copy, as `Wasm` does, because generating rewrites `Cargo.toml` and
+      # `src/generated/`, which would dirty tracked files in a checkout. The web crate's path
+      # dependency `..` resolves to the scratch workspace.
+      def scratch_copy(rust_dir)
+        scratch = Wasm.scratch_copy(rust_dir, "project_wasm_browser")
+        FileUtils.mkdir_p(File.join(scratch, "web"))
+        %w[Cargo.toml Cargo.lock src].each do |entry|
+          from = File.join(rust_dir, "web", entry)
+          FileUtils.cp_r(from, File.join(scratch, "web")) if File.exist?(from)
+        end
+        scratch
+      end
+
+      def require_target!
+        return if system("rustup target list --installed 2>/dev/null | grep -qx #{TARGET}")
+
+        raise Failure, <<~MSG
+          #{TARGET} isn't installed for this toolchain. Install it once with:
+
+              rustup target add #{TARGET}
+
+          then re-run the build.
+        MSG
+      end
+
+      # The wasm-bindgen CLI must match the crate version pinned in `rust/web/Cargo.toml`; a
+      # mismatch fails only when the CLI runs, so the pin is read back, not hardcoded.
+      def require_cli!(web_dir)
+        pinned = File.read(File.join(web_dir, "Cargo.toml"))[/^wasm-bindgen = "=(.+)"/, 1] or
+          raise Failure, "rust/web/Cargo.toml: couldn't find a pinned (\"=x.y.z\") wasm-bindgen version"
+        installed = `wasm-bindgen --version 2>/dev/null`[/[\d.]+/]
+        return if installed == pinned
+
+        raise Failure, <<~MSG
+          wasm-bindgen CLI #{installed.inspect} != the pinned crate version #{pinned.inspect}. Install the matching CLI once with:
+
+              cargo install wasm-bindgen-cli --version #{pinned} --locked
+
+          then re-run the build.
+        MSG
+      end
+
+      def compile(scratch_web, web_dir)
+        puts "== cargo build --release --target #{TARGET} (rust/web) =="
+        target_dir = ENV.fetch("CARGO_TARGET_DIR", File.join(web_dir, "target"))
+        RustBuild.command!("rustup", "run", "stable", "cargo", "build", "--release", "--target", TARGET,
+                           chdir: scratch_web, env: { "CARGO_TARGET_DIR" => target_dir })
+      end
+
+      def bind(web_dir, name)
+        out_dir = File.join(RustBuild.rust_dir, "dist", "browser", name)
+        FileUtils.mkdir_p(out_dir)
+        built = File.join(ENV.fetch("CARGO_TARGET_DIR", File.join(web_dir, "target")), TARGET, "release",
+                          "rust_web.wasm")
+        puts "== wasm-bindgen --target web =="
+        RustBuild.command!("wasm-bindgen", "--target", "web", "--out-dir", out_dir, "--out-name", name, built)
+        puts "wrote #{out_dir}/"
+        puts <<~MSG
+          use it in a browser:
+              <script type="module">
+                import init, { dispatch } from "./#{name}.js";
+                await init();
+                const result = dispatch(JSON.stringify({ steps: [...] }));
+              </script>
+        MSG
+      end
+    end
+  end
+end

data/lib/hecks/rust_build.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "monitor"
+require "tempfile"
+require_relative "hecks/adapters/console_capture"
+
+module Hecks
+  # The tools that generate a domain's Rust source, compile it and check the build against the Ruby
+  # engine, carried in the gem so a caller reaches them in this process.
+  #
+  # Each tool is a module with `call(argv) -> Integer`: it reads its arguments as a command line
+  # would give them, prints its report to `$stdout` and its complaints to `$stderr`, reads
+  # `ENV` for its settings, and answers an exit status. `run` and `capture` are the entry points:
+  # they hand a tool the streams and environment it is asked to use for the length of one call,
+  # and turn a refusal (`Failure`, `abort`, an unexpected error) into a status and a message. The
+  # `hecks project_rust` family of verbs runs them through `run`.
+  module RustBuild
+    # The gem's root directory, where `lib/` and `rust/` live.
+    ROOT = File.expand_path("../..", __dir__)
+
+    # Raised by a tool to refuse, with the reason as the message.
+    class Failure < StandardError; end
+
+    # What one captured call did: `out` and `err` (what the tool printed) and its exit `status`.
+    Result = Struct.new(:out, :err, :status) do
+      # @return [Boolean] whether the tool succeeded
+      def ok? = status.zero?
+    end
+
+    # Tool name to the file that defines it and the constant it defines.
+    TOOLS = {
+      "project_rust"          => ["rust_build/project_rust", :ProjectRust],
+      "project_wasm"          => ["rust_build/wasm", :Wasm],
+      "project_wasm_browser"  => ["rust_build/wasm_browser", :WasmBrowser],
+      "rust_conformance"      => ["rust_build/conformance", :Conformance],
+      "rust_conformance_fuzz" => ["rust_build/conformance_fuzz", :ConformanceFuzz],
+      "rust_coverage"         => ["rust_build/coverage", :Coverage]
+    }.freeze
+
+    # The process-wide lock a call holds while it swaps `ENV`, `$stdout` and `$stderr`; the same
+    # lock `ConsoleCapture` holds, so a build and a capture never interleave.
+    LOCK = Hecks::Adapters::ConsoleCapture::LOCK
+    private_constant :LOCK
+
+    class << self
+      # The workspace tools build in: `HECKS_RUST_DIR`, or the gem's own `rust/`.
+      #
+      # @return [String] the directory
+      def rust_dir = ENV.fetch("HECKS_RUST_DIR", File.join(ROOT, "rust"))
+
+      # Runs a tool with the given environment, writing to the given streams.
+      #
+      # The process's `ENV`, `$stdout` and `$stderr` are replaced for the length of the call, so
+      # calls are serialized.
+      #
+      # @param tool [String] a key of `TOOLS`
+      # @param argv [Array<String>] the tool's arguments
+      # @param env [Hash{String => String, nil}] variables to set for the call (nil removes one)
+      # @param out [IO] where the tool's stdout goes; must be a real IO (a file or the terminal)
+      # @param err [IO] where the tool's stderr goes
+      # @return [Integer] the exit status
+      def run(tool, argv, env: {}, out: $stdout, err: $stderr)
+        LOCK.synchronize do
+          with_env(env) do
+            with_streams(out, err) { finish(tool, argv) }
+          end
+        end
+      end
+
+      # Runs a program, its output going where the running tool's output goes.
+      #
+      # @param command [Array<String>] the program and its arguments
+      # @param chdir [String, nil] the directory to run it in
+      # @param env [Hash{String => String}] variables to set for it
+      # @return [void]
+      # @raise [Failure] when it cannot start or ends non-zero
+      def command!(*command, chdir: nil, env: {})
+        options = { out: $stdout, err: $stderr }
+        options[:chdir] = chdir if chdir
+        return if system(env, *command, **options)
+
+        raise Failure, "`#{command.join(' ')}` failed#{" in #{chdir}" if chdir}"
+      end
+
+      # Runs a tool and holds what it printed.
+      #
+      # @param tool [String] a key of `TOOLS`
+      # @param argv [Array<String>] the tool's arguments
+      # @param env [Hash{String => String, nil}] variables to set for the call
+      # @return [Result] what it printed, and how it ended
+      def capture(tool, argv, env: {})
+        Tempfile.create("rust_build_out") do |out|
+          Tempfile.create("rust_build_err") do |err|
+            status = run(tool, argv, env: env, out: out, err: err)
+            [out, err].each(&:flush)
+            Result.new(File.read(out.path), File.read(err.path), status)
+          end
+        end
+      end
+
+      # Sets variables for the block and puts them back after, for a nested tool call.
+      #
+      # @param env [Hash{String => String, nil}] variables to set (nil removes one)
+      # @yield the work to do under those variables
+      # @return [Object] the block's value
+      def with_env(env)
+        LOCK.synchronize do
+          saved = env.keys.to_h { |key| [key, ENV.fetch(key, nil)] }
+          begin
+            env.each { |key, value| ENV[key] = value }
+            yield
+          ensure
+            saved.each { |key, value| ENV[key] = value }
+          end
+        end
+      end
+
+      private
+
+      def with_streams(out, err)
+        saved = [$stdout, $stderr]
+        $stdout = out
+        $stderr = err
+        yield
+      ensure
+        $stdout, $stderr = saved
+      end
+
+      def finish(tool, argv)
+        file, constant = TOOLS.fetch(tool)
+        require_relative file
+        RustBuild.const_get(constant).call(argv)
+      rescue Failure => e
+        warn e.message
+        1
+      rescue SystemExit => e
+        e.status
+      rescue StandardError, ScriptError => e
+        warn "#{tool}: #{e.class}: #{e.message}"
+        1
+      end
+    end
+  end
+end