hecks 2.5.0 → 2.6.0

data/lib/hecks/bench/rust_runner.rb

@@ -0,0 +1,165 @@
+require "fileutils"
+require "open3"
+require_relative "run"
+
+module Hecks
+  module Bench
+    # Measures the native Rust binary through its streaming mode.
+    #
+    # `rust --serve` reads one JSON step per line and answers one JSON line each, keeping
+    # its in-memory store alive between them. This runner writes a step, waits for the
+    # answer line, and times the whole round trip, so the figures include two pipe
+    # crossings and the binary's own JSON parse and print. Process start-up is outside
+    # the timer.
+    #
+    # Alongside each run it measures a **round-trip floor**: the same round trip for a
+    # step the binary refuses immediately, which is what the pipes cost with almost no
+    # domain work behind them.
+    module RustRunner
+      # Where the Rust crate lives.
+      RUST_DIR = File.expand_path("../../../rust", __dir__)
+
+      # How many refused-at-once steps make up the round-trip floor's sample.
+      FLOOR_SAMPLES = 500
+
+      # A `cargo build` that exited non-zero.
+      class BuildFailed < StandardError; end
+
+      # A benchmarked step the binary refused or could not answer.
+      class StepRefused < StandardError; end
+
+      module_function
+
+      # Says why the Rust target cannot run, if it cannot.
+      #
+      # @param binary [String, nil] an explicit pre-built binary, which needs no toolchain
+      # @return [String, nil] nil when the target can run, otherwise one sentence saying why
+      def unavailable_reason(binary: nil)
+        return "the binary #{binary} does not exist or is not executable" if binary && !File.executable?(binary)
+        return nil if binary || Environment.capture("cargo", "-V") != "unknown"
+
+        "`cargo` is not on PATH and no --rust-binary was given"
+      end
+
+      # Builds the release binary for one domain's Cargo feature and pins a copy of it.
+      #
+      # The copy has its own name so building another domain's feature afterwards cannot
+      # replace it.
+      #
+      # @param feature [String] the domain's Cargo feature, its directory name
+      # @param rust_dir [String] the Rust crate's directory
+      # @return [String] the path of the pinned release binary
+      # @raise [BuildFailed] if cargo exits non-zero or leaves no binary
+      def build(feature, rust_dir: RUST_DIR)
+        output, status = Open3.capture2e("cargo", "build", "--release", "--no-default-features",
+                                         "--features", feature, chdir: rust_dir)
+        unless status.success?
+          raise BuildFailed,
+                "cargo build --release --features #{feature} failed:\n#{output.lines.last(15).join}"
+        end
+
+        target = File.join(ENV.fetch("CARGO_TARGET_DIR", File.join(rust_dir, "target")), "release")
+        pinned = File.join(target, "rust-bench-#{feature}")
+        FileUtils.cp(File.join(target, "rust"), pinned)
+        pinned
+      end
+
+      # Starts the binary in serve mode and times the workload's commands.
+      #
+      # @param workload [Workload] the commands to send
+      # @param binary [String] the path of a binary built for `workload`'s domain
+      # @param warmup [Integer] cycles sent and discarded before timing starts
+      # @param iterations [Integer] cycles timed
+      # @return [Run] the timings, with the round-trip floor in its extras as
+      #   `:roundtrip_floor_p50_us`
+      # @raise [StepRefused] if the binary refuses any step, warmup included
+      def call(workload, binary:, warmup:, iterations:)
+        IO.popen([binary, "--serve"], "r+") do |pipe|
+          pipe.sync = true
+          workload.setup.each { |step| ask(pipe, step.to_json_line) }
+          warmup.times { |n| workload.cycle(n).each { |step| ask(pipe, step.to_json_line) } }
+          run = measure(pipe, workload, warmup: warmup, iterations: iterations)
+          Run.new(samples: run.samples, wall_seconds: run.wall_seconds,
+                  extras: { roundtrip_floor_p50_us: floor(pipe) })
+        ensure
+          pipe.close_write unless pipe.closed?
+        end
+      end
+
+      # Times the measured cycles.
+      #
+      # @param pipe [IO] the binary's stdin and stdout
+      # @param workload [Workload] the commands to send
+      # @param warmup [Integer] cycles already sent, so record names continue from there
+      # @param iterations [Integer] cycles to time
+      # @return [Run] the timings, without extras
+      def measure(pipe, workload, warmup:, iterations:)
+        samples = []
+        started = now
+        iterations.times do |n|
+          workload.cycle(warmup + n).each do |step|
+            line = step.to_json_line
+            began = now
+            answer = round_trip(pipe, line)
+            samples << [step.verb, now - began]
+            check(answer, step.verb)
+          end
+        end
+        Run.new(samples: samples, wall_seconds: now - started)
+      end
+
+      # Measures the pipes with almost no work behind them.
+      #
+      # @param pipe [IO] the binary's stdin and stdout
+      # @return [Float] the p50 microseconds of {FLOOR_SAMPLES} round trips for an empty step,
+      #   which the binary refuses without touching its store
+      def floor(pipe)
+        seconds = Array.new(FLOOR_SAMPLES) do
+          began = now
+          round_trip(pipe, "{}")
+          now - began
+        end
+        Stats.micros(Stats.percentile(seconds, 0.5))
+      end
+
+      # Sends one step and returns the answer line, untimed.
+      #
+      # @param pipe [IO] the binary's stdin and stdout
+      # @param line [String] the step as one JSON line
+      # @return [void]
+      # @raise [StepRefused] if the binary refuses the step
+      def ask(pipe, line)
+        check(round_trip(pipe, line), line[0, 80])
+      end
+
+      # Writes one line and reads one line back.
+      #
+      # @param pipe [IO] the binary's stdin and stdout
+      # @param line [String] the step as one JSON line
+      # @return [String] the answer line, or an empty string if the binary closed its output
+      def round_trip(pipe, line)
+        pipe.write(line, "\n")
+        pipe.gets.to_s
+      end
+
+      # Refuses an answer that is not a success.
+      #
+      # @param answer [String] the binary's answer line
+      # @param what [String] what was asked, for the error message
+      # @return [void]
+      # @raise [StepRefused] unless `answer` starts with `{"ok":true`
+      def check(answer, what)
+        return if answer.start_with?('{"ok":true')
+
+        raise StepRefused, "#{what} was not accepted: #{answer.strip.empty? ? 'the binary closed its output' : answer.strip}"
+      end
+
+      # Reads the monotonic clock.
+      #
+      # @return [Float] seconds on the monotonic clock
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/hecks/bench/stats.rb

@@ -0,0 +1,75 @@
+module Hecks
+  module Bench
+    # Order statistics over a list of latency samples.
+    #
+    # Percentiles use the nearest-rank method: the smallest sample with at least
+    # that fraction of the samples at or below it. It never interpolates, so every
+    # reported figure is a latency that actually happened. With fewer than 100
+    # samples the p99 is simply the maximum.
+    module Stats
+      module_function
+
+      # Reads one percentile from a list of samples.
+      #
+      # @param samples [Array<Float>] the samples, in any order
+      # @param fraction [Float] the percentile as a fraction, e.g. `0.99`
+      # @return [Float, nil] the nearest-rank percentile, or nil when `samples` is empty
+      def percentile(samples, fraction)
+        return nil if samples.empty?
+
+        sorted = samples.sort
+        sorted[[(fraction * sorted.size).ceil - 1, 0].max]
+      end
+
+      # Finds the middle value of a list, averaging the two middle values of an even one.
+      #
+      # @param values [Array<Numeric>] the values, in any order
+      # @return [Float, nil] the median, or nil when `values` is empty
+      def median(values)
+        return nil if values.empty?
+
+        sorted = values.sort
+        middle = sorted.size / 2
+        sorted.size.odd? ? sorted[middle].to_f : (sorted[middle - 1] + sorted[middle]) / 2.0
+      end
+
+      # Summarizes one run's latency samples.
+      #
+      # @param seconds [Array<Float>] per-command latencies in seconds, in dispatch order
+      # @return [Hash{Symbol => Numeric}] `:count`, and `:p50_us`, `:p99_us`, `:mean_us` and
+      #   `:max_us` in microseconds
+      def latency(seconds)
+        {
+          count:   seconds.size,
+          p50_us:  micros(percentile(seconds, 0.50)),
+          p99_us:  micros(percentile(seconds, 0.99)),
+          mean_us: micros(seconds.sum / seconds.size),
+          max_us:  micros(seconds.max)
+        }
+      end
+
+      # Compares how slow the last tenth of a run was against its first tenth.
+      #
+      # A ratio well above 1.0 means latency grows as the store fills, which a single
+      # p50 over the whole run would hide.
+      #
+      # @param seconds [Array<Float>] per-command latencies in dispatch order
+      # @return [Float, nil] the last tenth's p50 over the first tenth's, or nil when there
+      #   are fewer than ten samples
+      def drift(seconds)
+        tenth = seconds.size / 10
+        return nil if tenth.zero?
+
+        (percentile(seconds.last(tenth), 0.5) / percentile(seconds.first(tenth), 0.5)).round(2)
+      end
+
+      # Converts seconds to microseconds, rounded to a tenth.
+      #
+      # @param seconds [Float, nil] a duration in seconds
+      # @return [Float, nil] the duration in microseconds, or nil when `seconds` is nil
+      def micros(seconds)
+        seconds && (seconds * 1_000_000).round(1)
+      end
+    end
+  end
+end

data/lib/hecks/bench/suite.rb

@@ -0,0 +1,172 @@
+module Hecks
+  module Bench
+    # Runs every requested target against every requested domain and aggregates the runs.
+    #
+    # Each domain's runs are interleaved across targets (memory, sqlite, ..., memory,
+    # sqlite, ...) rather than finished one target at a time, so a burst of unrelated load on
+    # the machine lands on every target instead of flattering one of them. Each figure in the
+    # result is the median across runs, with the fastest and slowest throughput kept so a noisy
+    # measurement shows.
+    module Suite
+      # Every target `bin/bench` can measure.
+      TARGETS = %w[ruby:memory ruby:sqlite ruby:postgres ruby:postgres_era rust].freeze
+
+      # What to measure and how hard.
+      #
+      # @!attribute domains
+      #   @return [Array<String>] workload names, keys of `Workload.all`
+      # @!attribute targets
+      #   @return [Array<String>] members of `TARGETS`
+      # @!attribute warmup
+      #   @return [Integer] cycles discarded before each run is timed
+      # @!attribute iterations
+      #   @return [Integer] cycles timed in each run
+      # @!attribute runs
+      #   @return [Integer] how many fresh boots of each target to measure per domain
+      # @!attribute rust_binary
+      #   @return [String, nil] a pre-built binary to use instead of building one; only valid
+      #     with a single domain, since a binary embeds exactly one
+      Config = Struct.new(:domains, :targets, :warmup, :iterations, :runs, :rust_binary, keyword_init: true)
+
+      module_function
+
+      # Measures everything `config` asks for.
+      #
+      # @param config [Config] what to measure
+      # @param log [#puts] where progress lines go; one line per run
+      # @return [Hash{Symbol => Object}] `:environment`, `:config`, `:results` (one entry per
+      #   domain and target, see `.aggregate`) and `:skipped` (each skipped target and why)
+      # @raise [ArgumentError] if a target or domain is unknown, a count is not positive, or
+      #   `rust_binary` is combined with more than one domain
+      def call(config, log: $stderr)
+        validate(config)
+        load_before = Environment.load_average
+        skipped = unavailable_targets(config)
+        active = config.targets - skipped.map { |entry| entry[:target] }
+        skipped.each { |entry| log.puts "skip #{entry[:target]}: #{entry[:reason]}" }
+        results = config.domains.flat_map { |domain| measure_domain(Workload.fetch(domain), active, config, log) }
+        environment = Environment.describe
+        environment[:load_average] = { before: load_before, after: environment[:load_average] }
+        { environment: environment.merge(postgres: postgres_version(active)),
+          config: config.to_h, results: results, skipped: skipped }
+      end
+
+      # Checks a configuration before anything is booted.
+      #
+      # @param config [Config] the configuration to check
+      # @return [void]
+      # @raise [ArgumentError] on the first problem found
+      def validate(config)
+        unknown = config.targets - TARGETS
+        raise ArgumentError, "unknown target #{unknown.first.inspect} — one of #{TARGETS.join(', ')}" if unknown.any?
+
+        config.domains.each { |domain| Workload.fetch(domain) }
+        if config.iterations < 1 || config.runs < 1 || config.warmup.negative?
+          raise ArgumentError, "iterations and runs must be at least 1, and warmup at least 0"
+        end
+        return unless config.rust_binary && config.domains.size > 1
+
+        raise ArgumentError, "--rust-binary embeds one domain, so it needs exactly one --domain"
+      end
+
+      # Finds the targets this machine cannot run.
+      #
+      # @param config [Config] the configuration whose targets to check
+      # @return [Array<Hash{Symbol => String}>] `{target:, reason:}` for each unavailable target
+      def unavailable_targets(config)
+        postgres = PostgresProbe.unavailable_reason if config.targets.any? { |t| t.start_with?("ruby:postgres") }
+        rust = RustRunner.unavailable_reason(binary: config.rust_binary) if config.targets.include?("rust")
+        config.targets.filter_map do |target|
+          reason = target == "rust" ? rust : (postgres if target.start_with?("ruby:postgres"))
+          { target: target, reason: reason } if reason
+        end
+      end
+
+      # Measures one domain across every active target, `config.runs` times each.
+      #
+      # @param workload [Workload] the domain's commands
+      # @param targets [Array<String>] the targets that can run here
+      # @param config [Config] warmup, iterations, runs and any binary override
+      # @param log [#puts] where progress lines go
+      # @return [Array<Hash>] one aggregated entry per target
+      def measure_domain(workload, targets, config, log)
+        binaries = {}
+        summaries = Hash.new { |hash, target| hash[target] = [] }
+        config.runs.times do |index|
+          targets.each do |target|
+            run = measure_once(workload, target, config, binaries)
+            summaries[target] << run.summary
+            log.puts progress_line("#{workload.name} #{target}", "run #{index + 1}/#{config.runs}", summaries[target].last)
+          end
+        end
+        targets.map { |target| aggregate(workload.name, target, summaries[target]) }
+      end
+
+      # Formats the one line printed as each run finishes.
+      #
+      # @param label [String] the domain and target
+      # @param position [String] which run of how many, e.g. `"2/3"`
+      # @param summary [Hash] the run's `Run#summary`
+      # @return [String] the progress line
+      def progress_line(label, position, summary)
+        "#{label.ljust(27)} #{position}  #{summary[:throughput_per_s].round.to_s.rjust(8)} cmds/s  " \
+          "p50 #{summary[:p50_us].to_s.rjust(9)} us  p99 #{summary[:p99_us].to_s.rjust(9)} us"
+      end
+
+      # Runs one target once.
+      #
+      # @param workload [Workload] the domain's commands
+      # @param target [String] a member of `TARGETS`
+      # @param config [Config] warmup, iterations and any binary override
+      # @param binaries [Hash{String => String}] Rust binaries already built, by domain name;
+      #   filled in as they are built
+      # @return [Run] the run's timings
+      def measure_once(workload, target, config, binaries)
+        if target == "rust"
+          binary = config.rust_binary || (binaries[workload.name] ||= RustRunner.build(workload.name))
+          RustRunner.call(workload, binary: binary, warmup: config.warmup, iterations: config.iterations)
+        else
+          RubyRunner.call(workload, adapter: target.delete_prefix("ruby:").to_sym,
+                                    warmup: config.warmup, iterations: config.iterations)
+        end
+      end
+
+      # Reduces one target's runs to a single entry.
+      #
+      # @param domain [String] the workload name
+      # @param target [String] the target name
+      # @param summaries [Array<Hash>] each run's `Run#summary`
+      # @return [Hash{Symbol => Object}] `:domain`, `:target`, `:runs` (the raw summaries),
+      #   `:median` (the median of each run's throughput, p50, p99 and drift, plus
+      #   `:by_verb`), and `:throughput_range` (`[slowest, fastest]` across runs)
+      def aggregate(domain, target, summaries)
+        rates = summaries.map { |summary| summary[:throughput_per_s] }
+        median = %i[throughput_per_s p50_us p99_us mean_us max_us drift].to_h do |key|
+          [key, Stats.median(summaries.filter_map { |summary| summary[key] })]
+        end
+        median[:roundtrip_floor_p50_us] = Stats.median(summaries.filter_map { |s| s[:roundtrip_floor_p50_us] })
+        median[:by_verb] = median_by_verb(summaries)
+        { domain: domain, target: target, runs: summaries, median: median.compact,
+          throughput_range: rates.minmax }
+      end
+
+      # Takes each verb's median p50 and p99 across runs.
+      #
+      # @param summaries [Array<Hash>] each run's `Run#summary`
+      # @return [Hash{String => Hash{Symbol => Float}}] `:p50_us` and `:p99_us` per verb
+      def median_by_verb(summaries)
+        summaries.first[:by_verb].keys.to_h do |verb|
+          [verb, %i[p50_us p99_us].to_h { |key| [key, Stats.median(summaries.map { |s| s[:by_verb][verb][key] })] }]
+        end
+      end
+
+      # Reads the Postgres server version if a Postgres target ran.
+      #
+      # @param active [Array<String>] the targets that ran
+      # @return [String, nil] the server version, or nil when no Postgres target ran
+      def postgres_version(active)
+        PostgresProbe.server_version if active.any? { |target| target.start_with?("ruby:postgres") }
+      end
+    end
+  end
+end

data/lib/hecks/bench/workload.rb

@@ -0,0 +1,137 @@
+require "json"
+
+module Hecks
+  module Bench
+    # A fixed command sequence for one example domain, identical for every runtime.
+    #
+    # `setup` runs once, untimed, before a run starts. `cycle(n)` answers the timed
+    # commands for the n-th unit of work. Every cycle names its own records by `n`, so
+    # cycles never collide and no command is ever refused: a refusal would mean the
+    # benchmark measured a rejection path rather than the work it claims.
+    #
+    # Steps are the same `{"verb", "args"}` shape `bin/run` and the corpus scripts use.
+    # Ruby dispatches them through `dispatch_flat` and the Rust binary reads them as
+    # JSON lines, so both runtimes see byte-identical input.
+    class Workload
+      # One command to dispatch.
+      #
+      # @!attribute verb
+      #   @return [String] the fully qualified command, e.g. `"Pizzas::Order.CreatePizza"`
+      # @!attribute args
+      #   @return [Hash{String => Object}] the command's arguments, string-keyed throughout
+      Step = Struct.new(:verb, :args) do
+        # Renders the step as one line of `rust --serve` input.
+        #
+        # @return [String] the step as compact JSON with no trailing newline
+        def to_json_line
+          JSON.generate("verb" => verb, "args" => args)
+        end
+
+        # Gives the arguments as `dispatch_flat` takes them.
+        #
+        # @return [Hash{Symbol => Object}] `args` with only its top-level keys symbolized,
+        #   the way the corpus replay hands them over
+        def ruby_args
+          args.transform_keys(&:to_sym)
+        end
+      end
+
+      # @return [String] the example domain's directory name, also its Cargo feature
+      attr_reader :name
+
+      # @return [String] the absolute path of the example domain directory
+      attr_reader :domain_path
+
+      # @return [Array<Step>] the commands run once, untimed, before any cycle
+      attr_reader :setup
+
+      # Builds a workload.
+      #
+      # @param name [String] the example domain's directory name, e.g. `"pizzas"`
+      # @param setup [Array<Step>] the untimed preparation commands
+      # @param cycle [#call] takes a cycle number and returns that cycle's `Array<Step>`
+      def initialize(name:, setup:, cycle:)
+        @name = name
+        @domain_path = File.expand_path("../../../examples/#{name}", __dir__)
+        @setup = setup
+        @cycle = cycle
+      end
+
+      # Gives the timed commands for one unit of work.
+      #
+      # @param number [Integer] the zero-based cycle number, unique across warmup and
+      #   measurement so record names never repeat
+      # @return [Array<Step>] the commands in dispatch order
+      def cycle(number) = @cycle.call(number)
+
+      # Gives how many commands each cycle dispatches.
+      #
+      # @return [Integer] the length of every cycle
+      def commands_per_cycle = cycle(0).size
+
+      # Lists the workloads `bin/bench` knows.
+      #
+      # @return [Hash{String => Workload}] each workload keyed by its domain name
+      def self.all
+        { "pizzas" => pizzas, "banking" => banking }
+      end
+
+      # Looks up one workload.
+      #
+      # @param name [String] a key of `.all`
+      # @return [Workload] the workload for `name`
+      # @raise [ArgumentError] if there is no workload with that name
+      def self.fetch(name)
+        all.fetch(name) do
+          raise ArgumentError, "unknown domain #{name.inspect} — one of #{all.keys.join(', ')}"
+        end
+      end
+
+      # A pizza is created, topped twice and bought: four commands and two aggregate updates.
+      #
+      # @return [Workload] the pizzas workload
+      def self.pizzas
+        new(name: "pizzas", setup: [], cycle: lambda { |n|
+          pizza = "pizza-#{n}"
+          [
+            Step.new("Pizzas::Order.CreatePizza",
+                     { "name"  => { "value" => pizza },
+                       "pizza" => { "price_cents" => { "cents" => 1200 }, "size" => { "value" => "large" } } }),
+            Step.new("Pizzas::Order.AddTopping",
+                     { "name" => pizza, "topping" => { "value" => "Basil" }, "amount" => { "value" => 3 } }),
+            Step.new("Pizzas::Order.AddTopping",
+                     { "name" => pizza, "topping" => { "value" => "Olive" }, "amount" => { "value" => 2 } }),
+            Step.new("Pizzas::Order.Purchase",
+                     { "name" => pizza, "customer_name" => { "value" => "Chris" },
+                       "amount" => { "cents" => 1200 } })
+          ]
+        })
+      end
+
+      # An account is opened, credited twice and debited once, all for one registered customer.
+      #
+      # @return [Workload] the banking workload
+      def self.banking
+        register = Step.new("Banking::Customer.Register",
+                            { "reference" => { "value" => "CUST-0001" },
+                              "name"      => { "given" => "Ada", "family" => "Lovelace" },
+                              "email"     => { "address" => "ada@example.com" } })
+        new(name: "banking", setup: [register], cycle: lambda { |n|
+          number = { "value" => "acct-#{n}" }
+          money = ->(cents) { { "cents" => cents, "currency" => "USD" } }
+          [
+            Step.new("Banking::Account.Open",
+                     { "number" => number, "kind" => { "name" => "current" },
+                       "daily_limit" => { "cents" => 50_000 }, "customer" => "CUST-0001" }),
+            Step.new("Banking::Account.Credit",
+                     { "number" => number, "amount" => money.call(10_000), "narrative" => { "text" => "Deposit" } }),
+            Step.new("Banking::Account.Credit",
+                     { "number" => number, "amount" => money.call(5000), "narrative" => { "text" => "Deposit" } }),
+            Step.new("Banking::Account.Debit",
+                     { "number" => number, "amount" => money.call(2500), "narrative" => { "text" => "Groceries" } })
+          ]
+        })
+      end
+    end
+  end
+end

data/lib/hecks/bench.rb

@@ -0,0 +1,28 @@
+module Hecks
+  # The throughput and latency harness `bin/bench` drives.
+  #
+  # It dispatches a fixed, valid command workload against the Ruby runtime on each
+  # persistence adapter and against the native Rust binary, and reports
+  # commands per second plus p50 and p99 latency for each. Not required by
+  # `lib/hecks.rb` on purpose: a booted domain never needs it.
+  #
+  # ## What is measured
+  #
+  # Every runtime is driven the same way: one caller, one command at a time, each
+  # waiting for the previous answer. The Ruby side times `dispatch_flat` in-process.
+  # The Rust side times a full round trip over the pipes of `rust --serve`. See
+  # `docs/benchmarks.md` for what that does and does not compare.
+  module Bench
+  end
+end
+
+require_relative "bench/stats"
+require_relative "bench/workload"
+require_relative "bench/environment"
+require_relative "bench/postgres_probe"
+require_relative "bench/run"
+require_relative "bench/ruby_runner"
+require_relative "bench/rust_runner"
+require_relative "bench/suite"
+require_relative "bench/report"
+require_relative "bench/cli"