hecks 2.9.0 → 3.0.0

data/lib/hecks/bench/cli.rb

@@ -0,0 +1,62 @@
+require "optparse"
+require_relative "suite"
+require_relative "report"
+
+module Hecks
+  module Bench
+    # The command line behind `hecks bench`: parses flags, runs the `Suite`, prints the report.
+    # Lives in `lib/` so a spec can drive it without shelling out.
+    module CLI
+      # The defaults `hecks bench` runs with when no flag says otherwise.
+      DEFAULTS = { domains: %w[pizzas banking], targets: Suite::TARGETS, warmup: 200, iterations: 1000,
+                   runs: 3 }.freeze
+
+      module_function
+
+      # Runs the benchmark described by `argv` and prints its report.
+      #
+      # @return [Integer] the process exit status: 0 on success, 2 for a usage error
+      def run(argv, out: $stdout, err: $stderr)
+        options = parse(argv)
+        result = Suite.call(options.fetch(:config), log: err)
+        out.puts(options[:format] == "json" ? Report.json(result) : Report.markdown(result))
+        File.write(options[:output], Report.json(result)) if options[:output]
+        0
+      rescue OptionParser::ParseError, ArgumentError => e
+        err.puts "hecks bench: #{e.message}"
+        2
+      end
+
+      # Turns command-line arguments into a configuration and output choices.
+      #
+      # @raise [OptionParser::ParseError] on an unknown or malformed flag
+      def parse(argv)
+        values = DEFAULTS.dup
+        extras = { format: "markdown", output: nil, rust_binary: nil }
+        parser(values, extras).parse(argv.dup)
+        { config: Suite::Config.new(**values, rust_binary: extras[:rust_binary]),
+          format: extras[:format], output: extras[:output] }
+      end
+
+      def parser(values, extras)
+        OptionParser.new do |opts|
+          opts.banner = "usage: hecks bench [options]"
+          opts.on("--domain NAMES", Array, "pizzas, banking (default: both)") { |v| values[:domains] = v }
+          opts.on("--targets NAMES", Array, "#{Suite::TARGETS.join(', ')} (default: all)") { |v| values[:targets] = v }
+          opts.on("--iterations N", Integer, "timed cycles per run (default #{DEFAULTS[:iterations]})") do |v|
+            values[:iterations] = v
+          end
+          opts.on("--warmup N", Integer, "discarded cycles before timing (default #{DEFAULTS[:warmup]})") do |v|
+            values[:warmup] = v
+          end
+          opts.on("--runs N", Integer, "fresh boots per target, median reported (default #{DEFAULTS[:runs]})") do |v|
+            values[:runs] = v
+          end
+          opts.on("--rust-binary PATH", "use this `rust` binary instead of building one") { |v| extras[:rust_binary] = v }
+          opts.on("--format FORMAT", %w[markdown json], "markdown (default) or json") { |v| extras[:format] = v }
+          opts.on("--output PATH", "also write the full JSON result here") { |v| extras[:output] = v }
+        end
+      end
+    end
+  end
+end

data/lib/hecks/bench/environment.rb

@@ -0,0 +1,71 @@
+require "etc"
+require "open3"
+
+module Hecks
+  module Bench
+    # Describes the machine and toolchain a benchmark ran on.
+    # Each probe is best effort: a missing tool yields `"unknown"`, not an error.
+    module Environment
+      module_function
+
+      def describe
+        {
+          cpu: cpu, cores: Etc.nprocessors, memory_gib: memory_gib, os: os,
+          ruby: RUBY_DESCRIPTION, yjit: yjit?,
+          rustc: capture("rustc", "-V"), cargo: capture("cargo", "-V"),
+          hecks: Hecks::VERSION, commit: commit, load_average: load_average
+        }
+      end
+
+      # Read before and after a run to show whether something else was busy.
+      def load_average
+        text = RUBY_PLATFORM.include?("darwin") ? capture("sysctl", "-n", "vm.loadavg").delete("{}") : File.read("/proc/loadavg")
+        text.split.first.to_f.round(2)
+      rescue SystemCallError
+        nil
+      end
+
+      def cpu
+        return capture("sysctl", "-n", "machdep.cpu.brand_string") if RUBY_PLATFORM.include?("darwin")
+
+        model = File.foreach("/proc/cpuinfo").find { |line| line.start_with?("model name") }
+        model ? model.split(":", 2).last.strip : "unknown"
+      rescue SystemCallError
+        "unknown"
+      end
+
+      def memory_gib
+        bytes =
+          if RUBY_PLATFORM.include?("darwin")
+            capture("sysctl", "-n", "hw.memsize").to_i
+          else
+            File.read("/proc/meminfo")[/MemTotal:\s+(\d+)/, 1].to_i * 1024
+          end
+        bytes.zero? ? "unknown" : (bytes / (1024.0**3)).round(1)
+      rescue SystemCallError
+        "unknown"
+      end
+
+      def os
+        return "macOS #{capture('sw_vers', '-productVersion')} (#{capture('uname', '-m')})" if RUBY_PLATFORM.include?("darwin")
+
+        capture("uname", "-sr")
+      end
+
+      def yjit?
+        defined?(RubyVM::YJIT) ? RubyVM::YJIT.enabled? : false
+      end
+
+      def commit
+        capture("git", "-C", File.expand_path("../../..", __dir__), "rev-parse", "--short", "HEAD")
+      end
+
+      def capture(*command)
+        out, status = Open3.capture2(*command, err: File::NULL)
+        status.success? && !out.strip.empty? ? out.lines.first.strip : "unknown"
+      rescue SystemCallError
+        "unknown"
+      end
+    end
+  end
+end

data/lib/hecks/bench/postgres_probe.rb

@@ -0,0 +1,30 @@
+module Hecks
+  module Bench
+    # Decides whether the Postgres targets can run at all.
+    # A missing server or `pg` gem skips the target with a reason; settings come from libpq's env.
+    module PostgresProbe
+      module_function
+
+      def unavailable_reason
+        require "pg"
+        PG.connect(dbname: "postgres", connect_timeout: 2).close
+        nil
+      rescue LoadError
+        "the `pg` gem is not installed"
+      rescue PG::Error => e
+        "no Postgres server is reachable (#{e.message.lines.first.to_s.strip}); start one, or point " \
+        "PGHOST, PGPORT and PGUSER at one"
+      end
+
+      def server_version
+        require "pg"
+        connection = PG.connect(dbname: "postgres", connect_timeout: 2)
+        connection.exec("SHOW server_version").getvalue(0, 0)
+      rescue LoadError, PG::Error
+        "unknown"
+      ensure
+        connection&.close
+      end
+    end
+  end
+end

data/lib/hecks/bench/report.rb

@@ -0,0 +1,76 @@
+require "json"
+
+module Hecks
+  module Bench
+    # Renders a `Suite.call` result as Markdown or JSON.
+    # The Markdown carries machine, counts and skips so a pasted table is never a bare number.
+    module Report
+      module_function
+
+      def markdown(result)
+        [environment_block(result[:environment]), configuration_block(result[:config]),
+         table(result[:results]), floor_block(result[:results]), skipped_block(result[:skipped])]
+          .compact.join("\n\n") << "\n"
+      end
+
+      def json(result)
+        JSON.pretty_generate(result)
+      end
+
+      def table(results)
+        header = ["| domain | target | commands/s | range across runs | p50 (us) | p99 (us) | drift |",
+                  "|---|---|---:|---:|---:|---:|---:|"]
+        (header + results.map { |entry| row(entry) }).join("\n")
+      end
+
+      def row(entry)
+        median = entry[:median]
+        low, high = entry[:throughput_range]
+        cells = [entry[:domain], entry[:target], number(median[:throughput_per_s], 0),
+                 "#{number(low, 0)} to #{number(high, 0)}", number(median[:p50_us], 1),
+                 number(median[:p99_us], 1), number(median[:drift], 2)]
+        "| #{cells.join(' | ')} |"
+      end
+
+      def number(value, places)
+        return "n/a" if value.nil?
+
+        whole, fraction = format("%.#{places}f", value).split(".")
+        [whole.gsub(/(\d)(?=(\d{3})+\z)/, '\1,'), fraction].compact.join(".")
+      end
+
+      def environment_block(environment)
+        load = environment[:load_average]
+        lines = [
+          "- CPU: #{environment[:cpu]} (#{environment[:cores]} cores), #{environment[:memory_gib]} GiB",
+          "- OS: #{environment[:os]}",
+          "- Ruby: #{environment[:ruby]} (YJIT #{environment[:yjit] ? 'on' : 'off'})",
+          "- Rust: #{environment[:rustc]}; #{environment[:cargo]}",
+          "- Hecks: #{environment[:hecks]} at #{environment[:commit]}",
+          "- Load average (1 min) before and after: #{load[:before]} and #{load[:after]}"
+        ]
+        lines << "- Postgres: #{environment[:postgres]}" if environment[:postgres]
+        lines.join("\n")
+      end
+
+      def configuration_block(config)
+        "Warmup #{config[:warmup]} cycles, then #{config[:iterations]} timed cycles per run, " \
+          "median of #{config[:runs]} run(s), domains: #{config[:domains].join(', ')}."
+      end
+
+      def floor_block(results)
+        floors = results.select { |entry| entry[:target] == "rust" }
+                        .map { |entry| "#{entry[:domain]} #{number(entry[:median][:roundtrip_floor_p50_us], 1)} us" }
+        return nil if floors.empty?
+
+        "Rust pipe round-trip floor (p50, a step refused at once): #{floors.join(', ')}."
+      end
+
+      def skipped_block(skipped)
+        return nil if skipped.empty?
+
+        (["Skipped:"] + skipped.map { |entry| "- #{entry[:target]}: #{entry[:reason]}" }).join("\n")
+      end
+    end
+  end
+end

data/lib/hecks/bench/ruby_runner.rb

@@ -0,0 +1,93 @@
+require_relative "run"
+
+module Hecks
+  module Bench
+    # Times the Ruby runtime's `dispatch_flat`, in-process, on one persistence adapter.
+    # Boots a throwaway copy via `Fuzzing::IsolatedBoot` so every run starts from an empty store.
+    module RubyRunner
+      ADAPTERS = %i[memory sqlite postgres postgres_era].freeze
+
+      POSTGRES_ADAPTERS = %i[postgres postgres_era].freeze
+
+      DATABASE = "hecks_bench".freeze
+
+      module_function
+
+      def call(workload, adapter:, warmup:, iterations:)
+        unless ADAPTERS.include?(adapter)
+          raise ArgumentError, "unknown adapter #{adapter.inspect} — one of #{ADAPTERS.join(', ')}"
+        end
+
+        load_dependencies
+        schema = "bench_#{Process.pid}_#{rand(1 << 32).to_s(16)}"
+        quietly_for_postgres do
+          boot(workload, adapter, schema) do |runtime|
+            workload.setup.each { |step| dispatch(runtime, step) }
+            measure(runtime, workload, warmup: warmup, iterations: iterations)
+          end
+        end
+      ensure
+        drop_schema(schema) if schema && POSTGRES_ADAPTERS.include?(adapter)
+      end
+
+      def boot(workload, adapter, schema)
+        Hecks::Fuzzing::IsolatedBoot.call(workload.domain_path, adapter: adapter, database: DATABASE,
+                                          schema: schema, scratch: { database: DATABASE, schema: schema }) do |copy|
+          yield Hecks.boot(copy, install_doors: false)
+        end
+      end
+
+      # Every aggregate opens its own connection and each repeats the `schema already exists`
+      # notice, burying progress lines; `PGOPTIONS` is restored afterwards.
+      def quietly_for_postgres
+        previous = ENV.fetch("PGOPTIONS", nil)
+        ENV["PGOPTIONS"] = [previous, "-c client_min_messages=warning"].compact.join(" ")
+        yield
+      ensure
+        ENV["PGOPTIONS"] = previous
+      end
+
+      def measure(runtime, workload, warmup:, iterations:)
+        warmup.times { |n| workload.cycle(n).each { |step| dispatch(runtime, step) } }
+        GC.start
+        samples = []
+        started = now
+        iterations.times do |n|
+          workload.cycle(warmup + n).each { |step| samples << [step.verb, time { dispatch(runtime, step) }] }
+        end
+        Run.new(samples: samples, wall_seconds: now - started)
+      end
+
+      def dispatch(runtime, step)
+        runtime.dispatch_flat(step.verb, step.ruby_args)
+      end
+
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def time
+        started = now
+        yield
+        now - started
+      end
+
+      def load_dependencies
+        require "hecks"
+        require "hecks/fuzzing"
+        require "hecks/ports/persistence/plugins/era"
+      end
+
+      def drop_schema(schema)
+        require "pg"
+        connection = PG.connect(dbname: DATABASE, connect_timeout: 2)
+        connection.exec("SET client_min_messages = warning")
+        connection.exec("DROP SCHEMA IF EXISTS #{connection.quote_ident(schema)} CASCADE")
+      rescue LoadError, PG::Error
+        nil
+      ensure
+        connection&.close
+      end
+    end
+  end
+end

data/lib/hecks/bench/run.rb

@@ -0,0 +1,55 @@
+module Hecks
+  module Bench
+    # The raw timings of one measured pass over a workload on one runtime.
+    #
+    # Runners produce these and know nothing about reporting. {#summary} is the only
+    # place raw samples become the figures a report prints.
+    class Run
+      # @return [Array<Array(String, Float)>] each timed command's verb and latency in
+      #   seconds, in dispatch order
+      attr_reader :samples
+
+      # @return [Float] seconds from the first timed command's start to the last one's end
+      attr_reader :wall_seconds
+
+      # @return [Hash{Symbol => Object}] facts a runner wants reported alongside the timings
+      attr_reader :extras
+
+      # Builds a run.
+      #
+      # @param samples [Array<Array(String, Float)>] verb and latency seconds per command
+      # @param wall_seconds [Float] the wall-clock duration of the whole timed pass
+      # @param extras [Hash{Symbol => Object}] runner-specific facts, such as a round-trip floor
+      def initialize(samples:, wall_seconds:, extras: {})
+        @samples = samples
+        @wall_seconds = wall_seconds
+        @extras = extras
+      end
+
+      # Reduces the samples to the figures a report prints.
+      #
+      # @return [Hash{Symbol => Object}] `:commands`, `:wall_seconds`, `:throughput_per_s`,
+      #   the `Stats.latency` figures, `:drift`, `:by_verb` (each verb's count, p50 and p99),
+      #   and any `extras`
+      def summary
+        seconds = samples.map(&:last)
+        {
+          commands:         samples.size,
+          wall_seconds:     wall_seconds.round(4),
+          throughput_per_s: (samples.size / wall_seconds).round(1),
+          **Stats.latency(seconds),
+          drift:            Stats.drift(seconds),
+          by_verb:          by_verb
+        }.merge(extras)
+      end
+
+      private
+
+      def by_verb
+        samples.group_by(&:first).transform_values do |group|
+          Stats.latency(group.map(&:last)).slice(:count, :p50_us, :p99_us)
+        end
+      end
+    end
+  end
+end

data/lib/hecks/bench/rust_runner.rb

@@ -0,0 +1,101 @@
+require "fileutils"
+require "open3"
+require_relative "run"
+
+module Hecks
+  module Bench
+    # Times the native Rust binary through `rust --serve`, one JSON step per line.
+    # Timings include two pipe crossings and the binary's own JSON parse; start-up is excluded.
+    module RustRunner
+      # HECKS_RUST_DIR names the workspace to build in; unset, this checkout's own rust/.
+      RUST_DIR = ENV.fetch("HECKS_RUST_DIR") { File.expand_path("../../../rust", __dir__) }
+
+      FLOOR_SAMPLES = 500
+
+      class BuildFailed < StandardError; end
+
+      class StepRefused < StandardError; end
+
+      module_function
+
+      def unavailable_reason(binary: nil)
+        return "the binary #{binary} does not exist or is not executable" if binary && !File.executable?(binary)
+        return nil if binary || Environment.capture("cargo", "-V") != "unknown"
+
+        "`cargo` is not on PATH and no --rust-binary was given"
+      end
+
+      # The pinned copy has its own name so building another domain's feature cannot replace it.
+      def build(feature, rust_dir: RUST_DIR)
+        output, status = Open3.capture2e("cargo", "build", "--release", "--no-default-features",
+                                         "--features", feature, chdir: rust_dir)
+        unless status.success?
+          raise BuildFailed,
+                "cargo build --release --features #{feature} failed:\n#{output.lines.last(15).join}"
+        end
+
+        target = File.join(ENV.fetch("CARGO_TARGET_DIR", File.join(rust_dir, "target")), "release")
+        pinned = File.join(target, "rust-bench-#{feature}")
+        FileUtils.cp(File.join(target, "rust"), pinned)
+        pinned
+      end
+
+      def call(workload, binary:, warmup:, iterations:)
+        IO.popen([binary, "--serve"], "r+") do |pipe|
+          pipe.sync = true
+          workload.setup.each { |step| ask(pipe, step.to_json_line) }
+          warmup.times { |n| workload.cycle(n).each { |step| ask(pipe, step.to_json_line) } }
+          run = measure(pipe, workload, warmup: warmup, iterations: iterations)
+          Run.new(samples: run.samples, wall_seconds: run.wall_seconds,
+                  extras: { roundtrip_floor_p50_us: floor(pipe) })
+        ensure
+          pipe.close_write unless pipe.closed?
+        end
+      end
+
+      def measure(pipe, workload, warmup:, iterations:)
+        samples = []
+        started = now
+        iterations.times do |n|
+          workload.cycle(warmup + n).each do |step|
+            line = step.to_json_line
+            began = now
+            answer = round_trip(pipe, line)
+            samples << [step.verb, now - began]
+            check(answer, step.verb)
+          end
+        end
+        Run.new(samples: samples, wall_seconds: now - started)
+      end
+
+      # An empty step is refused without touching the store: the cost of the pipes alone.
+      def floor(pipe)
+        seconds = Array.new(FLOOR_SAMPLES) do
+          began = now
+          round_trip(pipe, "{}")
+          now - began
+        end
+        Stats.micros(Stats.percentile(seconds, 0.5))
+      end
+
+      def ask(pipe, line)
+        check(round_trip(pipe, line), line[0, 80])
+      end
+
+      def round_trip(pipe, line)
+        pipe.write(line, "\n")
+        pipe.gets.to_s
+      end
+
+      def check(answer, what)
+        return if answer.start_with?('{"ok":true')
+
+        raise StepRefused, "#{what} was not accepted: #{answer.strip.empty? ? 'the binary closed its output' : answer.strip}"
+      end
+
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/hecks/bench/stats.rb

@@ -0,0 +1,46 @@
+module Hecks
+  module Bench
+    # Order statistics over latency samples.
+    # Percentiles are nearest-rank, never interpolated, so every figure is a latency that happened.
+    module Stats
+      module_function
+
+      def percentile(samples, fraction)
+        return nil if samples.empty?
+
+        sorted = samples.sort
+        sorted[[(fraction * sorted.size).ceil - 1, 0].max]
+      end
+
+      def median(values)
+        return nil if values.empty?
+
+        sorted = values.sort
+        middle = sorted.size / 2
+        sorted.size.odd? ? sorted[middle].to_f : (sorted[middle - 1] + sorted[middle]) / 2.0
+      end
+
+      def latency(seconds)
+        {
+          count:   seconds.size,
+          p50_us:  micros(percentile(seconds, 0.50)),
+          p99_us:  micros(percentile(seconds, 0.99)),
+          mean_us: micros(seconds.sum / seconds.size),
+          max_us:  micros(seconds.max)
+        }
+      end
+
+      # A ratio well above 1.0 means latency grows as the store fills, which a whole-run p50 hides.
+      def drift(seconds)
+        tenth = seconds.size / 10
+        return nil if tenth.zero?
+
+        (percentile(seconds.last(tenth), 0.5) / percentile(seconds.first(tenth), 0.5)).round(2)
+      end
+
+      def micros(seconds)
+        seconds && (seconds * 1_000_000).round(1)
+      end
+    end
+  end
+end

data/lib/hecks/bench/suite.rb

@@ -0,0 +1,98 @@
+module Hecks
+  module Bench
+    # Runs every requested target against every requested domain and aggregates the runs.
+    # Runs are interleaved across targets so unrelated machine load lands on all of them.
+    module Suite
+      TARGETS = %w[ruby:memory ruby:sqlite ruby:postgres ruby:postgres_era rust].freeze
+
+      # What to measure; `rust_binary` needs a single domain, since a binary embeds exactly one.
+      Config = Struct.new(:domains, :targets, :warmup, :iterations, :runs, :rust_binary, keyword_init: true)
+
+      module_function
+
+      def call(config, log: $stderr)
+        validate(config)
+        load_before = Environment.load_average
+        skipped = unavailable_targets(config)
+        active = config.targets - skipped.map { |entry| entry[:target] }
+        skipped.each { |entry| log.puts "skip #{entry[:target]}: #{entry[:reason]}" }
+        results = config.domains.flat_map { |domain| measure_domain(Workload.fetch(domain), active, config, log) }
+        environment = Environment.describe
+        environment[:load_average] = { before: load_before, after: environment[:load_average] }
+        { environment: environment.merge(postgres: postgres_version(active)),
+          config: config.to_h, results: results, skipped: skipped }
+      end
+
+      def validate(config)
+        unknown = config.targets - TARGETS
+        raise ArgumentError, "unknown target #{unknown.first.inspect} — one of #{TARGETS.join(', ')}" if unknown.any?
+
+        config.domains.each { |domain| Workload.fetch(domain) }
+        if config.iterations < 1 || config.runs < 1 || config.warmup.negative?
+          raise ArgumentError, "iterations and runs must be at least 1, and warmup at least 0"
+        end
+        return unless config.rust_binary && config.domains.size > 1
+
+        raise ArgumentError, "--rust-binary embeds one domain, so it needs exactly one --domain"
+      end
+
+      def unavailable_targets(config)
+        postgres = PostgresProbe.unavailable_reason if config.targets.any? { |t| t.start_with?("ruby:postgres") }
+        rust = RustRunner.unavailable_reason(binary: config.rust_binary) if config.targets.include?("rust")
+        config.targets.filter_map do |target|
+          reason = target == "rust" ? rust : (postgres if target.start_with?("ruby:postgres"))
+          { target: target, reason: reason } if reason
+        end
+      end
+
+      def measure_domain(workload, targets, config, log)
+        binaries = {}
+        summaries = Hash.new { |hash, target| hash[target] = [] }
+        config.runs.times do |index|
+          targets.each do |target|
+            run = measure_once(workload, target, config, binaries)
+            summaries[target] << run.summary
+            log.puts progress_line("#{workload.name} #{target}", "run #{index + 1}/#{config.runs}", summaries[target].last)
+          end
+        end
+        targets.map { |target| aggregate(workload.name, target, summaries[target]) }
+      end
+
+      def progress_line(label, position, summary)
+        "#{label.ljust(27)} #{position}  #{summary[:throughput_per_s].round.to_s.rjust(8)} cmds/s  " \
+          "p50 #{summary[:p50_us].to_s.rjust(9)} us  p99 #{summary[:p99_us].to_s.rjust(9)} us"
+      end
+
+      def measure_once(workload, target, config, binaries)
+        if target == "rust"
+          binary = config.rust_binary || (binaries[workload.name] ||= RustRunner.build(workload.name))
+          RustRunner.call(workload, binary: binary, warmup: config.warmup, iterations: config.iterations)
+        else
+          RubyRunner.call(workload, adapter: target.delete_prefix("ruby:").to_sym,
+                                    warmup: config.warmup, iterations: config.iterations)
+        end
+      end
+
+      def aggregate(domain, target, summaries)
+        rates = summaries.map { |summary| summary[:throughput_per_s] }
+        median = %i[throughput_per_s p50_us p99_us mean_us max_us drift].to_h do |key|
+          [key, Stats.median(summaries.filter_map { |summary| summary[key] })]
+        end
+        median[:roundtrip_floor_p50_us] = Stats.median(summaries.filter_map { |s| s[:roundtrip_floor_p50_us] })
+        median[:by_verb] = median_by_verb(summaries)
+        { domain: domain, target: target, runs: summaries, median: median.compact,
+          throughput_range: rates.minmax }
+      end
+
+      def median_by_verb(summaries)
+        summaries.first[:by_verb].keys.to_h do |verb|
+          [verb, %i[p50_us p99_us].to_h { |key| [key, Stats.median(summaries.map { |s| s[:by_verb][verb][key] })] }]
+        end
+      end
+
+      def postgres_version(active)
+        PostgresProbe.server_version if active.any? { |target| target.start_with?("ruby:postgres") }
+      end
+    end
+  end
+end