hecks 2.5.1 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b63cae5fd3cc67091185210b80a7fa73e42977f28db96563ad46c50d37dc462
-  data.tar.gz: 0b5ec871c1ba7bf000bc2a65fdd308c9618a14ae07977fe76a2ed06a7e434399
+  metadata.gz: 2220925ed910b9c63dd34c3fc75448f6cff634a64be3fb67461985d462e83563
+  data.tar.gz: 618319249d19f123171d300578ba8d70556391371457700504c92ba4bd389935
 SHA512:
-  metadata.gz: 47a19c9674469250e49357f8ea57087721274c0a749a4006d3674e5c8d19babe8d57a692cc4750c7de34960a863ddad0596e7587abe8f9da2c6392e6488d5b52
-  data.tar.gz: 320fbf1791da47bb4247d56ab7f3943dd38ec08dff630eda886955ac24b80b4bf69f59040305f231735e96e586b6f6ca3b6c0068fc44d08995163834dc305dd3
+  metadata.gz: a24cd51e0ba290e819645aec45cbc8a55c9978cb1c5fed673180db103a050a77516ebcd224d0102fad523161ce4776866b5702acf790dd0751282c2c399e73a9
+  data.tar.gz: a779523871cd0e81d87c26305bed16e51916723ef544e55ccc30d669b6fdb38b825ee2a05515ae9f8c2f40ff31e4faa57f4d5dffc0dac68146d900d112830d92

data/lib/hecks/bench/cli.rb

@@ -0,0 +1,74 @@
+require "optparse"
+require_relative "suite"
+require_relative "report"
+
+module Hecks
+  module Bench
+    # The command line behind `bin/bench`: parses flags, runs the `Suite`, prints the report.
+    #
+    # Kept in `lib/` so a spec can drive it with a tiny configuration and read what it
+    # prints, rather than shelling out to a script.
+    module CLI
+      # The defaults `bin/bench` runs with when no flag says otherwise.
+      DEFAULTS = { domains: %w[pizzas banking], targets: Suite::TARGETS, warmup: 200, iterations: 1000,
+                   runs: 3 }.freeze
+
+      module_function
+
+      # Runs the benchmark described by the command line and prints its report.
+      #
+      # @param argv [Array<String>] the command-line arguments
+      # @param out [#puts] where the report goes
+      # @param err [#puts] where progress and usage errors go
+      # @return [Integer] the process exit status: 0 on success, 2 for a usage error
+      def run(argv, out: $stdout, err: $stderr)
+        options = parse(argv)
+        result = Suite.call(options.fetch(:config), log: err)
+        out.puts(options[:format] == "json" ? Report.json(result) : Report.markdown(result))
+        File.write(options[:output], Report.json(result)) if options[:output]
+        0
+      rescue OptionParser::ParseError, ArgumentError => e
+        err.puts "bin/bench: #{e.message}"
+        2
+      end
+
+      # Turns command-line arguments into a configuration and output choices.
+      #
+      # @param argv [Array<String>] the command-line arguments
+      # @return [Hash{Symbol => Object}] `:config` (a `Suite::Config`), `:format` and `:output`
+      # @raise [OptionParser::ParseError] on an unknown or malformed flag
+      def parse(argv)
+        values = DEFAULTS.dup
+        extras = { format: "markdown", output: nil, rust_binary: nil }
+        parser(values, extras).parse(argv.dup)
+        { config: Suite::Config.new(**values, rust_binary: extras[:rust_binary]),
+          format: extras[:format], output: extras[:output] }
+      end
+
+      # Builds the flag parser.
+      #
+      # @param values [Hash{Symbol => Object}] the `Suite::Config` fields, filled in as flags parse
+      # @param extras [Hash{Symbol => Object}] the output choices, filled in as flags parse
+      # @return [OptionParser] the parser
+      def parser(values, extras)
+        OptionParser.new do |opts|
+          opts.banner = "usage: bin/bench [options]"
+          opts.on("--domain NAMES", Array, "pizzas, banking (default: both)") { |v| values[:domains] = v }
+          opts.on("--targets NAMES", Array, "#{Suite::TARGETS.join(', ')} (default: all)") { |v| values[:targets] = v }
+          opts.on("--iterations N", Integer, "timed cycles per run (default #{DEFAULTS[:iterations]})") do |v|
+            values[:iterations] = v
+          end
+          opts.on("--warmup N", Integer, "discarded cycles before timing (default #{DEFAULTS[:warmup]})") do |v|
+            values[:warmup] = v
+          end
+          opts.on("--runs N", Integer, "fresh boots per target, median reported (default #{DEFAULTS[:runs]})") do |v|
+            values[:runs] = v
+          end
+          opts.on("--rust-binary PATH", "use this `rust` binary instead of building one") { |v| extras[:rust_binary] = v }
+          opts.on("--format FORMAT", %w[markdown json], "markdown (default) or json") { |v| extras[:format] = v }
+          opts.on("--output PATH", "also write the full JSON result here") { |v| extras[:output] = v }
+        end
+      end
+    end
+  end
+end

data/lib/hecks/bench/environment.rb

@@ -0,0 +1,100 @@
+require "etc"
+require "open3"
+
+module Hecks
+  module Bench
+    # A description of the machine and toolchain a benchmark ran on.
+    #
+    # A number without its hardware is not reproducible, so every report carries
+    # this. Each probe is best effort: a missing tool becomes `"unknown"` rather
+    # than an error, since the benchmark itself is still valid without it.
+    module Environment
+      module_function
+
+      # Collects the machine and toolchain facts a report should carry.
+      #
+      # @return [Hash{Symbol => Object}] `:cpu`, `:cores`, `:memory_gib`, `:os`, `:ruby`,
+      #   `:yjit`, `:rustc`, `:cargo`, `:hecks`, `:commit` and `:load_average`
+      def describe
+        {
+          cpu: cpu, cores: Etc.nprocessors, memory_gib: memory_gib, os: os,
+          ruby: RUBY_DESCRIPTION, yjit: yjit?,
+          rustc: capture("rustc", "-V"), cargo: capture("cargo", "-V"),
+          hecks: Hecks::VERSION, commit: commit, load_average: load_average
+        }
+      end
+
+      # Reads the one-minute load average, the quickest sign that something else was
+      # busy while a benchmark ran. `Suite.call` reads it before and after and reports both.
+      #
+      # @return [Float, nil] the one-minute load average, or nil when it cannot be read
+      def load_average
+        text = RUBY_PLATFORM.include?("darwin") ? capture("sysctl", "-n", "vm.loadavg").delete("{}") : File.read("/proc/loadavg")
+        text.split.first.to_f.round(2)
+      rescue SystemCallError
+        nil
+      end
+
+      # Names the processor.
+      #
+      # @return [String] the CPU model, or `"unknown"`
+      def cpu
+        return capture("sysctl", "-n", "machdep.cpu.brand_string") if RUBY_PLATFORM.include?("darwin")
+
+        model = File.foreach("/proc/cpuinfo").find { |line| line.start_with?("model name") }
+        model ? model.split(":", 2).last.strip : "unknown"
+      rescue SystemCallError
+        "unknown"
+      end
+
+      # Reads installed memory.
+      #
+      # @return [Float, String] physical memory in GiB, or `"unknown"`
+      def memory_gib
+        bytes =
+          if RUBY_PLATFORM.include?("darwin")
+            capture("sysctl", "-n", "hw.memsize").to_i
+          else
+            File.read("/proc/meminfo")[/MemTotal:\s+(\d+)/, 1].to_i * 1024
+          end
+        bytes.zero? ? "unknown" : (bytes / (1024.0**3)).round(1)
+      rescue SystemCallError
+        "unknown"
+      end
+
+      # Names the operating system.
+      #
+      # @return [String] the OS name and version, or `"unknown"`
+      def os
+        return "macOS #{capture('sw_vers', '-productVersion')} (#{capture('uname', '-m')})" if RUBY_PLATFORM.include?("darwin")
+
+        capture("uname", "-sr")
+      end
+
+      # Says whether Ruby's `YJIT` compiler was running.
+      #
+      # @return [Boolean] true when `YJIT` is enabled in this process
+      def yjit?
+        defined?(RubyVM::YJIT) ? RubyVM::YJIT.enabled? : false
+      end
+
+      # Names the Hecks source revision under test.
+      #
+      # @return [String] the short commit hash, or `"unknown"` outside a git checkout
+      def commit
+        capture("git", "-C", File.expand_path("../../..", __dir__), "rev-parse", "--short", "HEAD")
+      end
+
+      # Runs a command and returns its first line of output.
+      #
+      # @param command [Array<String>] the program and its arguments
+      # @return [String] the trimmed first line of stdout, or `"unknown"` if it cannot run
+      def capture(*command)
+        out, status = Open3.capture2(*command, err: File::NULL)
+        status.success? && !out.strip.empty? ? out.lines.first.strip : "unknown"
+      rescue SystemCallError
+        "unknown"
+      end
+    end
+  end
+end

data/lib/hecks/bench/postgres_probe.rb

@@ -0,0 +1,41 @@
+module Hecks
+  module Bench
+    # Decides whether the Postgres targets can run at all.
+    #
+    # Postgres is optional. A machine with no server, or without the `pg` gem, gets a
+    # skipped target and a sentence saying why, never a stack trace. Connection
+    # settings come from libpq's own environment (`PGHOST`, `PGPORT`, `PGUSER`,
+    # `PGPASSWORD`), the same way the fuzzer's Postgres adapters read them.
+    module PostgresProbe
+      module_function
+
+      # Tries to reach the server's maintenance database.
+      #
+      # @return [String, nil] nil when a server answered, otherwise one sentence saying why
+      #   the Postgres targets cannot run
+      def unavailable_reason
+        require "pg"
+        PG.connect(dbname: "postgres", connect_timeout: 2).close
+        nil
+      rescue LoadError
+        "the `pg` gem is not installed"
+      rescue PG::Error => e
+        "no Postgres server is reachable (#{e.message.lines.first.to_s.strip}); start one, or point " \
+        "PGHOST, PGPORT and PGUSER at one"
+      end
+
+      # Reads the server's version for the report's environment block.
+      #
+      # @return [String] the `server_version` setting, or `"unknown"` if it cannot be read
+      def server_version
+        require "pg"
+        connection = PG.connect(dbname: "postgres", connect_timeout: 2)
+        connection.exec("SHOW server_version").getvalue(0, 0)
+      rescue LoadError, PG::Error
+        "unknown"
+      ensure
+        connection&.close
+      end
+    end
+  end
+end

data/lib/hecks/bench/report.rb

@@ -0,0 +1,116 @@
+require "json"
+
+module Hecks
+  module Bench
+    # Renders a `Suite.call` result as Markdown or JSON.
+    #
+    # The Markdown is what `docs/benchmarks.md` pastes in. It states the machine, the
+    # counts and the skipped targets next to the table, so a copied table is never a
+    # bare number.
+    module Report
+      module_function
+
+      # Renders the whole report as Markdown.
+      #
+      # @param result [Hash] the value `Suite.call` returned
+      # @return [String] the environment, the configuration, the table and any skipped targets
+      def markdown(result)
+        [environment_block(result[:environment]), configuration_block(result[:config]),
+         table(result[:results]), floor_block(result[:results]), skipped_block(result[:skipped])]
+          .compact.join("\n\n") << "\n"
+      end
+
+      # Renders the whole report as JSON.
+      #
+      # @param result [Hash] the value `Suite.call` returned
+      # @return [String] pretty-printed JSON of `result`
+      def json(result)
+        JSON.pretty_generate(result)
+      end
+
+      # Renders the results table.
+      #
+      # @param results [Array<Hash>] the `:results` of `Suite.call`
+      # @return [String] a Markdown table with one row per domain and target
+      def table(results)
+        header = ["| domain | target | commands/s | range across runs | p50 (us) | p99 (us) | drift |",
+                  "|---|---|---:|---:|---:|---:|---:|"]
+        (header + results.map { |entry| row(entry) }).join("\n")
+      end
+
+      # Renders one table row.
+      #
+      # @param entry [Hash] one aggregated result
+      # @return [String] the row as Markdown
+      def row(entry)
+        median = entry[:median]
+        low, high = entry[:throughput_range]
+        cells = [entry[:domain], entry[:target], number(median[:throughput_per_s], 0),
+                 "#{number(low, 0)} to #{number(high, 0)}", number(median[:p50_us], 1),
+                 number(median[:p99_us], 1), number(median[:drift], 2)]
+        "| #{cells.join(' | ')} |"
+      end
+
+      # Formats a number with thousands separators.
+      #
+      # @param value [Numeric, nil] the number
+      # @param places [Integer] digits after the decimal point
+      # @return [String] the formatted number, or `"n/a"` for nil
+      def number(value, places)
+        return "n/a" if value.nil?
+
+        whole, fraction = format("%.#{places}f", value).split(".")
+        [whole.gsub(/(\d)(?=(\d{3})+\z)/, '\1,'), fraction].compact.join(".")
+      end
+
+      # Renders the machine and toolchain.
+      #
+      # @param environment [Hash] the `:environment` of `Suite.call`
+      # @return [String] a Markdown bullet list
+      def environment_block(environment)
+        load = environment[:load_average]
+        lines = [
+          "- CPU: #{environment[:cpu]} (#{environment[:cores]} cores), #{environment[:memory_gib]} GiB",
+          "- OS: #{environment[:os]}",
+          "- Ruby: #{environment[:ruby]} (YJIT #{environment[:yjit] ? 'on' : 'off'})",
+          "- Rust: #{environment[:rustc]}; #{environment[:cargo]}",
+          "- Hecks: #{environment[:hecks]} at #{environment[:commit]}",
+          "- Load average (1 min) before and after: #{load[:before]} and #{load[:after]}"
+        ]
+        lines << "- Postgres: #{environment[:postgres]}" if environment[:postgres]
+        lines.join("\n")
+      end
+
+      # Renders how hard the run pushed.
+      #
+      # @param config [Hash] the `:config` of `Suite.call`
+      # @return [String] one sentence with the warmup, iteration and run counts
+      def configuration_block(config)
+        "Warmup #{config[:warmup]} cycles, then #{config[:iterations]} timed cycles per run, " \
+          "median of #{config[:runs]} run(s), domains: #{config[:domains].join(', ')}."
+      end
+
+      # Renders the Rust round-trip floor, if a Rust target ran.
+      #
+      # @param results [Array<Hash>] the `:results` of `Suite.call`
+      # @return [String, nil] a sentence naming each domain's floor, or nil without a Rust result
+      def floor_block(results)
+        floors = results.select { |entry| entry[:target] == "rust" }
+                        .map { |entry| "#{entry[:domain]} #{number(entry[:median][:roundtrip_floor_p50_us], 1)} us" }
+        return nil if floors.empty?
+
+        "Rust pipe round-trip floor (p50, a step refused at once): #{floors.join(', ')}."
+      end
+
+      # Renders the targets that did not run.
+      #
+      # @param skipped [Array<Hash>] the `:skipped` of `Suite.call`
+      # @return [String, nil] one line per skipped target, or nil when none were skipped
+      def skipped_block(skipped)
+        return nil if skipped.empty?
+
+        (["Skipped:"] + skipped.map { |entry| "- #{entry[:target]}: #{entry[:reason]}" }).join("\n")
+      end
+    end
+  end
+end

data/lib/hecks/bench/ruby_runner.rb

@@ -0,0 +1,151 @@
+require_relative "run"
+
+module Hecks
+  module Bench
+    # Measures the Ruby runtime, in-process, on one persistence adapter.
+    #
+    # The domain is booted from a throwaway copy through `Fuzzing::IsolatedBoot`, so a
+    # run always starts from an empty store and never touches the example's own data.
+    # That is the same isolation `bin/fuzz --adapter` uses. Both Postgres adapters write
+    # to a uniquely named schema in the `hecks_bench` database, which is dropped after the
+    # run, so a concurrent `bin/fuzz` and its shared `hecks_fuzz` schema are never touched.
+    #
+    # Only `dispatch_flat` is inside the timer. Boot, setup commands and warmup are not.
+    module RubyRunner
+      # The adapters this runner can bind, as `IsolatedBoot` names them.
+      ADAPTERS = %i[memory sqlite postgres postgres_era].freeze
+
+      # The adapters that need a server and a scratch schema.
+      POSTGRES_ADAPTERS = %i[postgres postgres_era].freeze
+
+      # The scratch database both Postgres adapters run in.
+      DATABASE = "hecks_bench".freeze
+
+      module_function
+
+      # Boots the workload's domain on `adapter` and times its commands.
+      #
+      # @param workload [Workload] the commands to dispatch
+      # @param adapter [Symbol] one of `ADAPTERS`
+      # @param warmup [Integer] cycles dispatched and discarded before timing starts
+      # @param iterations [Integer] cycles timed
+      # @return [Run] the timings of the measured cycles
+      # @raise [ArgumentError] if `adapter` is not one of `ADAPTERS`
+      # @raise [StandardError] whatever refusal a command raises, warmup included, since a
+      #   benchmark that swallowed one would be timing something other than its workload
+      def call(workload, adapter:, warmup:, iterations:)
+        unless ADAPTERS.include?(adapter)
+          raise ArgumentError, "unknown adapter #{adapter.inspect} — one of #{ADAPTERS.join(', ')}"
+        end
+
+        load_dependencies
+        schema = "bench_#{Process.pid}_#{rand(1 << 32).to_s(16)}"
+        quietly_for_postgres do
+          boot(workload, adapter, schema) do |runtime|
+            workload.setup.each { |step| dispatch(runtime, step) }
+            measure(runtime, workload, warmup: warmup, iterations: iterations)
+          end
+        end
+      ensure
+        drop_schema(schema) if schema && POSTGRES_ADAPTERS.include?(adapter)
+      end
+
+      # Boots the workload's domain on `adapter` in an isolated copy.
+      #
+      # @param workload [Workload] the domain to boot
+      # @param adapter [Symbol] one of `ADAPTERS`
+      # @param schema [String] the scratch schema name, used only by the Postgres adapters
+      # @yield [Hecks::Runtime] the booted domain
+      # @return [Object] whatever the block returns
+      def boot(workload, adapter, schema)
+        Hecks::Fuzzing::IsolatedBoot.call(workload.domain_path, adapter: adapter, database: DATABASE,
+                                          schema: schema, scratch: { database: DATABASE, schema: schema }) do |copy|
+          yield Hecks.boot(copy, install_facade: false)
+        end
+      end
+
+      # Keeps Postgres's `schema already exists` notices off stderr while a block runs.
+      #
+      # Every aggregate opens its own connection and each one repeats the notice, which
+      # buries the progress lines. The server setting is restored afterwards.
+      #
+      # @yield the work to run with notices quieted
+      # @return [Object] whatever the block returns
+      def quietly_for_postgres
+        previous = ENV.fetch("PGOPTIONS", nil)
+        ENV["PGOPTIONS"] = [previous, "-c client_min_messages=warning"].compact.join(" ")
+        yield
+      ensure
+        ENV["PGOPTIONS"] = previous
+      end
+
+      # Dispatches warmup cycles untimed, then times the measured ones.
+      #
+      # @param runtime [Hecks::Runtime] the booted domain
+      # @param workload [Workload] the commands to dispatch
+      # @param warmup [Integer] cycles to discard
+      # @param iterations [Integer] cycles to time
+      # @return [Run] the timings of the measured cycles
+      def measure(runtime, workload, warmup:, iterations:)
+        warmup.times { |n| workload.cycle(n).each { |step| dispatch(runtime, step) } }
+        GC.start
+        samples = []
+        started = now
+        iterations.times do |n|
+          workload.cycle(warmup + n).each { |step| samples << [step.verb, time { dispatch(runtime, step) }] }
+        end
+        Run.new(samples: samples, wall_seconds: now - started)
+      end
+
+      # Sends one step to the runtime.
+      #
+      # @param runtime [Hecks::Runtime] the booted domain
+      # @param step [Workload::Step] the command to dispatch
+      # @return [Object] whatever `dispatch_flat` answers
+      def dispatch(runtime, step)
+        runtime.dispatch_flat(step.verb, step.ruby_args)
+      end
+
+      # Reads the monotonic clock.
+      #
+      # @return [Float] seconds on the monotonic clock
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      # Times a block.
+      #
+      # @yield the work to time
+      # @return [Float] seconds the block took
+      def time
+        started = now
+        yield
+        now - started
+      end
+
+      # Loads what booting a domain on every adapter needs.
+      #
+      # @return [void]
+      def load_dependencies
+        require "hecks"
+        require "hecks/fuzzing"
+        require "hecks/ports/persistence/plugins/era"
+      end
+
+      # Removes the schema a Postgres run created, so repeated runs do not accumulate.
+      #
+      # @param schema [String] the schema name this run used
+      # @return [void]
+      def drop_schema(schema)
+        require "pg"
+        connection = PG.connect(dbname: DATABASE, connect_timeout: 2)
+        connection.exec("SET client_min_messages = warning")
+        connection.exec("DROP SCHEMA IF EXISTS #{connection.quote_ident(schema)} CASCADE")
+      rescue LoadError, PG::Error
+        nil
+      ensure
+        connection&.close
+      end
+    end
+  end
+end

data/lib/hecks/bench/run.rb

@@ -0,0 +1,55 @@
+module Hecks
+  module Bench
+    # The raw timings of one measured pass over a workload on one runtime.
+    #
+    # Runners produce these and know nothing about reporting. {#summary} is the only
+    # place raw samples become the figures a report prints.
+    class Run
+      # @return [Array<Array(String, Float)>] each timed command's verb and latency in
+      #   seconds, in dispatch order
+      attr_reader :samples
+
+      # @return [Float] seconds from the first timed command's start to the last one's end
+      attr_reader :wall_seconds
+
+      # @return [Hash{Symbol => Object}] facts a runner wants reported alongside the timings
+      attr_reader :extras
+
+      # Builds a run.
+      #
+      # @param samples [Array<Array(String, Float)>] verb and latency seconds per command
+      # @param wall_seconds [Float] the wall-clock duration of the whole timed pass
+      # @param extras [Hash{Symbol => Object}] runner-specific facts, such as a round-trip floor
+      def initialize(samples:, wall_seconds:, extras: {})
+        @samples = samples
+        @wall_seconds = wall_seconds
+        @extras = extras
+      end
+
+      # Reduces the samples to the figures a report prints.
+      #
+      # @return [Hash{Symbol => Object}] `:commands`, `:wall_seconds`, `:throughput_per_s`,
+      #   the `Stats.latency` figures, `:drift`, `:by_verb` (each verb's count, p50 and p99),
+      #   and any `extras`
+      def summary
+        seconds = samples.map(&:last)
+        {
+          commands:         samples.size,
+          wall_seconds:     wall_seconds.round(4),
+          throughput_per_s: (samples.size / wall_seconds).round(1),
+          **Stats.latency(seconds),
+          drift:            Stats.drift(seconds),
+          by_verb:          by_verb
+        }.merge(extras)
+      end
+
+      private
+
+      def by_verb
+        samples.group_by(&:first).transform_values do |group|
+          Stats.latency(group.map(&:last)).slice(:count, :p50_us, :p99_us)
+        end
+      end
+    end
+  end
+end