hecks 2.6.0 → 2.8.0

data/lib/hecks/bench/report.rb

@@ -1,116 +0,0 @@
-require "json"
-
-module Hecks
-  module Bench
-    # Renders a `Suite.call` result as Markdown or JSON.
-    #
-    # The Markdown is what `docs/benchmarks.md` pastes in. It states the machine, the
-    # counts and the skipped targets next to the table, so a copied table is never a
-    # bare number.
-    module Report
-      module_function
-
-      # Renders the whole report as Markdown.
-      #
-      # @param result [Hash] the value `Suite.call` returned
-      # @return [String] the environment, the configuration, the table and any skipped targets
-      def markdown(result)
-        [environment_block(result[:environment]), configuration_block(result[:config]),
-         table(result[:results]), floor_block(result[:results]), skipped_block(result[:skipped])]
-          .compact.join("\n\n") << "\n"
-      end
-
-      # Renders the whole report as JSON.
-      #
-      # @param result [Hash] the value `Suite.call` returned
-      # @return [String] pretty-printed JSON of `result`
-      def json(result)
-        JSON.pretty_generate(result)
-      end
-
-      # Renders the results table.
-      #
-      # @param results [Array<Hash>] the `:results` of `Suite.call`
-      # @return [String] a Markdown table with one row per domain and target
-      def table(results)
-        header = ["| domain | target | commands/s | range across runs | p50 (us) | p99 (us) | drift |",
-                  "|---|---|---:|---:|---:|---:|---:|"]
-        (header + results.map { |entry| row(entry) }).join("\n")
-      end
-
-      # Renders one table row.
-      #
-      # @param entry [Hash] one aggregated result
-      # @return [String] the row as Markdown
-      def row(entry)
-        median = entry[:median]
-        low, high = entry[:throughput_range]
-        cells = [entry[:domain], entry[:target], number(median[:throughput_per_s], 0),
-                 "#{number(low, 0)} to #{number(high, 0)}", number(median[:p50_us], 1),
-                 number(median[:p99_us], 1), number(median[:drift], 2)]
-        "| #{cells.join(' | ')} |"
-      end
-
-      # Formats a number with thousands separators.
-      #
-      # @param value [Numeric, nil] the number
-      # @param places [Integer] digits after the decimal point
-      # @return [String] the formatted number, or `"n/a"` for nil
-      def number(value, places)
-        return "n/a" if value.nil?
-
-        whole, fraction = format("%.#{places}f", value).split(".")
-        [whole.gsub(/(\d)(?=(\d{3})+\z)/, '\1,'), fraction].compact.join(".")
-      end
-
-      # Renders the machine and toolchain.
-      #
-      # @param environment [Hash] the `:environment` of `Suite.call`
-      # @return [String] a Markdown bullet list
-      def environment_block(environment)
-        load = environment[:load_average]
-        lines = [
-          "- CPU: #{environment[:cpu]} (#{environment[:cores]} cores), #{environment[:memory_gib]} GiB",
-          "- OS: #{environment[:os]}",
-          "- Ruby: #{environment[:ruby]} (YJIT #{environment[:yjit] ? 'on' : 'off'})",
-          "- Rust: #{environment[:rustc]}; #{environment[:cargo]}",
-          "- Hecks: #{environment[:hecks]} at #{environment[:commit]}",
-          "- Load average (1 min) before and after: #{load[:before]} and #{load[:after]}"
-        ]
-        lines << "- Postgres: #{environment[:postgres]}" if environment[:postgres]
-        lines.join("\n")
-      end
-
-      # Renders how hard the run pushed.
-      #
-      # @param config [Hash] the `:config` of `Suite.call`
-      # @return [String] one sentence with the warmup, iteration and run counts
-      def configuration_block(config)
-        "Warmup #{config[:warmup]} cycles, then #{config[:iterations]} timed cycles per run, " \
-          "median of #{config[:runs]} run(s), domains: #{config[:domains].join(', ')}."
-      end
-
-      # Renders the Rust round-trip floor, if a Rust target ran.
-      #
-      # @param results [Array<Hash>] the `:results` of `Suite.call`
-      # @return [String, nil] a sentence naming each domain's floor, or nil without a Rust result
-      def floor_block(results)
-        floors = results.select { |entry| entry[:target] == "rust" }
-                        .map { |entry| "#{entry[:domain]} #{number(entry[:median][:roundtrip_floor_p50_us], 1)} us" }
-        return nil if floors.empty?
-
-        "Rust pipe round-trip floor (p50, a step refused at once): #{floors.join(', ')}."
-      end
-
-      # Renders the targets that did not run.
-      #
-      # @param skipped [Array<Hash>] the `:skipped` of `Suite.call`
-      # @return [String, nil] one line per skipped target, or nil when none were skipped
-      def skipped_block(skipped)
-        return nil if skipped.empty?
-
-        (["Skipped:"] + skipped.map { |entry| "- #{entry[:target]}: #{entry[:reason]}" }).join("\n")
-      end
-    end
-  end
-end

data/lib/hecks/bench/ruby_runner.rb

@@ -1,151 +0,0 @@
-require_relative "run"
-
-module Hecks
-  module Bench
-    # Measures the Ruby runtime, in-process, on one persistence adapter.
-    #
-    # The domain is booted from a throwaway copy through `Fuzzing::IsolatedBoot`, so a
-    # run always starts from an empty store and never touches the example's own data.
-    # That is the same isolation `bin/fuzz --adapter` uses. Both Postgres adapters write
-    # to a uniquely named schema in the `hecks_bench` database, which is dropped after the
-    # run, so a concurrent `bin/fuzz` and its shared `hecks_fuzz` schema are never touched.
-    #
-    # Only `dispatch_flat` is inside the timer. Boot, setup commands and warmup are not.
-    module RubyRunner
-      # The adapters this runner can bind, as `IsolatedBoot` names them.
-      ADAPTERS = %i[memory sqlite postgres postgres_era].freeze
-
-      # The adapters that need a server and a scratch schema.
-      POSTGRES_ADAPTERS = %i[postgres postgres_era].freeze
-
-      # The scratch database both Postgres adapters run in.
-      DATABASE = "hecks_bench".freeze
-
-      module_function
-
-      # Boots the workload's domain on `adapter` and times its commands.
-      #
-      # @param workload [Workload] the commands to dispatch
-      # @param adapter [Symbol] one of `ADAPTERS`
-      # @param warmup [Integer] cycles dispatched and discarded before timing starts
-      # @param iterations [Integer] cycles timed
-      # @return [Run] the timings of the measured cycles
-      # @raise [ArgumentError] if `adapter` is not one of `ADAPTERS`
-      # @raise [StandardError] whatever refusal a command raises, warmup included, since a
-      #   benchmark that swallowed one would be timing something other than its workload
-      def call(workload, adapter:, warmup:, iterations:)
-        unless ADAPTERS.include?(adapter)
-          raise ArgumentError, "unknown adapter #{adapter.inspect} — one of #{ADAPTERS.join(', ')}"
-        end
-
-        load_dependencies
-        schema = "bench_#{Process.pid}_#{rand(1 << 32).to_s(16)}"
-        quietly_for_postgres do
-          boot(workload, adapter, schema) do |runtime|
-            workload.setup.each { |step| dispatch(runtime, step) }
-            measure(runtime, workload, warmup: warmup, iterations: iterations)
-          end
-        end
-      ensure
-        drop_schema(schema) if schema && POSTGRES_ADAPTERS.include?(adapter)
-      end
-
-      # Boots the workload's domain on `adapter` in an isolated copy.
-      #
-      # @param workload [Workload] the domain to boot
-      # @param adapter [Symbol] one of `ADAPTERS`
-      # @param schema [String] the scratch schema name, used only by the Postgres adapters
-      # @yield [Hecks::Runtime] the booted domain
-      # @return [Object] whatever the block returns
-      def boot(workload, adapter, schema)
-        Hecks::Fuzzing::IsolatedBoot.call(workload.domain_path, adapter: adapter, database: DATABASE,
-                                          schema: schema, scratch: { database: DATABASE, schema: schema }) do |copy|
-          yield Hecks.boot(copy, install_facade: false)
-        end
-      end
-
-      # Keeps Postgres's `schema already exists` notices off stderr while a block runs.
-      #
-      # Every aggregate opens its own connection and each one repeats the notice, which
-      # buries the progress lines. The server setting is restored afterwards.
-      #
-      # @yield the work to run with notices quieted
-      # @return [Object] whatever the block returns
-      def quietly_for_postgres
-        previous = ENV.fetch("PGOPTIONS", nil)
-        ENV["PGOPTIONS"] = [previous, "-c client_min_messages=warning"].compact.join(" ")
-        yield
-      ensure
-        ENV["PGOPTIONS"] = previous
-      end
-
-      # Dispatches warmup cycles untimed, then times the measured ones.
-      #
-      # @param runtime [Hecks::Runtime] the booted domain
-      # @param workload [Workload] the commands to dispatch
-      # @param warmup [Integer] cycles to discard
-      # @param iterations [Integer] cycles to time
-      # @return [Run] the timings of the measured cycles
-      def measure(runtime, workload, warmup:, iterations:)
-        warmup.times { |n| workload.cycle(n).each { |step| dispatch(runtime, step) } }
-        GC.start
-        samples = []
-        started = now
-        iterations.times do |n|
-          workload.cycle(warmup + n).each { |step| samples << [step.verb, time { dispatch(runtime, step) }] }
-        end
-        Run.new(samples: samples, wall_seconds: now - started)
-      end
-
-      # Sends one step to the runtime.
-      #
-      # @param runtime [Hecks::Runtime] the booted domain
-      # @param step [Workload::Step] the command to dispatch
-      # @return [Object] whatever `dispatch_flat` answers
-      def dispatch(runtime, step)
-        runtime.dispatch_flat(step.verb, step.ruby_args)
-      end
-
-      # Reads the monotonic clock.
-      #
-      # @return [Float] seconds on the monotonic clock
-      def now
-        Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      end
-
-      # Times a block.
-      #
-      # @yield the work to time
-      # @return [Float] seconds the block took
-      def time
-        started = now
-        yield
-        now - started
-      end
-
-      # Loads what booting a domain on every adapter needs.
-      #
-      # @return [void]
-      def load_dependencies
-        require "hecks"
-        require "hecks/fuzzing"
-        require "hecks/ports/persistence/plugins/era"
-      end
-
-      # Removes the schema a Postgres run created, so repeated runs do not accumulate.
-      #
-      # @param schema [String] the schema name this run used
-      # @return [void]
-      def drop_schema(schema)
-        require "pg"
-        connection = PG.connect(dbname: DATABASE, connect_timeout: 2)
-        connection.exec("SET client_min_messages = warning")
-        connection.exec("DROP SCHEMA IF EXISTS #{connection.quote_ident(schema)} CASCADE")
-      rescue LoadError, PG::Error
-        nil
-      ensure
-        connection&.close
-      end
-    end
-  end
-end

data/lib/hecks/bench/run.rb

@@ -1,55 +0,0 @@
-module Hecks
-  module Bench
-    # The raw timings of one measured pass over a workload on one runtime.
-    #
-    # Runners produce these and know nothing about reporting. {#summary} is the only
-    # place raw samples become the figures a report prints.
-    class Run
-      # @return [Array<Array(String, Float)>] each timed command's verb and latency in
-      #   seconds, in dispatch order
-      attr_reader :samples
-
-      # @return [Float] seconds from the first timed command's start to the last one's end
-      attr_reader :wall_seconds
-
-      # @return [Hash{Symbol => Object}] facts a runner wants reported alongside the timings
-      attr_reader :extras
-
-      # Builds a run.
-      #
-      # @param samples [Array<Array(String, Float)>] verb and latency seconds per command
-      # @param wall_seconds [Float] the wall-clock duration of the whole timed pass
-      # @param extras [Hash{Symbol => Object}] runner-specific facts, such as a round-trip floor
-      def initialize(samples:, wall_seconds:, extras: {})
-        @samples = samples
-        @wall_seconds = wall_seconds
-        @extras = extras
-      end
-
-      # Reduces the samples to the figures a report prints.
-      #
-      # @return [Hash{Symbol => Object}] `:commands`, `:wall_seconds`, `:throughput_per_s`,
-      #   the `Stats.latency` figures, `:drift`, `:by_verb` (each verb's count, p50 and p99),
-      #   and any `extras`
-      def summary
-        seconds = samples.map(&:last)
-        {
-          commands:         samples.size,
-          wall_seconds:     wall_seconds.round(4),
-          throughput_per_s: (samples.size / wall_seconds).round(1),
-          **Stats.latency(seconds),
-          drift:            Stats.drift(seconds),
-          by_verb:          by_verb
-        }.merge(extras)
-      end
-
-      private
-
-      def by_verb
-        samples.group_by(&:first).transform_values do |group|
-          Stats.latency(group.map(&:last)).slice(:count, :p50_us, :p99_us)
-        end
-      end
-    end
-  end
-end

data/lib/hecks/bench/rust_runner.rb

@@ -1,165 +0,0 @@
-require "fileutils"
-require "open3"
-require_relative "run"
-
-module Hecks
-  module Bench
-    # Measures the native Rust binary through its streaming mode.
-    #
-    # `rust --serve` reads one JSON step per line and answers one JSON line each, keeping
-    # its in-memory store alive between them. This runner writes a step, waits for the
-    # answer line, and times the whole round trip, so the figures include two pipe
-    # crossings and the binary's own JSON parse and print. Process start-up is outside
-    # the timer.
-    #
-    # Alongside each run it measures a **round-trip floor**: the same round trip for a
-    # step the binary refuses immediately, which is what the pipes cost with almost no
-    # domain work behind them.
-    module RustRunner
-      # Where the Rust crate lives.
-      RUST_DIR = File.expand_path("../../../rust", __dir__)
-
-      # How many refused-at-once steps make up the round-trip floor's sample.
-      FLOOR_SAMPLES = 500
-
-      # A `cargo build` that exited non-zero.
-      class BuildFailed < StandardError; end
-
-      # A benchmarked step the binary refused or could not answer.
-      class StepRefused < StandardError; end
-
-      module_function
-
-      # Says why the Rust target cannot run, if it cannot.
-      #
-      # @param binary [String, nil] an explicit pre-built binary, which needs no toolchain
-      # @return [String, nil] nil when the target can run, otherwise one sentence saying why
-      def unavailable_reason(binary: nil)
-        return "the binary #{binary} does not exist or is not executable" if binary && !File.executable?(binary)
-        return nil if binary || Environment.capture("cargo", "-V") != "unknown"
-
-        "`cargo` is not on PATH and no --rust-binary was given"
-      end
-
-      # Builds the release binary for one domain's Cargo feature and pins a copy of it.
-      #
-      # The copy has its own name so building another domain's feature afterwards cannot
-      # replace it.
-      #
-      # @param feature [String] the domain's Cargo feature, its directory name
-      # @param rust_dir [String] the Rust crate's directory
-      # @return [String] the path of the pinned release binary
-      # @raise [BuildFailed] if cargo exits non-zero or leaves no binary
-      def build(feature, rust_dir: RUST_DIR)
-        output, status = Open3.capture2e("cargo", "build", "--release", "--no-default-features",
-                                         "--features", feature, chdir: rust_dir)
-        unless status.success?
-          raise BuildFailed,
-                "cargo build --release --features #{feature} failed:\n#{output.lines.last(15).join}"
-        end
-
-        target = File.join(ENV.fetch("CARGO_TARGET_DIR", File.join(rust_dir, "target")), "release")
-        pinned = File.join(target, "rust-bench-#{feature}")
-        FileUtils.cp(File.join(target, "rust"), pinned)
-        pinned
-      end
-
-      # Starts the binary in serve mode and times the workload's commands.
-      #
-      # @param workload [Workload] the commands to send
-      # @param binary [String] the path of a binary built for `workload`'s domain
-      # @param warmup [Integer] cycles sent and discarded before timing starts
-      # @param iterations [Integer] cycles timed
-      # @return [Run] the timings, with the round-trip floor in its extras as
-      #   `:roundtrip_floor_p50_us`
-      # @raise [StepRefused] if the binary refuses any step, warmup included
-      def call(workload, binary:, warmup:, iterations:)
-        IO.popen([binary, "--serve"], "r+") do |pipe|
-          pipe.sync = true
-          workload.setup.each { |step| ask(pipe, step.to_json_line) }
-          warmup.times { |n| workload.cycle(n).each { |step| ask(pipe, step.to_json_line) } }
-          run = measure(pipe, workload, warmup: warmup, iterations: iterations)
-          Run.new(samples: run.samples, wall_seconds: run.wall_seconds,
-                  extras: { roundtrip_floor_p50_us: floor(pipe) })
-        ensure
-          pipe.close_write unless pipe.closed?
-        end
-      end
-
-      # Times the measured cycles.
-      #
-      # @param pipe [IO] the binary's stdin and stdout
-      # @param workload [Workload] the commands to send
-      # @param warmup [Integer] cycles already sent, so record names continue from there
-      # @param iterations [Integer] cycles to time
-      # @return [Run] the timings, without extras
-      def measure(pipe, workload, warmup:, iterations:)
-        samples = []
-        started = now
-        iterations.times do |n|
-          workload.cycle(warmup + n).each do |step|
-            line = step.to_json_line
-            began = now
-            answer = round_trip(pipe, line)
-            samples << [step.verb, now - began]
-            check(answer, step.verb)
-          end
-        end
-        Run.new(samples: samples, wall_seconds: now - started)
-      end
-
-      # Measures the pipes with almost no work behind them.
-      #
-      # @param pipe [IO] the binary's stdin and stdout
-      # @return [Float] the p50 microseconds of {FLOOR_SAMPLES} round trips for an empty step,
-      #   which the binary refuses without touching its store
-      def floor(pipe)
-        seconds = Array.new(FLOOR_SAMPLES) do
-          began = now
-          round_trip(pipe, "{}")
-          now - began
-        end
-        Stats.micros(Stats.percentile(seconds, 0.5))
-      end
-
-      # Sends one step and returns the answer line, untimed.
-      #
-      # @param pipe [IO] the binary's stdin and stdout
-      # @param line [String] the step as one JSON line
-      # @return [void]
-      # @raise [StepRefused] if the binary refuses the step
-      def ask(pipe, line)
-        check(round_trip(pipe, line), line[0, 80])
-      end
-
-      # Writes one line and reads one line back.
-      #
-      # @param pipe [IO] the binary's stdin and stdout
-      # @param line [String] the step as one JSON line
-      # @return [String] the answer line, or an empty string if the binary closed its output
-      def round_trip(pipe, line)
-        pipe.write(line, "\n")
-        pipe.gets.to_s
-      end
-
-      # Refuses an answer that is not a success.
-      #
-      # @param answer [String] the binary's answer line
-      # @param what [String] what was asked, for the error message
-      # @return [void]
-      # @raise [StepRefused] unless `answer` starts with `{"ok":true`
-      def check(answer, what)
-        return if answer.start_with?('{"ok":true')
-
-        raise StepRefused, "#{what} was not accepted: #{answer.strip.empty? ? 'the binary closed its output' : answer.strip}"
-      end
-
-      # Reads the monotonic clock.
-      #
-      # @return [Float] seconds on the monotonic clock
-      def now
-        Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      end
-    end
-  end
-end

data/lib/hecks/bench/stats.rb

@@ -1,75 +0,0 @@
-module Hecks
-  module Bench
-    # Order statistics over a list of latency samples.
-    #
-    # Percentiles use the nearest-rank method: the smallest sample with at least
-    # that fraction of the samples at or below it. It never interpolates, so every
-    # reported figure is a latency that actually happened. With fewer than 100
-    # samples the p99 is simply the maximum.
-    module Stats
-      module_function
-
-      # Reads one percentile from a list of samples.
-      #
-      # @param samples [Array<Float>] the samples, in any order
-      # @param fraction [Float] the percentile as a fraction, e.g. `0.99`
-      # @return [Float, nil] the nearest-rank percentile, or nil when `samples` is empty
-      def percentile(samples, fraction)
-        return nil if samples.empty?
-
-        sorted = samples.sort
-        sorted[[(fraction * sorted.size).ceil - 1, 0].max]
-      end
-
-      # Finds the middle value of a list, averaging the two middle values of an even one.
-      #
-      # @param values [Array<Numeric>] the values, in any order
-      # @return [Float, nil] the median, or nil when `values` is empty
-      def median(values)
-        return nil if values.empty?
-
-        sorted = values.sort
-        middle = sorted.size / 2
-        sorted.size.odd? ? sorted[middle].to_f : (sorted[middle - 1] + sorted[middle]) / 2.0
-      end
-
-      # Summarizes one run's latency samples.
-      #
-      # @param seconds [Array<Float>] per-command latencies in seconds, in dispatch order
-      # @return [Hash{Symbol => Numeric}] `:count`, and `:p50_us`, `:p99_us`, `:mean_us` and
-      #   `:max_us` in microseconds
-      def latency(seconds)
-        {
-          count:   seconds.size,
-          p50_us:  micros(percentile(seconds, 0.50)),
-          p99_us:  micros(percentile(seconds, 0.99)),
-          mean_us: micros(seconds.sum / seconds.size),
-          max_us:  micros(seconds.max)
-        }
-      end
-
-      # Compares how slow the last tenth of a run was against its first tenth.
-      #
-      # A ratio well above 1.0 means latency grows as the store fills, which a single
-      # p50 over the whole run would hide.
-      #
-      # @param seconds [Array<Float>] per-command latencies in dispatch order
-      # @return [Float, nil] the last tenth's p50 over the first tenth's, or nil when there
-      #   are fewer than ten samples
-      def drift(seconds)
-        tenth = seconds.size / 10
-        return nil if tenth.zero?
-
-        (percentile(seconds.last(tenth), 0.5) / percentile(seconds.first(tenth), 0.5)).round(2)
-      end
-
-      # Converts seconds to microseconds, rounded to a tenth.
-      #
-      # @param seconds [Float, nil] a duration in seconds
-      # @return [Float, nil] the duration in microseconds, or nil when `seconds` is nil
-      def micros(seconds)
-        seconds && (seconds * 1_000_000).round(1)
-      end
-    end
-  end
-end