hecks 2.7.0 → 2.8.0

data/lib/hecks/bench/suite.rb

@@ -1,172 +0,0 @@
-module Hecks
-  module Bench
-    # Runs every requested target against every requested domain and aggregates the runs.
-    #
-    # Each domain's runs are interleaved across targets (memory, sqlite, ..., memory,
-    # sqlite, ...) rather than finished one target at a time, so a burst of unrelated load on
-    # the machine lands on every target instead of flattering one of them. Each figure in the
-    # result is the median across runs, with the fastest and slowest throughput kept so a noisy
-    # measurement shows.
-    module Suite
-      # Every target `bin/bench` can measure.
-      TARGETS = %w[ruby:memory ruby:sqlite ruby:postgres ruby:postgres_era rust].freeze
-
-      # What to measure and how hard.
-      #
-      # @!attribute domains
-      #   @return [Array<String>] workload names, keys of `Workload.all`
-      # @!attribute targets
-      #   @return [Array<String>] members of `TARGETS`
-      # @!attribute warmup
-      #   @return [Integer] cycles discarded before each run is timed
-      # @!attribute iterations
-      #   @return [Integer] cycles timed in each run
-      # @!attribute runs
-      #   @return [Integer] how many fresh boots of each target to measure per domain
-      # @!attribute rust_binary
-      #   @return [String, nil] a pre-built binary to use instead of building one; only valid
-      #     with a single domain, since a binary embeds exactly one
-      Config = Struct.new(:domains, :targets, :warmup, :iterations, :runs, :rust_binary, keyword_init: true)
-
-      module_function
-
-      # Measures everything `config` asks for.
-      #
-      # @param config [Config] what to measure
-      # @param log [#puts] where progress lines go; one line per run
-      # @return [Hash{Symbol => Object}] `:environment`, `:config`, `:results` (one entry per
-      #   domain and target, see `.aggregate`) and `:skipped` (each skipped target and why)
-      # @raise [ArgumentError] if a target or domain is unknown, a count is not positive, or
-      #   `rust_binary` is combined with more than one domain
-      def call(config, log: $stderr)
-        validate(config)
-        load_before = Environment.load_average
-        skipped = unavailable_targets(config)
-        active = config.targets - skipped.map { |entry| entry[:target] }
-        skipped.each { |entry| log.puts "skip #{entry[:target]}: #{entry[:reason]}" }
-        results = config.domains.flat_map { |domain| measure_domain(Workload.fetch(domain), active, config, log) }
-        environment = Environment.describe
-        environment[:load_average] = { before: load_before, after: environment[:load_average] }
-        { environment: environment.merge(postgres: postgres_version(active)),
-          config: config.to_h, results: results, skipped: skipped }
-      end
-
-      # Checks a configuration before anything is booted.
-      #
-      # @param config [Config] the configuration to check
-      # @return [void]
-      # @raise [ArgumentError] on the first problem found
-      def validate(config)
-        unknown = config.targets - TARGETS
-        raise ArgumentError, "unknown target #{unknown.first.inspect} — one of #{TARGETS.join(', ')}" if unknown.any?
-
-        config.domains.each { |domain| Workload.fetch(domain) }
-        if config.iterations < 1 || config.runs < 1 || config.warmup.negative?
-          raise ArgumentError, "iterations and runs must be at least 1, and warmup at least 0"
-        end
-        return unless config.rust_binary && config.domains.size > 1
-
-        raise ArgumentError, "--rust-binary embeds one domain, so it needs exactly one --domain"
-      end
-
-      # Finds the targets this machine cannot run.
-      #
-      # @param config [Config] the configuration whose targets to check
-      # @return [Array<Hash{Symbol => String}>] `{target:, reason:}` for each unavailable target
-      def unavailable_targets(config)
-        postgres = PostgresProbe.unavailable_reason if config.targets.any? { |t| t.start_with?("ruby:postgres") }
-        rust = RustRunner.unavailable_reason(binary: config.rust_binary) if config.targets.include?("rust")
-        config.targets.filter_map do |target|
-          reason = target == "rust" ? rust : (postgres if target.start_with?("ruby:postgres"))
-          { target: target, reason: reason } if reason
-        end
-      end
-
-      # Measures one domain across every active target, `config.runs` times each.
-      #
-      # @param workload [Workload] the domain's commands
-      # @param targets [Array<String>] the targets that can run here
-      # @param config [Config] warmup, iterations, runs and any binary override
-      # @param log [#puts] where progress lines go
-      # @return [Array<Hash>] one aggregated entry per target
-      def measure_domain(workload, targets, config, log)
-        binaries = {}
-        summaries = Hash.new { |hash, target| hash[target] = [] }
-        config.runs.times do |index|
-          targets.each do |target|
-            run = measure_once(workload, target, config, binaries)
-            summaries[target] << run.summary
-            log.puts progress_line("#{workload.name} #{target}", "run #{index + 1}/#{config.runs}", summaries[target].last)
-          end
-        end
-        targets.map { |target| aggregate(workload.name, target, summaries[target]) }
-      end
-
-      # Formats the one line printed as each run finishes.
-      #
-      # @param label [String] the domain and target
-      # @param position [String] which run of how many, e.g. `"2/3"`
-      # @param summary [Hash] the run's `Run#summary`
-      # @return [String] the progress line
-      def progress_line(label, position, summary)
-        "#{label.ljust(27)} #{position}  #{summary[:throughput_per_s].round.to_s.rjust(8)} cmds/s  " \
-          "p50 #{summary[:p50_us].to_s.rjust(9)} us  p99 #{summary[:p99_us].to_s.rjust(9)} us"
-      end
-
-      # Runs one target once.
-      #
-      # @param workload [Workload] the domain's commands
-      # @param target [String] a member of `TARGETS`
-      # @param config [Config] warmup, iterations and any binary override
-      # @param binaries [Hash{String => String}] Rust binaries already built, by domain name;
-      #   filled in as they are built
-      # @return [Run] the run's timings
-      def measure_once(workload, target, config, binaries)
-        if target == "rust"
-          binary = config.rust_binary || (binaries[workload.name] ||= RustRunner.build(workload.name))
-          RustRunner.call(workload, binary: binary, warmup: config.warmup, iterations: config.iterations)
-        else
-          RubyRunner.call(workload, adapter: target.delete_prefix("ruby:").to_sym,
-                                    warmup: config.warmup, iterations: config.iterations)
-        end
-      end
-
-      # Reduces one target's runs to a single entry.
-      #
-      # @param domain [String] the workload name
-      # @param target [String] the target name
-      # @param summaries [Array<Hash>] each run's `Run#summary`
-      # @return [Hash{Symbol => Object}] `:domain`, `:target`, `:runs` (the raw summaries),
-      #   `:median` (the median of each run's throughput, p50, p99 and drift, plus
-      #   `:by_verb`), and `:throughput_range` (`[slowest, fastest]` across runs)
-      def aggregate(domain, target, summaries)
-        rates = summaries.map { |summary| summary[:throughput_per_s] }
-        median = %i[throughput_per_s p50_us p99_us mean_us max_us drift].to_h do |key|
-          [key, Stats.median(summaries.filter_map { |summary| summary[key] })]
-        end
-        median[:roundtrip_floor_p50_us] = Stats.median(summaries.filter_map { |s| s[:roundtrip_floor_p50_us] })
-        median[:by_verb] = median_by_verb(summaries)
-        { domain: domain, target: target, runs: summaries, median: median.compact,
-          throughput_range: rates.minmax }
-      end
-
-      # Takes each verb's median p50 and p99 across runs.
-      #
-      # @param summaries [Array<Hash>] each run's `Run#summary`
-      # @return [Hash{String => Hash{Symbol => Float}}] `:p50_us` and `:p99_us` per verb
-      def median_by_verb(summaries)
-        summaries.first[:by_verb].keys.to_h do |verb|
-          [verb, %i[p50_us p99_us].to_h { |key| [key, Stats.median(summaries.map { |s| s[:by_verb][verb][key] })] }]
-        end
-      end
-
-      # Reads the Postgres server version if a Postgres target ran.
-      #
-      # @param active [Array<String>] the targets that ran
-      # @return [String, nil] the server version, or nil when no Postgres target ran
-      def postgres_version(active)
-        PostgresProbe.server_version if active.any? { |target| target.start_with?("ruby:postgres") }
-      end
-    end
-  end
-end

data/lib/hecks/bench/workload.rb

@@ -1,137 +0,0 @@
-require "json"
-
-module Hecks
-  module Bench
-    # A fixed command sequence for one example domain, identical for every runtime.
-    #
-    # `setup` runs once, untimed, before a run starts. `cycle(n)` answers the timed
-    # commands for the n-th unit of work. Every cycle names its own records by `n`, so
-    # cycles never collide and no command is ever refused: a refusal would mean the
-    # benchmark measured a rejection path rather than the work it claims.
-    #
-    # Steps are the same `{"verb", "args"}` shape `bin/run` and the corpus scripts use.
-    # Ruby dispatches them through `dispatch_flat` and the Rust binary reads them as
-    # JSON lines, so both runtimes see byte-identical input.
-    class Workload
-      # One command to dispatch.
-      #
-      # @!attribute verb
-      #   @return [String] the fully qualified command, e.g. `"Pizzas::Order.CreatePizza"`
-      # @!attribute args
-      #   @return [Hash{String => Object}] the command's arguments, string-keyed throughout
-      Step = Struct.new(:verb, :args) do
-        # Renders the step as one line of `rust --serve` input.
-        #
-        # @return [String] the step as compact JSON with no trailing newline
-        def to_json_line
-          JSON.generate("verb" => verb, "args" => args)
-        end
-
-        # Gives the arguments as `dispatch_flat` takes them.
-        #
-        # @return [Hash{Symbol => Object}] `args` with only its top-level keys symbolized,
-        #   the way the corpus replay hands them over
-        def ruby_args
-          args.transform_keys(&:to_sym)
-        end
-      end
-
-      # @return [String] the example domain's directory name, also its Cargo feature
-      attr_reader :name
-
-      # @return [String] the absolute path of the example domain directory
-      attr_reader :domain_path
-
-      # @return [Array<Step>] the commands run once, untimed, before any cycle
-      attr_reader :setup
-
-      # Builds a workload.
-      #
-      # @param name [String] the example domain's directory name, e.g. `"pizzas"`
-      # @param setup [Array<Step>] the untimed preparation commands
-      # @param cycle [#call] takes a cycle number and returns that cycle's `Array<Step>`
-      def initialize(name:, setup:, cycle:)
-        @name = name
-        @domain_path = File.expand_path("../../../examples/#{name}", __dir__)
-        @setup = setup
-        @cycle = cycle
-      end
-
-      # Gives the timed commands for one unit of work.
-      #
-      # @param number [Integer] the zero-based cycle number, unique across warmup and
-      #   measurement so record names never repeat
-      # @return [Array<Step>] the commands in dispatch order
-      def cycle(number) = @cycle.call(number)
-
-      # Gives how many commands each cycle dispatches.
-      #
-      # @return [Integer] the length of every cycle
-      def commands_per_cycle = cycle(0).size
-
-      # Lists the workloads `bin/bench` knows.
-      #
-      # @return [Hash{String => Workload}] each workload keyed by its domain name
-      def self.all
-        { "pizzas" => pizzas, "banking" => banking }
-      end
-
-      # Looks up one workload.
-      #
-      # @param name [String] a key of `.all`
-      # @return [Workload] the workload for `name`
-      # @raise [ArgumentError] if there is no workload with that name
-      def self.fetch(name)
-        all.fetch(name) do
-          raise ArgumentError, "unknown domain #{name.inspect} — one of #{all.keys.join(', ')}"
-        end
-      end
-
-      # A pizza is created, topped twice and bought: four commands and two aggregate updates.
-      #
-      # @return [Workload] the pizzas workload
-      def self.pizzas
-        new(name: "pizzas", setup: [], cycle: lambda { |n|
-          pizza = "pizza-#{n}"
-          [
-            Step.new("Pizzas::Order.CreatePizza",
-                     { "name"  => { "value" => pizza },
-                       "pizza" => { "price_cents" => { "cents" => 1200 }, "size" => { "value" => "large" } } }),
-            Step.new("Pizzas::Order.AddTopping",
-                     { "name" => pizza, "topping" => { "value" => "Basil" }, "amount" => { "value" => 3 } }),
-            Step.new("Pizzas::Order.AddTopping",
-                     { "name" => pizza, "topping" => { "value" => "Olive" }, "amount" => { "value" => 2 } }),
-            Step.new("Pizzas::Order.Purchase",
-                     { "name" => pizza, "customer_name" => { "value" => "Chris" },
-                       "amount" => { "cents" => 1200 } })
-          ]
-        })
-      end
-
-      # An account is opened, credited twice and debited once, all for one registered customer.
-      #
-      # @return [Workload] the banking workload
-      def self.banking
-        register = Step.new("Banking::Customer.Register",
-                            { "reference" => { "value" => "CUST-0001" },
-                              "name"      => { "given" => "Ada", "family" => "Lovelace" },
-                              "email"     => { "address" => "ada@example.com" } })
-        new(name: "banking", setup: [register], cycle: lambda { |n|
-          number = { "value" => "acct-#{n}" }
-          money = ->(cents) { { "cents" => cents, "currency" => "USD" } }
-          [
-            Step.new("Banking::Account.Open",
-                     { "number" => number, "kind" => { "name" => "current" },
-                       "daily_limit" => { "cents" => 50_000 }, "customer" => "CUST-0001" }),
-            Step.new("Banking::Account.Credit",
-                     { "number" => number, "amount" => money.call(10_000), "narrative" => { "text" => "Deposit" } }),
-            Step.new("Banking::Account.Credit",
-                     { "number" => number, "amount" => money.call(5000), "narrative" => { "text" => "Deposit" } }),
-            Step.new("Banking::Account.Debit",
-                     { "number" => number, "amount" => money.call(2500), "narrative" => { "text" => "Groceries" } })
-          ]
-        })
-      end
-    end
-  end
-end

data/lib/hecks/bench.rb

@@ -1,28 +0,0 @@
-module Hecks
-  # The throughput and latency harness `bin/bench` drives.
-  #
-  # It dispatches a fixed, valid command workload against the Ruby runtime on each
-  # persistence adapter and against the native Rust binary, and reports
-  # commands per second plus p50 and p99 latency for each. Not required by
-  # `lib/hecks.rb` on purpose: a booted domain never needs it.
-  #
-  # ## What is measured
-  #
-  # Every runtime is driven the same way: one caller, one command at a time, each
-  # waiting for the previous answer. The Ruby side times `dispatch_flat` in-process.
-  # The Rust side times a full round trip over the pipes of `rust --serve`. See
-  # `docs/benchmarks.md` for what that does and does not compare.
-  module Bench
-  end
-end
-
-require_relative "bench/stats"
-require_relative "bench/workload"
-require_relative "bench/environment"
-require_relative "bench/postgres_probe"
-require_relative "bench/run"
-require_relative "bench/ruby_runner"
-require_relative "bench/rust_runner"
-require_relative "bench/suite"
-require_relative "bench/report"
-require_relative "bench/cli"