hecks 2.5.1 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/hecks/adapters/driven/google_authentication.rb +1 -1
- data/lib/hecks/adapters/driven/lambda/client.rb +1 -1
- data/lib/hecks/adapters/driven/prism.rb +1 -1
- data/lib/hecks/bench/cli.rb +74 -0
- data/lib/hecks/bench/environment.rb +100 -0
- data/lib/hecks/bench/postgres_probe.rb +41 -0
- data/lib/hecks/bench/report.rb +116 -0
- data/lib/hecks/bench/ruby_runner.rb +151 -0
- data/lib/hecks/bench/run.rb +55 -0
- data/lib/hecks/bench/rust_runner.rb +165 -0
- data/lib/hecks/bench/stats.rb +75 -0
- data/lib/hecks/bench/suite.rb +172 -0
- data/lib/hecks/bench/workload.rb +137 -0
- data/lib/hecks/bench.rb +28 -0
- data/lib/hecks/bluebook/dsl/bootstrap_table.rb +2 -0
- data/lib/hecks/bluebook/dsl/command_builder.rb +3 -3
- data/lib/hecks/bluebook/dsl/port_operation_builder.rb +1 -1
- data/lib/hecks/bluebook/dsl/world_builder.rb +42 -3
- data/lib/hecks/bluebook/hexagon.rb +15 -7
- data/lib/hecks/bluebook/meta_validator/world_judge.rb +3 -1
- data/lib/hecks/bluebook/model_check/client_profile.rb +153 -0
- data/lib/hecks/bluebook/model_check.rb +21 -2
- data/lib/hecks/embryonaut_bluebook/lock.rb +72 -0
- data/lib/hecks/embryonaut_bluebook/shape.rb +47 -0
- data/lib/hecks/embryonaut_bluebook/vendor.rb +194 -0
- data/lib/hecks/embryonaut_bluebook/vendor_cli.rb +88 -0
- data/lib/hecks/embryonaut_bluebook.rb +29 -3
- data/lib/hecks/forms/value_object_shape.rb +1 -1
- data/lib/hecks/framework/bluebook/console_settings.bluebook +3 -3
- data/lib/hecks/fuzzing/era_boundary.rb +1 -1
- data/lib/hecks/fuzzing/isolated_boot.rb +136 -69
- data/lib/hecks/language/world/world.bluebook +32 -2
- data/lib/hecks/mcp_stdio_guard.rb +160 -0
- data/lib/hecks/ports/key_vault.rb +1 -1
- data/lib/hecks/ports/persistence/binding_policy.rb +16 -5
- data/lib/hecks/ports/persistence/plugins/era/era_check.rb +2 -1
- data/lib/hecks/ports/persistence/plugins/era/expected_era.rb +132 -0
- data/lib/hecks/ports/persistence/plugins/era/lineage.rb +1 -1
- data/lib/hecks/ports/persistence/plugins/era/translation/rule_compiler.rb +1 -1
- data/lib/hecks/ports/persistence/plugins/era.rb +7 -4
- data/lib/hecks/ports/persistence/postgres_dump/connection.rb +73 -0
- data/lib/hecks/ports/persistence/postgres_dump.rb +125 -0
- data/lib/hecks/ports/persistence/repository_factory.rb +2 -2
- data/lib/hecks/ports/persistence.rb +4 -0
- data/lib/hecks/projections/deploy/fargate/assembly.rb +193 -0
- data/lib/hecks/projections/deploy/fargate/cdn.rb +318 -0
- data/lib/hecks/projections/deploy/fargate/cdn_default.rb +87 -0
- data/lib/hecks/projections/deploy/fargate/check.rb +172 -0
- data/lib/hecks/projections/deploy/fargate/containers.rb +412 -0
- data/lib/hecks/projections/deploy/fargate/extras.rb +268 -0
- data/lib/hecks/projections/deploy/fargate/monitoring.rb +448 -0
- data/lib/hecks/projections/deploy/fargate/settings.rb +235 -0
- data/lib/hecks/projections/deploy/fargate/yaml.rb +116 -0
- data/lib/hecks/projections/deploy/fargate.rb +187 -138
- data/lib/hecks/projections/deploy/lambda.rb +5 -5
- data/lib/hecks/projections/deploy/preview/access_yaml.rb +145 -0
- data/lib/hecks/projections/deploy/preview/containers.rb +218 -0
- data/lib/hecks/projections/deploy/preview/preview.sh.erb +446 -0
- data/lib/hecks/projections/deploy/preview/routing_yaml.rb +216 -0
- data/lib/hecks/projections/deploy/preview/script.rb +40 -0
- data/lib/hecks/projections/deploy/preview/settings.rb +140 -0
- data/lib/hecks/projections/deploy/preview/task_yaml.rb +215 -0
- data/lib/hecks/projections/deploy/preview/template.rb +247 -0
- data/lib/hecks/projections/deploy/preview/yaml_text.rb +48 -0
- data/lib/hecks/projections/deploy/preview.rb +103 -0
- data/lib/hecks/projections/deploy/scripts/deploy-service.sh.tmpl +182 -0
- data/lib/hecks/projections/deploy/scripts/expected-era.header +17 -0
- data/lib/hecks/projections/deploy/scripts/hosting.mk.tmpl +54 -0
- data/lib/hecks/projections/deploy/scripts/settings.rb +114 -0
- data/lib/hecks/projections/deploy/scripts/smoke-after-deploy.sh.tmpl +232 -0
- data/lib/hecks/projections/deploy/scripts.rb +171 -0
- data/lib/hecks/projections/deploy/smoke/harness.js +366 -0
- data/lib/hecks/projections/deploy/smoke/workflow.yml.tmpl +56 -0
- data/lib/hecks/projections/deploy/smoke.rb +159 -0
- data/lib/hecks/projections/deploy/template_diff/comparer.rb +231 -0
- data/lib/hecks/projections/deploy/template_diff/loader.rb +105 -0
- data/lib/hecks/projections/deploy/template_diff.rb +173 -0
- data/lib/hecks/projections/glossary/sensitivity.rb +1 -1
- data/lib/hecks/projections.rb +2 -0
- data/lib/hecks/release/gem_pin/published_versions.rb +58 -0
- data/lib/hecks/release/gem_pin.rb +112 -0
- data/lib/hecks/runtime/command_rules/arithmetic.rb +2 -2
- data/lib/hecks/runtime/loader.rb +26 -2
- data/lib/hecks/runtime/registry/saga_persistence.rb +8 -2
- data/lib/hecks/runtime/registry/verification.rb +4 -2
- data/lib/hecks/runtime/registry/world_defaults.rb +106 -0
- data/lib/hecks/runtime/registry.rb +14 -9
- data/lib/hecks/runtime/tenant_check.rb +1 -1
- data/lib/hecks/vendoring/git_environment.rb +35 -0
- data/lib/hecks/vendoring/git_source.rb +102 -0
- data/lib/hecks/vendoring.rb +102 -0
- data/lib/hecks/version.rb +1 -1
- data/lib/hecks.rb +3 -1
- metadata +59 -2
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
require "fileutils"
|
|
2
|
+
require "open3"
|
|
3
|
+
require_relative "run"
|
|
4
|
+
|
|
5
|
+
module Hecks
|
|
6
|
+
module Bench
|
|
7
|
+
# Measures the native Rust binary through its streaming mode.
|
|
8
|
+
#
|
|
9
|
+
# `rust --serve` reads one JSON step per line and answers one JSON line each, keeping
|
|
10
|
+
# its in-memory store alive between them. This runner writes a step, waits for the
|
|
11
|
+
# answer line, and times the whole round trip, so the figures include two pipe
|
|
12
|
+
# crossings and the binary's own JSON parse and print. Process start-up is outside
|
|
13
|
+
# the timer.
|
|
14
|
+
#
|
|
15
|
+
# Alongside each run it measures a **round-trip floor**: the same round trip for a
|
|
16
|
+
# step the binary refuses immediately, which is what the pipes cost with almost no
|
|
17
|
+
# domain work behind them.
|
|
18
|
+
module RustRunner
|
|
19
|
+
# Where the Rust crate lives.
|
|
20
|
+
RUST_DIR = File.expand_path("../../../rust", __dir__)
|
|
21
|
+
|
|
22
|
+
# How many refused-at-once steps make up the round-trip floor's sample.
|
|
23
|
+
FLOOR_SAMPLES = 500
|
|
24
|
+
|
|
25
|
+
# A `cargo build` that exited non-zero.
|
|
26
|
+
class BuildFailed < StandardError; end
|
|
27
|
+
|
|
28
|
+
# A benchmarked step the binary refused or could not answer.
|
|
29
|
+
class StepRefused < StandardError; end
|
|
30
|
+
|
|
31
|
+
module_function
|
|
32
|
+
|
|
33
|
+
# Says why the Rust target cannot run, if it cannot.
|
|
34
|
+
#
|
|
35
|
+
# @param binary [String, nil] an explicit pre-built binary, which needs no toolchain
|
|
36
|
+
# @return [String, nil] nil when the target can run, otherwise one sentence saying why
|
|
37
|
+
def unavailable_reason(binary: nil)
|
|
38
|
+
return "the binary #{binary} does not exist or is not executable" if binary && !File.executable?(binary)
|
|
39
|
+
return nil if binary || Environment.capture("cargo", "-V") != "unknown"
|
|
40
|
+
|
|
41
|
+
"`cargo` is not on PATH and no --rust-binary was given"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Builds the release binary for one domain's Cargo feature and pins a copy of it.
|
|
45
|
+
#
|
|
46
|
+
# The copy has its own name so building another domain's feature afterwards cannot
|
|
47
|
+
# replace it.
|
|
48
|
+
#
|
|
49
|
+
# @param feature [String] the domain's Cargo feature, its directory name
|
|
50
|
+
# @param rust_dir [String] the Rust crate's directory
|
|
51
|
+
# @return [String] the path of the pinned release binary
|
|
52
|
+
# @raise [BuildFailed] if cargo exits non-zero or leaves no binary
|
|
53
|
+
def build(feature, rust_dir: RUST_DIR)
|
|
54
|
+
output, status = Open3.capture2e("cargo", "build", "--release", "--no-default-features",
|
|
55
|
+
"--features", feature, chdir: rust_dir)
|
|
56
|
+
unless status.success?
|
|
57
|
+
raise BuildFailed,
|
|
58
|
+
"cargo build --release --features #{feature} failed:\n#{output.lines.last(15).join}"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
target = File.join(ENV.fetch("CARGO_TARGET_DIR", File.join(rust_dir, "target")), "release")
|
|
62
|
+
pinned = File.join(target, "rust-bench-#{feature}")
|
|
63
|
+
FileUtils.cp(File.join(target, "rust"), pinned)
|
|
64
|
+
pinned
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Starts the binary in serve mode and times the workload's commands.
|
|
68
|
+
#
|
|
69
|
+
# @param workload [Workload] the commands to send
|
|
70
|
+
# @param binary [String] the path of a binary built for `workload`'s domain
|
|
71
|
+
# @param warmup [Integer] cycles sent and discarded before timing starts
|
|
72
|
+
# @param iterations [Integer] cycles timed
|
|
73
|
+
# @return [Run] the timings, with the round-trip floor in its extras as
|
|
74
|
+
# `:roundtrip_floor_p50_us`
|
|
75
|
+
# @raise [StepRefused] if the binary refuses any step, warmup included
|
|
76
|
+
def call(workload, binary:, warmup:, iterations:)
|
|
77
|
+
IO.popen([binary, "--serve"], "r+") do |pipe|
|
|
78
|
+
pipe.sync = true
|
|
79
|
+
workload.setup.each { |step| ask(pipe, step.to_json_line) }
|
|
80
|
+
warmup.times { |n| workload.cycle(n).each { |step| ask(pipe, step.to_json_line) } }
|
|
81
|
+
run = measure(pipe, workload, warmup: warmup, iterations: iterations)
|
|
82
|
+
Run.new(samples: run.samples, wall_seconds: run.wall_seconds,
|
|
83
|
+
extras: { roundtrip_floor_p50_us: floor(pipe) })
|
|
84
|
+
ensure
|
|
85
|
+
pipe.close_write unless pipe.closed?
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Times the measured cycles.
|
|
90
|
+
#
|
|
91
|
+
# @param pipe [IO] the binary's stdin and stdout
|
|
92
|
+
# @param workload [Workload] the commands to send
|
|
93
|
+
# @param warmup [Integer] cycles already sent, so record names continue from there
|
|
94
|
+
# @param iterations [Integer] cycles to time
|
|
95
|
+
# @return [Run] the timings, without extras
|
|
96
|
+
def measure(pipe, workload, warmup:, iterations:)
|
|
97
|
+
samples = []
|
|
98
|
+
started = now
|
|
99
|
+
iterations.times do |n|
|
|
100
|
+
workload.cycle(warmup + n).each do |step|
|
|
101
|
+
line = step.to_json_line
|
|
102
|
+
began = now
|
|
103
|
+
answer = round_trip(pipe, line)
|
|
104
|
+
samples << [step.verb, now - began]
|
|
105
|
+
check(answer, step.verb)
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
Run.new(samples: samples, wall_seconds: now - started)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Measures the pipes with almost no work behind them.
|
|
112
|
+
#
|
|
113
|
+
# @param pipe [IO] the binary's stdin and stdout
|
|
114
|
+
# @return [Float] the p50 microseconds of {FLOOR_SAMPLES} round trips for an empty step,
|
|
115
|
+
# which the binary refuses without touching its store
|
|
116
|
+
def floor(pipe)
|
|
117
|
+
seconds = Array.new(FLOOR_SAMPLES) do
|
|
118
|
+
began = now
|
|
119
|
+
round_trip(pipe, "{}")
|
|
120
|
+
now - began
|
|
121
|
+
end
|
|
122
|
+
Stats.micros(Stats.percentile(seconds, 0.5))
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Sends one step and returns the answer line, untimed.
|
|
126
|
+
#
|
|
127
|
+
# @param pipe [IO] the binary's stdin and stdout
|
|
128
|
+
# @param line [String] the step as one JSON line
|
|
129
|
+
# @return [void]
|
|
130
|
+
# @raise [StepRefused] if the binary refuses the step
|
|
131
|
+
def ask(pipe, line)
|
|
132
|
+
check(round_trip(pipe, line), line[0, 80])
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Writes one line and reads one line back.
|
|
136
|
+
#
|
|
137
|
+
# @param pipe [IO] the binary's stdin and stdout
|
|
138
|
+
# @param line [String] the step as one JSON line
|
|
139
|
+
# @return [String] the answer line, or an empty string if the binary closed its output
|
|
140
|
+
def round_trip(pipe, line)
|
|
141
|
+
pipe.write(line, "\n")
|
|
142
|
+
pipe.gets.to_s
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Refuses an answer that is not a success.
|
|
146
|
+
#
|
|
147
|
+
# @param answer [String] the binary's answer line
|
|
148
|
+
# @param what [String] what was asked, for the error message
|
|
149
|
+
# @return [void]
|
|
150
|
+
# @raise [StepRefused] unless `answer` starts with `{"ok":true`
|
|
151
|
+
def check(answer, what)
|
|
152
|
+
return if answer.start_with?('{"ok":true')
|
|
153
|
+
|
|
154
|
+
raise StepRefused, "#{what} was not accepted: #{answer.strip.empty? ? 'the binary closed its output' : answer.strip}"
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Reads the monotonic clock.
|
|
158
|
+
#
|
|
159
|
+
# @return [Float] seconds on the monotonic clock
|
|
160
|
+
def now
|
|
161
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
module Hecks
|
|
2
|
+
module Bench
|
|
3
|
+
# Order statistics over a list of latency samples.
|
|
4
|
+
#
|
|
5
|
+
# Percentiles use the nearest-rank method: the smallest sample with at least
|
|
6
|
+
# that fraction of the samples at or below it. It never interpolates, so every
|
|
7
|
+
# reported figure is a latency that actually happened. With fewer than 100
|
|
8
|
+
# samples the p99 is simply the maximum.
|
|
9
|
+
module Stats
|
|
10
|
+
module_function
|
|
11
|
+
|
|
12
|
+
# Reads one percentile from a list of samples.
|
|
13
|
+
#
|
|
14
|
+
# @param samples [Array<Float>] the samples, in any order
|
|
15
|
+
# @param fraction [Float] the percentile as a fraction, e.g. `0.99`
|
|
16
|
+
# @return [Float, nil] the nearest-rank percentile, or nil when `samples` is empty
|
|
17
|
+
def percentile(samples, fraction)
|
|
18
|
+
return nil if samples.empty?
|
|
19
|
+
|
|
20
|
+
sorted = samples.sort
|
|
21
|
+
sorted[[(fraction * sorted.size).ceil - 1, 0].max]
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Finds the middle value of a list, averaging the two middle values of an even one.
|
|
25
|
+
#
|
|
26
|
+
# @param values [Array<Numeric>] the values, in any order
|
|
27
|
+
# @return [Float, nil] the median, or nil when `values` is empty
|
|
28
|
+
def median(values)
|
|
29
|
+
return nil if values.empty?
|
|
30
|
+
|
|
31
|
+
sorted = values.sort
|
|
32
|
+
middle = sorted.size / 2
|
|
33
|
+
sorted.size.odd? ? sorted[middle].to_f : (sorted[middle - 1] + sorted[middle]) / 2.0
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Summarizes one run's latency samples.
|
|
37
|
+
#
|
|
38
|
+
# @param seconds [Array<Float>] per-command latencies in seconds, in dispatch order
|
|
39
|
+
# @return [Hash{Symbol => Numeric}] `:count`, and `:p50_us`, `:p99_us`, `:mean_us` and
|
|
40
|
+
# `:max_us` in microseconds
|
|
41
|
+
def latency(seconds)
|
|
42
|
+
{
|
|
43
|
+
count: seconds.size,
|
|
44
|
+
p50_us: micros(percentile(seconds, 0.50)),
|
|
45
|
+
p99_us: micros(percentile(seconds, 0.99)),
|
|
46
|
+
mean_us: micros(seconds.sum / seconds.size),
|
|
47
|
+
max_us: micros(seconds.max)
|
|
48
|
+
}
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Compares how slow the last tenth of a run was against its first tenth.
|
|
52
|
+
#
|
|
53
|
+
# A ratio well above 1.0 means latency grows as the store fills, which a single
|
|
54
|
+
# p50 over the whole run would hide.
|
|
55
|
+
#
|
|
56
|
+
# @param seconds [Array<Float>] per-command latencies in dispatch order
|
|
57
|
+
# @return [Float, nil] the last tenth's p50 over the first tenth's, or nil when there
|
|
58
|
+
# are fewer than ten samples
|
|
59
|
+
def drift(seconds)
|
|
60
|
+
tenth = seconds.size / 10
|
|
61
|
+
return nil if tenth.zero?
|
|
62
|
+
|
|
63
|
+
(percentile(seconds.last(tenth), 0.5) / percentile(seconds.first(tenth), 0.5)).round(2)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Converts seconds to microseconds, rounded to a tenth.
|
|
67
|
+
#
|
|
68
|
+
# @param seconds [Float, nil] a duration in seconds
|
|
69
|
+
# @return [Float, nil] the duration in microseconds, or nil when `seconds` is nil
|
|
70
|
+
def micros(seconds)
|
|
71
|
+
seconds && (seconds * 1_000_000).round(1)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
module Hecks
|
|
2
|
+
module Bench
|
|
3
|
+
# Runs every requested target against every requested domain and aggregates the runs.
|
|
4
|
+
#
|
|
5
|
+
# Each domain's runs are interleaved across targets (memory, sqlite, ..., memory,
|
|
6
|
+
# sqlite, ...) rather than finished one target at a time, so a burst of unrelated load on
|
|
7
|
+
# the machine lands on every target instead of flattering one of them. Each figure in the
|
|
8
|
+
# result is the median across runs, with the fastest and slowest throughput kept so a noisy
|
|
9
|
+
# measurement shows.
|
|
10
|
+
module Suite
|
|
11
|
+
# Every target `bin/bench` can measure.
|
|
12
|
+
TARGETS = %w[ruby:memory ruby:sqlite ruby:postgres ruby:postgres_era rust].freeze
|
|
13
|
+
|
|
14
|
+
# What to measure and how hard.
|
|
15
|
+
#
|
|
16
|
+
# @!attribute domains
|
|
17
|
+
# @return [Array<String>] workload names, keys of `Workload.all`
|
|
18
|
+
# @!attribute targets
|
|
19
|
+
# @return [Array<String>] members of `TARGETS`
|
|
20
|
+
# @!attribute warmup
|
|
21
|
+
# @return [Integer] cycles discarded before each run is timed
|
|
22
|
+
# @!attribute iterations
|
|
23
|
+
# @return [Integer] cycles timed in each run
|
|
24
|
+
# @!attribute runs
|
|
25
|
+
# @return [Integer] how many fresh boots of each target to measure per domain
|
|
26
|
+
# @!attribute rust_binary
|
|
27
|
+
# @return [String, nil] a pre-built binary to use instead of building one; only valid
|
|
28
|
+
# with a single domain, since a binary embeds exactly one
|
|
29
|
+
Config = Struct.new(:domains, :targets, :warmup, :iterations, :runs, :rust_binary, keyword_init: true)
|
|
30
|
+
|
|
31
|
+
module_function
|
|
32
|
+
|
|
33
|
+
# Measures everything `config` asks for.
|
|
34
|
+
#
|
|
35
|
+
# @param config [Config] what to measure
|
|
36
|
+
# @param log [#puts] where progress lines go; one line per run
|
|
37
|
+
# @return [Hash{Symbol => Object}] `:environment`, `:config`, `:results` (one entry per
|
|
38
|
+
# domain and target, see `.aggregate`) and `:skipped` (each skipped target and why)
|
|
39
|
+
# @raise [ArgumentError] if a target or domain is unknown, a count is not positive, or
|
|
40
|
+
# `rust_binary` is combined with more than one domain
|
|
41
|
+
def call(config, log: $stderr)
|
|
42
|
+
validate(config)
|
|
43
|
+
load_before = Environment.load_average
|
|
44
|
+
skipped = unavailable_targets(config)
|
|
45
|
+
active = config.targets - skipped.map { |entry| entry[:target] }
|
|
46
|
+
skipped.each { |entry| log.puts "skip #{entry[:target]}: #{entry[:reason]}" }
|
|
47
|
+
results = config.domains.flat_map { |domain| measure_domain(Workload.fetch(domain), active, config, log) }
|
|
48
|
+
environment = Environment.describe
|
|
49
|
+
environment[:load_average] = { before: load_before, after: environment[:load_average] }
|
|
50
|
+
{ environment: environment.merge(postgres: postgres_version(active)),
|
|
51
|
+
config: config.to_h, results: results, skipped: skipped }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Checks a configuration before anything is booted.
|
|
55
|
+
#
|
|
56
|
+
# @param config [Config] the configuration to check
|
|
57
|
+
# @return [void]
|
|
58
|
+
# @raise [ArgumentError] on the first problem found
|
|
59
|
+
def validate(config)
|
|
60
|
+
unknown = config.targets - TARGETS
|
|
61
|
+
raise ArgumentError, "unknown target #{unknown.first.inspect} — one of #{TARGETS.join(', ')}" if unknown.any?
|
|
62
|
+
|
|
63
|
+
config.domains.each { |domain| Workload.fetch(domain) }
|
|
64
|
+
if config.iterations < 1 || config.runs < 1 || config.warmup.negative?
|
|
65
|
+
raise ArgumentError, "iterations and runs must be at least 1, and warmup at least 0"
|
|
66
|
+
end
|
|
67
|
+
return unless config.rust_binary && config.domains.size > 1
|
|
68
|
+
|
|
69
|
+
raise ArgumentError, "--rust-binary embeds one domain, so it needs exactly one --domain"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Finds the targets this machine cannot run.
|
|
73
|
+
#
|
|
74
|
+
# @param config [Config] the configuration whose targets to check
|
|
75
|
+
# @return [Array<Hash{Symbol => String}>] `{target:, reason:}` for each unavailable target
|
|
76
|
+
def unavailable_targets(config)
|
|
77
|
+
postgres = PostgresProbe.unavailable_reason if config.targets.any? { |t| t.start_with?("ruby:postgres") }
|
|
78
|
+
rust = RustRunner.unavailable_reason(binary: config.rust_binary) if config.targets.include?("rust")
|
|
79
|
+
config.targets.filter_map do |target|
|
|
80
|
+
reason = target == "rust" ? rust : (postgres if target.start_with?("ruby:postgres"))
|
|
81
|
+
{ target: target, reason: reason } if reason
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Measures one domain across every active target, `config.runs` times each.
|
|
86
|
+
#
|
|
87
|
+
# @param workload [Workload] the domain's commands
|
|
88
|
+
# @param targets [Array<String>] the targets that can run here
|
|
89
|
+
# @param config [Config] warmup, iterations, runs and any binary override
|
|
90
|
+
# @param log [#puts] where progress lines go
|
|
91
|
+
# @return [Array<Hash>] one aggregated entry per target
|
|
92
|
+
def measure_domain(workload, targets, config, log)
|
|
93
|
+
binaries = {}
|
|
94
|
+
summaries = Hash.new { |hash, target| hash[target] = [] }
|
|
95
|
+
config.runs.times do |index|
|
|
96
|
+
targets.each do |target|
|
|
97
|
+
run = measure_once(workload, target, config, binaries)
|
|
98
|
+
summaries[target] << run.summary
|
|
99
|
+
log.puts progress_line("#{workload.name} #{target}", "run #{index + 1}/#{config.runs}", summaries[target].last)
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
targets.map { |target| aggregate(workload.name, target, summaries[target]) }
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Formats the one line printed as each run finishes.
|
|
106
|
+
#
|
|
107
|
+
# @param label [String] the domain and target
|
|
108
|
+
# @param position [String] which run of how many, e.g. `"2/3"`
|
|
109
|
+
# @param summary [Hash] the run's `Run#summary`
|
|
110
|
+
# @return [String] the progress line
|
|
111
|
+
def progress_line(label, position, summary)
|
|
112
|
+
"#{label.ljust(27)} #{position} #{summary[:throughput_per_s].round.to_s.rjust(8)} cmds/s " \
|
|
113
|
+
"p50 #{summary[:p50_us].to_s.rjust(9)} us p99 #{summary[:p99_us].to_s.rjust(9)} us"
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Runs one target once.
|
|
117
|
+
#
|
|
118
|
+
# @param workload [Workload] the domain's commands
|
|
119
|
+
# @param target [String] a member of `TARGETS`
|
|
120
|
+
# @param config [Config] warmup, iterations and any binary override
|
|
121
|
+
# @param binaries [Hash{String => String}] Rust binaries already built, by domain name;
|
|
122
|
+
# filled in as they are built
|
|
123
|
+
# @return [Run] the run's timings
|
|
124
|
+
def measure_once(workload, target, config, binaries)
|
|
125
|
+
if target == "rust"
|
|
126
|
+
binary = config.rust_binary || (binaries[workload.name] ||= RustRunner.build(workload.name))
|
|
127
|
+
RustRunner.call(workload, binary: binary, warmup: config.warmup, iterations: config.iterations)
|
|
128
|
+
else
|
|
129
|
+
RubyRunner.call(workload, adapter: target.delete_prefix("ruby:").to_sym,
|
|
130
|
+
warmup: config.warmup, iterations: config.iterations)
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# Reduces one target's runs to a single entry.
|
|
135
|
+
#
|
|
136
|
+
# @param domain [String] the workload name
|
|
137
|
+
# @param target [String] the target name
|
|
138
|
+
# @param summaries [Array<Hash>] each run's `Run#summary`
|
|
139
|
+
# @return [Hash{Symbol => Object}] `:domain`, `:target`, `:runs` (the raw summaries),
|
|
140
|
+
# `:median` (the median of each run's throughput, p50, p99 and drift, plus
|
|
141
|
+
# `:by_verb`), and `:throughput_range` (`[slowest, fastest]` across runs)
|
|
142
|
+
def aggregate(domain, target, summaries)
|
|
143
|
+
rates = summaries.map { |summary| summary[:throughput_per_s] }
|
|
144
|
+
median = %i[throughput_per_s p50_us p99_us mean_us max_us drift].to_h do |key|
|
|
145
|
+
[key, Stats.median(summaries.filter_map { |summary| summary[key] })]
|
|
146
|
+
end
|
|
147
|
+
median[:roundtrip_floor_p50_us] = Stats.median(summaries.filter_map { |s| s[:roundtrip_floor_p50_us] })
|
|
148
|
+
median[:by_verb] = median_by_verb(summaries)
|
|
149
|
+
{ domain: domain, target: target, runs: summaries, median: median.compact,
|
|
150
|
+
throughput_range: rates.minmax }
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
# Takes each verb's median p50 and p99 across runs.
|
|
154
|
+
#
|
|
155
|
+
# @param summaries [Array<Hash>] each run's `Run#summary`
|
|
156
|
+
# @return [Hash{String => Hash{Symbol => Float}}] `:p50_us` and `:p99_us` per verb
|
|
157
|
+
def median_by_verb(summaries)
|
|
158
|
+
summaries.first[:by_verb].keys.to_h do |verb|
|
|
159
|
+
[verb, %i[p50_us p99_us].to_h { |key| [key, Stats.median(summaries.map { |s| s[:by_verb][verb][key] })] }]
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Reads the Postgres server version if a Postgres target ran.
|
|
164
|
+
#
|
|
165
|
+
# @param active [Array<String>] the targets that ran
|
|
166
|
+
# @return [String, nil] the server version, or nil when no Postgres target ran
|
|
167
|
+
def postgres_version(active)
|
|
168
|
+
PostgresProbe.server_version if active.any? { |target| target.start_with?("ruby:postgres") }
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
|
|
3
|
+
module Hecks
|
|
4
|
+
module Bench
|
|
5
|
+
# A fixed command sequence for one example domain, identical for every runtime.
|
|
6
|
+
#
|
|
7
|
+
# `setup` runs once, untimed, before a run starts. `cycle(n)` answers the timed
|
|
8
|
+
# commands for the n-th unit of work. Every cycle names its own records by `n`, so
|
|
9
|
+
# cycles never collide and no command is ever refused: a refusal would mean the
|
|
10
|
+
# benchmark measured a rejection path rather than the work it claims.
|
|
11
|
+
#
|
|
12
|
+
# Steps are the same `{"verb", "args"}` shape `bin/run` and the corpus scripts use.
|
|
13
|
+
# Ruby dispatches them through `dispatch_flat` and the Rust binary reads them as
|
|
14
|
+
# JSON lines, so both runtimes see byte-identical input.
|
|
15
|
+
class Workload
|
|
16
|
+
# One command to dispatch.
|
|
17
|
+
#
|
|
18
|
+
# @!attribute verb
|
|
19
|
+
# @return [String] the fully qualified command, e.g. `"Pizzas::Order.CreatePizza"`
|
|
20
|
+
# @!attribute args
|
|
21
|
+
# @return [Hash{String => Object}] the command's arguments, string-keyed throughout
|
|
22
|
+
Step = Struct.new(:verb, :args) do
|
|
23
|
+
# Renders the step as one line of `rust --serve` input.
|
|
24
|
+
#
|
|
25
|
+
# @return [String] the step as compact JSON with no trailing newline
|
|
26
|
+
def to_json_line
|
|
27
|
+
JSON.generate("verb" => verb, "args" => args)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Gives the arguments as `dispatch_flat` takes them.
|
|
31
|
+
#
|
|
32
|
+
# @return [Hash{Symbol => Object}] `args` with only its top-level keys symbolized,
|
|
33
|
+
# the way the corpus replay hands them over
|
|
34
|
+
def ruby_args
|
|
35
|
+
args.transform_keys(&:to_sym)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @return [String] the example domain's directory name, also its Cargo feature
|
|
40
|
+
attr_reader :name
|
|
41
|
+
|
|
42
|
+
# @return [String] the absolute path of the example domain directory
|
|
43
|
+
attr_reader :domain_path
|
|
44
|
+
|
|
45
|
+
# @return [Array<Step>] the commands run once, untimed, before any cycle
|
|
46
|
+
attr_reader :setup
|
|
47
|
+
|
|
48
|
+
# Builds a workload.
|
|
49
|
+
#
|
|
50
|
+
# @param name [String] the example domain's directory name, e.g. `"pizzas"`
|
|
51
|
+
# @param setup [Array<Step>] the untimed preparation commands
|
|
52
|
+
# @param cycle [#call] takes a cycle number and returns that cycle's `Array<Step>`
|
|
53
|
+
def initialize(name:, setup:, cycle:)
|
|
54
|
+
@name = name
|
|
55
|
+
@domain_path = File.expand_path("../../../examples/#{name}", __dir__)
|
|
56
|
+
@setup = setup
|
|
57
|
+
@cycle = cycle
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Gives the timed commands for one unit of work.
|
|
61
|
+
#
|
|
62
|
+
# @param number [Integer] the zero-based cycle number, unique across warmup and
|
|
63
|
+
# measurement so record names never repeat
|
|
64
|
+
# @return [Array<Step>] the commands in dispatch order
|
|
65
|
+
def cycle(number) = @cycle.call(number)
|
|
66
|
+
|
|
67
|
+
# Gives how many commands each cycle dispatches.
|
|
68
|
+
#
|
|
69
|
+
# @return [Integer] the length of every cycle
|
|
70
|
+
def commands_per_cycle = cycle(0).size
|
|
71
|
+
|
|
72
|
+
# Lists the workloads `bin/bench` knows.
|
|
73
|
+
#
|
|
74
|
+
# @return [Hash{String => Workload}] each workload keyed by its domain name
|
|
75
|
+
def self.all
|
|
76
|
+
{ "pizzas" => pizzas, "banking" => banking }
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Looks up one workload.
|
|
80
|
+
#
|
|
81
|
+
# @param name [String] a key of `.all`
|
|
82
|
+
# @return [Workload] the workload for `name`
|
|
83
|
+
# @raise [ArgumentError] if there is no workload with that name
|
|
84
|
+
def self.fetch(name)
|
|
85
|
+
all.fetch(name) do
|
|
86
|
+
raise ArgumentError, "unknown domain #{name.inspect} — one of #{all.keys.join(', ')}"
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# A pizza is created, topped twice and bought: four commands and two aggregate updates.
|
|
91
|
+
#
|
|
92
|
+
# @return [Workload] the pizzas workload
|
|
93
|
+
def self.pizzas
|
|
94
|
+
new(name: "pizzas", setup: [], cycle: lambda { |n|
|
|
95
|
+
pizza = "pizza-#{n}"
|
|
96
|
+
[
|
|
97
|
+
Step.new("Pizzas::Order.CreatePizza",
|
|
98
|
+
{ "name" => { "value" => pizza },
|
|
99
|
+
"pizza" => { "price_cents" => { "cents" => 1200 }, "size" => { "value" => "large" } } }),
|
|
100
|
+
Step.new("Pizzas::Order.AddTopping",
|
|
101
|
+
{ "name" => pizza, "topping" => { "value" => "Basil" }, "amount" => { "value" => 3 } }),
|
|
102
|
+
Step.new("Pizzas::Order.AddTopping",
|
|
103
|
+
{ "name" => pizza, "topping" => { "value" => "Olive" }, "amount" => { "value" => 2 } }),
|
|
104
|
+
Step.new("Pizzas::Order.Purchase",
|
|
105
|
+
{ "name" => pizza, "customer_name" => { "value" => "Chris" },
|
|
106
|
+
"amount" => { "cents" => 1200 } })
|
|
107
|
+
]
|
|
108
|
+
})
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# An account is opened, credited twice and debited once, all for one registered customer.
|
|
112
|
+
#
|
|
113
|
+
# @return [Workload] the banking workload
|
|
114
|
+
def self.banking
|
|
115
|
+
register = Step.new("Banking::Customer.Register",
|
|
116
|
+
{ "reference" => { "value" => "CUST-0001" },
|
|
117
|
+
"name" => { "given" => "Ada", "family" => "Lovelace" },
|
|
118
|
+
"email" => { "address" => "ada@example.com" } })
|
|
119
|
+
new(name: "banking", setup: [register], cycle: lambda { |n|
|
|
120
|
+
number = { "value" => "acct-#{n}" }
|
|
121
|
+
money = ->(cents) { { "cents" => cents, "currency" => "USD" } }
|
|
122
|
+
[
|
|
123
|
+
Step.new("Banking::Account.Open",
|
|
124
|
+
{ "number" => number, "kind" => { "name" => "current" },
|
|
125
|
+
"daily_limit" => { "cents" => 50_000 }, "customer" => "CUST-0001" }),
|
|
126
|
+
Step.new("Banking::Account.Credit",
|
|
127
|
+
{ "number" => number, "amount" => money.call(10_000), "narrative" => { "text" => "Deposit" } }),
|
|
128
|
+
Step.new("Banking::Account.Credit",
|
|
129
|
+
{ "number" => number, "amount" => money.call(5000), "narrative" => { "text" => "Deposit" } }),
|
|
130
|
+
Step.new("Banking::Account.Debit",
|
|
131
|
+
{ "number" => number, "amount" => money.call(2500), "narrative" => { "text" => "Groceries" } })
|
|
132
|
+
]
|
|
133
|
+
})
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
data/lib/hecks/bench.rb
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
module Hecks
|
|
2
|
+
# The throughput and latency harness `bin/bench` drives.
|
|
3
|
+
#
|
|
4
|
+
# It dispatches a fixed, valid command workload against the Ruby runtime on each
|
|
5
|
+
# persistence adapter and against the native Rust binary, and reports
|
|
6
|
+
# commands per second plus p50 and p99 latency for each. Not required by
|
|
7
|
+
# `lib/hecks.rb` on purpose: a booted domain never needs it.
|
|
8
|
+
#
|
|
9
|
+
# ## What is measured
|
|
10
|
+
#
|
|
11
|
+
# Every runtime is driven the same way: one caller, one command at a time, each
|
|
12
|
+
# waiting for the previous answer. The Ruby side times `dispatch_flat` in-process.
|
|
13
|
+
# The Rust side times a full round trip over the pipes of `rust --serve`. See
|
|
14
|
+
# `docs/benchmarks.md` for what that does and does not compare.
|
|
15
|
+
module Bench
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
require_relative "bench/stats"
|
|
20
|
+
require_relative "bench/workload"
|
|
21
|
+
require_relative "bench/environment"
|
|
22
|
+
require_relative "bench/postgres_probe"
|
|
23
|
+
require_relative "bench/run"
|
|
24
|
+
require_relative "bench/ruby_runner"
|
|
25
|
+
require_relative "bench/rust_runner"
|
|
26
|
+
require_relative "bench/suite"
|
|
27
|
+
require_relative "bench/report"
|
|
28
|
+
require_relative "bench/cli"
|
|
@@ -82,6 +82,8 @@ module Hecks
|
|
|
82
82
|
["Type", "one_of"] => :one_of_impl,
|
|
83
83
|
["World", "realm"] => :realm_impl,
|
|
84
84
|
["World", "latest"] => :latest_impl,
|
|
85
|
+
["World", "default_database"] => :default_database_impl,
|
|
86
|
+
["World", "default_adapter"] => :default_adapter_impl,
|
|
85
87
|
["DomainPort", "operation"] => :tells_impl,
|
|
86
88
|
["DomainPort", "tells"] => :tells_impl,
|
|
87
89
|
["DomainPort", "asks"] => :asks_impl,
|
|
@@ -22,7 +22,7 @@ module Hecks
|
|
|
22
22
|
# same as an absent `to:` and falls through to `from` (also
|
|
23
23
|
# absent), so `then_set :accepted, to: false` raised "names no
|
|
24
24
|
# operation" for the one value most likely to be written that way
|
|
25
|
-
# (a boolean flip). Confirmed real, live:
|
|
25
|
+
# (a boolean flip). Confirmed real, live: an external project's
|
|
26
26
|
# dream_interpretation.bluebook (`then_set :accepted, to: false`)
|
|
27
27
|
# and transparency.bluebook (`then_set :always, to: false`). TODO
|
|
28
28
|
# upstream via bin/evolve.
|
|
@@ -262,7 +262,7 @@ module Hecks
|
|
|
262
262
|
# era keeps booting, which is the whole point of the rename column.
|
|
263
263
|
#
|
|
264
264
|
# Vendored addition, not (yet) upstream hecks: `then_set
|
|
265
|
-
# :target, from: :source_field` (
|
|
265
|
+
# :target, from: :source_field` (an external project, found
|
|
266
266
|
# live in body/doctor/bluebook/doctor.bluebook) -- semantically
|
|
267
267
|
# identical to `to:` (copy this argument/field into the target),
|
|
268
268
|
# different word. TODO upstream via bin/evolve (migration plan
|
|
@@ -271,7 +271,7 @@ module Hecks
|
|
|
271
271
|
#
|
|
272
272
|
# Vendored addition, not (yet) upstream hecks (migration plan
|
|
273
273
|
# task 4, i106 in-DSL math): `multiply:`/`clamp:` -- per-tick organ
|
|
274
|
-
# math (
|
|
274
|
+
# math (an external project's organ bluebook: strength decays ×0.98,
|
|
275
275
|
# weight/strength clamp to [0, 1]), moved into the bluebook itself
|
|
276
276
|
# from shell-side awk. `multiply:` mirrors
|
|
277
277
|
# increment/decrement's shape exactly (a Numeric amount, applied
|
|
@@ -34,7 +34,7 @@ module Hecks
|
|
|
34
34
|
# only external facts with attribute" the refusal message
|
|
35
35
|
# describes) rather than folding `to:` into identity_attribute's
|
|
36
36
|
# existing Reference-attribute scan, which every operation already
|
|
37
|
-
# in the corpus (Banking, pizzas,
|
|
37
|
+
# in the corpus (Banking, pizzas, a client site's own vendored
|
|
38
38
|
# PaymentGateway) still relies on unchanged.
|
|
39
39
|
#
|
|
40
40
|
# @param name [String] the operation's name
|