devrites 5.13.1 → 5.14.1

package/engine/internal/overhaul/bench/run.go

@@ -1,11 +1,14 @@
 // Package bench implements the /overhaul bench tool: ratio of medians with a
-// seeded percentile-bootstrap 95% confidence interval.
+// seeded percentile-bootstrap 95% confidence interval, checked against the
+// distribution-free Hodges-Lehmann ratio bounds.
 //
 // Input: {"direction": "lower_is_better"|"higher_is_better", "baseline": [...],
 // "candidate": [...], "order": ["A","B",...], "target": 2.0, "materiality": 1.05,
-// "resamples": 10000, "seed": 1, "resolution": 0}. The unit of resampling is one
-// independent invocation per sample; never pass inner iterations. A k-fold claim
-// is supported only when the CI lower bound >= k.
+// "resamples": 10000, "seed": 1, "resolution": 0}, plus optional "quantile"
+// with "requests_per_invocation" (tail metrics), "tolerance" (non-inferiority
+// guardrail), "budget" (absolute target), "error_rates" and "load". The unit of
+// resampling is one independent invocation per sample; never pass inner
+// iterations. A k-fold claim is supported only when both lower bounds are >= k.
 package bench
 
 import (
@@ -16,6 +19,7 @@ import (
 	"math"
 	"math/rand/v2"
 	"reflect"
+	"slices"
 	"sort"
 	"strconv"
 	"strings"
@@ -52,6 +56,8 @@ type result struct {
 	Verdict         string     `json:"verdict"`
 	Ratio           float64    `json:"ratio"`
 	CI95            [2]float64 `json:"ci95"`
+	HodgesLehmann   *interval  `json:"hodges_lehmann,omitempty"`
+	Quantile        float64    `json:"quantile,omitempty"`
 	BaselineMedian  float64    `json:"baseline_median"`
 	CandidateMedian float64    `json:"candidate_median"`
 	AbsoluteDelta   float64    `json:"absolute_delta"`
@@ -59,20 +65,53 @@ type result struct {
 	N               [2]int     `json:"n"`
 	Target          float64    `json:"target"`
 	Materiality     float64    `json:"materiality"`
+	Guardrail       *guardrail `json:"guardrail,omitempty"`
+	Budget          *budget    `json:"budget,omitempty"`
+	ErrorRate       *errorRate `json:"error_rate,omitempty"`
 	Notes           []string   `json:"notes"`
 }
 
+type interval struct {
+	Ratio float64    `json:"ratio"`
+	CI95  [2]float64 `json:"ci95"`
+}
+
+type guardrail struct {
+	Verdict   string  `json:"verdict"`
+	Tolerance float64 `json:"tolerance"`
+	Floor     float64 `json:"floor"`
+}
+
+type budget struct {
+	Verdict  string  `json:"verdict"`
+	Value    float64 `json:"value"`
+	Quantile float64 `json:"quantile"`
+	Bounds   [2]any  `json:"bounds95"`
+}
+
+type errorRate struct {
+	Verdict   string     `json:"verdict"`
+	Tolerance float64    `json:"tolerance"`
+	Delta     float64    `json:"delta"`
+	CI95      [2]float64 `json:"ci95"`
+}
+
 type noRatio struct {
-	Verdict         string   `json:"verdict"`
-	Reason          string   `json:"reason"`
-	BaselineMedian  float64  `json:"baseline_median"`
-	CandidateMedian float64  `json:"candidate_median"`
-	Notes           []string `json:"notes"`
+	Verdict         string     `json:"verdict"`
+	Reason          string     `json:"reason"`
+	BaselineMedian  float64    `json:"baseline_median"`
+	CandidateMedian float64    `json:"candidate_median"`
+	Budget          *budget    `json:"budget,omitempty"`
+	ErrorRate       *errorRate `json:"error_rate,omitempty"`
+	Notes           []string   `json:"notes"`
 }
 
 const (
-	noteFew     = "fewer than 10 invocations per arm: exploratory only"
-	noteBlocked = "arms not recorded as interleaved: contamination risk, verdict capped at inconclusive"
+	noteFew       = "fewer than 10 invocations per arm: exploratory only"
+	noteBlocked   = "arms not recorded as interleaved: contamination risk, verdict capped at inconclusive"
+	noteThinTail  = "tail quantile rests on too few requests per invocation: verdict capped at inconclusive"
+	noteSaturated = "completed load below 98% of offered load: saturated run, verdict capped at inconclusive"
+	noteNoHL      = "no Hodges-Lehmann bounds (non-positive samples or arms too small): verdict capped at inconclusive"
 )
 
 func compare(d map[string]any) (any, error) {
@@ -81,18 +120,28 @@ func compare(d map[string]any) (any, error) {
 		return nil, fmt.Errorf("direction must be lower_is_better or higher_is_better, got %#v", dir)
 	}
 	lower := dir == "lower_is_better"
-	a, err := samples(d, "baseline")
+	q, err := number(d, "quantile", 0)
+	if err != nil {
+		return nil, err
+	}
+	if q < 0 || q >= 1 {
+		return nil, fmt.Errorf("quantile must be in (0, 1), got %v", q)
+	}
+	notes := []string{}
+	a, thinA, err := samples(d, "baseline", q)
 	if err != nil {
 		return nil, err
 	}
-	b, err := samples(d, "candidate")
+	b, thinB, err := samples(d, "candidate", q)
 	if err != nil {
 		return nil, err
 	}
+	if thinA || thinB {
+		notes = append(notes, noteThinTail)
+	}
 	if len(a) == 0 || len(b) == 0 {
 		return nil, errors.New("no median for empty data")
 	}
-	notes := []string{}
 	n := min(len(a), len(b))
 	if n < 10 {
 		notes = append(notes, noteFew)
@@ -107,22 +156,25 @@ func compare(d map[string]any) (any, error) {
 	if len(order) != len(a)+len(b) || switches < n/2 {
 		notes = append(notes, noteBlocked)
 	}
+	saturated, err := saturation(d)
+	if err != nil {
+		return nil, err
+	}
+	if saturated {
+		notes = append(notes, noteSaturated)
+	}
+	bud, err := checkBudget(d, b, lower, saturated || thinB)
+	if err != nil {
+		return nil, err
+	}
 	ma, mb := median(a), median(b)
 	floor, err := number(d, "resolution", 0)
 	if err != nil {
 		return nil, err
 	}
-	if min(ma, mb) <= floor {
-		return noRatio{"NO_RATIO", "median at or below measurement resolution", ma, mb, notes}, nil
-	}
-	seed := int64(1)
-	if v, ok := d["seed"]; ok {
-		num, isNum := v.(json.Number)
-		s, err := num.Int64()
-		if !isNum || err != nil {
-			return nil, fmt.Errorf("seed must be an integer, got %v", v)
-		}
-		seed = s
+	rng, err := seeded(d)
+	if err != nil {
+		return nil, err
 	}
 	resamples, err := number(d, "resamples", 10000)
 	if err != nil {
@@ -131,7 +183,13 @@ func compare(d map[string]any) (any, error) {
 	if int(resamples) < 1 {
 		return nil, fmt.Errorf("resamples must be at least 1, got %v", resamples)
 	}
-	rng := rand.New(rand.NewPCG(uint64(seed), 0)) // #nosec G404 -- bootstrap resampling needs a seeded, reproducible PRNG, not secrecy
+	er, err := checkErrors(d, rng, int(resamples), saturated)
+	if err != nil {
+		return nil, err
+	}
+	if min(ma, mb) <= floor {
+		return noRatio{"NO_RATIO", "median at or below measurement resolution", ma, mb, bud, er, notes}, nil
+	}
 	boots := make([]float64, int(resamples))
 	ra, rb := make([]float64, len(a)), make([]float64, len(b))
 	for i := range boots {
@@ -148,7 +206,8 @@ func compare(d map[string]any) (any, error) {
 	if hiIdx < 0 { // Python's boots[-1]
 		hiIdx += len(boots)
 	}
-	lo, hi := boots[int(0.025*float64(len(boots)))], boots[hiIdx]
+	bootLo, bootHi := boots[int(0.025*float64(len(boots)))], boots[hiIdx]
+	lo, hi := bootLo, bootHi
 	target, err := number(d, "target", 1)
 	if err != nil {
 		return nil, err
@@ -157,6 +216,19 @@ func compare(d map[string]any) (any, error) {
 	if err != nil {
 		return nil, err
 	}
+	// Claims need both methods: the lower bound is the smaller of the two, and
+	// either method's upper bound below 1 is a regression.
+	num, den := a, b // the ratio is baseline/candidate when lower is better
+	if !lower {
+		num, den = b, a
+	}
+	var hl *interval
+	if est, hlLo, hlHi, ok := hlRatio(num, den, 0.05); ok {
+		hl = &interval{round(est, 4), [2]float64{round(hlLo, 4), round(hlHi, 4)}}
+		lo, hi = min(lo, hlLo), min(hi, hlHi)
+	} else {
+		notes = append(notes, noteNoHL)
+	}
 	var verdict string
 	switch {
 	case hi < 1:
@@ -173,15 +245,198 @@ func compare(d map[string]any) (any, error) {
 	if len(notes) > 0 && verdict != "REGRESSION" {
 		verdict = "INCONCLUSIVE"
 	}
+	var guard *guardrail
+	if _, ok := d["tolerance"]; ok {
+		tol, err := number(d, "tolerance", 0)
+		if err != nil || tol < 0 {
+			return nil, fmt.Errorf("tolerance must be a number >= 0, got %v", d["tolerance"])
+		}
+		fl := 1 / (1 + tol)
+		guard = &guardrail{"INCONCLUSIVE", tol, round(fl, 4)}
+		switch {
+		case hi < fl:
+			guard.Verdict = "REGRESSION"
+		case lo >= fl && len(notes) == 0:
+			guard.Verdict = "NON_INFERIOR"
+		}
+	}
 	delta := mb - ma
 	return result{
-		Verdict: verdict, Ratio: round(ratio(ma, mb, lower), 4), CI95: [2]float64{round(lo, 4), round(hi, 4)},
+		Verdict: verdict, Ratio: round(ratio(ma, mb, lower), 4), CI95: [2]float64{round(bootLo, 4), round(bootHi, 4)},
+		HodgesLehmann: hl, Quantile: q,
 		BaselineMedian: ma, CandidateMedian: mb, AbsoluteDelta: delta,
 		PercentChange: round(100*delta/ma, 2), N: [2]int{len(a), len(b)},
-		Target: target, Materiality: mat, Notes: notes,
+		Target: target, Materiality: mat, Guardrail: guard, Budget: bud, ErrorRate: er, Notes: notes,
 	}, nil
 }
 
+func seeded(d map[string]any) (*rand.Rand, error) {
+	seed := int64(1)
+	if v, ok := d["seed"]; ok {
+		num, isNum := v.(json.Number)
+		s, err := num.Int64()
+		if !isNum || err != nil {
+			return nil, fmt.Errorf("seed must be an integer, got %v", v)
+		}
+		seed = s
+	}
+	return rand.New(rand.NewPCG(uint64(seed), 0)), nil // #nosec G404 -- bootstrap resampling needs a seeded, reproducible PRNG, not secrecy
+}
+
+// saturation reports whether any arm completed less than 98% of its offered
+// load; latency from a saturated run supports no claim.
+func saturation(d map[string]any) (bool, error) {
+	l := ovio.Obj(d["load"])
+	if l == nil {
+		return false, nil
+	}
+	for _, arm := range []string{"baseline", "candidate"} {
+		off, err := toFloat(ovio.Obj(l["offered"])[arm], "load.offered."+arm)
+		if err != nil {
+			return false, err
+		}
+		done, err := toFloat(ovio.Obj(l["completed"])[arm], "load.completed."+arm)
+		if err != nil {
+			return false, err
+		}
+		if done < 0.98*off {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// checkBudget tests the candidate against an absolute target with one-sided
+// 95% distribution-free bounds on the chosen quantile of its invocations; a
+// deterministic metric (bytes, counts) must repeat exactly and needs no bound.
+func checkBudget(d map[string]any, b []float64, lower, capped bool) (*budget, error) {
+	m := ovio.Obj(d["budget"])
+	if m == nil {
+		return nil, nil
+	}
+	v, err := toFloat(m["value"], "budget.value")
+	if err != nil {
+		return nil, err
+	}
+	p := 0.5
+	if _, ok := m["quantile"]; ok {
+		if p, err = toFloat(m["quantile"], "budget.quantile"); err != nil || p <= 0 || p >= 1 {
+			return nil, fmt.Errorf("budget.quantile must be in (0, 1), got %v", m["quantile"])
+		}
+	}
+	out := &budget{Value: v, Quantile: p}
+	lcb, ucb := quantileBounds(b, p, 0.05)
+	if ovio.Truthy(m["deterministic"]) {
+		lcb, ucb = b[0], b[0]
+		for _, x := range b {
+			if x != b[0] {
+				out.Verdict = "NONDETERMINISTIC"
+				return out, nil
+			}
+		}
+	}
+	out.Bounds = [2]any{bound(lcb), bound(ucb)}
+	good, bad := ucb <= v, lcb > v // lower is better
+	if !lower {
+		good, bad = lcb >= v, ucb < v
+	}
+	switch {
+	case bad:
+		out.Verdict = "FAIL"
+	case capped:
+		out.Verdict = "INCONCLUSIVE"
+	case good:
+		out.Verdict = "PASS"
+	case math.IsInf(lcb, 0) || math.IsInf(ucb, 0):
+		out.Verdict = "INSUFFICIENT_DATA"
+	default:
+		out.Verdict = "INCONCLUSIVE"
+	}
+	return out, nil
+}
+
+// bound renders an infinite bound as null.
+func bound(x float64) any {
+	if math.IsInf(x, 0) {
+		return nil
+	}
+	return x
+}
+
+// checkErrors compares per-invocation error rates (candidate minus baseline)
+// against an absolute tolerance. The interval is the wider of a bootstrap over
+// invocations and, with request totals, the Newcombe hybrid score interval.
+func checkErrors(d map[string]any, rng *rand.Rand, resamples int, saturated bool) (*errorRate, error) {
+	m := ovio.Obj(d["error_rates"])
+	if m == nil {
+		return nil, nil
+	}
+	ea, _, err := samples(m, "baseline", 0)
+	if err != nil {
+		return nil, err
+	}
+	eb, _, err := samples(m, "candidate", 0)
+	if err != nil {
+		return nil, err
+	}
+	if len(ea) == 0 || len(eb) == 0 {
+		return nil, errors.New("error_rates needs baseline and candidate rates")
+	}
+	for _, x := range append(slices.Clone(ea), eb...) {
+		if x < 0 || x > 1 {
+			return nil, fmt.Errorf("error_rates must be proportions in [0, 1], got %v", x)
+		}
+	}
+	tol, err := toFloat(m["tolerance"], "error_rates.tolerance")
+	if err != nil {
+		return nil, err
+	}
+	mean := func(xs []float64) float64 {
+		s := 0.0
+		for _, x := range xs {
+			s += x
+		}
+		return s / float64(len(xs))
+	}
+	boots := make([]float64, resamples)
+	ra, rb := make([]float64, len(ea)), make([]float64, len(eb))
+	for i := range boots {
+		for j := range ra {
+			ra[j] = ea[rng.IntN(len(ea))]
+		}
+		for j := range rb {
+			rb[j] = eb[rng.IntN(len(eb))]
+		}
+		boots[i] = mean(rb) - mean(ra)
+	}
+	sort.Float64s(boots)
+	lo, hi := boots[int(0.025*float64(len(boots)))], boots[max(0, int(0.975*float64(len(boots)))-1)]
+	pa, pb := mean(ea), mean(eb)
+	if req := ovio.Obj(m["requests"]); req != nil {
+		na, err := toFloat(req["baseline"], "error_rates.requests.baseline")
+		if err != nil {
+			return nil, err
+		}
+		nb, err := toFloat(req["candidate"], "error_rates.requests.candidate")
+		if err != nil {
+			return nil, err
+		}
+		if na <= 0 || nb <= 0 {
+			return nil, fmt.Errorf("error_rates.requests must be positive, got %v and %v", na, nb)
+		}
+		nl, nu := newcombe(pb, nb, pa, na, 1.96)
+		lo, hi = min(lo, nl), max(hi, nu)
+	}
+	out := &errorRate{"INCONCLUSIVE", tol, round(pb-pa, 6), [2]float64{round(lo, 6), round(hi, 6)}}
+	switch {
+	case lo > tol:
+		out.Verdict = "FAIL"
+	case hi <= tol && !saturated:
+		out.Verdict = "PASS"
+	}
+	return out, nil
+}
+
 func ratio(base, cand float64, lowerBetter bool) float64 {
 	if lowerBetter {
 		return base / cand
@@ -204,24 +459,58 @@ func round(x float64, places int) float64 {
 	return math.Round(x*p) / p
 }
 
-func samples(d map[string]any, key string) ([]float64, error) {
+// samples reads one value per invocation. With a quantile q, an invocation may
+// be given as its raw request latencies (reduced to their q-quantile) or as its
+// q-quantile with requests_per_invocation; thin reports an invocation with
+// fewer than 10/(1-q) requests, too few for a stable tail.
+func samples(d map[string]any, key string, q float64) (out []float64, thin bool, err error) {
 	v, ok := d[key]
 	if !ok {
-		return nil, fmt.Errorf("missing %q", key)
+		return nil, false, fmt.Errorf("missing %q", key)
 	}
 	list, ok := v.([]any)
 	if !ok {
-		return nil, fmt.Errorf("%s must be an array", key)
+		return nil, false, fmt.Errorf("%s must be an array", key)
+	}
+	need := 0.0
+	nested := slices.ContainsFunc(list, func(x any) bool { _, ok := x.([]any); return ok })
+	if nested && slices.ContainsFunc(list, func(x any) bool { _, ok := x.([]any); return !ok }) {
+		return nil, false, fmt.Errorf("%s mixes raw request arrays with per-invocation values", key)
+	}
+	if q > 0 {
+		need = math.Ceil(10/(1-q) - 1e-9)
+		if !nested {
+			n, err := toFloat(d["requests_per_invocation"], "requests_per_invocation")
+			if err != nil {
+				return nil, false, fmt.Errorf("a quantile over per-invocation values needs requests_per_invocation: %w", err)
+			}
+			thin = n < need
+		}
 	}
-	out := make([]float64, len(list))
+	out = make([]float64, len(list))
 	for i, x := range list {
-		f, err := toFloat(x, key)
-		if err != nil {
-			return nil, err
+		if inner, isList := x.([]any); isList {
+			if q == 0 {
+				return nil, false, fmt.Errorf("%s: raw request arrays need a quantile", key)
+			}
+			reqs := make([]float64, len(inner))
+			for j, r := range inner {
+				if reqs[j], err = toFloat(r, key); err != nil {
+					return nil, false, err
+				}
+			}
+			if len(reqs) == 0 {
+				return nil, false, fmt.Errorf("%s: invocation %d has no requests", key, i)
+			}
+			thin = thin || float64(len(reqs)) < need
+			out[i] = quantile7(reqs, q)
+			continue
+		}
+		if out[i], err = toFloat(x, key); err != nil {
+			return nil, false, err
 		}
-		out[i] = f
 	}
-	return out, nil
+	return out, thin, nil
 }
 
 func number(d map[string]any, key string, def float64) (float64, error) {

package/engine/internal/overhaul/bench/run_test.go

@@ -3,11 +3,15 @@ package bench
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
+	"math"
 	"math/rand/v2"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
+
+	"github.com/devrites/devrites/internal/overhaul/ovio"
 )
 
 func gauss(rng *rand.Rand, mean, sd float64, n int) []float64 {
@@ -137,3 +141,130 @@ func TestBench(t *testing.T) {
 		}
 	})
 }
+
+func TestStats(t *testing.T) {
+	if k := tailCount(10, 10, 0.05); k != 24 {
+		t.Fatalf("n=m=10 alpha .05: k=%d, want 24 (Hollander-Wolfe D_(24), D_(77))", k)
+	}
+	if k := tailCount(30, 30, 1e-3); 2*k != 464 {
+		t.Fatalf("n=m=30 misrate 1e-3: margin %d, want 464 (Pragmastat PairwiseMargin)", 2*k)
+	}
+	if k := tailCount(3, 3, 0.05); k != 0 {
+		t.Fatalf("n=m=3 has no finite 95%% interval, got k=%d", k)
+	}
+	xs := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+	if lcb, ucb := quantileBounds(xs, 0.5, 0.05); lcb != 2 || ucb != 9 {
+		t.Fatalf("median bounds n=10: [%v, %v], want [2, 9]", lcb, ucb)
+	}
+	if _, ucb := quantileBounds(xs, 0.99, 0.05); !math.IsInf(ucb, 1) {
+		t.Fatalf("p99 with n=10 has no upper bound, got %v", ucb)
+	}
+}
+
+func TestBenchExtensions(t *testing.T) {
+	fast, rng := fixture()
+	if _, res := verdict(t, fast); res["hodges_lehmann"] == nil {
+		t.Fatalf("no Hodges-Lehmann bounds: %v", res)
+	}
+	raw := func(mean float64, invocations, requests int) []any {
+		var arm []any
+		for range invocations {
+			arm = append(arm, gauss(rng, mean, mean/10, requests))
+		}
+		return arm
+	}
+	tail := map[string]any{"direction": "lower_is_better", "quantile": 0.99, "order": interleaved(12), "target": 2.0,
+		"baseline": raw(100, 12, 1000), "candidate": raw(30, 12, 1000)}
+
+	t.Run("tail quantile from raw requests", func(t *testing.T) {
+		if v, res := verdict(t, tail); v != "SUPPORTS_TARGET" || res["quantile"] != 0.99 {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("thin tail is inconclusive", func(t *testing.T) {
+		thin := with(tail, "baseline", raw(100, 12, 50), "candidate", raw(30, 12, 50))
+		if v, res := verdict(t, thin); v != "INCONCLUSIVE" || !strings.Contains(fmt.Sprint(res["notes"]), "too few requests") {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("per-invocation quantiles need request counts", func(t *testing.T) {
+		code, _, stderr := run(t, with(fast, "quantile", 0.95))
+		if code != 2 || !strings.Contains(stderr, "requests_per_invocation") {
+			t.Fatalf("exit %d %q", code, stderr)
+		}
+	})
+	t.Run("guardrail", func(t *testing.T) {
+		same := with(fast, "candidate", gauss(rng, 30, 1, 20), "tolerance", 0.1)
+		if _, res := verdict(t, same); ovio.Get(res, "guardrail.verdict") != "NON_INFERIOR" {
+			t.Fatalf("equal arms: %v", res["guardrail"])
+		}
+		slow := with(fast, "candidate", gauss(rng, 45, 1, 20), "tolerance", 0.1)
+		if _, res := verdict(t, slow); ovio.Get(res, "guardrail.verdict") != "REGRESSION" {
+			t.Fatalf("1.5x slower: %v", res["guardrail"])
+		}
+	})
+	t.Run("budget", func(t *testing.T) {
+		bytes := func(xs ...float64) []float64 { return xs }
+		for name, tc := range map[string]struct {
+			cand   []float64
+			budget map[string]any
+			want   string
+		}{
+			"deterministic pass":   {bytes(100, 100), map[string]any{"value": 120, "deterministic": true}, "PASS"},
+			"deterministic fail":   {bytes(130, 130), map[string]any{"value": 120, "deterministic": true}, "FAIL"},
+			"nondeterministic":     {bytes(100, 101), map[string]any{"value": 120, "deterministic": true}, "NONDETERMINISTIC"},
+			"p75 under budget":     {gauss(rng, 10, 0.4, 20), map[string]any{"value": 12, "quantile": 0.75}, "PASS"},
+			"p75 over budget":      {gauss(rng, 10, 0.4, 20), map[string]any{"value": 8, "quantile": 0.75}, "FAIL"},
+			"p99 needs more runs":  {gauss(rng, 10, 0.4, 20), map[string]any{"value": 12, "quantile": 0.99}, "INSUFFICIENT_DATA"},
+			"budget straddles p75": {bytes(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20), map[string]any{"value": 15, "quantile": 0.75}, "INCONCLUSIVE"},
+		} {
+			if _, res := verdict(t, with(fast, "candidate", tc.cand, "budget", tc.budget)); ovio.Get(res, "budget.verdict") != tc.want {
+				t.Fatalf("%s: %v", name, res["budget"])
+			}
+		}
+	})
+	t.Run("error-rate guardrail", func(t *testing.T) {
+		zeros := make([]float64, 20)
+		clean := map[string]any{"baseline": zeros, "candidate": zeros, "tolerance": 0.001,
+			"requests": map[string]any{"baseline": 20000, "candidate": 20000}}
+		if _, res := verdict(t, with(fast, "error_rates", clean)); ovio.Get(res, "error_rate.verdict") != "PASS" {
+			t.Fatalf("clean: %v", res["error_rate"])
+		}
+		few := with(clean, "requests", map[string]any{"baseline": 200, "candidate": 200})
+		if _, res := verdict(t, with(fast, "error_rates", few)); ovio.Get(res, "error_rate.verdict") != "INCONCLUSIVE" {
+			t.Fatalf("zero errors in 200 requests proves nothing at 0.1%%: %v", res["error_rate"])
+		}
+		worse := with(clean, "candidate", gauss(rng, 0.05, 0.005, 20))
+		if _, res := verdict(t, with(fast, "error_rates", worse)); ovio.Get(res, "error_rate.verdict") != "FAIL" {
+			t.Fatalf("5%% errors: %v", res["error_rate"])
+		}
+	})
+	t.Run("saturated load caps every verdict", func(t *testing.T) {
+		load := map[string]any{"offered": map[string]any{"baseline": 1000, "candidate": 1000},
+			"completed": map[string]any{"baseline": 1000, "candidate": 900}}
+		clean := map[string]any{"baseline": make([]float64, 20), "candidate": make([]float64, 20), "tolerance": 0.01}
+		v, res := verdict(t, with(fast, "load", load, "error_rates", clean))
+		if v != "INCONCLUSIVE" || !strings.Contains(fmt.Sprint(res["notes"]), "saturated") || ovio.Get(res, "error_rate.verdict") != "INCONCLUSIVE" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("thin tail caps the budget", func(t *testing.T) {
+		thin := with(tail, "baseline", raw(100, 12, 50), "candidate", raw(30, 12, 50), "budget", map[string]any{"value": 1000})
+		if _, res := verdict(t, thin); ovio.Get(res, "budget.verdict") != "INCONCLUSIVE" {
+			t.Fatalf("got %v", res["budget"])
+		}
+	})
+	t.Run("invalid inputs are errors", func(t *testing.T) {
+		rates := func(x float64) []float64 { return []float64{x, x, x} }
+		for want, d := range map[string]map[string]any{
+			"proportions in [0, 1]": with(fast, "error_rates", map[string]any{"baseline": rates(1), "candidate": rates(1.5), "tolerance": 1}),
+			"must be positive": with(fast, "error_rates", map[string]any{"baseline": rates(0), "candidate": rates(0), "tolerance": 0.01,
+				"requests": map[string]any{"baseline": 0, "candidate": 1000}}),
+			"mixes raw request arrays": with(tail, "candidate", append(raw(30, 11, 1000), 30.0)),
+		} {
+			if code, _, stderr := run(t, d); code != 2 || !strings.Contains(stderr, want) {
+				t.Fatalf("%s: exit %d %q", want, code, stderr)
+			}
+		}
+	})
+}