puma-plus-core 0.1.0

data/go/internal/control/decide_test.go

@@ -0,0 +1,526 @@
+package control
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+var base = time.Unix(1700000000, 0)
+
+// harness drives Decide over a sequence of ticks, the way the real loop does.
+type harness struct {
+	cfg     Config
+	st      State
+	now     time.Time
+	workers []WorkerInfo
+	nextID  int
+	actions []Action
+}
+
+func newHarness(cfg Config) *harness {
+	h := &harness{cfg: cfg, now: base, nextID: 1}
+	for i := 0; i < cfg.MinWorkers; i++ {
+		h.addWorker(true)
+	}
+	return h
+}
+
+func (h *harness) addWorker(booted bool) int {
+	id := h.nextID
+	h.nextID++
+	h.workers = append(h.workers, WorkerInfo{
+		ID: id, StartedAt: h.now, Booted: booted,
+		Slots: h.cfg.ThreadsPerWorker, RSSBytes: 100 << 20, LastSeen: h.now,
+	})
+	return id
+}
+
+func (h *harness) removeWorker(id int) {
+	out := h.workers[:0]
+	for _, w := range h.workers {
+		if w.ID != id {
+			out = append(out, w)
+		}
+	}
+	h.workers = out
+}
+
+// tick advances one second and runs the controller, applying any action to the
+// simulated worker set so the next tick sees the consequence.
+func (h *harness) tick(in Inputs) []Action {
+	h.now = h.now.Add(time.Second)
+	in.Now = h.now
+	in.Workers = h.workers
+	if in.Capacity == 0 {
+		in.Capacity = len(h.workers) * h.cfg.ThreadsPerWorker
+	}
+	acts, st := Decide(h.cfg, in, h.st)
+	h.st = st
+	for _, a := range acts {
+		switch a.Kind {
+		case ActionSpawnWorker:
+			h.addWorker(true) // boots instantly unless a test says otherwise
+		case ActionRetireWorker:
+			h.removeWorker(a.WorkerID)
+		}
+	}
+	h.actions = append(h.actions, acts...)
+	return acts
+}
+
+func (h *harness) countKind(k ActionKind) int {
+	n := 0
+	for _, a := range h.actions {
+		if a.Kind == k {
+			n++
+		}
+	}
+	return n
+}
+
+func breaching(cfg Config) Inputs {
+	return Inputs{
+		QueueP95:      cfg.TargetQueueP95 * 4,
+		PredictedWait: cfg.TargetQueueP95 * 4,
+		Depth:         50,
+		ArrivalRate:   200,
+		ServiceMean:   0.025,
+	}
+}
+
+func quiet(cfg Config) Inputs {
+	return Inputs{
+		QueueP95:      cfg.TargetQueueP95 / 100,
+		PredictedWait: 0,
+		Depth:         0,
+		ArrivalRate:   1,
+		ServiceMean:   0.025,
+	}
+}
+
+// TestNoActionWhenHealthy is the most important negative test: a controller that
+// fidgets when nothing is wrong is worse than no controller.
+func TestNoActionWhenHealthy(t *testing.T) {
+	cfg := DefaultConfig()
+	h := newHarness(cfg)
+
+	for i := 0; i < 30; i++ {
+		in := Inputs{
+			QueueP95:      cfg.TargetQueueP95 / 2, // under target, but not idle
+			PredictedWait: cfg.TargetQueueP95 / 2,
+			ArrivalRate:   100,
+			ServiceMean:   0.025,
+		}
+		if acts := h.tick(in); len(acts) > 0 {
+			t.Fatalf("tick %d: acted on a healthy system: %+v", i, acts)
+		}
+	}
+}
+
+// TestScaleUpRequiresSustainedBreach pins the hysteresis: a single bad tick must
+// not fork a process.
+func TestScaleUpRequiresSustainedBreach(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.MinWorkers = 1
+	h := newHarness(cfg)
+
+	// One breach, with the predictor NOT confirming, then recovery.
+	in := breaching(cfg)
+	in.PredictedWait = 0
+	if acts := h.tick(in); len(acts) > 0 {
+		t.Fatalf("forked on a single unconfirmed breach: %+v", acts)
+	}
+	if acts := h.tick(quiet(cfg)); len(acts) > 0 {
+		t.Fatalf("acted after recovery: %+v", acts)
+	}
+	if h.st.BreachTicks != 0 {
+		t.Errorf("breach counter should reset on recovery, got %d", h.st.BreachTicks)
+	}
+}
+
+// TestScaleUpOnSustainedBreach: three consecutive breaches must fork.
+func TestScaleUpOnSustainedBreach(t *testing.T) {
+	cfg := DefaultConfig()
+	h := newHarness(cfg)
+
+	in := breaching(cfg)
+	in.PredictedWait = 0 // measured signal only
+	for i := 0; i < cfg.BreachTicksToScale-1; i++ {
+		if acts := h.tick(in); len(acts) > 0 {
+			t.Fatalf("tick %d: forked too early: %+v", i, acts)
+		}
+	}
+	acts := h.tick(in)
+	if len(acts) != 1 || acts[0].Kind != ActionSpawnWorker {
+		t.Fatalf("expected a spawn on tick %d, got %+v", cfg.BreachTicksToScale, acts)
+	}
+	if acts[0].Reason == "" {
+		t.Error("every action must carry a reason; it is the primary debugging artifact")
+	}
+}
+
+// TestPredictorShortCircuitsBreachCount: measured pain plus predicted pain is a
+// stronger signal than either alone, so it may act on the first tick.
+func TestPredictorShortCircuits(t *testing.T) {
+	cfg := DefaultConfig()
+	h := newHarness(cfg)
+
+	in := breaching(cfg)
+	in.PredictedWait = cfg.TargetQueueP95 * 5 // well past the 2x confirmation bar
+	acts := h.tick(in)
+	if len(acts) != 1 || acts[0].Kind != ActionSpawnWorker {
+		t.Fatalf("confirmed breach should scale immediately, got %+v", acts)
+	}
+}
+
+// TestPredictorAloneNeverScales is the guard derived from the Phase 2 finding
+// that the predictor lags workload regime changes badly (it reported a 7ms wait
+// while requests were actually waiting 600ms). A high prediction with a healthy
+// measurement must not fork anything.
+func TestPredictorAloneNeverScales(t *testing.T) {
+	cfg := DefaultConfig()
+	h := newHarness(cfg)
+
+	for i := 0; i < 20; i++ {
+		in := Inputs{
+			QueueP95:      cfg.TargetQueueP95 / 2, // measurement says fine
+			PredictedWait: time.Second,            // predictor says catastrophe
+			ArrivalRate:   100,
+			ServiceMean:   0.025,
+		}
+		if acts := h.tick(in); len(acts) > 0 {
+			t.Fatalf("tick %d: scaled on the predictor alone: %+v", i, acts)
+		}
+	}
+}
+
+// TestSpawnCooldown: after a fork, further forks wait out the cooldown even
+// under continuous breach. This is what stops fork thrash.
+func TestSpawnCooldown(t *testing.T) {
+	cfg := DefaultConfig()
+	h := newHarness(cfg)
+
+	in := breaching(cfg)
+	in.PredictedWait = cfg.TargetQueueP95 * 5
+	if acts := h.tick(in); len(acts) != 1 || acts[0].Kind != ActionSpawnWorker {
+		t.Fatalf("expected first spawn, got %+v", acts)
+	}
+
+	// Keep breaching hard for the cooldown window.
+	spawns := 0
+	for i := 0; i < int(cfg.SpawnCooldown.Seconds())-1; i++ {
+		for _, a := range h.tick(in) {
+			if a.Kind == ActionSpawnWorker {
+				spawns++
+			}
+		}
+	}
+	if spawns != 0 {
+		t.Errorf("spawned %d more workers inside the %s cooldown", spawns, cfg.SpawnCooldown)
+	}
+
+	// Past the cooldown, it may act again.
+	for i := 0; i < 3; i++ {
+		for _, a := range h.tick(in) {
+			if a.Kind == ActionSpawnWorker {
+				spawns++
+			}
+		}
+	}
+	if spawns == 0 {
+		t.Error("cooldown expired but the controller never scaled again")
+	}
+}
+
+// TestSettledGate: never fork on top of a worker that is still booting.
+func TestSettledGateBlocksSpawn(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.SpawnCooldown = 0
+	h := newHarness(cfg)
+	h.workers = append(h.workers, WorkerInfo{
+		ID: 99, StartedAt: h.now, Booted: false,
+		Slots: cfg.ThreadsPerWorker, RSSBytes: 100 << 20,
+	})
+
+	in := breaching(cfg)
+	in.PredictedWait = cfg.TargetQueueP95 * 5
+	for i := 0; i < 10; i++ {
+		for _, a := range h.tick(in) {
+			if a.Kind == ActionSpawnWorker {
+				t.Fatalf("tick %d: forked while worker 99 was still booting", i)
+			}
+			if a.Kind == ActionNone && a.Reason == "" {
+				t.Error("a declined decision must still explain itself")
+			}
+		}
+	}
+}
+
+// TestMemoryGuard: the projection, not current usage, must veto the fork --
+// once a worker exists it is too late to un-fork it.
+func TestMemoryGuardBlocksSpawn(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.SpawnCooldown = 0
+	cfg.MemLimitBytes = 250 << 20 // room for two 100MB workers, not three
+	h := newHarness(cfg)
+	h.addWorker(true) // now 2 workers x 100MB = 200MB
+
+	in := breaching(cfg)
+	in.PredictedWait = cfg.TargetQueueP95 * 5
+
+	sawVeto := false
+	for i := 0; i < 10; i++ {
+		for _, a := range h.tick(in) {
+			if a.Kind == ActionSpawnWorker {
+				t.Fatalf("tick %d: forked past the memory limit", i)
+			}
+			if a.Kind == ActionNone {
+				sawVeto = true
+			}
+		}
+	}
+	if !sawVeto {
+		t.Error("memory guard should have logged a declined decision")
+	}
+}
+
+func TestMaxWorkersRespected(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.SpawnCooldown = 0
+	cfg.MaxWorkers = 3
+	h := newHarness(cfg)
+
+	in := breaching(cfg)
+	in.PredictedWait = cfg.TargetQueueP95 * 5
+	for i := 0; i < 50; i++ {
+		h.tick(in)
+	}
+	if len(h.workers) > cfg.MaxWorkers {
+		t.Errorf("grew to %d workers past max_workers=%d", len(h.workers), cfg.MaxWorkers)
+	}
+}
+
+// TestScaleDownIsSlow checks the deliberate asymmetry: up in a few seconds,
+// down over a minute. Queues hurt users; idle memory only costs money.
+func TestScaleDownIsSlow(t *testing.T) {
+	cfg := DefaultConfig()
+	h := newHarness(cfg)
+	h.addWorker(true)
+	h.addWorker(true)
+	h.now = h.now.Add(2 * cfg.MinWorkerLifetime) // past the lifetime floor
+	for i := range h.workers {
+		h.workers[i].StartedAt = base
+	}
+
+	for i := 0; i < cfg.IdleTicksToRetire-1; i++ {
+		for _, a := range h.tick(quiet(cfg)) {
+			if a.Kind == ActionRetireWorker {
+				t.Fatalf("tick %d: retired before %d idle ticks", i, cfg.IdleTicksToRetire)
+			}
+		}
+	}
+	acts := h.tick(quiet(cfg))
+	if len(acts) != 1 || acts[0].Kind != ActionRetireWorker {
+		t.Fatalf("expected a retire after %d idle ticks, got %+v", cfg.IdleTicksToRetire, acts)
+	}
+}
+
+// TestRetiresNewestNotOldest: the oldest worker is the warmest, so retiring it
+// would trade away the best-performing process.
+func TestRetiresNewestWorker(t *testing.T) {
+	cfg := DefaultConfig()
+	h := newHarness(cfg)
+
+	oldest := h.workers[0].ID
+	h.now = h.now.Add(10 * time.Minute)
+	newestID := h.addWorker(true)
+	h.now = h.now.Add(10 * time.Minute)
+
+	for i := 0; i < cfg.IdleTicksToRetire+1; i++ {
+		for _, a := range h.tick(quiet(cfg)) {
+			if a.Kind == ActionRetireWorker {
+				if a.WorkerID != newestID {
+					t.Fatalf("retired worker %d; must retire the newest (%d), not the warmest (%d)",
+						a.WorkerID, newestID, oldest)
+				}
+				return
+			}
+		}
+	}
+	t.Fatal("never retired")
+}
+
+func TestNeverRetiresBelowMinWorkers(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.MinWorkers = 2
+	h := newHarness(cfg)
+	h.now = h.now.Add(10 * time.Minute)
+
+	for i := 0; i < 300; i++ {
+		h.tick(quiet(cfg))
+	}
+	if len(h.workers) < cfg.MinWorkers {
+		t.Errorf("shrank to %d workers below min_workers=%d", len(h.workers), cfg.MinWorkers)
+	}
+}
+
+// TestMinWorkerLifetime stops a just-forked worker from being retired
+// immediately when load subsides, which would be pure churn.
+func TestMinWorkerLifetime(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.IdleTicksToRetire = 2
+	h := newHarness(cfg)
+	h.addWorker(true) // born now, so inside the lifetime floor
+
+	for i := 0; i < 10; i++ {
+		for _, a := range h.tick(quiet(cfg)) {
+			if a.Kind == ActionRetireWorker {
+				t.Fatalf("retired a worker only %s old (floor is %s)",
+					h.now.Sub(base), cfg.MinWorkerLifetime)
+			}
+		}
+	}
+}
+
+// TestNoOscillation is the property that matters most and is hardest to get
+// right: alternating load must not produce alternating spawn/retire.
+func TestNoOscillation(t *testing.T) {
+	cfg := DefaultConfig()
+	h := newHarness(cfg)
+
+	// 10 minutes of load flipping every 5 seconds between breaching and quiet.
+	for i := 0; i < 600; i++ {
+		if (i/5)%2 == 0 {
+			h.tick(breaching(cfg))
+		} else {
+			h.tick(quiet(cfg))
+		}
+	}
+
+	spawns := h.countKind(ActionSpawnWorker)
+	retires := h.countKind(ActionRetireWorker)
+	// Over 10 minutes with a 15s spawn cooldown, ~40 spawns is the theoretical
+	// ceiling. Anything approaching that is thrash.
+	if spawns > 12 {
+		t.Errorf("thrashing: %d spawns in 10 minutes of alternating load", spawns)
+	}
+	if retires > 6 {
+		t.Errorf("thrashing: %d retires in 10 minutes of alternating load", retires)
+	}
+	t.Logf("alternating load over 10min: %d spawns, %d retires, ended at %d workers",
+		spawns, retires, len(h.workers))
+}
+
+func TestShouldShed(t *testing.T) {
+	cfg := DefaultConfig()
+	limit := cfg.ShedLimit()
+
+	if ShouldShed(cfg, limit/2) {
+		t.Error("shed below the limit")
+	}
+	if !ShouldShed(cfg, limit*2) {
+		t.Error("failed to shed past the limit")
+	}
+	// The floor matters: with a tiny target, 10x would be an absurdly twitchy
+	// shed threshold.
+	tiny := Config{TargetQueueP95: time.Millisecond}
+	if tiny.ShedLimit() < time.Second {
+		t.Errorf("shed limit %s should never fall below 1s", tiny.ShedLimit())
+	}
+}
+
+// TestDecideIsPure is the property the replay tooling depends on.
+func TestDecideIsPure(t *testing.T) {
+	cfg := DefaultConfig()
+	in := breaching(cfg)
+	in.Now = base
+	in.Workers = []WorkerInfo{{ID: 1, StartedAt: base, Booted: true, Slots: 5}}
+	st := State{BreachTicks: 5}
+
+	a1, s1 := Decide(cfg, in, st)
+	a2, s2 := Decide(cfg, in, st)
+
+	if len(a1) != len(a2) {
+		t.Fatalf("same inputs produced %d and %d actions", len(a1), len(a2))
+	}
+	for i := range a1 {
+		if a1[i] != a2[i] {
+			t.Errorf("action %d differs between identical calls: %+v vs %+v", i, a1[i], a2[i])
+		}
+	}
+	if s1 != s2 {
+		t.Errorf("same inputs produced different state: %+v vs %+v", s1, s2)
+	}
+}
+
+// TestThreadsWouldHelpUsesGVLWhenAvailable is the payoff of the native
+// instrumentation, and it encodes a case where the cheap heuristic is not merely
+// imprecise but backwards.
+//
+// Measured on this machine: eight threads each burning 50ms of PURE CPU produce
+// service 125.3ms = cpu 50.0ms + gvl_wait 75.3ms. cpu/service is 0.40, so
+// io_fraction reads 0.60 and concludes "IO-bound, add threads" about work where
+// more threads deepen the GVL queue and raise latency. The measurement says the
+// opposite, and it is right.
+func TestThreadsWouldHelpUsesGVLWhenAvailable(t *testing.T) {
+	// The exact figures from that run.
+	const service, cpu, gvl = 125.3, 50.0, 75.3
+
+	measured := Inputs{
+		CPUFraction: cpu / service,
+		GVLFraction: gvl / service,
+		GVLKnown:    true,
+	}
+	help, reason := ThreadsWouldHelp(measured)
+	if help {
+		t.Errorf("with GVL measured at %.2f, threads should NOT help: %s",
+			measured.GVLFraction, reason)
+	}
+	t.Logf("measured  -> threads help = %v (%s)", help, reason)
+
+	// The same request without the extension: only cpu/service is available.
+	guessed := Inputs{CPUFraction: cpu / service, GVLKnown: false}
+	helpGuess, reasonGuess := ThreadsWouldHelp(guessed)
+	t.Logf("heuristic -> threads help = %v (%s)", helpGuess, reasonGuess)
+
+	if !helpGuess {
+		t.Fatal("this test is meant to capture the heuristic getting it WRONG; " +
+			"if io_fraction now says CPU-bound here, the premise has changed")
+	}
+	if helpGuess == help {
+		t.Error("the whole point is that the two disagree on this input")
+	}
+}
+
+// TestThreadsWouldHelpOnGenuineIO: low GVL wait means threads really do help,
+// and the measurement must not be so conservative that it blocks that.
+func TestThreadsWouldHelpOnGenuineIO(t *testing.T) {
+	in := Inputs{
+		CPUFraction: 0.02, // 2% on CPU: waiting on a database
+		GVLFraction: 0.01, // barely any GVL contention
+		GVLKnown:    true,
+	}
+	help, reason := ThreadsWouldHelp(in)
+	if !help {
+		t.Errorf("genuinely IO-bound work should welcome threads: %s", reason)
+	}
+}
+
+// TestUnknownGVLFallsBackToHeuristic: without the extension the controller must
+// still decide, using io_fraction, and say that it is unverified.
+func TestUnknownGVLFallsBackToHeuristic(t *testing.T) {
+	cpuBound := Inputs{CPUFraction: 0.95, GVLKnown: false}
+	if help, reason := ThreadsWouldHelp(cpuBound); help {
+		t.Errorf("io_fraction 0.05 should read as CPU-bound: %s", reason)
+	}
+
+	ioBound := Inputs{CPUFraction: 0.05, GVLKnown: false}
+	help, reason := ThreadsWouldHelp(ioBound)
+	if !help {
+		t.Errorf("io_fraction 0.95 should read as IO-bound: %s", reason)
+	}
+	if !strings.Contains(reason, "unverified") {
+		t.Errorf("a guess must be labelled as one: %q", reason)
+	}
+}