puma-plus-core 0.1.0

data/go/internal/control/booted_test.go

@@ -0,0 +1,97 @@
+package control
+
+import (
+	"bytes"
+	"log"
+	"testing"
+	"time"
+)
+
+// newBreachingLoop builds a loop under a confirmed, sustained breach, so the
+// only thing that can hold a spawn back is the settled gate.
+func newBreachingLoop(cfg Config, sh *fakeShepherd, q *fakeQueue) *Loop {
+	m := &fakeMetrics{
+		p95: cfg.TargetQueueP95 * 40, pred: cfg.TargetQueueP95 * 80,
+		arrival: 700, service: 0.025,
+	}
+	return NewLoop(cfg, sh, q, m, nil, log.New(&bytes.Buffer{}, "", 0))
+}
+
+func spawnsIn(l *Loop, from time.Time, ticks int) int {
+	n := 0
+	now := from
+	for i := 0; i < ticks; i++ {
+		now = now.Add(time.Second)
+		for _, a := range l.Tick(now) {
+			if a.Kind == ActionSpawnWorker {
+				n++
+			}
+		}
+	}
+	return n
+}
+
+// TestSettledGateReleasesWhenCapacityArrives is the fix for the measured cost of
+// approximating readiness with a timer.
+//
+// A worker dials in only once its app is loadable, so the connection appearing
+// in the registry IS the readiness signal. Previously the controller ignored
+// that and waited a flat 2s, which cost 7 of the 11 seconds it took to scale
+// 5 -> 30 slots against Ractors that boot in ~4ms.
+func TestSettledGateReleasesWhenCapacityArrives(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.MinWorkers, cfg.MaxWorkers = 1, 8
+	cfg.SpawnCooldown = 0 // isolate the settled gate from the cooldown
+
+	sh := &fakeShepherd{workers: 1}
+	q := &fakeQueue{depth: 400, capacity: cfg.ThreadsPerWorker}
+	loop := newBreachingLoop(cfg, sh, q)
+	now := time.Unix(1700000000, 0)
+
+	// First spawn: one worker exists and its capacity is present, so nothing is
+	// pending and the gate is open.
+	if got := spawnsIn(loop, now, 4); got == 0 {
+		t.Fatal("no spawn under a confirmed sustained breach")
+	}
+	now = now.Add(4 * time.Second)
+
+	// The shepherd now claims 2 workers but the new one has not dialed in, so
+	// observed capacity still reflects only the first. The gate must hold.
+	if got := spawnsIn(loop, now, 4); got != 0 {
+		t.Errorf("%d spawns while the previous worker had not connected; "+
+			"the settled gate should have vetoed every one", got)
+	}
+	now = now.Add(4 * time.Second)
+
+	// Its connections appear. The gate must open on that alone -- no waiting.
+	q.capacity = sh.workers * cfg.ThreadsPerWorker
+	if got := spawnsIn(loop, now, 2); got == 0 {
+		t.Error("capacity arrived but the gate stayed shut; readiness is still " +
+			"being inferred from elapsed time rather than observed connections")
+	}
+}
+
+// TestSettledGateGivesUpOnAWorkerThatNeverConnects: a worker that dies before
+// dialing in must not wedge the controller. Without a deadline the shortfall
+// never closes and every future spawn is vetoed forever.
+func TestSettledGateGivesUpOnAWorkerThatNeverConnects(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.MinWorkers, cfg.MaxWorkers = 1, 8
+	cfg.SpawnCooldown = 0
+	cfg.BootDeadline = 5 * time.Second
+
+	sh := &fakeShepherd{workers: 2} // two claimed...
+	q := &fakeQueue{depth: 400, capacity: cfg.ThreadsPerWorker}
+	loop := newBreachingLoop(cfg, sh, q) // ...one connected, and it never will
+	now := time.Unix(1700000000, 0)
+
+	if got := spawnsIn(loop, now, 3); got != 0 {
+		t.Errorf("%d spawns before the boot deadline; the gate should still hold", got)
+	}
+
+	now = now.Add(3 * time.Second)
+	if got := spawnsIn(loop, now, 6); got == 0 {
+		t.Error("still vetoing past the boot deadline: a worker that never connects " +
+			"has wedged the controller permanently")
+	}
+}

data/go/internal/control/decide.go

@@ -0,0 +1,427 @@
+// Package control implements puma-plus's scaling controller.
+//
+// The controller is split deliberately: decide.go is a pure function with no
+// clocks, no IO and no goroutines, and actuators.go does everything effectful.
+// That split is what makes the decision log replayable -- a recorded sequence of
+// Inputs can be fed back through Decide and asserted on, so a controller
+// regression becomes an ordinary unit test instead of a load test somebody has
+// to watch.
+package control
+
+import (
+	"fmt"
+	"sort"
+	"time"
+)
+
+// Config is the controller's tuning. One knob is meant to be operator-facing
+// (TargetQueueP95); the rest are safety rails.
+type Config struct {
+	// TargetQueueP95 is the SLO: the queue-time p95 we are steering toward.
+	// Latency, not utilization, because a deploy that doubles service time at
+	// constant utilization doubles user-visible latency, and latency is the
+	// contract.
+	TargetQueueP95 time.Duration
+
+	MinWorkers       int
+	MaxWorkers       int
+	ThreadsPerWorker int
+
+	// BreachTicksToScale is how many consecutive breaching ticks are needed
+	// before scaling up on the measured signal alone. The predictor can
+	// short-circuit this (see Decide).
+	BreachTicksToScale int
+	// IdleTicksToRetire is deliberately far larger than BreachTicksToScale.
+	// Queues hurt users; idle memory only costs money.
+	IdleTicksToRetire int
+
+	// BootDeadline caps how long a worker may be counted as "still booting"
+	// before the settled gate stops honouring it. Without it a worker that dies
+	// before dialing in would block every future spawn.
+	BootDeadline time.Duration
+
+	SpawnCooldown     time.Duration
+	RetireCooldown    time.Duration
+	MinWorkerLifetime time.Duration
+
+	// MemLimitBytes caps total worker RSS. Zero disables the guard.
+	MemLimitBytes int64
+}
+
+// DefaultConfig returns the spike's defaults.
+func DefaultConfig() Config {
+	return Config{
+		TargetQueueP95:     25 * time.Millisecond,
+		MinWorkers:         1,
+		MaxWorkers:         8,
+		ThreadsPerWorker:   5,
+		BreachTicksToScale: 3,
+		IdleTicksToRetire:  60,
+		// 3s, not the 15s this started at. A live Phase 4 run showed the
+		// controller sitting out fifteen consecutive breaching ticks logging
+		// "spawn cooldown has N left", and the simulator quantified the cost:
+		// at a 1s fork latency, dropping 15s -> 2s improved queue p95 from
+		// 1.597s to 405ms and cut shedding from 2529 to 387, for two extra forks
+		// under deliberately flapping load.
+		//
+		// The reason it was safe to shorten is that the cooldown was never the
+		// real protection against fork thrash -- the settled gate is. At an 8s
+		// fork latency every cooldown from 2s to 5s produces identical behavior,
+		// because "do not fork while a worker is still booting" is what actually
+		// binds. The cooldown is a backstop for the fast-fork case only.
+		// See sim.TestCooldownTradeoff.
+		BootDeadline:      10 * time.Second,
+		SpawnCooldown:     3 * time.Second,
+		RetireCooldown:    60 * time.Second,
+		MinWorkerLifetime: 60 * time.Second,
+	}
+}
+
+// ShedLimit is the predicted wait past which arrivals are rejected outright.
+//
+// Derived from the target rather than configured separately: an operator who
+// says "I want p95 queue time under 25ms" has already said everything needed to
+// know that a predicted one-second wait is not worth serving.
+func (c Config) ShedLimit() time.Duration {
+	if l := 10 * c.TargetQueueP95; l > time.Second {
+		return l
+	}
+	return time.Second
+}
+
+// WorkerInfo is the controller's view of one Ruby worker process.
+type WorkerInfo struct {
+	ID        int
+	StartedAt time.Time
+	// Booted reports whether the worker has all its threads connected. A worker
+	// still booting is capacity we cannot count on yet.
+	Booted   bool
+	Slots    int
+	Busy     int
+	RSSBytes int64
+	LastSeen time.Time
+	Draining bool
+}
+
+// Inputs is everything Decide is allowed to look at. Now is passed in rather
+// than read from a clock, which is what keeps the function pure and replayable.
+type Inputs struct {
+	Now time.Time
+
+	// QueueP95 is measured over the scaling window. Lagging but authoritative.
+	QueueP95 time.Duration
+	// PredictedWait is Little's Law applied to the next arrival. Leading, but
+	// only as good as the service-time average behind it -- see the note on
+	// metrics.Collector.PredictedWait about how badly it lags a regime change.
+	PredictedWait time.Duration
+
+	Depth       int
+	Capacity    int
+	Busy        int
+	ArrivalRate float64
+	ServiceMean float64
+
+	// GVLFraction is the share of service time spent queued behind the GVL.
+	// Measured by the optional native extension; 0 means "unknown", not
+	// "no contention", which is why GVLKnown exists separately.
+	GVLFraction float64
+	// CPUFraction is the share of service time actually spent on CPU.
+	CPUFraction float64
+	// GVLKnown reports whether GVLFraction is a real measurement. Without it the
+	// controller must fall back to io_fraction and gain probing, which is what
+	// it did before the extension existed.
+	GVLKnown bool
+
+	Workers []WorkerInfo
+}
+
+// Utilization is rho = lambda*S/c. A guard rail and a diagnostic, never the
+// thing we steer on.
+func (in Inputs) Utilization() float64 {
+	if in.Capacity <= 0 {
+		return 0
+	}
+	return in.ArrivalRate * in.ServiceMean / float64(in.Capacity)
+}
+
+// TotalRSS sums worker resident memory.
+func (in Inputs) TotalRSS() int64 {
+	var total int64
+	for _, w := range in.Workers {
+		total += w.RSSBytes
+	}
+	return total
+}
+
+// medianWorkerRSS estimates what one more worker would cost. The median rather
+// than the mean, so one bloated outlier does not block all further scaling.
+func (in Inputs) medianWorkerRSS() int64 {
+	if len(in.Workers) == 0 {
+		return 0
+	}
+	vals := make([]int64, 0, len(in.Workers))
+	for _, w := range in.Workers {
+		vals = append(vals, w.RSSBytes)
+	}
+	sort.Slice(vals, func(i, j int) bool { return vals[i] < vals[j] })
+	return vals[len(vals)/2]
+}
+
+// live counts workers that are neither draining nor presumed dead.
+func (in Inputs) live() []WorkerInfo {
+	out := make([]WorkerInfo, 0, len(in.Workers))
+	for _, w := range in.Workers {
+		if !w.Draining {
+			out = append(out, w)
+		}
+	}
+	return out
+}
+
+// State is the controller's memory between ticks. Carried explicitly rather
+// than held in fields, so a replay can start from any recorded state.
+type State struct {
+	Tick        int64
+	BreachTicks int
+	IdleTicks   int
+	LastSpawn   time.Time
+	LastRetire  time.Time
+}
+
+// ActionKind enumerates what the controller can command.
+type ActionKind string
+
+const (
+	// ActionSpawnWorker forks a new Ruby worker process.
+	ActionSpawnWorker ActionKind = "spawn_worker"
+	// ActionRetireWorker drains and stops a worker.
+	ActionRetireWorker ActionKind = "retire_worker"
+	// ActionNone records a considered-but-declined decision, so the decision log
+	// explains inaction as well as action.
+	ActionNone ActionKind = "none"
+)
+
+// Action is one command, with the reason it was issued. The reason is not
+// decoration: it is the primary debugging artifact when the controller does
+// something surprising at 3am.
+type Action struct {
+	Kind     ActionKind `json:"kind"`
+	WorkerID int        `json:"worker_id,omitempty"`
+	Reason   string     `json:"reason"`
+}
+
+// Decide is the controller. Pure: same inputs and state always produce the same
+// actions and next state.
+//
+// The shape is tiered thresholds with hysteresis and cooldowns, not PID. PID
+// assumes a roughly linear plant with a known gain sign; this plant has discrete
+// actuators whose costs differ by three orders of magnitude, seconds of variable
+// dead time from fork and warmup, and -- the disqualifying part -- an unknown
+// gain sign, because under the GVL adding a thread to CPU-bound work makes queue
+// time worse. A PID would wind up during fork dead-time and oscillate across
+// that regime change. What replaces PID's sophistication here is verification:
+// the controller checks whether its previous action actually helped.
+func Decide(cfg Config, in Inputs, st State) ([]Action, State) {
+	st.Tick++
+
+	target := cfg.TargetQueueP95
+	live := in.live()
+
+	breaching := in.QueueP95 > target
+	confirming := in.PredictedWait > 2*target
+	// Idle needs both a quiet queue and low utilization. Queue time alone is not
+	// enough: a queue can read empty simply because nothing has arrived yet.
+	idling := in.QueueP95 < target/5 && in.Utilization() < 0.5
+
+	if breaching {
+		st.BreachTicks++
+	} else {
+		st.BreachTicks = 0
+	}
+	if idling {
+		st.IdleTicks++
+	} else {
+		st.IdleTicks = 0
+	}
+
+	// --- Scale up -------------------------------------------------------
+	//
+	// Two ways in. The slow way needs BreachTicksToScale consecutive breaches,
+	// which filters out a single unlucky request. The fast way needs one breach
+	// AND the predictor agreeing that the next arrival is in for at least twice
+	// the target -- measured pain plus predicted pain, which is a much stronger
+	// signal than either alone. The predictor never scales on its own, because
+	// it lags workload regime changes badly.
+	wantUp := st.BreachTicks >= cfg.BreachTicksToScale || (st.BreachTicks >= 1 && confirming)
+
+	if wantUp {
+		if act, ok := considerSpawn(cfg, in, st, live); ok {
+			st.LastSpawn = in.Now
+			st.BreachTicks = 0 // let the new capacity be observed before deciding again
+			return []Action{act}, st
+		} else {
+			return []Action{act}, st
+		}
+	}
+
+	// --- Scale down -----------------------------------------------------
+	if st.IdleTicks >= cfg.IdleTicksToRetire {
+		if act, ok := considerRetire(cfg, in, st, live); ok {
+			st.LastRetire = in.Now
+			st.IdleTicks = 0
+			return []Action{act}, st
+		} else if act.Reason != "" {
+			return []Action{act}, st
+		}
+	}
+
+	return nil, st
+}
+
+// considerSpawn applies every guard that can veto a fork. Each veto returns an
+// ActionNone carrying its reason, so the decision log shows why the controller
+// sat still while the queue was breaching.
+func considerSpawn(cfg Config, in Inputs, st State, live []WorkerInfo) (Action, bool) {
+	if len(live) >= cfg.MaxWorkers {
+		return Action{Kind: ActionNone, Reason: fmt.Sprintf(
+			"breach for %d ticks but already at max_workers=%d", st.BreachTicks, cfg.MaxWorkers)}, false
+	}
+
+	// Settled gate: never fork on top of a worker that has not finished booting.
+	// Forking during another worker's warmup stacks GC and CPU load exactly when
+	// the system is already struggling, and hides whether the previous fork
+	// helped.
+	for _, w := range live {
+		if !w.Booted {
+			return Action{Kind: ActionNone, Reason: fmt.Sprintf(
+				"breach for %d ticks but worker %d is still booting", st.BreachTicks, w.ID)}, false
+		}
+	}
+
+	if !st.LastSpawn.IsZero() && in.Now.Sub(st.LastSpawn) < cfg.SpawnCooldown {
+		return Action{Kind: ActionNone, Reason: fmt.Sprintf(
+			"breach for %d ticks but spawn cooldown has %s left",
+			st.BreachTicks, (cfg.SpawnCooldown - in.Now.Sub(st.LastSpawn)).Round(time.Second))}, false
+	}
+
+	// Memory guard. Checked against a projection, not against current usage,
+	// because by the time the new worker's RSS shows up it is too late to
+	// un-fork it. RSS comes from /proc, read by Go -- never trusted from the
+	// process we might have to kill.
+	if cfg.MemLimitBytes > 0 {
+		projected := in.TotalRSS() + in.medianWorkerRSS()
+		if projected > cfg.MemLimitBytes {
+			return Action{Kind: ActionNone, Reason: fmt.Sprintf(
+				"breach for %d ticks but forking would reach %dMB of a %dMB limit",
+				st.BreachTicks, projected>>20, cfg.MemLimitBytes>>20)}, false
+		}
+	}
+
+	return Action{Kind: ActionSpawnWorker, Reason: fmt.Sprintf(
+		"queue p95 %s over target %s for %d ticks (predicted %s, rho %.2f, %d workers)",
+		in.QueueP95.Round(time.Millisecond), cfg.TargetQueueP95, st.BreachTicks,
+		in.PredictedWait.Round(time.Millisecond), in.Utilization(), len(live))}, true
+}
+
+// considerRetire picks a worker to drain, or explains why it will not.
+func considerRetire(cfg Config, in Inputs, st State, live []WorkerInfo) (Action, bool) {
+	if len(live) <= cfg.MinWorkers {
+		return Action{}, false // at the floor; unremarkable, do not log it every tick
+	}
+	if !st.LastRetire.IsZero() && in.Now.Sub(st.LastRetire) < cfg.RetireCooldown {
+		return Action{}, false
+	}
+
+	// Retire the NEWEST worker, never the oldest.
+	//
+	// The oldest worker is the warmest: its heap is settled, its caches are
+	// populated, its JIT (where there is one) has done its work. Retiring it
+	// would trade away the best-performing process and leave a cold one to
+	// absorb the load.
+	victim := newest(live)
+	if victim == nil {
+		return Action{}, false
+	}
+	if in.Now.Sub(victim.StartedAt) < cfg.MinWorkerLifetime {
+		return Action{Kind: ActionNone, Reason: fmt.Sprintf(
+			"idle for %d ticks but worker %d is only %s old",
+			st.IdleTicks, victim.ID, in.Now.Sub(victim.StartedAt).Round(time.Second))}, false
+	}
+
+	return Action{Kind: ActionRetireWorker, WorkerID: victim.ID, Reason: fmt.Sprintf(
+		"queue p95 %s under target for %d ticks (rho %.2f, %d workers)",
+		in.QueueP95.Round(time.Millisecond), st.IdleTicks, in.Utilization(), len(live))}, true
+}
+
+func newest(workers []WorkerInfo) *WorkerInfo {
+	var out *WorkerInfo
+	for i := range workers {
+		if out == nil || workers[i].StartedAt.After(out.StartedAt) {
+			out = &workers[i]
+		}
+	}
+	return out
+}
+
+// GVLBound reports whether this process's threads are queueing on each other
+// rather than doing work.
+//
+// The threshold is deliberately low. Once a third of service time is spent
+// waiting for the GVL, an added thread does not get a third of a core -- it
+// joins the same queue and lengthens it for everyone, so throughput is flat
+// while latency rises.
+const GVLBoundThreshold = 0.3
+
+// ThreadsWouldHelp answers the controller's hardest question: given more
+// capacity, should it go to threads in an existing process or to a new process?
+//
+// The honest answer needs GVL measurement, and this is exactly where the cheap
+// heuristic is not merely imprecise but backwards. io_fraction = 1 - cpu/service
+// treats "not on CPU" as "blocked on IO", but a thread queued behind the GVL is
+// also not on CPU. Measured on this machine, eight threads each burning 50ms of
+// pure CPU reported:
+//
+//	service 125.3ms = cpu 50.0ms + gvl_wait 75.3ms
+//	io_fraction  = 0.60  -> "IO-bound, add threads"   (wrong)
+//	gvl_fraction = 0.60  -> "GVL-bound, fork"         (right)
+//
+// Adding threads there deepens the queue and raises latency. So when the
+// measurement is available it decides; without it we fall back to io_fraction,
+// which is better than nothing but must be paired with gain probing to catch
+// exactly this case after the fact.
+func ThreadsWouldHelp(in Inputs) (bool, string) {
+	if in.GVLKnown {
+		if in.GVLFraction >= GVLBoundThreshold {
+			return false, fmt.Sprintf(
+				"%.0f%% of service time is spent waiting for the GVL; "+
+					"more threads would join that queue, not shorten it",
+				in.GVLFraction*100)
+		}
+		return true, fmt.Sprintf(
+			"only %.0f%% of service time is GVL wait, so threads still have room",
+			in.GVLFraction*100)
+	}
+
+	// No measurement. io_fraction is the fallback, and it is unreliable in
+	// precisely the GVL-bound case -- pair it with gain probing.
+	io := 1 - in.CPUFraction
+	if io < 0.5 {
+		return false, fmt.Sprintf(
+			"no GVL measurement; io_fraction %.2f suggests CPU-bound", io)
+	}
+	return true, fmt.Sprintf(
+		"no GVL measurement; io_fraction %.2f suggests IO-bound (unverified)", io)
+}
+
+// ShouldShed reports whether an arriving request should be rejected rather than
+// queued.
+//
+// Keyed off the predictor, not the measured percentile, because a measured
+// percentile only knows about requests that already suffered. Shedding is also
+// intentionally independent of the scaler: scaling can be frozen by a cooldown,
+// a memory limit or the worker ceiling, and when it is, shedding is the only
+// thing standing between a saturated server and unbounded latency. Scaling may
+// pause; shedding may not.
+func ShouldShed(cfg Config, predictedWait time.Duration) bool {
+	return predictedWait > cfg.ShedLimit()
+}