puma-plus-core 0.1.0

data/go/internal/control/loop.go

@@ -0,0 +1,343 @@
+package control
+
+import (
+	"encoding/json"
+	"errors"
+	"io"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Shepherd is what the loop uses to actuate. Implemented by the real control
+// connection; a fake in tests.
+type Shepherd interface {
+	// SetWorkers asks for n units of capacity.
+	SetWorkers(n int) error
+	// WorkerCount reports how many units currently exist.
+	//
+	// Separate from len(WorkerPIDs) because the two stopped being the same thing
+	// once a unit could be a Ractor: Ractor mode has N units in ONE process, so
+	// counting pids would peg the controller's model at 1 forever.
+	WorkerCount() int
+	// WorkerPIDs reports the process ids to read RSS from, for the memory guard.
+	// One entry per process, which is not one entry per unit under Ractors.
+	WorkerPIDs() []int
+	// Connected reports whether a shepherd is attached at all.
+	Connected() bool
+}
+
+// QueueSource supplies live queue state.
+type QueueSource interface {
+	Depth() int
+	Busy() int
+	Capacity() int
+}
+
+// MetricSource supplies the measured signals.
+type MetricSource interface {
+	// QueueP95 over the scaling window.
+	QueueP95() time.Duration
+	// PredictedWait for the next arrival.
+	PredictedWait(depth, capacity int) time.Duration
+	ArrivalRate() float64
+	ServiceMean() float64
+	// GVLFraction is the share of service time spent queued behind the GVL, and
+	// whether that is a real measurement rather than an absent extension.
+	GVLFraction() (float64, bool)
+	CPUFraction() float64
+}
+
+// Loop runs the controller on a ticker and applies its decisions.
+//
+// Everything time-dependent, IO-bound and effectful lives here; the decision
+// itself is Decide, which is pure. That boundary is what makes the decision log
+// replayable.
+type Loop struct {
+	cfg   Config
+	sh    Shepherd
+	queue QueueSource
+	mets  MetricSource
+
+	mu      sync.Mutex
+	st      State
+	workers map[int]*WorkerInfo // by worker id
+	nextID  int
+
+	journal *Journal
+	logger  *log.Logger
+}
+
+// NewLoop builds a controller loop.
+func NewLoop(cfg Config, sh Shepherd, q QueueSource, m MetricSource,
+	journal *Journal, logger *log.Logger) *Loop {
+	if logger == nil {
+		logger = log.Default()
+	}
+	return &Loop{
+		cfg: cfg, sh: sh, queue: q, mets: m,
+		workers: map[int]*WorkerInfo{}, journal: journal, logger: logger,
+	}
+}
+
+// Run ticks until stop is closed.
+func (l *Loop) Run(tick time.Duration, stop <-chan struct{}) {
+	t := time.NewTicker(tick)
+	defer t.Stop()
+	for {
+		select {
+		case now := <-t.C:
+			l.Tick(now)
+		case <-stop:
+			return
+		}
+	}
+}
+
+// Tick gathers inputs, decides, and actuates. Exported so tests can drive it
+// with a controlled clock.
+func (l *Loop) Tick(now time.Time) []Action {
+	l.mu.Lock()
+	defer l.mu.Unlock()
+
+	if !l.sh.Connected() {
+		return nil // nothing to actuate; the shepherd owns processes
+	}
+
+	l.refreshWorkers(now)
+
+	depth, capacity := l.queue.Depth(), l.queue.Capacity()
+	in := Inputs{
+		Now:           now,
+		QueueP95:      l.mets.QueueP95(),
+		PredictedWait: l.mets.PredictedWait(depth, capacity),
+		Depth:         depth,
+		Capacity:      capacity,
+		Busy:          l.queue.Busy(),
+		ArrivalRate:   l.mets.ArrivalRate(),
+		ServiceMean:   l.mets.ServiceMean(),
+		CPUFraction:   l.mets.CPUFraction(),
+		Workers:       l.workerSlice(),
+	}
+	in.GVLFraction, in.GVLKnown = l.mets.GVLFraction()
+
+	actions, next := Decide(l.cfg, in, l.st)
+	l.st = next
+
+	for _, a := range actions {
+		l.apply(a, now)
+	}
+
+	if l.journal != nil {
+		l.journal.Write(Entry{
+			At: now, Tick: l.st.Tick, Inputs: in, State: l.st, Actions: actions,
+		})
+	}
+	for _, a := range actions {
+		if a.Kind != ActionNone {
+			l.logger.Printf("control: %s (%s)", a.Kind, a.Reason)
+		}
+	}
+	return actions
+}
+
+func (l *Loop) apply(a Action, now time.Time) {
+	switch a.Kind {
+	case ActionSpawnWorker:
+		target := len(l.workers) + 1
+		if err := l.sh.SetWorkers(target); err != nil {
+			l.logger.Printf("control: spawn to %d failed: %v", target, err)
+			return
+		}
+		id := l.nextID
+		l.nextID++
+		l.workers[id] = &WorkerInfo{ID: id, StartedAt: now, Booted: false}
+	case ActionRetireWorker:
+		target := len(l.workers) - 1
+		if target < l.cfg.MinWorkers {
+			target = l.cfg.MinWorkers
+		}
+		if err := l.sh.SetWorkers(target); err != nil {
+			l.logger.Printf("control: retire to %d failed: %v", target, err)
+			return
+		}
+		delete(l.workers, a.WorkerID)
+	}
+}
+
+// refreshWorkers reconciles the controller's model with reality: which workers
+// the shepherd actually has, and how much memory each is using.
+//
+// RSS is read from /proc by us, never taken from the worker's own heartbeat. A
+// process reporting its own memory is exactly the process we may have to kill
+// for using too much, and trusting it would let a runaway worker talk its way
+// out of the memory guard.
+func (l *Loop) refreshWorkers(now time.Time) {
+	pids := l.sh.WorkerPIDs()
+	count := l.sh.WorkerCount()
+
+	// Reconcile count. The shepherd is authoritative about how many exist.
+	for len(l.workers) < count {
+		id := l.nextID
+		l.nextID++
+		l.workers[id] = &WorkerInfo{ID: id, StartedAt: now, Booted: true}
+	}
+	for len(l.workers) > count {
+		// Drop the newest surplus entry, matching the retire policy.
+		var newest int = -1
+		for id, w := range l.workers {
+			if newest == -1 || w.StartedAt.After(l.workers[newest].StartedAt) {
+				newest = id
+			}
+		}
+		if newest == -1 {
+			break
+		}
+		delete(l.workers, newest)
+	}
+
+	// Mark booted from the connections that have actually appeared, not from a
+	// timer.
+	//
+	// A worker dials in only once its Rack app is loaded and callable, so a
+	// connection showing up in the registry *is* the readiness signal -- that is
+	// the whole reason this server needs none of puma-dev's dial-polling. The
+	// controller was nonetheless approximating it as "old enough that a fork
+	// would have finished" (2s), which is wrong in both directions: it blocks
+	// scaling for 2s after a worker is already serving, and it declares a worker
+	// ready that may still be booting a slow application.
+	//
+	// The cost was measured. Scaling from 5 to 30 slots, 7 of the 11 seconds
+	// were spent vetoing spawns with "worker N is still booting" -- against
+	// Ractors that boot in about four milliseconds.
+	//
+	// Which specific workers are still coming up cannot be known (the model's
+	// ids are ours, not the shepherd's), but the count can: any shortfall
+	// between the capacity we expect and the capacity we see is capacity still
+	// on its way, and it belongs to the newest workers, since those are the ones
+	// just spawned.
+	perWorker := l.cfg.ThreadsPerWorker
+	if perWorker < 1 {
+		perWorker = 1
+	}
+	missing := len(l.workers)*perWorker - l.queue.Capacity()
+	unbooted := 0
+	if missing > 0 {
+		unbooted = (missing + perWorker - 1) / perWorker
+	}
+
+	sorted := l.sortedIDs()
+	for i, id := range sorted {
+		w := l.workers[id]
+		if i < len(pids) {
+			w.RSSBytes = readRSS(pids[i])
+		}
+
+		// Ids increase monotonically, so the tail of this slice is the newest.
+		booted := i < len(sorted)-unbooted
+
+		// A worker that never dials in must not block scaling forever. Past the
+		// deadline stop counting it as pending: either it is dead, and the
+		// shepherd will drop it, or the app is slow enough that waiting longer
+		// helps nobody who is currently queued.
+		if !booted && now.Sub(w.StartedAt) > l.cfg.BootDeadline {
+			booted = true
+		}
+		w.Booted = booted
+		w.LastSeen = now
+	}
+}
+
+func (l *Loop) sortedIDs() []int {
+	ids := make([]int, 0, len(l.workers))
+	for id := range l.workers {
+		ids = append(ids, id)
+	}
+	for i := 1; i < len(ids); i++ {
+		for j := i; j > 0 && ids[j] < ids[j-1]; j-- {
+			ids[j], ids[j-1] = ids[j-1], ids[j]
+		}
+	}
+	return ids
+}
+
+func (l *Loop) workerSlice() []WorkerInfo {
+	out := make([]WorkerInfo, 0, len(l.workers))
+	for _, id := range l.sortedIDs() {
+		out = append(out, *l.workers[id])
+	}
+	return out
+}
+
+// readRSS reads resident set size from /proc/<pid>/statm.
+func readRSS(pid int) int64 {
+	data, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/statm")
+	if err != nil {
+		return 0
+	}
+	fields := strings.Fields(string(data))
+	if len(fields) < 2 {
+		return 0
+	}
+	pages, err := strconv.ParseInt(fields[1], 10, 64)
+	if err != nil {
+		return 0
+	}
+	return pages * int64(os.Getpagesize())
+}
+
+// Entry is one line of the decision log.
+//
+// It records the complete Inputs, not a summary. When the controller does
+// something surprising, this file is the entire debugging story -- and because
+// Decide is pure, the recorded inputs can be replayed through it to reproduce
+// the decision exactly.
+type Entry struct {
+	At      time.Time `json:"at"`
+	Tick    int64     `json:"tick"`
+	Inputs  Inputs    `json:"inputs"`
+	State   State     `json:"state"`
+	Actions []Action  `json:"actions"`
+}
+
+// Journal appends decision-log entries as JSON lines.
+type Journal struct {
+	mu  sync.Mutex
+	w   io.WriteCloser
+	enc *json.Encoder
+}
+
+// NewJournal opens a decision log at path.
+func NewJournal(path string) (*Journal, error) {
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
+	if err != nil {
+		return nil, err
+	}
+	return &Journal{w: f, enc: json.NewEncoder(f)}, nil
+}
+
+// Write appends one entry.
+func (j *Journal) Write(e Entry) {
+	if j == nil {
+		return
+	}
+	j.mu.Lock()
+	defer j.mu.Unlock()
+	_ = j.enc.Encode(e)
+}
+
+// Close flushes and closes the log.
+func (j *Journal) Close() error {
+	if j == nil {
+		return nil
+	}
+	j.mu.Lock()
+	defer j.mu.Unlock()
+	return j.w.Close()
+}
+
+// ErrShepherdFail is returned by a Shepherd that cannot accept commands.
+var ErrShepherdFail = errors.New("control: shepherd unavailable")

data/go/internal/control/loop_test.go

@@ -0,0 +1,228 @@
+package control
+
+import (
+	"bytes"
+	"log"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// fakeShepherd records SetWorkers calls and pretends the requested processes
+// exist.
+type fakeShepherd struct {
+	workers int
+	calls   []int
+	fail    bool
+}
+
+func (f *fakeShepherd) SetWorkers(n int) error {
+	if f.fail {
+		return ErrShepherdFail
+	}
+	f.calls = append(f.calls, n)
+	f.workers = n
+	return nil
+}
+
+func (f *fakeShepherd) WorkerPIDs() []int {
+	// PIDs that do not exist, so readRSS returns 0 and the memory guard is
+	// inert unless a test sets it up deliberately.
+	out := make([]int, f.workers)
+	for i := range out {
+		out[i] = 1 << 30
+	}
+	return out
+}
+
+// WorkerCount matches WorkerPIDs here, since a forking shepherd has one process
+// per unit. The two only diverge under Ractors, where N units share one process.
+func (f *fakeShepherd) WorkerCount() int { return f.workers }
+
+func (f *fakeShepherd) Connected() bool { return true }
+
+type fakeQueue struct{ depth, busy, capacity int }
+
+func (q *fakeQueue) Depth() int    { return q.depth }
+func (q *fakeQueue) Busy() int     { return q.busy }
+func (q *fakeQueue) Capacity() int { return q.capacity }
+
+type fakeMetrics struct {
+	p95      time.Duration
+	pred     time.Duration
+	arrival  float64
+	service  float64
+	gvl      float64
+	gvlKnown bool
+	cpu      float64
+}
+
+func (m *fakeMetrics) QueueP95() time.Duration              { return m.p95 }
+func (m *fakeMetrics) PredictedWait(_, _ int) time.Duration { return m.pred }
+func (m *fakeMetrics) ArrivalRate() float64                 { return m.arrival }
+func (m *fakeMetrics) ServiceMean() float64                 { return m.service }
+func (m *fakeMetrics) GVLFraction() (float64, bool)         { return m.gvl, m.gvlKnown }
+func (m *fakeMetrics) CPUFraction() float64                 { return m.cpu }
+
+func TestLoopScalesUpAndActuates(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.MinWorkers = 1
+	sh := &fakeShepherd{workers: 1}
+	q := &fakeQueue{depth: 40, capacity: 5}
+	m := &fakeMetrics{
+		p95: cfg.TargetQueueP95 * 4, pred: cfg.TargetQueueP95 * 6,
+		arrival: 200, service: 0.025,
+	}
+
+	loop := NewLoop(cfg, sh, q, m, nil, log.New(&bytes.Buffer{}, "", 0))
+
+	now := time.Unix(1700000000, 0)
+	var spawned bool
+	for i := 0; i < 5; i++ {
+		now = now.Add(time.Second)
+		for _, a := range loop.Tick(now) {
+			if a.Kind == ActionSpawnWorker {
+				spawned = true
+			}
+		}
+	}
+	if !spawned {
+		t.Fatal("loop never scaled up under sustained confirmed breach")
+	}
+	if len(sh.calls) == 0 {
+		t.Fatal("loop decided to spawn but never told the shepherd")
+	}
+	if sh.calls[0] != 2 {
+		t.Errorf("first SetWorkers = %d, want 2", sh.calls[0])
+	}
+}
+
+func TestLoopDoesNothingWithoutShepherd(t *testing.T) {
+	cfg := DefaultConfig()
+	loop := NewLoop(cfg, disconnectedShepherd{}, &fakeQueue{depth: 100, capacity: 1},
+		&fakeMetrics{p95: time.Second, pred: time.Second}, nil, log.New(&bytes.Buffer{}, "", 0))
+
+	now := time.Unix(1700000000, 0)
+	for i := 0; i < 10; i++ {
+		now = now.Add(time.Second)
+		if acts := loop.Tick(now); len(acts) > 0 {
+			t.Fatalf("acted with no shepherd attached: %+v", acts)
+		}
+	}
+}
+
+type disconnectedShepherd struct{}
+
+func (disconnectedShepherd) SetWorkers(int) error { return ErrShepherdFail }
+func (disconnectedShepherd) WorkerPIDs() []int    { return nil }
+func (disconnectedShepherd) WorkerCount() int     { return 0 }
+func (disconnectedShepherd) Connected() bool      { return false }
+
+// TestJournalRoundTripAndReplay is the payoff for keeping Decide pure: a
+// recorded run must replay to the same decisions.
+func TestJournalRoundTripAndReplay(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.MinWorkers = 1
+
+	path := filepath.Join(t.TempDir(), "decisions.jsonl")
+	j, err := NewJournal(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	sh := &fakeShepherd{workers: 1}
+	q := &fakeQueue{depth: 40, capacity: 5}
+	m := &fakeMetrics{arrival: 200, service: 0.025}
+	loop := NewLoop(cfg, sh, q, m, j, log.New(&bytes.Buffer{}, "", 0))
+
+	// A run with a plausible shape: quiet, then a sustained breach, then quiet.
+	now := time.Unix(1700000000, 0)
+	for i := 0; i < 40; i++ {
+		now = now.Add(time.Second)
+		switch {
+		case i < 10:
+			m.p95, m.pred = cfg.TargetQueueP95/10, 0
+		case i < 25:
+			m.p95, m.pred = cfg.TargetQueueP95*4, cfg.TargetQueueP95*6
+		default:
+			m.p95, m.pred = cfg.TargetQueueP95/10, 0
+		}
+		loop.Tick(now)
+	}
+	if err := j.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	entries, err := LoadJournal(path)
+	if err != nil {
+		t.Fatalf("loading the decision log: %v", err)
+	}
+	if len(entries) != 40 {
+		t.Fatalf("journal has %d entries, want 40", len(entries))
+	}
+
+	// Same config replayed over the same inputs must reproduce every decision.
+	if div := Replay(cfg, entries); len(div) > 0 {
+		for _, d := range div {
+			t.Errorf("replay diverged: %s", d)
+		}
+	}
+
+	spawns := 0
+	for _, e := range entries {
+		for _, a := range e.Actions {
+			if a.Kind == ActionSpawnWorker {
+				spawns++
+			}
+		}
+	}
+	if spawns == 0 {
+		t.Error("the recorded run never scaled; the replay proves nothing")
+	}
+	t.Logf("replayed %d ticks containing %d spawns with no divergence", len(entries), spawns)
+}
+
+// TestReplayDetectsTuningChange: replaying with a different config MUST
+// diverge, or the tool is useless for evaluating a tuning change.
+func TestReplayDetectsTuningChange(t *testing.T) {
+	cfg := DefaultConfig()
+	cfg.MinWorkers = 1
+
+	path := filepath.Join(t.TempDir(), "decisions.jsonl")
+	j, _ := NewJournal(path)
+	sh := &fakeShepherd{workers: 1}
+	q := &fakeQueue{depth: 40, capacity: 5}
+	m := &fakeMetrics{p95: cfg.TargetQueueP95 * 4, pred: 0, arrival: 200, service: 0.025}
+	loop := NewLoop(cfg, sh, q, m, j, log.New(&bytes.Buffer{}, "", 0))
+
+	now := time.Unix(1700000000, 0)
+	for i := 0; i < 20; i++ {
+		now = now.Add(time.Second)
+		loop.Tick(now)
+	}
+	j.Close()
+
+	entries, err := LoadJournal(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// A far higher target means the same load is no longer a breach, so the
+	// recorded spawns should not reproduce.
+	relaxed := cfg
+	relaxed.TargetQueueP95 = time.Second
+	div := Replay(relaxed, entries)
+	if len(div) == 0 {
+		t.Error("replaying with a 40x looser target produced no divergence; " +
+			"the replay tool cannot be used to evaluate tuning changes")
+	} else {
+		t.Logf("relaxing the target from %s to %s changed %d of %d decisions",
+			cfg.TargetQueueP95, relaxed.TargetQueueP95, len(div), len(entries))
+	}
+}
+
+func TestParseJournalRejectsGarbage(t *testing.T) {
+	if _, err := ParseJournal(bytes.NewBufferString("{not json}\n")); err == nil {
+		t.Error("expected an error on malformed journal input")
+	}
+}

data/go/internal/control/replay.go

@@ -0,0 +1,121 @@
+package control
+
+import (
+	"bufio"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+)
+
+// Replay feeds a recorded decision log back through Decide and reports where
+// the outcome differs from what was recorded.
+//
+// This is the payoff for keeping Decide pure. A production incident produces a
+// journal; that journal is a reproducible test case. Tuning changes can be
+// evaluated against real recorded load before being deployed, and a controller
+// regression becomes an ordinary unit test rather than something you have to
+// recreate under load.
+func Replay(cfg Config, entries []Entry) []Divergence {
+	var out []Divergence
+	var st State
+
+	for i, e := range entries {
+		// Start from the recorded state on the first entry so a replay can begin
+		// mid-log; afterwards, carry our own state forward, since divergence is
+		// exactly what we are looking for.
+		if i == 0 {
+			st = e.State
+			st.Tick-- // Decide increments; undo so ticks line up
+			st.BreachTicks = e.State.BreachTicks
+		}
+
+		actions, next := Decide(cfg, e.Inputs, st)
+		st = next
+
+		if !sameActions(actions, e.Actions) {
+			out = append(out, Divergence{
+				Index:    i,
+				Tick:     e.Tick,
+				Recorded: e.Actions,
+				Replayed: actions,
+				Inputs:   e.Inputs,
+			})
+		}
+	}
+	return out
+}
+
+// Divergence is one point where a replay disagreed with the recording.
+type Divergence struct {
+	Index    int
+	Tick     int64
+	Recorded []Action
+	Replayed []Action
+	Inputs   Inputs
+}
+
+func (d Divergence) String() string {
+	return fmt.Sprintf("tick %d: recorded %s, replayed %s (queue p95 %s, depth %d, %d workers)",
+		d.Tick, summarize(d.Recorded), summarize(d.Replayed),
+		d.Inputs.QueueP95, d.Inputs.Depth, len(d.Inputs.Workers))
+}
+
+func summarize(as []Action) string {
+	if len(as) == 0 {
+		return "[]"
+	}
+	s := ""
+	for i, a := range as {
+		if i > 0 {
+			s += ","
+		}
+		s += string(a.Kind)
+	}
+	return "[" + s + "]"
+}
+
+func sameActions(a, b []Action) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		// Compare the decision, not its prose. Reason strings are for humans and
+		// will legitimately change when messages are reworded; the Kind and the
+		// target are the decision.
+		if a[i].Kind != b[i].Kind || a[i].WorkerID != b[i].WorkerID {
+			return false
+		}
+	}
+	return true
+}
+
+// LoadJournal reads a decision log written by Journal.
+func LoadJournal(path string) ([]Entry, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return ParseJournal(f)
+}
+
+// ParseJournal reads JSON-lines entries from r.
+func ParseJournal(r io.Reader) ([]Entry, error) {
+	var out []Entry
+	sc := bufio.NewScanner(r)
+	// Entries carry the full Inputs including every worker, so lines can be long.
+	sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
+	for sc.Scan() {
+		line := sc.Bytes()
+		if len(line) == 0 {
+			continue
+		}
+		var e Entry
+		if err := json.Unmarshal(line, &e); err != nil {
+			return nil, fmt.Errorf("line %d: %w", len(out)+1, err)
+		}
+		out = append(out, e)
+	}
+	return out, sc.Err()
+}