puma-plus-core 0.1.0

data/go/internal/metrics/hist_test.go

@@ -0,0 +1,353 @@
+package metrics
+
+import (
+	"fmt"
+	"math/rand"
+	"sync"
+	"testing"
+	"time"
+)
+
+// TestBucketBoundsAreContiguous is the invariant that makes the histogram
+// correct: every bucket's upper bound must be the next bucket's lower bound,
+// with no gap and no overlap. The linear and log regions meet at subCount*2, and
+// an off-by-one there would silently misbin every small value.
+func TestBucketBoundsAreContiguous(t *testing.T) {
+	_, prevHi := bucketBounds(0)
+	if lo, _ := bucketBounds(0); lo != 0 {
+		t.Fatalf("bucket 0 starts at %d, want 0", lo)
+	}
+	for i := 1; i < bucketCount; i++ {
+		lo, hi := bucketBounds(i)
+		if lo != prevHi {
+			t.Fatalf("gap at bucket %d: previous hi=%d, this lo=%d", i, prevHi, lo)
+		}
+		if hi <= lo {
+			t.Fatalf("bucket %d is empty or inverted: [%d,%d)", i, lo, hi)
+		}
+		prevHi = hi
+	}
+}
+
+// TestBucketIndexMatchesBounds checks that a value lands in the bucket whose
+// range contains it -- the other half of the binning contract.
+func TestBucketIndexMatchesBounds(t *testing.T) {
+	check := func(v uint64) {
+		t.Helper()
+		i := bucketIndex(v)
+		if i < 0 || i >= bucketCount {
+			t.Fatalf("value %d produced out-of-range index %d", v, i)
+		}
+		lo, hi := bucketBounds(i)
+		if v < lo || v >= hi {
+			t.Fatalf("value %d landed in bucket %d = [%d,%d)", v, i, lo, hi)
+		}
+	}
+
+	// Exhaustive across the linear region and the first few octaves, where the
+	// two mappings meet.
+	for v := uint64(0); v < 1024; v++ {
+		check(v)
+	}
+	// Powers of two and their neighbours, where an off-by-one would hide.
+	for shift := 0; shift < 63; shift++ {
+		p := uint64(1) << shift
+		check(p)
+		if p > 0 {
+			check(p - 1)
+		}
+		check(p + 1)
+	}
+	// Realistic durations, in nanoseconds.
+	for _, d := range []time.Duration{
+		time.Microsecond, 100 * time.Microsecond, time.Millisecond,
+		25 * time.Millisecond, time.Second, time.Minute,
+	} {
+		check(uint64(d.Nanoseconds()))
+	}
+
+	r := rand.New(rand.NewSource(1))
+	for i := 0; i < 200_000; i++ {
+		check(r.Uint64())
+	}
+}
+
+// TestQuantileRelativeError pins the accuracy claim: reporting bucket midpoints
+// bounds relative error at 1/(2*subCount) = 12.5%. If that ever regresses, the
+// control loop's thresholds are built on sand.
+func TestQuantileRelativeError(t *testing.T) {
+	const tolerance = 0.125
+
+	for _, base := range []uint64{
+		1_000, 100_000, 1_000_000, 25_000_000, 1_000_000_000, 60_000_000_000,
+	} {
+		h := &Histogram{}
+		// All observations identical, so every quantile must return that value
+		// within the bucket's relative error.
+		for i := 0; i < 1000; i++ {
+			h.Record(base)
+		}
+		for _, q := range []float64{0.5, 0.95, 0.99} {
+			got := h.Quantile(q)
+			relErr := float64(int64(got)-int64(base)) / float64(base)
+			if relErr < 0 {
+				relErr = -relErr
+			}
+			if relErr > tolerance {
+				t.Errorf("value %d, q=%.2f: got %d, relative error %.4f > %.4f",
+					base, q, got, relErr, tolerance)
+			}
+		}
+	}
+}
+
+// TestQuantileDistribution checks quantiles against a known distribution.
+func TestQuantileDistribution(t *testing.T) {
+	h := &Histogram{}
+	// 1..10000 ns, uniform. p50 ~ 5000, p95 ~ 9500, p99 ~ 9900.
+	for v := uint64(1); v <= 10_000; v++ {
+		h.Record(v)
+	}
+
+	for _, tc := range []struct {
+		q    float64
+		want uint64
+	}{
+		{0.50, 5000},
+		{0.95, 9500},
+		{0.99, 9900},
+	} {
+		got := h.Quantile(tc.q)
+		relErr := float64(int64(got)-int64(tc.want)) / float64(tc.want)
+		if relErr < 0 {
+			relErr = -relErr
+		}
+		if relErr > 0.125 {
+			t.Errorf("q=%.2f: got %d, want ~%d (relative error %.4f)", tc.q, got, tc.want, relErr)
+		}
+	}
+
+	// Count is exact; Max is not. Max reports the midpoint of the highest
+	// non-empty bucket, so it carries the same 12.5% relative error as the
+	// percentiles rather than pretending to know the exact tail.
+	if h.Count() != 10_000 {
+		t.Errorf("Count = %d, want 10000", h.Count())
+	}
+	maxRelErr := float64(10_000-h.Max()) / 10_000
+	if maxRelErr < 0 {
+		maxRelErr = -maxRelErr
+	}
+	if maxRelErr > 0.125 {
+		t.Errorf("Max = %d, want ~10000 (relative error %.4f)", h.Max(), maxRelErr)
+	}
+}
+
+func TestQuantileEmpty(t *testing.T) {
+	h := &Histogram{}
+	if got := h.Quantile(0.95); got != 0 {
+		t.Errorf("empty histogram q95 = %d, want 0", got)
+	}
+	if h.Mean() != 0 {
+		t.Errorf("empty histogram mean = %v, want 0", h.Mean())
+	}
+}
+
+// TestConcurrentRecord runs under -race to prove the hot path is safe without
+// locking, which is the whole justification for the atomic-array design.
+func TestConcurrentRecord(t *testing.T) {
+	h := &Histogram{}
+	const goroutines, each = 16, 5000
+
+	var wg sync.WaitGroup
+	for g := 0; g < goroutines; g++ {
+		wg.Add(1)
+		go func(seed int64) {
+			defer wg.Done()
+			r := rand.New(rand.NewSource(seed))
+			for i := 0; i < each; i++ {
+				h.Record(uint64(r.Intn(1_000_000)))
+			}
+		}(int64(g))
+	}
+	wg.Wait()
+
+	if got := h.Count(); got != goroutines*each {
+		t.Errorf("Count = %d, want %d", got, goroutines*each)
+	}
+}
+
+// TestBucketsAreCumulative checks the Prometheus export contract: counts must be
+// monotonically non-decreasing and end at the total.
+func TestBucketsAreCumulative(t *testing.T) {
+	h := &Histogram{}
+	for _, v := range []uint64{5, 50, 500, 5000, 50000} {
+		h.Record(v)
+	}
+
+	buckets := h.Buckets()
+	if len(buckets) == 0 {
+		t.Fatal("no buckets exported")
+	}
+	var prev uint64
+	for i, b := range buckets {
+		if b.CumulativeCount < prev {
+			t.Fatalf("bucket %d count %d decreased from %d", i, b.CumulativeCount, prev)
+		}
+		prev = b.CumulativeCount
+	}
+	if prev != h.Count() {
+		t.Errorf("final cumulative count %d != total %d", prev, h.Count())
+	}
+}
+
+func TestRollingMerge(t *testing.T) {
+	r := NewRolling(WindowDepth)
+
+	// One observation per window, distinguishable by value.
+	for i := 0; i < WindowDepth; i++ {
+		r.Record(uint64((i + 1) * 1_000_000))
+		r.Rotate()
+	}
+
+	var merged Histogram
+	r.Merge(3, &merged)
+	// After WindowDepth rotations the current slot is empty, so merging 3
+	// windows picks up the current (empty) plus the two most recent writes.
+	if got := merged.Count(); got != 2 {
+		t.Errorf("merged 3 windows: count = %d, want 2", got)
+	}
+
+	r.Merge(WindowDepth, &merged)
+	if got := merged.Count(); got != WindowDepth-1 {
+		t.Errorf("merged all windows: count = %d, want %d", got, WindowDepth-1)
+	}
+
+	// The lifetime histogram is never windowed away.
+	if got := r.Lifetime().Count(); got != uint64(WindowDepth) {
+		t.Errorf("lifetime count = %d, want %d", got, WindowDepth)
+	}
+}
+
+// TestRollingEvictsOldData is the point of a sliding window: data must leave.
+func TestRollingEvictsOldData(t *testing.T) {
+	r := NewRolling(3)
+	r.Record(42)
+	for i := 0; i < 3; i++ {
+		r.Rotate()
+	}
+	var merged Histogram
+	r.Merge(3, &merged)
+	if got := merged.Count(); got != 0 {
+		t.Errorf("after a full lap the window still holds %d observations", got)
+	}
+}
+
+func TestEWMAConverges(t *testing.T) {
+	e := NewEWMA(time.Second)
+	now := time.Unix(0, 0)
+
+	e.Update(100, now)
+	if got := e.Value(); got != 100 {
+		t.Fatalf("first sample should seed the average: got %v", got)
+	}
+
+	// Feeding a constant must converge to that constant.
+	for i := 0; i < 100; i++ {
+		now = now.Add(100 * time.Millisecond)
+		e.Update(50, now)
+	}
+	if got := e.Value(); got < 49.9 || got > 50.1 {
+		t.Errorf("after converging on 50, got %v", got)
+	}
+}
+
+// TestEWMATauOrdering checks the property the control loop depends on: a fast
+// EWMA must track a step change sooner than a slow one. That separation is what
+// lets shedding react in one round trip while scaling ignores a brief burst.
+func TestEWMATauOrdering(t *testing.T) {
+	fast := NewEWMA(1 * time.Second)
+	slow := NewEWMA(15 * time.Second)
+	now := time.Unix(0, 0)
+
+	fast.Update(0, now)
+	slow.Update(0, now)
+
+	for i := 0; i < 20; i++ {
+		now = now.Add(100 * time.Millisecond)
+		fast.Update(100, now)
+		slow.Update(100, now)
+	}
+
+	if fast.Value() <= slow.Value() {
+		t.Errorf("fast EWMA (%v) should lead slow (%v) after a step up",
+			fast.Value(), slow.Value())
+	}
+	if fast.Value() < 50 {
+		t.Errorf("fast EWMA should be well up the step after 2s, got %v", fast.Value())
+	}
+	if slow.Value() > 50 {
+		t.Errorf("slow EWMA should still be lagging after 2s, got %v", slow.Value())
+	}
+}
+
+// TestEWMAHandlesNonAdvancingClock guards the divide-by-zero path.
+func TestEWMAHandlesNonAdvancingClock(t *testing.T) {
+	e := NewEWMA(time.Second)
+	now := time.Unix(0, 0)
+	e.Update(10, now)
+	e.Update(20, now)
+	e.Update(30, now.Add(-time.Second)) // clock went backwards
+	if v := e.Value(); v <= 0 {
+		t.Errorf("EWMA collapsed to %v on a non-advancing clock", v)
+	}
+}
+
+func BenchmarkHistogramRecord(b *testing.B) {
+	h := &Histogram{}
+	b.ReportAllocs()
+	b.RunParallel(func(pb *testing.PB) {
+		var v uint64 = 1
+		for pb.Next() {
+			h.Record(v)
+			v = v*6364136223846793005 + 1442695040888963407
+		}
+	})
+}
+
+// ExampleHistogram_Quantile shows the API and, deliberately, its approximation:
+// the true p50 of 1..1000ms is 500ms and the histogram reports 503ms. That 0.6%
+// error is the price of a lock-free fixed-array histogram, and it is two orders
+// of magnitude below anything the control loop would act on differently.
+func ExampleHistogram_Quantile() {
+	h := &Histogram{}
+	for i := 1; i <= 1000; i++ {
+		h.Record(uint64(i) * 1_000_000) // 1ms..1000ms
+	}
+	fmt.Printf("p50 ~ %dms (true value 500ms)\n", h.Quantile(0.5)/1_000_000)
+	// Output: p50 ~ 503ms (true value 500ms)
+}
+
+func BenchmarkHistogramRecordSerial(b *testing.B) {
+	h := &Histogram{}
+	b.ReportAllocs()
+	var v uint64 = 1
+	for i := 0; i < b.N; i++ {
+		h.Record(v)
+		v = v*6364136223846793005 + 1442695040888963407
+	}
+}
+
+// BenchmarkHistogramRecordSteadyMax isolates the cost of the running-max CAS
+// loop, which is contended only while the max is climbing. Random values keep it
+// climbing constantly; a fixed value models steady state.
+func BenchmarkHistogramRecordSteadyMax(b *testing.B) {
+	h := &Histogram{}
+	h.Record(^uint64(0)) // seed the max so the CAS loop never spins
+	b.ResetTimer()
+	b.ReportAllocs()
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			h.Record(25_000_000)
+		}
+	})
+}

data/go/internal/metrics/window.go

@@ -0,0 +1,120 @@
+package metrics
+
+import (
+	"math"
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// Rolling is a ring of tumbling histogram snapshots that together give sliding
+// window semantics.
+//
+// A true per-observation sliding window would need to remember and retract
+// individual observations; tumbling buckets plus a merge give the same answer
+// for percentile purposes at a fraction of the cost, with bounded memory and no
+// per-record bookkeeping.
+type Rolling struct {
+	mu   sync.Mutex
+	ring []*Histogram
+	cur  atomic.Int32 // index into ring, read on the hot path
+	// lifetime accumulates every observation ever, for the /metrics endpoint
+	// where Prometheus expects monotonic counters rather than windows.
+	lifetime Histogram
+}
+
+// NewRolling creates a ring of depth snapshots.
+func NewRolling(depth int) *Rolling {
+	r := &Rolling{ring: make([]*Histogram, depth)}
+	for i := range r.ring {
+		r.ring[i] = &Histogram{}
+	}
+	return r
+}
+
+// Record adds an observation to the current snapshot and to the lifetime total.
+func (r *Rolling) Record(v uint64) {
+	r.ring[r.cur.Load()].Record(v)
+	r.lifetime.Record(v)
+}
+
+// Rotate advances to the next snapshot, clearing it first. Called once per tick.
+func (r *Rolling) Rotate() {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	next := (int(r.cur.Load()) + 1) % len(r.ring)
+	// Clear before publishing, so no in-flight Record lands in a slot that is
+	// about to be zeroed. A record racing the swap may still be counted in the
+	// outgoing slot; for a metrics window that is benign.
+	r.ring[next].Reset()
+	r.cur.Store(int32(next))
+}
+
+// Merge sums the most recent k snapshots (including the current, partial one)
+// into dst, which the caller supplies to avoid allocating per scrape.
+func (r *Rolling) Merge(k int, dst *Histogram) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	if k > len(r.ring) {
+		k = len(r.ring)
+	}
+	dst.Reset()
+	idx := int(r.cur.Load())
+	for i := 0; i < k; i++ {
+		r.ring[((idx-i)%len(r.ring)+len(r.ring))%len(r.ring)].AddTo(dst)
+	}
+}
+
+// Lifetime returns the all-time histogram.
+func (r *Rolling) Lifetime() *Histogram { return &r.lifetime }
+
+// EWMA is an exponentially weighted moving average with a wall-clock time
+// constant, so its behavior does not change if the tick rate does.
+//
+// Two instances with different taus are the primary anti-oscillation device in
+// the control loop: a fast one (tau ~1s) drives shedding, which must react
+// inside one round trip of pain, and a slow one (tau ~15s) drives scaling, which
+// must not fork a process in response to a two-second burst that shedding
+// already absorbed.
+type EWMA struct {
+	mu    sync.Mutex
+	tau   time.Duration
+	value float64
+	last  time.Time
+	init  bool
+}
+
+// NewEWMA creates an EWMA with time constant tau.
+func NewEWMA(tau time.Duration) *EWMA {
+	return &EWMA{tau: tau}
+}
+
+// Update folds in a new sample observed at time now.
+func (e *EWMA) Update(sample float64, now time.Time) {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+
+	if !e.init {
+		e.value, e.last, e.init = sample, now, true
+		return
+	}
+	dt := now.Sub(e.last)
+	if dt <= 0 {
+		// Clock did not advance; fold in without decay rather than dividing by
+		// zero or letting an out-of-order sample rewind the average.
+		e.value = (e.value + sample) / 2
+		return
+	}
+	alpha := 1 - math.Exp(-float64(dt)/float64(e.tau))
+	e.value += alpha * (sample - e.value)
+	e.last = now
+}
+
+// Value returns the current average, or 0 before the first sample.
+func (e *EWMA) Value() float64 {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	return e.value
+}

data/go/internal/queue/drain_test.go

@@ -0,0 +1,141 @@
+package queue
+
+import (
+	"context"
+	"net"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/evanphx/puma-plus/internal/wire"
+	"golang.org/x/sys/unix"
+)
+
+// socketPair returns two connected unix sockets.
+//
+// Real sockets rather than net.Pipe: net.Pipe is unbuffered and synchronous, so
+// writing a GOAWAY to it would block forever unless the test happened to be
+// reading at that exact moment -- which would make the test pass or fail on
+// scheduling rather than on behaviour.
+func socketPair(t *testing.T) (net.Conn, net.Conn) {
+	t.Helper()
+	fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var out [2]net.Conn
+	for i, fd := range fds {
+		f := os.NewFile(uintptr(fd), "socketpair")
+		c, err := net.FileConn(f)
+		f.Close()
+		if err != nil {
+			t.Fatal(err)
+		}
+		out[i] = c
+	}
+	t.Cleanup(func() { out[0].Close(); out[1].Close() })
+	return out[0], out[1]
+}
+
+// addIdle registers n connections and returns the worker-side ends, so a test
+// can assert on what the worker actually received.
+func addIdle(t *testing.T, r *Registry, n int) []net.Conn {
+	t.Helper()
+	peers := make([]net.Conn, 0, n)
+	for i := 0; i < n; i++ {
+		goSide, rubySide := socketPair(t)
+		r.Add(NewConn(goSide, 4242, 0, i))
+		peers = append(peers, rubySide)
+	}
+	return peers
+}
+
+// TestDrainIdleSendsGoawayAndRetires is the scale-down actuator for Ractor
+// mode. A drained connection must be told to go away and must stop counting as
+// capacity; getting either half wrong leaves the controller scaling against a
+// number that does not match reality.
+func TestDrainIdleSendsGoaway(t *testing.T) {
+	r := NewRegistry(16)
+	peers := addIdle(t, r, 4)
+
+	if got := r.Capacity(); got != 4 {
+		t.Fatalf("capacity before drain = %d, want 4", got)
+	}
+
+	if got := r.DrainIdle(3); got != 3 {
+		t.Fatalf("DrainIdle(3) = %d, want 3", got)
+	}
+	if got := r.Capacity(); got != 1 {
+		t.Errorf("capacity after draining 3 = %d, want 1", got)
+	}
+	if got := r.Idle(); got != 1 {
+		t.Errorf("idle after draining 3 = %d, want 1", got)
+	}
+
+	// Each drained peer must actually have received a GOAWAY, not merely been
+	// dropped from the registry. A worker that is never told will sit blocked on
+	// read holding a Ractor open forever.
+	drained := 0
+	for _, p := range peers {
+		p.SetReadDeadline(time.Now().Add(200 * time.Millisecond))
+		typ, _, err := wire.NewReader(p).ReadFrame()
+		if err != nil {
+			continue // the one survivor: nothing written, read times out
+		}
+		if typ != wire.TypeGoaway {
+			t.Errorf("drained peer got frame %#x, want GOAWAY (%#x)", typ, wire.TypeGoaway)
+		}
+		drained++
+	}
+	if drained != 3 {
+		t.Errorf("%d peers received GOAWAY, want 3", drained)
+	}
+}
+
+// TestDrainIdleNeverBlocks: a drain asks for more than are idle. The surplus is
+// busy serving requests, and blocking to wait for it would stall the control
+// loop behind application work -- so the call must return what it got.
+func TestDrainIdleReturnsShortWithoutBlocking(t *testing.T) {
+	r := NewRegistry(16)
+	addIdle(t, r, 2)
+
+	done := make(chan int, 1)
+	go func() { done <- r.DrainIdle(5) }()
+
+	select {
+	case got := <-done:
+		if got != 2 {
+			t.Errorf("DrainIdle(5) with 2 idle = %d, want 2", got)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("DrainIdle blocked waiting for connections that were not idle")
+	}
+}
+
+// TestDrainIdleLeavesBusyConnsAlone is the safety property. GOAWAY is only safe
+// because it goes to connections that are between requests; draining one that
+// is mid-request would cut off a response in flight.
+func TestDrainIdleLeavesBusyConnsAlone(t *testing.T) {
+	r := NewRegistry(16)
+	addIdle(t, r, 2)
+
+	// Take one out of the idle set, as a dispatched request would.
+	busy, _, err := r.Acquire(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if got := r.DrainIdle(2); got != 1 {
+		t.Fatalf("DrainIdle(2) with 1 idle and 1 busy = %d, want 1", got)
+	}
+	if busy.IsClosed() {
+		t.Error("the busy connection was closed by a drain; a response in flight would be cut off")
+	}
+
+	// Once released it becomes drainable, so capacity really does come down --
+	// the controller just has to wait a tick.
+	r.Release(busy, true)
+	if got := r.DrainIdle(1); got != 1 {
+		t.Errorf("DrainIdle(1) after release = %d, want 1", got)
+	}
+}