puma-plus-core 0.1.0

data/go/internal/frontend/tlscert.go

@@ -0,0 +1,184 @@
+package frontend
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"fmt"
+	"math/big"
+	"net"
+	"strings"
+	"time"
+)
+
+// DevCertLifetime is how long a generated development certificate stays valid.
+//
+// Long enough that no server outlives its own certificate, which is the failure
+// this replaced -- see DevCertificate for why a long window is safe here.
+const DevCertLifetime = 10 * 365 * 24 * time.Hour
+
+// DevCertificate mints a self-signed certificate for local development.
+//
+// HTTP/3 has no cleartext mode -- QUIC always runs TLS -- so a certificate is
+// not optional the way it is for h1 and h2c. Rather than make every developer
+// produce one before they can try h3, the spike generates one in memory at boot.
+//
+// Deliberately in-memory: nothing is written to disk, so there is no private key
+// lying around to be mistaken for something trustworthy and none of it outlives
+// the process that made it. puma-dev's approach (a real local CA, added to the
+// system trust store, with per-SNI leaf certs cached in an LRU --
+// puma-dev/dev/ssl.go) is what a real product needs; this is the smallest thing
+// that makes h2 and h3 testable.
+//
+// Validity is deliberately long, which reads backwards until you notice what
+// bounds the lifetime here. The key never leaves memory and dies with the
+// process, so it is the process, not NotAfter, that limits exposure. An earlier
+// 24-hour window therefore protected nothing and broke something: the
+// certificate is minted once at boot and never refreshed, so any server running
+// longer than a day served an expired cert and failed every client. Given a
+// choice between expiring and re-minting on a timer, a long window is the one
+// with no moving parts.
+//
+// It is emphatically NOT a CA. A self-signed leaf used directly as a server
+// certificate needs neither IsCA nor KeyUsageCertSign, and setting them is what
+// would turn "harmless throwaway" into "ten-year signing key" for anyone who
+// added it to a trust store.
+func DevCertificate(hosts ...string) (tls.Certificate, error) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		return tls.Certificate{}, fmt.Errorf("generating dev key: %w", err)
+	}
+
+	serialMax := new(big.Int).Lsh(big.NewInt(1), 128)
+	serial, err := rand.Int(rand.Reader, serialMax)
+	if err != nil {
+		return tls.Certificate{}, err
+	}
+
+	tmpl := x509.Certificate{
+		SerialNumber: serial,
+		Subject: pkix.Name{
+			Organization: []string{"puma-plus development"},
+			CommonName:   "puma-plus dev",
+		},
+		// An hour of backdating absorbs clock skew between this machine and
+		// whatever is connecting to it.
+		NotBefore: time.Now().Add(-time.Hour),
+		NotAfter:  time.Now().Add(DevCertLifetime),
+
+		KeyUsage:              x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+		IsCA:                  false,
+	}
+
+	if len(hosts) == 0 {
+		hosts = []string{"localhost"}
+	}
+	for _, h := range hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			tmpl.IPAddresses = append(tmpl.IPAddresses, ip)
+		} else {
+			tmpl.DNSNames = append(tmpl.DNSNames, h)
+		}
+	}
+	// Loopback is always included: every local client reaches us one of these
+	// ways, and a cert that fails on 127.0.0.1 is useless for testing.
+	tmpl.IPAddresses = append(tmpl.IPAddresses, net.IPv4(127, 0, 0, 1), net.IPv6loopback)
+
+	der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key)
+	if err != nil {
+		return tls.Certificate{}, fmt.Errorf("creating dev certificate: %w", err)
+	}
+
+	return tls.Certificate{
+		Certificate: [][]byte{der},
+		PrivateKey:  key,
+		Leaf:        mustParse(der),
+	}, nil
+}
+
+// LoadCertificate reads a PEM certificate chain and its private key.
+//
+// The leaf is parsed and attached, which tls.LoadX509KeyPair does not do. It
+// costs one parse at boot and buys the ability to say which certificate was
+// loaded and when it expires -- the two questions asked whenever TLS is
+// mysteriously failing.
+func LoadCertificate(certPath, keyPath string) (tls.Certificate, error) {
+	cert, err := tls.LoadX509KeyPair(certPath, keyPath)
+	if err != nil {
+		// The wrapped error already names which file failed and why, including
+		// the common "private key does not match public key" mismatch.
+		return tls.Certificate{}, fmt.Errorf("loading certificate %s with key %s: %w",
+			certPath, keyPath, err)
+	}
+
+	if len(cert.Certificate) > 0 {
+		leaf, err := x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return tls.Certificate{}, fmt.Errorf("parsing certificate %s: %w", certPath, err)
+		}
+		cert.Leaf = leaf
+	}
+	return cert, nil
+}
+
+// DescribeCertificate summarises a loaded certificate for the boot log, and
+// reports whether it is currently usable.
+//
+// Expiry is checked and surfaced rather than left to fail at handshake time,
+// where it appears as an opaque client-side error on a machine that is not this
+// one.
+func DescribeCertificate(cert tls.Certificate) (desc string, problem string) {
+	leaf := cert.Leaf
+	if leaf == nil {
+		return "certificate (unparsed)", ""
+	}
+
+	names := append([]string{}, leaf.DNSNames...)
+	for _, ip := range leaf.IPAddresses {
+		names = append(names, ip.String())
+	}
+	if len(names) == 0 {
+		names = []string{leaf.Subject.CommonName}
+	}
+
+	desc = fmt.Sprintf("%s, expires %s", strings.Join(names, ","),
+		leaf.NotAfter.Format("2006-01-02"))
+
+	now := time.Now()
+	switch {
+	case now.After(leaf.NotAfter):
+		problem = fmt.Sprintf("certificate EXPIRED on %s", leaf.NotAfter.Format("2006-01-02"))
+	case now.Before(leaf.NotBefore):
+		problem = fmt.Sprintf("certificate is not valid until %s", leaf.NotBefore.Format("2006-01-02"))
+	case leaf.NotAfter.Sub(now) < 14*24*time.Hour:
+		problem = fmt.Sprintf("certificate expires in %d days",
+			int(leaf.NotAfter.Sub(now).Hours()/24))
+	}
+	return desc, problem
+}
+
+func mustParse(der []byte) *x509.Certificate {
+	c, err := x509.ParseCertificate(der)
+	if err != nil {
+		return nil
+	}
+	return c
+}
+
+// TLSConfig builds a server TLS config advertising the given ALPN protocols.
+//
+// ALPN is how the client and server agree on the HTTP version, so the protocol
+// list is the actual switch that decides whether a TLS connection becomes h2 or
+// h1. Getting it wrong means a benchmark labelled "h2" silently measures h1.
+func TLSConfig(cert tls.Certificate, protos ...string) *tls.Config {
+	return &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		NextProtos:   protos,
+		MinVersion:   tls.VersionTLS12,
+	}
+}

data/go/internal/frontend/tlscert_test.go

@@ -0,0 +1,81 @@
+package frontend
+
+import (
+	"crypto/x509"
+	"net"
+	"testing"
+	"time"
+)
+
+// TestDevCertificateOutlivesTheProcess is the regression test for a certificate
+// that expired while its server was still running.
+//
+// The cert is minted once at boot and never refreshed, so a 24-hour NotAfter
+// meant any server up for longer than a day served an expired certificate and
+// failed every client. Nothing caught it because no test ran for a day.
+func TestDevCertificateOutlivesTheProcess(t *testing.T) {
+	cert, err := DevCertificate("localhost")
+	if err != nil {
+		t.Fatal(err)
+	}
+	leaf := cert.Leaf
+	if leaf == nil {
+		t.Fatal("no parsed leaf on the certificate")
+	}
+
+	// A year is far more than any plausible uptime, and far less than the ten
+	// the constant asks for -- so this fails on a short window without pinning
+	// the exact value.
+	if until := time.Until(leaf.NotAfter); until < 365*24*time.Hour {
+		t.Errorf("certificate expires in %s; a server running longer than that "+
+			"would serve an expired cert, because it is minted once at boot and "+
+			"never refreshed", until.Round(time.Hour))
+	}
+
+	// Backdated, so a client whose clock is slightly behind still accepts it.
+	if !leaf.NotBefore.Before(time.Now()) {
+		t.Errorf("NotBefore %s is not in the past; clock skew would reject it", leaf.NotBefore)
+	}
+}
+
+// TestDevCertificateIsNotACA: a self-signed leaf used directly as a server
+// certificate needs neither IsCA nor KeyUsageCertSign. With a ten-year window,
+// setting them is the difference between a harmless throwaway and a long-lived
+// signing key for anyone who adds it to a trust store.
+func TestDevCertificateIsNotACA(t *testing.T) {
+	cert, err := DevCertificate("localhost")
+	if err != nil {
+		t.Fatal(err)
+	}
+	leaf := cert.Leaf
+
+	if leaf.IsCA {
+		t.Error("dev certificate is marked as a CA; it signs nothing and should not be able to")
+	}
+	if leaf.KeyUsage&x509.KeyUsageCertSign != 0 {
+		t.Error("dev certificate carries KeyUsageCertSign; it issues no certificates")
+	}
+	if leaf.KeyUsage&x509.KeyUsageDigitalSignature == 0 {
+		t.Error("dev certificate lacks KeyUsageDigitalSignature, which TLS needs")
+	}
+}
+
+// TestDevCertificateCoversLoopback: every local client arrives over loopback
+// under some name, and a certificate that fails on 127.0.0.1 is useless for the
+// testing it exists to enable.
+func TestDevCertificateCoversLoopback(t *testing.T) {
+	cert, err := DevCertificate("example.test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	leaf := cert.Leaf
+
+	for _, ip := range []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback} {
+		if err := leaf.VerifyHostname(ip.String()); err != nil {
+			t.Errorf("certificate does not cover %s: %v", ip, err)
+		}
+	}
+	if err := leaf.VerifyHostname("example.test"); err != nil {
+		t.Errorf("certificate does not cover the requested host: %v", err)
+	}
+}

data/go/internal/frontend/wscontrol.go

@@ -0,0 +1,198 @@
+package frontend
+
+import (
+	"log"
+	"net"
+	"strconv"
+
+	"github.com/evanphx/puma-plus/internal/realtime"
+	"github.com/evanphx/puma-plus/internal/websocket"
+	"github.com/evanphx/puma-plus/internal/wire"
+)
+
+// WSControlConn is a worker process's outbound WebSocket command channel.
+//
+// A worker's data connections are busy carrying requests and are blocked reading
+// between them, so they cannot be used to push commands out. Each worker process
+// therefore opens one extra connection with role "ws", which exists purely so
+// the application can address connections it is not currently handling --
+// answering a different client, or broadcasting to a topic long after the
+// request that subscribed to it finished.
+type WSControlConn struct {
+	hub    *realtime.Hub
+	conn   net.Conn
+	r      *wire.Reader
+	w      *wire.Writer
+	logger *log.Logger
+	pid    int
+}
+
+// ServeWSControl reads commands from a worker's ws control connection until it
+// closes.
+func ServeWSControl(hub *realtime.Hub, nc net.Conn, pid int, logger *log.Logger) {
+	if logger == nil {
+		logger = log.Default()
+	}
+	c := &WSControlConn{
+		hub: hub, conn: nc, r: wire.NewReader(nc), w: wire.NewWriter(nc),
+		logger: logger, pid: pid,
+	}
+	defer nc.Close()
+
+	for {
+		typ, payload, err := c.r.ReadFrame()
+		if err != nil {
+			return
+		}
+		if err := c.handle(typ, payload); err != nil {
+			logger.Printf("ws control pid=%d: %v", pid, err)
+			return
+		}
+	}
+}
+
+func (c *WSControlConn) handle(typ uint8, payload []byte) error {
+	switch typ {
+	case wire.TypeWSSend:
+		args, body, err := splitArgs(payload)
+		if err != nil {
+			return err
+		}
+		if conn := c.hub.Get(args["conn_id"]); conn != nil {
+			binary := args["opcode"] == "binary"
+			// An explicit stream targets one stream of a WebTransport session;
+			// without it the message goes to the session's default channel,
+			// which is what a WebSocket has anyway.
+			if sid := args["stream_id"]; sid != "" {
+				_ = conn.SendOnStream(sid, body, binary)
+			} else {
+				_ = conn.Send(body, binary)
+			}
+		}
+
+	case wire.TypeWSPublish:
+		args, body, err := splitArgs(payload)
+		if err != nil {
+			return err
+		}
+		// One frame in, N writes out, all inside Go, across BOTH transports:
+		// a topic may hold WebSocket and WebTransport clients at once and each
+		// gets the message in its native form. This is the operation that would
+		// otherwise cost the application N frames across the process boundary
+		// per broadcast.
+		c.hub.Publish(args["topic"], body, args["opcode"] == "binary")
+
+	case wire.TypeWSSubscribe:
+		args, _, err := splitArgs(payload)
+		if err != nil {
+			return err
+		}
+		c.hub.Subscribe(args["conn_id"], args["topic"])
+
+	case wire.TypeWSUnsubscribe:
+		args, _, err := splitArgs(payload)
+		if err != nil {
+			return err
+		}
+		c.hub.Unsubscribe(args["conn_id"], args["topic"])
+
+	case wire.TypeWSClose:
+		args, _, err := splitArgs(payload)
+		if err != nil {
+			return err
+		}
+		if conn := c.hub.Get(args["conn_id"]); conn != nil {
+			code, _ := strconv.Atoi(args["code"])
+			if code == 0 {
+				code = websocket.CloseNormal
+			}
+			conn.CloseWith(code, args["reason"])
+		}
+
+	case wire.TypeWSDatagram:
+		args, body, err := splitArgs(payload)
+		if err != nil {
+			return err
+		}
+		if conn := c.hub.Get(args["conn_id"]); conn != nil {
+			// May fail with ErrNoDatagrams on a WebSocket client. Not an error
+			// worth killing the channel over -- the app asked for something the
+			// client's transport cannot do, and it can find that out via
+			// puma_plus.ws.transport.
+			_ = conn.SendDatagram(body)
+		}
+
+	case wire.TypeWSPublishDatagram:
+		args, body, err := splitArgs(payload)
+		if err != nil {
+			return err
+		}
+		c.hub.PublishDatagram(args["topic"], body)
+
+	case wire.TypeWSOpenStream:
+		args, _, err := splitArgs(payload)
+		if err != nil {
+			return err
+		}
+		if conn := c.hub.Get(args["conn_id"]); conn != nil {
+			id, err := conn.OpenStream()
+			result := id
+			if err != nil {
+				result = ""
+			}
+			return c.w.WriteFrame(wire.TypeWSQueryReply, wire.AppendKV(nil, []wire.Pair{
+				{Key: []byte("seq"), Value: []byte(args["seq"])},
+				{Key: []byte("result"), Value: []byte(result)},
+			}))
+		}
+
+	case wire.TypeWSQuery:
+		args, _, err := splitArgs(payload)
+		if err != nil {
+			return err
+		}
+		return c.reply(args)
+	}
+	return nil
+}
+
+// reply answers a synchronous query. The sequence number lets Ruby match the
+// answer to its question without needing a connection per in-flight query.
+func (c *WSControlConn) reply(args map[string]string) error {
+	var result string
+	switch args["kind"] {
+	case "subscribers":
+		result = strconv.Itoa(c.hub.Subscribers(args["topic"]))
+	case "connections":
+		result = strconv.Itoa(c.hub.Count())
+	case "exists":
+		result = "false"
+		if c.hub.Get(args["conn_id"]) != nil {
+			result = "true"
+		}
+	case "transport":
+		if conn := c.hub.Get(args["conn_id"]); conn != nil {
+			result = conn.Kind()
+		}
+	default:
+		result = ""
+	}
+	return c.w.WriteFrame(wire.TypeWSQueryReply, wire.AppendKV(nil, []wire.Pair{
+		{Key: []byte("seq"), Value: []byte(args["seq"])},
+		{Key: []byte("result"), Value: []byte(result)},
+	}))
+}
+
+// splitArgs decodes the leading kv blob and returns whatever follows as the
+// payload. The blob is self-delimiting, so a message body needs no length field.
+func splitArgs(payload []byte) (map[string]string, []byte, error) {
+	pairs, rest, err := wire.DecodeKV(payload)
+	if err != nil {
+		return nil, nil, err
+	}
+	args := make(map[string]string, len(pairs))
+	for _, p := range pairs {
+		args[string(p.Key)] = string(p.Value)
+	}
+	return args, rest, nil
+}

data/go/internal/metrics/collector.go

@@ -0,0 +1,207 @@
+package metrics
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// WindowDepth is the number of 1s snapshots retained.
+const WindowDepth = 10
+
+// ScaleWindow is how many snapshots the control loop's percentile view merges.
+// Five seconds is long enough that a single slow request cannot trip a scale-up
+// and short enough to react inside the fork cooldown.
+const ScaleWindow = 5
+
+// Collector owns every measurement puma-plus takes.
+//
+// The design constraint is that the hot path costs two time.Now() calls (vDSO,
+// no syscall) plus a handful of atomic adds, with zero allocation. Everything
+// expensive -- percentiles, merges, rendering -- happens on the tick or on a
+// scrape, never per request.
+type Collector struct {
+	QueueTime   *Rolling // Q = T3-T2b, the product metric
+	ServiceTime *Rolling // T5-T4, from the RESP_END trailer
+	TTFB        *Rolling // T2b-T5, the user-visible number
+
+	// Counters. RequestsCount matches puma's stat of the same name.
+	RequestsCount atomic.Uint64
+	ShedCount     atomic.Uint64
+	ErrorCount    atomic.Uint64
+	BytesOut      atomic.Uint64
+
+	// Arrival rate and mean service time, at two time constants each. The fast
+	// pair feeds the shed decision; the slow pair feeds scaling.
+	ArrivalFast *EWMA
+	ArrivalSlow *EWMA
+	ServiceSlow *EWMA
+
+	// GVLFraction is the share of service time spent queued behind the GVL,
+	// averaged slowly. Above ~0.3 the process is GVL-bound: its threads are
+	// waiting on each other rather than working, so adding more of them deepens
+	// the queue instead of raising throughput.
+	//
+	// This is the one signal that distinguishes "blocked on a database" from
+	// "blocked on another Ruby thread". Both look like not-on-CPU to every
+	// cheaper heuristic. Zero when the worker has no native GVL extension, in
+	// which case the controller falls back to io_fraction and gain probing.
+	// CPUFraction is the share of service time actually spent on CPU.
+	CPUFraction *EWMA
+
+	// GVL turns each worker's cumulative counter into a fraction of available
+	// thread-time. See gvl.go for why it is a rate over thread-time rather than
+	// a per-request delta.
+	GVL *GVLTracker
+
+	startedAt   time.Time
+	lastRotate  time.Time
+	lastRequest atomic.Uint64 // RequestsCount at the previous tick
+
+	// Two scratch histograms, not one. Tick runs on the ticker goroutine while
+	// QuantilesNs runs on whichever goroutine is serving /stats, so a single
+	// shared scratch would be a data race. Each is reused to keep scrapes and
+	// ticks allocation-free.
+	tickScratch  Histogram
+	queryMu      sync.Mutex
+	queryScratch Histogram
+}
+
+// NewCollector creates a collector with the standard windows.
+func NewCollector(now time.Time) *Collector {
+	return &Collector{
+		QueueTime:   NewRolling(WindowDepth),
+		ServiceTime: NewRolling(WindowDepth),
+		TTFB:        NewRolling(WindowDepth),
+		ArrivalFast: NewEWMA(1 * time.Second),
+		ArrivalSlow: NewEWMA(15 * time.Second),
+		ServiceSlow: NewEWMA(15 * time.Second),
+		CPUFraction: NewEWMA(15 * time.Second),
+		GVL:         NewGVLTracker(),
+		startedAt:   now,
+		lastRotate:  now,
+	}
+}
+
+// ObserveWork records where a request's service time actually went, so the
+// controller can tell threads-will-help from threads-will-not-help.
+func (c *Collector) ObserveWork(service, cpu time.Duration, now time.Time) {
+	if service <= 0 {
+		return
+	}
+	c.CPUFraction.Update(float64(cpu)/float64(service), now)
+}
+
+// ObserveGVL records a worker's cumulative GVL-wait counter.
+func (c *Collector) ObserveGVL(workerID int, cumulative uint64, threads int, now time.Time) {
+	c.GVL.Observe(workerID, cumulative, threads, now)
+}
+
+// GVLFractionValue reports the measured GVL fraction and whether it is real.
+func (c *Collector) GVLFractionValue() (float64, bool) { return c.GVL.Fraction() }
+
+// ObserveRequest records one completed request. Called once per response, off
+// the latency-critical path (the client already has its bytes).
+func (c *Collector) ObserveRequest(queue, service, ttfb time.Duration, bytesOut int) {
+	c.RequestsCount.Add(1)
+	c.QueueTime.Record(uint64(queue.Nanoseconds()))
+	c.TTFB.Record(uint64(ttfb.Nanoseconds()))
+	if service > 0 {
+		c.ServiceTime.Record(uint64(service.Nanoseconds()))
+	}
+	if bytesOut > 0 {
+		c.BytesOut.Add(uint64(bytesOut))
+	}
+}
+
+// ObserveShed records a request rejected without dispatch.
+func (c *Collector) ObserveShed() { c.ShedCount.Add(1) }
+
+// ObserveError records a backend failure.
+func (c *Collector) ObserveError() { c.ErrorCount.Add(1) }
+
+// Tick rotates the windows and updates the rate averages. Called once per second.
+func (c *Collector) Tick(now time.Time) {
+	elapsed := now.Sub(c.lastRotate).Seconds()
+	if elapsed <= 0 {
+		elapsed = 1
+	}
+
+	total := c.RequestsCount.Load()
+	delta := total - c.lastRequest.Load()
+	c.lastRequest.Store(total)
+
+	rate := float64(delta) / elapsed
+	c.ArrivalFast.Update(rate, now)
+	c.ArrivalSlow.Update(rate, now)
+
+	// Mean service time over the window just closed. Falls back to holding the
+	// previous value when no requests completed, rather than decaying toward
+	// zero -- a zero service time would make the Little's Law predictor claim
+	// infinite capacity precisely when we have no evidence either way.
+	c.ServiceTime.Merge(1, &c.tickScratch)
+	if c.tickScratch.Count() > 0 {
+		c.ServiceSlow.Update(c.tickScratch.Mean()/1e9, now)
+	}
+
+	c.QueueTime.Rotate()
+	c.ServiceTime.Rotate()
+	c.TTFB.Rotate()
+	c.lastRotate = now
+}
+
+// PredictedWait estimates the queue time the *next* arrival will experience,
+// via Little's Law: W = (L+1) * S / c.
+//
+// This matters because measured percentiles are lagging indicators -- a request
+// must suffer the queue before it can report it, so when the queue is growing,
+// p95 describes the past. The predictor describes the request arriving now,
+// which is why shedding keys off this and scaling keys off measured p95
+// confirmed by this.
+//
+// We can compute it honestly only because we own both sides of L = lambda*W: L
+// is an integer we maintain, not an estimate. Puma cannot compute this.
+//
+// KNOWN LIMITATION, measured in Phase 2 and load-bearing for the controller:
+// the estimate is only as good as S-bar, and S-bar comes from a 15s EWMA. When
+// the workload's service time changes regime, the predictor lags badly -- an
+// observed run went from a 4us hello workload to a 100ms sleep workload and
+// reported S = 1.1ms (predicting a 7ms wait) while requests were actually
+// waiting ~600ms. It recovers within a few seconds once the new regime
+// dominates the average.
+//
+// So the predictor is least trustworthy exactly when the workload shifts, which
+// is when a controller is most tempted to act. This is the concrete reason the
+// control loop must scale on *measured* p95 confirmed by this predictor, and
+// never on the predictor alone. Shedding may use it directly because a shed is
+// cheap and instantly reversible; a fork is neither.
+func (c *Collector) PredictedWait(depth, capacity int) time.Duration {
+	if capacity <= 0 {
+		// No workers connected: any arrival waits indefinitely. Report a large
+		// but finite value so callers compare rather than special-case.
+		return time.Hour
+	}
+	s := c.ServiceSlow.Value()
+	if s <= 0 {
+		return 0
+	}
+	w := float64(depth+1) * s / float64(capacity)
+	return time.Duration(w * float64(time.Second))
+}
+
+// Uptime since the collector was created.
+func (c *Collector) Uptime(now time.Time) time.Duration { return now.Sub(c.startedAt) }
+
+// StartedAt reports when collection began.
+func (c *Collector) StartedAt() time.Time { return c.startedAt }
+
+// QuantilesNs merges the last `windows` snapshots of r and returns p50/p95/p99
+// in nanoseconds. Safe for concurrent callers.
+func (c *Collector) QuantilesNs(r *Rolling, windows int) (p50, p95, p99 uint64) {
+	c.queryMu.Lock()
+	defer c.queryMu.Unlock()
+	r.Merge(windows, &c.queryScratch)
+	return c.queryScratch.Quantile(0.50),
+		c.queryScratch.Quantile(0.95),
+		c.queryScratch.Quantile(0.99)
+}