@optimystic/db-core 0.13.5 → 0.14.0

package/src/cohort-topic/traffic.ts

@@ -0,0 +1,211 @@
+/**
+ * Cohort-topic substrate — topic-traffic signal.
+ *
+ * Transcribed from `docs/cohort-topic.md` §Topic traffic signal. A cohort tracks per-topic flow
+ * rates alongside the stock `directParticipants` count and returns them on registration replies so
+ * applications can decide whether the current tier is dense enough to settle on.
+ *
+ * Shape (`TopicTrafficV1` from the wire ticket):
+ * - `arrivalsPerMin` — **combines** fresh registrations and renewals into one scalar (the seeker
+ *   uses renewals as a proxy for active matchable supply); the caller invokes {@link recordArrival}
+ *   for both.
+ * - `queriesPerMin` — application-level query rate.
+ * - counts are **exact integers** over `windowSeconds` (default 60), *not* log-bucketed like the
+ *   load barometer — the consumer-side matchmaking formulas are numeric.
+ *
+ * Pipeline: each member counts arrivals/queries locally ({@link recordArrival} / {@link recordQuery})
+ * over a sliding window; {@link TrafficCounters.publish} freezes the windowed counts into the
+ * member's gossiped per-topic summary each gossip round; {@link TrafficCounters.snapshot} returns the
+ * **gossip-derived** view (own last-published summary + siblings' last-gossiped summaries from the
+ * {@link CohortView}), so the reply lags by at most one round and never recomputes from raw counters
+ * at reply time.
+ *
+ * Resolved open question (GROUNDING): the signal is returned **only** on `accepted` and `promoted`
+ * replies (see {@link attachTopicTraffic}); a participant getting `Promoted(d+1)` still receives the
+ * outgoing cohort's traffic so it can estimate whether the redirect target is hot.
+ *
+ * Counters reset to zero on `cohortEpoch` change ({@link TrafficCounters.reset}); the first round
+ * after a rotation may under-report, and consumers tolerate a single zero reading.
+ */
+
+import { bytesKey } from "./registration/bytes.js";
+import type { CohortView } from "./gossip/view.js";
+import type { RegistrationStore } from "./registration/types.js";
+import { bytesToB64url } from "./wire/codec.js";
+import type { CohortTopicSummary, RegisterReplyV1, TopicTrafficV1 } from "./wire/types.js";
+
+/** Default observation window for the rate fields (seconds). */
+export const DEFAULT_TRAFFIC_WINDOW_SECONDS = 60;
+
+/** Frozen per-topic counts the member last published to gossip. */
+interface PublishedTopic {
+	readonly arrivals: number;
+	readonly queries: number;
+}
+
+/** Local windowed event timestamps (unix ms) for one topic. */
+interface TopicWindow {
+	arrivals: number[];
+	queries: number[];
+}
+
+/** Per-topic traffic counters with a gossip-derived snapshot. */
+export interface TrafficCounters {
+	/** Record a fresh registration **or** a renewal for `topicId` at `now` (combined arrivals). */
+	recordArrival(topicId: Uint8Array, now: number): void;
+	/** Record an application-level query against `topicId` at `now`. */
+	recordQuery(topicId: Uint8Array, now: number): void;
+	/**
+	 * Freeze the current windowed counts into the member's gossiped summary (call once per gossip
+	 * round, at `now`). The frozen values are what {@link snapshot} and the gossip frame read.
+	 */
+	publish(topicId: Uint8Array, now: number): PublishedTopic;
+	/** Own most-recent published `(arrivals, queries)` for `topicId`, or zeros if never published. */
+	published(topicId: Uint8Array): PublishedTopic;
+	/**
+	 * Gossip-derived traffic for `topicId`: own last-published counts plus siblings' last-gossiped
+	 * summaries (exact-integer sums), with `directParticipants` from the replicated store. Lags ≤ one
+	 * round; never recomputes from raw counters.
+	 */
+	snapshot(topicId: Uint8Array): TopicTrafficV1;
+	/** Reset all counters to zero (call on `cohortEpoch` change). */
+	reset(): void;
+}
+
+export interface TrafficCountersDeps {
+	/** Merged per-member gossip view — siblings' last-gossiped per-topic summaries. */
+	view: CohortView;
+	/** Replicated registration store — supplies the cohort-wide `directParticipants` stock count. */
+	store: Pick<RegistrationStore, "directParticipants">;
+	/** This member's own id, base64url — excluded from the sibling summary scan (counted via own published). */
+	selfMember: string;
+	/** Observation window (seconds). Default {@link DEFAULT_TRAFFIC_WINDOW_SECONDS}. */
+	windowSeconds?: number;
+	/** Tier-(d+1) cohort count for a topic (0 if not promoted); the promotion ticket owns this. */
+	childCohortCount?: (topicId: Uint8Array) => number;
+}
+
+class WindowedTrafficCounters implements TrafficCounters {
+	private readonly windows = new Map<string, TopicWindow>();
+	private readonly lastPublished = new Map<string, PublishedTopic>();
+	private readonly windowMs: number;
+
+	constructor(private readonly deps: TrafficCountersDeps) {
+		const ws = deps.windowSeconds ?? DEFAULT_TRAFFIC_WINDOW_SECONDS;
+		if (!(ws > 0)) {
+			throw new RangeError(`windowSeconds must be > 0, got ${ws}`);
+		}
+		this.windowMs = ws * 1000;
+	}
+
+	recordArrival(topicId: Uint8Array, now: number): void {
+		this.windowFor(topicId).arrivals.push(now);
+	}
+
+	recordQuery(topicId: Uint8Array, now: number): void {
+		this.windowFor(topicId).queries.push(now);
+	}
+
+	publish(topicId: Uint8Array, now: number): PublishedTopic {
+		const w = this.windows.get(bytesKey(topicId));
+		const frozen: PublishedTopic = {
+			arrivals: w === undefined ? 0 : countWithin(w.arrivals, now, this.windowMs),
+			queries: w === undefined ? 0 : countWithin(w.queries, now, this.windowMs),
+		};
+		this.lastPublished.set(bytesKey(topicId), frozen);
+		return frozen;
+	}
+
+	published(topicId: Uint8Array): PublishedTopic {
+		return this.lastPublished.get(bytesKey(topicId)) ?? { arrivals: 0, queries: 0 };
+	}
+
+	snapshot(topicId: Uint8Array): TopicTrafficV1 {
+		const topicB64 = bytesToB64url(topicId);
+		const own = this.published(topicId);
+		let arrivals = own.arrivals;
+		let queries = own.queries;
+		let childCohortCount = 0;
+		for (const [member, contribution] of this.deps.view.all()) {
+			if (member === this.deps.selfMember) continue; // own contribution comes from lastPublished
+			const summary = contribution.topicSummaries.find((s) => s.topicId === topicB64);
+			if (summary === undefined) continue;
+			arrivals += summary.arrivalsPerMin;
+			queries += summary.queriesPerMin;
+			if (summary.childCohortCount > childCohortCount) {
+				childCohortCount = summary.childCohortCount;
+			}
+		}
+		const childOverride = this.deps.childCohortCount?.(topicId);
+		return {
+			windowSeconds: this.windowMs / 1000,
+			arrivalsPerMin: arrivals,
+			queriesPerMin: queries,
+			directParticipants: this.deps.store.directParticipants(topicId),
+			childCohortCount: childOverride ?? childCohortCount,
+		};
+	}
+
+	reset(): void {
+		this.windows.clear();
+		this.lastPublished.clear();
+	}
+
+	private windowFor(topicId: Uint8Array): TopicWindow {
+		const key = bytesKey(topicId);
+		let w = this.windows.get(key);
+		if (w === undefined) {
+			w = { arrivals: [], queries: [] };
+			this.windows.set(key, w);
+		}
+		return w;
+	}
+}
+
+/** Count timestamps within `[now - windowMs, now]`, pruning anything older from `events` in place. */
+function countWithin(events: number[], now: number, windowMs: number): number {
+	const cutoff = now - windowMs;
+	// Events accumulate in arrival order; drop the stale prefix so the array stays bounded.
+	let drop = 0;
+	while (drop < events.length && events[drop]! < cutoff) {
+		drop++;
+	}
+	if (drop > 0) {
+		events.splice(0, drop);
+	}
+	return events.length;
+}
+
+/** Construct empty {@link TrafficCounters}. */
+export function createTrafficCounters(deps: TrafficCountersDeps): TrafficCounters {
+	return new WindowedTrafficCounters(deps);
+}
+
+/**
+ * Attach the topic-traffic signal to a reply **only** when the result is `accepted` or `promoted`
+ * (GROUNDING-resolved: absent on `no_state`, `unwilling_member`, `unwilling_cohort`). Mutates and
+ * returns `reply`. A no-op for non-traffic-bearing results, so callers can pipe every reply through.
+ */
+export function attachTopicTraffic(reply: RegisterReplyV1, traffic: TopicTrafficV1): RegisterReplyV1 {
+	if (reply.result === "accepted" || reply.result === "promoted") {
+		reply.topicTraffic = traffic;
+	}
+	return reply;
+}
+
+/** The own published counts as a `CohortTopicSummary` for the gossip frame. Callers supply tier/promotion fields. */
+export function toCohortTopicSummary(
+	topicId: Uint8Array,
+	published: PublishedTopic,
+	fields: { tier: number; directParticipants: number; promoted: boolean; childCohortCount: number },
+): CohortTopicSummary {
+	return {
+		topicId: bytesToB64url(topicId),
+		tier: fields.tier,
+		directParticipants: fields.directParticipants,
+		arrivalsPerMin: published.arrivals,
+		queriesPerMin: published.queries,
+		promoted: fields.promoted,
+		childCohortCount: fields.childCohortCount,
+	};
+}

package/src/cohort-topic/walk.ts

@@ -0,0 +1,255 @@
+/**
+ * Cohort-topic substrate — walk-toward-root lookup / registration.
+ *
+ * Transcribed from `docs/cohort-topic.md` §Tree growth and lookup (§Lookup loop) and folded back
+ * from the simulator-validated `packages/substrate-simulator/src/walk.ts`. A participant resolves a
+ * topic by walking **toward the root** from `d_max`, probing one tier coordinate per RPC. The walk's
+ * single-direction discipline is the anti-flood backbone: it only ever moves inward (toward the root)
+ * on `NoState`, and the *only* outward move is following an explicit `Promoted` redirect.
+ *
+ * Reply handling (§Lookup):
+ * - **`accepted`** → done; return the reply (carries `primary`/`backups`/`cohortEpoch` to cache).
+ * - **`no_state`** at tier `d` → step one tier toward the root (`d − 1`); no traffic signal. At the
+ *   root (`d − 1 < 0`) re-issue once at tier 0 with `bootstrap: true` (cold-start request).
+ * - **`promoted(targetTier)`** → recompute `coord_targetTier(self, topicId)` and register there — the
+ *   one outward move, taken only on this explicit redirect.
+ * - **`unwilling_member(candidates)`** → retry the **same** coord at a named alternative member
+ *   (spatial move within the cohort), via a direct dial.
+ * - **`unwilling_cohort(retryAfter)`** → back off in **time**; the walk terminates with
+ *   {@link RetryLaterOutcome} and the caller restarts a fresh {@link WalkEngine.register} after
+ *   `afterMs` — which begins again at `d_max`, decorrelating retries across the ring (§Anti-flood
+ *   claim 4: never re-hit the declined coord immediately).
+ *
+ * This module is FRET-free: it drives the {@link ITopicRouter} port (db-p2p binds it to FRET's
+ * `RouteAndMaybeAct` / direct dial) and the {@link TierAddressing} math, and delegates building +
+ * signing the {@link RegisterV1} to the injected {@link RegisterMessageFactory} (participant identity
+ * and crypto live there, not here).
+ */
+
+import type { ITopicRouter, PeerRef } from "./ports.js";
+import type { TierAddressing } from "./addressing.js";
+import type { DMaxComputer } from "./dmax.js";
+import { backoffRetryMs } from "./willingness.js";
+import { b64urlToBytes, decodeRegisterReplyV1, encodeCohortMessage } from "./wire/codec.js";
+import type { RegisterReplyV1, RegisterV1 } from "./wire/types.js";
+
+/** The walk landed: the cohort accepted the registration. `reply` carries the cohort cache fields. */
+export interface AcceptedWalkOutcome {
+	readonly kind: "accepted";
+	readonly reply: RegisterReplyV1;
+}
+
+/**
+ * The walk hit a `Promoted` redirect and the engine was configured **not** to follow it
+ * ({@link WalkConfig.followPromoted} `= false`): the caller recomputes `coord_targetTier` and
+ * registers there itself. With the default (`followPromoted = true`) the engine follows the redirect
+ * internally and this outcome never surfaces.
+ */
+export interface PromotedWalkOutcome {
+	readonly kind: "promoted";
+	readonly targetTier: number;
+}
+
+/**
+ * The walk backed off in time (`unwilling_cohort`, exhausted sibling retries, a failed cold-start, or
+ * the safety step cap). The caller waits `afterMs` then calls {@link WalkEngine.register} again, which
+ * restarts at `d_max`.
+ */
+export interface RetryLaterOutcome {
+	readonly kind: "retry_later";
+	readonly afterMs: number;
+}
+
+export type WalkOutcome = AcceptedWalkOutcome | PromotedWalkOutcome | RetryLaterOutcome;
+
+/** Builds (and signs) the {@link RegisterV1} for one probe; owns participant identity + crypto. */
+export interface RegisterMessageFactory {
+	/**
+	 * Produce a signed `RegisterV1` for this participant at walk position `treeTier`. `bootstrap` is
+	 * set only on the root cold-start re-issue. `appPayload` is the opaque application slot. On the
+	 * bootstrap re-issue the factory may additionally mint and attach the signed `bootstrapEvidence`
+	 * envelope (§Anti-DoS) via the injected builder seam before signing — keyed off `bootstrap`, so no
+	 * extra build parameter is needed (the walk decides `bootstrap` internally, not the application).
+	 */
+	build(params: {
+		topicId: Uint8Array;
+		tier: number;
+		treeTier: number;
+		bootstrap: boolean;
+		/** Read-only lookup probe: the factory stamps `RegisterV1.probe` and never mints bootstrap evidence. */
+		probe: boolean;
+		appPayload?: Uint8Array;
+	}): Promise<RegisterV1>;
+}
+
+export interface WalkConfig {
+	/** Cohort size requested from the router (`wantK`). Default 16. */
+	wantK?: number;
+	/** Threshold signers requested from the router (`minSigs = k − x`). Default 14. */
+	minSigs?: number;
+	/** Max `unwilling_member` sibling retries at one coord before treating it as a cohort decline. Default `wantK`. */
+	maxMemberRetries?: number;
+	/**
+	 * Whether to follow a `Promoted` redirect internally (recompute coord + continue) or surface it as
+	 * a {@link PromotedWalkOutcome} for the caller to drive. Default `true` (self-contained walk).
+	 */
+	followPromoted?: boolean;
+	/**
+	 * Hard cap on probe RPCs in one walk — a safety valve against pathological oscillation between an
+	 * inward `NoState` step and an outward `Promoted` redirect in a malformed tree. Default scales with
+	 * `d_max`. Exceeding it yields a {@link RetryLaterOutcome}.
+	 */
+	maxSteps?: number;
+	/** `max_message_bytes` ceiling for the encoded register frame. Defaults to the codec default. */
+	maxMessageBytes?: number;
+}
+
+export interface WalkEngineDeps {
+	router: ITopicRouter;
+	addressing: TierAddressing;
+	/** Computes the walk start tier `d_max` from the current network-size estimate. */
+	dmax: DMaxComputer;
+	/** This participant's peer id — the `P` in `coord_d(P, topicId)`. */
+	self: Uint8Array;
+	/** Builds + signs the per-probe `RegisterV1`. */
+	factory: RegisterMessageFactory;
+	config?: WalkConfig;
+}
+
+/** Drives a participant's walk-toward-root registration over the injected router + addressing. */
+export interface WalkEngine {
+	/**
+	 * Walk from `d_max` toward the root registering for `topicId` at op `tier`, following `Promoted`
+	 * redirects outward. Resolves with the terminal {@link WalkOutcome}. With `opts.probe` the walk is a
+	 * **read-only lookup**: identical routing discipline, but the terminal cohort classifies rather than
+	 * admits and the root `no_state` branch backs off instead of issuing a `bootstrap: true` cold-start
+	 * (a probe never instantiates a cold root).
+	 */
+	register(topicId: Uint8Array, tier: number, appPayload?: Uint8Array, opts?: { probe?: boolean }): Promise<WalkOutcome>;
+}
+
+class RouterWalkEngine implements WalkEngine {
+	private readonly wantK: number;
+	private readonly minSigs: number;
+	private readonly maxMemberRetries: number;
+	private readonly followPromoted: boolean;
+	private readonly configuredMaxSteps?: number;
+	private readonly maxMessageBytes?: number;
+
+	constructor(private readonly deps: WalkEngineDeps) {
+		const cfg = deps.config ?? {};
+		this.wantK = cfg.wantK ?? 16;
+		this.minSigs = cfg.minSigs ?? 14;
+		this.maxMemberRetries = cfg.maxMemberRetries ?? this.wantK;
+		this.followPromoted = cfg.followPromoted ?? true;
+		this.configuredMaxSteps = cfg.maxSteps;
+		this.maxMessageBytes = cfg.maxMessageBytes;
+	}
+
+	async register(topicId: Uint8Array, tier: number, appPayload?: Uint8Array, opts?: { probe?: boolean }): Promise<WalkOutcome> {
+		const dMax = this.deps.dmax.dMax();
+		const maxSteps = this.configuredMaxSteps ?? 2 * (dMax + 2) + this.maxMemberRetries + 8;
+		const probe = opts?.probe ?? false;
+
+		let d = dMax;
+		let bootstrap = false;
+		let memberAttempts = 0;
+		let dialTarget: PeerRef | undefined;
+		let steps = 0;
+		let probeFollowedPromoted = false;
+
+		for (;;) {
+			if (++steps > maxSteps) {
+				// Safety valve: a well-formed tree converges well within this bound. Surface a temporal
+				// back-off rather than spin.
+				return { kind: "retry_later", afterMs: backoffRetryMs(0) };
+			}
+
+			const reg = await this.deps.factory.build({ topicId, tier, treeTier: d, bootstrap, probe, appPayload });
+			const activity = encodeCohortMessage(reg, this.maxMessageBytes);
+			const raw = dialTarget !== undefined
+				? await this.deps.router.dialMember(dialTarget, activity)
+				: await this.deps.router.routeAndAct(this.deps.addressing.coord(d, this.deps.self, topicId), activity, {
+					wantK: this.wantK,
+					minSigs: this.minSigs,
+				});
+			const reply = decodeRegisterReplyV1(raw, this.maxMessageBytes);
+
+			switch (reply.result) {
+				case "accepted": {
+					return { kind: "accepted", reply };
+				}
+				case "no_state": {
+					// Step toward the root. The cohort served nothing here; no spatial sibling state.
+					dialTarget = undefined;
+					memberAttempts = 0;
+					if (probe && probeFollowedPromoted) {
+						// The promoted child is cold (not yet instantiated). A probe never instantiates it,
+						// so back off immediately — walking inward to the promoting ancestor would just
+						// re-trigger the Promoted redirect and loop.
+						return { kind: "retry_later", afterMs: backoffRetryMs(0) };
+					}
+					const next = d - 1;
+					if (next < 0) {
+						if (bootstrap) {
+							// Already re-issued at the root as a bootstrap and still nothing — no cohort
+							// anywhere will instantiate right now. Back off in time.
+							return { kind: "retry_later", afterMs: backoffRetryMs(0) };
+						}
+						if (probe) {
+							// A read-only probe never instantiates a cold root: the topic exists nowhere, so
+							// resolve "not found / back off" rather than re-issuing with bootstrap:true.
+							return { kind: "retry_later", afterMs: backoffRetryMs(0) };
+						}
+						// Root returned NoState → cold-start: re-issue once at tier 0 with bootstrap:true.
+						d = 0;
+						bootstrap = true;
+						break;
+					}
+					d = next;
+					bootstrap = false;
+					break;
+				}
+				case "promoted": {
+					dialTarget = undefined;
+					memberAttempts = 0;
+					bootstrap = false;
+					const targetTier = reply.targetTier ?? d + 1;
+					if (probe) {
+						probeFollowedPromoted = true;
+					}
+					if (!this.followPromoted) {
+						return { kind: "promoted", targetTier };
+					}
+					d = targetTier; // the one outward move — recompute coord at the redirect target
+					break;
+				}
+				case "unwilling_member": {
+					const candidates = reply.candidateMembers ?? [];
+					if (candidates.length === 0 || memberAttempts >= this.maxMemberRetries) {
+						// Exhausted the named siblings (or none offered) → fall through to a cohort-level
+						// temporal back-off, restarting at d_max on the caller's retry.
+						return { kind: "retry_later", afterMs: backoffRetryMs(0) };
+					}
+					// Retry the SAME coord at a named alternative member (spatial move within the cohort).
+					dialTarget = { id: b64urlToBytes(candidates[memberAttempts % candidates.length]!) };
+					memberAttempts++;
+					break;
+				}
+				case "unwilling_cohort": {
+					// Back off in TIME, no spatial move; the caller restarts at d_max after the delay.
+					return { kind: "retry_later", afterMs: reply.retryAfterMs ?? backoffRetryMs(0) };
+				}
+				default: {
+					// Exhaustive over RegisterResult; an unknown result is treated as a temporal decline.
+					return { kind: "retry_later", afterMs: backoffRetryMs(0) };
+				}
+			}
+		}
+	}
+}
+
+/** Build a {@link WalkEngine} over the injected router, addressing, `d_max`, and message factory. */
+export function createWalkEngine(deps: WalkEngineDeps): WalkEngine {
+	return new RouterWalkEngine(deps);
+}