cursedbelt-server 4.18.0 → 4.19.0

package/src/server/bench/tail.ts

@@ -0,0 +1,237 @@
+/**
+ * The REAL reading: Worker CPU-ms off a tail worker's trace events, fed into the same recorder
+ * and the same assertion the local proxy uses.
+ *
+ * ## Why this shape, and not a clock
+ *
+ * {@link workerCpuClock} is a clock — `start()` then read again — and the isolate never hands a
+ * handler its own CPU time, so there is nothing for it to read inside a request. What Cloudflare
+ * DOES publish is one `cpuTime` per invocation on the trace event a TAIL worker receives (the
+ * same field `wrangler tail --format json` prints — `apps/binary-server/scripts/edgeTail.ts`
+ * already reads it there). So the real number arrives AFTER the request, per request, and the
+ * honest API is to record it then: {@link recordTailTraces} turns trace events into samples on a
+ * recorder built with {@link tailCpuClock}, which stamps `proxy: false`.
+ *
+ * Two ways to get the events, and a consumer needs neither code nor a new Worker for the first:
+ *
+ *   · **`wrangler tail <worker> --format json`** during a bench or a smoke against the live
+ *     Worker — one JSON trace per line, parsed with {@link parseTailLines}.
+ *   · **a Tail Worker** (`tail_consumers` in the producer's wrangler config) whose `tail(events)`
+ *     hands the array straight to {@link recordTailTraces}.
+ *
+ * 🔴 The degenerate guard still applies, deliberately. `cpuTime` is whole milliseconds, so a
+ * route that genuinely costs under 1 ms reads 0 — and a report where EVERY reading is 0 is still
+ * refused, because that is indistinguishable from a reader that is not reading. One real route
+ * above a millisecond anywhere in the window clears it; `allowZeroCpu` is not the fix.
+ *
+ * ## The traffic half
+ *
+ * {@link tailTrafficRate} is requests per day over the window the events span — the number
+ * `deriveCpuBudgetMs({ requestsPerDay })` needs to re-derive the default from THIS generation's
+ * Worker traffic (tasks 342/377) instead of the retired generation's corpus.
+ */
+
+import type { CpuClock } from './cpuClock.js';
+import type { CpuRecorder } from './recorder.js';
+
+/** The fields of a Cloudflare `TraceItem` this reads — structural, so no workers-types import. */
+export interface TailTraceLike {
+  /** CPU-ms of the whole invocation, as billed. Absent on runtimes that do not report it. */
+  cpuTime?: number | null;
+  wallTime?: number | null;
+  /** Epoch ms. */
+  eventTimestamp?: number | null;
+  outcome?: string | null;
+  scriptName?: string | null;
+  /** A fetch invocation carries `request`; a cron or queue invocation does not. */
+  event?: { request?: { url?: string; method?: string } | null } | null;
+}
+
+/** Provenance printed on every report built from tail events. */
+export const TAIL_CPU_SOURCE = 'tail-worker cpuTime';
+
+/**
+ * The clock for a recorder that is FED rather than started. `proxy: false`: this is the quantity
+ * Cloudflare bills. Its `start()` throws — a tail recorder mounted as `cpuBudget()` middleware is
+ * a wiring mistake, and a span that silently measured nothing is the lie this module refuses.
+ */
+export function tailCpuClock(source = TAIL_CPU_SOURCE): CpuClock {
+  return {
+    source,
+    proxy: false,
+    available: true,
+    start() {
+      throw new Error(
+        'tailCpuClock: a tail recorder is FED by recordTailTraces(), never started — the isolate ' +
+          'does not hand a handler its own CPU time. Mount cpuBudget() with processCpuClock() ' +
+          'for the local proxy, and feed this recorder from a tail.',
+      );
+    },
+  };
+}
+
+/** A route key's pattern half: `'GET /api/items/:id'` → `/api/items/:id`, with its method. */
+function splitKey(key: string): { method: string | null; pattern: string } {
+  const m = /^([A-Z]+)\s+(\S+)$/.exec(key.trim());
+  return m ? { method: m[1] ?? null, pattern: m[2] ?? key } : { method: null, pattern: key.trim() };
+}
+
+/** Hono-style pattern → anchored regex. `:param` (with an optional `{re}`) is one segment, `*` is the rest. */
+function patternRegex(pattern: string): RegExp {
+  let out = '';
+  for (let i = 0; i < pattern.length; ) {
+    const rest = pattern.slice(i);
+    const param = /^:[A-Za-z0-9_]+(\{[^}]*\})?\??/.exec(rest);
+    if (param) {
+      out += '[^/]+';
+      i += param[0].length;
+      continue;
+    }
+    const ch = pattern[i] as string;
+    out += ch === '*' ? '.*' : ch.replace(/[.+?^${}()|[\]\\]/g, '\\$&');
+    i += 1;
+  }
+  return new RegExp(`^${out}$`);
+}
+
+/** How literal a pattern is — the more literal characters, the earlier it is tried. */
+const specificity = (pattern: string): number =>
+  pattern.replace(/:[A-Za-z0-9_]+(\{[^}]*\})?\??/g, '').replace(/\*/g, '').length -
+  (pattern.includes('*') ? 1000 : 0);
+
+/**
+ * A `(method, path) → route label` matcher over declared route keys — the recorder's own
+ * `config.routes`, by default — so an app gets tail attribution with no code of its own. The
+ * most literal pattern wins, and a method-specific key beats an any-method one.
+ */
+export function routeMatcher(keys: readonly string[]): (method: string, path: string) => string | null {
+  const compiled = keys
+    .map((key) => ({ ...splitKey(key), key }))
+    .map((k) => ({ ...k, re: patternRegex(k.pattern), score: specificity(k.pattern) + (k.method ? 0.5 : 0) }))
+    .sort((a, b) => b.score - a.score);
+  return (method, path) => {
+    for (const k of compiled) {
+      if (k.method && k.method !== method.toUpperCase()) continue;
+      if (k.re.test(path)) return k.pattern;
+    }
+    return null;
+  };
+}
+
+/** The label traffic that matched no declared route is recorded under — so `requireDeclared` names it. */
+export const UNMATCHED_ROUTE = '(no declared route)';
+
+export interface RecordTailTracesOpts {
+  /** Map a request to its route label. Default: {@link routeMatcher} over the recorder's declared routes. */
+  route?: (method: string, path: string) => string | null;
+  /** Only this Worker's invocations — a tail consumer may receive several producers. */
+  scriptName?: string;
+}
+
+export interface TailRecordResult {
+  recorded: number;
+  /** Invocations with no request (cron, queue) — CPU, but no route to charge it to. */
+  notFetch: number;
+  /** Fetch invocations whose trace carried no finite `cpuTime`. */
+  noCpuTime: number;
+  /** Recorded under {@link UNMATCHED_ROUTE}; up to ten distinct paths, so the gap is nameable. */
+  unmatched: number;
+  unmatchedPaths: string[];
+}
+
+/**
+ * Charge each fetch invocation's billed `cpuTime` to its route on `recorder`.
+ *
+ * 🔴 A trace with no finite `cpuTime` is NOT a zero — it is skipped and counted, so a runtime
+ * that stopped reporting the field reads as "nothing recorded" (and the assertion's `no-samples`
+ * guard reds it) rather than as a confident route that costs nothing.
+ */
+export function recordTailTraces(
+  recorder: CpuRecorder,
+  traces: readonly TailTraceLike[],
+  opts: RecordTailTracesOpts = {},
+): TailRecordResult {
+  if (recorder.clock.proxy) {
+    throw new Error(
+      `recordTailTraces: the recorder's clock '${recorder.clock.source}' is a PROXY. Tail cpuTime is the ` +
+        'billed quantity — build this recorder with tailCpuClock(), or its report will call real numbers a proxy.',
+    );
+  }
+  const route = opts.route ?? routeMatcher(Object.keys(recorder.config.routes ?? {}));
+  const result: TailRecordResult = { recorded: 0, notFetch: 0, noCpuTime: 0, unmatched: 0, unmatchedPaths: [] };
+  for (const trace of traces) {
+    if (opts.scriptName && trace.scriptName && trace.scriptName !== opts.scriptName) continue;
+    const request = trace.event?.request;
+    if (!request?.url) {
+      result.notFetch += 1;
+      continue;
+    }
+    const cpu = trace.cpuTime;
+    if (typeof cpu !== 'number' || !Number.isFinite(cpu)) {
+      result.noCpuTime += 1;
+      continue;
+    }
+    const method = (request.method ?? 'GET').toUpperCase();
+    let path: string;
+    try {
+      path = new URL(request.url).pathname;
+    } catch {
+      path = request.url;
+    }
+    const label = route(method, path);
+    if (label === null) {
+      result.unmatched += 1;
+      if (result.unmatchedPaths.length < 10 && !result.unmatchedPaths.includes(`${method} ${path}`)) {
+        result.unmatchedPaths.push(`${method} ${path}`);
+      }
+    }
+    recorder.record(method, label ?? UNMATCHED_ROUTE, cpu);
+    result.recorded += 1;
+  }
+  return result;
+}
+
+/**
+ * `wrangler tail --format json` output → trace events. Tolerates the banner lines wrangler
+ * prints and blank lines; a line that is not a JSON object is skipped, never guessed at.
+ */
+export function parseTailLines(text: string): TailTraceLike[] {
+  const out: TailTraceLike[] = [];
+  for (const line of text.split('\n')) {
+    const trimmed = line.trim();
+    if (!trimmed.startsWith('{')) continue;
+    try {
+      const parsed = JSON.parse(trimmed) as unknown;
+      if (parsed && typeof parsed === 'object') out.push(parsed as TailTraceLike);
+    } catch {
+      // a partial line from an interrupted tail — not a trace
+    }
+  }
+  return out;
+}
+
+export interface TailTrafficRate {
+  /** Fetch invocations in the window. */
+  requests: number;
+  /** First to last `eventTimestamp`, ms. */
+  spanMs: number;
+  /** `requests` scaled to a day — `NaN` when the window is too short to scale honestly. */
+  requestsPerDay: number;
+}
+
+/**
+ * Requests per day over the window the traces span — what `deriveCpuBudgetMs({ requestsPerDay })`
+ * takes. `NaN` below an hour of window: scaling ten minutes of traffic to a day is a guess with a
+ * decimal point, and a budget derived from it would be quoted as a measurement.
+ */
+export function tailTrafficRate(traces: readonly TailTraceLike[], minSpanMs = 60 * 60 * 1000): TailTrafficRate {
+  const stamps = traces
+    .filter((t) => t.event?.request?.url)
+    .map((t) => t.eventTimestamp)
+    .filter((s): s is number => typeof s === 'number' && Number.isFinite(s))
+    .sort((a, b) => a - b);
+  const requests = stamps.length;
+  const spanMs = requests > 1 ? (stamps[requests - 1] as number) - (stamps[0] as number) : 0;
+  const requestsPerDay = spanMs >= minSpanMs ? (requests / spanMs) * 86_400_000 : Number.NaN;
+  return { requests, spanMs, requestsPerDay };
+}

package/src/server/d1/index.ts

@@ -7,11 +7,10 @@
  * local driver is deliberately as strict as D1 rather than as lenient as Bun.
  *
  * ```ts
- * import { createLocalD1, createRemoteD1, createD1Kysely } from 'cursedbelt-server/d1';
+ * import { createLocalD1, createRemoteD1 } from 'cursedbelt-server/d1';
  *
  * const db = env.DB ? createRemoteD1(env.DB) : createLocalD1(sqlite);
  * const row = await db.prepare('SELECT * FROM people WHERE id = ?').bind(id).first();
- * const ky  = createD1Kysely(db);
  * ```
  */
 
@@ -32,13 +31,14 @@ export {
 // as the note below, one step wider: a barrel must reach neither an optional peer nor a
 // runtime the target platform lacks. `barrelsReachNoOptionalPeer.spec.ts` checks both.
 //
-// 🔴 `./kysely` is deliberately NOT re-exported here — import it from
-// `cursedbelt-server/d1/kysely`. It statically imports `kysely` (real values: `Kysely`,
-// `SqliteAdapter`, `SqliteQueryCompiler`), which is an OPTIONAL peer, so re-exporting it
-// made `import 'cursedbelt-server/d1'` throw `Cannot find package 'kysely'` for every app
-// that does not use the query builder — which per `../db/kysely.ts`'s own header is most
-// of them, since business tables stay on raw statements. Measured 2026-09-18 against the
-// published 4.3.0 tarball. `barrelsReachNoOptionalPeer.spec.ts` is what keeps it out.
+// `./kysely` — a Kysely dialect over this seam, split out as its own `d1/kysely` subpath on
+// 2026-09-18 because it dragged the OPTIONAL `kysely` peer into every `import '…/d1'` — was
+// REMOVED in 4.19.0 (task 472). Measured 2026-09-23: no file under the generation imported
+// it, no app depends on `kysely` at all, and every ported Worker (patterns, collections,
+// music, vault) reached D1 through raw statements on this seam. A dialect nobody wants is
+// surface with a peer attached; if a ported app ever wants a query builder, it comes back
+// with that app as its first consumer, and `barrelsReachNoOptionalPeer.spec.ts` still keeps
+// it out of this barrel.
 export { createPendingWrites, type InvocationD1, type PendingWrites, perInvocation, type WriteCollector } from './invocation.js';
 export { createHttpD1, D1HttpError, type HttpD1Options } from './http.js';
 export { assertBatchSize, assertWithinLimits, chunkForBind, inArray, LIMITS } from './limits.js';

package/src/server/d1/pullD1.spec.ts

@@ -172,6 +172,26 @@ describe('pulling D1 down — table by table, because a whole-database export re
 		).rejects.toThrow(/lists no `tracks` table/);
 	});
 
+	test('an export that says success and writes NO file is asked again — and refuses if it never does', async () => {
+		const dir = scratch();
+		let flaky = 1;
+		const run: Wrangler = (args) => {
+			if (args[1] === 'execute') return { code: 0, stdout: JSON.stringify([{ results: MASTER }]), stderr: '' };
+			const table = args[args.indexOf('--table') + 1] as string;
+			const out = args[args.indexOf('--output') + 1] as string;
+			// `meta` "succeeds" once without writing — measured on collections, 2026-09-23.
+			if (table === 'meta' && flaky-- > 0) return { code: 0, stdout: 'Downloaded successfully', stderr: '' };
+			writeFileSync(out, table === 'items' ? "INSERT INTO items (id) VALUES ('i1');" : '');
+			return { code: 0, stdout: '', stderr: '' };
+		};
+		await pull(dir, run);
+		const never: Wrangler = (args) =>
+			args[1] === 'execute'
+				? { code: 0, stdout: JSON.stringify([{ results: MASTER }]), stderr: '' }
+				: { code: 0, stdout: 'Downloaded successfully', stderr: '' };
+		await expect(pull(scratch(), never)).rejects.toThrow(/reported success 3 times and wrote no file/);
+	});
+
 	test('🔴 one table failing to export THROWS — never a snapshot missing that table', async () => {
 		const dir = scratch();
 		const { run } = fake((table) => (table === 'meta' ? null : "INSERT INTO items (id) VALUES ('x');"));

package/src/server/d1/pullD1.ts

@@ -97,6 +97,9 @@ export async function servedRuntime(
 	return null;
 }
 
+/** How many times one table's export is asked for when wrangler says success and writes nothing. */
+const EXPORT_ATTEMPTS = 3;
+
 /** One `wrangler` invocation, as {@link pullD1ToSqlite} needs it. Injected so a spec runs none. */
 export type Wrangler = (args: string[]) => { code: number; stdout: string; stderr: string };
 
@@ -160,11 +163,24 @@ export async function pullD1ToSqlite(options: PullD1Options): Promise<void> {
 	let sql = '';
 	for (const table of tables) {
 		const dump = `${destination}.${table}.sql`;
-		const exported = wrangler(['d1', 'export', database, '--remote', '--env=', '--table', table, '--no-schema', '--output', dump, '-y']);
+		// 🔴 `wrangler d1 export` has been measured answering SUCCESS and writing no file —
+		// collections' nightly of 2026-09-23T15:35Z refused on `derivative_retries`, and the same
+		// command wrote the file (32 bytes, an empty table) an hour later. So a missing file is
+		// asked for again, twice, before it is a refusal; a failure that persists still refuses,
+		// because a snapshot missing a table is not a backup.
+		let exported = { code: 1, stdout: '', stderr: 'not attempted' };
+		for (let attempt = 1; attempt <= EXPORT_ATTEMPTS; attempt++) {
+			exported = wrangler(['d1', 'export', database, '--remote', '--env=', '--table', table, '--no-schema', '--output', dump, '-y']);
+			if (exported.code !== 0 || existsSync(dump)) break;
+		}
 		if (exported.code !== 0) {
 			throw new Error(`wrangler d1 export --table ${table} failed: ${exported.stderr.trim() || exported.stdout.trim()}`);
 		}
-		if (!existsSync(dump)) throw new Error(`wrangler d1 export --table ${table} reported success and wrote no file at ${dump}`);
+		if (!existsSync(dump)) {
+			throw new Error(
+				`wrangler d1 export --table ${table} reported success ${EXPORT_ATTEMPTS} times and wrote no file at ${dump}`,
+			);
+		}
 		sql += `${readFileSync(dump, 'utf8')}\n`;
 	}
 	if (!/INSERT INTO/i.test(sql)) {

package/src/server/engagement/api.ts

@@ -0,0 +1,119 @@
+/**
+ * The one door engagement needs, and the in-process read of what it recorded.
+ *
+ *   POST /api/engagement/view   → the beacon. SESSION-gated; the user id comes
+ *                                 from the session, never from the body.
+ *
+ * 🔴 **There is no HTTP read surface, on purpose** (task `113`/`268`, 2026-09-22).
+ * `GET /api/admin/engagement` used to sit here behind `SATELLITE_METRICS_TOKEN` —
+ * ONE bearer shared across five production env files, so a leak from any app was
+ * a read on all of them — for a fleet console that lives in the retired
+ * generation. Nothing in this generation read it over HTTP: station's telemetry
+ * page reads each app's sqlite straight off the disk, and the analytics reader
+ * planned after it does the same with `engagement.sqlite`. Recording carries on
+ * regardless; `readAppEngagement` below is the snapshot a same-machine reader
+ * computes from the store. If an HTTP reader is ever wanted, it gets a PER-APP
+ * credential from the generation's secrets store, never a fleet-wide one.
+ *
+ * ── Why the user id is never in the body ────────────────────────────────────
+ * A beacon whose payload names its own user is an endpoint that lets anyone
+ * write anyone's history — and on a private household app that history is the
+ * data. `resolveUser` is supplied by the app's own SSO consumer and is the ONLY
+ * source of a user key; a request that resolves to nobody is a 401 and writes
+ * nothing. `engagement.test.ts` pins that a body-supplied `userKey` is ignored.
+ *
+ * ── Why the recorder is mounted AFTER the session gate ──────────────────────
+ * The opposite of the feedback-kit sync receiver, and for the mirrored reason:
+ * this door has no credential of its own and wants the gate to have run,
+ * because that is what puts the user where `resolveUser` can read them — while
+ * a sync receiver carries its own bearer and must mount BEFORE the gate. The
+ * rule and its reason: `docs/engagement.md` §2 in this package.
+ */
+import { Hono } from "hono";
+import type { Context } from "hono";
+import { OTHER_PLACE, type PlaceRef, resolvePlace } from "./policy.js";
+import type { EngagementStore } from "./store.js";
+import { type AppEngagement, returnCurve, summarize } from "./summary.js";
+
+export interface EngagementApiOptions {
+	store: EngagementStore;
+	/** The app's own name. */
+	app: string;
+	/**
+	 * The allowlist of place ids — the app's `routes.manifest.json` ids. Anything
+	 * a client claims that is not in here records as `other`; see `policy.ts`
+	 * rule 1 for why this is the whole privacy story.
+	 */
+	places: readonly PlaceRef[];
+	/** Resolve the CURRENT request's account. Null → 401, nothing recorded. */
+	resolveUser: (c: Context) => string | null;
+	/** Injectable clock, so the tests are not timing-dependent. */
+	now?: () => number;
+}
+
+/** The recorder half — session-gated, mounted at `/api/engagement`. */
+export function createEngagementRecorder(options: EngagementApiOptions): Hono {
+	const app = new Hono();
+	const now = options.now ?? (() => Date.now());
+
+	app.post("/view", async (c) => {
+		const userKey = options.resolveUser(c);
+		if (!userKey) return c.json({ error: "unauthenticated" }, 401);
+
+		let body: unknown;
+		try {
+			body = await c.req.json();
+		} catch {
+			return c.json({ error: "expected a json body" }, 400);
+		}
+		const claimed = (body as { place?: unknown } | null)?.place;
+		if (typeof claimed !== "string") return c.json({ error: "place must be a string" }, 400);
+		// 🔴 Resolution happens HERE, at the door, against the app's own manifest.
+		// The store does not re-check; see its `recordView` header for why one
+		// check in the right place beats two in the wrong ones.
+		const place = resolvePlace(claimed, options.places);
+
+		const fold = options.store.recordView({ userKey, place, ts: now() });
+		// Echo the RESOLVED place, not the claimed one. A client that sent
+		// something off-manifest learns that it was bucketed, which is how a
+		// missing manifest entry gets noticed instead of quietly reading as an
+		// unpopular feature.
+		return c.json({ ok: true, place, bucketed: place === OTHER_PLACE, ...fold });
+	});
+
+	return app;
+}
+
+/** The whole snapshot one app publishes about itself. */
+export interface AppEngagementSnapshot extends AppEngagement {
+	/** The manifest ids this app CAN report — so a reader can show a place
+	 *  with zero views, which is the half of the question that is easy to lose. */
+	knownPlaces: string[];
+	returnCurve: Array<{ day: string; joined: number; returned: number }>;
+	retention: { days: number; places: number; users: number; oldestDay: string | null };
+}
+
+export function readAppEngagement(
+	store: EngagementStore,
+	app: string,
+	now: number,
+	knownPlaces: readonly PlaceRef[] = [],
+): AppEngagementSnapshot {
+	const users = store.users();
+	const summary = summarize({
+		app,
+		now,
+		users,
+		places: store.places(),
+		// 120 days is the store's whole daily retention — asking for more would
+		// silently return less, which reads as a quiet period rather than as a
+		// window boundary.
+		days: store.days(now - 120 * 86_400_000),
+	});
+	return {
+		...summary,
+		knownPlaces: knownPlaces.map((p) => p.id),
+		returnCurve: returnCurve(users, now),
+		retention: store.stats(),
+	};
+}