cursedbelt-server 4.27.0 → 4.29.0

package/src/server/activity/store.ts

@@ -34,80 +34,36 @@
  *     cannot be filtered, and filtering is the whole point of the column.
  */
 import type { Database, SQLQueryBindings } from "bun:sqlite";
-import {
-	type ActivityEvent,
-	type ActivityEventInput,
-	type ActivityOutcome,
-	isActivityAction,
-	isActivityOutcome,
-} from "cursedbelt-core/activity/model";
+import type { ActivityEvent, ActivityEventInput } from "cursedbelt-core/activity/model";
 import { runMigrations } from "../migration/runner.js";
-import { ACTIVITY_EVENTS_TABLE, ACTIVITY_MIGRATIONS } from "./migrations.js";
-
-/**
- * A read-only stream contributor: a SELECT projecting some other table into the
- * union's eleven columns, in this order and with these names —
- *
- *   `source, seq, at, app, actor, action, kind, resource_id, outcome, note, data`
- *
- * `seq` is that table's own row id: it breaks ties within one `at`, and it is
- * what makes the event id (`"<source>:<seq>"`) stable and unique across sources.
- */
-export interface ActivitySource {
-	/** Short, stable, unique within the log — it prefixes every id it produces. */
-	name: string;
-	/** The projection. See the column contract above. */
-	sql: string;
-	/** Bound in order, before any filter parameter. */
-	params?: readonly SQLQueryBindings[];
-	/**
-	 * A table this projection needs. When it does not exist the source is skipped
-	 * for that call rather than throwing — see property 3 in the header.
-	 */
-	requiresTable?: string;
-}
-
-/** Everything the stream can be narrowed by. Every field is optional; the empty
- *  query is "the newest `limit` events". */
-export interface ActivityQuery {
-	/** ISO-8601, inclusive. */
-	since?: string;
-	/** ISO-8601, inclusive. */
-	until?: string;
-	/** Case-insensitive exact match on the account. */
-	actor?: string;
-	/** Exact, or a PREFIX when it ends in a dot: `"share."` matches `share.grant`. */
-	action?: string;
-	kind?: string;
-	resourceId?: string;
-	outcome?: ActivityOutcome;
-	/** Restrict to one contributor (`"log"`, `"share"`). */
-	source?: string;
-	/** Substring, over actor + action + note + resource id. */
-	q?: string;
-	/** Default 100, hard cap 1000. */
-	limit?: number;
-}
-
-export interface ActivityPage {
-	events: ActivityEvent[];
-	/** The limit actually applied, after defaulting and capping. */
-	limit: number;
-	/** There is at least one more row older than the last one returned. */
-	truncated: boolean;
-}
-
-/** One dimension of "what is in this result", for the filter UI. */
-export interface ActivityFacet {
-	value: string;
-	count: number;
-}
-
-export interface ActivityFacets {
-	actions: ActivityFacet[];
-	actors: ActivityFacet[];
-	kinds: ActivityFacet[];
-}
+import { ACTIVITY_MIGRATIONS } from "./migrations.js";
+import {
+	ACTIVITY_SQL,
+	type ActivityFacet,
+	type ActivityFacets,
+	type ActivityPage,
+	type ActivityQuery,
+	type ActivitySource,
+	buildStream,
+	facetLimit,
+	MAX_FACET,
+	pageLimit,
+	prepareRecord,
+	type StreamRow,
+	toStreamEvent,
+} from "./query.js";
+
+// The contract and the SQL have no driver in them, so `./d1.ts` (which runs on `workerd`)
+// shares them without reaching this file. Re-exported so `cursedbelt-server/activity` is
+// unchanged.
+export type {
+	ActivityBindable,
+	ActivityFacet,
+	ActivityFacets,
+	ActivityPage,
+	ActivityQuery,
+	ActivitySource,
+} from "./query.js";
 
 export interface ActivityLogOptions {
 	db: Database;
@@ -136,59 +92,6 @@ export interface ActivityLog {
 	sources(): string[];
 }
 
-interface StreamRow {
-	source: string;
-	seq: number;
-	at: string;
-	app: string;
-	actor: string;
-	action: string;
-	kind: string | null;
-	resource_id: string | null;
-	outcome: ActivityOutcome;
-	note: string;
-	data: string | null;
-}
-
-const DEFAULT_LIMIT = 100;
-const MAX_LIMIT = 1000;
-const MAX_FACET = 25;
-
-/** `%` and `_` are wildcards and `\` is the escape — a note containing any of
- *  them must still search literally. */
-const likeContains = (value: string): string =>
-	`%${value.replace(/[\\%_]/g, (ch) => `\\${ch}`)}%`;
-
-const toEvent = (row: StreamRow): ActivityEvent => ({
-	id: `${row.source}:${row.seq}`,
-	source: row.source,
-	at: row.at,
-	app: row.app,
-	actor: row.actor,
-	action: row.action,
-	resource:
-		row.kind === null || row.resource_id === null
-			? null
-			: { app: row.app, kind: row.kind, id: row.resource_id },
-	outcome: isActivityOutcome(row.outcome) ? row.outcome : "ok",
-	note: row.note ?? "",
-	data: parseData(row.data),
-});
-
-/** A row whose `data` is unreadable must not take the page down with it — the
- *  event still says who did what, which is the part that matters. */
-function parseData(raw: string | null): Record<string, unknown> | null {
-	if (!raw) return null;
-	try {
-		const parsed = JSON.parse(raw) as unknown;
-		return parsed && typeof parsed === "object" && !Array.isArray(parsed)
-			? (parsed as Record<string, unknown>)
-			: { value: parsed };
-	} catch {
-		return { unparsed: raw };
-	}
-}
-
 export function createActivityLog(options: ActivityLogOptions): ActivityLog {
 	const { db, app } = options;
 	const now = options.now ?? (() => new Date().toISOString());
@@ -196,146 +99,37 @@ export function createActivityLog(options: ActivityLogOptions): ActivityLog {
 
 	runMigrations(db, ACTIVITY_MIGRATIONS);
 
-	const insert = db.query(
-		`INSERT INTO ${ACTIVITY_EVENTS_TABLE}
-		   (at, app, actor, action, kind, resource_id, outcome, note, data)
-		 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-	);
+	const insert = db.query(ACTIVITY_SQL.insert);
 
-	const tableExists = (name: string): boolean =>
-		!!db
-			.query("SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = ?")
-			.get(name);
+	const tableExists = (name: string): boolean => !!db.query(ACTIVITY_SQL.tableExists).get(name);
 
 	/** The contributors that can actually answer right now. */
 	const liveSources = (): ActivitySource[] =>
 		registered.filter((source) => !source.requiresTable || tableExists(source.requiresTable));
 
-	const OWN_SELECT = `SELECT 'log' AS source, event_id AS seq, at, app, actor, action,
-	                           kind, resource_id, outcome, note, data
-	                      FROM ${ACTIVITY_EVENTS_TABLE}`;
-
-	/** The union, plus the WHERE the caller asked for, plus their parameters in
-	 *  bind order (sources first — they are inside the FROM). */
-	const stream = (query: ActivityQuery): { sql: string; params: SQLQueryBindings[] } => {
-		const params: SQLQueryBindings[] = [];
-		const selects = [OWN_SELECT];
-		for (const source of liveSources()) {
-			if (query.source && query.source !== source.name) continue;
-			selects.push(source.sql);
-			params.push(...(source.params ?? []));
-		}
-		const where: string[] = [];
-		if (query.source) {
-			where.push("source = ?");
-			params.push(query.source);
-		}
-		if (query.since) {
-			where.push("at >= ?");
-			params.push(query.since);
-		}
-		if (query.until) {
-			where.push("at <= ?");
-			params.push(query.until);
-		}
-		if (query.actor) {
-			where.push("LOWER(actor) = LOWER(?)");
-			params.push(query.actor);
-		}
-		if (query.action) {
-			if (query.action.endsWith(".")) {
-				where.push("action LIKE ? ESCAPE '\\'");
-				params.push(`${query.action.replace(/[\\%_]/g, (ch) => `\\${ch}`)}%`);
-			} else {
-				where.push("action = ?");
-				params.push(query.action);
-			}
-		}
-		if (query.kind) {
-			where.push("kind = ?");
-			params.push(query.kind);
-		}
-		if (query.resourceId) {
-			where.push("resource_id = ?");
-			params.push(query.resourceId);
-		}
-		if (query.outcome) {
-			where.push("outcome = ?");
-			params.push(query.outcome);
-		}
-		if (query.q) {
-			where.push(
-				"(actor LIKE ? ESCAPE '\\' OR action LIKE ? ESCAPE '\\' OR note LIKE ? ESCAPE '\\' OR IFNULL(resource_id, '') LIKE ? ESCAPE '\\')",
-			);
-			const like = likeContains(query.q);
-			params.push(like, like, like, like);
-		}
-		// `query.source` naming a contributor that is not live leaves only the own
-		// select, filtered to a source it cannot be — an empty result, which is the
-		// honest answer rather than every row.
-		const sql = `SELECT * FROM (${selects.join("\n UNION ALL\n")}) AS stream${
-			where.length ? ` WHERE ${where.join(" AND ")}` : ""
-		}`;
-		return { sql, params };
-	};
+	const stream = (query: ActivityQuery): { sql: string; params: SQLQueryBindings[] } =>
+		buildStream(query, liveSources());
 
 	return {
 		app,
 
 		record(input: ActivityEventInput): ActivityEvent {
-			if (!isActivityAction(input.action)) {
-				throw new Error(
-					`activity: "${input.action}" is not an action — use lowercase subject.verb, e.g. "backup.delete"`,
-				);
-			}
-			const actor = input.actor?.trim();
-			if (!actor) {
-				// An event that cannot say who is a log line, not an audit trail.
-				throw new Error(`activity: ${input.action} has no actor`);
-			}
-			const at = input.at ?? now();
-			const outcome = input.outcome ?? "ok";
-			const data = input.data ? JSON.stringify(input.data) : null;
-			const info = insert.run(
-				at,
-				app,
-				actor,
-				input.action,
-				input.resource?.kind ?? null,
-				input.resource?.id ?? null,
-				outcome,
-				input.note ?? "",
-				data,
-			);
-			return {
-				id: `log:${Number(info.lastInsertRowid)}`,
-				source: "log",
-				at,
-				app,
-				actor,
-				action: input.action,
-				resource: input.resource ? { app, ...input.resource } : null,
-				outcome,
-				note: input.note ?? "",
-				data: input.data ?? null,
-			};
+			const { binds, event } = prepareRecord(app, input, now);
+			const info = insert.run(...binds);
+			return { id: `log:${Number(info.lastInsertRowid)}`, ...event };
 		},
 
 		query(query: ActivityQuery = {}): ActivityPage {
-			const limit = Math.min(MAX_LIMIT, Math.max(1, Math.floor(query.limit ?? DEFAULT_LIMIT)));
+			const limit = pageLimit(query.limit);
 			const { sql, params } = stream(query);
-			// One row past the limit, so "there is more" is a fact rather than a
-			// guess made from `rows.length === limit`.
-			const rows = db
-				.query(`${sql} ORDER BY at DESC, seq DESC LIMIT ?`)
-				.all(...params, limit + 1) as StreamRow[];
+			const rows = db.query(ACTIVITY_SQL.page(sql)).all(...params, limit + 1) as StreamRow[];
 			const truncated = rows.length > limit;
-			return { events: rows.slice(0, limit).map(toEvent), limit, truncated };
+			return { events: rows.slice(0, limit).map(toStreamEvent), limit, truncated };
 		},
 
 		count(query: ActivityQuery = {}): number {
 			const { sql, params } = stream(query);
-			const row = db.query(`SELECT COUNT(*) AS n FROM (${sql})`).get(...params) as {
+			const row = db.query(ACTIVITY_SQL.count(sql)).get(...params) as {
 				n: number;
 			} | null;
 			return row?.n ?? 0;
@@ -343,17 +137,11 @@ export function createActivityLog(options: ActivityLogOptions): ActivityLog {
 
 		facets(query: ActivityQuery = {}, limit = MAX_FACET): ActivityFacets {
 			const { sql, params } = stream(query);
-			const capped = Math.min(MAX_FACET, Math.max(1, Math.floor(limit)));
-			const dimension = (column: string): ActivityFacet[] =>
-				(
-					db
-						.query(
-							`SELECT ${column} AS value, COUNT(*) AS count FROM (${sql})
-							  WHERE ${column} IS NOT NULL AND ${column} <> ''
-							  GROUP BY ${column} ORDER BY count DESC, value ASC LIMIT ?`,
-						)
-						.all(...params, capped) as { value: string; count: number }[]
-				).map((row) => ({ value: row.value, count: row.count }));
+			const capped = facetLimit(limit);
+			const dimension = (column: "action" | "actor" | "kind"): ActivityFacet[] =>
+				(db.query(ACTIVITY_SQL.facet(sql, column)).all(...params, capped) as { value: string; count: number }[]).map(
+					(row) => ({ value: row.value, count: row.count }),
+				);
 			return {
 				actions: dimension("action"),
 				actors: dimension("actor"),

package/src/server/metrics/metricsBuffer.ts

@@ -1,4 +1,5 @@
 import type { Database } from 'bun:sqlite';
+import { PLACE_COLUMNS } from './requestPlace.js';
 
 /**
  * In-memory, batched writer for `request_metrics`. To keep request latency
@@ -30,8 +31,10 @@ export interface MetricRow {
   city?: string | null;
 }
 
-/** The optional place columns, in insert order. Each is written only when the table HAS it. */
-export const PLACE_COLUMNS = ['country', 'region', 'city'] as const;
+// Every place column is a `MetricRow` field — a column declared in `requestPlace.ts` that this
+// row cannot carry fails to compile here rather than inserting `undefined` for ever.
+const _placeColumnsAreRowFields: readonly (keyof MetricRow)[] = PLACE_COLUMNS;
+void _placeColumnsAreRowFields;
 
 export interface MetricsBufferOptions {
   db: Database;

package/src/server/metrics/requestPlace.ts

@@ -33,6 +33,15 @@ export interface RequestPlace {
 /** Place one request, or `null` when it cannot. Must not throw — but a throw is caught and read as `null`. */
 export type RequestLocator = (request: Request) => RequestPlace | null;
 
+/**
+ * The place columns, in insert order — the ONE declaration every reader and migration uses
+ * (`metricsBuffer` writes them, an app's reader allows them, an app's migration adds them).
+ * Exported through `cursedbelt-server/telemetry` since 4.29.0 (task 2141): until then it was
+ * private to `metricsBuffer.ts`, so flix, station and desk each declared their own copy, and a
+ * fourth column added here would have been written by the buffer and dropped by all three.
+ */
+export const PLACE_COLUMNS = ['country', 'region', 'city'] as const;
+
 /** Each place field is capped so three of them can never crowd the 5120-byte Analytics Engine blob budget. */
 export const PLACE_FIELD_MAX_CHARS = 96;
 

package/src/server/notifications/d1.spec.ts

@@ -0,0 +1,179 @@
+/**
+ * The D1 notification service is the sync one over D1 — the same table, the same rows for the
+ * same notifications, the same routes minus the live stream (task 393, `apps/family`).
+ *
+ * Parity is proven at the STORE seam, which is the only thing that changed: cwip's own center
+ * drives cwip's sqlite store and the D1 store through one scenario, with one clock and one id
+ * sequence, and every result and every row is compared. It runs over the local driver and over
+ * the fake D1 binding, which refuses `undefined` binds exactly as D1 does.
+ */
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+import {
+	createNotificationCenter,
+	createSqliteNotificationStore,
+	type NotificationCenter,
+	type NotificationDb,
+} from "cwip/notifications";
+import { Hono } from "hono";
+import { createFakeD1Binding } from "../d1/fakeD1.js";
+import { createLocalD1 } from "../d1/local.js";
+import { createRemoteD1 } from "../d1/remote.js";
+import type { D1LikeDatabase } from "../d1/types.js";
+import {
+	createD1NotificationService,
+	createD1NotificationStore,
+	NOTIFICATIONS_DDL,
+	notificationsDdl,
+} from "./d1.js";
+import { NOTIFICATION_MIGRATIONS } from "./migrations.js";
+
+const DRIVERS: Record<string, (sqlite: Database) => D1LikeDatabase> = {
+	local: (sqlite) => createLocalD1(sqlite),
+	"remote (fake D1 binding)": (sqlite) => createRemoteD1(createFakeD1Binding(sqlite)),
+};
+const local = DRIVERS.local as (sqlite: Database) => D1LikeDatabase;
+
+const withDdl = (ddl: readonly string[] = NOTIFICATIONS_DDL): Database => {
+	const sqlite = new Database(":memory:");
+	for (const statement of ddl) sqlite.run(statement);
+	return sqlite;
+};
+
+/** A clock that ticks 1 ms per call and a sequential id — identical for both centers. */
+const seams = () => {
+	let t = 1_000;
+	let n = 0;
+	return { now: () => t++, newId: () => `n${++n}`, onError: (error: unknown) => { throw error; } };
+};
+
+const SCENARIO: ReadonlyArray<(c: NotificationCenter) => Promise<unknown>> = [
+	(c) => c.notify("u1", { type: "share.grant", title: "Shared with you", body: "Grandma", link: "/people/p1" }),
+	(c) => c.notify("u1", { type: "share.grant", title: "Two things", groupKey: "g1", severity: "warning" }),
+	(c) => c.notify("u1", { type: "share.grant", title: "Three things", groupKey: "g1" }),
+	(c) => c.notify("u2", { type: "auth.login", title: "New sign-in", source: "auth", actorId: "a1", actorName: "Ann", resourceType: "session", resourceId: "s1" }),
+	(c) => c.notify("u1", { type: "plain", title: "A later row" }),
+	(c) => c.list({ userId: "u1" }),
+	(c) => c.list({ userId: "u1", limit: 1 }),
+	(c) => c.list({ userId: "u1", limit: 1, offset: 1 }),
+	(c) => c.unreadCount("u1"),
+	(c) => c.get("u1", "n1"),
+	(c) => c.get("u2", "n1"),
+	(c) => c.markRead("u1", "n1", true),
+	(c) => c.markRead("u1", "n1", true),
+	(c) => c.list({ userId: "u1", filter: "unread" }),
+	(c) => c.markAllRead("u1"),
+	(c) => c.markRead("u1", "n1", false),
+	(c) => c.setDone("u1", "n2", true),
+	(c) => c.list({ userId: "u1", filter: "done" }),
+	(c) => c.notify("u1", { type: "share.grant", title: "g1 again, after done", groupKey: "g1" }),
+	(c) => c.doneAll("u2"),
+	(c) => c.remove("u1", "n4"),
+	(c) => c.remove("u1", "nothing"),
+	(c) => c.pruneDone(0),
+	(c) => c.list({ userId: "u2", filter: "all" }),
+	(c) => c.unreadCount("nobody"),
+];
+
+describe("cursedbelt-server/notifications/d1", () => {
+	for (const [name, driver] of Object.entries(DRIVERS)) {
+		test(`🔴 parity over the ${name} driver: every result and every row match cwip's sqlite store`, async () => {
+			const syncDb = new Database(":memory:");
+			const sync = createNotificationCenter({
+				store: createSqliteNotificationStore(syncDb as unknown as NotificationDb),
+				...seams(),
+			});
+			const d1Sqlite = withDdl();
+			const d1 = createNotificationCenter({ store: createD1NotificationStore(driver(d1Sqlite)), ...seams() });
+			const results: unknown[] = [];
+			for (const [index, step] of SCENARIO.entries()) {
+				const expected = await step(sync);
+				const actual = await step(d1);
+				expect({ step: index, result: actual }).toEqual({ step: index, result: expected });
+				results.push(actual);
+			}
+			const table = (db: Database) => db.query("SELECT * FROM notifications ORDER BY rowid").all();
+			expect(table(d1Sqlite)).toEqual(table(syncDb));
+			// Not vacuous: a group coalesced to a count of 2, the prune removed a done row, and
+			// rows survived it.
+			expect(JSON.stringify(results)).toContain('"count":2');
+			expect(results[SCENARIO.length - 3]).toBeGreaterThan(0);
+			expect(table(d1Sqlite).length).toBeGreaterThan(1);
+		});
+	}
+
+	test("the DDL builds exactly the table cwip's store and NOTIFICATION_MIGRATIONS build", () => {
+		const shape = (db: Database) =>
+			(
+				db
+					.query("SELECT type, name, sql FROM sqlite_master WHERE tbl_name = 'notifications' AND sql IS NOT NULL ORDER BY name")
+					.all() as { type: string; name: string; sql: string }[]
+			).map((r) => ({ ...r, sql: r.sql.replace(/\s+/g, " ").replace("IF NOT EXISTS ", "").trim() }));
+		const viaStore = new Database(":memory:");
+		createSqliteNotificationStore(viaStore as unknown as NotificationDb);
+		const viaMigration = new Database(":memory:");
+		for (const migration of NOTIFICATION_MIGRATIONS) migration.up(viaMigration);
+		expect(shape(withDdl())).toEqual(shape(viaStore));
+		expect(shape(withDdl())).toEqual(shape(viaMigration));
+		expect(shape(viaStore).length).toBe(3);
+		// A renamed table carries its name into both indexes.
+		expect(notificationsDdl("inbox").join("\n")).toContain("idx_inbox_group ON inbox(");
+	});
+
+	test("🔴 a table name is an identifier or it is refused — it is interpolated into SQL", () => {
+		expect(() => notificationsDdl("notifications; DROP TABLE x")).toThrow(/invalid table identifier/);
+		expect(() => createD1NotificationStore(local(withDdl()), { table: "a b" })).toThrow(/invalid table identifier/);
+	});
+
+	test("🔴 coalescing is enforced by the index on D1 too — a second open row in a group is refused", async () => {
+		const store = createD1NotificationStore(local(withDdl()));
+		const row = (id: string) => ({
+			id,
+			userId: "u1",
+			createdAt: 0,
+			updatedAt: 0,
+			type: "t",
+			severity: "info" as const,
+			title: "x",
+			groupKey: "g",
+			count: 1,
+			read: false,
+			done: false,
+		});
+		await store.insert(row("a"));
+		await expect(store.insert(row("b"))).rejects.toThrow(/UNIQUE constraint/);
+		expect(await store.unreadCount("u1")).toBe(1);
+	});
+
+	test("🔴 runs no DDL: on a database without the table it creates nothing and fails loudly", async () => {
+		const sqlite = new Database(":memory:");
+		const service = createD1NotificationService({ db: local(sqlite) });
+		expect(sqlite.query("SELECT COUNT(*) AS n FROM sqlite_master").get()).toEqual({ n: 0 });
+		await expect(service.store.unreadCount("u1")).rejects.toThrow(/no such table/);
+	});
+
+	test("the service mounts the inbox routes, user-scoped, with no hub and no stream route", async () => {
+		let who: string | null = "u1";
+		const service = createD1NotificationService({ db: local(withDdl()), onError: () => {} });
+		expect(service.hub).toBeNull();
+		const app = new Hono();
+		service.mount(app, { authorize: () => who });
+		await service.center.notify("u1", { type: "share.grant", title: "Shared with you", link: "/people/p1" });
+		await service.center.notify("u2", { type: "share.grant", title: "Not yours" });
+
+		const list = (await (await app.request("/api/notifications")).json()) as { items: { id: string; title: string }[] };
+		expect(list.items.map((n) => n.title)).toEqual(["Shared with you"]);
+		expect(await (await app.request("/api/notifications/unread-count")).json()).toEqual({ count: 1 });
+		const id = list.items[0]?.id as string;
+		expect(await (await app.request(`/api/notifications/${id}/read`, { method: "POST" })).json()).toEqual({ changed: true });
+		expect(await (await app.request("/api/notifications/unread-count")).json()).toEqual({ count: 0 });
+		// u2's row cannot be reached by u1's session, even by id.
+		const u2 = (await service.center.list({ userId: "u2" })).items[0]?.id as string;
+		expect((await app.request(`/api/notifications/${u2}`)).status).toBe(404);
+		// No hub, so no stream: the client's poll transport falls back from the 404.
+		expect((await app.request("/api/notifications/stream")).status).not.toBe(200);
+		who = null;
+		expect((await app.request("/api/notifications")).status).toBe(401);
+		service.close();
+	});
+});