cursedbelt-server 1.0.0

package/src/server/retention/store.spec.ts

@@ -0,0 +1,236 @@
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { resolveAppDataHome } from "../paths/appDataHome";
+import type { RetentionDeclaration } from "./policy";
+import {
+	publishRetention,
+	readRetentionOverrides,
+	readRetentionRegistry,
+	resolveRetentionHome,
+	setRetentionHomeForTests,
+	writeRetentionOverride,
+} from "./store";
+
+const decl = (app: string, id: string): RetentionDeclaration => ({
+	app,
+	id,
+	label: id,
+	category: "run-log",
+	unit: "rows",
+	location: `${app}.sqlite · ${id}`,
+	consequence: "past run outcomes",
+});
+
+let home: string;
+
+beforeEach(() => {
+	home = mkdtempSync(join(tmpdir(), "retention-home-"));
+	setRetentionHomeForTests(home);
+});
+afterEach(() => {
+	setRetentionHomeForTests(null);
+	rmSync(home, { recursive: true, force: true });
+});
+
+describe("publications — one file per app", () => {
+	test("an app publishes and can be read back", () => {
+		expect(
+			publishRetention({ app: "orch", resources: [{ declaration: decl("orch", "events") }] }),
+		).toBe(true);
+		const registry = readRetentionRegistry(home);
+		expect(registry).toHaveLength(1);
+		expect(registry[0]?.app).toBe("orch");
+		expect(registry[0]?.resources[0]?.declaration.id).toBe("events");
+		expect(registry[0]?.publishedAt).toBeGreaterThan(0);
+	});
+
+	test("two apps publishing concurrently cannot lose each other's records", () => {
+		// The reason this is one file per app: a shared registry that concurrent
+		// writers read-modify-write loses records, which is the exact bug the agent
+		// coordination board is one-file-per-record to avoid.
+		publishRetention({ app: "orch", resources: [{ declaration: decl("orch", "events") }] });
+		publishRetention({ app: "notes", resources: [{ declaration: decl("notes", "ingest-runs") }] });
+		expect(
+			readRetentionRegistry(home)
+				.map((p) => p.app)
+				.sort(),
+		).toEqual(["notes", "orch"]);
+	});
+
+	test("republishing replaces only that app's file", () => {
+		publishRetention({ app: "orch", resources: [{ declaration: decl("orch", "events") }] });
+		publishRetention({ app: "notes", resources: [{ declaration: decl("notes", "ingest-runs") }] });
+		publishRetention({ app: "orch", resources: [{ declaration: decl("orch", "job-runs") }] });
+		const orch = readRetentionRegistry(home).find((p) => p.app === "orch");
+		expect(orch?.resources.map((r) => r.declaration.id)).toEqual(["job-runs"]);
+		expect(readRetentionRegistry(home)).toHaveLength(2);
+	});
+
+	test("an app key that is not a safe file name is refused", () => {
+		expect(publishRetention({ app: "../escape", resources: [] })).toBe(false);
+		expect(readRetentionRegistry(home)).toHaveLength(0);
+	});
+
+	test("a corrupt publication is skipped, never fatal to the whole view", () => {
+		publishRetention({ app: "orch", resources: [{ declaration: decl("orch", "events") }] });
+		writeFileSync(join(home, "apps", "broken.json"), "{ this is not json");
+		expect(readRetentionRegistry(home).map((p) => p.app)).toEqual(["orch"]);
+	});
+
+	test("publications carry the last sweep so the UI can show it while the app is down", () => {
+		publishRetention({
+			app: "orch",
+			instance: "local",
+			resources: [
+				{
+					declaration: decl("orch", "events"),
+					measurement: { items: 3000, bytes: 4096 },
+					lastSweep: { at: 123, aged: 12, overflow: 0 },
+				},
+			],
+		});
+		const row = readRetentionRegistry(home)[0]?.resources[0];
+		expect(row?.measurement).toEqual({ items: 3000, bytes: 4096 });
+		expect(row?.lastSweep).toEqual({ at: 123, aged: 12, overflow: 0 });
+	});
+});
+
+describe("overrides — the single-writer file", () => {
+	test("a write round-trips and stamps who and when", () => {
+		writeRetentionOverride("orch", "events", { days: 30 }, { home, updatedBy: "curt", now: 5 });
+		const file = readRetentionOverrides(home);
+		expect(file.overrides["orch:events"]).toMatchObject({
+			days: 30,
+			updatedBy: "curt",
+			updatedAt: 5,
+		});
+		expect(file.updatedAt).toBe(5);
+	});
+
+	test("editing one resource never loses another's setting", () => {
+		writeRetentionOverride("orch", "events", { days: 30 }, { home });
+		writeRetentionOverride("notes", "ingest-runs", { days: 3 }, { home });
+		const file = readRetentionOverrides(home);
+		expect(file.overrides["orch:events"]?.days).toBe(30);
+		expect(file.overrides["notes:ingest-runs"]?.days).toBe(3);
+	});
+
+	test("null CLEARS a field so the declaration's default becomes authoritative again", () => {
+		// Storing a value that merely equals today's default would freeze the
+		// resource at that number if the default ever moved.
+		writeRetentionOverride("orch", "events", { days: 30, maxItems: 500 }, { home });
+		writeRetentionOverride("orch", "events", { days: null }, { home });
+		expect(readRetentionOverrides(home).overrides["orch:events"]).toMatchObject({ maxItems: 500 });
+		expect(readRetentionOverrides(home).overrides["orch:events"]?.days).toBeUndefined();
+	});
+
+	test("a null patch removes the whole override — the UI's Reset", () => {
+		writeRetentionOverride("orch", "events", { days: 30 }, { home });
+		writeRetentionOverride("orch", "events", null, { home });
+		expect(readRetentionOverrides(home).overrides["orch:events"]).toBeUndefined();
+	});
+
+	test("an override left with nothing meaningful in it is dropped, not stored empty", () => {
+		writeRetentionOverride("orch", "events", { days: 30 }, { home });
+		writeRetentionOverride("orch", "events", { days: null }, { home });
+		expect(readRetentionOverrides(home).overrides["orch:events"]).toBeUndefined();
+	});
+
+	test("garbage values are ignored rather than written through to a sweep", () => {
+		writeRetentionOverride("orch", "events", { days: "thirty", maxItems: Number.NaN }, { home });
+		expect(readRetentionOverrides(home).overrides["orch:events"]).toBeUndefined();
+	});
+
+	test("enabled:false is storable and distinct from an absent value", () => {
+		writeRetentionOverride("orch", "events", { enabled: false }, { home });
+		expect(readRetentionOverrides(home).overrides["orch:events"]?.enabled).toBe(false);
+	});
+
+	test("a corrupt overrides file reads as no overrides — never a boot failure", () => {
+		writeFileSync(join(home, "overrides.json"), "}{");
+		expect(readRetentionOverrides(home).overrides).toEqual({});
+	});
+
+	test("the file is written whole and valid — a reader never sees half of it", () => {
+		writeRetentionOverride("orch", "events", { days: 30, note: "keeping a month" }, { home });
+		const raw = readFileSync(join(home, "overrides.json"), "utf8");
+		expect(() => JSON.parse(raw)).not.toThrow();
+		expect(raw.endsWith("\n")).toBe(true);
+	});
+});
+
+describe("resolveRetentionHome", () => {
+	afterEach(() => setRetentionHomeForTests(home));
+
+	test("RETENTION_HOME wins outright", () => {
+		setRetentionHomeForTests(null);
+		const explicit = join(home, "explicit");
+		expect(resolveRetentionHome({ RETENTION_HOME: explicit })).toBe(explicit);
+	});
+
+	test("APP_DATA_HOME — the one env var — is honored before any convention", () => {
+		setRetentionHomeForTests(null);
+		expect(resolveRetentionHome({ APP_DATA_HOME: home })).toBe(join(home, "retention"));
+	});
+
+	test("a configured home is used even before it exists — naming it is the statement", () => {
+		setRetentionHomeForTests(null);
+		const fresh = join(home, "not-yet");
+		expect(resolveRetentionHome({ APP_DATA_HOME: fresh, APP_DATA_DIR: join(home, "data") })).toBe(
+			join(fresh, "retention"),
+		);
+	});
+
+	test("the shared OS-convention home wins over APP_DATA_DIR so the station sees ONE fleet", () => {
+		// An app's own APP_DATA_DIR is private to it; publishing there instead of the
+		// shared home would hide it from every other app's view of the fleet.
+		setRetentionHomeForTests(null);
+		const conventional = resolveAppDataHome({ env: { HOME: home } });
+		mkdirSync(conventional, { recursive: true });
+		expect(resolveRetentionHome({ HOME: home, APP_DATA_DIR: join(home, "data") })).toBe(
+			join(conventional, "retention"),
+		);
+	});
+
+	test("a deployed box — no shared data home — falls through to APP_DATA_DIR", () => {
+		// prod-ec2 has no shared data home, and under ProtectSystem=strict only the
+		// data dir is writable. The app still sweeps; its fleet is that box's.
+		setRetentionHomeForTests(null);
+		const dataDir = join(home, "data");
+		expect(
+			resolveRetentionHome({ HOME: join(home, "nonexistent-home"), APP_DATA_DIR: dataDir }),
+		).toBe(join(dataDir, "retention"));
+	});
+
+	test("an unwritable candidate is probed and skipped, never assumed", () => {
+		setRetentionHomeForTests(null);
+		const dataDir = join(home, "data2");
+		expect(
+			resolveRetentionHome({
+				APP_DATA_HOME: "/proc/definitely-not-writable",
+				HOME: join(home, "nonexistent-home"),
+				APP_DATA_DIR: dataDir,
+			}),
+		).toBe(join(dataDir, "retention"));
+	});
+
+	/**
+	 * 🔴 The published defect, asserted on the OUTPUT rather than on the source: up
+	 * to 2026-09-13 this resolved a hardcoded dotfolder in the owner's home, so
+	 * every installer of `cursedbelt` got a library reaching into a directory that
+	 * had nothing to do with them.
+	 */
+	test("resolves nothing from a previous generation, whatever the env says", () => {
+		for (const env of [
+			{ HOME: join(home, "nonexistent-home") },
+			{ HOME: join(home, "nonexistent-home"), APP_MANAGED_HOME: "/nowhere/inherited" },
+		]) {
+			setRetentionHomeForTests(null);
+			const resolved = resolveRetentionHome(env);
+			expect(resolved).not.toContain("app-managed");
+			expect(resolved).not.toContain("/nowhere/inherited");
+		}
+	});
+});

package/src/server/retention/store.ts

@@ -0,0 +1,338 @@
+/**
+ * The fleet's retention SOURCE OF TRUTH on disk — where declarations are
+ * published and where the owner's overrides are read from.
+ *
+ * ── Why a file and not an API ───────────────────────────────────────────────
+ * The station has to show, and let the owner change, the policy for apps that
+ * are not running. Most satellites are started on demand; `orch` is a daemon but
+ * `notes`, `roms`, `vault` and the rest are usually down, and the whole point of
+ * the UI is to answer "what is keeping my disk full" without booting anything.
+ * An HTTP endpoint per app could not answer that, would need an auth model per
+ * app, and would make a stopped app look policy-less rather than idle.
+ *
+ * So an app PUBLISHES its declarations (a description of what it owns, plus what
+ * it last measured and swept) and READS overrides. The station reads every
+ * publication and is the only writer of overrides. Nothing has to be up for the
+ * owner to see the fleet, and no app parses another app's database.
+ *
+ * ── One file per app, deliberately ──────────────────────────────────────────
+ * `apps/<app>.json`, never one shared registry file. Several satellites boot at
+ * once (the station's start-all, a `bun run dev` fan-out, launchd at login), and
+ * a shared file that concurrent writers read-modify-write loses records — the
+ * exact bug the agent-coordination board is one-file-per-record to avoid. Each
+ * app owns exactly one path and can only ever clobber itself.
+ *
+ * `overrides.json` IS a single file, because it has a single writer (the
+ * station) and because the owner's settings for the whole fleet are one thing to
+ * back up, copy to a box, or read at a glance.
+ *
+ * ── Where it lives ──────────────────────────────────────────────────────────
+ * `<app-data-home>/retention` — one root shared by every local app, so the
+ * station sees one fleet. The home is CONFIGURATION ({@link resolveAppDataHome}):
+ * `$APP_DATA_HOME` when a machine keeps every app's data under one root, else the
+ * OS's own convention (`$XDG_DATA_HOME`, `~/Library/Application Support`).
+ *
+ * 🔴 It was `~/<a private dotfolder>` until 2026-09-13, hardcoded, in a PUBLISHED
+ * package — so anyone who installed cursedbelt got a library hunting for one
+ * person's directory layout. A library may know what its caller told it and what
+ * the OS says; it may not know the name of a folder in somebody's home.
+ *
+ * A deployed box under `ProtectSystem=strict` has only its own data dir writable,
+ * so it falls back to `$APP_DATA_DIR/retention`: the app still sweeps on the same
+ * code and the same defaults, and a station instance on that box reads that box's
+ * fleet. The resolution probes WRITABILITY rather than assuming, because the
+ * failure it prevents — an app whose every publish silently EROFS-es — is
+ * invisible until someone asks why the UI is empty.
+ */
+import {
+	existsSync,
+	mkdirSync,
+	readdirSync,
+	readFileSync,
+	renameSync,
+	rmSync,
+	writeFileSync,
+} from "node:fs";
+import { join } from "node:path";
+import { APP_DATA_HOME_ENV, resolveAppDataHome } from "../paths/appDataHome";
+import type { RetentionDeclaration, RetentionOverride } from "./policy";
+import { retentionKey } from "./policy";
+
+/** What one sweep of one resource did. Published so the UI can show it. */
+export interface RetentionSweepReport {
+	at: number;
+	/** Items removed because they aged past the cutoff. */
+	aged: number;
+	/** Items removed by the item-count backstop, liveness ignored. */
+	overflow: number;
+	/**
+	 * Items the sweep deliberately left alone because they are still live — a
+	 * running worker's transcript, an unsettled task's trail. Reported so a
+	 * resource that stays large has an explanation rather than looking like a
+	 * broken sweep.
+	 */
+	protectedItems?: number;
+	bytesFreed?: number;
+	/**
+	 * Items written to the archive before they were deleted — the `archived-history`
+	 * category's receipt. Reported so the station can show "moved" rather than
+	 * "removed": for an archiving resource those two words are the whole difference,
+	 * and a UI that said the second would be describing data loss that did not happen.
+	 */
+	archived?: number;
+	/** Bytes those archived items occupy in the archive. */
+	archivedBytes?: number;
+	/**
+	 * Why this sweep deliberately removed nothing, when it otherwise would have.
+	 * An archiving resource whose archive is not being collected REFUSES to trim,
+	 * and a resource that quietly stops working is indistinguishable from one with
+	 * nothing to do — so the reason is carried out to the UI rather than logged and
+	 * lost.
+	 */
+	blocked?: string;
+	error?: string;
+}
+
+/** A live measurement of a resource, taken at sweep time. */
+export interface RetentionMeasurement {
+	items: number;
+	bytes?: number;
+	/** Timestamp of the oldest item, when the target can report it cheaply. */
+	oldestAt?: number;
+}
+
+/** One resource as published by its owning app. */
+export interface PublishedResource {
+	declaration: RetentionDeclaration;
+	measurement?: RetentionMeasurement;
+	lastSweep?: RetentionSweepReport;
+}
+
+/** One app's publication file. */
+export interface RetentionPublication {
+	app: string;
+	version: 1;
+	publishedAt: number;
+	/** The instance that published — `local`, `prod`, `dev`; purely informational. */
+	instance?: string;
+	/** True when the process that published it has sweeping disabled entirely. */
+	sweepDisabled?: boolean;
+	resources: PublishedResource[];
+}
+
+export interface RetentionOverridesFile {
+	version: 1;
+	updatedAt: number;
+	/** Keyed by `<app>:<id>` — `retentionKey()`. */
+	overrides: Record<string, RetentionOverride>;
+}
+
+/**
+ * A FUNCTION, not a shared constant. `writeRetentionOverride` mutates what
+ * `readRetentionOverrides` hands it, so a module-level empty object would be
+ * aliased into every caller: the first write on a machine with no overrides file
+ * would edit the constant itself, and a later read of an unrelated home would
+ * return those overrides. Caught by `store.test.ts`.
+ */
+const emptyOverrides = (): RetentionOverridesFile => ({ version: 1, updatedAt: 0, overrides: {} });
+
+let cachedHome: string | null = null;
+
+/** Can we create and write inside this directory? Probed, never assumed. */
+function isUsable(dir: string): boolean {
+	try {
+		mkdirSync(dir, { recursive: true });
+		const probe = join(dir, ".write-probe");
+		writeFileSync(probe, "");
+		rmSync(probe, { force: true });
+		return true;
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Resolve the retention home. See the module note for the ordering rationale.
+ * Cached per process — it is consulted on every sweep and cannot change under a
+ * running app.
+ */
+export function resolveRetentionHome(
+	env: Record<string, string | undefined> = process.env,
+): string {
+	if (cachedHome) return cachedHome;
+	const candidates: string[] = [];
+	if (env.RETENTION_HOME) candidates.push(env.RETENTION_HOME);
+	// An explicitly CONFIGURED home is used whether or not it exists yet — the
+	// operator naming it is the statement that it is the right place.
+	const configured = env[APP_DATA_HOME_ENV]?.trim();
+	if (configured) candidates.push(join(configured, "retention"));
+	const conventional = resolveAppDataHome({ env });
+	// The OS convention is only PREFERRED when it is already there: on a deployed
+	// box creating it would either fail (read-only namespace) or strand a
+	// publication somewhere no station instance looks.
+	if (!configured && existsSync(conventional)) candidates.push(join(conventional, "retention"));
+	if (env.APP_DATA_DIR) candidates.push(join(env.APP_DATA_DIR, "retention"));
+	candidates.push(join(conventional, "retention"));
+
+	for (const candidate of candidates) {
+		if (isUsable(candidate)) {
+			cachedHome = candidate;
+			return candidate;
+		}
+	}
+	// Nothing is writable — return the conventional path anyway. Every writer here
+	// is best-effort and will log its failure; retention must not stop an app from
+	// booting.
+	cachedHome = join(conventional, "retention");
+	return cachedHome;
+}
+
+/** Test seam — point the home somewhere else for this process. */
+export function setRetentionHomeForTests(home: string | null): void {
+	cachedHome = home;
+}
+
+const appsDir = (home: string): string => join(home, "apps");
+const overridesPath = (home: string): string => join(home, "overrides.json");
+
+/** A file name cannot be derived from an app key without pinning the charset. */
+const SAFE_APP = /^[a-z0-9][a-z0-9-]*$/;
+
+/**
+ * Write whole, atomically: a reader must never see half a file. `rename` inside
+ * one directory is atomic on every filesystem this fleet runs on, and the temp
+ * name carries the pid so two writers cannot share a scratch path.
+ */
+function writeJsonAtomic(path: string, value: unknown): void {
+	const tmp = `${path}.${process.pid}.tmp`;
+	writeFileSync(tmp, `${JSON.stringify(value, null, 2)}\n`);
+	renameSync(tmp, path);
+}
+
+function readJson<T>(path: string, fallback: T): T {
+	try {
+		return JSON.parse(readFileSync(path, "utf8")) as T;
+	} catch {
+		// Absent is the normal first-run case; corrupt is a truncated write from a
+		// killed process. Both mean "no settings yet" — never a boot failure.
+		return fallback;
+	}
+}
+
+/**
+ * Publish one app's resources. Best-effort by contract: a retention publication
+ * that cannot be written is a missing row in a dashboard, and must never take
+ * down the app that owns the data.
+ */
+export function publishRetention(
+	publication: Omit<RetentionPublication, "version" | "publishedAt">,
+	options: { home?: string; log?: (line: string) => void } = {},
+): boolean {
+	const home = options.home ?? resolveRetentionHome();
+	if (!SAFE_APP.test(publication.app)) return false;
+	try {
+		mkdirSync(appsDir(home), { recursive: true });
+		writeJsonAtomic(join(appsDir(home), `${publication.app}.json`), {
+			version: 1,
+			publishedAt: Date.now(),
+			...publication,
+		} satisfies RetentionPublication);
+		return true;
+	} catch (error) {
+		options.log?.(
+			`⚠️  retention: could not publish ${publication.app} — ${error instanceof Error ? error.message : String(error)}`,
+		);
+		return false;
+	}
+}
+
+/** Every app's publication, newest-published first. */
+export function readRetentionRegistry(home = resolveRetentionHome()): RetentionPublication[] {
+	const dir = appsDir(home);
+	let names: string[];
+	try {
+		names = readdirSync(dir).filter((n) => n.endsWith(".json"));
+	} catch {
+		return [];
+	}
+	const publications: RetentionPublication[] = [];
+	for (const name of names) {
+		const parsed = readJson<RetentionPublication | null>(join(dir, name), null);
+		if (parsed && Array.isArray(parsed.resources) && typeof parsed.app === "string") {
+			publications.push(parsed);
+		}
+	}
+	return publications.sort((a, b) => (b.publishedAt ?? 0) - (a.publishedAt ?? 0));
+}
+
+export function readRetentionOverrides(home = resolveRetentionHome()): RetentionOverridesFile {
+	const file = readJson<RetentionOverridesFile>(overridesPath(home), emptyOverrides());
+	if (!file || typeof file !== "object" || typeof file.overrides !== "object" || !file.overrides) {
+		return emptyOverrides();
+	}
+	return { version: 1, updatedAt: file.updatedAt ?? 0, overrides: file.overrides };
+}
+
+/**
+ * Apply one override. Read-modify-write, so an unrelated resource's setting is
+ * never lost to a concurrent edit of this one; the station is the only writer,
+ * which is what keeps that honest.
+ *
+ * A patch field set to `undefined` is left alone; set to `null` it is CLEARED,
+ * which is how the UI's "reset to default" works — deleting the key is the only
+ * way to make the declaration's default authoritative again, since a stored
+ * value that merely equals the default would freeze the resource at today's
+ * number if the default ever moves.
+ */
+export function writeRetentionOverride(
+	app: string,
+	id: string,
+	patch: Partial<Record<"days" | "maxItems" | "maxBytes" | "enabled" | "note", unknown>> | null,
+	options: { home?: string; updatedBy?: string; now?: number } = {},
+): RetentionOverridesFile {
+	const home = options.home ?? resolveRetentionHome();
+	const now = options.now ?? Date.now();
+	const file = readRetentionOverrides(home);
+	const key = retentionKey(app, id);
+
+	if (patch === null) {
+		delete file.overrides[key];
+	} else {
+		const current: RetentionOverride = { ...(file.overrides[key] ?? {}) };
+		for (const field of ["days", "maxItems", "maxBytes"] as const) {
+			if (field in patch) {
+				const value = patch[field];
+				if (value === null) delete current[field];
+				else if (typeof value === "number" && Number.isFinite(value)) current[field] = value;
+			}
+		}
+		if ("enabled" in patch) {
+			const value = patch.enabled;
+			if (value === null) delete current.enabled;
+			else if (typeof value === "boolean") current.enabled = value;
+		}
+		if ("note" in patch) {
+			const value = patch.note;
+			if (value === null || value === "") delete current.note;
+			else if (typeof value === "string") current.note = value.slice(0, 500);
+		}
+		current.updatedAt = now;
+		if (options.updatedBy) current.updatedBy = options.updatedBy;
+		// An override with nothing left in it is not an override — drop the key so
+		// the resource reads as "default" rather than as a customized row that
+		// happens to change nothing.
+		const meaningful =
+			current.days !== undefined ||
+			current.maxItems !== undefined ||
+			current.maxBytes !== undefined ||
+			current.enabled !== undefined ||
+			current.note !== undefined;
+		if (meaningful) file.overrides[key] = current;
+		else delete file.overrides[key];
+	}
+
+	file.updatedAt = now;
+	mkdirSync(home, { recursive: true });
+	writeJsonAtomic(overridesPath(home), file);
+	return file;
+}