@coffre/vault 0.14.5 → 0.15.0

package/dist/node.js

@@ -1,9 +1,291 @@
-import { a as verifyCheckpoint, c as resolveVaultConfig, i as checkpointMessage, n as KekUnavailableError, r as awsKms, s as derivedSigningKey, t as KekBadClaimError } from "./src-Bg0ZeMJW.js";
-import { n as prepareVault, t as openVault } from "./vault-Cbi45kLB.js";
+import { a as verifyCheckpoint, c as resolveVaultConfig, i as checkpointMessage, n as KekUnavailableError, r as awsKms, s as derivedSigningKey, t as KekBadClaimError } from "./src-DheG92EM.js";
+import { _ as Unsealed, a as replayStored, b as sealingOf, c as openLogCheckpointer, f as Projector, g as TICK_BOUND_MS, h as Stalled, i as logStore, l as reasonOf, n as prepareVault, p as Fence, r as verifierKeys, s as daysAround, t as openVault, u as upkeep, x as within, y as heldIn } from "./vault-kmtojYnW.js";
 import { isUnavailable } from "@coffre/db/dialect";
 import { chmodSync, existsSync, lstatSync, rmSync } from "node:fs";
 import { Agent, createServer, request } from "node:http";
-import { openDatabase } from "@coffre/db/connect";
+import { engineOfUrl, openDatabase } from "@coffre/db/connect";
+import { appendFile, open, readFile, rename, stat, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+//#region src/checkpointer-node.ts
+/**
+* The 0.15 checkpointer on Node (docs/design/audit-tree.md): one process
+* seals the log every 500 ms, its memory a file beside the vault, synced
+* before the database has the pair it holds (RememberFirst). The vault's
+* process runs it (`keepCheckpointer`, node.ts). Nothing it awaits is
+* unbounded (T169): a tick past its bound stalls the checkpointer, and the
+* vault's process opens another.
+*/
+/** The signer's memory in a file: written whole, synced, renamed over the last, and the directory synced, so a power loss keeps it. */
+function fileMemory(path) {
+	return {
+		last: () => readFile(path, "utf8").catch((error) => {
+			if (error.code === "ENOENT") return null;
+			throw error;
+		}),
+		async remember(record) {
+			const file = await open(`${path}.part`, "w");
+			try {
+				await file.writeFile(record);
+				await file.sync();
+			} finally {
+				await file.close();
+			}
+			await rename(`${path}.part`, path);
+			const dir = await open(dirname(path), "r");
+			try {
+				await dir.sync();
+			} finally {
+				await dir.close();
+			}
+		}
+	};
+}
+/** How long the runner waits on an open, the next days of uses, or the log's upkeep after a tick (T169). */
+const OPEN_BOUND_MS = TICK_BOUND_MS;
+const UPKEEP_BOUND_MS = 6e4;
+/**
+* Opens the checkpointer over `database` (the vault's login) and ticks it
+* every `intervalMs`, a tick after the last one ends, each followed by the
+* log's upkeep (days made ahead, retention past `retainDays`, the bulk
+* cap's rows pruned). Each alarm is appended to `<memory>.alarms`, one JSON
+* line each. Given `connect` instead, over a session of its own, cut once
+* it stalls or stops, so that what it still awaits holds no connection of
+* the vault's (review r1).
+*/
+async function runCheckpointer(options) {
+	const alarmsPath = `${options.memory}.alarms`;
+	let appending = Promise.resolve();
+	if (options.connect === void 0 === (options.database === void 0)) throw new Error("the checkpointer runs over its database or a session it connects, one of the two");
+	const session = options.connect === void 0 ? null : await within(options.connect(), options.tickBoundMs ?? OPEN_BOUND_MS, "connecting");
+	const store = logStore(session?.db ?? options.database, { tileRange: options.tileRange });
+	const fence = new Fence();
+	const checkpointer = await within(store.makeDays(daysAround(Date.now())), options.tickBoundMs ?? OPEN_BOUND_MS, "making the next days of uses").then(() => within(openLogCheckpointer({
+		store,
+		seed: options.seed,
+		memory: (options.memoryOf ?? fileMemory)(options.memory),
+		witnesses: options.witnesses,
+		record: (alarm) => record(alarm),
+		fence,
+		...options.tickBoundMs === void 0 ? {} : { tickBoundMs: options.tickBoundMs }
+	}), options.tickBoundMs ?? OPEN_BOUND_MS, "the open")).catch(async (error) => {
+		await session?.cut().catch(() => {});
+		await fence.close("its open failed");
+		throw error;
+	});
+	/** In order, one after the other; a file grown past a megabyte keeps its last 30 days. */
+	function record(alarm) {
+		appending = appending.then(async () => {
+			await appendFile(alarmsPath, `${JSON.stringify(alarm)}\n`);
+			if ((await stat(alarmsPath)).size > 1e6) {
+				const kept = (await readAlarms(alarmsPath)).filter((a) => a.at >= alarm.at - 2592e6);
+				await writeFile(`${alarmsPath}.part`, kept.map((a) => `${JSON.stringify(a)}\n`).join(""));
+				await rename(`${alarmsPath}.part`, alarmsPath);
+			}
+		}).catch((error) => console.warn(`coffre audit: an alarm could not be kept: ${error.message}`));
+		return appending;
+	}
+	const keep = upkeep(checkpointer, fence.store(store), {
+		keepDays: options.retainDays ?? null,
+		seed: options.seed,
+		alert: (message) => {
+			console.warn(`coffre audit: ${message}`);
+			record({
+				at: Date.now(),
+				message
+			});
+		}
+	});
+	const keys = await verifierKeys([options.seed]);
+	const projector = new Projector(store, keys, (message) => {
+		console.warn(`coffre audit: ${message}`);
+		record({
+			at: Date.now(),
+			message
+		});
+	});
+	const interval = options.intervalMs ?? 500;
+	let stopped = false;
+	let timer;
+	let ticking = Promise.resolve();
+	let failed = null;
+	let last = null;
+	let stalled = false;
+	let upkeeping = null;
+	let retired = false;
+	const retire = (why) => {
+		if (retired) return;
+		retired = true;
+		options.stalled?.(why);
+	};
+	const upkeepOnce = async () => {
+		if (upkeeping !== null) return;
+		const run = upkeeping = keep().finally(() => {
+			if (upkeeping === run) upkeeping = null;
+		});
+		await within(run, options.upkeepBoundMs ?? UPKEEP_BOUND_MS, "the log's upkeep").catch(async (error) => {
+			const message = `a checkpoint failed: ${error.message}`;
+			console.warn(`coffre audit: ${message}`);
+			failed = message;
+			stalled = true;
+			clearTimeout(timer);
+			await record({
+				at: Date.now(),
+				message
+			});
+			ticking.finally(() => retire(message));
+		});
+	};
+	let more = false;
+	const loop = () => {
+		ticking = checkpointer.tick().then((t) => {
+			failed = null;
+			last = {
+				at: Date.now(),
+				failures: t.failures
+			};
+			projector.kick();
+			more = t.more;
+			if (!more) upkeepOnce();
+		}, async (error) => {
+			more = false;
+			const message = `a checkpoint failed: ${reasonOf(error)}`;
+			if (message !== failed) {
+				failed = message;
+				console.warn(`coffre audit: ${message}`);
+				await record({
+					at: Date.now(),
+					message
+				});
+			}
+			if (error instanceof Stalled) stalled = true;
+			await within(store.makeDays(daysAround(Date.now())), options.tickBoundMs ?? OPEN_BOUND_MS, "making the next days of uses").catch(() => {});
+		}).finally(() => {
+			if (stalled) retire(failed ?? "the log's upkeep did not end in time");
+			else if (!stopped) timer = setTimeout(loop, more ? 0 : interval);
+		});
+	};
+	loop();
+	return {
+		checkpointer,
+		sealThrough: (number, deadline) => checkpointer.sealThrough(number, deadline),
+		async heldOutside() {
+			return heldIn(await fileMemory(options.memory).last());
+		},
+		async projected() {
+			for (let run = 0; run < 100; run += 1) {
+				await projector.settled();
+				projector.kick();
+				await projector.settled();
+				const [cursor, newest] = await Promise.all([store.readsCursor(), store.newest()]);
+				if (newest === null || cursor.through >= newest.upto) return;
+			}
+			throw new Error("the reads' projection did not catch up with the newest pair");
+		},
+		sealing: () => sealingOf({
+			open: !stalled && !stopped,
+			failed: failed ?? checkpointer.standing,
+			last
+		}),
+		async alarms(since = 0) {
+			await appending;
+			return (await readAlarms(alarmsPath)).filter((a) => a.at >= since);
+		},
+		async stop() {
+			stopped = true;
+			clearTimeout(timer);
+			if (stalled) await session?.cut().catch(() => {});
+			await ticking;
+			const projecting = projector.stop();
+			if (!stalled) await within(projecting, options.tickBoundMs ?? OPEN_BOUND_MS, "the reads' projection").catch(() => {});
+			await checkpointer.close();
+			await session?.cut().catch(() => {});
+		}
+	};
+}
+/** The alarms in the file, a line each; a line a crash left torn is skipped. */
+async function readAlarms(path) {
+	return (await readFile(path, "utf8").catch((error) => {
+		if (error.code === "ENOENT") return "";
+		throw error;
+	})).split("\n").flatMap((line) => {
+		try {
+			return line.length > 0 ? [JSON.parse(line)] : [];
+		} catch {
+			return [];
+		}
+	});
+}
+/** How long after a refused open the vault's checkpointer tries again (`keepCheckpointer`). */
+const OPEN_AGAIN_MS = 1e4;
+/**
+* The checkpointer as the vault's process keeps it: `runCheckpointer`, tried
+* again every `retryMs` while it cannot open (a log it refuses, as after a
+* restore, docs/restore.md; the database out of reach), each try making the
+* next days of uses first, so that uses go on; its failure an alarm, kept
+* once while it stands. Meanwhile a change's seal fails with why, as on
+* Workers, and the pair held outside is read from the file as ever. One
+* that stalls (T169) is stopped, fenced, and another opened a tick later.
+*/
+async function keepCheckpointer(options) {
+	let running = null;
+	let failed = null;
+	let stopped = false;
+	let timer;
+	const alarmsPath = `${options.memory}.alarms`;
+	let retiring = Promise.resolve();
+	const tryOpen = async () => {
+		try {
+			const opened = await runCheckpointer({
+				...options,
+				stalled: (why) => {
+					if (running === opened) running = null;
+					failed = why;
+					const retired = opened.stop().catch(() => {});
+					retiring = Promise.all([retiring, retired]).then(() => {});
+					retired.finally(() => {
+						if (!stopped) timer = setTimeout(() => void (trying = tryOpen()), 500);
+					});
+				}
+			});
+			running = opened;
+			failed = null;
+		} catch (error) {
+			const message = `the checkpointer cannot open: ${reasonOf(error)}`;
+			if (message !== failed) {
+				failed = message;
+				console.warn(`coffre audit: ${message}`);
+				await appendFile(alarmsPath, `${JSON.stringify({
+					at: Date.now(),
+					message
+				})}\n`).catch((e) => console.warn(`coffre audit: an alarm could not be kept: ${e.message}`));
+			}
+			if (!stopped) timer = setTimeout(() => void (trying = tryOpen()), options.retryMs ?? OPEN_AGAIN_MS);
+		}
+	};
+	let trying = tryOpen();
+	await trying;
+	return {
+		sealThrough: (number, deadline) => running === null ? Promise.reject(new Unsealed(number, failed ?? "the checkpointer is not open")) : running.sealThrough(number, deadline),
+		sealing: () => running === null ? {
+			ok: false,
+			reason: failed ?? "the checkpointer is not open"
+		} : running.sealing(),
+		async heldOutside() {
+			return heldIn(await fileMemory(options.memory).last());
+		},
+		async alarms(since = 0) {
+			return running === null ? (await readAlarms(alarmsPath)).filter((a) => a.at >= since) : running.alarms(since);
+		},
+		async stop() {
+			stopped = true;
+			clearTimeout(timer);
+			await trying;
+			await running?.stop();
+			await retiring;
+		}
+	};
+}
+//#endregion
 //#region src/failure.ts
 /**
 * A vault call's failure, as `serveVault` answers it: its message, and the
@@ -55,16 +337,31 @@ const METHODS = [
 	"checkpoint",
 	"about",
 	"keyChecks",
-	"verifyLog"
+	"verifyLog",
+	"use",
+	"useNamed",
+	"useMany",
+	"prepare",
+	"waitSealed",
+	"signIn",
+	"revokeCredential",
+	"issueToken",
+	"changeBindings",
+	"cutover",
+	"verifyTrees",
+	"verifyReads",
+	"vouch",
+	"sealing"
 ];
 /**
-* The vault in this process, over `db`. Every argument and result goes
+* The vault in this process, over `db`, and `hot` for 0.15's uses (vault.ts,
+* `openVault`). Every argument and result goes
 * through JSON on the way, as it would over RPC or a socket, so what works
 * here does not rely on sharing objects with the caller. `close` is
 * whatever lets go of the database, when the vault opened it.
 */
-async function openLocalVault(db, config, options = {}, close = async () => {}) {
-	const vault = openVault(db, await prepareVault(config, options));
+async function openLocalVault(db, config, options = {}, close = async () => {}, hot) {
+	const vault = openVault(db, await prepareVault(config, options), hot);
 	const local = Object.fromEntries(METHODS.map((name) => [name, async (...args) => json(await vault[name](...json(args)))]));
 	return Object.assign(local, { close });
 }
@@ -75,11 +372,12 @@ function json(value) {
 //#region src/node.ts
 /**
 * The vault on Node, one of two ways, each over the database the server
-* uses, through the vault's own login:
+* uses, through the vault's own login and its uses' (`hotDatabase`), with
+* the checkpointer that seals its log in the same process (`checkpointer`):
 *
-* - in the server's own process: `vault: await localVault({ database, ...keys })`;
+* - in the server's own process: `vault: await localVault({ database, hotDatabase, checkpointer, ...keys })`;
 * - as a process of its own, which the server reaches over a Unix socket:
-*   `serveVault({ socket, database, ...keys })` there, and
+*   `serveVault({ socket, database, hotDatabase, checkpointer, ...keys })` there, and
 *   `vault: connectVault(socket)` in the server.
 *
 * The second keeps every key out of the process that faces the network. The
@@ -89,15 +387,59 @@ function json(value) {
 * with the arguments as a JSON array, and refusals come back as values, as
 * from any vault; only a failure is an error.
 */
-/** The vault in this process. */
+/**
+* The vault in this process, and under 0.15's log, the default, its
+* checkpointer, opened before the vault serves, so that no call meets a log
+* with no checkpoint yet. A configuration without what 0.15 needs is refused.
+*/
 async function localVault(config, options = {}) {
 	const resolved = resolveVaultConfig(config);
-	if (typeof config.database !== "string") return openLocalVault(config.database, resolved, options);
-	const { db, close } = await openDatabase(config.database);
-	return openLocalVault(db, resolved, options, close).catch(async (error) => {
+	const log015 = (options.log ?? "0.15") === "0.15";
+	const own = log015 && options.sealThrough === void 0;
+	if (log015 && config.hotDatabase === void 0) throw new Error("the vault's uses run as a login of their own: set hotDatabase in the vault's configuration, the database as coffre_vault_hot (HOT_DATABASE_URL)");
+	if (own && config.checkpointer?.memory === void 0) throw new Error("the vault answers a change once its checkpointer has sealed it: set checkpointer: { memory } in the vault's configuration, the file its signer remembers in (CHECKPOINTER_MEMORY)");
+	const opened = [];
+	const open = async (database) => {
+		if (typeof database !== "string") return database;
+		const { db, close } = await openDatabase(database);
+		opened.push(close);
+		return db;
+	};
+	let checkpointer = null;
+	const close = async () => {
+		await checkpointer?.stop();
+		await Promise.all(opened.map((done) => done()));
+	};
+	try {
+		const db = await open(config.database);
+		const hot = config.hotDatabase === void 0 ? void 0 : config.hotDatabase === config.database ? db : await open(config.hotDatabase);
+		const connections = hot?.$client?.options?.max;
+		const uses = {
+			...connections === void 0 ? {} : { connections },
+			...options.uses
+		};
+		if (own) {
+			const url = typeof config.database === "string" && engineOfUrl(config.database) === "postgres" ? config.database : null;
+			const kept = checkpointer = await keepCheckpointer({
+				...url === null ? { database: db } : { connect: () => openDatabase(url, { max: 2 }) },
+				seed: resolved.signingKeys[0],
+				memory: config.checkpointer.memory
+			});
+			options = {
+				...options,
+				sealThrough: kept.sealThrough,
+				heldOutside: kept.heldOutside,
+				sealing: kept.sealing
+			};
+		}
+		return await openLocalVault(db, resolved, {
+			...options,
+			uses
+		}, close, hot);
+	} catch (error) {
 		await close();
 		throw error;
-	});
+	}
 }
 const METHOD_NAMES = new Set(METHODS);
 const MAX_BODY = 8388608;
@@ -182,4 +524,4 @@ async function readBody(stream) {
 	return Buffer.concat(chunks).toString("utf8");
 }
 //#endregion
-export { KekBadClaimError, KekUnavailableError, awsKms, checkpointMessage, connectVault, derivedSigningKey, localVault, serveVault, verifyCheckpoint };
+export { KekBadClaimError, KekUnavailableError, awsKms, checkpointMessage, connectVault, derivedSigningKey, fileMemory, keepCheckpointer, localVault, replayStored, runCheckpointer, serveVault, verifyCheckpoint };

package/dist/{src-Bg0ZeMJW.js → src-DheG92EM.js}

@@ -83,7 +83,7 @@ function signingKeys(given, keks) {
 	return [signing, ...derived].filter((key, i, all) => all.findIndex((other) => other.equals(key)) === i);
 }
 function checkBulkLimit({ count, windowMinutes }) {
-	if (!Number.isInteger(count) || count < 1 || !(windowMinutes > 0) || windowMinutes > 1440) throw new Error(`bulkLimit needs a whole count of at least 1 and a window above 0 minutes, a day (${MAX_BULK_WINDOW_MINUTES}) at most`);
+	if (!Number.isInteger(count) || count < 1 || !(windowMinutes > 0) || windowMinutes > 1440 || !Number.isInteger(windowMinutes * 6e4)) throw new Error(`bulkLimit needs a whole count of at least 1 and a window above 0 minutes, a day (${MAX_BULK_WINDOW_MINUTES}) at most, a whole number of milliseconds`);
 	return {
 		count,
 		windowMs: windowMinutes * 6e4