@flowdular/sdk 0.2.4 → 0.3.1

package/modules/audit/src/services/anchor-rotation.ts

@@ -0,0 +1,147 @@
+import type { AnchorSigner } from './anchor-key.ts';
+import type { AuditRepository } from './repository.ts';
+
+/** Anchors re-signed per routing page. */
+export const ANCHOR_ROTATION_BATCH = 200;
+
+/** Anchor ids one report names, so a broken ring does not print a table. */
+export const ANCHOR_ROTATION_REPORT_LIMIT = 50;
+
+export interface AnchorKeyCountReport {
+	readonly keyId: string;
+	readonly anchors: number;
+}
+
+export interface AnchorRotationReport {
+	readonly table: string;
+	/** Key id every anchor should end on: the current key of the signer. */
+	readonly currentKeyId: string;
+	readonly counts: readonly AnchorKeyCountReport[];
+	/** Anchors on a retired key when the run started. */
+	readonly stale: number;
+	readonly rotated: number;
+	/** Anchors another process re-signed between the read and the update. */
+	readonly skipped: number;
+	/** Anchors whose recorded key is in no ring, which a re-sign cannot fix. */
+	readonly unknownKeys: readonly string[];
+	/**
+	 * Anchors whose stored signature does not verify under the ring, by id and
+	 * bounded by `ANCHOR_ROTATION_REPORT_LIMIT`. None of them is re-signed: a
+	 * signature that does not verify is evidence, and signing the anchor hash
+	 * again would replace that evidence with a valid signature of this
+	 * deployment's own making.
+	 */
+	readonly unverified: readonly string[];
+	/** How many failed to verify, including the ones past the report limit. */
+	readonly unverifiedCount: number;
+}
+
+export interface AnchorRotationOptions {
+	readonly repository: AuditRepository;
+	readonly signer: AnchorSigner;
+	readonly apply?: boolean;
+	readonly batchSize?: number;
+}
+
+/**
+ * Re-signs every anchor that is not on the current key, once its stored
+ * signature verifies under the ring. Only the signature and the key id change:
+ * the anchor hash, the segment hash and the chain are what the segment files
+ * already carry, so a re-signed anchor still verifies against a file written
+ * years ago. It is idempotent, so a second run finds nothing.
+ */
+export async function rotateAnchorSignatures(
+	options: AnchorRotationOptions,
+): Promise<AnchorRotationReport> {
+	const currentKeyId = options.signer.keyId;
+	const batchSize = options.batchSize ?? ANCHOR_ROTATION_BATCH;
+	const counts = await options.repository.countAnchorsByKey();
+	const stale = counts.reduce(
+		(total, entry) =>
+			entry.keyId === currentKeyId ? total : total + entry.anchors,
+		0,
+	);
+	const unknownKeys = counts
+		.filter(
+			(entry) =>
+				entry.keyId !== currentKeyId && !options.signer.knows(entry.keyId),
+		)
+		.map((entry) => entry.keyId);
+	const report = {
+		table: 'audit_anchors',
+		currentKeyId,
+		counts: counts.map((entry) => ({
+			keyId: entry.keyId,
+			anchors: entry.anchors,
+		})),
+		stale,
+		unknownKeys,
+	};
+	if (options.apply !== true) {
+		return {
+			...report,
+			rotated: 0,
+			skipped: 0,
+			unverified: [],
+			unverifiedCount: 0,
+		};
+	}
+	let rotated = 0;
+	let skipped = 0;
+	let unverifiedCount = 0;
+	const unverified: string[] = [];
+	let cursor = '';
+	for (;;) {
+		/* Paged by primary key: an anchor the optimistic update skipped stays
+		   stale, so a query that only asked for stale rows would return it for
+		   ever. The routing read sees the tenant, the id and the key id alone. */
+		const page = await options.repository.listAnchorsNotOnKey(
+			currentKeyId,
+			cursor,
+			batchSize,
+		);
+		for (const routing of page) {
+			cursor = routing.id;
+			/* Read again under the workspace the routing row named; the anchor
+			   hash it signs is never visible to the cross-tenant lease. */
+			const anchor = await options.repository.getAnchor(
+				routing.tenantId,
+				routing.id,
+			);
+			if (!anchor || anchor.keyId === currentKeyId) {
+				skipped += 1;
+				continue;
+			}
+			/* A signature is re-made only once the stored one is proved. A key the
+			   ring does not hold, or an anchor whose signature does not verify
+			   under the key it names, is reported and left exactly as it is: a new
+			   signature over the same anchor hash would turn a row nobody can
+			   account for into one this deployment vouches for. */
+			if (
+				!options.signer.knows(anchor.keyId) ||
+				!options.signer.verify(
+					anchor.anchorHash,
+					anchor.signature,
+					anchor.keyId,
+				)
+			) {
+				unverifiedCount += 1;
+				if (unverified.length < ANCHOR_ROTATION_REPORT_LIMIT) {
+					unverified.push(anchor.id);
+				}
+				continue;
+			}
+			const written = await options.repository.resignAnchor({
+				tenantId: anchor.tenantId,
+				id: anchor.id,
+				signature: options.signer.sign(anchor.anchorHash),
+				keyId: currentKeyId,
+				expectedSignature: anchor.signature,
+			});
+			if (written) rotated += 1;
+			else skipped += 1;
+		}
+		if (page.length < batchSize) break;
+	}
+	return { ...report, rotated, skipped, unverified, unverifiedCount };
+}

package/modules/audit/src/services/audit-runners.ts

@@ -0,0 +1,278 @@
+/**
+ * The three background loops of audit.core on the platform job runner
+ * (`@flowdular/sdk/server`). The runner owns the timer and its unref, the guard
+ * against overlapping passes, the bound on the work one pass takes, the
+ * per-item isolation, the renewal timer with the abort it raises when the fence
+ * answers false, and the drain on stop. This module keeps its tables, its
+ * routing reads, its claim statements with their stale windows, its renewal
+ * statements and every outcome it records.
+ *
+ * One runner per loop rather than one for audit.core: the intervals, the
+ * routing pages and the claims differ, and a retention sweep that ran every
+ * five seconds or an export that waited an hour would both be wrong.
+ */
+import {
+	createJobRunner,
+	createJobTraceSink,
+	jobBackoff,
+	serverLogger,
+	type JobEvent,
+	type JobRunner,
+	type Tracer,
+} from '@flowdular/sdk/server';
+import type { AuditErasureRun, AuditExportRun } from '../domain/types.ts';
+import {
+	ERASURE_CLAIM_TIMEOUT_MS,
+	ERASURE_POLL_INTERVAL_MS,
+	ERASURE_ROUTING_PAGE,
+	type AuditErasureService,
+} from './erasure-service.ts';
+import {
+	EXPORT_CLAIM_TIMEOUT_MS,
+	EXPORT_POLL_INTERVAL_MS,
+	EXPORT_ROUTING_PAGE,
+	type AuditExportService,
+} from './export-service.ts';
+import type { AuditRepository } from './repository.ts';
+import {
+	SWEEP_ROUTING_PAGE,
+	type AuditSweepService,
+	type DueDataClass,
+} from './sweep-service.ts';
+
+interface RunnerSeams {
+	readonly now?: (() => number) | undefined;
+	/** Absent turns every pass into spans; a sink of its own replaces that. */
+	readonly onEvent?: ((event: JobEvent) => void) | undefined;
+	/** Defaults to the process tracer, which is the one `context.tracer` carries. */
+	readonly tracer?: Tracer | undefined;
+}
+
+/* One sink per loop: the pass spans it keeps open are that loop's, and the
+   three loops here run at cadences of their own. */
+function traceSink(options: RunnerSeams): (event: JobEvent) => void {
+	return (
+		options.onEvent ??
+		createJobTraceSink(options.tracer ? { tracer: options.tracer } : {})
+	);
+}
+
+export interface AuditSweepRunnerOptions extends RunnerSeams {
+	readonly sweeps: () => Promise<AuditSweepService>;
+	/**
+	 * Read once, as the cadence was before the runner: the setting's description
+	 * says an edited value applies the next time the platform starts. The same
+	 * setting is read live per pass for the due cutoff.
+	 */
+	readonly intervalMs: number;
+}
+
+export interface AuditExportRunnerOptions extends RunnerSeams {
+	readonly exports: () => Promise<AuditExportService>;
+	readonly repository: () => Promise<AuditRepository>;
+	readonly pollIntervalMs?: number | undefined;
+	readonly claimTimeoutMs?: number | undefined;
+	readonly heartbeatEveryMs?: number | undefined;
+}
+
+export interface AuditErasureRunnerOptions extends RunnerSeams {
+	readonly erasures: () => Promise<AuditErasureService>;
+	readonly repository: () => Promise<AuditRepository>;
+	readonly pollIntervalMs?: number | undefined;
+	readonly claimTimeoutMs?: number | undefined;
+	readonly heartbeatEveryMs?: number | undefined;
+}
+
+/** A claimed run and the claim instant every renewal of it is fenced on. */
+interface ClaimedRun<Run> {
+	readonly run: Run;
+	held: number;
+}
+
+/** Routing columns are all a cross-workspace read answers. */
+interface RunRouting {
+	readonly tenantId: string;
+	readonly id: string;
+}
+
+interface ClaimRunInput extends RunRouting {
+	readonly claimedAt: number;
+	readonly staleBefore: number;
+}
+
+/**
+ * The retention sweep. It claims nothing: a class is taken by the compare and
+ * swap on the sweep stamp the routing row carried, so there is no lease to hold
+ * and none to renew. One pass reads the routing page once and refuses,
+ * withholds or sweeps each class of it under the backup evidence that page was
+ * read with.
+ */
+export function createAuditSweepRunner(
+	options: AuditSweepRunnerOptions,
+): JobRunner {
+	/* The page one pass works through. It refills only once drained, and a page
+	   shorter than the bound means nothing more was due, so a pass reads the
+	   routing table exactly once however many classes it examines. */
+	let queue: DueDataClass[] = [];
+	let refillable = true;
+
+	return createJobRunner<DueDataClass>({
+		name: 'audit.core.sweep',
+		intervalMs: options.intervalMs,
+		/* No claim and no renewal, so no window: the runner reads the field only
+		   to space heartbeats this loop does not have. */
+		staleAfterMs: options.intervalMs,
+		backoff: jobBackoff(options.intervalMs),
+		batchLimit: SWEEP_ROUTING_PAGE,
+		logger: serverLogger,
+		now: options.now,
+		onEvent: traceSink(options),
+		claim: async () => {
+			if (queue.length === 0 && refillable) {
+				const page = await (await options.sweeps()).due(SWEEP_ROUTING_PAGE);
+				refillable = page.length === SWEEP_ROUTING_PAGE;
+				queue = [...page];
+			}
+			const next = queue.shift();
+			if (!next) {
+				refillable = true;
+				return null;
+			}
+			return next;
+		},
+		perform: async (due) => {
+			await (await options.sweeps()).sweep(due);
+		},
+	});
+}
+
+/** Requested exports, claimed under the workspace the routing row named. */
+export function createAuditExportRunner(
+	options: AuditExportRunnerOptions,
+): JobRunner {
+	const staleAfterMs = options.claimTimeoutMs ?? EXPORT_CLAIM_TIMEOUT_MS;
+	const intervalMs = options.pollIntervalMs ?? EXPORT_POLL_INTERVAL_MS;
+	const claim = claimQueue<AuditExportRun>({
+		limit: EXPORT_ROUTING_PAGE,
+		staleAfterMs,
+		page: async (limit) =>
+			(await options.repository()).listPendingExportRuns(limit),
+		take: async (input) => (await options.repository()).claimExportRun(input),
+	});
+
+	return createJobRunner<ClaimedRun<AuditExportRun>>({
+		name: 'audit.core.export',
+		intervalMs,
+		staleAfterMs,
+		backoff: jobBackoff(intervalMs),
+		batchLimit: EXPORT_ROUTING_PAGE,
+		heartbeatEveryMs: options.heartbeatEveryMs,
+		logger: serverLogger,
+		now: options.now,
+		onEvent: traceSink(options),
+		claim,
+		heartbeat: async (item, at) => {
+			const renewed = await (
+				await options.repository()
+			).heartbeatExportRun(item.run.tenantId, item.run.id, at, item.held);
+			if (renewed) item.held = at;
+			return renewed;
+		},
+		perform: async (item, signal) => {
+			await (await options.exports()).perform(item.run, signal);
+			/* The stage records nothing when the claim changed hands and leaves the
+			   run as the loop that owns it now will find it. Raising the abort here
+			   is how the pass learns the item was contention, not work performed. */
+			signal.throwIfAborted();
+		},
+	});
+}
+
+/** Requested erasures, including the expiry of one no platform answered. */
+export function createAuditErasureRunner(
+	options: AuditErasureRunnerOptions,
+): JobRunner {
+	const staleAfterMs = options.claimTimeoutMs ?? ERASURE_CLAIM_TIMEOUT_MS;
+	const intervalMs = options.pollIntervalMs ?? ERASURE_POLL_INTERVAL_MS;
+	const claim = claimQueue<AuditErasureRun>({
+		limit: ERASURE_ROUTING_PAGE,
+		staleAfterMs,
+		page: async (limit) =>
+			(await options.repository()).listPendingErasureRuns(limit),
+		take: async (input) => (await options.repository()).claimErasureRun(input),
+	});
+
+	return createJobRunner<ClaimedRun<AuditErasureRun>>({
+		name: 'audit.core.erasure',
+		intervalMs,
+		staleAfterMs,
+		backoff: jobBackoff(intervalMs),
+		batchLimit: ERASURE_ROUTING_PAGE,
+		heartbeatEveryMs: options.heartbeatEveryMs,
+		logger: serverLogger,
+		now: options.now,
+		onEvent: traceSink(options),
+		claim,
+		heartbeat: async (item, at) => {
+			const renewed = await (
+				await options.repository()
+			).heartbeatErasureRun(item.run.tenantId, item.run.id, at, item.held);
+			if (renewed) item.held = at;
+			return renewed;
+		},
+		perform: async (item, signal) => {
+			await (await options.erasures()).perform(item.run, signal);
+			signal.throwIfAborted();
+		},
+	});
+}
+
+/**
+ * One claim attempt over a routing page held between calls. Both request loops
+ * read routing columns across workspaces on the background lease and claim
+ * under the workspace the routing row named, so they share the walk and keep
+ * their own statements.
+ */
+function claimQueue<Run>(options: {
+	readonly limit: number;
+	readonly staleAfterMs: number;
+	readonly page: (limit: number) => Promise<readonly RunRouting[]>;
+	readonly take: (input: ClaimRunInput) => Promise<Run | null>;
+}): (at: number) => Promise<ClaimedRun<Run> | null> {
+	let queue: RunRouting[] = [];
+	let refillable = true;
+	/* Rows this pass walked past without taking them. The runner bounds the
+	   claims it is handed; this bounds the rows read to find them, so a page of
+	   runs other processes hold cannot cost one routing read per row. */
+	let skipped = 0;
+
+	return async (at) => {
+		while (skipped < options.limit) {
+			if (queue.length === 0) {
+				if (!refillable) break;
+				const page = await options.page(options.limit);
+				refillable = page.length === options.limit;
+				queue = [...page];
+				if (queue.length === 0) break;
+			}
+			const routing = queue.shift()!;
+			const run = await options.take({
+				tenantId: routing.tenantId,
+				id: routing.id,
+				claimedAt: at,
+				staleBefore: at - options.staleAfterMs,
+			});
+			/* The claim statement wrote `at`, so that is the instant every renewal
+			   fences on. */
+			if (run) return { run, held: at };
+			/* Another process holds it, or it was answered between the routing read
+			   and the claim. Either way it is not this pass's work. */
+			skipped += 1;
+		}
+		/* Null ends the pass, so the next one walks a budget of its own and asks
+		   the queue again for whatever is left here. */
+		skipped = 0;
+		refillable = true;
+		return null;
+	};
+}

package/modules/audit/src/services/backup-guard.ts

@@ -0,0 +1,95 @@
+import { readFile, stat } from 'node:fs/promises';
+import { isAbsolute, resolve } from 'node:path';
+import {
+	BACKUP_MANIFEST_FILE,
+	parseBackupManifest,
+	type BackupKeyFingerprint,
+} from '@flowdular/sdk/database';
+import { AUDIT_REASONS } from '../domain/types.ts';
+
+/**
+ * How a deployment records that a backup exists. It points at the directory or
+ * at the `backup.json` that `flowdular database backup --apply` wrote. The
+ * sweep and the export read it and refuse while it names nothing, so a policy
+ * defect can never destroy data no backup holds.
+ */
+export const AUDIT_BACKUP_MANIFEST_VARIABLE = 'FD_AUDIT_BACKUP_MANIFEST';
+
+/* A manifest is a few hundred bytes; anything this size is not one, and the
+   read is bounded before the file reaches memory. */
+const MANIFEST_SIZE_LIMIT = 1_048_576;
+
+export interface BackupEvidence {
+	readonly manifestPath: string;
+	readonly createdAt: string;
+	readonly adapter: string;
+	readonly platformVersion: string;
+	/** The key fingerprints the backup was taken under; never key material. */
+	readonly keys: readonly BackupKeyFingerprint[];
+}
+
+export type BackupGuardResult =
+	| { readonly ok: true; readonly evidence: BackupEvidence }
+	| { readonly ok: false; readonly reason: string; readonly detail: string };
+
+export type BackupGuard = () => Promise<BackupGuardResult>;
+
+function missing(detail: string): BackupGuardResult {
+	return {
+		ok: false,
+		reason: AUDIT_REASONS.backupManifestMissing,
+		detail,
+	};
+}
+
+function manifestPath(value: string, workspaceRoot: string): string {
+	const base = isAbsolute(value) ? value : resolve(workspaceRoot, value);
+	return base.endsWith('.json') ? base : resolve(base, BACKUP_MANIFEST_FILE);
+}
+
+/**
+ * Reads the recorded manifest on every call. It is asked once per sweep pass
+ * and once per export, so a backup taken while the platform runs is picked up
+ * without a restart and a backup directory that disappeared stops the sweep at
+ * the next interval.
+ */
+export function createBackupGuard(
+	environment: NodeJS.ProcessEnv,
+	workspaceRoot: string,
+): BackupGuard {
+	return async () => {
+		const configured = environment[AUDIT_BACKUP_MANIFEST_VARIABLE]?.trim();
+		if (!configured) {
+			return missing(
+				`${AUDIT_BACKUP_MANIFEST_VARIABLE} is not set, so this deployment records no backup.`,
+			);
+		}
+		const path = manifestPath(configured, workspaceRoot);
+		try {
+			const stats = await stat(path);
+			if (!stats.isFile()) return missing(`${path} is not a backup manifest.`);
+			if (stats.size > MANIFEST_SIZE_LIMIT) {
+				return missing(`${path} is too large to be a backup manifest.`);
+			}
+			const manifest = parseBackupManifest(
+				JSON.parse(await readFile(path, 'utf8')),
+			);
+			return {
+				ok: true,
+				evidence: {
+					manifestPath: path,
+					createdAt: manifest.createdAt,
+					adapter: manifest.adapter,
+					platformVersion: manifest.platformVersion,
+					keys: manifest.keys,
+				},
+			};
+		} catch (error) {
+			return missing(
+				`${path} could not be read as a backup manifest: ${
+					error instanceof Error ? error.message : String(error)
+				}`,
+			);
+		}
+	};
+}