@flowdular/sdk 0.2.3 → 0.3.0

package/modules/audit/src/services/service-error.ts

@@ -0,0 +1,61 @@
+export class AuditServiceError extends Error {
+	constructor(
+		readonly code: string,
+		message: string,
+		readonly status = 400,
+	) {
+		super(message);
+		this.name = 'AuditServiceError';
+	}
+}
+
+export function bounded(
+	value: string,
+	field: string,
+	minimum: number,
+	maximum: number,
+): string {
+	const normalized = value.trim();
+	if (normalized.length < minimum || normalized.length > maximum) {
+		throw new AuditServiceError(
+			'INVALID_INPUT',
+			`${field} must contain between ${minimum} and ${maximum} characters.`,
+		);
+	}
+	if (normalized.includes('\u0000')) {
+		throw new AuditServiceError(
+			'INVALID_INPUT',
+			`${field} contains an unsupported character.`,
+		);
+	}
+	return normalized;
+}
+
+export function boundedInteger(
+	value: number,
+	field: string,
+	minimum: number,
+	maximum: number,
+): number {
+	if (!Number.isSafeInteger(value) || value < minimum || value > maximum) {
+		throw new AuditServiceError(
+			'INVALID_INPUT',
+			`${field} must be a whole number between ${minimum} and ${maximum}.`,
+		);
+	}
+	return value;
+}
+
+export function oneOf<T extends string>(
+	value: string,
+	field: string,
+	allowed: readonly T[],
+): T {
+	if (!(allowed as readonly string[]).includes(value)) {
+		throw new AuditServiceError(
+			'INVALID_INPUT',
+			`${field} must be one of: ${allowed.join(', ')}.`,
+		);
+	}
+	return value as T;
+}

package/modules/audit/src/services/subject-keys.ts

@@ -0,0 +1,90 @@
+import { createHash, randomBytes } from 'node:crypto';
+import { createKeyring, KeyringError } from '@flowdular/sdk/kernel';
+
+/**
+ * The envelope a sealed audit event carries, as one stored string. A per-subject
+ * data key seals it; destroying that key is the erasure, so the material lives
+ * beside the ciphertext on purpose. It buys crypto-shredding, not confidentiality
+ * at rest, which the deployment provides for the volume and the backup.
+ */
+export const SUBJECT_KEY_BYTES = 32;
+
+export function generateSubjectKey(): Buffer {
+	return randomBytes(SUBJECT_KEY_BYTES);
+}
+
+export function encodeSubjectKey(key: Buffer): string {
+	return key.toString('base64');
+}
+
+export function decodeSubjectKey(value: string): Buffer {
+	const key = Buffer.from(value, 'base64');
+	if (key.byteLength !== SUBJECT_KEY_BYTES) {
+		key.fill(0);
+		throw new Error('A stored audit subject key is not 32 bytes.');
+	}
+	return key;
+}
+
+/**
+ * The name a subject keeps once its key is gone. The key row, the erasure run
+ * and the certificate file all carry it, so a destroyed key is still findable
+ * without storing the account it belonged to: a subject whose key was destroyed
+ * must never be given a new one, and the marker is what proves it had one.
+ */
+export function erasureSubjectMarker(
+	tenantId: string,
+	subject: string,
+): string {
+	return createHash('sha256')
+		.update(`${tenantId}:${subject}`)
+		.digest('hex')
+		.slice(0, 12);
+}
+
+/**
+ * The additional data every sealed event is bound to. It is part of the stored
+ * envelope, so it can never change for a row that already exists.
+ */
+export function subjectSealContext(tenantId: string, eventId: string): string {
+	return `${tenantId}:${eventId}:audit-event`;
+}
+
+/** `iv.tag.ciphertext`, base64 each, so one column holds the whole envelope. */
+export function sealSubjectPayload(
+	key: Buffer,
+	context: string,
+	plaintext: string,
+): string {
+	const sealed = createKeyring({ current: key }).seal(plaintext, context);
+	return [
+		sealed.iv.toString('base64'),
+		sealed.tag.toString('base64'),
+		sealed.ciphertext.toString('base64'),
+	].join('.');
+}
+
+/** Null for an envelope this key cannot open, which a destroyed key never can. */
+export function openSubjectPayload(
+	key: Buffer,
+	context: string,
+	payload: string,
+): string | null {
+	const parts = payload.split('.');
+	if (parts.length !== 3) return null;
+	try {
+		return createKeyring({ current: key })
+			.open(
+				{
+					iv: Buffer.from(parts[0]!, 'base64'),
+					tag: Buffer.from(parts[1]!, 'base64'),
+					ciphertext: Buffer.from(parts[2]!, 'base64'),
+				},
+				context,
+			)
+			.toString('utf8');
+	} catch (error) {
+		if (error instanceof KeyringError) return null;
+		throw error;
+	}
+}

package/modules/audit/src/services/sweep-service.ts

@@ -0,0 +1,298 @@
+import type {
+	DataClassDeclaration,
+	PlatformDataClassRegistry,
+} from '@flowdular/sdk/kernel';
+import {
+	AUDIT_EVENT_ACTIONS,
+	AUDIT_REASONS,
+	type AuditDataClass,
+	type SweepStatus,
+} from '../domain/types.ts';
+import type { BackupGuard, BackupGuardResult } from './backup-guard.ts';
+import { DeclaredDataClasses } from './declared-classes.ts';
+import { noLegalHolds, type LegalHoldCheck } from './hold-service.ts';
+import { AUDIT_EVENTS_CLASS_ID } from './own-classes.ts';
+import type { AuditRepository, DataClassRouting } from './repository.ts';
+import { AuditServiceError } from './service-error.ts';
+
+const DAY_MS = 86_400_000;
+
+/**
+ * Classes one pass looks at. The pass is bounded so a deployment with many
+ * workspaces spreads its work over intervals instead of holding the loop.
+ */
+export const SWEEP_ROUTING_PAGE = 50;
+
+/**
+ * Batches one class may take in one pass. Without it a class whose owner keeps
+ * answering a full batch would hold the pass for as long as it has rows; the
+ * class is recorded as `partial` and picked up again at the next interval.
+ */
+export const SWEEP_MAX_BATCHES = 20;
+
+export interface SweepServiceOptions {
+	readonly repository: AuditRepository;
+	readonly registry: PlatformDataClassRegistry;
+	readonly backup: BackupGuard;
+	/** Live platform settings, read again on every pass. */
+	readonly batchSize: () => number;
+	readonly intervalMs: () => number;
+	readonly holds?: LegalHoldCheck;
+	readonly now?: () => number;
+}
+
+/**
+ * One due class as the pass that found it saw it: the routing row, the instant
+ * the pass read, and the backup evidence it refuses under. The instant and the
+ * evidence are the pass's own, so every class of one pass sweeps against the
+ * same cutoff and one backup read answers for all of them.
+ */
+export interface DueDataClass {
+	readonly routing: DataClassRouting;
+	readonly at: number;
+	readonly backup: BackupGuardResult;
+}
+
+/**
+ * The retention work of one class at a time. Due classes are found across
+ * workspaces on the routing lease, every class is read again under the
+ * workspace the routing row named, and the removal itself happens inside the
+ * owning module. The loop over them is the platform job runner's.
+ */
+export class AuditSweepService {
+	readonly #repository: AuditRepository;
+	readonly #declared: DeclaredDataClasses;
+	readonly #backup: BackupGuard;
+	readonly #batchSize: () => number;
+	readonly #intervalMs: () => number;
+	readonly #holds: LegalHoldCheck;
+	readonly #now: () => number;
+
+	constructor(options: SweepServiceOptions) {
+		this.#repository = options.repository;
+		this.#declared = new DeclaredDataClasses(options.registry);
+		this.#backup = options.backup;
+		this.#batchSize = options.batchSize;
+		this.#intervalMs = options.intervalMs;
+		this.#holds = options.holds ?? noLegalHolds;
+		this.#now = options.now ?? Date.now;
+	}
+
+	/**
+	 * The classes one pass looks at, found across workspaces on the routing
+	 * lease. The backup evidence is read once here and carried on every class,
+	 * not asked per class: the answer is a property of the deployment and reading
+	 * it per class would be a file read per row. An idle pass reads neither.
+	 */
+	async due(limit = SWEEP_ROUTING_PAGE): Promise<readonly DueDataClass[]> {
+		const at = this.#now();
+		const due = await this.#repository.listDueDataClasses(
+			at - this.#intervalMs(),
+			limit,
+		);
+		if (due.length === 0) return [];
+		const backup = await this.#backup();
+		return due.map((routing) => ({ routing, at, backup }));
+	}
+
+	/**
+	 * One due class: refused for the deployment, withheld under an active hold,
+	 * left to the next pass when the workspace moved it, or swept.
+	 */
+	async sweep(due: DueDataClass): Promise<void> {
+		const { routing, at, backup } = due;
+		if (!backup.ok) {
+			await this.#refuse(
+				routing.tenantId,
+				routing.classId,
+				AUDIT_REASONS.backupManifestMissing,
+				at,
+			);
+			return;
+		}
+		/* The routing read crosses tenants and returns routing columns only.
+		   The class is read again under the workspace that row named, and one
+		   that moved in between is left to the next pass rather than swept
+		   against a period that is no longer current. */
+		const record = await this.#repository.getDataClass(
+			routing.tenantId,
+			routing.classId,
+		);
+		if (
+			!record ||
+			record.lastSweptAt !== routing.lastSweptAt ||
+			record.effectiveRetentionDays === null
+		) {
+			return;
+		}
+		const owner = this.#declared.resolve(record.classId);
+		if (!owner?.declaration.sweep) return;
+		const hold = await this.#holds({
+			tenantId: record.tenantId,
+			classId: record.classId,
+		});
+		if (hold.held) {
+			const heldBack =
+				hold.heldBack ??
+				(await this.#countHeld(record, record.effectiveRetentionDays, at));
+			await this.#refuse(
+				record.tenantId,
+				record.classId,
+				AUDIT_REASONS.holdActive,
+				at,
+				heldBack,
+			);
+			return;
+		}
+		await this.#sweepClass(
+			record,
+			record.effectiveRetentionDays,
+			owner.declaration.sweep,
+			at,
+		);
+	}
+
+	/**
+	 * Rows the hold withheld, for the one class audit.core owns and sweeps. A
+	 * foreign class is withheld whole and only its owner could count it, which
+	 * the ledger records as an absent count rather than a zero.
+	 */
+	async #countHeld(
+		record: AuditDataClass,
+		retentionDays: number,
+		now: number,
+	): Promise<number | null> {
+		if (record.classId !== AUDIT_EVENTS_CLASS_ID) return null;
+		return this.#repository.countAuditEventsBefore(
+			record.tenantId,
+			now - retentionDays * DAY_MS,
+			null,
+		);
+	}
+
+	async #sweepClass(
+		record: AuditDataClass,
+		retentionDays: number,
+		sweep: NonNullable<DataClassDeclaration['sweep']>,
+		now: number,
+	): Promise<void> {
+		const limit = this.#batchSize();
+		const cutoffMs = now - retentionDays * DAY_MS;
+		const cutoff = new Date(cutoffMs);
+		let removed = 0;
+		let status: SweepStatus = 'completed';
+		let reason: string | null = null;
+		for (let batch = 1; batch <= SWEEP_MAX_BATCHES; batch += 1) {
+			/* The event is written before the rows are gone, so the trail records
+			   what was about to be removed even if the removal itself fails. */
+			await this.#repository.appendAuditEvent({
+				tenantId: record.tenantId,
+				actorId: 'audit.core',
+				action: AUDIT_EVENT_ACTIONS.retentionSweep,
+				subjectType: 'data-class',
+				subjectId: record.classId,
+				metadata: {
+					classId: record.classId,
+					cutoff: cutoff.toISOString(),
+					retentionDays,
+					batch,
+					limit,
+				},
+				occurredAt: this.#now(),
+			});
+			let batchRemoved: number;
+			try {
+				/* Foreign code. A faulty owner must not stop the pass or leave the
+				   class looking swept, so the failure becomes the ledger outcome. */
+				const answer = await sweep({
+					tenantId: record.tenantId,
+					cutoff,
+					limit,
+				});
+				batchRemoved = Math.max(0, Math.trunc(answer?.removed ?? 0));
+			} catch (error) {
+				status = removed > 0 ? 'partial' : 'refused';
+				/* Foreign code, but an owner that names a stable reason gets it
+				   recorded: the ledger is what an operator reads, and "the owner
+				   failed" hides a condition an operator can actually answer. */
+				reason =
+					error instanceof AuditServiceError
+						? error.code
+						: AUDIT_REASONS.ownerSweepFailed;
+				break;
+			}
+			removed += batchRemoved;
+			if (batchRemoved < limit) break;
+			if (batch === SWEEP_MAX_BATCHES) {
+				status = 'partial';
+				reason = AUDIT_REASONS.batchCapReached;
+			}
+		}
+		/* A refusal that stands until the deployment answers it is recorded once;
+		   repeating it every interval would grow the ledger without adding
+		   anything a reader did not already have. */
+		const standing =
+			status === 'refused' &&
+			(await this.#repository.latestSweepRun(record.tenantId, record.classId));
+		if (
+			!standing ||
+			standing.status !== 'refused' ||
+			standing.reason !== reason
+		) {
+			await this.#repository.appendSweepRun({
+				tenantId: record.tenantId,
+				classId: record.classId,
+				cutoff: cutoffMs,
+				removed,
+				status,
+				reason,
+				heldBack: null,
+				occurredAt: this.#now(),
+			});
+		}
+		/* A refused pass leaves the stamp alone so the class stays due; anything
+		   that removed rows moves on to the next interval. */
+		if (status !== 'refused') {
+			await this.#repository.stampSwept(
+				record.tenantId,
+				record.classId,
+				this.#now(),
+			);
+		}
+	}
+
+	/**
+	 * Records a standing refusal once. Repeating it every interval would grow
+	 * the ledger without adding anything: the reason is answered by fixing the
+	 * deployment, not by the next pass.
+	 */
+	async #refuse(
+		tenantId: string,
+		classId: string,
+		reason: string,
+		now: number,
+		heldBack: number | null = null,
+	): Promise<void> {
+		const latest = await this.#repository.latestSweepRun(tenantId, classId);
+		if (latest?.status === 'refused' && latest.reason === reason) return;
+		await this.#repository.appendSweepRun({
+			tenantId,
+			classId,
+			cutoff: now,
+			removed: 0,
+			status: 'refused',
+			reason,
+			heldBack,
+			occurredAt: now,
+		});
+		await this.#repository.appendAuditEvent({
+			tenantId,
+			actorId: 'audit.core',
+			action: AUDIT_EVENT_ACTIONS.retentionRefused,
+			subjectType: 'data-class',
+			subjectId: classId,
+			metadata: { classId, reason },
+			occurredAt: now,
+		});
+	}
+}

package/modules/audit/src/services/zip.ts

@@ -0,0 +1,217 @@
+import { crc32 } from 'node:zlib';
+
+/**
+ * A store-only ZIP writer built on node built-ins. The workspace carries no zip
+ * dependency and an export must not add one, so this writes the format itself:
+ * no compression, one entry per data class plus the manifest.
+ *
+ * Entries are streamed. The size and the checksum of an entry are not known
+ * until its last row is written, so every local header sets the streaming flag
+ * and the real values follow in a data descriptor; the central directory at the
+ * end carries them again, which is where a reader takes them from.
+ */
+const LOCAL_HEADER = 0x04034b50;
+const DATA_DESCRIPTOR = 0x08074b50;
+const CENTRAL_HEADER = 0x02014b50;
+const END_OF_CENTRAL_DIRECTORY = 0x06054b50;
+const STREAMING_FLAG = 0x0008;
+const STORE = 0;
+const VERSION = 20;
+/* Unix regular file, owner read and write. A reader that restores modes gets
+   the same owner-only file the archive itself is written with. */
+const EXTERNAL_ATTRIBUTES = 0o100600 << 16;
+
+/** No ZIP64: an entry or an archive at 4 GiB is a refusal, never a wrap. */
+const SIZE_LIMIT = 0xffff_fffe;
+/* One syscall per exported row would dominate the export; rows are gathered
+   into a reused window first. */
+const WRITE_WINDOW = 65_536;
+
+/**
+ * Where the archive bytes go. A file handle satisfies it directly; the export
+ * passes a wrapper that digests every byte on the way out, so the archive is
+ * fingerprinted without a second pass over the finished file.
+ */
+export interface ZipOutput {
+	write(data: Buffer): Promise<unknown>;
+}
+
+interface ZipEntryRecord {
+	readonly name: Buffer;
+	readonly offset: number;
+	readonly size: number;
+	readonly checksum: number;
+	readonly time: number;
+	readonly date: number;
+}
+
+function dosTime(at: Date): { readonly time: number; readonly date: number } {
+	/* MS-DOS stamps start in 1980 and keep two-second resolution. */
+	const year = Math.max(1980, at.getFullYear());
+	return {
+		time:
+			(at.getHours() << 11) |
+			(at.getMinutes() << 5) |
+			(Math.floor(at.getSeconds() / 2) & 0x1f),
+		date: ((year - 1980) << 9) | ((at.getMonth() + 1) << 5) | at.getDate(),
+	};
+}
+
+export class ZipWriteError extends Error {
+	constructor(
+		readonly code: string,
+		message: string,
+	) {
+		super(message);
+		this.name = 'ZipWriteError';
+	}
+}
+
+export class StoredZipWriter {
+	readonly #output: ZipOutput;
+	readonly #at: Date;
+	readonly #entries: ZipEntryRecord[] = [];
+	#offset = 0;
+	#pending: Buffer[] = [];
+	#pendingBytes = 0;
+	#open: {
+		readonly name: Buffer;
+		readonly offset: number;
+		readonly time: number;
+		readonly date: number;
+		size: number;
+		checksum: number;
+	} | null = null;
+	#closed = false;
+
+	constructor(output: ZipOutput, at: Date = new Date()) {
+		this.#output = output;
+		this.#at = at;
+	}
+
+	async addEntry(name: string): Promise<void> {
+		if (this.#open) {
+			throw new ZipWriteError(
+				'ZIP_ENTRY_OPEN',
+				`The entry ${this.#open.name.toString('utf8')} is still open.`,
+			);
+		}
+		const encoded = Buffer.from(name, 'utf8');
+		if (encoded.byteLength === 0 || encoded.byteLength > 0xffff) {
+			throw new ZipWriteError(
+				'ZIP_ENTRY_NAME',
+				`"${name}" is not an entry name.`,
+			);
+		}
+		const stamp = dosTime(this.#at);
+		const header = Buffer.alloc(30);
+		header.writeUInt32LE(LOCAL_HEADER, 0);
+		header.writeUInt16LE(VERSION, 4);
+		header.writeUInt16LE(STREAMING_FLAG, 6);
+		header.writeUInt16LE(STORE, 8);
+		header.writeUInt16LE(stamp.time, 10);
+		header.writeUInt16LE(stamp.date, 12);
+		header.writeUInt16LE(encoded.byteLength, 26);
+		this.#open = {
+			name: encoded,
+			offset: this.#offset,
+			time: stamp.time,
+			date: stamp.date,
+			size: 0,
+			checksum: 0,
+		};
+		await this.#push(header);
+		await this.#push(encoded);
+	}
+
+	async write(chunk: Buffer): Promise<void> {
+		const open = this.#open;
+		if (!open) {
+			throw new ZipWriteError('ZIP_NO_ENTRY', 'No archive entry is open.');
+		}
+		if (open.size + chunk.byteLength > SIZE_LIMIT) {
+			throw new ZipWriteError(
+				'ZIP_ENTRY_TOO_LARGE',
+				`${open.name.toString('utf8')} is larger than this archive format carries.`,
+			);
+		}
+		open.checksum = crc32(chunk, open.checksum);
+		open.size += chunk.byteLength;
+		await this.#push(chunk);
+	}
+
+	async closeEntry(): Promise<void> {
+		const open = this.#open;
+		if (!open) return;
+		const descriptor = Buffer.alloc(16);
+		descriptor.writeUInt32LE(DATA_DESCRIPTOR, 0);
+		descriptor.writeUInt32LE(open.checksum >>> 0, 4);
+		descriptor.writeUInt32LE(open.size, 8);
+		descriptor.writeUInt32LE(open.size, 12);
+		await this.#push(descriptor);
+		this.#entries.push({
+			name: open.name,
+			offset: open.offset,
+			size: open.size,
+			checksum: open.checksum >>> 0,
+			time: open.time,
+			date: open.date,
+		});
+		this.#open = null;
+	}
+
+	/** Writes the central directory. The archive is unreadable without it. */
+	async finish(): Promise<void> {
+		if (this.#closed) return;
+		await this.closeEntry();
+		const start = this.#offset;
+		for (const entry of this.#entries) {
+			const header = Buffer.alloc(46);
+			header.writeUInt32LE(CENTRAL_HEADER, 0);
+			header.writeUInt16LE(VERSION, 4);
+			header.writeUInt16LE(VERSION, 6);
+			header.writeUInt16LE(STREAMING_FLAG, 8);
+			header.writeUInt16LE(STORE, 10);
+			header.writeUInt16LE(entry.time, 12);
+			header.writeUInt16LE(entry.date, 14);
+			header.writeUInt32LE(entry.checksum, 16);
+			header.writeUInt32LE(entry.size, 20);
+			header.writeUInt32LE(entry.size, 24);
+			header.writeUInt16LE(entry.name.byteLength, 28);
+			header.writeUInt32LE(EXTERNAL_ATTRIBUTES >>> 0, 38);
+			header.writeUInt32LE(entry.offset, 42);
+			await this.#push(header);
+			await this.#push(entry.name);
+		}
+		const end = Buffer.alloc(22);
+		end.writeUInt32LE(END_OF_CENTRAL_DIRECTORY, 0);
+		end.writeUInt16LE(this.#entries.length, 8);
+		end.writeUInt16LE(this.#entries.length, 10);
+		end.writeUInt32LE(this.#offset - start, 12);
+		end.writeUInt32LE(start, 16);
+		await this.#push(end);
+		await this.#flush();
+		this.#closed = true;
+	}
+
+	async #push(chunk: Buffer): Promise<void> {
+		this.#offset += chunk.byteLength;
+		if (this.#offset > SIZE_LIMIT) {
+			throw new ZipWriteError(
+				'ZIP_ARCHIVE_TOO_LARGE',
+				'The archive is larger than this archive format carries.',
+			);
+		}
+		this.#pending.push(chunk);
+		this.#pendingBytes += chunk.byteLength;
+		if (this.#pendingBytes >= WRITE_WINDOW) await this.#flush();
+	}
+
+	async #flush(): Promise<void> {
+		if (this.#pendingBytes === 0) return;
+		const payload = Buffer.concat(this.#pending, this.#pendingBytes);
+		this.#pending = [];
+		this.#pendingBytes = 0;
+		await this.#output.write(payload);
+	}
+}