@flowdular/sdk 0.2.4 → 0.3.0

package/modules/exports/src/services/export-service.ts

@@ -0,0 +1,356 @@
+import { randomUUID } from 'node:crypto';
+import type { AuthPrincipal } from '@flowdular/sdk/modules/auth';
+import {
+	runListExport,
+	ListExportError,
+	type ListExportPrincipal,
+} from '@flowdular/sdk/server';
+import { StorageError, type StoragePort } from '@flowdular/sdk/storage';
+import {
+	EXPORT_LIMITS,
+	EXPORT_UNEXPECTED_FAILURE,
+	type ClaimedExportJob,
+	type ExportJob,
+	type ExportListView,
+	type ExportRequester,
+} from '../domain/types.ts';
+import type { ExportListRegistry } from './list-registry.ts';
+import type {
+	ExportJobPage,
+	ExportJobQuery,
+	ExportRepository,
+} from './repository.ts';
+
+export class ExportServiceError extends Error {
+	constructor(
+		readonly code: string,
+		message: string,
+		readonly status = 400,
+	) {
+		super(message);
+		this.name = 'ExportServiceError';
+	}
+}
+
+/** The module id every object of this module lives under in the storage port. */
+export const EXPORT_OWNER_MODULE = 'exports.core';
+
+export const EXPORT_CSV_CONTENT_TYPE = 'text/csv';
+
+/** How long a minted read URL lives. Long enough to click, short enough to lose. */
+export const EXPORT_READ_URL_SECONDS = 300;
+
+export interface ExportServiceOptions {
+	readonly repository: ExportRepository;
+	readonly lists: ExportListRegistry;
+	readonly storage: StoragePort;
+	readonly maxRows: () => number;
+	readonly maxBytes: () => number;
+	/** The storage port's own object ceiling, which bounds the same bytes. */
+	readonly maxObjectBytes: () => number;
+	readonly now?: () => number;
+	readonly newId?: () => string;
+}
+
+function requesterOf(principal: AuthPrincipal): ExportRequester {
+	const requester: ExportRequester = {
+		accountId: principal.accountId,
+		email: principal.email,
+		displayName: principal.displayName,
+		role: principal.role,
+		scopes: [...principal.scopes],
+	};
+	/* The job row bounds the snapshot at 8 KB. A principal carrying more than
+	   that is refused here with a code the screen can read, rather than by the
+	   column check once the job is half written. */
+	if (JSON.stringify(requester).length > EXPORT_LIMITS.json) {
+		throw new ExportServiceError(
+			'REQUESTER_TOO_LARGE',
+			'Your account carries more grants than an export job can record.',
+		);
+	}
+	return requester;
+}
+
+/**
+ * The requester the list sees in the background stage. It carries the identity
+ * and the grants held when the job was started; the workspace is the job's own,
+ * and other workspace memberships are never part of an export.
+ */
+function principalOf(job: ExportJob): ListExportPrincipal {
+	return {
+		accountId: job.requester.accountId,
+		tenantId: job.tenantId,
+		scopes: job.requester.scopes,
+	};
+}
+
+/* One stable code per way a stage can end badly. A storage refusal on size is
+   the byte bound the operator already knows about, reported as that bound
+   rather than as a second one underneath it. */
+function failureCodeOf(error: unknown): string {
+	if (error instanceof ListExportError) return error.code;
+	if (error instanceof StorageError) {
+		return error.code === 'OBJECT_TOO_LARGE'
+			? 'EXPORT_BYTES_EXCEEDED'
+			: 'EXPORT_STORAGE_FAILED';
+	}
+	if (error instanceof ExportServiceError) return error.code;
+	return EXPORT_UNEXPECTED_FAILURE;
+}
+
+/**
+ * The catalogue a reader sees. It needs the sealed registry and the caller's
+ * scopes only, so the endpoint answers it without a repository or a lease.
+ */
+export function exportCatalogue(
+	lists: Pick<ExportListRegistry, 'list'>,
+	principal: AuthPrincipal,
+): readonly ExportListView[] {
+	const held = new Set(principal.scopes);
+	return lists.list().map((entry) => ({
+		id: entry.id,
+		label: entry.definition.label,
+		moduleId: entry.moduleId,
+		permitted: held.has(entry.definition.permission),
+	}));
+}
+
+export class ExportService {
+	readonly #repository: ExportRepository;
+	readonly #lists: ExportListRegistry;
+	readonly #storage: StoragePort;
+	readonly #maxRows: () => number;
+	readonly #maxBytes: () => number;
+	readonly #maxObjectBytes: () => number;
+	readonly #now: () => number;
+	readonly #newId: () => string;
+
+	constructor(options: ExportServiceOptions) {
+		this.#repository = options.repository;
+		this.#lists = options.lists;
+		this.#storage = options.storage;
+		this.#maxRows = options.maxRows;
+		this.#maxBytes = options.maxBytes;
+		this.#maxObjectBytes = options.maxObjectBytes;
+		this.#now = options.now ?? (() => Date.now());
+		this.#newId = options.newId ?? (() => randomUUID());
+	}
+
+	/* The storage port refuses an object above its own ceiling, and it counts the
+	   same plaintext bytes the walk does. A walk bounded above the ceiling would
+	   read the list to its end and build the whole file only to have the write
+	   refused, so the lower of the two is applied before the first page. Both
+	   refusals are reported under EXPORT_BYTES_EXCEEDED either way. */
+	#bytesBound(): number {
+		return Math.min(this.#maxBytes(), this.#maxObjectBytes());
+	}
+
+	async start(principal: AuthPrincipal, listId: string): Promise<ExportJob> {
+		const registered = this.#lists.find(listId);
+		if (!registered) {
+			throw new ExportServiceError(
+				'EXPORT_LIST_UNKNOWN',
+				'No module registered a list export under that id.',
+				404,
+			);
+		}
+		/* The list's own permission, checked on the acting principal before
+		   anything is recorded, so an export never widens what a member may
+		   read. */
+		if (!principal.scopes.includes(registered.definition.permission)) {
+			throw new ExportServiceError(
+				'EXPORT_LIST_FORBIDDEN',
+				'You do not hold the permission this list requires.',
+				403,
+			);
+		}
+		const startedAt = this.#now();
+		return this.#repository.createJob({
+			id: this.#newId(),
+			tenantId: principal.tenantId,
+			listId: registered.id,
+			status: 'requested',
+			rowCount: 0,
+			byteCount: 0,
+			objectId: null,
+			requesterAccountId: principal.accountId,
+			requester: requesterOf(principal),
+			failureCode: null,
+			claimedAt: null,
+			startedAt,
+			completedAt: null,
+		});
+	}
+
+	/**
+	 * The catalogue as one principal sees it: every registered list, each marked
+	 * with whether that principal holds the permission the list declares. The
+	 * permission is decided again on the live principal when a job is started, so
+	 * this is what a screen offers and never what authorizes an export.
+	 */
+	jobs(tenantId: string, query: ExportJobQuery): Promise<ExportJobPage> {
+		return this.#repository.listJobs(tenantId, query);
+	}
+
+	async job(tenantId: string, id: string): Promise<ExportJob> {
+		const job = await this.#repository.findJob(tenantId, id);
+		if (!job) {
+			throw new ExportServiceError(
+				'EXPORT_JOB_NOT_FOUND',
+				'That export job does not exist in this workspace.',
+				404,
+			);
+		}
+		return job;
+	}
+
+	/**
+	 * A signed platform route for the file of a completed job. It is minted per
+	 * request and never stored, so a signed credential does not reach a log line
+	 * or a page of results.
+	 */
+	async readUrl(principal: AuthPrincipal, id: string): Promise<string> {
+		const tenantId = principal.tenantId;
+		const job = await this.job(tenantId, id);
+		/* The file holds the rows of the exported list, so opening it needs that
+		   list's own permission on the live principal and not only the permission
+		   to read jobs. Otherwise a member who may see the job list could download
+		   rows the list itself would never have shown them, and a grant revoked
+		   after the export would still hand the file over. */
+		const registered = this.#lists.find(job.listId);
+		if (!registered) {
+			throw new ExportServiceError(
+				'EXPORT_LIST_UNKNOWN',
+				'No module registered the list this file was exported from.',
+				404,
+			);
+		}
+		if (!principal.scopes.includes(registered.definition.permission)) {
+			throw new ExportServiceError(
+				'EXPORT_LIST_FORBIDDEN',
+				'You do not hold the permission this list requires.',
+				403,
+			);
+		}
+		const objectId = job.objectId;
+		if (job.status !== 'completed' || objectId === null) {
+			throw new ExportServiceError(
+				'EXPORT_NOT_READY',
+				'That export has no file to open.',
+				409,
+			);
+		}
+		const reference = {
+			tenantId,
+			moduleId: EXPORT_OWNER_MODULE,
+			objectId,
+		};
+		/* Retention deletes the object before its row, so a job can name a file
+		   that is already gone; the reader is told that rather than handed a URL
+		   that answers nothing. */
+		if ((await this.#storage.stat(reference)) === null) {
+			throw new ExportServiceError(
+				'EXPORT_OBJECT_GONE',
+				'That export file is no longer stored.',
+				404,
+			);
+		}
+		return this.#storage.readUrl({
+			...reference,
+			expiresInSeconds: EXPORT_READ_URL_SECONDS,
+		});
+	}
+
+	/**
+	 * One claimed job: walk the list under the requester's snapshot, write the
+	 * file and settle. It never throws for work it performed; a claim taken over
+	 * mid-stage settles nothing and leaves the job to whoever holds it now.
+	 */
+	async perform(job: ClaimedExportJob, signal: AbortSignal): Promise<void> {
+		const registered = this.#lists.find(job.listId);
+		if (!registered) {
+			/* A list that was registered when the job started and is not any more
+			   means the module was disabled between the two. */
+			await this.#fail(job, 'EXPORT_LIST_UNKNOWN');
+			return;
+		}
+		let objectId: string | null = null;
+		let recorded = false;
+		try {
+			const result = await runListExport({
+				definition: registered.definition,
+				principal: principalOf(job),
+				bounds: { maxRows: this.#maxRows(), maxBytes: this.#bytesBound() },
+				signal,
+			});
+			/* Nothing is written for a claim this stage no longer holds. */
+			signal.throwIfAborted();
+			objectId = this.#newId();
+			const stored = await this.#storage.put({
+				tenantId: job.tenantId,
+				moduleId: EXPORT_OWNER_MODULE,
+				objectId,
+				contentType: EXPORT_CSV_CONTENT_TYPE,
+				body: result.body,
+				declaredBytes: result.bytes,
+			});
+			const settled = await this.#repository.settleJob(job.tenantId, job.id, {
+				status: 'completed',
+				rowCount: result.rows,
+				byteCount: stored.bytes,
+				objectId,
+				failureCode: null,
+				completedAt: this.#now(),
+			});
+			recorded = settled !== null;
+		} catch (error) {
+			if (!signal.aborted) await this.#fail(job, failureCodeOf(error));
+		} finally {
+			/* Every way this stage can end without the job row naming the object:
+			   the claim changed hands, the settle raised, the abort landed between
+			   the write and the settle, or the stage failed after the file existed.
+			   In each of them nothing would ever reference the object and no sweep
+			   would ever find it, so it goes now. */
+			if (objectId !== null && !recorded) {
+				await this.#discard(job.tenantId, objectId);
+			}
+		}
+	}
+
+	async #fail(job: ClaimedExportJob, failureCode: string): Promise<void> {
+		await this.#repository.settleJob(job.tenantId, job.id, {
+			status: 'failed',
+			rowCount: 0,
+			byteCount: 0,
+			objectId: null,
+			failureCode,
+			completedAt: this.#now(),
+		});
+	}
+
+	/* Best effort: a file that could not be deleted is one the retention sweep
+	   never sees, but failing the stage over it would settle a job whose work is
+	   already decided. */
+	async #discard(tenantId: string, objectId: string): Promise<void> {
+		try {
+			await this.#storage.delete({
+				tenantId,
+				moduleId: EXPORT_OWNER_MODULE,
+				objectId,
+			});
+		} catch {
+			/* The object outlives its job; nothing else depends on the delete. */
+		}
+	}
+
+	/** Deletes the files of a swept batch before its rows go. */
+	async discardObjects(
+		tenantId: string,
+		objectIds: readonly string[],
+	): Promise<void> {
+		for (const objectId of objectIds) {
+			await this.#discard(tenantId, objectId);
+		}
+	}
+}

package/modules/exports/src/services/index.ts

@@ -0,0 +1,6 @@
+export { ExportService, ExportServiceError } from './export-service.ts';
+export type { ExportRepository } from './repository.ts';
+export {
+	DatabaseExportRepository,
+	migrateExportsDatabase,
+} from './database-repository.ts';

package/modules/exports/src/services/list-registry.ts

@@ -0,0 +1,105 @@
+import type { DefinedListExport } from '@flowdular/sdk/server';
+import { EXPORT_LIST_LIMITS, type ExportLists } from '../domain/lists.ts';
+
+export class ExportListError extends Error {
+	constructor(
+		readonly code: string,
+		message: string,
+		readonly status = 400,
+	) {
+		super(message);
+		this.name = 'ExportListError';
+	}
+}
+
+/** A registered list and the module that declared it. */
+export interface RegisteredExportList {
+	readonly id: string;
+	readonly moduleId: string;
+	readonly definition: DefinedListExport;
+}
+
+export interface ExportListRegistry extends ExportLists {
+	/** Closes registration. Every later `register` throws. */
+	seal(): void;
+	find(id: string): RegisteredExportList | null;
+	list(): readonly RegisteredExportList[];
+}
+
+/**
+ * The lists a workspace can export. Registration is open while the platform
+ * composes and sealed before the first request, so the catalogue a job was
+ * started against cannot change under it. Lookup is a map read, O(1).
+ */
+export function createExportListRegistry(): ExportListRegistry {
+	const lists = new Map<string, RegisteredExportList>();
+	let sealed = false;
+	return {
+		register(moduleId, registered) {
+			if (sealed) {
+				throw new ExportListError(
+					'EXPORT_LISTS_SEALED',
+					'List exports are registered while the platform composes.',
+					500,
+				);
+			}
+			if (
+				typeof moduleId !== 'string' ||
+				moduleId.length === 0 ||
+				moduleId.length > EXPORT_LIST_LIMITS.moduleId
+			) {
+				throw new ExportListError(
+					'EXPORT_LIST_INVALID',
+					`A module id is 1 to ${EXPORT_LIST_LIMITS.moduleId} characters.`,
+					500,
+				);
+			}
+			if (registered.length > EXPORT_LIST_LIMITS.listsPerModule) {
+				throw new ExportListError(
+					'EXPORT_LIST_INVALID',
+					`A module registers at most ${EXPORT_LIST_LIMITS.listsPerModule} lists.`,
+					500,
+				);
+			}
+			for (const definition of registered) {
+				/* The id is the registrar's own namespace. Without this a module
+				   could declare an export under another module's id and answer for a
+				   list it does not own. */
+				if (!definition.id.startsWith(`${moduleId}.`)) {
+					throw new ExportListError(
+						'EXPORT_LIST_INVALID',
+						`The list ${definition.id} is not inside the namespace of ${moduleId}.`,
+						500,
+					);
+				}
+				if (lists.has(definition.id)) {
+					throw new ExportListError(
+						'EXPORT_LIST_DUPLICATE',
+						`The list export ${definition.id} is already registered.`,
+						500,
+					);
+				}
+				if (lists.size >= EXPORT_LIST_LIMITS.lists) {
+					throw new ExportListError(
+						'EXPORT_LIST_INVALID',
+						`The catalogue holds at most ${EXPORT_LIST_LIMITS.lists} lists.`,
+						500,
+					);
+				}
+				lists.set(definition.id, {
+					id: definition.id,
+					moduleId,
+					definition,
+				});
+			}
+		},
+		seal() {
+			sealed = true;
+		},
+		find: (id) => lists.get(id) ?? null,
+		list: () =>
+			[...lists.values()].sort((left, right) =>
+				left.id.localeCompare(right.id),
+			),
+	};
+}

package/modules/exports/src/services/migration.ts

@@ -0,0 +1,89 @@
+import type { DatabaseMigration } from '@flowdular/sdk/database';
+import { postgresTenantTableState } from '@flowdular/sdk/database';
+
+/* Every constant mirrors its migrations/<id>.up.sql file byte for byte;
+   tests/migrations.test.ts fails on drift. */
+
+export const EXPORTS_MIGRATION_001 = `CREATE TABLE IF NOT EXISTS exports_jobs (
+  id TEXT PRIMARY KEY,
+  tenant_id TEXT NOT NULL,
+  list_id TEXT NOT NULL CHECK (length(list_id) BETWEEN 3 AND 96),
+  status TEXT NOT NULL CHECK (status IN ('requested', 'running', 'completed', 'failed')),
+  row_count BIGINT NOT NULL CHECK (row_count >= 0),
+  byte_count BIGINT NOT NULL CHECK (byte_count >= 0),
+  object_id TEXT CHECK (object_id IS NULL OR length(object_id) BETWEEN 1 AND 128),
+  requester_account_id TEXT NOT NULL CHECK (length(requester_account_id) BETWEEN 1 AND 128),
+  requester_json TEXT NOT NULL CHECK (length(requester_json) <= 8192 AND requester_json::jsonb IS NOT NULL),
+  failure_code TEXT CHECK (failure_code IS NULL OR length(failure_code) BETWEEN 1 AND 64),
+  claimed_at BIGINT,
+  started_at BIGINT NOT NULL,
+  completed_at BIGINT,
+  UNIQUE (tenant_id, object_id),
+  CHECK ((status IN ('completed', 'failed')) = (completed_at IS NOT NULL)),
+  CHECK ((failure_code IS NOT NULL) = (status = 'failed')),
+  CHECK (object_id IS NULL OR status = 'completed')
+);
+CREATE INDEX IF NOT EXISTS exports_jobs_tenant_started_idx
+  ON exports_jobs (tenant_id, started_at DESC, id);
+CREATE INDEX IF NOT EXISTS exports_jobs_tenant_export_idx
+  ON exports_jobs (tenant_id, started_at, id);
+ALTER TABLE exports_jobs ENABLE ROW LEVEL SECURITY;
+ALTER TABLE exports_jobs FORCE ROW LEVEL SECURITY;
+CREATE POLICY exports_jobs_tenant_policy ON exports_jobs
+  USING (tenant_id = current_setting('coreloom.tenant_id', true))
+  WITH CHECK (tenant_id = current_setting('coreloom.tenant_id', true));
+`;
+
+export const EXPORTS_MIGRATION_002_JOB_ROUTING_ROLE = `-- The poll loop must find waiting jobs across workspaces before it knows whose
+-- they are, so it reads the routing columns alone on the background role: the
+-- list, the requester snapshot, the object and every count stay invisible to
+-- it, and each job it picks is claimed and run again under the workspace the
+-- routing row named. PostgreSQL checks column privileges in WHERE too, so
+-- \`status\` is part of the grant. A job already running is routed as well, so a
+-- claim whose process is gone is found again once its lease lapses.
+CREATE INDEX IF NOT EXISTS exports_jobs_routing_idx
+  ON exports_jobs (status, started_at, tenant_id, id);
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'coreloom_background') THEN
+    RAISE EXCEPTION 'The coreloom_background role must exist before this migration.';
+  END IF;
+END
+$$;
+CREATE POLICY exports_jobs_background_policy ON exports_jobs
+  FOR SELECT TO coreloom_background
+  USING (status IN ('requested', 'running'));
+REVOKE SELECT ON exports_jobs FROM coreloom_background;
+GRANT SELECT (tenant_id, id, status, started_at) ON exports_jobs TO coreloom_background;
+`;
+
+export const databaseMigrations: readonly DatabaseMigration[] = [
+	{
+		id: '0001_exports_core',
+		sql: { postgresql: EXPORTS_MIGRATION_001 },
+		inspectExisting: (database) =>
+			postgresTenantTableState(
+				database,
+				'exports_jobs',
+				'exports_jobs_tenant_policy',
+				[
+					() => database.schema.hasIndex('exports_jobs_tenant_started_idx'),
+					() => database.schema.hasIndex('exports_jobs_tenant_export_idx'),
+				],
+			),
+	},
+	{
+		id: '0002_exports_job_routing_role',
+		sql: { postgresql: EXPORTS_MIGRATION_002_JOB_ROUTING_ROLE },
+		/* A policy and a column grant leave no schema object behind, so the
+		   privilege itself is what proves this migration ran. */
+		inspectExisting: async (database) => {
+			const result = await database.query<{ granted: boolean }>({
+				text: `SELECT CASE WHEN to_regclass('exports_jobs') IS NOT NULL THEN
+				  has_column_privilege('coreloom_background', 'exports_jobs', 'status', 'SELECT')
+				ELSE false END AS granted`,
+			});
+			return result.rows[0]?.granted ? 'complete' : 'absent';
+		},
+	},
+];

package/modules/exports/src/services/repository.ts

@@ -0,0 +1,97 @@
+import type {
+	DataClassExportSink,
+	DataClassExportSummary,
+} from '@flowdular/sdk/kernel';
+import type {
+	ClaimedExportJob,
+	ExportJob,
+	ExportJobRouting,
+	ExportJobStatus,
+} from '../domain/types.ts';
+
+/** The keyset of the last job on a page: newest first, id breaking ties. */
+export interface ExportJobCursor {
+	readonly startedAt: number;
+	readonly id: string;
+}
+
+export interface ExportJobQuery {
+	readonly status?: ExportJobStatus | undefined;
+	readonly limit: number;
+	readonly cursor?: ExportJobCursor | undefined;
+}
+
+export interface ExportJobPage {
+	readonly items: readonly ExportJob[];
+	readonly nextCursor: ExportJobCursor | null;
+}
+
+export interface ClaimExportJobInput {
+	readonly tenantId: string;
+	readonly id: string;
+	readonly claimedAt: number;
+	/** A claim at or before this belonged to a process that is gone. */
+	readonly staleBefore: number;
+}
+
+export interface SettleExportJobInput {
+	readonly status: Extract<ExportJobStatus, 'completed' | 'failed'>;
+	readonly rowCount: number;
+	readonly byteCount: number;
+	readonly objectId: string | null;
+	readonly failureCode: string | null;
+	readonly completedAt: number;
+}
+
+export interface ExportSweepInput {
+	/** Settled jobs started strictly before this go. */
+	readonly settledBefore: number;
+	/** Upper bound on the jobs one pass removes. */
+	readonly limit: number;
+}
+
+export interface ExportSweepBatch {
+	readonly ids: readonly string[];
+	/** The objects those jobs hold, for the caller to delete before the rows. */
+	readonly objectIds: readonly string[];
+}
+
+/**
+ * The business port: async, database-agnostic, and unaware of SQL. Every method
+ * is tenant-scoped except `listPendingJobs`, which crosses workspaces on the
+ * background role and answers routing columns alone.
+ */
+export interface ExportRepository {
+	createJob(job: ExportJob): Promise<ExportJob>;
+	findJob(tenantId: string, id: string): Promise<ExportJob | null>;
+	listJobs(tenantId: string, query: ExportJobQuery): Promise<ExportJobPage>;
+	/** Routing columns of the jobs waiting or in flight, across workspaces. */
+	listPendingJobs(limit: number): Promise<readonly ExportJobRouting[]>;
+	/** Takes one job for this process; null when another already holds it. */
+	claimJob(input: ClaimExportJobInput): Promise<ClaimedExportJob | null>;
+	/** Renews a claim, answering false once the job changed hands. */
+	heartbeatJob(
+		tenantId: string,
+		id: string,
+		at: number,
+		claimedAt: number,
+	): Promise<boolean>;
+	/** Settles a running job; null when it is no longer this process's work. */
+	settleJob(
+		tenantId: string,
+		id: string,
+		input: SettleExportJobInput,
+	): Promise<ExportJob | null>;
+	/** The batch one retention pass removes, chosen once, oldest first. */
+	claimSweepBatch(
+		tenantId: string,
+		input: ExportSweepInput,
+	): Promise<ExportSweepBatch>;
+	/** Removes the jobs of a chosen batch; answers how many rows went. */
+	deleteJobs(tenantId: string, ids: readonly string[]): Promise<number>;
+	/** The whole workspace, oldest first, for the data class export. */
+	exportJobs(
+		tenantId: string,
+		sink: DataClassExportSink,
+	): Promise<DataClassExportSummary>;
+}