@flowdular/sdk 0.2.3 → 0.3.0

package/modules/exports/src/domain/types.ts

@@ -0,0 +1,120 @@
+/** The stages a job moves through; the transitions live in the service. */
+export const EXPORT_JOB_STATUSES = [
+	'requested',
+	'running',
+	'completed',
+	'failed',
+] as const;
+export type ExportJobStatus = (typeof EXPORT_JOB_STATUSES)[number];
+
+/** Longest values this module accepts; over any of them is a rejection. */
+export const EXPORT_LIMITS = {
+	id: 64,
+	listId: 96,
+	accountId: 128,
+	objectId: 128,
+	failureCode: 64,
+	/** Serialized principal snapshot, matching the schema. */
+	json: 8_192,
+	/** Jobs one page may carry; the platform page ceiling. */
+	page: 200,
+	/** Jobs one routing pass considers. */
+	routingPage: 25,
+} as const;
+
+/** What a stage records when the failure carries no stable code of its own. */
+export const EXPORT_UNEXPECTED_FAILURE = 'JOB_FAILED';
+
+/**
+ * The acting principal as it stood when the job was started, kept so the list
+ * sees the same identity in the background stage that it would have seen in the
+ * request. Authorization is decided on the live principal at start; this
+ * snapshot carries the decision forward, it does not make one.
+ */
+export interface ExportRequester {
+	readonly accountId: string;
+	readonly email: string;
+	readonly displayName: string;
+	readonly role: string;
+	readonly scopes: readonly string[];
+}
+
+/**
+ * One registered list as the catalogue answers it. The permission the list
+ * declares is not carried: a reader needs to know whether they may export the
+ * list, not the id of the grant that decides it.
+ */
+export interface ExportListView {
+	readonly id: string;
+	readonly label: string;
+	/** The module that registered the list, so a reader can place it. */
+	readonly moduleId: string;
+	readonly permitted: boolean;
+}
+
+export interface ExportJob {
+	readonly id: string;
+	readonly tenantId: string;
+	readonly listId: string;
+	readonly status: ExportJobStatus;
+	/** Rows written, the header record excluded. */
+	readonly rowCount: number;
+	readonly byteCount: number;
+	/** The storage object holding the file; null until the job completes. */
+	readonly objectId: string | null;
+	readonly requesterAccountId: string;
+	readonly requester: ExportRequester;
+	/** The stable code behind a failed job; null in every other status. */
+	readonly failureCode: string | null;
+	readonly claimedAt: number | null;
+	readonly startedAt: number;
+	readonly completedAt: number | null;
+}
+
+/**
+ * A job as the poll loop holds it. The claim instant is the fence every renewal
+ * of that claim is made under, so a stage can tell that its lease lapsed and the
+ * job changed hands.
+ */
+export type ClaimedExportJob = ExportJob & { readonly claimedAt: number };
+
+/** What the cross-tenant routing read may see, and nothing else. */
+export interface ExportJobRouting {
+	readonly tenantId: string;
+	readonly id: string;
+	readonly status: ExportJobStatus;
+	readonly startedAt: number;
+}
+
+/**
+ * The job as an API response carries it. The requester snapshot is bookkeeping
+ * the list needs in a background stage and no reader of the screen does, the
+ * claim is the poll loop's own lease, and the object id addresses bytes that
+ * are only ever reached through a signed route, so none of the three crosses
+ * the wire: a job read names its requester by account id alone.
+ */
+export type ExportJobView = Omit<
+	ExportJob,
+	'requester' | 'claimedAt' | 'objectId'
+> & {
+	/** Whether a file exists to open, without naming where it lives. */
+	readonly downloadable: boolean;
+};
+
+/* Every field is named rather than spread, so a field added to the job is a
+   decision taken here instead of an addition the wire makes silently. */
+export function exportJobView(job: ExportJob): ExportJobView {
+	return {
+		id: job.id,
+		tenantId: job.tenantId,
+		listId: job.listId,
+		status: job.status,
+		rowCount: job.rowCount,
+		byteCount: job.byteCount,
+		requesterAccountId: job.requesterAccountId,
+		failureCode: job.failureCode,
+		startedAt: job.startedAt,
+		completedAt: job.completedAt,
+		downloadable: job.objectId !== null,
+	};
+}

package/modules/exports/src/index.ts

@@ -0,0 +1,41 @@
+import type { ModuleManifest, RegisteredModule } from '@flowdular/sdk/contracts';
+import manifest from '../module.json' with { type: 'json' };
+import { EXPORTS_PERMISSIONS } from './acl/permissions.ts';
+
+export const moduleDefinition = {
+	manifest: manifest as ModuleManifest,
+	navigation: [
+		{
+			id: 'exports.navigation',
+			label: 'Exports',
+			href: '/exports',
+			order: 45,
+			permission: EXPORTS_PERMISSIONS.read,
+		},
+	],
+	permissions: Object.values(EXPORTS_PERMISSIONS),
+} satisfies RegisteredModule;
+
+export { EXPORTS_PERMISSIONS } from './acl/permissions.ts';
+
+/* The cross-module contract. A module that owns a list imports the identifier
+   and the service type from here, builds its declaration with defineListExport
+   from @flowdular/sdk/server, and registers it through the capability registry;
+   nothing else in this module is meant to be imported by another. */
+export { EXPORT_LISTS_CAPABILITY, EXPORT_LIST_LIMITS } from './domain/lists.ts';
+export type { ExportLists } from './domain/lists.ts';
+
+export {
+	exportJobView,
+	EXPORT_JOB_STATUSES,
+	EXPORT_LIMITS,
+	EXPORT_UNEXPECTED_FAILURE,
+} from './domain/types.ts';
+export type {
+	ClaimedExportJob,
+	ExportJob,
+	ExportJobRouting,
+	ExportJobStatus,
+	ExportJobView,
+	ExportRequester,
+} from './domain/types.ts';

package/modules/exports/src/platform.ts

@@ -0,0 +1,80 @@
+import type {
+	PlatformServerComposition,
+	PlatformServerContext,
+} from '@flowdular/sdk/modules/auth/server';
+import {
+	DEFAULT_STORAGE_MAX_OBJECT_BYTES,
+	storageConfigFromEnvironment,
+} from '@flowdular/sdk/storage';
+import { EXPORT_LISTS_CAPABILITY, type ExportLists } from './domain/lists.ts';
+import { createExportRoutes, createExportsRuntime } from './server/index.ts';
+import { exportsDataClass } from './services/data-classes.ts';
+import {
+	exportMaxBytes,
+	exportMaxRows,
+	EXPORTS_MODULE_SETTINGS,
+} from './settings.ts';
+
+export function createServerComposition(
+	context: PlatformServerContext,
+): PlatformServerComposition {
+	const objectCeiling = storageObjectCeiling(
+		context.environment,
+		context.workspaceRoot,
+	);
+	const runtime = createExportsRuntime({
+		databases: context.databases,
+		storage: context.storage,
+		purpose:
+			context.environment.NODE_ENV === 'test'
+				? 'test'
+				: context.environment.NODE_ENV === 'production'
+					? 'runtime'
+					: 'preview',
+		maxRows: () => exportMaxRows(context.settings),
+		maxBytes: () => exportMaxBytes(context.settings),
+		maxObjectBytes: () => objectCeiling,
+	});
+	/* Registered while the platform composes, so a module that owns a list can
+	   declare its export in its own composition. */
+	context.capabilities.register<ExportLists>(EXPORT_LISTS_CAPABILITY, {
+		register: (moduleId, exports) => runtime.lists.register(moduleId, exports),
+	});
+	/* The catalogue is sealed before start hooks run, so what this module holds
+	   is declared here rather than on the first request. */
+	context.dataClasses.declare([
+		exportsDataClass(
+			() => runtime.repository(),
+			() => runtime.service(),
+		),
+	]);
+	return {
+		routes: createExportRoutes(context.auth, runtime),
+		settings: EXPORTS_MODULE_SETTINGS,
+		start: () => {
+			/* Every module has composed by now, so the catalogue is complete and
+			   closed: a job can never be started against a list that appears
+			   afterwards. */
+			runtime.lists.seal();
+			runtime.start();
+		},
+		stop: () => runtime.quiesce(),
+		dispose: () => runtime.dispose(),
+	};
+}
+
+/* The ceiling the storage port enforces, read from the same configuration the
+   platform built the port from. A build or a preview composes under a NODE_ENV
+   the adapter check refuses; the port itself never reached a running platform
+   with that configuration, so the default ceiling is the one it would apply. */
+export function storageObjectCeiling(
+	environment: NodeJS.ProcessEnv,
+	workspaceRoot: string,
+): number {
+	try {
+		return storageConfigFromEnvironment(environment, workspaceRoot)
+			.maxObjectBytes;
+	} catch {
+		return DEFAULT_STORAGE_MAX_OBJECT_BYTES;
+	}
+}

package/modules/exports/src/server/index.ts

@@ -0,0 +1,51 @@
+export { createExportRoutes, endpoints } from '../api/endpoints.ts';
+export {
+	DatabaseExportRepository,
+	migrateExportsDatabase,
+} from '../services/database-repository.ts';
+export type { ExportDatabaseHandles } from '../services/database-repository.ts';
+export {
+	ExportService,
+	ExportServiceError,
+	EXPORT_CSV_CONTENT_TYPE,
+	EXPORT_OWNER_MODULE,
+	EXPORT_READ_URL_SECONDS,
+} from '../services/export-service.ts';
+export type { ExportServiceOptions } from '../services/export-service.ts';
+export {
+	createExportJobRunner,
+	EXPORT_CLAIM_TIMEOUT_MS,
+	EXPORT_POLL_INTERVAL_MS,
+} from '../services/export-runner.ts';
+export type { ExportRunnerOptions } from '../services/export-runner.ts';
+export {
+	createExportListRegistry,
+	ExportListError,
+} from '../services/list-registry.ts';
+export type {
+	ExportListRegistry,
+	RegisteredExportList,
+} from '../services/list-registry.ts';
+export {
+	exportsDataClass,
+	EXPORT_DATA_CLASS_KEY,
+	EXPORT_RETENTION_DAYS,
+	EXPORT_SWEEP_JOBS,
+} from '../services/data-classes.ts';
+export type {
+	ExportJobCursor,
+	ExportJobPage,
+	ExportJobQuery,
+	ExportRepository,
+	ExportSweepBatch,
+	ExportSweepInput,
+	SettleExportJobInput,
+} from '../services/repository.ts';
+export { databaseMigrations } from '../services/migration.ts';
+export {
+	exportMaxBytes,
+	exportMaxRows,
+	EXPORTS_MODULE_SETTINGS,
+} from '../settings.ts';
+export { createExportsRuntime } from './runtime.ts';
+export type { ExportsRuntime, ExportsRuntimeOptions } from './runtime.ts';

package/modules/exports/src/server/runtime.ts

@@ -0,0 +1,160 @@
+import type {
+	DatabaseAdapterLease,
+	DatabaseProvider,
+	DatabaseProviderRequest,
+} from '@flowdular/sdk/database';
+import {
+	DATABASE_CAPABILITY_IDS,
+	DATABASE_DIALECT_IDS,
+} from '@flowdular/sdk/database';
+import type { StoragePort } from '@flowdular/sdk/storage';
+import { ExportService } from '../services/export-service.ts';
+import { createExportJobRunner } from '../services/export-runner.ts';
+import {
+	DatabaseExportRepository,
+	migrateExportsDatabase,
+} from '../services/database-repository.ts';
+import type { ExportRepository } from '../services/repository.ts';
+import {
+	createExportListRegistry,
+	type ExportListRegistry,
+} from '../services/list-registry.ts';
+
+export interface ExportsRuntimeOptions {
+	/** Platform-owned provider. Modules never receive a DSN or a pool. */
+	readonly databases: DatabaseProvider;
+	readonly purpose: Exclude<DatabaseProviderRequest['purpose'], 'migration'>;
+	/** Platform-owned object storage; this module writes under its own id. */
+	readonly storage: StoragePort;
+	readonly maxRows: () => number;
+	readonly maxBytes: () => number;
+	/** The object ceiling the platform configured the storage port with. */
+	readonly maxObjectBytes: () => number;
+	/** Test seams: an already migrated repository, and a faster poll. */
+	readonly repository?: ExportRepository;
+	readonly pollIntervalMs?: number;
+	readonly claimTimeoutMs?: number;
+}
+
+export interface ExportsRuntime {
+	readonly lists: ExportListRegistry;
+	service(): Promise<ExportService>;
+	repository(): Promise<ExportRepository>;
+	/** One poll pass, for a test that must not wait on a timer. */
+	tick(): Promise<void>;
+	start(): void;
+	stop(): void;
+	quiesce(): Promise<void>;
+	dispose(): Promise<void>;
+}
+
+export function createExportsRuntime(
+	options: ExportsRuntimeOptions,
+): ExportsRuntime {
+	const lists = createExportListRegistry();
+	let disposed = false;
+	let leases: DatabaseAdapterLease[] = [];
+	let repositoryPromise: Promise<ExportRepository> | undefined;
+	let servicePromise: Promise<ExportService> | undefined;
+
+	/* Schema work runs on the migrator role and that lease is released before the
+	   runtime one is taken, so request handling never holds a schema owner. */
+	const openRepository = async (): Promise<ExportRepository> => {
+		if (options.repository) return options.repository;
+		const migrationLease = await options.databases.acquire({
+			namespace: 'exports.core',
+			purpose: 'migration',
+			requirements: {
+				dialectIds: [DATABASE_DIALECT_IDS.postgresql],
+				capabilities: [
+					DATABASE_CAPABILITY_IDS.MIGRATION_LOCK,
+					DATABASE_CAPABILITY_IDS.SCHEMA_INTROSPECTION,
+					DATABASE_CAPABILITY_IDS.TRANSACTIONAL_DDL,
+				],
+			},
+		});
+		try {
+			await migrateExportsDatabase(migrationLease.database);
+		} finally {
+			await migrationLease.release();
+		}
+		const acquire = (purpose: DatabaseProviderRequest['purpose']) =>
+			options.databases.acquire({
+				namespace: 'exports.core',
+				purpose,
+				requirements: {
+					dialectIds: [DATABASE_DIALECT_IDS.postgresql],
+					capabilities: [DATABASE_CAPABILITY_IDS.TRANSACTIONS],
+				},
+			});
+		const runtimeLease = await acquire(options.purpose);
+		/* The job poll reads across tenants; every write that follows uses the
+		   tenant carried by the routing row it returned. */
+		const backgroundLease = await acquire('background');
+		leases = [runtimeLease, backgroundLease];
+		return new DatabaseExportRepository({
+			runtime: runtimeLease.database,
+			background: backgroundLease.database,
+		});
+	};
+
+	const repository = (): Promise<ExportRepository> => {
+		if (disposed)
+			return Promise.reject(new Error('Exports runtime is disposed.'));
+		repositoryPromise ??= openRepository();
+		return repositoryPromise;
+	};
+
+	const service = (): Promise<ExportService> => {
+		if (disposed)
+			return Promise.reject(new Error('Exports runtime is disposed.'));
+		servicePromise ??= repository().then(
+			(resolved) =>
+				new ExportService({
+					repository: resolved,
+					lists,
+					storage: options.storage,
+					maxRows: options.maxRows,
+					maxBytes: options.maxBytes,
+					maxObjectBytes: options.maxObjectBytes,
+				}),
+		);
+		return servicePromise;
+	};
+
+	/* The platform runner owns the loop: the interval and its unref, the guard
+	   against overlapping passes, the bound on claims, the renewal timer, the
+	   backoff and the drain. This module keeps its table, its statements and its
+	   stale window. */
+	const jobs = createExportJobRunner({
+		repository,
+		service,
+		pollIntervalMs: options.pollIntervalMs,
+		claimTimeoutMs: options.claimTimeoutMs,
+	});
+
+	return {
+		lists,
+		service,
+		repository,
+		async tick() {
+			await jobs.tick();
+		},
+		start: () => jobs.start(),
+		stop: () => jobs.stop(),
+		quiesce: () => jobs.quiesce(),
+		async dispose() {
+			if (disposed) return;
+			disposed = true;
+			await jobs.dispose();
+			/* An open still in flight would assign its leases after this read, so
+			   settle it first; a failed open must not surface as an unhandled
+			   rejection during teardown. */
+			await repositoryPromise?.catch(() => undefined);
+			for (const lease of leases) await lease.release();
+			leases = [];
+			repositoryPromise = undefined;
+			servicePromise = undefined;
+		},
+	};
+}

package/modules/exports/src/services/data-classes.ts

@@ -0,0 +1,56 @@
+import type { DataClassDeclaration } from '@flowdular/sdk/kernel';
+import type { ExportService } from './export-service.ts';
+import type { ExportRepository } from './repository.ts';
+
+/** The class id is `exports.core.jobs`. */
+export const EXPORT_DATA_CLASS_KEY = 'jobs';
+
+export const EXPORT_RETENTION_DAYS = 30;
+
+/**
+ * Jobs one pass may remove. Each carries one stored file, so the bound on jobs
+ * is what keeps the storage work of a pass bounded as well.
+ */
+export const EXPORT_SWEEP_JOBS = 200;
+
+/**
+ * What this module holds: the export jobs of one workspace and the files they
+ * name. The rows of the exported lists belong to the modules that own them and
+ * are swept by their own classes, never by this one.
+ *
+ * Only a settled job is swept. A job still running is work in flight, and
+ * removing it under the poll loop would leave a stage writing a file against a
+ * row that is gone.
+ */
+export function exportsDataClass(
+	repository: () => Promise<ExportRepository>,
+	service: () => Promise<ExportService>,
+): DataClassDeclaration {
+	return {
+		key: EXPORT_DATA_CLASS_KEY,
+		label: 'Export jobs',
+		defaultRetentionDays: EXPORT_RETENTION_DAYS,
+		exportable: true,
+		sweep: async ({ tenantId, cutoff, limit }) => {
+			const open = await repository();
+			const batch = await open.claimSweepBatch(tenantId, {
+				settledBefore: cutoff.getTime(),
+				/* The platform's number is the caller's; this is the module's own
+				   bound on one pass. */
+				limit: Math.min(Math.max(Math.trunc(limit), 1), EXPORT_SWEEP_JOBS),
+			});
+			if (batch.ids.length === 0) return { removed: 0 };
+			/* The file goes before the row it belongs to. An interrupted pass then
+			   leaves a row naming a file that is gone, which the next pass removes
+			   and a read answers as gone; the other order would leave a file
+			   nothing references and no pass could ever find. */
+			await (await service()).discardObjects(tenantId, batch.objectIds);
+			return { removed: await open.deleteJobs(tenantId, batch.ids) };
+		},
+		/* Metadata only. The requester snapshot is bookkeeping for a background
+		   stage, and the exported files are the lists' own data, already covered
+		   by the classes of the modules that own them. */
+		export: async ({ tenantId, sink }) =>
+			(await repository()).exportJobs(tenantId, sink),
+	};
+}