@flowdular/sdk 0.2.4 → 0.3.1

package/modules/import/src/services/index.ts

@@ -0,0 +1,6 @@
+export { ImportService, ImportServiceError } from './import-service.ts';
+export type { ImportRepository } from './repository.ts';
+export {
+	DatabaseImportRepository,
+	migrateImportDatabase,
+} from './database-repository.ts';

package/modules/import/src/services/migration.ts

@@ -0,0 +1,156 @@
+import type { DatabaseMigration } from '@flowdular/sdk/database';
+import { postgresTenantTableState } from '@flowdular/sdk/database';
+
+/* Every constant mirrors its migrations/<id>.up.sql file byte for byte;
+   tests/migrations.test.ts fails on drift. */
+
+export const IMPORT_MIGRATION_001 = `CREATE TABLE IF NOT EXISTS import_jobs (
+  id TEXT PRIMARY KEY,
+  tenant_id TEXT NOT NULL,
+  target TEXT NOT NULL CHECK (length(target) BETWEEN 3 AND 96),
+  document_id TEXT NOT NULL CHECK (length(document_id) BETWEEN 1 AND 128),
+  document_ref TEXT NOT NULL CHECK (length(document_ref) BETWEEN 1 AND 200),
+  mode TEXT NOT NULL CHECK (mode IN ('create-only', 'update-existing', 'skip-existing')),
+  dry_run SMALLINT NOT NULL CHECK (dry_run IN (0, 1)),
+  valid_only SMALLINT NOT NULL CHECK (valid_only IN (0, 1)),
+  status TEXT NOT NULL CHECK (status IN ('parsing', 'validated', 'writing', 'completed', 'failed', 'cancelled')),
+  total_rows BIGINT NOT NULL CHECK (total_rows >= 0),
+  valid_rows BIGINT NOT NULL CHECK (valid_rows >= 0),
+  written_rows BIGINT NOT NULL CHECK (written_rows >= 0),
+  failed_rows BIGINT NOT NULL CHECK (failed_rows >= 0),
+  requester_account_id TEXT NOT NULL CHECK (length(requester_account_id) BETWEEN 1 AND 128),
+  requester_json TEXT NOT NULL CHECK (length(requester_json) <= 8192 AND requester_json::jsonb IS NOT NULL),
+  columns_json TEXT NOT NULL CHECK (length(columns_json) <= 8192 AND columns_json::jsonb IS NOT NULL),
+  failure_code TEXT CHECK (failure_code IS NULL OR length(failure_code) BETWEEN 1 AND 64),
+  claimed_at BIGINT,
+  started_at BIGINT NOT NULL,
+  completed_at BIGINT,
+  CHECK ((status IN ('completed', 'failed', 'cancelled')) = (completed_at IS NOT NULL))
+);
+CREATE INDEX IF NOT EXISTS import_jobs_tenant_started_idx
+  ON import_jobs (tenant_id, started_at DESC, id);
+ALTER TABLE import_jobs ENABLE ROW LEVEL SECURITY;
+ALTER TABLE import_jobs FORCE ROW LEVEL SECURITY;
+CREATE POLICY import_jobs_tenant_policy ON import_jobs
+  USING (tenant_id = current_setting('coreloom.tenant_id', true))
+  WITH CHECK (tenant_id = current_setting('coreloom.tenant_id', true));
+
+CREATE TABLE IF NOT EXISTS import_job_rows (
+  id TEXT PRIMARY KEY,
+  tenant_id TEXT NOT NULL,
+  job_id TEXT NOT NULL,
+  row_number BIGINT NOT NULL CHECK (row_number >= 1),
+  outcome TEXT NOT NULL CHECK (outcome IN ('valid', 'invalid', 'created', 'updated', 'skipped', 'failed')),
+  field TEXT CHECK (field IS NULL OR length(field) BETWEEN 1 AND 64),
+  reason TEXT CHECK (reason IS NULL OR length(reason) BETWEEN 1 AND 200),
+  record_ref TEXT CHECK (record_ref IS NULL OR length(record_ref) BETWEEN 1 AND 200),
+  UNIQUE (tenant_id, job_id, row_number)
+);
+ALTER TABLE import_job_rows ENABLE ROW LEVEL SECURITY;
+ALTER TABLE import_job_rows FORCE ROW LEVEL SECURITY;
+CREATE POLICY import_job_rows_tenant_policy ON import_job_rows
+  USING (tenant_id = current_setting('coreloom.tenant_id', true))
+  WITH CHECK (tenant_id = current_setting('coreloom.tenant_id', true));
+
+CREATE TABLE IF NOT EXISTS import_mappings (
+  id TEXT PRIMARY KEY,
+  tenant_id TEXT NOT NULL,
+  target TEXT NOT NULL CHECK (length(target) BETWEEN 3 AND 96),
+  columns_json TEXT NOT NULL CHECK (length(columns_json) <= 8192 AND columns_json::jsonb IS NOT NULL),
+  updated_at BIGINT NOT NULL,
+  UNIQUE (tenant_id, target)
+);
+ALTER TABLE import_mappings ENABLE ROW LEVEL SECURITY;
+ALTER TABLE import_mappings FORCE ROW LEVEL SECURITY;
+CREATE POLICY import_mappings_tenant_policy ON import_mappings
+  USING (tenant_id = current_setting('coreloom.tenant_id', true))
+  WITH CHECK (tenant_id = current_setting('coreloom.tenant_id', true));
+`;
+
+export const IMPORT_MIGRATION_002_JOB_ROUTING_ROLE = `-- The poll loop must find queued jobs across workspaces before it knows whose
+-- they are, so it reads the routing columns alone on the background role: the
+-- document, the target, the mapping, the requester and every row outcome stay
+-- invisible to it, and each job it picks is claimed and processed again under
+-- the workspace the routing row named. PostgreSQL checks column privileges in
+-- WHERE too, so \`status\` is part of the grant.
+CREATE INDEX IF NOT EXISTS import_jobs_routing_idx
+  ON import_jobs (status, started_at, tenant_id, id);
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'coreloom_background') THEN
+    RAISE EXCEPTION 'The coreloom_background role must exist before this migration.';
+  END IF;
+END
+$$;
+CREATE POLICY import_jobs_background_policy ON import_jobs
+  FOR SELECT TO coreloom_background
+  USING (status IN ('parsing', 'writing'));
+REVOKE SELECT ON import_jobs FROM coreloom_background;
+GRANT SELECT (tenant_id, id, status, started_at) ON import_jobs TO coreloom_background;
+`;
+
+export const IMPORT_MIGRATION_003_EXPORT_INDEX = `-- The data class export walks a workspace oldest first by (started_at, id) and
+-- the retention sweep picks its batch by the same key. The listing index orders
+-- started_at descending, and reading it backwards pairs an ascending started_at
+-- with a descending id, so neither walk can be served by it. The outcome and
+-- mapping walks keep using the unique keys their tables already carry.
+CREATE INDEX IF NOT EXISTS import_jobs_tenant_export_idx
+  ON import_jobs (tenant_id, started_at, id);
+`;
+
+export const IMPORT_MIGRATION_004_TRACEPARENT = `-- The trace that enqueued the job, as a W3C traceparent header value, so the
+-- pass that claims it can be read against the request that started it. Nullable
+-- because a job enqueued outside a traced scope, and every job that predates
+-- this column, is a new root. It is written once at insert and never used as a
+-- predicate, so it carries no index and stays off the background routing grant:
+-- the claim returns it under the tenant that owns the job.
+ALTER TABLE import_jobs ADD COLUMN IF NOT EXISTS traceparent text;
+`;
+
+export const databaseMigrations: readonly DatabaseMigration[] = [
+	{
+		id: '0001_import_core',
+		sql: { postgresql: IMPORT_MIGRATION_001 },
+		inspectExisting: (database) =>
+			postgresTenantTableState(
+				database,
+				'import_jobs',
+				'import_jobs_tenant_policy',
+				[
+					() => database.schema.hasTable('import_job_rows'),
+					() => database.schema.hasTable('import_mappings'),
+					() => database.schema.hasIndex('import_jobs_tenant_started_idx'),
+				],
+			),
+	},
+	{
+		id: '0002_import_job_routing_role',
+		sql: { postgresql: IMPORT_MIGRATION_002_JOB_ROUTING_ROLE },
+		/* A policy and a column grant leave no schema object behind, so the
+		   privilege itself is what proves this migration ran. */
+		inspectExisting: async (database) => {
+			const result = await database.query<{ granted: boolean }>({
+				text: `SELECT CASE WHEN to_regclass('import_jobs') IS NOT NULL THEN
+				  has_column_privilege('coreloom_background', 'import_jobs', 'status', 'SELECT')
+				ELSE false END AS granted`,
+			});
+			return result.rows[0]?.granted ? 'complete' : 'absent';
+		},
+	},
+	{
+		id: '0003_import_jobs_export_index',
+		sql: { postgresql: IMPORT_MIGRATION_003_EXPORT_INDEX },
+		inspectExisting: async (database) =>
+			(await database.schema.hasIndex('import_jobs_tenant_export_idx'))
+				? 'complete'
+				: 'absent',
+	},
+	{
+		id: '0004_import_jobs_traceparent',
+		sql: { postgresql: IMPORT_MIGRATION_004_TRACEPARENT },
+		inspectExisting: async (database) =>
+			(await database.schema.hasColumn('import_jobs', 'traceparent'))
+				? 'complete'
+				: 'absent',
+	},
+];

package/modules/import/src/services/port-registry.ts

@@ -0,0 +1,158 @@
+import {
+	IMPORT_FIELD_TYPES,
+	IMPORT_PORT_LIMITS,
+	importTargetId,
+	type ImportPort,
+	type ImportPorts,
+} from '../domain/ports.ts';
+
+export class ImportPortError extends Error {
+	constructor(
+		readonly code: string,
+		message: string,
+		readonly status = 400,
+	) {
+		super(message);
+		this.name = 'ImportPortError';
+	}
+}
+
+/** A target and the module that registered it, resolved by target id. */
+export interface RegisteredImportPort {
+	readonly target: string;
+	readonly moduleId: string;
+	readonly port: ImportPort;
+}
+
+export interface ImportPortRegistry extends ImportPorts {
+	/** Closes registration. Every later `register` throws. */
+	seal(): void;
+	find(target: string): RegisteredImportPort | null;
+	list(): readonly RegisteredImportPort[];
+}
+
+function bounded(value: string, label: string, max: number): string {
+	if (typeof value !== 'string' || value.length === 0 || value.length > max) {
+		throw new ImportPortError(
+			'IMPORT_PORT_INVALID',
+			`An import port ${label} must be 1 to ${max} characters.`,
+			500,
+		);
+	}
+	return value;
+}
+
+function assertPort(moduleId: string, port: ImportPort): void {
+	bounded(port.key, 'key', IMPORT_PORT_LIMITS.key);
+	bounded(port.label, 'label', IMPORT_PORT_LIMITS.label);
+	bounded(port.permission, 'permission', IMPORT_PORT_LIMITS.permission);
+	if (
+		port.fields.length === 0 ||
+		port.fields.length > IMPORT_PORT_LIMITS.fields
+	) {
+		throw new ImportPortError(
+			'IMPORT_PORT_INVALID',
+			`An import port declares 1 to ${IMPORT_PORT_LIMITS.fields} fields.`,
+			500,
+		);
+	}
+	const ids = new Set<string>();
+	for (const field of port.fields) {
+		bounded(field.id, 'field id', IMPORT_PORT_LIMITS.fieldId);
+		bounded(field.label, 'field label', IMPORT_PORT_LIMITS.fieldLabel);
+		if (!(IMPORT_FIELD_TYPES as readonly string[]).includes(field.type)) {
+			throw new ImportPortError(
+				'IMPORT_PORT_INVALID',
+				`The field ${field.id} declares an unknown type.`,
+				500,
+			);
+		}
+		if (ids.has(field.id)) {
+			throw new ImportPortError(
+				'IMPORT_PORT_INVALID',
+				`The field ${field.id} is declared twice.`,
+				500,
+			);
+		}
+		ids.add(field.id);
+	}
+	if (
+		port.naturalKey.length === 0 ||
+		port.naturalKey.length > IMPORT_PORT_LIMITS.naturalKey
+	) {
+		throw new ImportPortError(
+			'IMPORT_PORT_INVALID',
+			'An import port declares a natural key of 1 to ' +
+				`${IMPORT_PORT_LIMITS.naturalKey} fields.`,
+			500,
+		);
+	}
+	/* A natural key naming a field the port does not accept can never make a
+	   repeat idempotent, so it is refused where it is declared rather than
+	   discovered on the second import. */
+	for (const field of port.naturalKey) {
+		if (!ids.has(field)) {
+			throw new ImportPortError(
+				'IMPORT_PORT_INVALID',
+				`The natural key names ${field}, which is not a declared field.`,
+				500,
+			);
+		}
+	}
+	if (importTargetId(moduleId, port.key).length > IMPORT_PORT_LIMITS.target) {
+		throw new ImportPortError(
+			'IMPORT_PORT_INVALID',
+			`The target id is longer than ${IMPORT_PORT_LIMITS.target} characters.`,
+			500,
+		);
+	}
+}
+
+/**
+ * The targets a workspace can import into. Registration is open while the
+ * platform composes and sealed before the first request, so the target list a
+ * job was started against cannot change under it. Lookup is a map read, O(1).
+ */
+export function createImportPortRegistry(): ImportPortRegistry {
+	const ports = new Map<string, RegisteredImportPort>();
+	let sealed = false;
+	return {
+		register(moduleId, registered) {
+			if (sealed) {
+				throw new ImportPortError(
+					'IMPORT_PORTS_SEALED',
+					'Import ports are registered while the platform composes.',
+					500,
+				);
+			}
+			bounded(moduleId, 'module id', IMPORT_PORT_LIMITS.moduleId);
+			if (registered.length > IMPORT_PORT_LIMITS.portsPerModule) {
+				throw new ImportPortError(
+					'IMPORT_PORT_INVALID',
+					`A module registers at most ${IMPORT_PORT_LIMITS.portsPerModule} ports.`,
+					500,
+				);
+			}
+			for (const port of registered) {
+				assertPort(moduleId, port);
+				const target = importTargetId(moduleId, port.key);
+				if (ports.has(target)) {
+					throw new ImportPortError(
+						'IMPORT_PORT_DUPLICATE',
+						`The import target ${target} is already registered.`,
+						500,
+					);
+				}
+				ports.set(target, { target, moduleId, port });
+			}
+		},
+		seal() {
+			sealed = true;
+		},
+		find: (target) => ports.get(target) ?? null,
+		list: () =>
+			[...ports.values()].sort((left, right) =>
+				left.target.localeCompare(right.target),
+			),
+	};
+}

package/modules/import/src/services/repository.ts

@@ -0,0 +1,147 @@
+import type { DataClassExportSink } from '@flowdular/sdk/kernel';
+import type {
+	ClaimedImportJob,
+	ImportJob,
+	ImportJobRouting,
+	ImportJobRow,
+	ImportJobStatus,
+	ImportMapping,
+	ImportRowOutcome,
+} from '../domain/types.ts';
+
+export interface ImportJobPage {
+	readonly items: readonly ImportJob[];
+	readonly nextCursor: JobCursor | null;
+}
+
+/** The keyset of the last job on a page: newest first, id breaking ties. */
+export interface JobCursor {
+	readonly startedAt: number;
+	readonly id: string;
+}
+
+export interface ImportJobQuery {
+	readonly status?: ImportJobStatus | undefined;
+	readonly target?: string | undefined;
+	readonly limit: number;
+	readonly cursor?: JobCursor | undefined;
+}
+
+export interface ImportRowPage {
+	readonly items: readonly ImportJobRow[];
+	readonly nextCursor: number | null;
+}
+
+export interface ImportJobResult {
+	readonly status: ImportJobStatus;
+	readonly totalRows: number;
+	readonly validRows: number;
+	readonly writtenRows: number;
+	readonly failedRows: number;
+	readonly failureCode: string | null;
+	readonly completedAt: number | null;
+}
+
+/** How many rows of one job ended in each outcome; every outcome is present. */
+export type ImportRowOutcomeCounts = Readonly<Record<ImportRowOutcome, number>>;
+
+export interface ImportSweepInput {
+	/** Settled jobs started strictly before this go, with their outcomes. */
+	readonly settledBefore: number;
+	/**
+	 * A job validated but never continued by this instant is cancelled, so an
+	 * abandoned job settles instead of waiting for its retention alone.
+	 */
+	readonly abandonedBefore: number;
+	/** What an expiry records as the job's completion. */
+	readonly at: number;
+	/** Upper bound on the jobs one pass expires and on the jobs it removes. */
+	readonly limit: number;
+}
+
+export interface ClaimJobInput {
+	readonly tenantId: string;
+	readonly id: string;
+	readonly claimedAt: number;
+	/** A claim at or before this belonged to a process that is gone. */
+	readonly staleBefore: number;
+}
+
+/**
+ * The business port: async, database-agnostic, and unaware of SQL. Every method
+ * is tenant-scoped except `listPendingJobs`, which crosses workspaces on the
+ * background role and answers routing columns alone.
+ */
+export interface ImportRepository {
+	createJob(job: ImportJob): Promise<ImportJob>;
+	findJob(tenantId: string, id: string): Promise<ImportJob | null>;
+	listJobs(tenantId: string, query: ImportJobQuery): Promise<ImportJobPage>;
+	/** Moves a job between stages; answers null when the status no longer matches. */
+	advanceJob(
+		tenantId: string,
+		id: string,
+		from: readonly ImportJobStatus[],
+		result: ImportJobResult,
+	): Promise<ImportJob | null>;
+	/** Records the requester's choice when a validated job is continued. */
+	continueJob(
+		tenantId: string,
+		id: string,
+		validOnly: boolean,
+	): Promise<ImportJob | null>;
+	claimJob(input: ClaimJobInput): Promise<ClaimedImportJob | null>;
+	/**
+	 * Renews the claim of a job that is still in the stage it was claimed for and
+	 * still carries the claim the caller was given, so a stage longer than the
+	 * claim lease is not taken up twice and a stage whose lease already lapsed
+	 * cannot renew over the loop that reclaimed the job. Answers whether the
+	 * renewal landed; a job that left the stage or changed hands is not touched.
+	 */
+	heartbeatJob(
+		tenantId: string,
+		id: string,
+		at: number,
+		claimedAt: number,
+	): Promise<boolean>;
+	listPendingJobs(limit: number): Promise<readonly ImportJobRouting[]>;
+
+	replaceJobRows(
+		tenantId: string,
+		jobId: string,
+		rows: readonly ImportJobRow[],
+	): Promise<void>;
+	recordJobRows(tenantId: string, rows: readonly ImportJobRow[]): Promise<void>;
+	listJobRows(
+		tenantId: string,
+		jobId: string,
+		limit: number,
+		after?: number,
+	): Promise<ImportRowPage>;
+	/** The row numbers a job called valid, in ascending order. */
+	listValidRowNumbers(
+		tenantId: string,
+		jobId: string,
+	): Promise<readonly number[]>;
+	/**
+	 * The outcomes of one job counted in one read, so a stage that resumed after
+	 * a crash reports the whole job rather than the batches it ran itself.
+	 */
+	countRowOutcomes(
+		tenantId: string,
+		jobId: string,
+	): Promise<ImportRowOutcomeCounts>;
+
+	findMapping(tenantId: string, target: string): Promise<ImportMapping | null>;
+	saveMapping(mapping: ImportMapping): Promise<ImportMapping>;
+
+	/**
+	 * One retention pass: abandoned validated jobs are cancelled, then settled
+	 * jobs older than the cutoff go with their outcomes. Answers how many jobs
+	 * were removed.
+	 */
+	sweepJobs(tenantId: string, input: ImportSweepInput): Promise<number>;
+	exportJobs(
+		tenantId: string,
+		sink: DataClassExportSink,
+	): Promise<{ rows: number; from: Date | null; to: Date | null }>;
+}

package/modules/import/src/settings.ts

@@ -0,0 +1,70 @@
+import {
+	defineModuleSettings,
+	PLATFORM_SETTINGS_TENANT,
+	type ModuleSettingsRuntime,
+} from '@flowdular/sdk/kernel';
+
+export const IMPORT_MODULE_ID = 'import.core';
+
+export const DEFAULT_IMPORT_MAX_ROWS = 50_000;
+export const DEFAULT_IMPORT_BATCH_SIZE = 500;
+
+export const IMPORT_MODULE_SETTINGS = defineModuleSettings({
+	moduleId: IMPORT_MODULE_ID,
+	settings: {
+		maxRows: {
+			type: 'number',
+			defaultValue: DEFAULT_IMPORT_MAX_ROWS,
+			min: 100,
+			max: 500_000,
+			visibility: 'shared',
+			client: false,
+			scope: 'platform',
+			labelKey: 'import.settings.maxRows.label',
+			label: 'Rows per job',
+			descriptionKey: 'import.settings.maxRows.description',
+			description: 'Rows one import job may hold; a larger file is refused.',
+		},
+		batchSize: {
+			type: 'number',
+			defaultValue: DEFAULT_IMPORT_BATCH_SIZE,
+			min: 50,
+			max: 5_000,
+			visibility: 'shared',
+			client: false,
+			scope: 'platform',
+			labelKey: 'import.settings.batchSize.label',
+			label: 'Rows per batch',
+			descriptionKey: 'import.settings.batchSize.description',
+			description:
+				'Rows handed to an import port in one validate or write call.',
+		},
+	},
+});
+
+/* A setting read must never take a running job down, so both accessors fall
+   back to the declared default when the runtime has not registered the module
+   yet. */
+function value(
+	settings: ModuleSettingsRuntime,
+	key: 'maxRows' | 'batchSize',
+	fallback: number,
+): number {
+	try {
+		return settings.get<number>(
+			PLATFORM_SETTINGS_TENANT,
+			IMPORT_MODULE_ID,
+			key,
+		);
+	} catch {
+		return fallback;
+	}
+}
+
+export function importMaxRows(settings: ModuleSettingsRuntime): number {
+	return value(settings, 'maxRows', DEFAULT_IMPORT_MAX_ROWS);
+}
+
+export function importBatchSize(settings: ModuleSettingsRuntime): number {
+	return value(settings, 'batchSize', DEFAULT_IMPORT_BATCH_SIZE);
+}