@flowdular/sdk 0.2.3 → 0.3.0

package/modules/import/src/domain/csv.ts

@@ -0,0 +1,258 @@
+/**
+ * An RFC 4180 reader for the one shape this module accepts: UTF-8, a header
+ * row, comma separated, `"` quoting with `""` for a literal quote, CRLF or LF
+ * record separators. Anything else is a refusal with a stable code rather than
+ * a guess, because a file parsed the wrong way writes the wrong records.
+ */
+
+export type CsvErrorCode =
+	| 'CSV_EMPTY'
+	| 'CSV_HEADER_BLANK'
+	| 'CSV_HEADER_DUPLICATE'
+	| 'CSV_NOT_UTF8'
+	| 'CSV_QUOTE_UNTERMINATED'
+	| 'CSV_QUOTE_UNEXPECTED'
+	| 'CSV_TOO_LARGE'
+	| 'CSV_TOO_MANY_ROWS';
+
+export class CsvError extends Error {
+	constructor(
+		readonly code: CsvErrorCode,
+		message: string,
+	) {
+		super(message);
+		this.name = 'CsvError';
+	}
+}
+
+export interface CsvRow {
+	/** 1-based data row number, header excluded. */
+	readonly row: number;
+	readonly cells: readonly string[];
+}
+
+export interface CsvDocument {
+	readonly header: readonly string[];
+	readonly rows: readonly CsvRow[];
+}
+
+export interface CsvLimits {
+	/** Bytes of encoded CSV; the stream is abandoned the moment it passes this. */
+	readonly maxBytes: number;
+	/** Data rows, header excluded. */
+	readonly maxRows: number;
+}
+
+const BOM = '';
+
+/**
+ * Reads at most `maxBytes` and refuses anything longer, so a hostile body costs
+ * the cap rather than the file. The whole CSV is held in memory on purpose: the
+ * cap is 25 MB, one job's rows are held anyway, and a streaming parser would
+ * buy nothing against a bound this small.
+ */
+export async function readCsvBody(
+	body: ReadableStream<Uint8Array>,
+	maxBytes: number,
+): Promise<Uint8Array> {
+	const reader = body.getReader();
+	const chunks: Uint8Array[] = [];
+	let total = 0;
+	try {
+		for (;;) {
+			const { done, value } = await reader.read();
+			if (done) break;
+			if (!value) continue;
+			total += value.byteLength;
+			if (total > maxBytes) {
+				throw new CsvError(
+					'CSV_TOO_LARGE',
+					`The file is larger than ${maxBytes} bytes.`,
+				);
+			}
+			chunks.push(value);
+		}
+	} finally {
+		reader.releaseLock();
+		/* The body is abandoned on a refusal as well, so a producer blocked on
+		   backpressure is released rather than left holding the connection. */
+		await body.cancel().catch(() => undefined);
+	}
+	const bytes = new Uint8Array(total);
+	let offset = 0;
+	for (const chunk of chunks) {
+		bytes.set(chunk, offset);
+		offset += chunk.byteLength;
+	}
+	return bytes;
+}
+
+export function decodeCsv(bytes: Uint8Array): string {
+	let text: string;
+	try {
+		text = new TextDecoder('utf-8', { fatal: true }).decode(bytes);
+	} catch {
+		throw new CsvError('CSV_NOT_UTF8', 'The file is not valid UTF-8 text.');
+	}
+	return text.startsWith(BOM) ? text.slice(BOM.length) : text;
+}
+
+/* One pass, no regular expressions and no per-character array: O(n) in the
+   characters of the file and O(1) beyond the cells it produces.
+
+   A cell is cut out of the source text rather than appended character by
+   character: appending leaves every cell a tree of cons strings whose retained
+   size is an order of magnitude past the file (measured 124 MB of heap for a
+   5 MB CSV against 8.7 MB for these slices). `pending` carries only the pieces
+   a doubled quote forces this reader to join. */
+function parseRecords(text: string, maxRows: number): string[][] {
+	const records: string[][] = [];
+	let cells: string[] = [];
+	let start = 0;
+	let pending = '';
+	let quoted = false;
+	let started = false;
+	let index = 0;
+
+	/** Whether the cell being read has taken no character yet. */
+	const empty = () => pending === '' && start === index;
+	const endField = () => {
+		cells.push(
+			pending === ''
+				? text.slice(start, index)
+				: pending + text.slice(start, index),
+		);
+		pending = '';
+		started = true;
+	};
+	const endRecord = () => {
+		endField();
+		records.push(cells);
+		cells = [];
+		started = false;
+		/* The header is the first record, so the data bound is one past it. */
+		if (records.length > maxRows + 1) {
+			throw new CsvError(
+				'CSV_TOO_MANY_ROWS',
+				`The file holds more than ${maxRows} rows.`,
+			);
+		}
+	};
+
+	while (index < text.length) {
+		const character = text[index]!;
+		if (quoted) {
+			if (character !== '"') {
+				index += 1;
+				continue;
+			}
+			if (text[index + 1] === '"') {
+				/* The pair's first quote closes the piece and is the one literal
+				   character the cell keeps. */
+				pending += text.slice(start, index + 1);
+				index += 2;
+				start = index;
+				continue;
+			}
+			pending =
+				pending === ''
+					? text.slice(start, index)
+					: pending + text.slice(start, index);
+			quoted = false;
+			index += 1;
+			start = index;
+			/* Only a separator may follow a closing quote. `a"b"c` is a file whose
+			   author meant something this reader cannot know. */
+			const next = text[index];
+			if (
+				next !== undefined &&
+				next !== ',' &&
+				next !== '\n' &&
+				next !== '\r'
+			) {
+				throw new CsvError(
+					'CSV_QUOTE_UNEXPECTED',
+					`Unexpected text after a closing quote at offset ${index}.`,
+				);
+			}
+			continue;
+		}
+		if (character === '"') {
+			if (!empty()) {
+				throw new CsvError(
+					'CSV_QUOTE_UNEXPECTED',
+					`A quote may only open a field, at offset ${index}.`,
+				);
+			}
+			quoted = true;
+			started = true;
+			index += 1;
+			start = index;
+			continue;
+		}
+		if (character === ',') {
+			endField();
+			index += 1;
+			start = index;
+			continue;
+		}
+		if (character === '\r' || character === '\n') {
+			endRecord();
+			index += character === '\r' && text[index + 1] === '\n' ? 2 : 1;
+			start = index;
+			continue;
+		}
+		started = true;
+		index += 1;
+	}
+
+	if (quoted) {
+		throw new CsvError(
+			'CSV_QUOTE_UNTERMINATED',
+			'A quoted field is never closed.',
+		);
+	}
+	/* A file ending in a newline has no trailing empty record; one ending in
+	   text does have a last record. */
+	if (started || !empty() || cells.length > 0) endRecord();
+	return records;
+}
+
+/**
+ * Header cells are trimmed because whitespace around a column name is never
+ * what the author meant, and a mapping keyed on `" email"` would silently miss.
+ * Data cells keep every character they carried.
+ */
+export function parseCsv(text: string, limits: CsvLimits): CsvDocument {
+	const records = parseRecords(text, limits.maxRows);
+	const headerRecord = records[0];
+	if (!headerRecord || (headerRecord.length === 1 && headerRecord[0] === '')) {
+		throw new CsvError('CSV_EMPTY', 'The file carries no header row.');
+	}
+	const header = headerRecord.map((cell) => cell.trim());
+	const seen = new Set<string>();
+	for (const name of header) {
+		if (name === '') {
+			throw new CsvError('CSV_HEADER_BLANK', 'A column header is blank.');
+		}
+		if (seen.has(name)) {
+			throw new CsvError(
+				'CSV_HEADER_DUPLICATE',
+				`The column ${name} appears more than once.`,
+			);
+		}
+		seen.add(name);
+	}
+	const rows: CsvRow[] = [];
+	for (let index = 1; index < records.length; index += 1) {
+		rows.push({ row: index, cells: records[index]! });
+	}
+	return { header, rows };
+}
+
+export async function readCsv(
+	body: ReadableStream<Uint8Array>,
+	limits: CsvLimits,
+): Promise<CsvDocument> {
+	return parseCsv(decodeCsv(await readCsvBody(body, limits.maxBytes)), limits);
+}

package/modules/import/src/domain/ports.ts

@@ -0,0 +1,157 @@
+import type { AuthPrincipal } from '@flowdular/sdk/modules/auth';
+
+/**
+ * The public cross-module surface. A module that owns records registers its own
+ * import targets while it composes:
+ * `context.capabilities.get<ImportPorts>(IMPORT_PORTS_CAPABILITY)?.register(...)`.
+ *
+ * import.core never writes another module's records. It parses the CSV, maps
+ * the columns, checks the port's declared permission on the acting principal
+ * and then hands bounded batches of rows to the port, which applies its own
+ * invariants, its own natural key and its own tenant-scoped transaction.
+ */
+export const IMPORT_PORTS_CAPABILITY = 'import.ports.v1';
+
+/**
+ * What a field holds. The type is the port's declaration of shape, not its
+ * validation: import.core refuses a cell that cannot be that type at all, and
+ * the port still decides what a well-shaped value means.
+ */
+export const IMPORT_FIELD_TYPES = [
+	'string',
+	'integer',
+	'boolean',
+	'date',
+	'email',
+] as const;
+export type ImportFieldType = (typeof IMPORT_FIELD_TYPES)[number];
+
+/** How a row that already exists under the port's natural key is treated. */
+export const IMPORT_MODES = [
+	'create-only',
+	'update-existing',
+	'skip-existing',
+] as const;
+export type ImportMode = (typeof IMPORT_MODES)[number];
+
+export interface ImportField {
+	readonly id: string;
+	readonly label: string;
+	readonly required: boolean;
+	readonly type: ImportFieldType;
+}
+
+/**
+ * One CSV row after mapping. `row` is the 1-based data row number with the
+ * header excluded, so it is the number the reader counts in their file, and
+ * every verdict and outcome refers back to it. Values are the raw cell text
+ * keyed by field id; a column the mapping did not name is absent rather than
+ * empty, so a port can tell "not supplied" from "supplied blank".
+ */
+export interface ImportRow {
+	readonly row: number;
+	readonly values: Readonly<Record<string, string>>;
+}
+
+export interface ImportValidateInput {
+	readonly tenantId: string;
+	/**
+	 * The principal that started the job, as it stood at that moment. A port
+	 * checks its own record-level conditions against it; the declared
+	 * `permission` is already checked by import.core before this call.
+	 */
+	readonly principal: AuthPrincipal;
+	readonly rows: readonly ImportRow[];
+}
+
+/**
+ * One verdict per row. A port may answer fewer entries than it was given; a row
+ * it says nothing about is taken as valid, because a port reports what it
+ * refuses rather than restating what it accepts.
+ */
+export interface ImportValidation {
+	readonly row: number;
+	readonly verdict: 'valid' | 'invalid';
+	readonly field?: string;
+	readonly reason?: string;
+}
+
+export interface ImportWriteInput {
+	readonly tenantId: string;
+	readonly principal: AuthPrincipal;
+	readonly rows: readonly ImportRow[];
+	readonly mode: ImportMode;
+}
+
+/**
+ * One outcome per row. A row the port says nothing about is recorded as failed
+ * with a stable reason rather than silently counted as written: a port that
+ * drops a row has to be visible in the job.
+ */
+export interface ImportWriteOutcome {
+	readonly row: number;
+	readonly outcome: 'created' | 'updated' | 'skipped' | 'failed';
+	readonly recordRef?: string;
+	readonly reason?: string;
+}
+
+/**
+ * One import target. The target id a job names is `<moduleId>.<key>`, for
+ * example `users.core.members`.
+ *
+ * Guaranteed to the port: `validate` runs before any `write`; `write` is called
+ * only for rows the port itself called valid; both are called with batches no
+ * larger than the platform `batchSize` setting, in ascending row order, and
+ * never concurrently for one job. Deliberately unspecified: how many batches a
+ * job takes, whether two jobs of one workspace overlap, and how long the gap
+ * between validate and write is, so a port must not carry state between calls.
+ */
+export interface ImportPort {
+	readonly key: string;
+	readonly label: string;
+	/** The target module's own permission; import never widens what a member may write. */
+	readonly permission: string;
+	readonly fields: readonly ImportField[];
+	/** The fields whose values make a repeat idempotent. */
+	readonly naturalKey: readonly string[];
+	/**
+	 * The natural key of one row as this port compares two records of its own.
+	 * import.core refuses a row repeating an earlier row's key across the whole
+	 * file, and only the port knows what two values mean to it: an address it
+	 * stores folded matches whatever case the file carries. Left out, the key
+	 * fields are compared exactly, length prefixed so two fields cannot run into
+	 * one another.
+	 */
+	naturalKeyOf?(values: ImportRow['values']): string;
+	validate(input: ImportValidateInput): Promise<readonly ImportValidation[]>;
+	write(input: ImportWriteInput): Promise<readonly ImportWriteOutcome[]>;
+}
+
+export interface ImportPorts {
+	/**
+	 * Registers a module's targets. Registration is open while the platform
+	 * composes and sealed before the first request; a later call throws, so the
+	 * set of targets a workspace can choose from never changes under it.
+	 */
+	register(moduleId: string, ports: readonly ImportPort[]): void;
+}
+
+/** Longest values a port declaration may carry; over any of them is refused. */
+export const IMPORT_PORT_LIMITS = {
+	moduleId: 64,
+	key: 48,
+	target: 96,
+	label: 120,
+	permission: 96,
+	fieldId: 64,
+	fieldLabel: 120,
+	fields: 64,
+	naturalKey: 8,
+	portsPerModule: 32,
+	reason: 200,
+	recordRef: 200,
+} as const;
+
+export function importTargetId(moduleId: string, key: string): string {
+	return `${moduleId}.${key}`;
+}

package/modules/import/src/domain/types.ts

@@ -0,0 +1,179 @@
+import type { ImportMode } from './ports.ts';
+
+/** The stages a job moves through; the transitions live in the service. */
+export const IMPORT_JOB_STATUSES = [
+	'parsing',
+	'validated',
+	'writing',
+	'completed',
+	'failed',
+	'cancelled',
+] as const;
+export type ImportJobStatus = (typeof IMPORT_JOB_STATUSES)[number];
+
+/** What one row ended as. The first two are verdicts, the rest are writes. */
+export const IMPORT_ROW_OUTCOMES = [
+	'valid',
+	'invalid',
+	'created',
+	'updated',
+	'skipped',
+	'failed',
+] as const;
+export type ImportRowOutcome = (typeof IMPORT_ROW_OUTCOMES)[number];
+
+/** Longest values this module accepts; over any of them is a rejection. */
+export const IMPORT_LIMITS = {
+	id: 64,
+	target: 96,
+	documentId: 128,
+	accountId: 128,
+	field: 64,
+	reason: 200,
+	recordRef: 200,
+	failureCode: 64,
+	/** Serialized principal snapshot and column mapping, matching the schema. */
+	json: 8_192,
+	/** Rows one outcome page may carry; the platform page ceiling. */
+	page: 200,
+	/** Jobs one routing pass considers. */
+	routingPage: 25,
+} as const;
+
+/**
+ * The CSV bound the spec fixes. It is not a setting: the storage port refuses a
+ * larger object at upload, so raising it here would only move the refusal.
+ */
+export const IMPORT_MAX_CSV_BYTES = 25 * 1024 * 1024;
+
+/**
+ * The one content type an import source may carry. The screen uploads under it,
+ * the start request refuses anything else, and the stage that opens the file
+ * decides the same way, so a document refused at start is never a document a
+ * stage goes on to read.
+ */
+export const IMPORT_CSV_CONTENT_TYPE = 'text/csv';
+
+/**
+ * The acting principal as it stood when the job was started, kept so the port
+ * sees the same identity in a later stage that it would have seen in the
+ * request. Authorization is decided on the live principal at start and at
+ * continue; this snapshot carries the decision forward, it does not make one.
+ */
+export interface ImportRequester {
+	readonly accountId: string;
+	readonly email: string;
+	readonly displayName: string;
+	readonly role: string;
+	readonly scopes: readonly string[];
+}
+
+export interface ImportJob {
+	readonly id: string;
+	readonly tenantId: string;
+	readonly target: string;
+	readonly documentId: string;
+	/**
+	 * The record reference the CSV was uploaded against. documents.core
+	 * addresses an attachment by owner module, record reference and id, so the
+	 * job carries the reference its `documentId` is only resolvable under.
+	 */
+	readonly documentRef: string;
+	readonly mode: ImportMode;
+	readonly dryRun: boolean;
+	/** Whether `continue` asked for the valid rows alone. */
+	readonly validOnly: boolean;
+	readonly status: ImportJobStatus;
+	readonly totalRows: number;
+	readonly validRows: number;
+	readonly writtenRows: number;
+	readonly failedRows: number;
+	readonly requesterAccountId: string;
+	readonly requester: ImportRequester;
+	/** Port field id to CSV header, as the job was started with. */
+	readonly columns: Readonly<Record<string, string>>;
+	/** The stable code behind a failed job; null in every other status. */
+	readonly failureCode: string | null;
+	readonly claimedAt: number | null;
+	readonly startedAt: number;
+	readonly completedAt: number | null;
+	/**
+	 * The trace that enqueued the job, as a `traceparent` header value. Null
+	 * when the job was started outside a traced scope, which the claim reads as
+	 * a new root.
+	 */
+	readonly traceparent: string | null;
+}
+
+/**
+ * A job as the poll loop holds it. The claim instant is the fence every renewal
+ * of that claim is made under, so a stage can tell that its lease lapsed and the
+ * job changed hands.
+ */
+export type ClaimedImportJob = ImportJob & { readonly claimedAt: number };
+
+/** What a stage records when the failure carries no stable code of its own. */
+export const IMPORT_UNEXPECTED_FAILURE = 'JOB_FAILED';
+
+/**
+ * The job as an API response carries it. The requester snapshot is bookkeeping
+ * a port needs in a background stage and no reader of the screen does, and the
+ * claim and the trace it was enqueued in are the poll loop's own, so none of
+ * them crosses the wire: a job read names the requester by account id alone.
+ */
+export type ImportJobView = Omit<
+	ImportJob,
+	'requester' | 'claimedAt' | 'traceparent'
+>;
+
+/* Every field is named rather than spread, so a field added to the job is a
+   decision taken here instead of an addition the wire makes silently. */
+export function importJobView(job: ImportJob): ImportJobView {
+	return {
+		id: job.id,
+		tenantId: job.tenantId,
+		target: job.target,
+		documentId: job.documentId,
+		documentRef: job.documentRef,
+		mode: job.mode,
+		dryRun: job.dryRun,
+		validOnly: job.validOnly,
+		status: job.status,
+		totalRows: job.totalRows,
+		validRows: job.validRows,
+		writtenRows: job.writtenRows,
+		failedRows: job.failedRows,
+		requesterAccountId: job.requesterAccountId,
+		columns: job.columns,
+		failureCode: job.failureCode,
+		startedAt: job.startedAt,
+		completedAt: job.completedAt,
+	};
+}
+
+export interface ImportJobRow {
+	readonly id: string;
+	readonly tenantId: string;
+	readonly jobId: string;
+	readonly rowNumber: number;
+	readonly outcome: ImportRowOutcome;
+	readonly field: string | null;
+	readonly reason: string | null;
+	readonly recordRef: string | null;
+}
+
+export interface ImportMapping {
+	readonly id: string;
+	readonly tenantId: string;
+	readonly target: string;
+	readonly columns: Readonly<Record<string, string>>;
+	readonly updatedAt: number;
+}
+
+/** What the cross-tenant routing read may see, and nothing else. */
+export interface ImportJobRouting {
+	readonly tenantId: string;
+	readonly id: string;
+	readonly status: ImportJobStatus;
+	readonly startedAt: number;
+}

package/modules/import/src/index.ts

@@ -0,0 +1,58 @@
+import type { ModuleManifest, RegisteredModule } from '@flowdular/sdk/contracts';
+import manifest from '../module.json' with { type: 'json' };
+import { IMPORT_PERMISSIONS } from './acl/permissions.ts';
+
+export const moduleDefinition = {
+	manifest: manifest as ModuleManifest,
+	navigation: [
+		{
+			id: 'import.navigation',
+			label: 'Imports',
+			href: '/imports',
+			order: 60,
+			permission: IMPORT_PERMISSIONS.read,
+		},
+	],
+	permissions: Object.values(IMPORT_PERMISSIONS),
+} satisfies RegisteredModule;
+
+export { IMPORT_PERMISSIONS } from './acl/permissions.ts';
+
+/* The cross-module contract. A module that owns records imports the identifier
+   and the types from here and registers its targets through the capability
+   registry; nothing else in this module is meant to be imported by another. */
+export {
+	IMPORT_FIELD_TYPES,
+	IMPORT_MODES,
+	IMPORT_PORTS_CAPABILITY,
+	IMPORT_PORT_LIMITS,
+	importTargetId,
+} from './domain/ports.ts';
+export type {
+	ImportField,
+	ImportFieldType,
+	ImportMode,
+	ImportPort,
+	ImportPorts,
+	ImportRow,
+	ImportValidateInput,
+	ImportValidation,
+	ImportWriteInput,
+	ImportWriteOutcome,
+} from './domain/ports.ts';
+
+export {
+	IMPORT_CSV_CONTENT_TYPE,
+	IMPORT_JOB_STATUSES,
+	IMPORT_LIMITS,
+	IMPORT_MAX_CSV_BYTES,
+	IMPORT_ROW_OUTCOMES,
+} from './domain/types.ts';
+export type {
+	ClaimedImportJob,
+	ImportJob,
+	ImportJobRow,
+	ImportJobStatus,
+	ImportMapping,
+	ImportRowOutcome,
+} from './domain/types.ts';