@flowdular/sdk 0.2.4 → 0.3.1

package/packages/storage/src/port.ts

@@ -0,0 +1,262 @@
+import { KeyringError, type Keyring } from '@flowdular/sdk/kernel';
+import {
+	StorageError,
+	storageObjectKey,
+	type StorageObjectRef,
+	type StoragePort,
+	type StoragePutInput,
+	type StorageReadResult,
+	type StorageReadUrlInput,
+	type StorageScanVerdict,
+	type StoredObject,
+} from './contracts.ts';
+import {
+	contentMatchesType,
+	contentTypeAllowed,
+	normalizeContentType,
+} from './content-type.ts';
+import {
+	encodeStoredObject,
+	openStoredObject,
+	readStoredObjectHeader,
+	STORAGE_HEADER_PREFIX_BYTES,
+} from './envelope.ts';
+import { createLocalObjectStore } from './local.ts';
+import { createS3ObjectStore } from './s3.ts';
+import { mintStorageReadToken, storageReadUrl } from './read-token.ts';
+import { createStorageResealer, type StorageResealPort } from './reseal.ts';
+import { unscannedStorageScanner, type StorageScanner } from './scanner.ts';
+import type { StorageConfig } from './config.ts';
+import type { ObjectStore } from './store.ts';
+
+export interface StoragePortOptions {
+	readonly keyring: Keyring;
+	readonly scanner?: StorageScanner | undefined;
+	readonly clock?: (() => Date) | undefined;
+	/** Test seam for the S3 adapter. A deployment never sets it. */
+	readonly fetch?: typeof globalThis.fetch | undefined;
+}
+
+/**
+ * Reads at most `limit` bytes and refuses as soon as one more arrives. The
+ * source is cancelled at that point, so an oversized upload is not read to the
+ * end and never reaches the object store.
+ */
+async function readBounded(
+	body: ReadableStream<Uint8Array> | Uint8Array,
+	limit: number,
+): Promise<Uint8Array> {
+	if (!(body instanceof ReadableStream)) {
+		if (body.byteLength > limit) throw tooLarge(limit);
+		return body;
+	}
+	const reader = body.getReader();
+	const chunks: Uint8Array[] = [];
+	let total = 0;
+	try {
+		for (;;) {
+			const { value, done } = await reader.read();
+			if (done) break;
+			if (!value) continue;
+			total += value.byteLength;
+			if (total > limit) {
+				await reader.cancel();
+				throw tooLarge(limit);
+			}
+			chunks.push(value);
+		}
+	} finally {
+		reader.releaseLock();
+	}
+	return Buffer.concat(chunks, total);
+}
+
+function tooLarge(limit: number): StorageError {
+	return new StorageError(
+		'OBJECT_TOO_LARGE',
+		`An object may not exceed ${limit} bytes.`,
+	);
+}
+
+function streamOf(bytes: Uint8Array): ReadableStream<Uint8Array> {
+	return new ReadableStream<Uint8Array>({
+		start(controller) {
+			controller.enqueue(bytes);
+			controller.close();
+		},
+	});
+}
+
+async function verdictOf(
+	scanner: StorageScanner,
+	plaintext: Uint8Array,
+): Promise<StorageScanVerdict> {
+	/* A scanner is deployment-supplied code. A throw is not a clean verdict and
+	   must not be read as one, so the write is refused rather than downgraded. */
+	let result;
+	try {
+		result = await scanner.scan(streamOf(plaintext));
+	} catch (error) {
+		throw new StorageError(
+			'SCAN_FAILED',
+			'The configured scanner did not return a verdict.',
+			{ cause: error },
+		);
+	}
+	if (result.verdict === 'infected') {
+		throw new StorageError(
+			'OBJECT_INFECTED',
+			'The scanner reported the content as infected.',
+		);
+	}
+	return result.verdict;
+}
+
+function objectStoreFor(
+	config: StorageConfig,
+	options: StoragePortOptions,
+): ObjectStore {
+	if (config.adapter === 'local') {
+		return createLocalObjectStore(config.local.directory);
+	}
+	return createS3ObjectStore({
+		...config.s3,
+		fetch: options.fetch,
+		clock: options.clock,
+	});
+}
+
+/** The port a deployment builds: the module contract plus the operator pass. */
+export interface ManagedStoragePort extends StoragePort, StorageResealPort {}
+
+/**
+ * The one place the storage rules live: the tenant-first key layout, the size
+ * limit, the content type allowlist with magic byte verification, the scanner
+ * verdict and encryption under the keyring. Adapters below it only move frames.
+ */
+export function createStoragePort(
+	config: StorageConfig,
+	options: StoragePortOptions,
+): ManagedStoragePort {
+	const store = objectStoreFor(config, options);
+	const scanner = options.scanner ?? unscannedStorageScanner;
+	const clock = options.clock ?? (() => new Date());
+	const keyring = options.keyring;
+	const resealer = createStorageResealer(store, keyring);
+	let disposed = false;
+
+	const live = (): void => {
+		if (disposed) {
+			throw new StorageError(
+				'STORAGE_DISPOSED',
+				'The storage port was disposed.',
+			);
+		}
+	};
+
+	const openFrame = (
+		key: string,
+		frame: Uint8Array,
+	): { readonly object: StoredObject; readonly plaintext: Buffer } => {
+		try {
+			return openStoredObject(keyring, key, frame);
+		} catch (error) {
+			if (error instanceof KeyringError) {
+				throw new StorageError(
+					'OBJECT_CORRUPT',
+					`The stored object did not open under this keyring (${error.code}).`,
+					{ cause: error },
+				);
+			}
+			throw error;
+		}
+	};
+
+	return {
+		async put(input: StoragePutInput): Promise<StoredObject> {
+			live();
+			const key = storageObjectKey(input);
+			const contentType = normalizeContentType(input.contentType);
+			if (!contentType || !contentTypeAllowed(contentType)) {
+				throw new StorageError(
+					'CONTENT_TYPE_REFUSED',
+					`The content type ${input.contentType} is not stored by this platform.`,
+				);
+			}
+			if (
+				input.declaredBytes !== undefined &&
+				input.declaredBytes > config.maxObjectBytes
+			) {
+				throw tooLarge(config.maxObjectBytes);
+			}
+			const plaintext = await readBounded(input.body, config.maxObjectBytes);
+			if (!contentMatchesType(contentType, plaintext)) {
+				throw new StorageError(
+					'CONTENT_MISMATCH',
+					`The content does not carry the structure of ${contentType}.`,
+				);
+			}
+			const scan = await verdictOf(scanner, plaintext);
+			const { frame, object } = encodeStoredObject(keyring, key, plaintext, {
+				contentType,
+				scan,
+				createdAt: clock(),
+			});
+			await store.write(key, frame);
+			return object;
+		},
+
+		async get(input: StorageObjectRef): Promise<StorageReadResult | null> {
+			live();
+			const key = storageObjectKey(input);
+			const frame = await store.read(key);
+			if (!frame) return null;
+			/* The body is authenticated as a whole, so it is opened before a byte
+			   of plaintext leaves: streaming out an unverified prefix would hand a
+			   caller content the tag has not covered yet. The object limit bounds
+			   what this holds. */
+			const { object, plaintext } = openFrame(key, frame);
+			return { object, body: streamOf(plaintext) };
+		},
+
+		async delete(input: StorageObjectRef): Promise<boolean> {
+			live();
+			return store.remove(storageObjectKey(input));
+		},
+
+		readUrl(input: StorageReadUrlInput): Promise<string> {
+			live();
+			storageObjectKey(input);
+			return Promise.resolve(
+				storageReadUrl(
+					mintStorageReadToken(keyring, {
+						tenantId: input.tenantId,
+						moduleId: input.moduleId,
+						objectId: input.objectId,
+						expiresInSeconds: input.expiresInSeconds,
+						now: clock(),
+					}),
+				),
+			);
+		},
+
+		async stat(input: StorageObjectRef): Promise<StoredObject | null> {
+			live();
+			const key = storageObjectKey(input);
+			const prefix = await store.read(key, STORAGE_HEADER_PREFIX_BYTES);
+			return prefix ? readStoredObjectHeader(key, prefix) : null;
+		},
+
+		keyId: keyring.keyId,
+		reseal(objects, options) {
+			live();
+			return resealer.reseal(objects, options);
+		},
+
+		async dispose(): Promise<void> {
+			if (disposed) return;
+			disposed = true;
+			await store.close();
+		},
+	};
+}

package/packages/storage/src/read-token.ts

@@ -0,0 +1,130 @@
+import { KeyringError, type Keyring } from '@flowdular/sdk/kernel';
+import {
+	StorageError,
+	STORAGE_READ_ROUTE_PREFIX,
+	STORAGE_READ_URL_MAX_SECONDS,
+	type StorageObjectRef,
+} from './contracts.ts';
+
+/* The read token is the capability: it names the object, carries its own
+   expiry, and needs no session. It is sealed rather than signed, so the tenant
+   id never travels in a URL a proxy or a browser history would keep, and the
+   GCM tag is what makes a forged token impossible rather than merely unlikely.
+
+     version(1) | keyId(16 ascii) | iv(12) | tag(16) | ciphertext
+
+   The additional data is a constant unique to this use, so an object envelope
+   can never be replayed as a token and vice versa. */
+
+const TOKEN_VERSION = 1;
+const KEY_ID_BYTES = 16;
+const IV_BYTES = 12;
+const TAG_BYTES = 16;
+const HEADER_BYTES = 1 + KEY_ID_BYTES + IV_BYTES + TAG_BYTES;
+const TOKEN_AAD = 'flowdular-storage-read-token:v1';
+/* A payload is a fixed handful of ids; anything longer is not one, and parsing
+   stops before the decode rather than after it. */
+const TOKEN_CHARACTER_LIMIT = 1024;
+
+interface TokenPayload {
+	readonly t: string;
+	readonly m: string;
+	readonly o: string;
+	readonly e: number;
+}
+
+export interface StorageReadToken extends StorageObjectRef {
+	readonly expiresAt: Date;
+}
+
+export function mintStorageReadToken(
+	keyring: Keyring,
+	input: StorageObjectRef & {
+		readonly expiresInSeconds: number;
+		readonly now: Date;
+	},
+): string {
+	if (
+		!Number.isInteger(input.expiresInSeconds) ||
+		input.expiresInSeconds < 1 ||
+		input.expiresInSeconds > STORAGE_READ_URL_MAX_SECONDS
+	) {
+		throw new StorageError(
+			'EXPIRY_INVALID',
+			`A read URL expires in 1 to ${STORAGE_READ_URL_MAX_SECONDS} seconds.`,
+		);
+	}
+	const payload: TokenPayload = {
+		t: input.tenantId,
+		m: input.moduleId,
+		o: input.objectId,
+		e:
+			Math.floor(input.now.getTime() / 1000) +
+			Math.trunc(input.expiresInSeconds),
+	};
+	const sealed = keyring.seal(JSON.stringify(payload), TOKEN_AAD);
+	const keyId = Buffer.from(sealed.keyId.padEnd(KEY_ID_BYTES, '0'), 'latin1');
+	return Buffer.concat([
+		Buffer.of(TOKEN_VERSION),
+		keyId.subarray(0, KEY_ID_BYTES),
+		sealed.iv,
+		sealed.tag,
+		sealed.ciphertext,
+	]).toString('base64url');
+}
+
+/**
+ * The token's object reference, or null for anything a caller must not be able
+ * to tell apart: a malformed token, a key this ring does not hold, a forged tag
+ * and an expired capability all answer the same way.
+ */
+export function openStorageReadToken(
+	keyring: Keyring,
+	token: string,
+	now: Date,
+): StorageReadToken | null {
+	if (token.length === 0 || token.length > TOKEN_CHARACTER_LIMIT) return null;
+	const raw = Buffer.from(token, 'base64url');
+	if (raw.byteLength <= HEADER_BYTES || raw[0] !== TOKEN_VERSION) return null;
+	let opened: Buffer;
+	try {
+		opened = keyring.open(
+			{
+				keyId: raw.subarray(1, 1 + KEY_ID_BYTES).toString('latin1'),
+				iv: raw.subarray(1 + KEY_ID_BYTES, 1 + KEY_ID_BYTES + IV_BYTES),
+				tag: raw.subarray(1 + KEY_ID_BYTES + IV_BYTES, HEADER_BYTES),
+				ciphertext: raw.subarray(HEADER_BYTES),
+			},
+			TOKEN_AAD,
+		);
+	} catch (error) {
+		if (error instanceof KeyringError) return null;
+		throw error;
+	}
+	let payload: TokenPayload;
+	try {
+		payload = JSON.parse(opened.toString('utf8')) as TokenPayload;
+	} catch {
+		return null;
+	}
+	if (
+		typeof payload.t !== 'string' ||
+		typeof payload.m !== 'string' ||
+		typeof payload.o !== 'string' ||
+		!Number.isSafeInteger(payload.e)
+	) {
+		return null;
+	}
+	const expiresAt = new Date(payload.e * 1000);
+	if (expiresAt.getTime() <= now.getTime()) return null;
+	return {
+		tenantId: payload.t,
+		moduleId: payload.m,
+		objectId: payload.o,
+		expiresAt,
+	};
+}
+
+export function storageReadUrl(token: string): string {
+	return `${STORAGE_READ_ROUTE_PREFIX}${token}`;
+}

package/packages/storage/src/reseal.ts

@@ -0,0 +1,128 @@
+import { KeyringError, type Keyring } from '@flowdular/sdk/kernel';
+import {
+	StorageError,
+	storageObjectKey,
+	type StorageObjectRef,
+} from './contracts.ts';
+import {
+	readStoredObjectHeader,
+	resealStoredObject,
+	STORAGE_HEADER_PREFIX_BYTES,
+} from './envelope.ts';
+import type { ObjectStore } from './store.ts';
+
+export interface StorageResealCount {
+	readonly keyId: string;
+	readonly objects: number;
+}
+
+export interface StorageResealReport {
+	/** Key id every object should end on: the current key of the ring. */
+	readonly currentKeyId: string;
+	readonly counts: readonly StorageResealCount[];
+	/** Objects on a retired key the ring still holds. */
+	readonly stale: number;
+	readonly resealed: number;
+	/** Objects sealed under a key id this ring does not hold; left as they are. */
+	readonly unknown: number;
+	/** Objects that failed authentication under the key they name; left as they are. */
+	readonly refused: number;
+	/** References with no object behind them. */
+	readonly missing: number;
+}
+
+export interface StorageResealOptions {
+	readonly apply: boolean;
+}
+
+export interface StorageResealPort {
+	/** Key id every re-seal writes: the current key of the ring. */
+	readonly keyId: string;
+	/**
+	 * Re-seals every object of the batch that a retired key sealed, in place.
+	 * The caller owns the inventory: an adapter cannot list, so the references
+	 * come from the rows that name the objects. Without `apply` the headers are
+	 * read and counted and nothing is written.
+	 */
+	reseal(
+		objects: readonly StorageObjectRef[],
+		options: StorageResealOptions,
+	): Promise<StorageResealReport>;
+}
+
+/* A frame the port cannot parse is left exactly as the one it cannot open:
+   the pass counts it and moves on, and the operator restores it. */
+function corrupt(error: unknown): boolean {
+	return error instanceof StorageError && error.code === 'OBJECT_CORRUPT';
+}
+
+export function createStorageResealer(
+	store: ObjectStore,
+	keyring: Keyring,
+): StorageResealPort {
+	return {
+		keyId: keyring.keyId,
+		async reseal(objects, options) {
+			const counts = new Map<string, number>();
+			let stale = 0;
+			let resealed = 0;
+			let unknown = 0;
+			let refused = 0;
+			let missing = 0;
+			for (const reference of objects) {
+				const key = storageObjectKey(reference);
+				const prefix = await store.read(key, STORAGE_HEADER_PREFIX_BYTES);
+				if (!prefix) {
+					missing += 1;
+					continue;
+				}
+				let keyId: string;
+				try {
+					keyId = readStoredObjectHeader(key, prefix).keyId;
+				} catch (error) {
+					if (corrupt(error)) {
+						refused += 1;
+						continue;
+					}
+					throw error;
+				}
+				counts.set(keyId, (counts.get(keyId) ?? 0) + 1);
+				if (keyId === keyring.keyId) continue;
+				if (!keyring.knows(keyId)) {
+					unknown += 1;
+					continue;
+				}
+				stale += 1;
+				if (!options.apply) continue;
+				const frame = await store.read(key);
+				if (!frame) {
+					missing += 1;
+					continue;
+				}
+				let next;
+				try {
+					next = resealStoredObject(keyring, key, frame);
+				} catch (error) {
+					if (error instanceof KeyringError || corrupt(error)) {
+						refused += 1;
+						continue;
+					}
+					throw error;
+				}
+				await store.write(key, next.frame);
+				resealed += 1;
+			}
+			return {
+				currentKeyId: keyring.keyId,
+				counts: [...counts]
+					.sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
+					.map(([keyId, count]) => ({ keyId, objects: count })),
+				stale,
+				resealed,
+				unknown,
+				refused,
+				missing,
+			};
+		},
+	};
+}

package/packages/storage/src/s3.ts

@@ -0,0 +1,144 @@
+import { StorageError } from './contracts.ts';
+import {
+	encodeS3Path,
+	sha256Hex,
+	signS3Request,
+	type SigV4Credentials,
+} from './sigv4.ts';
+import type { ObjectStore } from './store.ts';
+
+const REQUEST_TIMEOUT_MS = 30_000;
+/* An S3 error body is XML naming a code and the resource. Only the code reaches
+   the message, bounded, so a provider cannot grow an exception text at will. */
+const ERROR_CODE_LIMIT = 64;
+
+export interface S3ObjectStoreOptions {
+	readonly bucket: string;
+	readonly region: string;
+	/** Absent uses the AWS endpoint for the region; MinIO and R2 set their own. */
+	readonly endpoint?: string | undefined;
+	readonly accessKeyId: string;
+	readonly secretAccessKey: string;
+	/** MinIO and most self-hosted gateways need `<endpoint>/<bucket>/<key>`. */
+	readonly forcePathStyle: boolean;
+	readonly fetch?: typeof globalThis.fetch | undefined;
+	readonly clock?: (() => Date) | undefined;
+}
+
+function errorCode(body: string): string {
+	return (
+		/<Code>([^<]{1,64})<\/Code>/.exec(body)?.[1]?.slice(0, ERROR_CODE_LIMIT) ??
+		'unknown'
+	);
+}
+
+async function unavailable(
+	action: string,
+	response: Response,
+): Promise<StorageError> {
+	const body = await response.text().catch(() => '');
+	return new StorageError(
+		'STORAGE_UNAVAILABLE',
+		`The object store refused to ${action}: HTTP ${response.status} ${errorCode(body)}.`,
+	);
+}
+
+export function createS3ObjectStore(
+	options: S3ObjectStoreOptions,
+): ObjectStore {
+	const credentials: SigV4Credentials = {
+		accessKeyId: options.accessKeyId,
+		secretAccessKey: options.secretAccessKey,
+		region: options.region,
+	};
+	const call = options.fetch ?? globalThis.fetch;
+	const clock = options.clock ?? (() => new Date());
+	const endpoint = new URL(
+		options.endpoint?.trim() || `https://s3.${options.region}.amazonaws.com`,
+	);
+	const origin = options.forcePathStyle
+		? `${endpoint.protocol}//${endpoint.host}`
+		: `${endpoint.protocol}//${options.bucket}.${endpoint.host}`;
+	const host = new URL(origin).host;
+	const basePath = options.forcePathStyle ? `/${options.bucket}` : '';
+
+	async function request(
+		method: string,
+		key: string,
+		init: {
+			readonly body?: Uint8Array | undefined;
+			readonly headers?: Readonly<Record<string, string>> | undefined;
+		} = {},
+	): Promise<Response> {
+		const canonicalUri = encodeS3Path(`${basePath}/${key}`);
+		const signed = signS3Request({
+			method,
+			canonicalUri,
+			headers: { host, ...init.headers },
+			payloadHash: sha256Hex(init.body ?? ''),
+			credentials,
+			now: clock(),
+		});
+		/* fetch owns Host and sets it from the URL, so sending the signed copy is
+		   both refused and unnecessary: the value the server reads is the one the
+		   signature covers. Content-Length is forbidden for the same reason, which
+		   is why it is neither sent nor signed. */
+		const { host: _signedHost, ...sendable } = signed;
+		try {
+			return await call(`${origin}${canonicalUri}`, {
+				method,
+				headers: sendable,
+				/* fetch accepts a typed array body at run time; BodyInit only spells
+				   the ArrayBuffer-backed view, and copying the frame to satisfy the
+				   type would double what one request holds. */
+				...(init.body ? { body: init.body as unknown as BodyInit } : {}),
+				signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+			});
+		} catch (error) {
+			throw new StorageError(
+				'STORAGE_UNAVAILABLE',
+				`The object store did not answer a ${method} request.`,
+				{ cause: error },
+			);
+		}
+	}
+
+	return {
+		async write(key, frame) {
+			const response = await request('PUT', key, {
+				body: frame,
+				headers: { 'content-type': 'application/octet-stream' },
+			});
+			if (!response.ok) throw await unavailable(`store ${key}`, response);
+			await response.body?.cancel();
+		},
+		async read(key, maxBytes) {
+			const response = await request('GET', key, {
+				...(maxBytes === undefined
+					? {}
+					: { headers: { range: `bytes=0-${maxBytes - 1}` } }),
+			});
+			if (response.status === 404) {
+				await response.body?.cancel();
+				return null;
+			}
+			if (!response.ok) throw await unavailable(`read ${key}`, response);
+			return new Uint8Array(await response.arrayBuffer());
+		},
+		async remove(key) {
+			/* S3 deletes are idempotent and answer 204 for an absent key, so the
+			   caller is told whether the object was there by looking first. */
+			const head = await request('HEAD', key);
+			await head.body?.cancel();
+			if (head.status === 404) return false;
+			if (!head.ok) throw await unavailable(`read ${key}`, head);
+			const response = await request('DELETE', key);
+			if (!response.ok && response.status !== 404) {
+				throw await unavailable(`delete ${key}`, response);
+			}
+			await response.body?.cancel();
+			return true;
+		},
+		close: () => Promise.resolve(),
+	};
+}

package/packages/storage/src/scanner.ts

@@ -0,0 +1,22 @@
+import type { StorageScanVerdict } from './contracts.ts';
+
+export interface StorageScanResult {
+	readonly verdict: StorageScanVerdict;
+}
+
+/**
+ * The seam a deployment fills with a malware scanner. It reads the plaintext
+ * stream and answers with a verdict; the platform refuses `infected` and stores
+ * the verdict with the object so a later policy change can find what was never
+ * scanned.
+ */
+export interface StorageScanner {
+	scan(
+		body: ReadableStream<Uint8Array>,
+	): StorageScanResult | Promise<StorageScanResult>;
+}
+
+/** No scanner configured. Every object is stored with the `unscanned` verdict. */
+export const unscannedStorageScanner: StorageScanner = Object.freeze({
+	scan: () => ({ verdict: 'unscanned' as const }),
+});