@flowdular/sdk 0.3.0 → 0.3.2

package/modules/connectors/migrations/0003_connectors_rotation_inventory.up.sql

@@ -0,0 +1,19 @@
+-- The credential key rotation has to find the instances still sealed with a
+-- retired key before it knows whose they are, so the cross-tenant role may read
+-- the key id of every row that holds an envelope and nothing else: the nonce,
+-- the tag, the ciphertext and the fingerprint stay unreadable on this
+-- connection, and every row it re-seals is read again under the workspace that
+-- row named. PostgreSQL checks column privileges in WHERE too, so the key id is
+-- part of the grant.
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'coreloom_background') THEN
+    RAISE EXCEPTION 'The coreloom_background role must exist before this migration.';
+  END IF;
+END
+$$;
+CREATE POLICY connectors_instances_background_policy ON connectors_instances
+  FOR SELECT TO coreloom_background
+  USING (credential_key_id IS NOT NULL);
+REVOKE SELECT ON connectors_instances FROM coreloom_background;
+GRANT SELECT (tenant_id, credential_key_id) ON connectors_instances TO coreloom_background;

package/modules/connectors/module.json

@@ -3,10 +3,10 @@
 	"schemaVersion": 1,
 	"id": "connectors.core",
 	"package": "@flowdular/module-connectors",
-	"version": "0.1.5",
+	"version": "0.1.6",
 	"platformApi": "^0.1.0",
 	"profile": "full",
-	"capabilities": ["api", "database", "client", "translations"],
+	"capabilities": ["api", "database", "client", "translations", "cli"],
 	"platform": {
 		"server": true,
 		"client": true
@@ -14,7 +14,7 @@
 	"dependencies": [
 		{
 			"id": "system.core",
-			"range": "^0.7.0"
+			"range": "^0.8.0"
 		},
 		{
 			"id": "auth.core",
@@ -24,5 +24,9 @@
 	"tenancy": "required",
 	"locales": ["en", "pl"],
 	"stability": "experimental",
-	"provides": ["connectors.definitions.v1", "connectors.calls.v1"]
+	"provides": ["connectors.definitions.v1", "connectors.calls.v1"],
+	"cli": {
+		"catalog": "src/cli/commands.json",
+		"entry": "src/cli/index.ts"
+	}
 }

package/modules/connectors/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@flowdular/module-connectors",
-	"version": "0.1.5",
+	"version": "0.1.6",
 	"private": true,
 	"type": "module",
 	"exports": {
@@ -28,6 +28,7 @@
 		"test": "vitest run"
 	},
 	"dependencies": {
+		"@flowdular/cli-protocol": "workspace:*",
 		"@flowdular/client": "workspace:*",
 		"@flowdular/contracts": "workspace:*",
 		"@flowdular/database": "workspace:*",

package/modules/connectors/spec/module.yaml

@@ -1,6 +1,6 @@
 schemaVersion: 2
 id: connectors.core
-specVersion: 0.1.5
+specVersion: 0.1.6
 status: approved
 name: Connectors Core
 description: Gives a workspace a governed way to call external systems through connector definitions a module or the platform ships, instances an owner configures with sealed credentials and a host allowlist, a test call, a call log without payloads, and an explicit per-instance consent that lets workflows and agents use the connector as an external action, with a generic HTTP JSON connector as the reference.
@@ -10,9 +10,10 @@ capabilities:
   - database
   - client
   - translations
+  - cli
 dependencies:
   - id: system.core
-    range: ^0.7.0
+    range: ^0.8.0
   - id: auth.core
     range: ^0.13.0
 tenancy: required

package/modules/connectors/src/cli/commands.json

@@ -0,0 +1,17 @@
+{
+	"protocolVersion": 1,
+	"moduleId": "connectors.core",
+	"commands": [
+		{
+			"path": ["connectors", "secrets-rotate"],
+			"capability": {
+				"id": "connectors.secrets.rotate",
+				"version": 1,
+				"summary": "Re-seal stored connector credentials with the current encryption key.",
+				"risk": "process",
+				"requiresApprovedSpec": false,
+				"supportsDryRun": true
+			}
+		}
+	]
+}

package/modules/connectors/src/cli/index.ts

@@ -0,0 +1,126 @@
+import {
+	defineCliExtension,
+	type CliExtensionContext,
+} from '@flowdular/sdk/cli-protocol';
+import {
+	DATABASE_CAPABILITY_IDS,
+	DATABASE_DIALECT_IDS,
+	type DatabaseAdapterLease,
+} from '@flowdular/sdk/database';
+import { migrateConnectorsDatabase } from '../services/database-repository.ts';
+import { rotateConnectorCredentials } from '../services/credential-rotation.ts';
+import { credentialVaultFromEnvironment } from '../services/credential-vault.ts';
+
+const rotateCapability = {
+	id: 'connectors.secrets.rotate',
+	version: 1,
+	summary:
+		'Re-seal stored connector credentials with the current encryption key.',
+	risk: 'process' as const,
+	requiresApprovedSpec: false,
+	supportsDryRun: true,
+};
+
+interface OpenDatabase {
+	readonly leases: readonly DatabaseAdapterLease[];
+	readonly runtime: DatabaseAdapterLease;
+	readonly background: DatabaseAdapterLease;
+}
+
+/* The operator command reads the same deployment database the platform does;
+   the runner owns the provider and a module owns no driver, so it arrives on
+   the context. It may be the first thing to touch a fresh database, so it
+   migrates before reading. */
+async function open(context: CliExtensionContext): Promise<OpenDatabase> {
+	const databases = context.databases;
+	if (!databases) {
+		throw new Error(
+			'connectors.core CLI commands read the deployment database, and this workspace has none configured.',
+		);
+	}
+	const requirements = {
+		dialectIds: [DATABASE_DIALECT_IDS.postgresql],
+		capabilities: [DATABASE_CAPABILITY_IDS.TRANSACTIONS],
+	};
+	const migration = await databases.acquire({
+		namespace: 'connectors.core',
+		purpose: 'migration',
+		requirements,
+	});
+	await migrateConnectorsDatabase(migration.database);
+	const runtime = await databases.acquire({
+		namespace: 'connectors.core',
+		purpose: 'runtime',
+		requirements,
+	});
+	/* The inventory counts rows across the whole deployment, which only the
+	   cross-tenant read-only role may do. */
+	const background = await databases.acquire({
+		namespace: 'connectors.core',
+		purpose: 'background',
+		requirements,
+	});
+	return { leases: [migration, runtime, background], runtime, background };
+}
+
+/* The provider belongs to the runner; only the leases this command took are
+   released here. */
+async function close(open: OpenDatabase): Promise<void> {
+	for (const lease of open.leases) await lease.release();
+}
+
+export const cliExtension = defineCliExtension({
+	protocolVersion: 1,
+	moduleId: 'connectors.core',
+	commands: [
+		{
+			path: ['connectors', 'secrets-rotate'],
+			capability: rotateCapability,
+			execute: async (context) => {
+				/* The report names key ids and row counts only; a credential never
+				   reaches the command output. */
+				const vault = credentialVaultFromEnvironment(
+					process.env,
+					context.workspaceRoot,
+				);
+				const opened = await open(context);
+				try {
+					const report = await rotateConnectorCredentials({
+						runtime: opened.runtime.database,
+						background: opened.background.database,
+						vault,
+						apply: context.apply,
+					});
+					const warnings: string[] = [];
+					if (report.skipped > 0) {
+						warnings.push(
+							`${report.skipped} rows were rewritten by the application while this ran and keep their own envelope. Run the command again.`,
+						);
+					}
+					if (report.unknown > 0) {
+						warnings.push(
+							`${report.unknown} rows are sealed under a key this ring does not hold and were left as they are. Put that key back in FD_CONNECTORS_SECRET_KEY_PREVIOUS before retiring it.`,
+						);
+					}
+					if (report.refused > 0) {
+						warnings.push(
+							`${report.refused} rows failed authentication under the key they name and were left as they are. Restore them from a database backup or replace the credential.`,
+						);
+					}
+					return {
+						data: { moduleId: 'connectors.core', ...report },
+						evidence: [
+							'modules/connectors/spec/module.yaml',
+							'docs/operations.md',
+						],
+						warnings,
+					};
+				} finally {
+					await close(opened);
+				}
+			},
+		},
+	],
+});
+
+export default cliExtension;

package/modules/connectors/src/services/credential-rotation.ts

@@ -0,0 +1,221 @@
+import type { DatabaseHandle } from '@flowdular/sdk/database';
+import { KeyringError } from '@flowdular/sdk/kernel';
+import { credentialContext, type CredentialVault } from './credential-vault.ts';
+
+/** Rows re-sealed inside one tenant-scoped transaction. */
+export const CREDENTIAL_ROTATION_BATCH = 200;
+
+export const CREDENTIAL_ROTATION_TABLE = 'connectors_instances';
+
+export interface CredentialKeyCount {
+	readonly keyId: string;
+	readonly rows: number;
+}
+
+export interface CredentialRotationReport {
+	readonly table: string;
+	/** Key id every row should end on: the current key of the vault. */
+	readonly currentKeyId: string;
+	readonly counts: readonly CredentialKeyCount[];
+	/** Rows on a retired key when the run started. */
+	readonly stale: number;
+	readonly tenants: number;
+	readonly rotated: number;
+	/** Rows a concurrent write changed between the read and the update. */
+	readonly skipped: number;
+	/** Rows sealed under a key id the vault does not hold; left as they are. */
+	readonly unknown: number;
+	/** Rows that failed authentication under the key they name; left as they are. */
+	readonly refused: number;
+}
+
+export interface CredentialRotationOptions {
+	/** Tenant-scoped handle. Every read and write of a row runs on it. */
+	readonly runtime: DatabaseHandle;
+	/** Cross-tenant handle. It is granted the key ids and nothing else. */
+	readonly background: DatabaseHandle;
+	readonly vault: CredentialVault;
+	readonly apply?: boolean;
+	readonly batchSize?: number;
+}
+
+interface StaleRow {
+	id: string;
+	credential_key_id: string;
+	credential_iv: string;
+	credential_tag: string;
+	credential_ciphertext: string;
+}
+
+/* Rows with auth_kind none hold no envelope and no key id, so every statement
+   here excludes them by the key id column. */
+const SQL = {
+	counts: `SELECT credential_key_id AS key_id, count(*) AS row_count
+	         FROM connectors_instances
+	         WHERE credential_key_id IS NOT NULL
+	         GROUP BY credential_key_id
+	         ORDER BY credential_key_id`,
+	/* The key ids the vault holds are bound as one comma-separated parameter
+	   (an id is a hex digest), so a row under a key it does not hold is neither
+	   stale nor walked: it is counted and left. */
+	staleTenants: `SELECT tenant_id, count(*) AS row_count
+	               FROM connectors_instances
+	               WHERE credential_key_id IS NOT NULL
+	                 AND credential_key_id <> $1
+	                 AND credential_key_id = ANY(string_to_array($2, ','))
+	               GROUP BY tenant_id
+	               ORDER BY tenant_id`,
+	/* Paged by primary key: a row the optimistic update skipped stays stale, so
+	   a query that only asked for stale rows would return it forever. */
+	staleBatch: `SELECT id, credential_key_id, credential_iv, credential_tag,
+	                    credential_ciphertext
+	             FROM connectors_instances
+	             WHERE tenant_id = $1 AND credential_key_id IS NOT NULL
+	               AND credential_key_id <> $2
+	               AND credential_key_id = ANY(string_to_array($3, ',')) AND id > $4
+	             ORDER BY id
+	             LIMIT $5`,
+	/* The envelope the row still holds is the optimistic check: a credential
+	   replaced by an owner in between keeps its own value. updated_at marks
+	   changes an owner can see, and a re-seal is not one. */
+	reseal: `UPDATE connectors_instances
+	         SET credential_key_id = $1, credential_iv = $2, credential_tag = $3,
+	             credential_ciphertext = $4, credential_fingerprint = $5
+	         WHERE tenant_id = $6 AND id = $7 AND credential_ciphertext = $8`,
+};
+
+function count(value: number | bigint | string): number {
+	const normalized = Number(value);
+	if (!Number.isSafeInteger(normalized)) {
+		throw new Error('The connectors database returned an invalid count.');
+	}
+	return normalized;
+}
+
+/**
+ * Re-seals every stored connector credential that is not on the current key.
+ * The inventory is read once across tenants, and each batch of rows is read,
+ * decrypted, re-sealed and written inside one transaction scoped to the tenant
+ * that owns them. The fingerprint is keyed by the current key, so it is
+ * recomputed while the plaintext is in hand: the value an owner compares
+ * changes with the key, the credential does not. It is idempotent: a second
+ * run finds nothing to do.
+ */
+export async function rotateConnectorCredentials(
+	options: CredentialRotationOptions,
+): Promise<CredentialRotationReport> {
+	const currentKeyId = options.vault.keyId;
+	const batchSize = options.batchSize ?? CREDENTIAL_ROTATION_BATCH;
+	const counts = await options.background.transaction(
+		async (transaction) =>
+			(
+				await transaction.query<{
+					key_id: string;
+					row_count: number | bigint | string;
+				}>({ text: SQL.counts })
+			).rows.map((row) => ({ keyId: row.key_id, rows: count(row.row_count) })),
+		{ access: 'read' },
+	);
+	const known = counts
+		.map((entry) => entry.keyId)
+		.filter((keyId) => options.vault.knows(keyId))
+		.join(',');
+	const unknown = counts
+		.filter((entry) => !options.vault.knows(entry.keyId))
+		.reduce((total, entry) => total + entry.rows, 0);
+	const inventory = await options.background.transaction(
+		async (transaction) =>
+			(
+				await transaction.query<{
+					tenant_id: string;
+					row_count: number | bigint | string;
+				}>({ text: SQL.staleTenants, parameters: [currentKeyId, known] })
+			).rows,
+		{ access: 'read' },
+	);
+	const report = {
+		table: CREDENTIAL_ROTATION_TABLE,
+		currentKeyId,
+		counts,
+		stale: inventory.reduce((total, row) => total + count(row.row_count), 0),
+		tenants: inventory.length,
+		unknown,
+	};
+	if (options.apply !== true) {
+		return { ...report, rotated: 0, skipped: 0, refused: 0 };
+	}
+	let rotated = 0;
+	let skipped = 0;
+	let refused = 0;
+	for (const { tenant_id: tenantId } of inventory) {
+		let cursor = '';
+		for (;;) {
+			const batch = await options.runtime.transaction(
+				async (transaction) => {
+					const rows = (
+						await transaction.query<StaleRow>({
+							text: SQL.staleBatch,
+							parameters: [tenantId, currentKeyId, known, cursor, batchSize],
+						})
+					).rows;
+					let written = 0;
+					let left = 0;
+					for (const row of rows) {
+						const context = credentialContext(tenantId, row.id);
+						let plaintext: string;
+						try {
+							plaintext = options.vault.open(
+								{
+									keyId: row.credential_key_id,
+									iv: row.credential_iv,
+									tag: row.credential_tag,
+									ciphertext: row.credential_ciphertext,
+								},
+								context,
+							);
+						} catch (error) {
+							/* A tag that fails under the key it names is evidence the pass
+							   must not replace with a fresh envelope; the row stays. */
+							if (
+								error instanceof KeyringError &&
+								error.code === 'ENVELOPE_INVALID'
+							) {
+								left += 1;
+								continue;
+							}
+							throw error;
+						}
+						const sealed = options.vault.seal(plaintext, context);
+						const result = await transaction.execute({
+							text: SQL.reseal,
+							parameters: [
+								sealed.keyId,
+								sealed.iv,
+								sealed.tag,
+								sealed.ciphertext,
+								options.vault.fingerprint(plaintext, context),
+								tenantId,
+								row.id,
+								row.credential_ciphertext,
+							],
+						});
+						written += result.affectedRows;
+					}
+					return {
+						read: rows.length,
+						written,
+						left,
+						last: rows.at(-1)?.id,
+					};
+				},
+				{ access: 'write', tenantId },
+			);
+			rotated += batch.written;
+			refused += batch.left;
+			skipped += batch.read - batch.left - batch.written;
+			if (batch.read < batchSize || batch.last === undefined) break;
+			cursor = batch.last;
+		}
+	}
+	return { ...report, rotated, skipped, refused };
+}

package/modules/connectors/src/services/credential-vault.ts

@@ -48,6 +48,8 @@ export function credentialContext(
 export interface CredentialVault {
 	/** Key id every new envelope is written with; stored rows may carry older ones. */
 	readonly keyId: string;
+	/** Whether the current key or a retired one in the ring carries this id. */
+	knows(keyId: string): boolean;
 	seal(secret: string, context: string): SealedCredential;
 	open(envelope: SealedCredential, context: string): string;
 	/**
@@ -95,6 +97,10 @@ export class AesGcmCredentialVault implements CredentialVault {
 		return this.#keyring.keyId;
 	}
 
+	knows(keyId: string): boolean {
+		return this.#keyring.knows(keyId);
+	}
+
 	fingerprint(secret: string, context: string): string {
 		return createHmac('sha256', this.#fingerprintKey)
 			.update(context, 'utf8')

package/modules/connectors/src/services/migration.ts

@@ -111,6 +111,28 @@ async function combined(
 	return 'partial';
 }
 
+/* Mirrors migrations/0003_connectors_rotation_inventory.up.sql byte for byte. */
+export const CONNECTORS_MIGRATION_003 = `-- The credential key rotation has to find the instances still sealed with a
+-- retired key before it knows whose they are, so the cross-tenant role may read
+-- the key id of every row that holds an envelope and nothing else: the nonce,
+-- the tag, the ciphertext and the fingerprint stay unreadable on this
+-- connection, and every row it re-seals is read again under the workspace that
+-- row named. PostgreSQL checks column privileges in WHERE too, so the key id is
+-- part of the grant.
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'coreloom_background') THEN
+    RAISE EXCEPTION 'The coreloom_background role must exist before this migration.';
+  END IF;
+END
+$$;
+CREATE POLICY connectors_instances_background_policy ON connectors_instances
+  FOR SELECT TO coreloom_background
+  USING (credential_key_id IS NOT NULL);
+REVOKE SELECT ON connectors_instances FROM coreloom_background;
+GRANT SELECT (tenant_id, credential_key_id) ON connectors_instances TO coreloom_background;
+`;
+
 export const databaseMigrations: readonly DatabaseMigration[] = [
 	{
 		id: '0001_connectors_core',
@@ -163,4 +185,18 @@ export const databaseMigrations: readonly DatabaseMigration[] = [
 				],
 			),
 	},
+	{
+		id: '0003_connectors_rotation_inventory',
+		sql: { postgresql: CONNECTORS_MIGRATION_003 },
+		/* A policy and a column grant leave no schema object behind, so the
+		   privilege itself is what proves this migration ran. */
+		inspectExisting: async (database) => {
+			const result = await database.query<{ granted: boolean }>({
+				text: `SELECT CASE WHEN to_regclass('connectors_instances') IS NOT NULL THEN
+				  has_column_privilege('coreloom_background', 'connectors_instances', 'credential_key_id', 'SELECT')
+				ELSE false END AS granted`,
+			});
+			return result.rows[0]?.granted === true ? 'complete' : 'absent';
+		},
+	},
 ];

package/modules/directory/module.json

@@ -14,7 +14,7 @@
 	"dependencies": [
 		{
 			"id": "system.core",
-			"range": "^0.7.0"
+			"range": "^0.8.0"
 		},
 		{
 			"id": "auth.core",

package/modules/directory/spec/module.yaml

@@ -12,7 +12,7 @@ capabilities:
   - translations
 dependencies:
   - id: system.core
-    range: ^0.7.0
+    range: ^0.8.0
   - id: auth.core
     range: ^0.13.0
 tenancy: required

package/modules/documents/migrations/0003_documents_rotation_inventory.down.sql

@@ -0,0 +1,2 @@
+REVOKE SELECT (tenant_id, status) ON documents_files FROM coreloom_background;
+DROP POLICY IF EXISTS documents_files_background_policy ON documents_files;

package/modules/documents/migrations/0003_documents_rotation_inventory.up.sql

@@ -0,0 +1,18 @@
+-- The storage key rotation has to find the workspaces that still hold objects
+-- before it knows which objects those are, so the cross-tenant role may count
+-- stored rows by workspace and nothing else: the storage key, the record and
+-- the file name stay invisible to it, and every object it names is read again
+-- under the workspace that row named. PostgreSQL checks column privileges in
+-- WHERE too, so `status` is part of the grant.
+DO $$
+BEGIN
+  IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'coreloom_background') THEN
+    RAISE EXCEPTION 'The coreloom_background role must exist before this migration.';
+  END IF;
+END
+$$;
+CREATE POLICY documents_files_background_policy ON documents_files
+  FOR SELECT TO coreloom_background
+  USING (status = 'stored');
+REVOKE SELECT ON documents_files FROM coreloom_background;
+GRANT SELECT (tenant_id, status) ON documents_files TO coreloom_background;

package/modules/documents/module.json

@@ -3,10 +3,10 @@
 	"schemaVersion": 1,
 	"id": "documents.core",
 	"package": "@flowdular/module-documents",
-	"version": "0.1.8",
+	"version": "0.1.9",
 	"platformApi": "^0.1.0",
 	"profile": "full",
-	"capabilities": ["api", "database", "client", "translations"],
+	"capabilities": ["api", "database", "client", "translations", "cli"],
 	"platform": {
 		"server": true,
 		"client": true
@@ -14,7 +14,7 @@
 	"dependencies": [
 		{
 			"id": "system.core",
-			"range": "^0.7.0"
+			"range": "^0.8.0"
 		},
 		{
 			"id": "auth.core",
@@ -24,5 +24,9 @@
 	"provides": ["documents.attachments.v1"],
 	"tenancy": "required",
 	"locales": ["en", "pl"],
-	"stability": "experimental"
+	"stability": "experimental",
+	"cli": {
+		"catalog": "src/cli/commands.json",
+		"entry": "src/cli/index.ts"
+	}
 }

package/modules/documents/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@flowdular/module-documents",
-	"version": "0.1.8",
+	"version": "0.1.9",
 	"private": true,
 	"type": "module",
 	"exports": {
@@ -28,6 +28,7 @@
 		"test": "vitest run"
 	},
 	"dependencies": {
+		"@flowdular/cli-protocol": "workspace:*",
 		"@flowdular/client": "workspace:*",
 		"@flowdular/contracts": "workspace:*",
 		"@flowdular/database": "workspace:*",

package/modules/documents/spec/module.yaml

@@ -1,6 +1,6 @@
 schemaVersion: 2
 id: documents.core
-specVersion: 0.1.8
+specVersion: 0.1.9
 status: approved
 name: Documents Core
 description: Lets a workspace attach files to its records through the platform storage port, owning only the document metadata and the reference to the record, with tenant-scoped keys, encrypted bytes, size and type limits, and a scan state, so a module attaches by reference and never holds bytes.
@@ -10,9 +10,10 @@ capabilities:
   - database
   - client
   - translations
+  - cli
 dependencies:
   - id: system.core
-    range: ^0.7.0
+    range: ^0.8.0
   - id: auth.core
     range: ^0.13.0
 tenancy: required

package/modules/documents/src/cli/commands.json

@@ -0,0 +1,17 @@
+{
+	"protocolVersion": 1,
+	"moduleId": "documents.core",
+	"commands": [
+		{
+			"path": ["documents", "secrets-rotate"],
+			"capability": {
+				"id": "documents.storage.rotate",
+				"version": 1,
+				"summary": "Re-seal stored document objects with the current storage encryption key.",
+				"risk": "process",
+				"requiresApprovedSpec": false,
+				"supportsDryRun": true
+			}
+		}
+	]
+}