@flowdular/sdk 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/.ai/platform-capabilities.md +2 -2
  2. package/.ai/policies/capabilities.yaml +30 -3
  3. package/.ai/references/catalog/module.json +1 -1
  4. package/.ai/references/catalog/package.json +2 -2
  5. package/.ai/references/catalog/spec/module.yaml +1 -1
  6. package/.ai/references/catalog.provenance.json +6 -6
  7. package/.ai/skills/cli-extension/SKILL.md +1 -1
  8. package/.ai/skills/deploy-operate/SKILL.md +7 -2
  9. package/modules/approvals/migrations/0005_approvals_grant_audit.down.sql +4 -0
  10. package/modules/approvals/migrations/0005_approvals_grant_audit.up.sql +20 -0
  11. package/modules/approvals/module.json +1 -1
  12. package/modules/approvals/package.json +1 -1
  13. package/modules/approvals/spec/module.yaml +11 -2
  14. package/modules/approvals/src/domain/capability.ts +12 -0
  15. package/modules/approvals/src/domain/grant.ts +69 -0
  16. package/modules/approvals/src/domain/types.ts +14 -0
  17. package/modules/approvals/src/index.ts +9 -0
  18. package/modules/approvals/src/platform.ts +8 -0
  19. package/modules/approvals/src/server/runtime.ts +4 -0
  20. package/modules/approvals/src/services/approvals-service.ts +80 -0
  21. package/modules/approvals/src/services/database-repository.ts +65 -6
  22. package/modules/approvals/src/services/migration.ts +34 -0
  23. package/modules/approvals/src/services/repository.ts +8 -0
  24. package/modules/connectors/migrations/0003_connectors_rotation_inventory.down.sql +2 -0
  25. package/modules/connectors/migrations/0003_connectors_rotation_inventory.up.sql +19 -0
  26. package/modules/connectors/module.json +7 -3
  27. package/modules/connectors/package.json +2 -1
  28. package/modules/connectors/spec/module.yaml +2 -1
  29. package/modules/connectors/src/cli/commands.json +17 -0
  30. package/modules/connectors/src/cli/index.ts +126 -0
  31. package/modules/connectors/src/services/credential-rotation.ts +221 -0
  32. package/modules/connectors/src/services/credential-vault.ts +6 -0
  33. package/modules/connectors/src/services/migration.ts +36 -0
  34. package/modules/documents/migrations/0003_documents_rotation_inventory.down.sql +2 -0
  35. package/modules/documents/migrations/0003_documents_rotation_inventory.up.sql +18 -0
  36. package/modules/documents/module.json +7 -3
  37. package/modules/documents/package.json +2 -1
  38. package/modules/documents/spec/module.yaml +2 -1
  39. package/modules/documents/src/cli/commands.json +17 -0
  40. package/modules/documents/src/cli/index.ts +145 -0
  41. package/modules/documents/src/services/database-repository.ts +15 -4
  42. package/modules/documents/src/services/documents-service.ts +13 -9
  43. package/modules/documents/src/services/migration.ts +35 -0
  44. package/modules/documents/src/services/repository.ts +12 -2
  45. package/modules/documents/src/services/storage-rotation.ts +157 -0
  46. package/modules/exports/migrations/0003_exports_rotation_inventory.down.sql +1 -0
  47. package/modules/exports/migrations/0003_exports_rotation_inventory.up.sql +9 -0
  48. package/modules/exports/module.json +7 -3
  49. package/modules/exports/package.json +2 -1
  50. package/modules/exports/spec/module.yaml +2 -1
  51. package/modules/exports/src/cli/commands.json +17 -0
  52. package/modules/exports/src/cli/index.ts +145 -0
  53. package/modules/exports/src/server/index.ts +0 -1
  54. package/modules/exports/src/services/data-classes.ts +16 -13
  55. package/modules/exports/src/services/database-repository.ts +30 -32
  56. package/modules/exports/src/services/migration.ts +27 -0
  57. package/modules/exports/src/services/repository.ts +9 -10
  58. package/modules/exports/src/services/storage-rotation.ts +138 -0
  59. package/package.json +1 -1
  60. package/packages/contracts/src/index.ts +1 -1
  61. package/packages/database/src/backup.ts +1 -0
  62. package/packages/database/src/migrations.ts +7 -0
  63. package/packages/harness/src/runtime.ts +169 -10
  64. package/packages/harness/src/tool-adapters.ts +6 -13
  65. package/packages/kernel/src/approval-grant.ts +310 -0
  66. package/packages/kernel/src/index.ts +20 -0
  67. package/packages/storage/src/envelope.ts +70 -21
  68. package/packages/storage/src/index.ts +7 -1
  69. package/packages/storage/src/port.ts +12 -1
  70. package/packages/storage/src/reseal.ts +128 -0
@@ -111,6 +111,28 @@ async function combined(
111
111
  return 'partial';
112
112
  }
113
113
 
114
+ /* Mirrors migrations/0003_connectors_rotation_inventory.up.sql byte for byte. */
115
+ export const CONNECTORS_MIGRATION_003 = `-- The credential key rotation has to find the instances still sealed with a
116
+ -- retired key before it knows whose they are, so the cross-tenant role may read
117
+ -- the key id of every row that holds an envelope and nothing else: the nonce,
118
+ -- the tag, the ciphertext and the fingerprint stay unreadable on this
119
+ -- connection, and every row it re-seals is read again under the workspace that
120
+ -- row named. PostgreSQL checks column privileges in WHERE too, so the key id is
121
+ -- part of the grant.
122
+ DO $$
123
+ BEGIN
124
+ IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'coreloom_background') THEN
125
+ RAISE EXCEPTION 'The coreloom_background role must exist before this migration.';
126
+ END IF;
127
+ END
128
+ $$;
129
+ CREATE POLICY connectors_instances_background_policy ON connectors_instances
130
+ FOR SELECT TO coreloom_background
131
+ USING (credential_key_id IS NOT NULL);
132
+ REVOKE SELECT ON connectors_instances FROM coreloom_background;
133
+ GRANT SELECT (tenant_id, credential_key_id) ON connectors_instances TO coreloom_background;
134
+ `;
135
+
114
136
  export const databaseMigrations: readonly DatabaseMigration[] = [
115
137
  {
116
138
  id: '0001_connectors_core',
@@ -163,4 +185,18 @@ export const databaseMigrations: readonly DatabaseMigration[] = [
163
185
  ],
164
186
  ),
165
187
  },
188
+ {
189
+ id: '0003_connectors_rotation_inventory',
190
+ sql: { postgresql: CONNECTORS_MIGRATION_003 },
191
+ /* A policy and a column grant leave no schema object behind, so the
192
+ privilege itself is what proves this migration ran. */
193
+ inspectExisting: async (database) => {
194
+ const result = await database.query<{ granted: boolean }>({
195
+ text: `SELECT CASE WHEN to_regclass('connectors_instances') IS NOT NULL THEN
196
+ has_column_privilege('coreloom_background', 'connectors_instances', 'credential_key_id', 'SELECT')
197
+ ELSE false END AS granted`,
198
+ });
199
+ return result.rows[0]?.granted === true ? 'complete' : 'absent';
200
+ },
201
+ },
166
202
  ];
@@ -0,0 +1,2 @@
1
+ REVOKE SELECT (tenant_id, status) ON documents_files FROM coreloom_background;
2
+ DROP POLICY IF EXISTS documents_files_background_policy ON documents_files;
@@ -0,0 +1,18 @@
1
+ -- The storage key rotation has to find the workspaces that still hold objects
2
+ -- before it knows which objects those are, so the cross-tenant role may count
3
+ -- stored rows by workspace and nothing else: the storage key, the record and
4
+ -- the file name stay invisible to it, and every object it names is read again
5
+ -- under the workspace that row named. PostgreSQL checks column privileges in
6
+ -- WHERE too, so `status` is part of the grant.
7
+ DO $$
8
+ BEGIN
9
+ IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'coreloom_background') THEN
10
+ RAISE EXCEPTION 'The coreloom_background role must exist before this migration.';
11
+ END IF;
12
+ END
13
+ $$;
14
+ CREATE POLICY documents_files_background_policy ON documents_files
15
+ FOR SELECT TO coreloom_background
16
+ USING (status = 'stored');
17
+ REVOKE SELECT ON documents_files FROM coreloom_background;
18
+ GRANT SELECT (tenant_id, status) ON documents_files TO coreloom_background;
@@ -3,10 +3,10 @@
3
3
  "schemaVersion": 1,
4
4
  "id": "documents.core",
5
5
  "package": "@flowdular/module-documents",
6
- "version": "0.1.8",
6
+ "version": "0.1.9",
7
7
  "platformApi": "^0.1.0",
8
8
  "profile": "full",
9
- "capabilities": ["api", "database", "client", "translations"],
9
+ "capabilities": ["api", "database", "client", "translations", "cli"],
10
10
  "platform": {
11
11
  "server": true,
12
12
  "client": true
@@ -24,5 +24,9 @@
24
24
  "provides": ["documents.attachments.v1"],
25
25
  "tenancy": "required",
26
26
  "locales": ["en", "pl"],
27
- "stability": "experimental"
27
+ "stability": "experimental",
28
+ "cli": {
29
+ "catalog": "src/cli/commands.json",
30
+ "entry": "src/cli/index.ts"
31
+ }
28
32
  }
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@flowdular/module-documents",
3
- "version": "0.1.8",
3
+ "version": "0.1.9",
4
4
  "private": true,
5
5
  "type": "module",
6
6
  "exports": {
@@ -28,6 +28,7 @@
28
28
  "test": "vitest run"
29
29
  },
30
30
  "dependencies": {
31
+ "@flowdular/cli-protocol": "workspace:*",
31
32
  "@flowdular/client": "workspace:*",
32
33
  "@flowdular/contracts": "workspace:*",
33
34
  "@flowdular/database": "workspace:*",
@@ -1,6 +1,6 @@
1
1
  schemaVersion: 2
2
2
  id: documents.core
3
- specVersion: 0.1.8
3
+ specVersion: 0.1.9
4
4
  status: approved
5
5
  name: Documents Core
6
6
  description: Lets a workspace attach files to its records through the platform storage port, owning only the document metadata and the reference to the record, with tenant-scoped keys, encrypted bytes, size and type limits, and a scan state, so a module attaches by reference and never holds bytes.
@@ -10,6 +10,7 @@ capabilities:
10
10
  - database
11
11
  - client
12
12
  - translations
13
+ - cli
13
14
  dependencies:
14
15
  - id: system.core
15
16
  range: ^0.7.0
@@ -0,0 +1,17 @@
1
+ {
2
+ "protocolVersion": 1,
3
+ "moduleId": "documents.core",
4
+ "commands": [
5
+ {
6
+ "path": ["documents", "secrets-rotate"],
7
+ "capability": {
8
+ "id": "documents.storage.rotate",
9
+ "version": 1,
10
+ "summary": "Re-seal stored document objects with the current storage encryption key.",
11
+ "risk": "process",
12
+ "requiresApprovedSpec": false,
13
+ "supportsDryRun": true
14
+ }
15
+ }
16
+ ]
17
+ }
@@ -0,0 +1,145 @@
1
+ import {
2
+ defineCliExtension,
3
+ type CliExtensionContext,
4
+ } from '@flowdular/sdk/cli-protocol';
5
+ import {
6
+ DATABASE_CAPABILITY_IDS,
7
+ DATABASE_DIALECT_IDS,
8
+ type DatabaseAdapterLease,
9
+ } from '@flowdular/sdk/database';
10
+ import {
11
+ createStorageKeyring,
12
+ createStoragePort,
13
+ storageConfigFromEnvironment,
14
+ } from '@flowdular/sdk/storage';
15
+ import { migrateDocumentsDatabase } from '../services/database-repository.ts';
16
+ import {
17
+ rotateDocumentObjects,
18
+ type StorageRotationReport,
19
+ } from '../services/storage-rotation.ts';
20
+
21
+ const rotateCapability = {
22
+ id: 'documents.storage.rotate',
23
+ version: 1,
24
+ summary:
25
+ 'Re-seal stored document objects with the current storage encryption key.',
26
+ risk: 'process' as const,
27
+ requiresApprovedSpec: false,
28
+ supportsDryRun: true,
29
+ };
30
+
31
+ interface OpenDatabase {
32
+ readonly leases: readonly DatabaseAdapterLease[];
33
+ readonly runtime: DatabaseAdapterLease;
34
+ readonly background: DatabaseAdapterLease;
35
+ }
36
+
37
+ /* The operator command reads the same deployment database the platform does;
38
+ the runner owns the provider and a module owns no driver, so it arrives on
39
+ the context. It may be the first thing to touch a fresh database, so it
40
+ migrates before reading. */
41
+ async function open(context: CliExtensionContext): Promise<OpenDatabase> {
42
+ const databases = context.databases;
43
+ if (!databases) {
44
+ throw new Error(
45
+ 'documents.core CLI commands read the deployment database, and this workspace has none configured.',
46
+ );
47
+ }
48
+ const requirements = {
49
+ dialectIds: [DATABASE_DIALECT_IDS.postgresql],
50
+ capabilities: [DATABASE_CAPABILITY_IDS.TRANSACTIONS],
51
+ };
52
+ const migration = await databases.acquire({
53
+ namespace: 'documents.core',
54
+ purpose: 'migration',
55
+ requirements,
56
+ });
57
+ await migrateDocumentsDatabase(migration.database);
58
+ const runtime = await databases.acquire({
59
+ namespace: 'documents.core',
60
+ purpose: 'runtime',
61
+ requirements,
62
+ });
63
+ /* The inventory lists workspaces across the whole deployment, which only
64
+ the cross-tenant read-only role may do. */
65
+ const background = await databases.acquire({
66
+ namespace: 'documents.core',
67
+ purpose: 'background',
68
+ requirements,
69
+ });
70
+ return { leases: [migration, runtime, background], runtime, background };
71
+ }
72
+
73
+ /* The provider belongs to the runner; only the leases this command took are
74
+ released here. */
75
+ async function close(open: OpenDatabase): Promise<void> {
76
+ for (const lease of open.leases) await lease.release();
77
+ }
78
+
79
+ export function storageRotationWarnings(
80
+ report: StorageRotationReport,
81
+ variable: string,
82
+ ): string[] {
83
+ const warnings: string[] = [];
84
+ if (report.unknown > 0) {
85
+ warnings.push(
86
+ `${report.unknown} objects are sealed under a key this ring does not hold and were left as they are. Put that key back in ${variable}_PREVIOUS before retiring it.`,
87
+ );
88
+ }
89
+ if (report.refused > 0) {
90
+ warnings.push(
91
+ `${report.refused} objects failed authentication under the key they name and were left as they are. Restore them from the object store backup.`,
92
+ );
93
+ }
94
+ if (report.missing > 0) {
95
+ warnings.push(
96
+ `${report.missing} rows name an object the store no longer holds.`,
97
+ );
98
+ }
99
+ return warnings;
100
+ }
101
+
102
+ export const cliExtension = defineCliExtension({
103
+ protocolVersion: 1,
104
+ moduleId: 'documents.core',
105
+ commands: [
106
+ {
107
+ path: ['documents', 'secrets-rotate'],
108
+ capability: rotateCapability,
109
+ execute: async (context) => {
110
+ /* The report names key ids and counts only; no object content
111
+ reaches the command output. The port is the platform's own, built
112
+ from the same environment the server reads. */
113
+ const storage = createStoragePort(
114
+ storageConfigFromEnvironment(process.env, context.workspaceRoot),
115
+ { keyring: createStorageKeyring(process.env, context.workspaceRoot) },
116
+ );
117
+ const opened = await open(context);
118
+ try {
119
+ const report = await rotateDocumentObjects({
120
+ runtime: opened.runtime.database,
121
+ background: opened.background.database,
122
+ storage,
123
+ apply: context.apply,
124
+ });
125
+ return {
126
+ data: { moduleId: 'documents.core', ...report },
127
+ evidence: [
128
+ 'modules/documents/spec/module.yaml',
129
+ 'docs/operations.md',
130
+ ],
131
+ warnings: storageRotationWarnings(
132
+ report,
133
+ 'FD_STORAGE_ENCRYPTION_KEY',
134
+ ),
135
+ };
136
+ } finally {
137
+ await close(opened);
138
+ await storage.dispose();
139
+ }
140
+ },
141
+ },
142
+ ],
143
+ });
144
+
145
+ export default cliExtension;
@@ -69,6 +69,9 @@ const SQL = {
69
69
  WHERE tenant_id = $1 AND owner_module = $2 AND record_ref = $3 AND id = $4`,
70
70
  create: `INSERT INTO documents_files (${COLUMNS})
71
71
  VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`,
72
+ lockStored: `SELECT id FROM documents_files
73
+ WHERE tenant_id = $1 AND id = $2 AND status = 'stored'
74
+ FOR UPDATE`,
72
75
  markDeleted: `UPDATE documents_files SET status = 'deleted'
73
76
  WHERE tenant_id = $1 AND id = $2 AND status = 'stored'
74
77
  RETURNING ${COLUMNS}`,
@@ -226,16 +229,24 @@ export class DatabaseDocumentsRepository implements DocumentsRepository {
226
229
  async markDeleted(
227
230
  tenantId: string,
228
231
  id: string,
232
+ discard: () => Promise<void>,
229
233
  ): Promise<DocumentsFile | null> {
230
234
  const result = await this.database.transaction(
231
- (transaction) =>
232
- transaction.query<DocumentsFileRow>({
235
+ async (transaction) => {
236
+ const locked = await transaction.query<{ id: string }>({
237
+ text: SQL.lockStored,
238
+ parameters: [tenantId, id],
239
+ });
240
+ if (locked.rows.length === 0) return null;
241
+ await discard();
242
+ return transaction.query<DocumentsFileRow>({
233
243
  text: SQL.markDeleted,
234
244
  parameters: [tenantId, id],
235
- }),
245
+ });
246
+ },
236
247
  { access: 'write', tenantId },
237
248
  );
238
- const row = result.rows[0];
249
+ const row = result?.rows[0];
239
250
  return row ? fromRow(row) : null;
240
251
  }
241
252
 
@@ -381,19 +381,20 @@ export class DocumentsService {
381
381
  };
382
382
  }
383
383
 
384
- /** The object first, the row after, so no row ever outlives its bytes. */
384
+ /**
385
+ * The object first, the row after, so no row ever outlives its bytes; both
386
+ * under the row lock, so a rewrite of the object cannot slip in between.
387
+ */
385
388
  async remove(tenantId: string, id: string): Promise<DocumentsFile> {
386
389
  const tenant = this.tenant(tenantId);
387
390
  const record = await this.require(tenant, id);
388
391
  if (record.status === 'deleted') return record;
389
- await this.storage.delete(this.referenceOf(record));
390
392
  /* A null means another request deleted the row between the read and the
391
- update; the object is gone either way, so the answer is the same. */
393
+ lock; the object is gone either way, so the answer is the same. */
392
394
  return (
393
- (await this.repository.markDeleted(tenant, id)) ?? {
394
- ...record,
395
- status: 'deleted',
396
- }
395
+ (await this.repository.markDeleted(tenant, id, () =>
396
+ this.storage.delete(this.referenceOf(record)).then(() => undefined),
397
+ )) ?? { ...record, status: 'deleted' }
397
398
  );
398
399
  }
399
400
 
@@ -469,8 +470,11 @@ export class DocumentsService {
469
470
  bounded(id, 'id', 1, DOCUMENT_LIMITS.id),
470
471
  );
471
472
  if (!record || record.status === 'deleted') return false;
472
- await this.storage.delete(this.referenceOf(record));
473
- return (await this.repository.markDeleted(tenant, id)) !== null;
473
+ return (
474
+ (await this.repository.markDeleted(tenant, id, () =>
475
+ this.storage.delete(this.referenceOf(record)).then(() => undefined),
476
+ )) !== null
477
+ );
474
478
  }
475
479
 
476
480
  private async remainingBytes(tenantId: string): Promise<number> {
@@ -52,6 +52,27 @@ CREATE INDEX IF NOT EXISTS documents_files_page_idx
52
52
  ON documents_files (tenant_id, created_at DESC, id DESC);
53
53
  `;
54
54
 
55
+ /* Mirrors migrations/0003_documents_rotation_inventory.up.sql byte for byte. */
56
+ export const DOCUMENTS_MIGRATION_003 = `-- The storage key rotation has to find the workspaces that still hold objects
57
+ -- before it knows which objects those are, so the cross-tenant role may count
58
+ -- stored rows by workspace and nothing else: the storage key, the record and
59
+ -- the file name stay invisible to it, and every object it names is read again
60
+ -- under the workspace that row named. PostgreSQL checks column privileges in
61
+ -- WHERE too, so \`status\` is part of the grant.
62
+ DO $$
63
+ BEGIN
64
+ IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'coreloom_background') THEN
65
+ RAISE EXCEPTION 'The coreloom_background role must exist before this migration.';
66
+ END IF;
67
+ END
68
+ $$;
69
+ CREATE POLICY documents_files_background_policy ON documents_files
70
+ FOR SELECT TO coreloom_background
71
+ USING (status = 'stored');
72
+ REVOKE SELECT ON documents_files FROM coreloom_background;
73
+ GRANT SELECT (tenant_id, status) ON documents_files TO coreloom_background;
74
+ `;
75
+
55
76
  export const databaseMigrations: readonly DatabaseMigration[] = [
56
77
  {
57
78
  id: '0001_documents_core',
@@ -79,4 +100,18 @@ export const databaseMigrations: readonly DatabaseMigration[] = [
79
100
  () => database.schema.hasIndex('documents_files_page_idx'),
80
101
  ]),
81
102
  },
103
+ {
104
+ id: '0003_documents_rotation_inventory',
105
+ sql: { postgresql: DOCUMENTS_MIGRATION_003 },
106
+ /* A policy and a column grant leave no schema object behind, so the
107
+ privilege itself is what proves this migration ran. */
108
+ inspectExisting: async (database) => {
109
+ const result = await database.query<{ granted: boolean }>({
110
+ text: `SELECT CASE WHEN to_regclass('documents_files') IS NOT NULL THEN
111
+ has_column_privilege('coreloom_background', 'documents_files', 'status', 'SELECT')
112
+ ELSE false END AS granted`,
113
+ });
114
+ return result.rows[0]?.granted === true ? 'complete' : 'absent';
115
+ },
116
+ },
82
117
  ];
@@ -41,8 +41,18 @@ export interface DocumentsRepository {
41
41
  id: string,
42
42
  ): Promise<DocumentsFile | null>;
43
43
  create(record: DocumentsFile): Promise<DocumentsFile>;
44
- /** The updated row, or null when it was already deleted or never existed. */
45
- markDeleted(tenantId: string, id: string): Promise<DocumentsFile | null>;
44
+ /**
45
+ * Locks the stored row, runs `discard` while it is held, then marks the row
46
+ * deleted. Null when it was already deleted or never existed, in which case
47
+ * `discard` does not run. The lock orders the delete against a pass that
48
+ * rewrites the object under the same lock, so neither can leave an object
49
+ * the other does not see.
50
+ */
51
+ markDeleted(
52
+ tenantId: string,
53
+ id: string,
54
+ discard: () => Promise<void>,
55
+ ): Promise<DocumentsFile | null>;
46
56
  /** Bytes the workspace still stores; the quota is measured against it. */
47
57
  storedBytes(tenantId: string): Promise<number>;
48
58
  }
@@ -0,0 +1,157 @@
1
+ import type { DatabaseHandle } from '@flowdular/sdk/database';
2
+ import type {
3
+ StorageObjectRef,
4
+ StorageResealCount,
5
+ StorageResealPort,
6
+ } from '@flowdular/sdk/storage';
7
+
8
+ /** Objects walked inside one tenant-scoped transaction. */
9
+ export const STORAGE_ROTATION_BATCH = 200;
10
+
11
+ export const STORAGE_ROTATION_TABLE = 'documents_files';
12
+
13
+ export interface StorageRotationReport {
14
+ readonly table: string;
15
+ /** Key id every object should end on: the current key of the storage ring. */
16
+ readonly currentKeyId: string;
17
+ readonly counts: readonly StorageResealCount[];
18
+ readonly tenants: number;
19
+ /** Stored rows the pass walked. */
20
+ readonly objects: number;
21
+ /** Objects on a retired key the ring still holds when the run started. */
22
+ readonly stale: number;
23
+ readonly resealed: number;
24
+ /** Objects under a key id the ring does not hold; left as they are. */
25
+ readonly unknown: number;
26
+ /** Objects that failed authentication; left as they are. */
27
+ readonly refused: number;
28
+ /** Stored rows whose object the store no longer holds. */
29
+ readonly missing: number;
30
+ }
31
+
32
+ export interface StorageRotationOptions {
33
+ /** Tenant-scoped handle. Every row is read on it. */
34
+ readonly runtime: DatabaseHandle;
35
+ /** Cross-tenant handle. It is granted the tenant id and the status and nothing else. */
36
+ readonly background: DatabaseHandle;
37
+ readonly storage: StorageResealPort;
38
+ readonly apply?: boolean;
39
+ readonly batchSize?: number;
40
+ }
41
+
42
+ const SQL = {
43
+ tenants: `SELECT tenant_id, count(*) AS row_count
44
+ FROM documents_files
45
+ WHERE status = 'stored'
46
+ GROUP BY tenant_id
47
+ ORDER BY tenant_id`,
48
+ /* Paged by primary key. The rows are locked while their objects are
49
+ rewritten, so a delete that marks one of them waits for the batch and
50
+ then removes the re-sealed object instead of racing its rewrite. */
51
+ batch: `SELECT id, storage_key
52
+ FROM documents_files
53
+ WHERE tenant_id = $1 AND status = 'stored' AND id > $2
54
+ ORDER BY id
55
+ LIMIT $3
56
+ FOR UPDATE`,
57
+ inventoryBatch: `SELECT id, storage_key
58
+ FROM documents_files
59
+ WHERE tenant_id = $1 AND status = 'stored' AND id > $2
60
+ ORDER BY id
61
+ LIMIT $3`,
62
+ };
63
+
64
+ function count(value: number | bigint | string): number {
65
+ const normalized = Number(value);
66
+ if (!Number.isSafeInteger(normalized)) {
67
+ throw new Error('The documents database returned an invalid count.');
68
+ }
69
+ return normalized;
70
+ }
71
+
72
+ /* The row is the inventory, and its key names the tenant it was written under;
73
+ a row whose key names another tenant is a defect, not an object to rewrite. */
74
+ function referenceOf(tenantId: string, storageKey: string): StorageObjectRef {
75
+ const [tenant, moduleId, objectId, ...rest] = storageKey.split('/');
76
+ if (tenant !== tenantId || !moduleId || !objectId || rest.length > 0) {
77
+ throw new Error(
78
+ `A documents_files row of ${tenantId} names the storage key ${storageKey}, which is not one of its objects.`,
79
+ );
80
+ }
81
+ return { tenantId, moduleId, objectId };
82
+ }
83
+
84
+ /**
85
+ * Re-seals every stored document object that is not on the current storage
86
+ * key. The workspaces are listed once on the cross-tenant role, and each batch
87
+ * of rows is read under its own tenant; the objects they name are read from
88
+ * the store, and the stale ones rewritten in place. It is idempotent: a second
89
+ * run finds nothing to do.
90
+ */
91
+ export async function rotateDocumentObjects(
92
+ options: StorageRotationOptions,
93
+ ): Promise<StorageRotationReport> {
94
+ const apply = options.apply === true;
95
+ const batchSize = options.batchSize ?? STORAGE_ROTATION_BATCH;
96
+ const inventory = await options.background.transaction(
97
+ (transaction) =>
98
+ transaction.query<{
99
+ tenant_id: string;
100
+ row_count: number | bigint | string;
101
+ }>({ text: SQL.tenants }),
102
+ { access: 'read' },
103
+ );
104
+ const counts = new Map<string, number>();
105
+ let objects = 0;
106
+ let stale = 0;
107
+ let resealed = 0;
108
+ let unknown = 0;
109
+ let refused = 0;
110
+ let missing = 0;
111
+ for (const { tenant_id: tenantId, row_count: rowCount } of inventory.rows) {
112
+ objects += count(rowCount);
113
+ let cursor = '';
114
+ for (;;) {
115
+ const batch = await options.runtime.transaction(
116
+ async (transaction) => {
117
+ const rows = (
118
+ await transaction.query<{ id: string; storage_key: string }>({
119
+ text: apply ? SQL.batch : SQL.inventoryBatch,
120
+ parameters: [tenantId, cursor, batchSize],
121
+ })
122
+ ).rows;
123
+ const report = await options.storage.reseal(
124
+ rows.map((row) => referenceOf(tenantId, row.storage_key)),
125
+ { apply },
126
+ );
127
+ return { read: rows.length, last: rows.at(-1)?.id, report };
128
+ },
129
+ { access: apply ? 'write' : 'read', tenantId },
130
+ );
131
+ for (const entry of batch.report.counts) {
132
+ counts.set(entry.keyId, (counts.get(entry.keyId) ?? 0) + entry.objects);
133
+ }
134
+ stale += batch.report.stale;
135
+ resealed += batch.report.resealed;
136
+ unknown += batch.report.unknown;
137
+ refused += batch.report.refused;
138
+ missing += batch.report.missing;
139
+ if (batch.read < batchSize || batch.last === undefined) break;
140
+ cursor = batch.last;
141
+ }
142
+ }
143
+ return {
144
+ table: STORAGE_ROTATION_TABLE,
145
+ currentKeyId: options.storage.keyId,
146
+ counts: [...counts]
147
+ .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
148
+ .map(([keyId, total]) => ({ keyId, objects: total })),
149
+ tenants: inventory.rows.length,
150
+ objects,
151
+ stale,
152
+ resealed,
153
+ unknown,
154
+ refused,
155
+ missing,
156
+ };
157
+ }
@@ -0,0 +1 @@
1
+ DROP POLICY IF EXISTS exports_jobs_rotation_policy ON exports_jobs;
@@ -0,0 +1,9 @@
1
+ -- The storage key rotation has to find the workspaces that still hold export
2
+ -- files before it knows which files those are. The routing policy shows the
3
+ -- background role only waiting and running jobs; this one adds the completed
4
+ -- jobs under the same four routing columns, so the object id, the list, the
5
+ -- requester and every count stay invisible to it, and every file it names is
6
+ -- read again under the workspace that row named.
7
+ CREATE POLICY exports_jobs_rotation_policy ON exports_jobs
8
+ FOR SELECT TO coreloom_background
9
+ USING (status = 'completed');
@@ -3,10 +3,10 @@
3
3
  "schemaVersion": 1,
4
4
  "id": "exports.core",
5
5
  "package": "@flowdular/module-exports",
6
- "version": "0.2.2",
6
+ "version": "0.2.3",
7
7
  "platformApi": "^0.1.2",
8
8
  "profile": "full",
9
- "capabilities": ["api", "database", "client", "translations"],
9
+ "capabilities": ["api", "database", "client", "translations", "cli"],
10
10
  "platform": {
11
11
  "server": true,
12
12
  "client": true
@@ -24,5 +24,9 @@
24
24
  "provides": ["exports.lists.v1"],
25
25
  "tenancy": "required",
26
26
  "locales": ["en", "pl"],
27
- "stability": "experimental"
27
+ "stability": "experimental",
28
+ "cli": {
29
+ "catalog": "src/cli/commands.json",
30
+ "entry": "src/cli/index.ts"
31
+ }
28
32
  }
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@flowdular/module-exports",
3
- "version": "0.2.2",
3
+ "version": "0.2.3",
4
4
  "private": true,
5
5
  "type": "module",
6
6
  "exports": {
@@ -23,6 +23,7 @@
23
23
  "test": "vitest run"
24
24
  },
25
25
  "dependencies": {
26
+ "@flowdular/cli-protocol": "workspace:*",
26
27
  "@flowdular/client": "workspace:*",
27
28
  "@flowdular/contracts": "workspace:*",
28
29
  "@flowdular/database": "workspace:*",
@@ -1,6 +1,6 @@
1
1
  schemaVersion: 2
2
2
  id: exports.core
3
- specVersion: 0.2.2
3
+ specVersion: 0.2.3
4
4
  status: approved
5
5
  name: Exports Core
6
6
  description: Exports any list endpoint that already pages with the platform keyset helpers as one CSV file, through an export job that streams the list's own pages under the requester's grants, writes an RFC 4180 file with a UTF-8 byte order mark to the storage port, bounds the result by rows and by bytes, and answers a short-lived signed read URL; the list itself stays with the module that owns it and registers a declaration through the public capability exports.lists.v1.
@@ -10,6 +10,7 @@ capabilities:
10
10
  - database
11
11
  - client
12
12
  - translations
13
+ - cli
13
14
  dependencies:
14
15
  - id: system.core
15
16
  range: ^0.7.0