@empire-builder-kit/nx 1.0.0-beta.10 → 1.0.0-beta.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,6 +4,7 @@ exports.STANDARD_LOCAL_DEV_ADAPTER = exports.PRODUCT_PACKAGE_MANAGED_JSON_POINTE
4
4
  exports.createBuiltInSlicePlan = createBuiltInSlicePlan;
5
5
  exports.sliceGenerator = sliceGenerator;
6
6
  const devkit_1 = require("@nx/devkit");
7
+ const node_fs_1 = require("node:fs");
7
8
  const node_path_1 = require("node:path");
8
9
  const index_js_1 = require("../../blueprints/index.js");
9
10
  const index_js_2 = require("../../foundation/index.js");
@@ -12,8 +13,8 @@ const index_js_4 = require("../../repository-policy/index.js");
12
13
  const PRODUCT_VERSIONS = {
13
14
  awsCognito: '^3.700.0',
14
15
  awsEventBridge: '^3.700.0',
15
- awsRdsData: '^3.700.0',
16
- awsSecretsManager: '^3.700.0',
16
+ awsRdsSigner: '^3.700.0',
17
+ awsSfn: '^3.700.0',
17
18
  awsSsm: '^3.700.0',
18
19
  drizzle: '^0.45.1',
19
20
  drizzleKit: '^0.31.4',
@@ -23,6 +24,7 @@ const PRODUCT_VERSIONS = {
23
24
  react: '19.2.4',
24
25
  reactDom: '19.2.4',
25
26
  typesNode: '25.5.0',
27
+ typesPg: '^8.15.5',
26
28
  typesReact: '19.2.14',
27
29
  typesReactDom: '19.2.3',
28
30
  vitest: '4.1.4',
@@ -33,20 +35,21 @@ exports.PRODUCT_PACKAGE_MANAGED_JSON_POINTERS = [
33
35
  '/private',
34
36
  '/dependencies/@aws-sdk~1client-cognito-identity-provider',
35
37
  '/dependencies/@aws-sdk~1client-eventbridge',
36
- '/dependencies/@aws-sdk~1client-rds-data',
37
- '/dependencies/@aws-sdk~1client-secrets-manager',
38
+ '/dependencies/@aws-sdk~1client-sfn',
39
+ '/dependencies/@aws-sdk~1rds-signer',
38
40
  '/dependencies/@aws-sdk~1client-ssm',
39
41
  '/dependencies/@empire-builder-kit~1runtime',
40
42
  '/dependencies/drizzle-orm',
41
43
  '/dependencies/next',
44
+ '/dependencies/pg',
42
45
  '/dependencies/react',
43
46
  '/dependencies/react-dom',
44
47
  '/devDependencies/@playwright~1test',
45
48
  '/devDependencies/@types~1node',
49
+ '/devDependencies/@types~1pg',
46
50
  '/devDependencies/@types~1react',
47
51
  '/devDependencies/@types~1react-dom',
48
52
  '/devDependencies/drizzle-kit',
49
- '/devDependencies/pg',
50
53
  '/devDependencies/vitest',
51
54
  '/exports/.~1contracts',
52
55
  ];
@@ -94,6 +97,18 @@ function workspaceScope(packageJson) {
94
97
  function json(value) {
95
98
  return `${JSON.stringify(value, null, 2)}\n`;
96
99
  }
100
+ const MIGRATION_PACKAGE_LOCK = JSON.parse((0, node_fs_1.readFileSync)(require.resolve('./migration-package-lock.json'), 'utf8'));
101
+ function renderMigrationPackageLock(name) {
102
+ const lock = structuredClone(MIGRATION_PACKAGE_LOCK);
103
+ const packageName = `@ebk-migration/${name}`;
104
+ lock.name = packageName;
105
+ const root = lock.packages[''];
106
+ if (!root) {
107
+ throw new Error('Migration package lock is missing its root package.');
108
+ }
109
+ root.name = packageName;
110
+ return json(lock);
111
+ }
97
112
  function reportSlicePolicyPosture(tree, name) {
98
113
  const contents = tree.read(index_js_4.REPOSITORY_POLICY_FILE, 'utf8');
99
114
  if (contents === null) {
@@ -135,12 +150,999 @@ function parseBlueprintInputs(inputs) {
135
150
  }
136
151
  function renderStandardProductFiles(name, pascalName) {
137
152
  return {
153
+ '.dockerignore': `.sst
154
+ .env
155
+ .env.*
156
+ !.env.example
157
+ dist
158
+ node_modules
159
+ web/.next
160
+ web/.open-next
161
+ `,
138
162
  'database/bindings.json': json({
139
163
  primary: {
140
164
  active: 'primary',
141
165
  provisioned: ['primary'],
142
166
  },
143
167
  }),
168
+ 'migration/Dockerfile': `FROM node:22-alpine@sha256:16e22a550f3863206a3f701448c45f7912c6896a62de43add43bb9c86130c3e2
169
+
170
+ WORKDIR /app
171
+ ADD --checksum=sha256:e5bb2084ccf45087bda1c9bffdea0eb15ee67f0b91646106e466714f9de3c7e3 https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem ./rds-global-bundle.pem
172
+ COPY migration/package.json migration/package-lock.json ./
173
+ RUN npm ci --omit=dev --omit=optional --ignore-scripts
174
+
175
+ COPY jobs/bootstrap-database.mjs ./jobs/bootstrap-database.mjs
176
+ COPY jobs/migrate-database.mjs ./jobs/migrate-database.mjs
177
+ COPY jobs/retire-database.mjs ./jobs/retire-database.mjs
178
+ COPY database/migrations ./database/migrations
179
+ USER node
180
+ CMD ["node", "jobs/migrate-database.mjs"]
181
+ `,
182
+ 'migration/package.json': json({
183
+ name: `@ebk-migration/${name}`,
184
+ version: '0.0.0',
185
+ private: true,
186
+ type: 'module',
187
+ dependencies: {
188
+ '@aws-sdk/client-secrets-manager': '3.1090.0',
189
+ '@aws-sdk/rds-signer': '3.1090.0',
190
+ pg: '8.22.0',
191
+ },
192
+ }),
193
+ 'migration/package-lock.json': renderMigrationPackageLock(name),
194
+ 'jobs/bootstrap-database.mjs': `import { randomBytes } from 'node:crypto';
195
+ import { readFile } from 'node:fs/promises';
196
+ import {
197
+ GetSecretValueCommand,
198
+ PutSecretValueCommand,
199
+ SecretsManagerClient,
200
+ } from '@aws-sdk/client-secrets-manager';
201
+ import { Signer } from '@aws-sdk/rds-signer';
202
+ import pg from 'pg';
203
+
204
+ const { Client } = pg;
205
+ const required = (name) => {
206
+ const value = process.env[name]?.trim();
207
+ if (!value) throw new Error(name + ' is required.');
208
+ return value;
209
+ };
210
+ const requiredIdentifier = (name) => {
211
+ const value = required(name);
212
+ if (!/^[a-z][a-z0-9_]{0,62}$/.test(value)) {
213
+ throw new Error(name + ' must be a normalized PostgreSQL identifier.');
214
+ }
215
+ return value;
216
+ };
217
+ const requiredPort = () => {
218
+ const value = required('EBK_DATABASE_PORT');
219
+ if (!/^\\d+$/.test(value)) {
220
+ throw new Error('EBK_DATABASE_PORT must be an integer.');
221
+ }
222
+ const port = Number(value);
223
+ if (!Number.isSafeInteger(port) || port < 1 || port > 65535) {
224
+ throw new Error('EBK_DATABASE_PORT must be from 1 through 65535.');
225
+ }
226
+ return port;
227
+ };
228
+ let bootstrapId = 'unresolved';
229
+ let bootstrapPhase = 'configuration';
230
+ let bootstrapLockClient;
231
+ let bootstrapLockKey;
232
+
233
+ const log = (level, event, fields = {}) => {
234
+ process.stdout.write(JSON.stringify({
235
+ bootstrapId,
236
+ event,
237
+ level,
238
+ timestamp: new Date().toISOString(),
239
+ ...fields,
240
+ }) + '\\n');
241
+ };
242
+ const safeErrorFields = (error) => {
243
+ const name = error instanceof Error ? error.name : 'UnknownError';
244
+ const code =
245
+ error &&
246
+ typeof error === 'object' &&
247
+ typeof error.code === 'string' &&
248
+ /^[A-Z0-9_]{2,32}$/.test(error.code)
249
+ ? error.code
250
+ : undefined;
251
+ return {
252
+ errorName: name.replace(/[^A-Za-z0-9_.-]/g, '').slice(0, 80) || 'Error',
253
+ ...(code ? { errorCode: code } : {}),
254
+ };
255
+ };
256
+ const retryableConnection = (error) => {
257
+ const code = error && typeof error === 'object' ? error.code : undefined;
258
+ const message = error instanceof Error ? error.message : String(error);
259
+ return new Set([
260
+ '08000',
261
+ '08001',
262
+ '08003',
263
+ '08006',
264
+ '57P03',
265
+ 'ECONNREFUSED',
266
+ 'ECONNRESET',
267
+ 'ETIMEDOUT',
268
+ ]).has(code) ||
269
+ /resum|starting up|temporarily unavailable|timeout|connection terminated unexpectedly|server closed the connection unexpectedly|terminating connection/i.test(message);
270
+ };
271
+ const sleep = (milliseconds) =>
272
+ new Promise((resolve) => setTimeout(resolve, milliseconds));
273
+ const quoteIdentifier = (value) => {
274
+ if (!/^[a-z][a-z0-9_]{0,62}$/.test(value)) {
275
+ throw new Error('Unsafe PostgreSQL identifier.');
276
+ }
277
+ return '"' + value + '"';
278
+ };
279
+ const quoteLiteral = (value) => "'" + value.replaceAll("'", "''") + "'";
280
+ const generatePassword = () =>
281
+ randomBytes(24)
282
+ .toString('base64')
283
+ .replace(/[^A-Za-z0-9]/g, '')
284
+ .slice(0, 32)
285
+ .padEnd(32, 'a');
286
+
287
+ const secrets = new SecretsManagerClient({});
288
+ const readJsonSecret = async (secretArn, missingAllowed = false) => {
289
+ try {
290
+ const result = await secrets.send(
291
+ new GetSecretValueCommand({ SecretId: secretArn }),
292
+ );
293
+ if (!result.SecretString) {
294
+ throw new Error('Secret must contain a JSON SecretString.');
295
+ }
296
+ const value = JSON.parse(result.SecretString);
297
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
298
+ throw new Error('Secret JSON must be an object.');
299
+ }
300
+ return value;
301
+ } catch (error) {
302
+ if (
303
+ missingAllowed &&
304
+ error &&
305
+ typeof error === 'object' &&
306
+ error.name === 'ResourceNotFoundException'
307
+ ) {
308
+ return undefined;
309
+ }
310
+ throw error;
311
+ }
312
+ };
313
+ const runtimeCredential = async ({
314
+ database,
315
+ host,
316
+ port,
317
+ rotationToken,
318
+ secretArn,
319
+ username,
320
+ }) => {
321
+ const current = await readJsonSecret(secretArn, true);
322
+ const identityMatches =
323
+ current &&
324
+ current.database === database &&
325
+ current.username === username &&
326
+ typeof current.password === 'string' &&
327
+ /^[A-Za-z0-9]{32}$/.test(current.password);
328
+ if (current && !identityMatches) {
329
+ throw new Error('Existing generated database credential has incompatible metadata.');
330
+ }
331
+ const rotate =
332
+ rotationToken !== '' && current?.rotationToken !== rotationToken;
333
+ const connectionMetadataChanged =
334
+ current && (current.host !== host || current.port !== port);
335
+ return {
336
+ needsWrite: !current || rotate || connectionMetadataChanged,
337
+ password: !current || rotate ? generatePassword() : current.password,
338
+ };
339
+ };
340
+ const writeRuntimeCredential = async ({
341
+ database,
342
+ host,
343
+ password,
344
+ port,
345
+ rotationToken,
346
+ secretArn,
347
+ username,
348
+ }) => {
349
+ await secrets.send(
350
+ new PutSecretValueCommand({
351
+ SecretId: secretArn,
352
+ SecretString: JSON.stringify({
353
+ database,
354
+ host,
355
+ password,
356
+ port,
357
+ ...(rotationToken ? { rotationToken } : {}),
358
+ username,
359
+ }),
360
+ }),
361
+ );
362
+ };
363
+ const publishRuntimeCredentials = async ({
364
+ database,
365
+ host,
366
+ port,
367
+ readonly,
368
+ readonlySecretArn,
369
+ readonlyRole,
370
+ readwrite,
371
+ readwriteSecretArn,
372
+ readwriteRole,
373
+ rotationToken,
374
+ }) => {
375
+ if (readwrite.needsWrite) {
376
+ await writeRuntimeCredential({
377
+ database,
378
+ host,
379
+ password: readwrite.password,
380
+ port,
381
+ rotationToken,
382
+ secretArn: readwriteSecretArn,
383
+ username: readwriteRole,
384
+ });
385
+ }
386
+ if (readonly.needsWrite) {
387
+ await writeRuntimeCredential({
388
+ database,
389
+ host,
390
+ password: readonly.password,
391
+ port,
392
+ rotationToken,
393
+ secretArn: readonlySecretArn,
394
+ username: readonlyRole,
395
+ });
396
+ }
397
+ };
398
+ const connectWithRetry = async ({
399
+ database,
400
+ host,
401
+ password,
402
+ port,
403
+ roleKind,
404
+ username,
405
+ ssl,
406
+ }) => {
407
+ const startedAt = Date.now();
408
+ let lastError;
409
+ for (let attempt = 1; attempt <= 10; attempt += 1) {
410
+ let candidate;
411
+ try {
412
+ candidate = new Client({
413
+ connectionTimeoutMillis: 10_000,
414
+ database,
415
+ host,
416
+ password:
417
+ typeof password === 'function' ? await password() : password,
418
+ port,
419
+ ssl: { ca: ssl, rejectUnauthorized: true },
420
+ statement_timeout: 10 * 60 * 1000,
421
+ user: username,
422
+ });
423
+ candidate.on('error', (error) => {
424
+ log('error', 'database_bootstrap_client_error', {
425
+ roleKind,
426
+ ...safeErrorFields(error),
427
+ });
428
+ });
429
+ await candidate.connect();
430
+ return candidate;
431
+ } catch (error) {
432
+ lastError = error;
433
+ await candidate?.end().catch(() => undefined);
434
+ if (!retryableConnection(error) || attempt === 10) throw error;
435
+ log('warn', 'database_bootstrap_connection_retry', {
436
+ attempt,
437
+ elapsedMs: Date.now() - startedAt,
438
+ roleKind,
439
+ ...safeErrorFields(error),
440
+ });
441
+ await sleep(Math.min(attempt * 5_000, 30_000));
442
+ }
443
+ }
444
+ throw lastError;
445
+ };
446
+ const roleExists = async (client, role) => {
447
+ const result = await client.query(
448
+ 'SELECT 1 FROM pg_roles WHERE rolname = ' + quoteLiteral(role),
449
+ );
450
+ return result.rowCount > 0;
451
+ };
452
+ const executeStatements = async (client, statements) => {
453
+ for (const statement of statements) await client.query(statement);
454
+ };
455
+ const releaseBootstrapLock = async () => {
456
+ const client = bootstrapLockClient;
457
+ const key = bootstrapLockKey;
458
+ bootstrapLockClient = undefined;
459
+ bootstrapLockKey = undefined;
460
+ if (!client) return;
461
+ if (key) {
462
+ await client
463
+ .query('SELECT pg_advisory_unlock(hashtextextended($1, 0))', [key])
464
+ .catch(() => undefined);
465
+ }
466
+ await client.end().catch(() => undefined);
467
+ };
468
+
469
+ async function main() {
470
+ const database = requiredIdentifier('EBK_DATABASE_NAME');
471
+ const host = required('EBK_DATABASE_HOST');
472
+ const masterSecretArn = required('EBK_MASTER_SECRET_ARN');
473
+ const migratorRole = requiredIdentifier('EBK_DATABASE_MIGRATOR_ROLE');
474
+ const port = requiredPort();
475
+ const readonlyRole = requiredIdentifier('EBK_DATABASE_READONLY_ROLE');
476
+ const readonlySecretArn = required('EBK_READONLY_SECRET_ARN');
477
+ const readwriteRole = requiredIdentifier('EBK_DATABASE_READWRITE_ROLE');
478
+ const readwriteSecretArn = required('EBK_READWRITE_SECRET_ARN');
479
+ const region = required('AWS_REGION');
480
+ const rotationToken = process.env.EBK_DATABASE_ROTATION_TOKEN?.trim() ?? '';
481
+ const runtimeHost = required('EBK_RUNTIME_DATABASE_HOST');
482
+ const ca = await readFile(required('EBK_RDS_CA_BUNDLE'), 'utf8');
483
+ bootstrapId = required('EBK_BOOTSTRAP_ID');
484
+
485
+ bootstrapPhase = 'credential-resolution';
486
+ const master = await readJsonSecret(masterSecretArn);
487
+ if (
488
+ typeof master.username !== 'string' ||
489
+ typeof master.password !== 'string' ||
490
+ master.password.length === 0
491
+ ) {
492
+ throw new Error('Foundation master secret lacks username or password.');
493
+ }
494
+ if (!/^[a-z][a-z0-9_]{0,62}$/.test(master.username)) {
495
+ throw new Error('Foundation master secret username is invalid.');
496
+ }
497
+ const masterUsername = master.username;
498
+
499
+ bootstrapPhase = 'database-convergence';
500
+ const postgres = await connectWithRetry({
501
+ database: 'postgres',
502
+ host,
503
+ password: master.password,
504
+ port,
505
+ roleKind: 'master',
506
+ ssl: ca,
507
+ username: masterUsername,
508
+ });
509
+ bootstrapLockClient = postgres;
510
+ bootstrapLockKey = 'ebk-bootstrap:' + database;
511
+ await postgres.query(
512
+ 'SELECT pg_advisory_lock(hashtextextended($1, 0))',
513
+ [bootstrapLockKey],
514
+ );
515
+ bootstrapPhase = 'credential-resolution';
516
+ const readwrite = await runtimeCredential({
517
+ database,
518
+ host: runtimeHost,
519
+ port,
520
+ rotationToken,
521
+ secretArn: readwriteSecretArn,
522
+ username: readwriteRole,
523
+ });
524
+ const readonly = await runtimeCredential({
525
+ database,
526
+ host: runtimeHost,
527
+ port,
528
+ rotationToken,
529
+ secretArn: readonlySecretArn,
530
+ username: readonlyRole,
531
+ });
532
+
533
+ bootstrapPhase = 'database-convergence';
534
+ const existing = await postgres.query(
535
+ 'SELECT 1 FROM pg_database WHERE datname = ' + quoteLiteral(database),
536
+ );
537
+ if (existing.rowCount === 0) {
538
+ await postgres.query('CREATE DATABASE ' + quoteIdentifier(database));
539
+ }
540
+
541
+ const masterDatabase = await connectWithRetry({
542
+ database,
543
+ host,
544
+ password: master.password,
545
+ port,
546
+ roleKind: 'master',
547
+ ssl: ca,
548
+ username: masterUsername,
549
+ });
550
+ try {
551
+ for (const [role, password] of [
552
+ [readwriteRole, readwrite.password],
553
+ [readonlyRole, readonly.password],
554
+ ]) {
555
+ if (await roleExists(masterDatabase, role)) {
556
+ await masterDatabase.query(
557
+ 'ALTER ROLE ' + quoteIdentifier(role) + ' WITH LOGIN PASSWORD ' + quoteLiteral(password),
558
+ );
559
+ } else {
560
+ await masterDatabase.query(
561
+ 'CREATE ROLE ' + quoteIdentifier(role) + ' WITH LOGIN PASSWORD ' + quoteLiteral(password),
562
+ );
563
+ }
564
+ }
565
+ if (await roleExists(masterDatabase, migratorRole)) {
566
+ await masterDatabase.query(
567
+ 'ALTER ROLE ' + quoteIdentifier(migratorRole) + ' WITH LOGIN PASSWORD NULL',
568
+ );
569
+ } else {
570
+ await masterDatabase.query(
571
+ 'CREATE ROLE ' + quoteIdentifier(migratorRole) + ' WITH LOGIN',
572
+ );
573
+ }
574
+
575
+ // PostgreSQL 16+ can record creator ADMIN memberships when a CREATEROLE
576
+ // principal creates a role. Keep that non-inheriting administration path
577
+ // available for the membership grants below, but explicitly remove SET
578
+ // and INHERIT options before the migrator receives rds_iam. Default
579
+ // privileges are then converged through direct role-owned sessions.
580
+ await executeStatements(masterDatabase, [
581
+ 'REVOKE INHERIT OPTION FOR ' + quoteIdentifier(migratorRole) +
582
+ ' FROM CURRENT_USER',
583
+ 'REVOKE SET OPTION FOR ' + quoteIdentifier(migratorRole) +
584
+ ' FROM CURRENT_USER',
585
+ 'REVOKE INHERIT OPTION FOR ' + quoteIdentifier(readwriteRole) +
586
+ ' FROM CURRENT_USER',
587
+ 'REVOKE SET OPTION FOR ' + quoteIdentifier(readwriteRole) +
588
+ ' FROM CURRENT_USER',
589
+ 'REVOKE ALL ON DATABASE ' + quoteIdentifier(database) + ' FROM PUBLIC',
590
+ 'REVOKE CREATE ON DATABASE ' + quoteIdentifier(database) + ' FROM ' +
591
+ quoteIdentifier(migratorRole) + ', ' + quoteIdentifier(readwriteRole),
592
+ 'GRANT CONNECT ON DATABASE ' + quoteIdentifier(database) + ' TO ' +
593
+ quoteIdentifier(migratorRole) + ', ' + quoteIdentifier(readwriteRole) +
594
+ ', ' + quoteIdentifier(readonlyRole),
595
+ 'GRANT ALL ON SCHEMA public TO ' + quoteIdentifier(migratorRole) +
596
+ ', ' + quoteIdentifier(readwriteRole),
597
+ 'GRANT USAGE ON SCHEMA public TO ' + quoteIdentifier(readonlyRole),
598
+ 'GRANT rds_iam TO ' + quoteIdentifier(migratorRole),
599
+ 'GRANT ' + quoteIdentifier(readwriteRole) + ' TO ' +
600
+ quoteIdentifier(migratorRole),
601
+ ]);
602
+ } finally {
603
+ await masterDatabase.end().catch(() => undefined);
604
+ }
605
+
606
+ // Publish immediately after ALTER ROLE convergence. A database password and
607
+ // Secrets Manager AWSCURRENT cannot change atomically, so keep this
608
+ // serialized stale window ahead of slower default-privilege work. Runtime
609
+ // authentication failure invalidates the bounded cache and retries once.
610
+ bootstrapPhase = 'runtime-secret-publication';
611
+ await publishRuntimeCredentials({
612
+ database,
613
+ host: runtimeHost,
614
+ port,
615
+ readonly,
616
+ readonlySecretArn,
617
+ readonlyRole,
618
+ readwrite,
619
+ readwriteSecretArn,
620
+ readwriteRole,
621
+ rotationToken,
622
+ });
623
+
624
+ bootstrapPhase = 'default-privilege-convergence';
625
+ const readwriteDatabase = await connectWithRetry({
626
+ database,
627
+ host,
628
+ password: readwrite.password,
629
+ port,
630
+ roleKind: 'readwrite',
631
+ ssl: ca,
632
+ username: readwriteRole,
633
+ });
634
+ try {
635
+ await executeStatements(readwriteDatabase, [
636
+ 'ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO ' +
637
+ quoteIdentifier(readwriteRole),
638
+ 'ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ' +
639
+ quoteIdentifier(readonlyRole),
640
+ 'ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO ' +
641
+ quoteIdentifier(readonlyRole),
642
+ ]);
643
+ } finally {
644
+ await readwriteDatabase.end().catch(() => undefined);
645
+ }
646
+
647
+ const signer = new Signer({
648
+ hostname: host,
649
+ port,
650
+ region,
651
+ username: migratorRole,
652
+ });
653
+ const migratorDatabase = await connectWithRetry({
654
+ database,
655
+ host,
656
+ password: () => signer.getAuthToken(),
657
+ port,
658
+ roleKind: 'migrator',
659
+ ssl: ca,
660
+ username: migratorRole,
661
+ });
662
+ try {
663
+ await executeStatements(migratorDatabase, [
664
+ 'ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO ' +
665
+ quoteIdentifier(readwriteRole),
666
+ 'ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ' +
667
+ quoteIdentifier(readonlyRole),
668
+ 'ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO ' +
669
+ quoteIdentifier(readwriteRole),
670
+ 'ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO ' +
671
+ quoteIdentifier(readonlyRole),
672
+ ]);
673
+ } finally {
674
+ await migratorDatabase.end().catch(() => undefined);
675
+ }
676
+
677
+ await releaseBootstrapLock();
678
+ log('info', 'database_bootstrap_succeeded', { database });
679
+ }
680
+
681
+ main().catch(async (error) => {
682
+ await releaseBootstrapLock();
683
+ log('error', 'database_bootstrap_failed', {
684
+ phase: bootstrapPhase,
685
+ ...safeErrorFields(error),
686
+ });
687
+ process.exitCode = 1;
688
+ });
689
+ `,
690
+ 'jobs/retire-database.mjs': `import { readFile } from 'node:fs/promises';
691
+ import {
692
+ GetSecretValueCommand,
693
+ SecretsManagerClient,
694
+ } from '@aws-sdk/client-secrets-manager';
695
+ import pg from 'pg';
696
+
697
+ const { Client } = pg;
698
+ const required = (name) => {
699
+ const value = process.env[name]?.trim();
700
+ if (!value) throw new Error(name + ' is required.');
701
+ return value;
702
+ };
703
+ const requiredIdentifier = (name) => {
704
+ const value = required(name);
705
+ if (!/^[a-z][a-z0-9_]{0,62}$/.test(value)) {
706
+ throw new Error(name + ' must be a normalized PostgreSQL identifier.');
707
+ }
708
+ return value;
709
+ };
710
+ const requiredPort = () => {
711
+ const value = required('EBK_DATABASE_PORT');
712
+ if (!/^\\d+$/.test(value)) {
713
+ throw new Error('EBK_DATABASE_PORT must be an integer.');
714
+ }
715
+ const port = Number(value);
716
+ if (!Number.isSafeInteger(port) || port < 1 || port > 65535) {
717
+ throw new Error('EBK_DATABASE_PORT must be from 1 through 65535.');
718
+ }
719
+ return port;
720
+ };
721
+ let retirementId = 'unresolved';
722
+ let retirementPhase = 'configuration';
723
+
724
+ const log = (level, event, fields = {}) => {
725
+ process.stdout.write(JSON.stringify({
726
+ event,
727
+ level,
728
+ retirementId,
729
+ timestamp: new Date().toISOString(),
730
+ ...fields,
731
+ }) + '\\n');
732
+ };
733
+ const safeErrorFields = (error) => {
734
+ const name = error instanceof Error ? error.name : 'UnknownError';
735
+ const code =
736
+ error &&
737
+ typeof error === 'object' &&
738
+ typeof error.code === 'string' &&
739
+ /^[A-Z0-9_]{2,32}$/.test(error.code)
740
+ ? error.code
741
+ : undefined;
742
+ return {
743
+ errorName: name.replace(/[^A-Za-z0-9_.-]/g, '').slice(0, 80) || 'Error',
744
+ ...(code ? { errorCode: code } : {}),
745
+ };
746
+ };
747
+ const quoteIdentifier = (value) => {
748
+ if (!/^[a-z][a-z0-9_]{0,62}$/.test(value)) {
749
+ throw new Error('Unsafe PostgreSQL identifier.');
750
+ }
751
+ return '"' + value + '"';
752
+ };
753
+ const quoteLiteral = (value) => "'" + value.replaceAll("'", "''") + "'";
754
+ const secrets = new SecretsManagerClient({});
755
+ const readJsonSecret = async (secretArn) => {
756
+ const result = await secrets.send(
757
+ new GetSecretValueCommand({ SecretId: secretArn }),
758
+ );
759
+ if (!result.SecretString) {
760
+ throw new Error('Secret must contain a JSON SecretString.');
761
+ }
762
+ const value = JSON.parse(result.SecretString);
763
+ if (!value || typeof value !== 'object' || Array.isArray(value)) {
764
+ throw new Error('Secret JSON must be an object.');
765
+ }
766
+ return value;
767
+ };
768
+
769
+ async function main() {
770
+ const database = requiredIdentifier('EBK_DATABASE_NAME');
771
+ const host = required('EBK_DATABASE_HOST');
772
+ const masterSecretArn = required('EBK_MASTER_SECRET_ARN');
773
+ const migratorRole = requiredIdentifier('EBK_DATABASE_MIGRATOR_ROLE');
774
+ const port = requiredPort();
775
+ const readonlyRole = requiredIdentifier('EBK_DATABASE_READONLY_ROLE');
776
+ const readwriteRole = requiredIdentifier('EBK_DATABASE_READWRITE_ROLE');
777
+ const ca = await readFile(required('EBK_RDS_CA_BUNDLE'), 'utf8');
778
+ retirementId = required('EBK_RETIREMENT_ID');
779
+
780
+ retirementPhase = 'credential-resolution';
781
+ const master = await readJsonSecret(masterSecretArn);
782
+ if (
783
+ typeof master.username !== 'string' ||
784
+ typeof master.password !== 'string' ||
785
+ master.password.length === 0 ||
786
+ !/^[a-z][a-z0-9_]{0,62}$/.test(master.username)
787
+ ) {
788
+ throw new Error('Foundation master secret is invalid.');
789
+ }
790
+ const postgres = new Client({
791
+ connectionTimeoutMillis: 10_000,
792
+ database: 'postgres',
793
+ host,
794
+ password: master.password,
795
+ port,
796
+ ssl: { ca, rejectUnauthorized: true },
797
+ statement_timeout: 10 * 60 * 1000,
798
+ user: master.username,
799
+ });
800
+ postgres.on('error', (error) => {
801
+ log('error', 'database_retirement_client_error', safeErrorFields(error));
802
+ });
803
+ await postgres.connect();
804
+ const lockKey = 'ebk-bootstrap:' + database;
805
+ try {
806
+ retirementPhase = 'serialized-retirement';
807
+ await postgres.query(
808
+ 'SELECT pg_advisory_lock(hashtextextended($1, 0))',
809
+ [lockKey],
810
+ );
811
+ const existing = await postgres.query(
812
+ 'SELECT 1 FROM pg_database WHERE datname = ' + quoteLiteral(database),
813
+ );
814
+ if (existing.rowCount > 0) {
815
+ const grantees = [migratorRole, readwriteRole, readonlyRole]
816
+ .map(quoteIdentifier)
817
+ .join(', ');
818
+ await postgres.query(
819
+ 'REVOKE ALL ON DATABASE ' + quoteIdentifier(database) +
820
+ ' FROM ' + grantees,
821
+ );
822
+ await postgres.query(
823
+ 'DROP DATABASE ' + quoteIdentifier(database) + ' WITH (FORCE)',
824
+ );
825
+ }
826
+ const roles = await postgres.query(
827
+ 'SELECT rolname FROM pg_roles WHERE rolname IN (' +
828
+ [migratorRole, readwriteRole, readonlyRole]
829
+ .map(quoteLiteral)
830
+ .join(', ') +
831
+ ')',
832
+ );
833
+ const existingRoles = new Set(roles.rows.map(({ rolname }) => rolname));
834
+ if (
835
+ existingRoles.has(readwriteRole) &&
836
+ existingRoles.has(migratorRole)
837
+ ) {
838
+ await postgres.query(
839
+ 'REVOKE ' + quoteIdentifier(readwriteRole) + ' FROM ' +
840
+ quoteIdentifier(migratorRole),
841
+ );
842
+ }
843
+ for (const role of [migratorRole, readwriteRole, readonlyRole]) {
844
+ await postgres.query('DROP ROLE IF EXISTS ' + quoteIdentifier(role));
845
+ }
846
+ log('info', 'database_retirement_succeeded', { database });
847
+ } finally {
848
+ await postgres
849
+ .query('SELECT pg_advisory_unlock(hashtextextended($1, 0))', [lockKey])
850
+ .catch(() => undefined);
851
+ await postgres.end().catch(() => undefined);
852
+ }
853
+ }
854
+
855
+ main().catch((error) => {
856
+ log('error', 'database_retirement_failed', {
857
+ phase: retirementPhase,
858
+ ...safeErrorFields(error),
859
+ });
860
+ process.exitCode = 1;
861
+ });
862
+ `,
863
+ 'jobs/migrate-database.mjs': `import { createHash } from 'node:crypto';
864
+ import { readdir, readFile } from 'node:fs/promises';
865
+ import { Signer } from '@aws-sdk/rds-signer';
866
+ import pg from 'pg';
867
+
868
+ const { Client } = pg;
869
+ const required = (name) => {
870
+ const value = process.env[name]?.trim();
871
+ if (!value) throw new Error(name + ' is required.');
872
+ return value;
873
+ };
874
+ let migrationId = 'unresolved';
875
+ let migrationPhase = 'configuration';
876
+
877
+ const log = (level, event, fields = {}) => {
878
+ process.stdout.write(JSON.stringify({
879
+ event,
880
+ level,
881
+ migrationId,
882
+ timestamp: new Date().toISOString(),
883
+ ...fields,
884
+ }) + '\\n');
885
+ };
886
+ const safeErrorFields = (error) => {
887
+ const name = error instanceof Error ? error.name : 'UnknownError';
888
+ const code =
889
+ error &&
890
+ typeof error === 'object' &&
891
+ typeof error.code === 'string' &&
892
+ /^[A-Z0-9_]{2,32}$/.test(error.code)
893
+ ? error.code
894
+ : undefined;
895
+ return {
896
+ errorName: name.replace(/[^A-Za-z0-9_.-]/g, '').slice(0, 80) || 'Error',
897
+ ...(code ? { errorCode: code } : {}),
898
+ };
899
+ };
900
+ const retryableConnection = (error) => {
901
+ const code = error && typeof error === 'object' ? error.code : undefined;
902
+ const message = error instanceof Error ? error.message : String(error);
903
+ return new Set([
904
+ '08000',
905
+ '08001',
906
+ '08003',
907
+ '08006',
908
+ '57P03',
909
+ 'ECONNREFUSED',
910
+ 'ECONNRESET',
911
+ 'ETIMEDOUT',
912
+ ]).has(code) ||
913
+ /resum|starting up|temporarily unavailable|timeout|connection terminated unexpectedly|server closed the connection unexpectedly|terminating connection/i.test(message);
914
+ };
915
+ const sleep = (milliseconds) =>
916
+ new Promise((resolve) => setTimeout(resolve, milliseconds));
917
+
918
+ const sqlCodeOnly = (sql) => {
919
+ let index = 0;
920
+ let result = '';
921
+ const masked = (value) => value.replace(/[^\\r\\n]/g, ' ');
922
+ while (index < sql.length) {
923
+ if (sql.startsWith('--', index)) {
924
+ const end = sql.indexOf('\\n', index);
925
+ const next = end === -1 ? sql.length : end;
926
+ result += masked(sql.slice(index, next));
927
+ index = next;
928
+ continue;
929
+ }
930
+ if (sql.startsWith('/*', index)) {
931
+ const start = index;
932
+ let depth = 1;
933
+ index += 2;
934
+ while (index < sql.length && depth > 0) {
935
+ if (sql.startsWith('/*', index)) {
936
+ depth += 1;
937
+ index += 2;
938
+ } else if (sql.startsWith('*/', index)) {
939
+ depth -= 1;
940
+ index += 2;
941
+ } else {
942
+ index += 1;
943
+ }
944
+ }
945
+ result += masked(sql.slice(start, index));
946
+ continue;
947
+ }
948
+ const quote = sql[index];
949
+ if (quote === "'" || quote === '"') {
950
+ const start = index;
951
+ index += 1;
952
+ while (index < sql.length) {
953
+ if (sql[index] === quote) {
954
+ if (sql[index + 1] === quote) {
955
+ index += 2;
956
+ continue;
957
+ }
958
+ index += 1;
959
+ break;
960
+ }
961
+ if (quote === "'" && sql[index] === '\\\\' && index + 1 < sql.length) {
962
+ index += 2;
963
+ } else {
964
+ index += 1;
965
+ }
966
+ }
967
+ result += masked(sql.slice(start, index));
968
+ continue;
969
+ }
970
+ if (sql[index] === '$') {
971
+ const tag = sql
972
+ .slice(index)
973
+ .match(/^\\$(?:[A-Za-z_][A-Za-z0-9_]*)?\\$/)?.[0];
974
+ if (tag) {
975
+ const start = index;
976
+ const close = sql.indexOf(tag, index + tag.length);
977
+ index = close === -1 ? sql.length : close + tag.length;
978
+ result += masked(sql.slice(start, index));
979
+ continue;
980
+ }
981
+ }
982
+ result += sql[index];
983
+ index += 1;
984
+ }
985
+ return result;
986
+ };
987
+ const hasExecutableSql = (sql) => sqlCodeOnly(sql).trim().length > 0;
988
+ const unsupportedNonTransactionalSql =
989
+ /\\b(?:CREATE|DROP)\\s+INDEX\\s+CONCURRENTLY\\b|\\bVACUUM\\b/i;
990
+
991
+ async function main() {
992
+ const host = required('EBK_DATABASE_HOST');
993
+ const database = required('EBK_DATABASE_NAME');
994
+ const username = required('EBK_DATABASE_USERNAME');
995
+ const region = required('AWS_REGION');
996
+ migrationId = required('EBK_MIGRATION_ID');
997
+ const caBundlePath = required('EBK_RDS_CA_BUNDLE');
998
+ const port = Number(required('EBK_DATABASE_PORT'));
999
+ if (!Number.isSafeInteger(port) || port < 1 || port > 65535) {
1000
+ throw new Error('EBK_DATABASE_PORT must be a valid TCP port.');
1001
+ }
1002
+ migrationPhase = 'load_ca_bundle';
1003
+ const ca = await readFile(caBundlePath, 'utf8');
1004
+ const connectDeadline = Date.now() + 8 * 60 * 1000;
1005
+ let attempt = 0;
1006
+ let client;
1007
+ while (!client) {
1008
+ attempt += 1;
1009
+ let candidate;
1010
+ try {
1011
+ migrationPhase = 'connect';
1012
+ const password = await new Signer({
1013
+ hostname: host,
1014
+ port,
1015
+ region,
1016
+ username,
1017
+ }).getAuthToken();
1018
+ candidate = new Client({
1019
+ application_name: migrationId,
1020
+ connectionTimeoutMillis: 15_000,
1021
+ database,
1022
+ host,
1023
+ password,
1024
+ port,
1025
+ ssl: { ca, rejectUnauthorized: true },
1026
+ user: username,
1027
+ });
1028
+ await candidate.connect();
1029
+ client = candidate;
1030
+ } catch (error) {
1031
+ await candidate?.end().catch(() => undefined);
1032
+ if (!retryableConnection(error) || Date.now() >= connectDeadline) {
1033
+ throw error;
1034
+ }
1035
+ const delayMs = Math.min(attempt * 5_000, 30_000);
1036
+ log('warn', 'database_connection_retry', {
1037
+ attempt,
1038
+ delayMs,
1039
+ ...safeErrorFields(error),
1040
+ });
1041
+ await sleep(delayMs);
1042
+ }
1043
+ }
1044
+
1045
+ let clientFailure;
1046
+ client.on('error', (error) => {
1047
+ clientFailure ??= error;
1048
+ log('error', 'database_client_error', {
1049
+ phase: migrationPhase,
1050
+ ...safeErrorFields(error),
1051
+ });
1052
+ });
1053
+
1054
+ let locked = false;
1055
+ try {
1056
+ migrationPhase = 'lock';
1057
+ await client.query("SET lock_timeout = '5min'");
1058
+ await client.query(
1059
+ 'SELECT pg_advisory_lock(hashtextextended($1, 0))',
1060
+ [migrationId],
1061
+ );
1062
+ locked = true;
1063
+ migrationPhase = 'ledger';
1064
+ await client.query(
1065
+ 'CREATE TABLE IF NOT EXISTS _ebk_migrations (' +
1066
+ 'name text PRIMARY KEY, ' +
1067
+ 'checksum text NOT NULL, ' +
1068
+ 'applied_at timestamptz NOT NULL DEFAULT now())',
1069
+ );
1070
+ const directory = new URL('../database/migrations/', import.meta.url);
1071
+ const files = (await readdir(directory))
1072
+ .filter((file) => file.endsWith('.sql'))
1073
+ .sort();
1074
+ for (const name of files) {
1075
+ migrationPhase = 'migration:' + name;
1076
+ const sql = await readFile(new URL(name, directory), 'utf8');
1077
+ if (!hasExecutableSql(sql)) {
1078
+ throw new Error('Migration contains no executable SQL.');
1079
+ }
1080
+ if (unsupportedNonTransactionalSql.test(sqlCodeOnly(sql))) {
1081
+ throw new Error(
1082
+ 'Migration requires an unsupported non-transactional operation.',
1083
+ );
1084
+ }
1085
+ const checksum = createHash('sha256').update(sql).digest('hex');
1086
+ const existing = await client.query(
1087
+ 'SELECT checksum FROM _ebk_migrations WHERE name = $1',
1088
+ [name],
1089
+ );
1090
+ const appliedChecksum = existing.rows[0]?.checksum;
1091
+ if (appliedChecksum === checksum) {
1092
+ log('info', 'migration_skipped', { checksum, name });
1093
+ continue;
1094
+ }
1095
+ if (appliedChecksum) {
1096
+ throw new Error('An applied migration checksum changed.');
1097
+ }
1098
+
1099
+ await client.query('BEGIN');
1100
+ try {
1101
+ const statements = sql
1102
+ .split(/^--> statement-breakpoint\\s*$/m)
1103
+ .map((value) => value.trim())
1104
+ .filter(Boolean);
1105
+ for (const statement of statements) {
1106
+ await client.query(statement);
1107
+ }
1108
+ await client.query(
1109
+ 'INSERT INTO _ebk_migrations (name, checksum) VALUES ($1, $2)',
1110
+ [name, checksum],
1111
+ );
1112
+ await client.query('COMMIT');
1113
+ log('info', 'migration_applied', { checksum, name });
1114
+ } catch (error) {
1115
+ await client.query('ROLLBACK').catch((rollbackError) => {
1116
+ log('error', 'migration_rollback_failed', {
1117
+ phase: migrationPhase,
1118
+ ...safeErrorFields(rollbackError),
1119
+ });
1120
+ });
1121
+ throw error;
1122
+ }
1123
+ }
1124
+ if (clientFailure) throw clientFailure;
1125
+ log('info', 'migration_completed', { count: files.length });
1126
+ } finally {
1127
+ migrationPhase = 'cleanup';
1128
+ if (locked) {
1129
+ await client.query(
1130
+ 'SELECT pg_advisory_unlock(hashtextextended($1, 0))',
1131
+ [migrationId],
1132
+ ).catch(() => undefined);
1133
+ }
1134
+ await client.end().catch(() => undefined);
1135
+ }
1136
+ }
1137
+
1138
+ await main().catch((error) => {
1139
+ log('error', 'migration_failed', {
1140
+ phase: migrationPhase,
1141
+ ...safeErrorFields(error),
1142
+ });
1143
+ process.exitCode = 1;
1144
+ });
1145
+ `,
144
1146
  'contracts/events.ts': `export interface ${pascalName}CreatedData {
145
1147
  id: string;
146
1148
  occurredBy: string;
@@ -307,6 +1309,80 @@ export const handler = createHttpJsonHandler({
307
1309
  return { message: 'Authenticated request accepted.', principal };
308
1310
  },
309
1311
  });
1312
+ `,
1313
+ 'database/runtime-health.ts': `import {
1314
+ databaseTlsOptions,
1315
+ invalidateDatabaseCredential,
1316
+ resolveDatabaseCredential,
1317
+ type LinkedDatabaseResource,
1318
+ } from '@empire-builder-kit/runtime/database';
1319
+ import { createBetterAuthConfig } from '@empire-builder-kit/runtime/auth';
1320
+ import { createAuthGuard } from '@empire-builder-kit/runtime/auth-backend';
1321
+ import { createHttpJsonHandler } from '@empire-builder-kit/runtime/lambda';
1322
+ import pg from 'pg';
1323
+ import { Resource } from 'sst';
1324
+
1325
+ const { Client } = pg;
1326
+ let guard: ReturnType<typeof createAuthGuard> | undefined;
1327
+ function getGuard() {
1328
+ return guard ??= createAuthGuard({
1329
+ config: createBetterAuthConfig({
1330
+ clientId: process.env.AUTH_CLIENT_ID ?? '',
1331
+ region: process.env.AWS_REGION ?? '',
1332
+ userPoolId: process.env.AUTH_USER_POOL_ID ?? '',
1333
+ }),
1334
+ policy: { defaultKind: 'customer' },
1335
+ });
1336
+ }
1337
+ function linkedDatabase(): LinkedDatabaseResource {
1338
+ const resource = (Resource as unknown as Record<string, unknown>)
1339
+ .${pascalName}PrimaryDatabase;
1340
+ if (!resource || typeof resource !== 'object') {
1341
+ throw new Error('Primary database resource is not linked.');
1342
+ }
1343
+ return resource as LinkedDatabaseResource;
1344
+ }
1345
+ async function queryDatabase(resource: LinkedDatabaseResource) {
1346
+ for (let attempt = 1; attempt <= 2; attempt += 1) {
1347
+ const credential = await resolveDatabaseCredential(resource);
1348
+ const client = new Client({
1349
+ ...credential,
1350
+ connectionTimeoutMillis: 10_000,
1351
+ ssl: databaseTlsOptions(resource),
1352
+ });
1353
+ try {
1354
+ await client.connect();
1355
+ await client.query('SELECT 1');
1356
+ return;
1357
+ } catch (error) {
1358
+ const code =
1359
+ error && typeof error === 'object' && 'code' in error
1360
+ ? error.code
1361
+ : undefined;
1362
+ if (
1363
+ attempt === 1 &&
1364
+ resource.provider === 'aws-secrets-manager' &&
1365
+ code === '28P01'
1366
+ ) {
1367
+ invalidateDatabaseCredential(resource);
1368
+ continue;
1369
+ }
1370
+ throw error;
1371
+ } finally {
1372
+ await client.end().catch(() => undefined);
1373
+ }
1374
+ }
1375
+ }
1376
+
1377
+ export const handler = createHttpJsonHandler({
1378
+ app: '${name}',
1379
+ operation: 'database-health',
1380
+ async handle({ event }) {
1381
+ await getGuard().authorize(event);
1382
+ await queryDatabase(linkedDatabase());
1383
+ return { service: '${name}', status: 'ok' };
1384
+ },
1385
+ });
310
1386
  `,
311
1387
  'api/event-consumer.ts': `import { parseSqsEventBridgeRecords } from '@empire-builder-kit/runtime/events';
312
1388
  import type { ${pascalName}CreatedData } from '../contracts/events.js';
@@ -378,7 +1454,11 @@ export const handler = createHttpJsonHandler({
378
1454
  `,
379
1455
  'infra/product.ts': `import { provisionDatabaseBindings } from '@empire-builder-kit/nx/foundation';
380
1456
  import databaseBindings from '../database/bindings.json';
381
- import { getFoundationInfrastructure } from './foundation.js';
1457
+ import {
1458
+ getFoundationInfrastructure,
1459
+ registerProductMigration,
1460
+ registerProductRetirement,
1461
+ } from './foundation.js';
382
1462
  // <ebk:composition-imports:start>
383
1463
  // <ebk:composition-imports:end>
384
1464
 
@@ -392,10 +1472,12 @@ export function product() {
392
1472
  throw new Error('EBK_DEV_PORT must be an integer from 1024 through 65535.');
393
1473
  }
394
1474
  const foundation = getFoundationInfrastructure();
395
- const { db: database } = provisionDatabaseBindings(foundation, {
1475
+ const databaseProvisions = provisionDatabaseBindings(foundation, {
396
1476
  bindings: databaseBindings,
397
1477
  slice: '${name}',
398
- }).primary;
1478
+ });
1479
+ const databaseProvision = databaseProvisions.bindings.primary;
1480
+ const { db: database } = databaseProvision;
399
1481
  // <ebk:workload-registrations:start>
400
1482
  // <ebk:workload-registrations:end>
401
1483
 
@@ -427,6 +1509,17 @@ export function product() {
427
1509
  link: [database],
428
1510
  logging: { format: 'json', retention: '1 month' },
429
1511
  });
1512
+ api.route('GET /database-health', {
1513
+ environment: {
1514
+ AUTH_CLIENT_ID: authClient.id,
1515
+ AUTH_USER_POOL_ID: foundation.userPool.id,
1516
+ EBK_STAGE: $app.stage,
1517
+ },
1518
+ handler: 'database/runtime-health.handler',
1519
+ link: [database],
1520
+ logging: { format: 'json', retention: '1 month' },
1521
+ vpc: foundation.vpc,
1522
+ });
430
1523
  api.route('POST /events/example', {
431
1524
  environment: {
432
1525
  AUTH_CLIENT_ID: authClient.id,
@@ -490,6 +1583,259 @@ export function product() {
490
1583
  vpc: foundation.vpc,
491
1584
  });
492
1585
 
1586
+ const migrationControls: Record<string, any> = {};
1587
+ if (!$dev) {
1588
+ const ephemeralStage = !new Set([
1589
+ 'development',
1590
+ 'staging',
1591
+ 'production',
1592
+ ]).has($app.stage);
1593
+ const accountId = aws.getCallerIdentityOutput({}).accountId;
1594
+ const partition = aws.getPartitionOutput({}).partition;
1595
+ const region = aws.getRegionOutput({}).name;
1596
+ const runTask = (name: string, task: any) =>
1597
+ sst.aws.StepFunctions.task({
1598
+ arguments: {
1599
+ Cluster: task.cluster,
1600
+ LaunchType: 'FARGATE',
1601
+ NetworkConfiguration: {
1602
+ AwsvpcConfiguration: {
1603
+ AssignPublicIp: task.assignPublicIp.apply(
1604
+ (value: boolean) => value ? 'ENABLED' : 'DISABLED',
1605
+ ),
1606
+ SecurityGroups: task.securityGroups,
1607
+ Subnets: task.subnets,
1608
+ },
1609
+ },
1610
+ TaskDefinition: task.taskDefinition,
1611
+ },
1612
+ integration: 'sync',
1613
+ name,
1614
+ permissions: [
1615
+ {
1616
+ actions: ['ecs:RunTask'],
1617
+ resources: [task.nodes.taskDefinition.arn],
1618
+ },
1619
+ {
1620
+ actions: ['iam:PassRole'],
1621
+ resources: [
1622
+ task.nodes.executionRole.arn,
1623
+ task.nodes.taskRole.arn,
1624
+ ],
1625
+ },
1626
+ {
1627
+ actions: ['ecs:DescribeTasks', 'ecs:StopTask'],
1628
+ resources: ['*'],
1629
+ },
1630
+ {
1631
+ actions: [
1632
+ 'events:DescribeRule',
1633
+ 'events:PutRule',
1634
+ 'events:PutTargets',
1635
+ ],
1636
+ resources: [
1637
+ $interpolate\`arn:\${partition}:events:\${region}:\${accountId}:rule/StepFunctionsGetEventsForECSTaskRule\`,
1638
+ ],
1639
+ },
1640
+ ],
1641
+ resource: 'arn:aws:states:::ecs:runTask',
1642
+ timeout: '45 minutes',
1643
+ });
1644
+
1645
+ for (const [databaseIdentifier, target] of Object.entries(
1646
+ databaseProvisions.databases,
1647
+ )) {
1648
+ if (!target.readwriteSecretArn || !target.readonlySecretArn) {
1649
+ throw new Error(
1650
+ 'Deployed database target ' + databaseIdentifier +
1651
+ ' is missing its generated credential references.',
1652
+ );
1653
+ }
1654
+ const databaseUserArn =
1655
+ $interpolate\`arn:\${partition}:rds-db:\${region}:\${accountId}:dbuser:\${foundation.database.clusterResourceId}/\${target.migratorRole}\`;
1656
+ const bootstrapTask = new sst.aws.Task(
1657
+ target.resourceName + 'BootstrapTask',
1658
+ {
1659
+ cluster: foundation.cluster,
1660
+ command: ['node', 'jobs/bootstrap-database.mjs'],
1661
+ environment: {
1662
+ AWS_REGION: region,
1663
+ EBK_BOOTSTRAP_ID:
1664
+ $app.name + ':' + $app.stage + ':' + databaseIdentifier,
1665
+ EBK_DATABASE_HOST: foundation.database.clusterEndpoint,
1666
+ EBK_DATABASE_MIGRATOR_ROLE: target.migratorRole,
1667
+ EBK_DATABASE_NAME: target.databaseName,
1668
+ EBK_DATABASE_PORT:
1669
+ $interpolate\`\${foundation.database.port}\`,
1670
+ EBK_DATABASE_READONLY_ROLE: target.readonlyRole,
1671
+ EBK_DATABASE_READWRITE_ROLE: target.readwriteRole,
1672
+ EBK_DATABASE_ROTATION_TOKEN: target.rotationToken ?? '',
1673
+ EBK_MASTER_SECRET_ARN: foundation.database.masterSecretArn,
1674
+ EBK_READONLY_SECRET_ARN: target.readonlySecretArn,
1675
+ EBK_READWRITE_SECRET_ARN: target.readwriteSecretArn,
1676
+ EBK_RDS_CA_BUNDLE: '/app/rds-global-bundle.pem',
1677
+ EBK_RUNTIME_DATABASE_HOST: foundation.database.host,
1678
+ },
1679
+ image: {
1680
+ context: '.',
1681
+ dockerfile: 'migration/Dockerfile',
1682
+ },
1683
+ logging: {
1684
+ name:
1685
+ '/ebk/' + $app.stage + '/${name}/database-bootstrap/' +
1686
+ databaseIdentifier,
1687
+ retention: '1 month',
1688
+ },
1689
+ permissions: [
1690
+ {
1691
+ actions: ['secretsmanager:GetSecretValue'],
1692
+ resources: [
1693
+ foundation.database.masterSecretArn,
1694
+ target.readonlySecretArn,
1695
+ target.readwriteSecretArn,
1696
+ ],
1697
+ },
1698
+ {
1699
+ actions: ['secretsmanager:PutSecretValue'],
1700
+ resources: [
1701
+ target.readonlySecretArn,
1702
+ target.readwriteSecretArn,
1703
+ ],
1704
+ },
1705
+ {
1706
+ actions: ['rds-db:connect'],
1707
+ resources: [databaseUserArn],
1708
+ },
1709
+ ],
1710
+ },
1711
+ );
1712
+ const migrationTask = new sst.aws.Task(
1713
+ target.resourceName + 'MigrationTask',
1714
+ {
1715
+ cluster: foundation.cluster,
1716
+ environment: {
1717
+ AWS_REGION: region,
1718
+ EBK_DATABASE_HOST: foundation.database.clusterEndpoint,
1719
+ EBK_DATABASE_NAME: target.databaseName,
1720
+ EBK_DATABASE_PORT:
1721
+ $interpolate\`\${foundation.database.port}\`,
1722
+ EBK_DATABASE_USERNAME: target.migratorRole,
1723
+ EBK_MIGRATION_ID:
1724
+ $app.name + ':' + $app.stage + ':' + databaseIdentifier,
1725
+ EBK_RDS_CA_BUNDLE: '/app/rds-global-bundle.pem',
1726
+ },
1727
+ image: {
1728
+ context: '.',
1729
+ dockerfile: 'migration/Dockerfile',
1730
+ },
1731
+ logging: {
1732
+ name:
1733
+ '/ebk/' + $app.stage + '/${name}/database-migration/' +
1734
+ databaseIdentifier,
1735
+ retention: '1 month',
1736
+ },
1737
+ permissions: [
1738
+ {
1739
+ actions: ['rds-db:connect'],
1740
+ resources: [databaseUserArn],
1741
+ },
1742
+ ],
1743
+ },
1744
+ );
1745
+ const retirementTask = ephemeralStage
1746
+ ? new sst.aws.Task(target.resourceName + 'RetirementTask', {
1747
+ cluster: foundation.cluster,
1748
+ command: ['node', 'jobs/retire-database.mjs'],
1749
+ environment: {
1750
+ EBK_DATABASE_HOST: foundation.database.clusterEndpoint,
1751
+ EBK_DATABASE_MIGRATOR_ROLE: target.migratorRole,
1752
+ EBK_DATABASE_NAME: target.databaseName,
1753
+ EBK_DATABASE_PORT:
1754
+ $interpolate\`\${foundation.database.port}\`,
1755
+ EBK_DATABASE_READONLY_ROLE: target.readonlyRole,
1756
+ EBK_DATABASE_READWRITE_ROLE: target.readwriteRole,
1757
+ EBK_MASTER_SECRET_ARN: foundation.database.masterSecretArn,
1758
+ EBK_RDS_CA_BUNDLE: '/app/rds-global-bundle.pem',
1759
+ EBK_RETIREMENT_ID:
1760
+ $app.name + ':' + $app.stage + ':' + databaseIdentifier,
1761
+ },
1762
+ image: {
1763
+ context: '.',
1764
+ dockerfile: 'migration/Dockerfile',
1765
+ },
1766
+ logging: {
1767
+ name:
1768
+ '/ebk/' + $app.stage + '/${name}/database-retirement/' +
1769
+ databaseIdentifier,
1770
+ retention: '1 month',
1771
+ },
1772
+ permissions: [
1773
+ {
1774
+ actions: ['secretsmanager:GetSecretValue'],
1775
+ resources: [foundation.database.masterSecretArn],
1776
+ },
1777
+ ],
1778
+ })
1779
+ : undefined;
1780
+ const bootstrap = runTask('BootstrapDatabase', bootstrapTask);
1781
+ const migrate = runTask('RunDatabaseMigration', migrationTask);
1782
+ const succeeded = sst.aws.StepFunctions.succeed({
1783
+ name: 'DatabaseMigrationSucceeded',
1784
+ });
1785
+ const stateMachine = new sst.aws.StepFunctions(
1786
+ target.resourceName + 'Migration',
1787
+ {
1788
+ definition: bootstrap.next(migrate).next(succeeded),
1789
+ logging: {
1790
+ includeData: false,
1791
+ level: 'error',
1792
+ retention: '1 month',
1793
+ },
1794
+ },
1795
+ );
1796
+ const registration = registerProductMigration(
1797
+ target.resourceName + 'MigrationRegistration',
1798
+ databaseIdentifier,
1799
+ stateMachine.arn,
1800
+ );
1801
+ let retirementControl;
1802
+ if (retirementTask) {
1803
+ const retirement = runTask('RetireDatabase', retirementTask);
1804
+ const retirementSucceeded = sst.aws.StepFunctions.succeed({
1805
+ name: 'DatabaseRetirementSucceeded',
1806
+ });
1807
+ const retirementStateMachine = new sst.aws.StepFunctions(
1808
+ target.resourceName + 'MigrationRetirement',
1809
+ {
1810
+ definition: retirement.next(retirementSucceeded),
1811
+ logging: {
1812
+ includeData: false,
1813
+ level: 'error',
1814
+ retention: '1 month',
1815
+ },
1816
+ },
1817
+ );
1818
+ const retirementRegistration = registerProductRetirement(
1819
+ target.resourceName + 'RetirementRegistration',
1820
+ databaseIdentifier,
1821
+ retirementStateMachine.arn,
1822
+ );
1823
+ retirementControl = {
1824
+ retirementRegistration,
1825
+ retirementStateMachine,
1826
+ retirementTask,
1827
+ };
1828
+ }
1829
+ migrationControls[databaseIdentifier] = {
1830
+ bootstrapTask,
1831
+ migrationTask,
1832
+ registration,
1833
+ ...(retirementControl ?? {}),
1834
+ stateMachine,
1835
+ };
1836
+ }
1837
+ }
1838
+
493
1839
  const apiAlarm = new aws.cloudwatch.MetricAlarm('${pascalName}ApiErrors', {
494
1840
  alarmActions: [foundation.discovery.alertTopicArn],
495
1841
  comparisonOperator: 'GreaterThanOrEqualToThreshold',
@@ -532,6 +1878,7 @@ export function product() {
532
1878
  dlqAlarm,
533
1879
  eventDlq,
534
1880
  eventQueue,
1881
+ migrationControls,
535
1882
  apiAlarm,
536
1883
  web,
537
1884
  // <ebk:composition-outputs:start>
@@ -842,10 +2189,12 @@ signal, callback/logout URLs, and allocated internal port; do not persist a deve
842
2189
  this project.
843
2190
 
844
2191
  \`database/bindings.json\` is the adopter-owned database selection contract. Add a candidate
845
- identifier to \`primary.provisioned\` to provision it without moving workload traffic. After the
846
- candidate is migrated and verified through the database-replacement runbook, change only
847
- \`primary.active\` to select it on the next reviewed deployment. Do not edit managed
848
- \`infra/product.ts\` or provision databases with Foundation master credentials.
2192
+ identifier to \`primary.provisioned\` to provision it without moving workload traffic. Every
2193
+ provisioned identifier receives its own AWS-internal bootstrap-and-migration workflow. Run
2194
+ \`pnpm nx run ${name}:db-migrate --stage=<stage> --database=<identifier>\` to migrate and verify
2195
+ a non-active candidate before switching \`primary.active\`. Without \`--database\`, the target
2196
+ selects the active \`primary\` identifier. Do not edit managed \`infra/product.ts\` or provision
2197
+ databases with Foundation master credentials.
849
2198
 
850
2199
  The readiness route is framework-managed for local orchestration. Put product-specific
851
2200
  dependency and status checks under \`/${name}/api/status\` rather than editing that route.
@@ -853,6 +2202,29 @@ dependency and status checks under \`/${name}/api/status\` rather than editing t
853
2202
  Run \`pnpm nx run ${name}:db-local-down\` when finished. The named volume preserves local data
854
2203
  between sessions.
855
2204
 
2205
+ For a deployed canonical stage, \`${name}:db-migrate\` does not connect from the
2206
+ local or GitHub process. It starts the generated Standard Step Functions
2207
+ workflow, waits for its one-off Fargate task, and reports the execution ARN.
2208
+ The workflow first runs an AWS-internal bootstrap task that converges the
2209
+ database, roles, and reference-only runtime credentials, then runs the
2210
+ IAM-authenticated migration task. Neither password resolves in GitHub, SST, or
2211
+ Pulumi state. The authenticated \`GET /database-health\` API route uses the
2212
+ server-only resolver, linked secret, and runtime-embedded AWS RDS global CA
2213
+ bundle to prove the deployed runtime path with hostname-verified TLS.
2214
+ Staging and production use the existing environment-scoped
2215
+ \`AWS_DEPLOY_ROLE_ARN\`; no second migration OIDC role or database password is
2216
+ configured by the adopter. That role needs only \`ssm:GetParameter\` for the
2217
+ exact generated migration-registration path and \`states:StartExecution\` /
2218
+ \`states:DescribeExecution\` for the exact generated state machine and its
2219
+ execution namespace; it never receives SQL, database-token, or arbitrary ECS
2220
+ authority.
2221
+
2222
+ The Product Slice \`remove\` target depends on \`db-retire\`. Canonical
2223
+ development, staging, and production retain logical data. A deployed
2224
+ preview/personal stage runs its AWS-internal retirement workflow for every
2225
+ declared database before SST removal; do not bypass this with direct
2226
+ \`sst remove\`.
2227
+
856
2228
  Use \`pnpm nx run ${name}:dev --stage=<personal-name>\` for a personal live relay. The target
857
2229
  maps that consumer stage to the \`development\` Foundation and reports both stages during
858
2230
  preflight. It never deploys a personal Foundation. A normal full deploy at a mapped personal
@@ -885,6 +2257,50 @@ Record consumed slice contracts here before adding a source dependency.
885
2257
 
886
2258
  Document deployment smoke checks, alarms, rollback, event redrive, database migration, and
887
2259
  Service or Job recovery procedures as those capabilities are added.
2260
+
2261
+ ## Database migration
2262
+
2263
+ Protected migration runs from generated immutable bootstrap and migration
2264
+ Fargate tasks through one Standard Step Functions workflow per provisioned
2265
+ database identifier. The bootstrap task owns exact master-secret read,
2266
+ runtime-secret read/write, and migrator-connect permissions; it converges the
2267
+ database and roles under a database-scoped advisory lock. PostgreSQL may retain
2268
+ a creator ADMIN membership for role administration; bootstrap removes its
2269
+ \`SET\` and \`INHERIT\` options before granting \`rds_iam\`. The migration task
2270
+ receives only exact migrator-connect permission.
2271
+
2272
+ Retain the workflow execution ARN, both task identities, image identity, source
2273
+ commit, migration checksums, sanitized CloudWatch logs, backup reference, and
2274
+ verification result. The GitHub job reuses
2275
+ \`AWS_DEPLOY_ROLE_ARN\` only to start and describe the exact workflow; it must
2276
+ not have direct SQL, secret-value, \`rds-db:connect\`, or arbitrary
2277
+ \`ecs:RunTask\` authority.
2278
+
2279
+ The default \`db-migrate\` target selects the active \`primary\` database.
2280
+ During replacement, first add the candidate to \`primary.provisioned\`, deploy
2281
+ infrastructure, then run \`db-migrate --database=<candidate>\` and complete
2282
+ candidate verification before changing \`primary.active\`.
2283
+
2284
+ Preview/personal removal invokes the generated retirement state machine for
2285
+ every provisioned identifier before infrastructure removal. Canonical stages
2286
+ retain their logical data. A preview created by a pre-bootstrap candidate must
2287
+ be removed before upgrading the Foundation and disabling its legacy Data API.
2288
+ If an interrupted current-candidate deploy never registered the retirement
2289
+ state machine in SSM, rerun the same Slice infrastructure deployment to restore
2290
+ the registration, then retry \`remove\`; do not bypass retirement.
2291
+
2292
+ The authenticated \`GET /database-health\` route uses the server-only runtime
2293
+ resolver and its checksum-pinned AWS RDS global CA bundle. Remote Aurora
2294
+ connections retain hostname verification; only local loopback database
2295
+ connections disable TLS.
2296
+
2297
+ Migrations are lexically ordered, checksum-locked, transaction-scoped, and
2298
+ serialized with a Postgres advisory lock. v1 schema migrations are restricted
2299
+ to the \`public\` schema. Known non-transactional operations fail closed; use a
2300
+ reviewed replacement or forward-compatible design instead of bypassing the
2301
+ transaction. Apply expand/migrate/contract. Repair failures with a forward
2302
+ migration or the documented restore/replacement procedure; do not edit an
2303
+ applied SQL file.
888
2304
  `,
889
2305
  'ebk-globals.d.ts': `declare const sst: any;
890
2306
  declare const aws: any;
@@ -899,7 +2315,9 @@ declare const $util: any;
899
2315
  `,
900
2316
  'infra/foundation.ts': `import {
901
2317
  foundationSsmPath,
2318
+ productSliceMigrationPath,
902
2319
  productSliceRegistrationPath,
2320
+ productSliceRetirementPath,
903
2321
  resolveFoundationStages,
904
2322
  type FoundationDiscovery,
905
2323
  } from '@empire-builder-kit/nx/foundation';
@@ -924,8 +2342,14 @@ export function getFoundation(): FoundationDiscovery {
924
2342
  clusterId: foundationParameter('clusterId'),
925
2343
  contractVersion: foundationParameter('contractVersion'),
926
2344
  databaseClusterArn: foundationParameter('databaseClusterArn'),
2345
+ databaseClusterEndpoint: foundationParameter('databaseClusterEndpoint'),
927
2346
  databaseClusterId: foundationParameter('databaseClusterId'),
2347
+ databaseClusterResourceId: foundationParameter(
2348
+ 'databaseClusterResourceId',
2349
+ ),
2350
+ databaseHost: foundationParameter('databaseHost'),
928
2351
  databaseMasterSecretArn: foundationParameter('databaseMasterSecretArn'),
2352
+ databasePort: foundationParameter('databasePort').apply(Number),
929
2353
  eventBusArn: foundationParameter('eventBusArn'),
930
2354
  eventBusName: foundationParameter('eventBusName'),
931
2355
  hasPublicDomain: foundationParameter('hasPublicDomain').apply(
@@ -962,17 +2386,15 @@ export function getFoundationInfrastructure() {
962
2386
  'FoundationEventBus',
963
2387
  discovery.eventBusName,
964
2388
  );
965
- const database = sst.aws.Aurora.get(
966
- 'FoundationDatabase',
967
- discovery.databaseClusterId,
968
- );
969
-
970
2389
  return {
971
2390
  cluster,
972
2391
  database: {
973
- cluster: database,
974
2392
  clusterArn: discovery.databaseClusterArn,
2393
+ clusterEndpoint: discovery.databaseClusterEndpoint,
2394
+ clusterResourceId: discovery.databaseClusterResourceId,
2395
+ host: discovery.databaseHost,
975
2396
  masterSecretArn: discovery.databaseMasterSecretArn,
2397
+ port: discovery.databasePort,
976
2398
  },
977
2399
  discovery,
978
2400
  eventBus,
@@ -999,6 +2421,40 @@ export function registerProductSlice() {
999
2421
  }),
1000
2422
  });
1001
2423
  }
2424
+
2425
+ export function registerProductMigration(
2426
+ resourceName: string,
2427
+ database: string,
2428
+ stateMachineArn: any,
2429
+ ) {
2430
+ return new aws.ssm.Parameter(resourceName, {
2431
+ name: productSliceMigrationPath(
2432
+ stages.foundationStage,
2433
+ stages.consumerStage,
2434
+ '${name}',
2435
+ database,
2436
+ ),
2437
+ type: 'String',
2438
+ value: stateMachineArn,
2439
+ });
2440
+ }
2441
+
2442
+ export function registerProductRetirement(
2443
+ name: string,
2444
+ database: string,
2445
+ stateMachineArn: any,
2446
+ ) {
2447
+ return new aws.ssm.Parameter(name, {
2448
+ name: productSliceRetirementPath(
2449
+ stages.foundationStage,
2450
+ stages.consumerStage,
2451
+ '${name}',
2452
+ database,
2453
+ ),
2454
+ type: 'String',
2455
+ value: stateMachineArn,
2456
+ });
2457
+ }
1002
2458
  `,
1003
2459
  'infra/index.ts': `import { getFoundation, registerProductSlice } from './foundation.js';
1004
2460
  import { product } from './product.js';
@@ -1025,22 +2481,23 @@ export function app() {
1025
2481
  dependencies: {
1026
2482
  '@aws-sdk/client-cognito-identity-provider': PRODUCT_VERSIONS.awsCognito,
1027
2483
  '@aws-sdk/client-eventbridge': PRODUCT_VERSIONS.awsEventBridge,
1028
- '@aws-sdk/client-rds-data': PRODUCT_VERSIONS.awsRdsData,
1029
- '@aws-sdk/client-secrets-manager': PRODUCT_VERSIONS.awsSecretsManager,
2484
+ '@aws-sdk/client-sfn': PRODUCT_VERSIONS.awsSfn,
1030
2485
  '@aws-sdk/client-ssm': PRODUCT_VERSIONS.awsSsm,
2486
+ '@aws-sdk/rds-signer': PRODUCT_VERSIONS.awsRdsSigner,
1031
2487
  '@empire-builder-kit/runtime': runtimeVersion,
1032
2488
  'drizzle-orm': PRODUCT_VERSIONS.drizzle,
1033
2489
  next: PRODUCT_VERSIONS.next,
2490
+ pg: PRODUCT_VERSIONS.pg,
1034
2491
  react: PRODUCT_VERSIONS.react,
1035
2492
  'react-dom': PRODUCT_VERSIONS.reactDom,
1036
2493
  },
1037
2494
  devDependencies: {
1038
2495
  '@playwright/test': PRODUCT_VERSIONS.playwright,
1039
2496
  '@types/node': PRODUCT_VERSIONS.typesNode,
2497
+ '@types/pg': PRODUCT_VERSIONS.typesPg,
1040
2498
  '@types/react': PRODUCT_VERSIONS.typesReact,
1041
2499
  '@types/react-dom': PRODUCT_VERSIONS.typesReactDom,
1042
2500
  'drizzle-kit': PRODUCT_VERSIONS.drizzleKit,
1043
- pg: PRODUCT_VERSIONS.pg,
1044
2501
  vitest: PRODUCT_VERSIONS.vitest,
1045
2502
  },
1046
2503
  exports: {
@@ -1123,6 +2580,14 @@ export function app() {
1123
2580
  cwd: `packages/${name}`,
1124
2581
  },
1125
2582
  },
2583
+ 'db-retire': {
2584
+ executor: 'nx:run-commands',
2585
+ options: {
2586
+ command: 'node scripts/retire-databases.mjs',
2587
+ cwd: `packages/${name}`,
2588
+ forwardAllArgs: true,
2589
+ },
2590
+ },
1126
2591
  deploy: {
1127
2592
  executor: 'nx:run-commands',
1128
2593
  dependsOn: [{ target: 'db-migrate', params: 'forward' }],
@@ -1203,6 +2668,7 @@ export function app() {
1203
2668
  },
1204
2669
  remove: {
1205
2670
  executor: '@empire-builder-kit/nx:remove',
2671
+ dependsOn: [{ target: 'db-retire', params: 'forward' }],
1206
2672
  options: {
1207
2673
  cwd: `packages/${name}`,
1208
2674
  stage: 'development',
@@ -1338,122 +2804,366 @@ const result = spawnSync('docker', args, {
1338
2804
  });
1339
2805
  if (result.error) throw result.error;
1340
2806
  if (result.status !== 0) process.exit(result.status ?? 1);
2807
+ `,
2808
+ 'scripts/retire-databases.mjs': `import { createHash } from 'node:crypto';
2809
+ import {
2810
+ DescribeExecutionCommand,
2811
+ SFNClient,
2812
+ StartExecutionCommand,
2813
+ } from '@aws-sdk/client-sfn';
2814
+ import { GetParameterCommand, SSMClient } from '@aws-sdk/client-ssm';
2815
+ import {
2816
+ createDatabaseBindingPlan,
2817
+ productSliceRetirementPath,
2818
+ resolveFoundationStages,
2819
+ } from '@empire-builder-kit/nx/foundation';
2820
+ import databaseBindings from '../database/bindings.json' with { type: 'json' };
2821
+
2822
+ const stageArg = process.argv.find((value) => value.startsWith('--stage='));
2823
+ const foundationStageArg = process.argv.find((value) =>
2824
+ value.startsWith('--foundationStage='),
2825
+ );
2826
+ const { consumerStage: stage, foundationStage } = resolveFoundationStages({
2827
+ consumerStage: stageArg?.slice('--stage='.length) ?? 'development',
2828
+ foundationStage:
2829
+ foundationStageArg?.slice('--foundationStage='.length) ??
2830
+ process.env.EBK_FOUNDATION_STAGE,
2831
+ });
2832
+ if (new Set(['development', 'staging', 'production']).has(stage)) {
2833
+ process.stdout.write(
2834
+ '[database-retirement] canonical stage ' + stage +
2835
+ ' retains logical databases and roles.\\n',
2836
+ );
2837
+ process.exit(0);
2838
+ }
2839
+
2840
+ const bindingPlan = createDatabaseBindingPlan(databaseBindings, {
2841
+ slice: '${name}',
2842
+ stage,
2843
+ });
2844
+ const sourceCommit = (
2845
+ process.env.GITHUB_SHA ??
2846
+ process.env.EBK_SOURCE_COMMIT ??
2847
+ 'local'
2848
+ ).replace(/[^A-Za-z0-9_-]/g, '-');
2849
+ const runIdentity = (
2850
+ process.env.GITHUB_RUN_ID ??
2851
+ process.env.EBK_RETIREMENT_RUN_ID ??
2852
+ String(Date.now())
2853
+ ).replace(/[^A-Za-z0-9_-]/g, '-');
2854
+ const runAttempt = (
2855
+ process.env.GITHUB_RUN_ATTEMPT ??
2856
+ '1'
2857
+ ).replace(/[^A-Za-z0-9_-]/g, '-');
2858
+ const sfn = new SFNClient({});
2859
+ const ssm = new SSMClient({});
2860
+ const terminal = new Set(['ABORTED', 'FAILED', 'SUCCEEDED', 'TIMED_OUT']);
2861
+
2862
+ for (const database of bindingPlan.provisioned) {
2863
+ const parameterName = productSliceRetirementPath(
2864
+ foundationStage,
2865
+ stage,
2866
+ '${name}',
2867
+ database,
2868
+ );
2869
+ const missingRegistration = () =>
2870
+ new Error(
2871
+ 'Database retirement state machine ARN is missing at ' +
2872
+ parameterName +
2873
+ '. Redeploy the Slice infrastructure with its adopted framework candidate to restore retirement registration, then retry removal. A pre-bootstrap preview must instead be removed with its old candidate before Foundation cutover.',
2874
+ );
2875
+ let parameter;
2876
+ try {
2877
+ parameter = await ssm.send(
2878
+ new GetParameterCommand({ Name: parameterName }),
2879
+ );
2880
+ } catch (error) {
2881
+ if (
2882
+ error &&
2883
+ typeof error === 'object' &&
2884
+ error.name === 'ParameterNotFound'
2885
+ ) {
2886
+ throw missingRegistration();
2887
+ }
2888
+ throw error;
2889
+ }
2890
+ const stateMachineArn = parameter.Parameter?.Value;
2891
+ if (!stateMachineArn) {
2892
+ throw missingRegistration();
2893
+ }
2894
+ const identity = createHash('sha256')
2895
+ .update(['${name}', stage, database, runIdentity, runAttempt, sourceCommit].join('\\0'))
2896
+ .digest('hex')
2897
+ .slice(0, 16);
2898
+ const executionName = [
2899
+ 'ebk-retire',
2900
+ '${name}'.slice(0, 16),
2901
+ database.slice(0, 16),
2902
+ identity,
2903
+ ].join('-').slice(0, 80);
2904
+ const executionArn =
2905
+ stateMachineArn.replace(':stateMachine:', ':execution:') +
2906
+ ':' +
2907
+ executionName;
2908
+ let execution;
2909
+ try {
2910
+ execution = await sfn.send(
2911
+ new DescribeExecutionCommand({ executionArn }),
2912
+ );
2913
+ } catch (error) {
2914
+ if (
2915
+ !error ||
2916
+ typeof error !== 'object' ||
2917
+ error.name !== 'ExecutionDoesNotExist'
2918
+ ) {
2919
+ throw error;
2920
+ }
2921
+ }
2922
+ if (!execution) {
2923
+ try {
2924
+ await sfn.send(
2925
+ new StartExecutionCommand({
2926
+ input: JSON.stringify({
2927
+ databaseIdentifier: database,
2928
+ sourceCommit,
2929
+ stage,
2930
+ slice: '${name}',
2931
+ workflowRunAttempt: runAttempt,
2932
+ workflowRunId: runIdentity,
2933
+ }),
2934
+ name: executionName,
2935
+ stateMachineArn,
2936
+ }),
2937
+ );
2938
+ } catch (error) {
2939
+ if (
2940
+ !error ||
2941
+ typeof error !== 'object' ||
2942
+ error.name !== 'ExecutionAlreadyExists'
2943
+ ) {
2944
+ throw error;
2945
+ }
2946
+ }
2947
+ }
2948
+ let status = execution?.status ?? 'RUNNING';
2949
+ const deadline = Date.now() + 50 * 60 * 1000;
2950
+ while (!terminal.has(status)) {
2951
+ if (Date.now() >= deadline) {
2952
+ throw new Error(
2953
+ 'Timed out waiting for database retirement execution ' + executionArn,
2954
+ );
2955
+ }
2956
+ await new Promise((resolve) => setTimeout(resolve, 5_000));
2957
+ execution = await sfn.send(
2958
+ new DescribeExecutionCommand({ executionArn }),
2959
+ );
2960
+ status = execution.status ?? 'UNKNOWN';
2961
+ }
2962
+ if (status !== 'SUCCEEDED') {
2963
+ throw new Error(
2964
+ 'Database retirement execution ' + executionArn +
2965
+ ' ended with status ' + status + '. Infrastructure removal is blocked.',
2966
+ );
2967
+ }
2968
+ process.stdout.write(
2969
+ '[database-retirement] database=' + database +
2970
+ ' execution=' + executionArn + ' status=SUCCEEDED\\n',
2971
+ );
2972
+ }
1341
2973
  `,
1342
2974
  'scripts/confirm-deploy.mjs': `const stageArg = process.argv.find((value) => value.startsWith('--stage='));
1343
2975
  const stage = stageArg?.slice('--stage='.length) ?? 'development';
1344
2976
  process.stdout.write('${name} infrastructure and database migrations completed for ' + stage + '.\\n');
1345
2977
  `,
1346
2978
  'scripts/migrate-database.mjs': `import { createHash } from 'node:crypto';
1347
- import { readdir, readFile } from 'node:fs/promises';
1348
- import { createDatabaseBindingPlan, resolveFoundationStages } from '@empire-builder-kit/nx/foundation';
1349
2979
  import {
1350
- BeginTransactionCommand,
1351
- CommitTransactionCommand,
1352
- ExecuteStatementCommand,
1353
- RDSDataClient,
1354
- RollbackTransactionCommand,
1355
- } from '@aws-sdk/client-rds-data';
1356
- import { DescribeSecretCommand, SecretsManagerClient } from '@aws-sdk/client-secrets-manager';
2980
+ DescribeExecutionCommand,
2981
+ SFNClient,
2982
+ StartExecutionCommand,
2983
+ } from '@aws-sdk/client-sfn';
1357
2984
  import { GetParameterCommand, SSMClient } from '@aws-sdk/client-ssm';
2985
+ import {
2986
+ createDatabaseBindingPlan,
2987
+ productSliceMigrationPath,
2988
+ resolveFoundationStages,
2989
+ selectDatabaseTarget,
2990
+ } from '@empire-builder-kit/nx/foundation';
1358
2991
  import databaseBindings from '../database/bindings.json' with { type: 'json' };
1359
2992
 
1360
2993
  const stageArg = process.argv.find((value) => value.startsWith('--stage='));
1361
- const foundationStageArg = process.argv.find((value) => value.startsWith('--foundationStage='));
2994
+ const foundationStageArg = process.argv.find((value) =>
2995
+ value.startsWith('--foundationStage='),
2996
+ );
1362
2997
  const { consumerStage: stage, foundationStage } = resolveFoundationStages({
1363
2998
  consumerStage: stageArg?.slice('--stage='.length) ?? 'development',
1364
- foundationStage: foundationStageArg?.slice('--foundationStage='.length) ?? process.env.EBK_FOUNDATION_STAGE,
2999
+ foundationStage:
3000
+ foundationStageArg?.slice('--foundationStage='.length) ??
3001
+ process.env.EBK_FOUNDATION_STAGE,
1365
3002
  });
1366
3003
  const bindingPlan = createDatabaseBindingPlan(databaseBindings, {
1367
3004
  slice: '${name}',
1368
3005
  stage,
1369
3006
  });
1370
- const activeDatabase = bindingPlan.active.primary;
1371
- if (!activeDatabase) throw new Error('Database binding "primary" is required.');
1372
- const database = bindingPlan.identifiers[activeDatabase].database;
1373
- const clusterParameter = '/ebk/' + foundationStage + '/foundation/databaseClusterArn';
1374
- const clusterResponse = await new SSMClient({}).send(
1375
- new GetParameterCommand({ Name: clusterParameter }),
3007
+ const databaseArg = process.argv.find((value) =>
3008
+ value.startsWith('--database='),
1376
3009
  );
1377
- const resourceArn = clusterResponse.Parameter?.Value;
1378
- if (!resourceArn) throw new Error('Foundation database cluster ARN is missing at ' + clusterParameter + '.');
1379
- const secretId = 'ebk/' + stage + '/${name}/database/' + activeDatabase + '/readwrite';
1380
- const secretResponse = await new SecretsManagerClient({}).send(
1381
- new DescribeSecretCommand({ SecretId: secretId }),
3010
+ const selectedDatabase = selectDatabaseTarget(bindingPlan, {
3011
+ database: databaseArg?.slice('--database='.length),
3012
+ });
3013
+
3014
+ const parameterName = productSliceMigrationPath(
3015
+ foundationStage,
3016
+ stage,
3017
+ '${name}',
3018
+ selectedDatabase,
1382
3019
  );
1383
- const secretArn = secretResponse.ARN;
1384
- if (!secretArn) throw new Error('Logical database secret ARN is missing for ' + secretId + '.');
1385
-
1386
- const client = new RDSDataClient({});
1387
- const resumeRetryDelaysMs = [1_000, 2_000, 4_000, 8_000, 16_000];
1388
- const sleep = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
1389
- const send = async (command) => {
1390
- for (let attempt = 0; ; attempt += 1) {
1391
- try {
1392
- return await client.send(command);
1393
- } catch (error) {
1394
- const resumable = error && typeof error === 'object' && error.name === 'DatabaseResumingException';
1395
- if (!resumable || attempt >= resumeRetryDelaysMs.length) throw error;
1396
- await sleep(resumeRetryDelaysMs[attempt]);
1397
- }
1398
- }
1399
- };
1400
- const execute = (sql, parameters = [], transactionId) => send(
1401
- new ExecuteStatementCommand({
1402
- database,
1403
- parameters,
1404
- resourceArn,
1405
- secretArn,
1406
- sql,
1407
- transactionId,
1408
- }),
3020
+ const parameter = await new SSMClient({}).send(
3021
+ new GetParameterCommand({ Name: parameterName }),
1409
3022
  );
1410
-
1411
- await execute('CREATE TABLE IF NOT EXISTS _ebk_migrations (name text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())');
1412
- const directory = new URL('../database/migrations/', import.meta.url);
1413
- const files = (await readdir(directory)).filter((file) => file.endsWith('.sql')).sort();
1414
- for (const name of files) {
1415
- const sql = await readFile(new URL(name, directory), 'utf8');
1416
- const checksum = createHash('sha256').update(sql).digest('hex');
1417
- const existing = await execute(
1418
- 'SELECT checksum FROM _ebk_migrations WHERE name = :name',
1419
- [{ name: 'name', value: { stringValue: name } }],
3023
+ const stateMachineArn = parameter.Parameter?.Value;
3024
+ if (!stateMachineArn) {
3025
+ throw new Error(
3026
+ 'Database migration state machine ARN is missing at ' +
3027
+ parameterName +
3028
+ '. Deploy Slice infrastructure before running migrations.',
1420
3029
  );
1421
- const appliedChecksum = existing.records?.[0]?.[0]?.stringValue;
1422
- if (appliedChecksum === checksum) continue;
1423
- if (appliedChecksum) throw new Error('Applied migration changed: ' + name);
3030
+ }
1424
3031
 
1425
- const transaction = await send(new BeginTransactionCommand({
1426
- database,
1427
- resourceArn,
1428
- secretArn,
1429
- }));
1430
- if (!transaction.transactionId) throw new Error('Data API did not return a transaction id.');
3032
+ const sourceCommit = (
3033
+ process.env.GITHUB_SHA ??
3034
+ process.env.EBK_SOURCE_COMMIT ??
3035
+ 'local'
3036
+ ).replace(/[^A-Za-z0-9_-]/g, '-');
3037
+ const runIdentity = (
3038
+ process.env.GITHUB_RUN_ID ??
3039
+ process.env.EBK_MIGRATION_RUN_ID ??
3040
+ String(Date.now())
3041
+ ).replace(/[^A-Za-z0-9_-]/g, '-');
3042
+ const runAttempt = (
3043
+ process.env.GITHUB_RUN_ATTEMPT ??
3044
+ '1'
3045
+ ).replace(/[^A-Za-z0-9_-]/g, '-');
3046
+ if (!/^\\d+$/.test(runAttempt)) {
3047
+ throw new Error('GITHUB_RUN_ATTEMPT must be a positive integer.');
3048
+ }
3049
+ const parsedRunAttempt = Number.parseInt(runAttempt, 10);
3050
+ if (!Number.isSafeInteger(parsedRunAttempt) || parsedRunAttempt < 1) {
3051
+ throw new Error('GITHUB_RUN_ATTEMPT must be a positive integer.');
3052
+ }
3053
+ const executionIdentity = createHash('sha256')
3054
+ .update([
3055
+ '${name}',
3056
+ selectedDatabase,
3057
+ runIdentity,
3058
+ sourceCommit,
3059
+ ].join('\\0'))
3060
+ .digest('hex')
3061
+ .slice(0, 16);
3062
+ const baseExecutionName = [
3063
+ 'ebk',
3064
+ '${name}'.slice(0, 20),
3065
+ selectedDatabase.slice(0, 20),
3066
+ executionIdentity,
3067
+ ].join('-').slice(0, 80);
3068
+ const sfn = new SFNClient({});
3069
+ const executionArnFor = (name) =>
3070
+ stateMachineArn.replace(':stateMachine:', ':execution:') + ':' + name;
3071
+ const describeExecution = async (name) => {
1431
3072
  try {
1432
- const statements = sql.split(/^--> statement-breakpoint\\s*$/m).map((value) => value.trim()).filter(Boolean);
1433
- for (const statement of statements) await execute(statement, [], transaction.transactionId);
1434
- await execute(
1435
- 'INSERT INTO _ebk_migrations (name, checksum) VALUES (:name, :checksum)',
1436
- [
1437
- { name: 'name', value: { stringValue: name } },
1438
- { name: 'checksum', value: { stringValue: checksum } },
1439
- ],
1440
- transaction.transactionId,
3073
+ return await sfn.send(
3074
+ new DescribeExecutionCommand({ executionArn: executionArnFor(name) }),
1441
3075
  );
1442
- await send(new CommitTransactionCommand({
1443
- resourceArn,
1444
- secretArn,
1445
- transactionId: transaction.transactionId,
1446
- }));
1447
3076
  } catch (error) {
1448
- await send(new RollbackTransactionCommand({
1449
- resourceArn,
1450
- secretArn,
1451
- transactionId: transaction.transactionId,
1452
- }));
3077
+ if (
3078
+ error &&
3079
+ typeof error === 'object' &&
3080
+ error.name === 'ExecutionDoesNotExist'
3081
+ ) {
3082
+ return undefined;
3083
+ }
1453
3084
  throw error;
1454
3085
  }
3086
+ };
3087
+ const startExecution = async (name) => {
3088
+ try {
3089
+ await sfn.send(
3090
+ new StartExecutionCommand({
3091
+ input: JSON.stringify({
3092
+ databaseIdentifier: selectedDatabase,
3093
+ sourceCommit,
3094
+ stage,
3095
+ slice: '${name}',
3096
+ workflowRunId: runIdentity,
3097
+ workflowRunAttempt: runAttempt,
3098
+ }),
3099
+ name,
3100
+ stateMachineArn,
3101
+ }),
3102
+ );
3103
+ } catch (error) {
3104
+ if (
3105
+ !error ||
3106
+ typeof error !== 'object' ||
3107
+ error.name !== 'ExecutionAlreadyExists'
3108
+ ) {
3109
+ throw error;
3110
+ }
3111
+ }
3112
+ };
3113
+ const terminal = new Set([
3114
+ 'ABORTED',
3115
+ 'FAILED',
3116
+ 'SUCCEEDED',
3117
+ 'TIMED_OUT',
3118
+ ]);
3119
+
3120
+ let executionName = baseExecutionName;
3121
+ let execution = await describeExecution(executionName);
3122
+ if (
3123
+ execution &&
3124
+ terminal.has(execution.status ?? 'UNKNOWN') &&
3125
+ execution.status !== 'SUCCEEDED' &&
3126
+ parsedRunAttempt > 1
3127
+ ) {
3128
+ const retrySuffix = '-retry-' + runAttempt;
3129
+ executionName =
3130
+ baseExecutionName.slice(0, 80 - retrySuffix.length) + retrySuffix;
3131
+ execution = await describeExecution(executionName);
1455
3132
  }
1456
- process.stdout.write('${name} database migrations completed for ' + stage + '.\\n');
3133
+ if (!execution) {
3134
+ await startExecution(executionName);
3135
+ }
3136
+
3137
+ const executionArn = executionArnFor(executionName);
3138
+ let status = execution?.status ?? 'RUNNING';
3139
+ process.stdout.write(
3140
+ '[database-migration] execution=' + executionArn + ' status=' + status + '\\n',
3141
+ );
3142
+ const deadline = Date.now() + 95 * 60 * 1000;
3143
+ while (!terminal.has(status)) {
3144
+ if (Date.now() >= deadline) {
3145
+ throw new Error(
3146
+ 'Timed out waiting for database migration execution ' + executionArn,
3147
+ );
3148
+ }
3149
+ await new Promise((resolve) => setTimeout(resolve, 5_000));
3150
+ const execution = await sfn.send(
3151
+ new DescribeExecutionCommand({ executionArn }),
3152
+ );
3153
+ status = execution.status ?? 'UNKNOWN';
3154
+ }
3155
+ if (status !== 'SUCCEEDED') {
3156
+ throw new Error(
3157
+ 'Database migration execution ' +
3158
+ executionArn +
3159
+ ' ended with status ' +
3160
+ status +
3161
+ '. Inspect its Step Functions and CloudWatch records.',
3162
+ );
3163
+ }
3164
+ process.stdout.write(
3165
+ '[database-migration] execution=' + executionArn + ' status=SUCCEEDED\\n',
3166
+ );
1457
3167
  `,
1458
3168
  'sst-env.d.ts': `// Bootstrap placeholder. SST overwrites this file after install, dev, or deploy.
1459
3169
  export {};