@empire-builder-kit/nx 1.0.0-rc.37 → 1.0.0-rc.39

package/src/generators/slice/built-in-plan-testing-files.js

@@ -2,36 +2,21 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.renderProductTestingFiles = renderProductTestingFiles;
 const built_in_plan_shared_js_1 = require("./built-in-plan-shared.js");
+const local_source_identity_js_1 = require("./local-source-identity.js");
+const built_in_plan_testing_persistence_files_js_1 = require("./built-in-plan-testing-persistence-files.js");
+const deployed_smoke_readiness_js_1 = require("./deployed-smoke-readiness.js");
 /** @internal */
-function renderProductTestingFiles(name, pascalName) {
+function renderProductTestingFiles(name, pascalName, sql = false) {
     return {
-        'database/test-compose.yml': `services:
-  database:
-    image: postgres:17-alpine
-    environment:
-      POSTGRES_DB: postgres
-      POSTGRES_PASSWORD: ebk_test
-      POSTGRES_USER: ebk_test
-    healthcheck:
-      test: [CMD-SHELL, "pg_isready -U ebk_test -d postgres"]
-      interval: 1s
-      timeout: 3s
-      retries: 30
-    labels:
-      dev.empirebuilderkit.owner: test-integration
-      dev.empirebuilderkit.run: \${EBK_TEST_RUN_ID}
-      dev.empirebuilderkit.slice: ${name}
-    ports:
-      - '127.0.0.1::5432'
-    tmpfs:
-      - /var/lib/postgresql/data
-`,
+        ...(0, built_in_plan_testing_persistence_files_js_1.renderTestingPersistenceFiles)(name, pascalName, sql),
+        'scripts/deployed-smoke-readiness.mjs': deployed_smoke_readiness_js_1.deployedSmokeReadinessScript,
         'docs/testing.md': `# ${name} testing
 
 Use the narrowest proof layer that can establish the behavior. Source tests are
 hermetic and credential-free. The \`test-integration\` target owns a disposable
-PostgreSQL container, creates one database per worker, applies the committed
-migrations, runs deterministic fixtures, and removes every resource it creates.
+DynamoDB Local container and one isolated table per worker. Declared SQL adds
+a PostgreSQL container and one migrated database per worker. It runs deterministic
+fixtures and removes every resource it creates.
 It never uses the persistent database started by \`ebk dev\`.
 
 Every fixture run records its seed and run scope. Keep fixture data synthetic.
@@ -51,11 +36,38 @@ deployment and migration. Local and GitHub execution use the same stage
 deployment role and exact deployed outputs; no separate smoke role, workflow,
 target registry, or staging-evidence reconciliation graph exists.
 
-Development and staging smoke wait up to sixteen minutes for the exact durable
-event on the fifteen-minute nonproduction outbox schedule. Production waits up
-to ninety seconds for the same proof on its one-minute schedule. These bounds
-include CloudWatch Logs request latency and the runner aborts an in-flight
-request when no bounded observation time remains.
+The generated Next.js component selects native SST invalidation completion
+before deployment returns. This is not a readiness or application guarantee;
+the subsequent readiness and browser gates still apply.
+
+Before synthetic users or browser assertions, a read-only prerequisite checks
+the exact API and web service, stage and deployed build. Transient gateway or
+network unavailability has one fixed thirty-second readiness bound. The one
+exception is the Router's own CloudFront error page for the web route: the
+pinned SST Router reads routing from an eventually consistent CloudFront
+KeyValueStore, so a newly deployed route can briefly reach the placeholder
+origin at an edge. Only that classified page may extend readiness, to at most
+three hundred seconds; any other failure ends the extension. Readiness then
+requires consecutive healthy rounds on fresh connections spanning thirty
+seconds; any fallthrough restarts that window. Wrong build
+identity, malformed responses, certificate errors, redirects and access denials
+fail immediately. Bounded status, request-start time, duration, valid
+CloudFront request-ID/POP/cache-result observations and the edge-routing
+classification are retained in \`deployed-readiness.json\`; no URLs, response
+bodies, cookies, arbitrary headers or credentials are retained. Duration
+measures the request through response headers (or transport failure), not
+response-body processing.
+This prerequisite is not application acceptance and does not change browser or
+event deadlines. Subsequent failed-then-passed assertions remain flaky/failing.
+
+Development and staging smoke observe the exact event for at most sixteen
+minutes; production observes it for at most ninety seconds. Ordinary delivery
+uses DynamoDB Streams or SQL post-commit publication/notification, not a wait
+for the recovery schedule. New SQL recovery defaults to hourly; a missed
+notification may therefore leave a bounded smoke unsuccessful while its durable
+intent remains recoverable. These observation limits include CloudWatch Logs
+request latency; the runner aborts an in-flight request when time runs out.
+Do not change maintenance pauses or increase timeouts merely to claim success.
 
 The runner creates one run-scoped synthetic customer in the Slice's narrow
 \`<slice>-test-writer\` group and one distinct synthetic reviewer in
@@ -78,78 +90,6 @@ Local quarantine uses reviewed date expiry. A run-count record binds an
 absolute \`.github/workflows/*.yml\` path and \`expiresAfterRun\`; it is accepted
 only when GitHub supplies a matching \`GITHUB_WORKFLOW_REF\` and positive
 \`GITHUB_RUN_NUMBER\`, and it must expire within 25 trusted workflow runs.
-`,
-        'integration/record.spec.ts': `import { Client } from 'pg';
-import { describe, expect, it } from 'vitest';
-import { build${pascalName}RecordFixture } from '../testing/fixtures/records.js';
-
-const databaseUrl = process.env.DATABASE_URL;
-if (!databaseUrl) {
-  throw new Error('DATABASE_URL is required for ${name} local integration proof.');
-}
-const seed = process.env.EBK_TEST_SEED;
-if (!seed) {
-  throw new Error('EBK_TEST_SEED is required for reproducible integration fixtures.');
-}
-
-describe('${name} database integration', () => {
-  it('writes and reads one deterministic record in its worker database', async () => {
-    const fixture = build${pascalName}RecordFixture(seed);
-    const client = new Client({ connectionString: databaseUrl });
-    await client.connect();
-    try {
-      await client.query(
-        \`INSERT INTO records
-           (id, name, owner_id, run_scope, test_expires_at)
-         VALUES ($1::uuid, $2, $3, $4, now() + INTERVAL '1 day')\`,
-        [fixture.id, fixture.name, 'synthetic-test-owner', seed],
-      );
-      const result = await client.query(
-        'SELECT id::text AS id, name FROM records WHERE id = $1::uuid',
-        [fixture.id],
-      );
-      expect(result.rows).toEqual([fixture]);
-    } finally {
-      await client.end();
-    }
-  });
-});
-`,
-        'integration/migrations.spec.ts': `import { Client } from 'pg';
-import { describe, expect, it } from 'vitest';
-
-const databaseUrl = process.env.DATABASE_URL;
-if (!databaseUrl) {
-  throw new Error('DATABASE_URL is required for ${name} local integration proof.');
-}
-
-describe('${name} migration integration', () => {
-  it('exposes the committed records table contract', async () => {
-    const requiredColumns = [
-      { column_name: 'id', data_type: 'uuid' },
-      { column_name: 'name', data_type: 'text' },
-      { column_name: 'owner_id', data_type: 'text' },
-      { column_name: 'run_scope', data_type: 'text' },
-      { column_name: 'created_at', data_type: 'timestamp with time zone' },
-    ];
-    const client = new Client({ connectionString: databaseUrl });
-    await client.connect();
-    try {
-      const result = await client.query(
-        \`SELECT column_name, data_type
-           FROM information_schema.columns
-          WHERE table_schema = 'public'
-            AND table_name = 'records'
-            AND column_name = ANY($1::text[])
-          ORDER BY ordinal_position\`,
-        [requiredColumns.map(({ column_name }) => column_name)],
-      );
-      expect(result.rows).toEqual(requiredColumns);
-    } finally {
-      await client.end();
-    }
-  });
-});
 `,
         'testing/fixtures/records.ts': `import { createTypedFixtureBuilder } from '@empire-builder-kit/nx/testing-evidence';
 
@@ -322,7 +262,7 @@ import {
   mkdirSync,
   readFileSync,
   realpathSync,
-  readdirSync,
+${sql ? '  readdirSync,\n' : ''}\
   rmSync,
   statSync,
   writeFileSync,
@@ -330,7 +270,9 @@ import {
 import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { createRequire } from 'node:module';
 import { TextDecoder } from 'node:util';
-import pg from 'pg';
+${sql ? "import pg from 'pg';" : ''}
+import { CreateTableCommand, DynamoDBClient, ListTablesCommand } from '@aws-sdk/client-dynamodb';
+import { DYNAMO_PENDING_INDEX } from '@empire-builder-kit/runtime/key-value';
 import {
   githubWorkflowRun,
   reconcileRunnerQuarantine,
@@ -339,14 +281,14 @@ import {
 
 const require = createRequire(import.meta.url);
 const {
-  createLocalTestScope,
+${sql ? '  createLocalTestScope,\n' : ''}\
   createTestingEvidenceFromRunnerAttempts,
   inspectTestingEvidenceArtifact,
   normalizeEbkIntegrationEvidenceAttempt,
   sanitizeTestingEvidenceSummary,
   validateTestingEvidenceQuarantine,
 } = require('@empire-builder-kit/nx/testing-evidence');
-const { Client } = pg;
+${sql ? 'const { Client } = pg;' : ''}
 const fatalUtf8Decoder = new TextDecoder('utf-8', { fatal: true });
 const projectRoot = process.cwd();
 const workspaceRoot = resolve(projectRoot, '../..');
@@ -382,9 +324,8 @@ const run = (command, args, options = {}) => {
   }
   return result.stdout.trim();
 };
-const gitCommit =
-  process.env.EBK_SOURCE_COMMIT ??
-  run('git', ['rev-parse', 'HEAD'], { cwd: workspaceRoot });
+${local_source_identity_js_1.localSourceCommitFunction}
+const gitCommit = localSourceCommit();
 const workspacePackage = readJson(join(workspaceRoot, 'package.json'));
 const ownership = readJson(join(projectRoot, '.ebk', 'ownership.json'));
 const frameworkCandidate =
@@ -642,7 +583,8 @@ function pruneStaleTestProjects() {
   }
 }
 
-async function prepareDatabase(adminUrl, databaseName) {
+${sql
+            ? `async function prepareDatabase(adminUrl, databaseName) {
   const admin = new Client({ connectionString: adminUrl });
   await admin.connect();
   try {
@@ -665,9 +607,54 @@ async function prepareDatabase(adminUrl, databaseName) {
     await client.end();
   }
   return databaseUrl;
+}`
+            : ''}
+
+function localDynamoClient(endpoint) {
+  const url = new URL(endpoint);
+  if (url.protocol !== 'http:' || url.hostname !== '127.0.0.1' || !url.port ||
+      url.username || url.password || url.search || url.hash || url.pathname !== '/')
+    throw new Error('Integration DynamoDB must use its owned loopback endpoint.');
+  return new DynamoDBClient({ endpoint, region: 'us-east-1', maxAttempts: 1,
+    credentials: { accessKeyId: 'LOCALFIXTURE', secretAccessKey: 'local-fixture-not-an-aws-credential' } });
 }
 
-function runWorker(worker, databaseUrl, reportPath, attempt) {
+async function waitForLocalDynamo(endpoint) {
+  const client = localDynamoClient(endpoint);
+  const deadline = Date.now() + 30_000;
+  try {
+    for (;;) {
+      try { await client.send(new ListTablesCommand({ Limit: 1 })); return; }
+      catch (error) {
+        if (Date.now() >= deadline) throw error;
+        await new Promise((resolveWait) => setTimeout(resolveWait, 200));
+      }
+    }
+  } finally { client.destroy(); }
+}
+
+async function prepareKeyValue(endpoint, attempt, worker) {
+  if (!validRunId || ![1, 2].includes(attempt) || !Number.isInteger(worker) || worker < 0 || worker > 7)
+    throw new Error('Integration table identity is invalid.');
+  const tableName = 'ebk-test-${name}-' + runId + '-a' + attempt + '-w' + worker;
+  const client = localDynamoClient(endpoint);
+  try {
+    await client.send(new CreateTableCommand({ TableName: tableName, BillingMode: 'PAY_PER_REQUEST',
+      AttributeDefinitions: [
+        { AttributeName: 'pk', AttributeType: 'S' }, { AttributeName: 'sk', AttributeType: 'S' },
+        { AttributeName: 'pendingPk', AttributeType: 'S' }, { AttributeName: 'pendingAt', AttributeType: 'N' },
+      ],
+      KeySchema: [{ AttributeName: 'pk', KeyType: 'HASH' }, { AttributeName: 'sk', KeyType: 'RANGE' }],
+      GlobalSecondaryIndexes: [{ IndexName: DYNAMO_PENDING_INDEX,
+        KeySchema: [{ AttributeName: 'pendingPk', KeyType: 'HASH' }, { AttributeName: 'pendingAt', KeyType: 'RANGE' }],
+        Projection: { ProjectionType: 'ALL' } }],
+      StreamSpecification: { StreamEnabled: true, StreamViewType: 'NEW_AND_OLD_IMAGES' },
+    }));
+    return tableName;
+  } finally { client.destroy(); }
+}
+
+function runWorker(worker, persistence, reportPath, attempt) {
   const arguments_ = [
     installedVitestCommand.entryPoint,
     'run',
@@ -708,7 +695,11 @@ function runWorker(worker, databaseUrl, reportPath, attempt) {
       cwd: projectRoot,
       env: {
         ...childEnvironment,
-        DATABASE_URL: databaseUrl,
+        ...(persistence.databaseUrl ? { DATABASE_URL: persistence.databaseUrl } : {}),
+        EBK_TEST_DYNAMODB_ENDPOINT: persistence.endpoint,
+        EBK_TEST_DYNAMODB_TABLE: persistence.tableName,
+        EBK_STAGE: 'development',
+        EVENT_BUS_NAME: 'local-integration-events',
         EBK_TEST_ATTEMPT: String(attempt),
         EBK_TEST_RUN_ID: runId,
         EBK_TEST_SEED: seed,
@@ -893,16 +884,17 @@ function readBoundedVitestReport(reportPath) {
   return { report };
 }
 
-async function runAttempt(attempt, adminUrl) {
+async function runAttempt(attempt, endpoint${sql ? ', adminUrl' : ''}) {
   const attemptStartedAt = new Date().toISOString();
   const findings = [];
   const reports = [];
   const reportPaths = [];
   let workerResults = [];
   try {
-    const databaseUrls = [];
+    const persistenceWorkers = [];
     for (let worker = 0; worker < workers; worker += 1) {
-      const scope = createLocalTestScope({
+      ${sql
+            ? `const scope = createLocalTestScope({
         capabilities: {
           canCreateDatabase: true,
           canForceDropDatabase: true,
@@ -919,18 +911,18 @@ async function runAttempt(attempt, adminUrl) {
         },
         sharedDatabaseName: 'postgres',
       });
-      databaseUrls.push(
-        await prepareDatabase(adminUrl, scope.databaseName),
-      );
+      const databaseUrl = await prepareDatabase(adminUrl, scope.databaseName);`
+            : ''}
+      persistenceWorkers.push({ endpoint, tableName: await prepareKeyValue(endpoint, attempt, worker)${sql ? ', databaseUrl' : ''} });
     }
     workerResults = await Promise.all(
-      databaseUrls.map((databaseUrl, worker) => {
+      persistenceWorkers.map((persistence, worker) => {
         const reportPath = join(
           rawRoot,
           'vitest-attempt-' + attempt + '-worker-' + worker + '.json',
         );
         reportPaths.push(reportPath);
-        return runWorker(worker, databaseUrl, reportPath, attempt);
+        return runWorker(worker, persistence, reportPath, attempt);
       }),
     );
     const workerDiagnostics = [];
@@ -1148,11 +1140,18 @@ if (process.exitCode === undefined) {
   const reports = [];
   const reportPaths = [];
   try {
+    // Even a partially failed compose up can create owned resources.
+    proofStarted = true;
     run('docker', [...composeArgs, 'up', '--detach', '--wait'], {
       env: { ...process.env, EBK_TEST_RUN_ID: runId },
     });
-    proofStarted = true;
-    const address = run('docker', [
+    const keyValueAddress = run('docker', [...composeArgs, 'port', 'key-value', '8000']);
+    const keyValuePort = /:(\\d+)\\s*$/.exec(keyValueAddress)?.[1];
+    if (!keyValuePort) throw new Error('Docker did not report the integration DynamoDB port.');
+    const endpoint = 'http://127.0.0.1:' + keyValuePort;
+    await waitForLocalDynamo(endpoint);
+    ${sql
+            ? `const address = run('docker', [
       ...composeArgs,
       'port',
       'database',
@@ -1161,13 +1160,14 @@ if (process.exitCode === undefined) {
     const port = /:(\\d+)\\s*$/.exec(address)?.[1];
     if (!port) throw new Error('Docker did not report the integration port.');
     const adminUrl =
-      'postgresql://ebk_test:ebk_test@127.0.0.1:' + port + '/postgres';
-    const first = await runAttempt(1, adminUrl);
+      'postgresql://ebk_test:ebk_test@127.0.0.1:' + port + '/postgres';`
+            : ''}
+    const first = await runAttempt(1, endpoint${sql ? ', adminUrl' : ''});
     attempts.push(first.normalized);
     reports.push(...first.reports);
     reportPaths.push(...first.reportPaths);
     if (first.normalized.failed > 0) {
-      const second = await runAttempt(2, adminUrl);
+      const second = await runAttempt(2, endpoint${sql ? ', adminUrl' : ''});
       attempts.push(second.normalized);
       reports.push(...second.reports);
       reportPaths.push(...second.reportPaths);
@@ -1294,7 +1294,9 @@ if (process.exitCode === undefined) {
   }
 }
 `,
-        'scripts/run-smoke-deployed.mjs': `import {
+        'scripts/run-smoke-deployed.mjs': `import { sliceAuthorizationGroupNames } from '../authorization/groups.mjs';
+import { waitForDeployedSmokeReadiness } from './deployed-smoke-readiness.mjs';
+import {
   AdminAddUserToGroupCommand,
   AdminCreateUserCommand,
   AdminDeleteUserCommand,
@@ -1409,8 +1411,10 @@ const evidenceRunId = validRunId ? runId : 'invalid-run-id';
 const evidenceSeed = validSeed ? seed : 'invalid-seed';
 const resultPath = join(evidenceRoot, 'smoke-deployed.json');
 const sanitizedReportPath = join(evidenceRoot, 'playwright-smoke.json');
+const readinessPath = join(evidenceRoot, 'deployed-readiness.json');
 rmSync(resultPath, { force: true });
 rmSync(sanitizedReportPath, { force: true });
+rmSync(readinessPath, { force: true });
 const rawPath = (attempt) =>
   join(rawRoot, 'playwright-smoke-' + attempt + '.json');
 const dataCleanupPath = (attempt) =>
@@ -1721,6 +1725,17 @@ function requiresScheduledDeliveryObservation(stage) {
   return ['development', 'staging', 'production'].includes(stage);
 }
 
+// A successful browser journey may have deleted its synthetic record, while
+// its mutation receipt remains. Replaying that same journey/run scope cannot
+// recreate the record, even when the first cleanup completed successfully.
+function shouldRetryDeployedSmoke(execution) {
+  return (
+    execution.failed > 0 &&
+    execution.dataCleanup?.outcome === 'not-required' &&
+    execution.observationFindings.length === 0
+  );
+}
+
 function scheduledDeliveryDeadlineFinding() {
   return {
     classification: 'assertion-failed',
@@ -1752,6 +1767,7 @@ async function observePublishedEvent(logs, observationPath, attemptStartedAt) {
   const startTime = Math.max(0, Date.parse(attemptStartedAt) - 5_000);
   const timeoutMilliseconds = stage === 'production' ? 90_000 : 960_000;
   const deadline = performance.now() + timeoutMilliseconds;
+  let consecutiveUnavailableReads = 0;
   while (true) {
     const remainingRequestMilliseconds = deadline - performance.now();
     if (remainingRequestMilliseconds <= 0) break;
@@ -1771,10 +1787,28 @@ async function observePublishedEvent(logs, observationPath, attemptStartedAt) {
         },
       );
     } catch (error) {
-      if (
-        performance.now() >= deadline ||
-        ['AbortError', 'TimeoutError'].includes(error?.name)
-      ) {
+      if (error?.name === 'ServiceUnavailableException') {
+        consecutiveUnavailableReads += 1;
+        // Retry the read, not the already-mutating browser journey. Keep this
+        // much shorter than the delivery deadline when Logs is unavailable.
+        const remainingDelayMilliseconds = deadline - performance.now();
+        const retryDelayMilliseconds = 1_000 * consecutiveUnavailableReads;
+        if (
+          consecutiveUnavailableReads >= 3 ||
+          remainingDelayMilliseconds <= retryDelayMilliseconds
+        ) {
+          return permissionFailure(
+            error,
+            'event-consumer observation',
+            'CloudWatch Logs',
+          );
+        }
+        await new Promise((resolveDelay) =>
+          setTimeout(resolveDelay, retryDelayMilliseconds),
+        );
+        continue;
+      }
+      if (['AbortError', 'TimeoutError'].includes(error?.name)) {
         return scheduledDeliveryDeadlineFinding();
       }
       return permissionFailure(
@@ -1783,6 +1817,7 @@ async function observePublishedEvent(logs, observationPath, attemptStartedAt) {
         'CloudWatch Logs',
       );
     }
+    consecutiveUnavailableReads = 0;
     if (performance.now() > deadline) {
       return scheduledDeliveryDeadlineFinding();
     }
@@ -1885,6 +1920,23 @@ if (process.exitCode === undefined) {
   }
 }
 
+if (process.exitCode === undefined) {
+  const readiness = await waitForDeployedSmokeReadiness({
+    apiUrl: smokeApiUrl,
+    webUrl: smokeUrl,
+    service: '${name}',
+    stage,
+    sourceCommit: deployedSourceCommit,
+  });
+  writeFileSync(readinessPath, JSON.stringify({
+    schemaVersion: 1, source, deployedSourceCommit, stage, slice: '${name}',
+    result: readiness,
+  }, null, 2) + '\\n', { encoding: 'utf8', mode: 0o600 });
+  if (readiness.outcome !== 'ready') {
+    failPreflight(readiness.classification, readiness.summary);
+  }
+}
+
 if (process.exitCode === undefined) {
   const cognito = new CognitoIdentityProviderClient({
     region: process.env.AWS_REGION,
@@ -1933,7 +1985,7 @@ if (process.exitCode === undefined) {
     );
     await cognito.send(
       new AdminAddUserToGroupCommand({
-        GroupName: '${name}-test-writer',
+        GroupName: sliceAuthorizationGroupNames(stage).testWriter,
         Username: principalEmail,
         UserPoolId: userPoolId,
       }),
@@ -1960,7 +2012,7 @@ if (process.exitCode === undefined) {
     );
     await cognito.send(
       new AdminAddUserToGroupCommand({
-        GroupName: '${name}-admin',
+        GroupName: sliceAuthorizationGroupNames(stage).admin,
         Username: reviewerEmail,
         UserPoolId: userPoolId,
       }),
@@ -2258,7 +2310,7 @@ if (process.exitCode === undefined) {
     let cleanup = { outcome: 'passed' };
     try {
       executions.push(await executeAttempt(1));
-      if (executions[0].failed > 0) {
+      if (shouldRetryDeployedSmoke(executions[0])) {
         executions.push(await executeAttempt(2));
       }
       const reports = executions

package/src/generators/slice/built-in-plan-testing-persistence-files.d.ts

@@ -0,0 +1,2 @@
+/** Disposable integration resources are independent of fleet or deployed data. */
+export declare function renderTestingPersistenceFiles(name: string, pascalName: string, sql: boolean): Readonly<Record<string, string>>;