@empire-builder-kit/nx 1.0.0-rc.39 → 1.0.0-rc.40

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@empire-builder-kit/nx",
-  "version": "1.0.0-rc.39",
+  "version": "1.0.0-rc.40",
   "description": "Nx generators, executors, and Blueprint contracts for Empire Builder Kit",
   "keywords": [
     "aws",
@@ -145,7 +145,7 @@
     "provenance": false
   },
   "dependencies": {
-    "@empire-builder-kit/runtime": "1.0.0-rc.39",
+    "@empire-builder-kit/runtime": "1.0.0-rc.40",
     "@aws-sdk/client-cloudfront": "^3.700.0",
     "@aws-sdk/client-cloudwatch": "^3.700.0",
     "@aws-sdk/client-cloudwatch-logs": "^3.700.0",
@@ -178,5 +178,5 @@
   "engines": {
     "node": ">=22.13.0"
   },
-  "gitHead": "2875c7b517e99e6eab20c6efa31dfeb932b410f8"
+  "gitHead": "56a2e45b04b8ce8849990baba025cecb5909f8c7"
 }

package/src/executors/local-deploy/local-deploy.js

@@ -25,6 +25,7 @@ const run_sst_command_js_1 = require("../sst-lifecycle/run-sst-command.js");
 const index_js_3 = require("../../workspace-deployment/index.js");
 const private_smoke_access_js_1 = require("../../foundation/private-smoke-access.js");
 const local_private_smoke_access_js_1 = require("../../foundation/local-private-smoke-access.js");
+const capability_plan_js_1 = require("../../foundation/capability-plan.js");
 const sst_environment_source_js_1 = require("../sst-lifecycle/sst-environment-source.js");
 const SHA = /^[a-f0-9]{40,64}$/u;
 const ROLE = /^arn:(aws|aws-us-gov|aws-cn):iam::\d{12}:role\/[A-Za-z0-9+=,.@_/-]{1,512}$/u;
@@ -668,6 +669,30 @@ async function runNxTarget(input) {
     if (primaryFailure)
         throw primaryFailure.error;
 }
+const FOUNDATION_DESCRIPTOR_PATH = 'packages/foundation/contracts/foundation.json';
+/** The committed Foundation declaration is the only SQL signal; the generated
+ * Foundation publishes database discovery exactly when it declares postgres. */
+function productionFoundationDeclaresPostgres(root) {
+    let descriptor;
+    try {
+        descriptor = JSON.parse((0, node_fs_1.readFileSync)((0, node_path_1.join)(root, FOUNDATION_DESCRIPTOR_PATH), 'utf8'));
+    }
+    catch {
+        throw new Error(`Production deployment requires the committed Foundation capability declaration ${FOUNDATION_DESCRIPTOR_PATH}.`);
+    }
+    return (0, capability_plan_js_1.planFoundationCapabilities)({ current: descriptor, requirements: [] })
+        .resources.postgres;
+}
+/** Declarations are monotonic: a SQL-free Foundation must never observe, gain,
+ * or lose a production database during a native deployment. The
+ * `priorDiscovery` check is a redundant second line: the preflight already
+ * rejects an existing database before any child deploys, so a prior `existing`
+ * state cannot reach the postflights today and no spec exercises that branch. */
+function assertSqlFreeProductionDatabase(discovery, priorDiscovery) {
+    if (discovery.state !== 'absent' || priorDiscovery?.state === 'existing') {
+        throw new Error('Production Foundation database discovery is present although the Foundation does not declare postgres; deployment is blocked.');
+    }
+}
 function productionConfirmation(account, region, commit) {
     return `deploy:production:${account}:${region}:${commit}`;
 }
@@ -728,6 +753,7 @@ async function runLocalDeploy(options, context, dependencies = {}) {
     let transport;
     let productionDatabaseBefore;
     let productionDatabaseAfterFoundation;
+    let productionPostgresDeclared = true;
     let outcome = 'failed';
     let failure;
     let finalPhase = 'validation';
@@ -803,6 +829,11 @@ async function runLocalDeploy(options, context, dependencies = {}) {
         if ((githubAuthority || options.stage !== 'development') && !source.clean) {
             throw new Error(`Local ${options.stage} deployment requires a completely clean source tree.`);
         }
+        if (options.stage === 'production') {
+            // Read only after the clean-source check so the declaration is bound to
+            // the digested tree, and before any AWS authority or transport exists.
+            productionPostgresDeclared = productionFoundationDeclaresPostgres(context.root);
+        }
         if (githubAuthority) {
             if (options.profile) {
                 throw new Error('Protected GitHub deployment must use ambient OIDC credentials, not a local AWS profile.');
@@ -947,6 +978,11 @@ async function runLocalDeploy(options, context, dependencies = {}) {
                 ...(transport ? { profile: transport.profile } : {}),
                 requireExisting: false,
             });
+            // Fail before any mutation: deploying a SQL-free Foundation over an
+            // existing database would retire it outside the separate retirement path.
+            if (!productionPostgresDeclared) {
+                assertSqlFreeProductionDatabase(productionDatabaseBefore);
+            }
         }
         const localContainerRunId = localProtectedContainerProjects.length === 0
             ? ''
@@ -1122,17 +1158,24 @@ async function runLocalDeploy(options, context, dependencies = {}) {
                             }
                             : {}),
                         expectedAccountId: account,
-                        expectedCapacity: {
-                            max: Number.parseFloat(profile.aurora.max),
-                            min: Number.parseFloat(profile.aurora.min),
-                        },
+                        ...(productionPostgresDeclared
+                            ? {
+                                expectedCapacity: {
+                                    max: Number.parseFloat(profile.aurora.max),
+                                    min: Number.parseFloat(profile.aurora.min),
+                                },
+                            }
+                            : {}),
                         expectedRegion: region,
                         expectedRoleArn,
                         now: now(),
                         priorDiscovery: productionDatabaseBefore,
                         ...(transport ? { profile: transport.profile } : {}),
-                        requireExisting: true,
+                        requireExisting: productionPostgresDeclared,
                     });
+                    if (!productionPostgresDeclared) {
+                        assertSqlFreeProductionDatabase(productionDatabaseAfterFoundation, productionDatabaseBefore);
+                    }
                     children.push({
                         completedAt: now().toISOString(),
                         outcome: 'succeeded',
@@ -1259,7 +1302,7 @@ async function runLocalDeploy(options, context, dependencies = {}) {
             const foundationProject = orderedProjects[0] ?? 'foundation';
             const profile = (0, index_js_2.productionFoundationProfileContract)(policy.productionFoundation.profile).contract;
             try {
-                await (dependencies.attestProductionDatabase ??
+                const finalDiscovery = await (dependencies.attestProductionDatabase ??
                     production_database_safeguard_js_1.attestProductionDatabaseLifecycle)({
                     ...(boundaryEnvironment.AWS_CONFIG_FILE
                         ? { configFilepath: boundaryEnvironment.AWS_CONFIG_FILE }
@@ -1270,17 +1313,24 @@ async function runLocalDeploy(options, context, dependencies = {}) {
                         }
                         : {}),
                     expectedAccountId: account,
-                    expectedCapacity: {
-                        max: Number.parseFloat(profile.aurora.max),
-                        min: Number.parseFloat(profile.aurora.min),
-                    },
+                    ...(productionPostgresDeclared
+                        ? {
+                            expectedCapacity: {
+                                max: Number.parseFloat(profile.aurora.max),
+                                min: Number.parseFloat(profile.aurora.min),
+                            },
+                        }
+                        : {}),
                     expectedRegion: region,
                     expectedRoleArn,
                     now: now(),
                     priorDiscovery: productionDatabaseAfterFoundation,
                     ...(transport ? { profile: transport.profile } : {}),
-                    requireExisting: true,
+                    requireExisting: productionPostgresDeclared,
                 });
+                if (!productionPostgresDeclared) {
+                    assertSqlFreeProductionDatabase(finalDiscovery, productionDatabaseAfterFoundation);
+                }
                 children.push({
                     completedAt: now().toISOString(),
                     outcome: 'succeeded',

package/src/executors/sst-lifecycle/run-sst-command.js

@@ -51,7 +51,7 @@ const SST_PROCESS_GROUP_POLL_MS = 25;
 const GOVERNED_PROCESS_GROUP_ENV = 'EBK_GOVERNED_PROCESS_GROUP';
 function assertGovernedAwsLifecyclePlatform(platform = process.platform) {
     if (platform === 'win32') {
-        throw new Error('Governed AWS infrastructure lifecycle is unavailable on native Windows; use macOS, Linux, or WSL2. Native Windows remains supported for local development and verification.');
+        throw new Error('Governed AWS infrastructure lifecycle is unavailable on native Windows; use macOS, Linux, or WSL2. Native-Windows task Fleet is not validated in this prerelease either.');
     }
 }
 function isMissingProcessError(error) {

package/src/generators/preset/workspace-files-adopter-skills.js

@@ -128,8 +128,9 @@ configuration. It is advisory, not proof that image authentication will fail:
 some valid helpers do not implement list. If Docker's actual image pull fails,
 repair/unlock that registry's selected helper; do not
 remove host credentials, drop CLI plugins or prune shared caches as a workaround.
-Docker is required only for declared container/local-SQL dependencies, not
-ordinary unit tests or DynamoDB feature editing.
+Docker is required for Slice \`verify\` (its \`test-integration\` runs DynamoDB
+Local, plus PostgreSQL when SQL is declared) and for declared container
+dependencies, not for ordinary unit tests or feature editing.
 
 \`\`\`bash
 ebk dev up billing
@@ -143,8 +144,9 @@ owned fleet origin and browser-scoped trust; do not guess ports or alter host
 certificate trust. Keep the persisted opaque task identity on restart. Inspect
 owned status after interruption; use supported task recovery, not lock deletion.
 
-Use affected unit/type/lint/ownership checks during edits. Require Docker only
-for local container dependencies or external Blueprint execution; require AWS
+Use affected unit/type/lint/ownership checks during edits. Require Docker for
+Slice integration tests (\`test-integration\`, part of \`verify\`), local container
+dependencies, or external Blueprint execution; require AWS
 credentials for actual task resources. Install browser binaries for browser
 proof. A Standard Slice includes a Python infrastructure provisioner even when
 its feature handlers use TypeScript: install uv ${tool_versions_js_1.WORKSPACE_UV_VERSION} (Python 3.12 or

package/src/generators/preset/workspace-files-thin-docs.js

@@ -29,7 +29,9 @@ ${tool_versions_js_1.WORKSPACE_UV_VERSION} (Python 3.12 or newer) on PATH before
 \`ebk deploy\`. Each Standard Slice stops before its SST child starts when
 \`uv --version\` fails; a full \`ebk deploy\` may already have deployed
 Foundation.
-Source-only feature checks do not require it.
+Source-only feature checks do not require it. Slice \`verify\` runs
+\`test-integration\`, which needs a running Docker engine for DynamoDB Local (and
+PostgreSQL when SQL is declared).
 `,
         'docs/git-workflow.md': `# Protected Git workflow
 
@@ -163,10 +165,11 @@ committed policy and gives SST an invocation-private temporary profile rather
 than the operator's Identity Center profile chain.
 
 Governed AWS infrastructure planning and mutation are supported on macOS,
-Linux, and WSL2. Native Windows remains supported for local development and
-verification, but run \`deploy-local\`, \`infrastructure-plan\`, \`refresh\`,
-\`remove\`, and \`unlock\` from WSL2 or a macOS/Linux host. Generated protected
-GitHub AWS jobs continue to run on Ubuntu.
+Linux, and WSL2. Native Windows can run local verification, but native-Windows
+task Fleet (\`ebk dev\`) is not validated in this prerelease; use WSL2 or a
+macOS/Linux host for Fleet development and for \`deploy-local\`,
+\`infrastructure-plan\`, \`refresh\`, \`remove\`, and \`unlock\`. Generated
+protected GitHub AWS jobs continue to run on Ubuntu.
 
 Development may deploy a dirty working tree without a prompt. Its receipt binds
 the exact HEAD and working-tree digest. Local staging requires a clean commit

package/src/generators/slice/built-in-plan-api-files.js

@@ -669,8 +669,11 @@ export const expireHoldHandler = createContractHttpJsonHandler<
 `,
         'database/runtime-client.ts': `import {
   databaseTlsOptions,
+  isDatabaseConnectFailure,
   runWithDatabaseCredential,
   toNodePostgresConnectionOptions,
+  withDatabaseClient,
+  type DatabaseConnectRetry,
   type LinkedDatabaseResource,
 } from '@empire-builder-kit/runtime/database';
 import { createLogger } from '@empire-builder-kit/runtime/logging';
@@ -729,22 +732,72 @@ function linkedDatabase(): LinkedDatabaseResource {
   return resource as LinkedDatabaseResource;
 }
 
+type PrimaryDatabaseWorkload =
+  | 'api-database-health'
+  | 'api-database-records'
+  | 'api-publish-created'
+  | 'async-inbox-cleanup'
+  | 'async-outbox-relay'
+  | 'async-provider-worker'
+  | 'data-export-cleanup'
+  | 'data-retention-job'
+  | 'event-consumer'
+  | 'web-server';
+
+// Aurora Serverless v2 resume can take 15-30 s or longer. Only connect()
+// retries, within a per-call wall-clock budget that leaves room for the
+// operation: HTTP routes and the web server run on SST's 20 s default
+// timeout, workers and the outbox relay on 1 minute, and jobs on 5 minutes.
+// HTTP attempts are short so a held or slow resume connect is retried
+// inside the budget instead of consuming it in a single attempt.
+const connectPolicy: Record<
+  PrimaryDatabaseWorkload,
+  { attemptTimeoutMs: number; budgetMs: number }
+> = {
+  'api-database-health': { attemptTimeoutMs: 5_000, budgetMs: 15_000 },
+  'api-database-records': { attemptTimeoutMs: 5_000, budgetMs: 15_000 },
+  'api-publish-created': { attemptTimeoutMs: 5_000, budgetMs: 15_000 },
+  'async-inbox-cleanup': { attemptTimeoutMs: 15_000, budgetMs: 120_000 },
+  'async-outbox-relay': { attemptTimeoutMs: 15_000, budgetMs: 40_000 },
+  'async-provider-worker': { attemptTimeoutMs: 15_000, budgetMs: 40_000 },
+  'data-export-cleanup': { attemptTimeoutMs: 15_000, budgetMs: 120_000 },
+  'data-retention-job': { attemptTimeoutMs: 15_000, budgetMs: 120_000 },
+  'event-consumer': { attemptTimeoutMs: 15_000, budgetMs: 40_000 },
+  'web-server': { attemptTimeoutMs: 5_000, budgetMs: 15_000 },
+};
+
+function logConnectionRetry(
+  context: { correlationId: string },
+  workload: PrimaryDatabaseWorkload,
+  retry: DatabaseConnectRetry,
+): void {
+  createLogger({
+    component: workload,
+    feature: 'database',
+    operation: 'database.connect',
+    slice: '${name}',
+    stage: process.env.EBK_STAGE ?? '',
+    workspace: '${workspace}',
+  }).warn('Retrying database connection', {
+    attributes: {
+      databaseEvent: 'database_runtime_connection_retry',
+      ...retry,
+    },
+    context: { correlationId: context.correlationId },
+    outcome: 'unknown',
+  });
+}
+
 export async function withPrimaryDatabase<Result>(
   context: { correlationId: string },
-  workload:
-    | 'api-database-health'
-    | 'api-database-records'
-    | 'api-publish-created'
-    | 'async-inbox-cleanup'
-    | 'async-outbox-relay'
-    | 'async-provider-worker'
-    | 'data-export-cleanup'
-    | 'data-retention-job'
-    | 'event-consumer'
-    | 'web-server',
+  workload: PrimaryDatabaseWorkload,
   operation: (client: PgClient) => Promise<Result>,
 ): Promise<Result> {
   const resource = linkedDatabase();
+  const policy = connectPolicy[workload];
+  // One per-call deadline spans credential resolution and any 28P01 refresh.
+  // Handlers that make several sequential calls bound their own total work.
+  const deadline = Date.now() + policy.budgetMs;
   return runWithDatabaseCredential(
     resource,
     resource.provider === 'aws-secrets-manager'
@@ -754,22 +807,25 @@ export async function withPrimaryDatabase<Result>(
           resolver: getSecretResolver(workload),
         }
       : undefined,
-    async (credential) => {
-      const client = new Client({
-        ...toNodePostgresConnectionOptions(credential),
-        connectionTimeoutMillis: 10_000,
-        ssl: databaseTlsOptions(resource),
-      });
-      try {
-        await client.connect();
-        return await operation(client);
-      } finally {
-        await client.end().catch(() => undefined);
-      }
-    },
+    (credential) =>
+      withDatabaseClient(
+        {
+          attemptTimeoutMs: policy.attemptTimeoutMs,
+          createClient: (connectionTimeoutMillis) =>
+            new Client({
+              ...toNodePostgresConnectionOptions(credential),
+              connectionTimeoutMillis,
+              ssl: databaseTlsOptions(resource),
+            }),
+          deadline,
+          onRetry: (retry) => logConnectionRetry(context, workload, retry),
+        },
+        operation,
+      ),
+    // Refresh credentials only when connect() itself failed authentication,
+    // so a started operation is never run a second time.
     (error) =>
-      error !== null &&
-      typeof error === 'object' &&
+      isDatabaseConnectFailure(error) &&
       'code' in error &&
       error.code === '28P01',
   );

package/src/generators/slice/built-in-plan-async-work-files.js

@@ -6045,6 +6045,55 @@ capacity/idempotency/provider-effect evidence, stop conditions, and the forward
 compensation plan. Archive-wide observed size is evidence, not an immutable
 range estimate, because the canonical archive continues receiving events while
 approval is pending.
+
+## Event delivery alarms
+
+Domain events are delivered by the key-value stream Function and recovered by
+the scheduled key-value reconciliation Function. The \`async/outbox-relay\`
+Function drains only the isolated performance sink; \`outbox-relay-errors\`
+does not describe domain delivery. These alarms observe delivery only. No alarm
+pauses, drops, quarantines, caps, or retries an event intent, and reconciliation
+does not fail because some intents remain undelivered.
+
+- \`key-value-stream-delivery-errors\` reports a failed invocation of the
+  stream delivery Function: the stream identity or record shape was rejected,
+  or the handler crashed. Per-record publish failures are partial batch
+  failures, not Lambda errors. They surface through
+  \`key-value-stream-iterator-age\` while retries hold the shard back, and
+  through \`outbox-backlog\` and \`event-delivery-attempts\` after the mapping
+  discards them (after three retries or one hour).
+- \`key-value-reconciliation-errors\` reports Lambda invocation errors for the
+  exact reconciliation Function.
+- \`key-value-stream-iterator-age\` reports a stream iterator at least ten
+  minutes old. The mapping discards records older than one hour; their retained
+  intents then wait for reconciliation.
+- \`outbox-backlog\` reports \`OutboxBacklog\`: intents a reconciliation run
+  attempted but could not deliver, plus rejected intents, in two consecutive
+  hours. Each run reads at most one 25-intent page per shard, so the value is a
+  lower bound rather than a global count. Intents leased by another delivery
+  owner are in flight and are not counted.
+- \`event-intent-rejected\` reports retained intents that fail validation.
+  They are never dispatched or deleted, so the alarm recurs until an owner
+  investigates the content-free \`ebk-key-value-invalid-intents\` record.
+- \`event-delivery-attempts\` reports \`EventDeliveryMaxAttempts\` of at least
+  ten delivery leases on one due intent. Attempts are reported, never enforced.
+
+SQL Slices only (see docs/runbooks/sql-events.md):
+
+- \`sql-outbox-relay-errors\` reports Lambda errors for the exact SQL outbox
+  relay Function, which fails every run that retains a failed delivery.
+- \`sql-outbox-backlog\` reports the Minimum of \`SqlOutboxBacklog\`, the
+  global unpublished-intent count, over two consecutive hours (3600 s x 2): it
+  alarms only when no relay run in either hour drained the outbox. A paused
+  relay reports nothing, so the alarm is absent rather than breaching while
+  paused.
+- \`sql-outbox-attempts\` reports \`SqlOutboxMaxAttempts\` of at least ten
+  relay attempts on one unpublished intent. It is observational; no intent is
+  capped or dropped.
+
+Diagnose with the content-free \`ebk-key-value-reconciliation\` log summary and
+the EventBridge target before any recovery. Do not delete or edit retained
+intents to clear an alarm.
 `,
     };
 }

package/src/generators/slice/built-in-plan-infrastructure-files.js

@@ -773,6 +773,142 @@ ${`  const keyValue = createKeyValueInfrastructure({
     lambdaLogging,
   });
 `}\
+  // Domain delivery is the key-value stream plus scheduled reconciliation; the
+  // async/outbox-relay Function drains only the isolated performance sink.
+  // Observation only: alarms never pause, drop, or retry an event intent.
+  const keyValueStreamDeliveryErrorAlarm = new aws.cloudwatch.MetricAlarm(
+    '${pascalName}KeyValueStreamDeliveryErrors',
+    {
+      alarmActions: [foundation.discovery.alertTopicArn],
+      alarmDescription: operationalAlarmDescription({
+        businessSymptom:
+          'The key-value stream delivery Function failed an invocation.',
+        id: 'key-value-stream-delivery-errors',
+        runbook: 'docs/runbooks/async-recovery.md',
+      }),
+      comparisonOperator: 'GreaterThanOrEqualToThreshold',
+      dimensions: { FunctionName: keyValue.dispatcher.name },
+      evaluationPeriods: 1,
+      metricName: 'Errors',
+      name: '${name}-' + $app.stage + '-key-value-stream-delivery-errors',
+      namespace: 'AWS/Lambda',
+      period: 300,
+      statistic: 'Sum',
+      threshold: 1,
+      treatMissingData: 'notBreaching',
+    },
+  );
+  const keyValueStreamIteratorAgeAlarm = new aws.cloudwatch.MetricAlarm(
+    '${pascalName}KeyValueStreamIteratorAge',
+    {
+      alarmActions: [foundation.discovery.alertTopicArn],
+      alarmDescription: operationalAlarmDescription({
+        businessSymptom:
+          'Prompt domain-event delivery from the key-value stream is falling behind.',
+        id: 'key-value-stream-iterator-age',
+        runbook: 'docs/runbooks/async-recovery.md',
+      }),
+      comparisonOperator: 'GreaterThanOrEqualToThreshold',
+      dimensions: { FunctionName: keyValue.dispatcher.name },
+      evaluationPeriods: 1,
+      metricName: 'IteratorAge',
+      name: '${name}-' + $app.stage + '-key-value-stream-iterator-age',
+      namespace: 'AWS/Lambda',
+      period: 300,
+      statistic: 'Maximum',
+      threshold: 600_000,
+      treatMissingData: 'notBreaching',
+    },
+  );
+  const keyValueReconciliationErrorAlarm = new aws.cloudwatch.MetricAlarm(
+    '${pascalName}KeyValueReconciliationErrors',
+    {
+      alarmActions: [foundation.discovery.alertTopicArn],
+      alarmDescription: operationalAlarmDescription({
+        businessSymptom: 'Durable domain-event recovery is failing.',
+        id: 'key-value-reconciliation-errors',
+        runbook: 'docs/runbooks/async-recovery.md',
+      }),
+      comparisonOperator: 'GreaterThanOrEqualToThreshold',
+      dimensions: { FunctionName: keyValue.reconciler.name },
+      evaluationPeriods: 1,
+      metricName: 'Errors',
+      name: '${name}-' + $app.stage + '-key-value-reconciliation-errors',
+      namespace: 'AWS/Lambda',
+      period: 300,
+      statistic: 'Sum',
+      threshold: 1,
+      treatMissingData: 'notBreaching',
+    },
+  );
+  // Reconciliation gauges are bounded per-run samples (one page per shard),
+  // so hourly periods cover the default and any sub-hourly reconciliation
+  // cadence. A slower configured cadence leaves empty periods, so these alarms
+  // can miss a backlog (see each registration's expectedFalseNegativeRisk).
+  const outboxBacklogAlarm = new aws.cloudwatch.MetricAlarm(
+    '${pascalName}OutboxBacklog',
+    {
+      alarmActions: [foundation.discovery.alertTopicArn],
+      alarmDescription: operationalAlarmDescription({
+        businessSymptom: 'Durable domain-event intents remain undelivered.',
+        id: 'outbox-backlog',
+        runbook: 'docs/runbooks/async-recovery.md',
+      }),
+      comparisonOperator: 'GreaterThanOrEqualToThreshold',
+      dimensions: { Slice: '${name}', Stage: $app.stage },
+      evaluationPeriods: 2,
+      metricName: 'OutboxBacklog',
+      name: '${name}-' + $app.stage + '-outbox-backlog',
+      namespace: 'EBK/Async',
+      period: 3600,
+      statistic: 'Maximum',
+      threshold: 1,
+      treatMissingData: 'notBreaching',
+    },
+  );
+  const eventIntentRejectedAlarm = new aws.cloudwatch.MetricAlarm(
+    '${pascalName}EventIntentRejected',
+    {
+      alarmActions: [foundation.discovery.alertTopicArn],
+      alarmDescription: operationalAlarmDescription({
+        businessSymptom: 'A retained domain-event intent cannot be delivered.',
+        id: 'event-intent-rejected',
+        runbook: 'docs/runbooks/async-recovery.md',
+      }),
+      comparisonOperator: 'GreaterThanOrEqualToThreshold',
+      dimensions: { Slice: '${name}', Stage: $app.stage },
+      evaluationPeriods: 1,
+      metricName: 'EventReconciliationRejected',
+      name: '${name}-' + $app.stage + '-event-intent-rejected',
+      namespace: 'EBK/Async',
+      period: 3600,
+      statistic: 'Maximum',
+      threshold: 1,
+      treatMissingData: 'notBreaching',
+    },
+  );
+  const eventDeliveryAttemptsAlarm = new aws.cloudwatch.MetricAlarm(
+    '${pascalName}EventDeliveryAttempts',
+    {
+      alarmActions: [foundation.discovery.alertTopicArn],
+      alarmDescription: operationalAlarmDescription({
+        businessSymptom:
+          'A domain-event intent is repeatedly failing delivery.',
+        id: 'event-delivery-attempts',
+        runbook: 'docs/runbooks/async-recovery.md',
+      }),
+      comparisonOperator: 'GreaterThanOrEqualToThreshold',
+      dimensions: { Slice: '${name}', Stage: $app.stage },
+      evaluationPeriods: 1,
+      metricName: 'EventDeliveryMaxAttempts',
+      name: '${name}-' + $app.stage + '-event-delivery-attempts',
+      namespace: 'EBK/Async',
+      period: 3600,
+      statistic: 'Maximum',
+      threshold: 10,
+      treatMissingData: 'notBreaching',
+    },
+  );
   const nativeAuthority = createStandardSliceNativeRuntimeAuthority({
 ${sql
             ? `    clusterId: foundation.discovery.clusterId,
@@ -1840,6 +1976,72 @@ ${sql
     managedLogGroup, lambdaLogging,
 ${''}\
   });
+  const sqlOutboxRelayErrorAlarm = new aws.cloudwatch.MetricAlarm(
+    '${pascalName}SqlOutboxRelayErrors',
+    {
+      alarmActions: [foundation.discovery.alertTopicArn],
+      alarmDescription: operationalAlarmDescription({
+        businessSymptom: 'Relational event delivery is failing.',
+        id: 'sql-outbox-relay-errors',
+        runbook: 'docs/runbooks/async-recovery.md',
+      }),
+      comparisonOperator: 'GreaterThanOrEqualToThreshold',
+      dimensions: { FunctionName: sqlOutbox.relay.name },
+      evaluationPeriods: 1,
+      metricName: 'Errors',
+      name: '${name}-' + $app.stage + '-sql-outbox-relay-errors',
+      namespace: 'AWS/Lambda',
+      period: 300,
+      statistic: 'Sum',
+      threshold: 1,
+      treatMissingData: 'notBreaching',
+    },
+  );
+  // The relay counts every unpublished intent; Minimum ignores healthy
+  // in-flight batches and alarms only when no run in an hour drained it.
+  const sqlOutboxBacklogAlarm = new aws.cloudwatch.MetricAlarm(
+    '${pascalName}SqlOutboxBacklog',
+    {
+      alarmActions: [foundation.discovery.alertTopicArn],
+      alarmDescription: operationalAlarmDescription({
+        businessSymptom: 'Relational event intents remain undelivered.',
+        id: 'sql-outbox-backlog',
+        runbook: 'docs/runbooks/async-recovery.md',
+      }),
+      comparisonOperator: 'GreaterThanOrEqualToThreshold',
+      dimensions: { Slice: '${name}', Stage: $app.stage },
+      evaluationPeriods: 2,
+      metricName: 'SqlOutboxBacklog',
+      name: '${name}-' + $app.stage + '-sql-outbox-backlog',
+      namespace: 'EBK/Async',
+      period: 3600,
+      statistic: 'Minimum',
+      threshold: 1,
+      treatMissingData: 'notBreaching',
+    },
+  );
+  const sqlOutboxAttemptsAlarm = new aws.cloudwatch.MetricAlarm(
+    '${pascalName}SqlOutboxAttempts',
+    {
+      alarmActions: [foundation.discovery.alertTopicArn],
+      alarmDescription: operationalAlarmDescription({
+        businessSymptom:
+          'A relational event intent is repeatedly failing delivery.',
+        id: 'sql-outbox-attempts',
+        runbook: 'docs/runbooks/async-recovery.md',
+      }),
+      comparisonOperator: 'GreaterThanOrEqualToThreshold',
+      dimensions: { Slice: '${name}', Stage: $app.stage },
+      evaluationPeriods: 1,
+      metricName: 'SqlOutboxMaxAttempts',
+      name: '${name}-' + $app.stage + '-sql-outbox-attempts',
+      namespace: 'EBK/Async',
+      period: 3600,
+      statistic: 'Maximum',
+      threshold: 10,
+      treatMissingData: 'notBreaching',
+    },
+  );
 `
             : ''}\
   const asyncInboxCleanup = new sst.aws.Function(
@@ -2749,7 +2951,8 @@ ${''}\
     {
       alarmActions: [foundation.discovery.alertTopicArn],
       alarmDescription: operationalAlarmDescription({
-        businessSymptom: 'The transactional outbox relay is failing.',
+        businessSymptom:
+          'The isolated performance-sink outbox relay is failing.',
         id: 'outbox-relay-errors',
         runbook: 'docs/runbooks/async-recovery.md',
       }),
@@ -2834,27 +3037,6 @@ ${''}\
         treatMissingData: 'breaching',
       },
     );
-  const outboxBacklogAlarm = new aws.cloudwatch.MetricAlarm(
-    '${pascalName}OutboxBacklog',
-    {
-      alarmActions: [foundation.discovery.alertTopicArn],
-      alarmDescription: operationalAlarmDescription({
-        businessSymptom: 'Transactional outbox work is accumulating.',
-        id: 'outbox-backlog',
-        runbook: 'docs/runbooks/async-recovery.md',
-      }),
-      comparisonOperator: 'GreaterThanOrEqualToThreshold',
-      dimensions: { Slice: '${name}', Stage: $app.stage },
-      evaluationPeriods: 2,
-      metricName: 'OutboxBacklog',
-      name: '${name}-' + $app.stage + '-outbox-backlog',
-      namespace: 'EBK/Async',
-      period: 300,
-      statistic: 'Maximum',
-      threshold: 1,
-      treatMissingData: 'notBreaching',
-    },
-  );
   const inboxCleanupBacklogAlarm = new aws.cloudwatch.MetricAlarm(
     '${pascalName}InboxCleanupBacklog',
     {
@@ -3000,7 +3182,7 @@ ${''}\
     performanceLoadCleanupBacklogAlarm,
     performanceLoadCleanupDeadlineAlarm,
     asyncOutboxRelay,
-${sql ? '    sqlOutbox,\n' : ''}\
+${sql ? '    sqlOutbox,\n    sqlOutboxAttemptsAlarm,\n    sqlOutboxBacklogAlarm,\n    sqlOutboxRelayErrorAlarm,\n' : ''}\
     asyncRecoveryLedger,
     asyncScheduleTarget,
     asyncTargetDlq,
@@ -3027,6 +3209,11 @@ ${'    keyValue,\n'}\
     eventQueue,
     machineClientProvisioner,
     migrationControls,
+    eventDeliveryAttemptsAlarm,
+    eventIntentRejectedAlarm,
+    keyValueReconciliationErrorAlarm,
+    keyValueStreamDeliveryErrorAlarm,
+    keyValueStreamIteratorAgeAlarm,
     outboxBacklogAlarm,
     outboxRelayAlarm,
     outboxRelayRule,

package/src/generators/slice/built-in-plan-key-value-delivery.js

@@ -5,11 +5,13 @@ exports.renderKeyValueDeliverySource = renderKeyValueDeliverySource;
 function renderKeyValueDeliverySource(name) {
     return `import { EventBridgeClient, PutEventsCommand } from '@aws-sdk/client-eventbridge';
 import { DYNAMO_OUTBOX_SHARDS } from '@empire-builder-kit/runtime/key-value';
+import { createMetricEmitter } from '@empire-builder-kit/runtime/observability';
 import type { PublishableEventEntry } from '@empire-builder-kit/runtime/events';
 import { dataStore } from '../data/key-value.js';
 import { reconcileRecordDeletions } from '../data/record-deletion.js';
 
 const client = new EventBridgeClient({ maxAttempts: 2 });
+const metrics = createMetricEmitter();
 type Store = ReturnType<typeof dataStore>;
 type Cursor = NonNullable<Awaited<ReturnType<Store['listPending']>>['cursor']>;
 // Only the provider fields consumed below are part of this handler's input.
@@ -77,9 +79,16 @@ export async function stream(event: DynamoDBStreamEvent) {
   return { batchItemFailures };
 }
 
+// EMF gauges are observational. A lost metric returns undefined and never
+// changes delivery, checkpoints, or the reconciliation outcome.
+function observe(metricName: string, value: number) {
+  metrics.emit({ namespace: 'EBK/Async', metricName, unit: 'Count', value,
+    dimensions: { slice: '${name}', stage: process.env.EBK_STAGE ?? 'local' } });
+}
+
 export async function reconcile() {
   const store = dataStore();
-  const summary = { kind: 'ebk-key-value-reconciliation', delivered: 0, deliveryPending: 0, retryRequired: 0, rejected: 0, shards: 0 };
+  const summary = { kind: 'ebk-key-value-reconciliation', delivered: 0, deliveryPending: 0, retryRequired: 0, rejected: 0, maxAttempts: 0, shards: 0 };
   // One page per shard per run bounds work and prevents a poison-filled first
   // page from starving the remainder forever. Wrap after the final page.
   for (let shard = 0; shard < DYNAMO_OUTBOX_SHARDS; shard++) {
@@ -94,6 +103,7 @@ export async function reconcile() {
       kind: 'ebk-key-value-invalid-intents', shard, records: page.rejected,
     }));
     for (const intent of page.events) {
+      summary.maxAttempts = Math.max(summary.maxAttempts, intent.attempts);
       try {
         const result = await store.dispatch(intent.id, publish);
         if (delivered(result)) summary.delivered++;
@@ -110,6 +120,13 @@ export async function reconcile() {
     }] });
     summary.shards++;
   }
+  // One page per shard is a bounded sample, not a global count: each gauge is a
+  // lower bound for this run. Intents leased by another owner are in flight and
+  // excluded; a dead owner's intent is due again after its lease expires.
+  observe('OutboxBacklog', summary.retryRequired + summary.rejected);
+  observe('EventReconciliationRetryRequired', summary.retryRequired);
+  observe('EventReconciliationRejected', summary.rejected);
+  observe('EventDeliveryMaxAttempts', summary.maxAttempts);
   const deletionCompletion = await reconcileRecordDeletions();
   console.info(JSON.stringify({ ...summary, deletionCompletion }));
   return { ...summary, deletionCompletion,

package/src/generators/slice/built-in-plan-metadata-files.js

@@ -59,7 +59,10 @@ Run \`ebk dev up ${name}\`. The framework reuses this worktree's persisted opaqu
 task identity, starts the selected Slice and declared dependencies, and runs
 SST live mode through the workspace-pinned Nx task. Native persistence uses the
 task's real AWS DynamoDB resources; Docker PostgreSQL starts only for declared
-relational workloads. A caller cannot supply or persist a stage override. The
+relational workloads. \`verify\` runs \`test-integration\`, which starts DynamoDB
+Local (and PostgreSQL when declared) in disposable Docker containers, so
+verification needs a running Docker engine. A caller cannot supply or persist a
+stage override. The
 fleet supervisor allocates the internal Next.js port and reports the HTTPS
 readiness URL. Development accepts a dirty tree and requires no production
 confirmation or release artifact.

package/src/generators/slice/built-in-plan-reliability-files.d.ts

@@ -1,2 +1,2 @@
 /** @internal */
-export declare function renderProductReliabilityFiles(name: string, pascalName: string, reviewedAt: string): Readonly<Record<string, string>>;
+export declare function renderProductReliabilityFiles(name: string, pascalName: string, reviewedAt: string, sql?: boolean): Readonly<Record<string, string>>;

package/src/generators/slice/built-in-plan-reliability-files.js

@@ -127,7 +127,7 @@ function reliabilityContract(name, reviewedAt) {
         slice: name,
     };
 }
-function alarmRegistrations(name) {
+function alarmRegistrations(name, sql) {
     const owner = `${name}-product`;
     const route = `${name}-on-call`;
     const common = {
@@ -243,8 +243,8 @@ function alarmRegistrations(name) {
         },
         {
             ...common,
-            businessSymptom: 'A generated Function has a terminal invocation error.',
-            condition: 'AWS/Lambda reports one or more errors for the exact generated cleanup Function.',
+            businessSymptom: 'The generated confidential-export cleanup Function has a terminal invocation error.',
+            condition: 'AWS/Lambda reports one or more errors in one minute for the exact generated data-export cleanup Function. Event-delivery Functions have their own error alarms.',
             dependencyAlarmIds: [],
             id: 'function-failure',
             runbookRef: 'runbook:api-recovery',
@@ -306,8 +306,8 @@ function alarmRegistrations(name) {
         },
         {
             ...common,
-            businessSymptom: 'The transactional outbox relay is failing.',
-            condition: 'The generated outbox relay reports one or more errors.',
+            businessSymptom: 'The isolated performance-sink outbox relay is failing.',
+            condition: 'AWS/Lambda reports one or more errors in five minutes for the generated async/outbox-relay Function, which drains only the isolated performance sink. Domain-event delivery is the key-value stream and reconciliation.',
             dependencyAlarmIds: [],
             id: 'outbox-relay-errors',
             runbookRef: 'runbook:async-recovery',
@@ -324,13 +324,62 @@ function alarmRegistrations(name) {
         },
         {
             ...common,
-            businessSymptom: 'Transactional outbox work is accumulating.',
-            condition: 'The transactional outbox backlog remains nonempty.',
-            dependencyAlarmIds: ['outbox-relay-errors'],
+            businessSymptom: 'The key-value stream delivery Function failed an invocation.',
+            condition: 'AWS/Lambda reports one or more errors in five minutes for the exact key-value stream delivery Function: the stream identity or record shape was rejected, or the handler crashed. Per-record publish failures are partial batch failures, not Lambda errors; they surface through key-value-stream-iterator-age while retries hold the shard back, and through outbox-backlog and event-delivery-attempts after the mapping discards them.',
+            dependencyAlarmIds: [],
+            id: 'key-value-stream-delivery-errors',
+            runbookRef: 'runbook:async-recovery',
+            severity: 'high',
+        },
+        {
+            ...common,
+            businessSymptom: 'Prompt domain-event delivery from the key-value stream is falling behind.',
+            condition: 'AWS/Lambda reports a stream iterator age of at least ten minutes for the exact key-value stream delivery Function. The mapping discards records older than one hour; reconciliation then recovers their retained intents.',
+            dependencyAlarmIds: ['key-value-stream-delivery-errors'],
+            id: 'key-value-stream-iterator-age',
+            runbookRef: 'runbook:async-recovery',
+            severity: 'high',
+        },
+        {
+            ...common,
+            businessSymptom: 'Durable domain-event recovery is failing.',
+            condition: 'AWS/Lambda reports one or more errors in five minutes for the exact key-value reconciliation Function. Partial runs do not error; the backlog, rejected-intent, and attempts alarms report them.',
+            dependencyAlarmIds: [],
+            id: 'key-value-reconciliation-errors',
+            runbookRef: 'runbook:async-recovery',
+            severity: 'high',
+        },
+        {
+            ...common,
+            businessSymptom: 'Durable domain-event intents remain undelivered.',
+            condition: 'In two consecutive hours, key-value reconciliation reported one or more due intents it could not deliver (retry-required plus rejected). Each run reads at most one 25-intent page per shard, so the value is a lower bound, not a global count, and excludes intents leased by another delivery owner.',
+            dependencyAlarmIds: [
+                'key-value-reconciliation-errors',
+                'key-value-stream-delivery-errors',
+            ],
+            expectedFalseNegativeRisk: 'Intents beyond the sampled pages, or behind a paused outbox, are not counted until a later run reaches them. When the configured reconciliation cadence is slower than one hour, some hourly periods have no data and the two-period condition may never be met.',
             id: 'outbox-backlog',
             runbookRef: 'runbook:async-recovery',
             severity: 'high',
         },
+        {
+            ...common,
+            businessSymptom: 'A retained domain-event intent cannot be delivered.',
+            condition: 'Within an hour, key-value reconciliation reported one or more retained intents that fail validation. Rejected intents are never dispatched or deleted, so this recurs until an operator investigates.',
+            dependencyAlarmIds: [],
+            id: 'event-intent-rejected',
+            runbookRef: 'runbook:async-recovery',
+            severity: 'high',
+        },
+        {
+            ...common,
+            businessSymptom: 'A domain-event intent is repeatedly failing delivery.',
+            condition: 'Within an hour, key-value reconciliation observed a due intent with at least ten delivery leases (attempts). Attempts are reported only; no intent is capped, dropped, or quarantined.',
+            dependencyAlarmIds: ['key-value-reconciliation-errors'],
+            id: 'event-delivery-attempts',
+            runbookRef: 'runbook:async-recovery',
+            severity: 'high',
+        },
         {
             ...common,
             businessSymptom: 'Terminal asynchronous inbox records are accumulating.',
@@ -340,6 +389,38 @@ function alarmRegistrations(name) {
             runbookRef: 'runbook:async-recovery',
             severity: 'high',
         },
+        ...(sql
+            ? [
+                {
+                    ...common,
+                    businessSymptom: 'Relational event delivery is failing.',
+                    condition: 'AWS/Lambda reports one or more errors in five minutes for the exact SQL outbox relay Function, which fails every run that retains a failed delivery.',
+                    dependencyAlarmIds: [],
+                    id: 'sql-outbox-relay-errors',
+                    runbookRef: 'runbook:async-recovery',
+                    severity: 'high',
+                },
+                {
+                    ...common,
+                    businessSymptom: 'Relational event intents remain undelivered.',
+                    condition: 'Every SQL relay run in two consecutive hours ended with one or more unpublished intents (Minimum of a global count). A paused relay reports nothing.',
+                    dependencyAlarmIds: ['sql-outbox-relay-errors'],
+                    expectedFalseNegativeRisk: 'A paused relay reports nothing. When the configured sqlOutboxRecovery cadence is slower than one hour and no notification wakes the relay, some hourly periods have no data and the two-period condition may never be met.',
+                    id: 'sql-outbox-backlog',
+                    runbookRef: 'runbook:async-recovery',
+                    severity: 'high',
+                },
+                {
+                    ...common,
+                    businessSymptom: 'A relational event intent is repeatedly failing delivery.',
+                    condition: 'Within an hour, an unpublished SQL intent reached at least ten relay attempts. Attempts are reported only; no intent is capped or dropped.',
+                    dependencyAlarmIds: ['sql-outbox-relay-errors'],
+                    id: 'sql-outbox-attempts',
+                    runbookRef: 'runbook:async-recovery',
+                    severity: 'high',
+                },
+            ]
+            : []),
     ];
     return registrations.map((registration) => (0, reliability_1.validateAlarmRegistrationMetadata)(registration));
 }
@@ -514,7 +595,7 @@ function reliabilityExercises(name) {
         },
     ];
 }
-function deployedAlarmResources(name) {
+function deployedAlarmResources(name, sql) {
     const primary = {
         actions: 'primary-route',
         comparisonOperator: 'GreaterThanOrEqualToThreshold',
@@ -636,7 +717,7 @@ function deployedAlarmResources(name) {
                 Slice: name,
                 Stage: '{stage}',
             },
-            evaluationPeriods: 1,
+            evaluationPeriods: 2,
             id: 'performance-load-cleanup-backlog',
             metricName: 'PerformanceLoadCleanupBacklog',
             name: `${name}-{stage}-performance-load-cleanup-backlog`,
@@ -839,6 +920,51 @@ function deployedAlarmResources(name) {
             threshold: 1,
             treatMissingData: 'notBreaching',
         },
+        {
+            ...primary,
+            dimensionNames: ['FunctionName'],
+            evaluationPeriods: 1,
+            id: 'key-value-stream-delivery-errors',
+            metricName: 'Errors',
+            name: `${name}-{stage}-key-value-stream-delivery-errors`,
+            namespace: 'AWS/Lambda',
+            period: 300,
+            runbook: 'docs/runbooks/async-recovery.md',
+            signal: 'metric',
+            statistic: 'Sum',
+            threshold: 1,
+            treatMissingData: 'notBreaching',
+        },
+        {
+            ...primary,
+            dimensionNames: ['FunctionName'],
+            evaluationPeriods: 1,
+            id: 'key-value-stream-iterator-age',
+            metricName: 'IteratorAge',
+            name: `${name}-{stage}-key-value-stream-iterator-age`,
+            namespace: 'AWS/Lambda',
+            period: 300,
+            runbook: 'docs/runbooks/async-recovery.md',
+            signal: 'metric',
+            statistic: 'Maximum',
+            threshold: 600_000,
+            treatMissingData: 'notBreaching',
+        },
+        {
+            ...primary,
+            dimensionNames: ['FunctionName'],
+            evaluationPeriods: 1,
+            id: 'key-value-reconciliation-errors',
+            metricName: 'Errors',
+            name: `${name}-{stage}-key-value-reconciliation-errors`,
+            namespace: 'AWS/Lambda',
+            period: 300,
+            runbook: 'docs/runbooks/async-recovery.md',
+            signal: 'metric',
+            statistic: 'Sum',
+            threshold: 1,
+            treatMissingData: 'notBreaching',
+        },
         {
             ...primary,
             dimensionNames: ['Slice', 'Stage'],
@@ -848,13 +974,96 @@ function deployedAlarmResources(name) {
             metricName: 'OutboxBacklog',
             name: `${name}-{stage}-outbox-backlog`,
             namespace: 'EBK/Async',
-            period: 300,
+            period: 3600,
             runbook: 'docs/runbooks/async-recovery.md',
             signal: 'metric',
             statistic: 'Maximum',
             threshold: 1,
             treatMissingData: 'notBreaching',
         },
+        {
+            ...primary,
+            dimensionNames: ['Slice', 'Stage'],
+            dimensionValues: { Slice: name, Stage: '{stage}' },
+            evaluationPeriods: 1,
+            id: 'event-intent-rejected',
+            metricName: 'EventReconciliationRejected',
+            name: `${name}-{stage}-event-intent-rejected`,
+            namespace: 'EBK/Async',
+            period: 3600,
+            runbook: 'docs/runbooks/async-recovery.md',
+            signal: 'metric',
+            statistic: 'Maximum',
+            threshold: 1,
+            treatMissingData: 'notBreaching',
+        },
+        {
+            ...primary,
+            dimensionNames: ['Slice', 'Stage'],
+            dimensionValues: { Slice: name, Stage: '{stage}' },
+            evaluationPeriods: 1,
+            id: 'event-delivery-attempts',
+            metricName: 'EventDeliveryMaxAttempts',
+            name: `${name}-{stage}-event-delivery-attempts`,
+            namespace: 'EBK/Async',
+            period: 3600,
+            runbook: 'docs/runbooks/async-recovery.md',
+            signal: 'metric',
+            statistic: 'Maximum',
+            threshold: 10,
+            treatMissingData: 'notBreaching',
+        },
+        ...(sql
+            ? [
+                {
+                    ...primary,
+                    dimensionNames: ['FunctionName'],
+                    evaluationPeriods: 1,
+                    id: 'sql-outbox-relay-errors',
+                    metricName: 'Errors',
+                    name: `${name}-{stage}-sql-outbox-relay-errors`,
+                    namespace: 'AWS/Lambda',
+                    period: 300,
+                    runbook: 'docs/runbooks/async-recovery.md',
+                    signal: 'metric',
+                    statistic: 'Sum',
+                    threshold: 1,
+                    treatMissingData: 'notBreaching',
+                },
+                {
+                    ...primary,
+                    dimensionNames: ['Slice', 'Stage'],
+                    dimensionValues: { Slice: name, Stage: '{stage}' },
+                    evaluationPeriods: 2,
+                    id: 'sql-outbox-backlog',
+                    metricName: 'SqlOutboxBacklog',
+                    name: `${name}-{stage}-sql-outbox-backlog`,
+                    namespace: 'EBK/Async',
+                    period: 3600,
+                    runbook: 'docs/runbooks/async-recovery.md',
+                    signal: 'metric',
+                    statistic: 'Minimum',
+                    threshold: 1,
+                    treatMissingData: 'notBreaching',
+                },
+                {
+                    ...primary,
+                    dimensionNames: ['Slice', 'Stage'],
+                    dimensionValues: { Slice: name, Stage: '{stage}' },
+                    evaluationPeriods: 1,
+                    id: 'sql-outbox-attempts',
+                    metricName: 'SqlOutboxMaxAttempts',
+                    name: `${name}-{stage}-sql-outbox-attempts`,
+                    namespace: 'EBK/Async',
+                    period: 3600,
+                    runbook: 'docs/runbooks/async-recovery.md',
+                    signal: 'metric',
+                    statistic: 'Maximum',
+                    threshold: 10,
+                    treatMissingData: 'notBreaching',
+                },
+            ]
+            : []),
         {
             ...primary,
             dimensionNames: ['Component', 'Feature', 'Operation', 'Slice', 'Stage'],
@@ -898,9 +1107,9 @@ function deployedAlarmResources(name) {
     ];
 }
 /** @internal */
-function renderProductReliabilityFiles(name, pascalName, reviewedAt) {
+function renderProductReliabilityFiles(name, pascalName, reviewedAt, sql = false) {
     const contract = reliabilityContract(name, reviewedAt);
-    const alarms = alarmRegistrations(name);
+    const alarms = alarmRegistrations(name, sql);
     const exercises = reliabilityExercises(name);
     return {
         'docs/reliability/slo.json': (0, built_in_plan_shared_js_1.json)(contract),
@@ -917,7 +1126,7 @@ CloudWatch resources do not rewrite it.
         'operations/reliability.json': (0, built_in_plan_shared_js_1.json)({
             alarmRegistrations: alarms,
             consoleIndependent: true,
-            deployedAlarmResources: deployedAlarmResources(name),
+            deployedAlarmResources: deployedAlarmResources(name, sql),
             diagnostic: {
                 metadataSources: [
                     'alarm-state',
@@ -1314,7 +1523,8 @@ ${`      evaluationPeriods: 1,
     {
       alarmActions: [input.alertTopicArn],
       alarmDescription: alarmDescription({
-        businessSymptom: 'A generated Function has a terminal invocation error.',
+        businessSymptom:
+          'The generated confidential-export cleanup Function has a terminal invocation error.',
         id: 'function-failure',
         owner: '${name}-product',
         route: '${name}-on-call',

package/src/generators/slice/built-in-plan-sql-events.js

@@ -99,7 +99,7 @@ export function createSqlEventOutbox(dependencies: Dependencies) {
   }
   async function reconcile(context: Context) {
     const scope = dependencies.scope(); validateScope(scope);
-    if (!scope.enabled) return { outcome: 'paused', published: 0, failed: 0, remaining: null };
+    if (!scope.enabled) return { outcome: 'paused', published: 0, failed: 0, remaining: null, maxAttempts: null };
     const result = await dependencies.withDatabase(context, 'async-outbox-relay', async (client) => {
       await client.query('BEGIN');
       try {
@@ -124,9 +124,13 @@ export function createSqlEventOutbox(dependencies: Dependencies) {
             published_at = now(), expires_at = now() + interval '30 days' WHERE idempotency_key = $1 AND published_at IS NULL\`, [row.id]);
           published++;
         }
-        const count = await client.query<{ count: string }>('SELECT count(*)::text AS count FROM data_lifecycle_outbox WHERE published_at IS NULL');
+        const count = await client.query<{ count: string; max_attempts: string }>(\`SELECT count(*)::text AS count,
+          coalesce(max(attempts), 0)::text AS max_attempts FROM data_lifecycle_outbox WHERE published_at IS NULL\`);
         const remaining = Number(count.rows[0]?.count);
         if (!Number.isSafeInteger(remaining) || remaining < 0) throw new Error('SQL outbox backlog is invalid.');
+        // Reporting attempts is telemetry only; it never expires or drops an intent.
+        const attempts = Number(count.rows[0]?.max_attempts);
+        const maxAttempts = Number.isSafeInteger(attempts) && attempts >= 0 ? attempts : null;
         // Bounded control-metadata cleanup; pending intents never expire. Use
         // row locks so overlapping relays cannot delete another worker's batch.
         const pruned = await client.query(\`DELETE FROM data_lifecycle_outbox WHERE idempotency_key IN (
@@ -135,7 +139,7 @@ export function createSqlEventOutbox(dependencies: Dependencies) {
           ORDER BY expires_at, idempotency_key LIMIT 100 FOR UPDATE SKIP LOCKED
         )\`);
         await client.query('COMMIT');
-        return { outcome: failed ? 'retry-required' : remaining ? 'pending' : 'converged', published, failed, remaining, pruned: pruned.rowCount ?? 0 };
+        return { outcome: failed ? 'retry-required' : remaining ? 'pending' : 'converged', published, failed, remaining, maxAttempts, pruned: pruned.rowCount ?? 0 };
       } catch (error) { await client.query('ROLLBACK').catch(() => undefined); throw error; }
     });
     // Continue a healthy backlog promptly; failures remain retryable and recover hourly.
@@ -162,12 +166,21 @@ export const sqlEventOutbox = createSqlEventOutbox({
 });
 `,
         'async/sql-outbox-relay.ts': `import { randomUUID } from 'node:crypto';
+import { createMetricEmitter } from '@empire-builder-kit/runtime/observability';
 import { sqlEventOutbox } from '../database/sql-events.js';
 
+const metrics = createMetricEmitter();
+
 export async function handler(event: { id?: string } = {}) {
   const started = Date.now();
   const result = await sqlEventOutbox.reconcile({ correlationId: event.id ?? randomUUID() });
   console.info(JSON.stringify({ kind: 'ebk-sql-outbox-relay', ...result, durationMilliseconds: Date.now() - started }));
+  // Observational gauges precede the failure signal. A paused relay reads
+  // nothing and reports no backlog; a lost metric never changes delivery.
+  for (const [metricName, value] of [['SqlOutboxBacklog', result.remaining], ['SqlOutboxMaxAttempts', result.maxAttempts]] as const) {
+    if (value !== null) metrics.emit({ namespace: 'EBK/Async', metricName, unit: 'Count', value,
+      dimensions: { slice: '${name}', stage: process.env.EBK_STAGE ?? 'local' } });
+  }
   if (result.failed) throw new Error('SQL outbox retains failed deliveries; inspect content-free diagnostics and retry.');
   return result;
 }
@@ -240,7 +253,13 @@ Use stable event IDs and caller idempotency for repeat requests.
 Only after commit does a content-free notification wake the SQL relay. A missed
 notification does not undo committed data: the result reports reconciliation-
 required, and the retained outbox is recovered on the configured schedule.
-Pending intents are not discarded on age alone. Published intents retain thirty
+Pending intents are not discarded on age alone. A relay run that retains a
+failed delivery fails its invocation (\`sql-outbox-relay-errors\`). Each enabled
+run reports the global unpublished count as \`SqlOutboxBacklog\` and the highest
+unpublished attempt count as \`SqlOutboxMaxAttempts\`; \`sql-outbox-backlog\`
+alarms when every run in two consecutive hours ended with a backlog and
+\`sql-outbox-attempts\` at ten attempts. These are observations, never delivery
+rules. See docs/runbooks/async-recovery.md. Published intents retain thirty
 days of replay metadata. Each enabled SQL relay invocation prunes at most one
 hundred expired published intents in the same transaction. Pausing SQL recovery
 also defers this cleanup. Pending intents are never pruned. Delivery is at least

package/src/generators/slice/built-in-plan.js

@@ -149,7 +149,7 @@ function renderSliceFiles(name, scope, runtimeVersion, ownership, dataPolicyRevi
         ['API', (0, built_in_plan_api_files_js_1.renderProductApiFiles)(name, pascalName, scope.slice(1), sql)],
         [
             'reliability and incident response',
-            (0, built_in_plan_reliability_files_js_1.renderProductReliabilityFiles)(name, pascalName, dataPolicyReviewedAt),
+            (0, built_in_plan_reliability_files_js_1.renderProductReliabilityFiles)(name, pascalName, dataPolicyReviewedAt, sql),
         ],
         [
             'product infrastructure',

package/src/generators/upgrade/upgrade.js

@@ -351,7 +351,9 @@ const SEEDED_RECONCILIATION = {
             'alarm-registration',
             'database-capacity-dynamodb-table-and-pending-events-throttle-metrics',
             'diagnostic-authority',
+            'event-delivery-alarms-and-hourly-outbox-backlog-period',
             'incident-remediation',
+            'performance-load-cleanup-backlog-two-evaluation-periods',
         ],
     },
     'operations/release.json': {