@empire-builder-kit/nx 1.0.0-rc.4 → 1.0.0-rc.41

package/src/generators/slice/built-in-plan-reliability-files.js

@@ -127,7 +127,7 @@ function reliabilityContract(name, reviewedAt) {
         slice: name,
     };
 }
-function alarmRegistrations(name) {
+function alarmRegistrations(name, sql) {
     const owner = `${name}-product`;
     const route = `${name}-on-call`;
     const common = {
@@ -186,8 +186,8 @@ function alarmRegistrations(name) {
         },
         {
             ...common,
-            businessSymptom: 'The shared database is approaching its capacity bound.',
-            condition: 'Aurora capacity exceeds the reviewed warning threshold.',
+            businessSymptom: 'Slice data access or durable event delivery is being throttled.',
+            condition: 'The Slice DynamoDB table or PendingEvents index reports read or write throttling.',
             dependencyAlarmIds: [],
             id: 'database-capacity',
             runbookRef: 'runbook:database-recovery',
@@ -243,8 +243,8 @@ function alarmRegistrations(name) {
         },
         {
             ...common,
-            businessSymptom: 'A generated Function has a terminal invocation error.',
-            condition: 'AWS/Lambda reports one or more errors for the exact generated cleanup Function.',
+            businessSymptom: 'The generated confidential-export cleanup Function has a terminal invocation error.',
+            condition: 'AWS/Lambda reports one or more errors in one minute for the exact generated data-export cleanup Function. Event-delivery Functions have their own error alarms.',
             dependencyAlarmIds: [],
             id: 'function-failure',
             runbookRef: 'runbook:api-recovery',
@@ -306,8 +306,8 @@ function alarmRegistrations(name) {
         },
         {
             ...common,
-            businessSymptom: 'The transactional outbox relay is failing.',
-            condition: 'The generated outbox relay reports one or more errors.',
+            businessSymptom: 'The isolated performance-sink outbox relay is failing.',
+            condition: 'AWS/Lambda reports one or more errors in five minutes for the generated async/outbox-relay Function, which drains only the isolated performance sink. Domain-event delivery is the key-value stream and reconciliation.',
             dependencyAlarmIds: [],
             id: 'outbox-relay-errors',
             runbookRef: 'runbook:async-recovery',
@@ -324,13 +324,62 @@ function alarmRegistrations(name) {
         },
         {
             ...common,
-            businessSymptom: 'Transactional outbox work is accumulating.',
-            condition: 'The transactional outbox backlog remains nonempty.',
-            dependencyAlarmIds: ['outbox-relay-errors'],
+            businessSymptom: 'The key-value stream delivery Function failed an invocation.',
+            condition: 'AWS/Lambda reports one or more errors in five minutes for the exact key-value stream delivery Function: the stream identity or record shape was rejected, or the handler crashed. Per-record publish failures are partial batch failures, not Lambda errors; they surface through key-value-stream-iterator-age while retries hold the shard back, and through outbox-backlog and event-delivery-attempts after the mapping discards them.',
+            dependencyAlarmIds: [],
+            id: 'key-value-stream-delivery-errors',
+            runbookRef: 'runbook:async-recovery',
+            severity: 'high',
+        },
+        {
+            ...common,
+            businessSymptom: 'Prompt domain-event delivery from the key-value stream is falling behind.',
+            condition: 'AWS/Lambda reports a stream iterator age of at least ten minutes for the exact key-value stream delivery Function. The mapping discards records older than one hour; reconciliation then recovers their retained intents.',
+            dependencyAlarmIds: ['key-value-stream-delivery-errors'],
+            id: 'key-value-stream-iterator-age',
+            runbookRef: 'runbook:async-recovery',
+            severity: 'high',
+        },
+        {
+            ...common,
+            businessSymptom: 'Durable domain-event recovery is failing.',
+            condition: 'AWS/Lambda reports one or more errors in five minutes for the exact key-value reconciliation Function. Partial runs do not error; the backlog, rejected-intent, and attempts alarms report them.',
+            dependencyAlarmIds: [],
+            id: 'key-value-reconciliation-errors',
+            runbookRef: 'runbook:async-recovery',
+            severity: 'high',
+        },
+        {
+            ...common,
+            businessSymptom: 'Durable domain-event intents remain undelivered.',
+            condition: 'In two consecutive hours, key-value reconciliation reported one or more due intents it could not deliver (retry-required plus rejected). Each run reads at most one 25-intent page per shard, so the value is a lower bound, not a global count, and excludes intents leased by another delivery owner.',
+            dependencyAlarmIds: [
+                'key-value-reconciliation-errors',
+                'key-value-stream-delivery-errors',
+            ],
+            expectedFalseNegativeRisk: 'Intents beyond the sampled pages, or behind a paused outbox, are not counted until a later run reaches them. When the configured reconciliation cadence is slower than one hour, some hourly periods have no data and the two-period condition may never be met.',
             id: 'outbox-backlog',
             runbookRef: 'runbook:async-recovery',
             severity: 'high',
         },
+        {
+            ...common,
+            businessSymptom: 'A retained domain-event intent cannot be delivered.',
+            condition: 'Within an hour, key-value reconciliation reported one or more retained intents that fail validation. Rejected intents are never dispatched or deleted, so this recurs until an operator investigates.',
+            dependencyAlarmIds: [],
+            id: 'event-intent-rejected',
+            runbookRef: 'runbook:async-recovery',
+            severity: 'high',
+        },
+        {
+            ...common,
+            businessSymptom: 'A domain-event intent is repeatedly failing delivery.',
+            condition: 'Within an hour, key-value reconciliation observed a due intent with at least ten delivery leases (attempts). Attempts are reported only; no intent is capped, dropped, or quarantined.',
+            dependencyAlarmIds: ['key-value-reconciliation-errors'],
+            id: 'event-delivery-attempts',
+            runbookRef: 'runbook:async-recovery',
+            severity: 'high',
+        },
         {
             ...common,
             businessSymptom: 'Terminal asynchronous inbox records are accumulating.',
@@ -340,6 +389,38 @@ function alarmRegistrations(name) {
             runbookRef: 'runbook:async-recovery',
             severity: 'high',
         },
+        ...(sql
+            ? [
+                {
+                    ...common,
+                    businessSymptom: 'Relational event delivery is failing.',
+                    condition: 'AWS/Lambda reports one or more errors in five minutes for the exact SQL outbox relay Function, which fails every run that retains a failed delivery.',
+                    dependencyAlarmIds: [],
+                    id: 'sql-outbox-relay-errors',
+                    runbookRef: 'runbook:async-recovery',
+                    severity: 'high',
+                },
+                {
+                    ...common,
+                    businessSymptom: 'Relational event intents remain undelivered.',
+                    condition: 'Every SQL relay run in two consecutive hours ended with one or more unpublished intents (Minimum of a global count). A paused relay reports nothing.',
+                    dependencyAlarmIds: ['sql-outbox-relay-errors'],
+                    expectedFalseNegativeRisk: 'A paused relay reports nothing. When the configured sqlOutboxRecovery cadence is slower than one hour and no notification wakes the relay, some hourly periods have no data and the two-period condition may never be met.',
+                    id: 'sql-outbox-backlog',
+                    runbookRef: 'runbook:async-recovery',
+                    severity: 'high',
+                },
+                {
+                    ...common,
+                    businessSymptom: 'A relational event intent is repeatedly failing delivery.',
+                    condition: 'Within an hour, an unpublished SQL intent reached at least ten relay attempts. Attempts are reported only; no intent is capped or dropped.',
+                    dependencyAlarmIds: ['sql-outbox-relay-errors'],
+                    id: 'sql-outbox-attempts',
+                    runbookRef: 'runbook:async-recovery',
+                    severity: 'high',
+                },
+            ]
+            : []),
     ];
     return registrations.map((registration) => (0, reliability_1.validateAlarmRegistrationMetadata)(registration));
 }
@@ -514,7 +595,7 @@ function reliabilityExercises(name) {
         },
     ];
 }
-function deployedAlarmResources(name) {
+function deployedAlarmResources(name, sql) {
     const primary = {
         actions: 'primary-route',
         comparisonOperator: 'GreaterThanOrEqualToThreshold',
@@ -612,18 +693,19 @@ function deployedAlarmResources(name) {
         },
         {
             ...primary,
-            dimensionNames: ['DBClusterIdentifier'],
-            evaluationPeriods: 3,
+            dimensionNames: ['GlobalSecondaryIndexName', 'TableName'],
+            dimensionValues: { GlobalSecondaryIndexName: 'PendingEvents' },
+            evaluationPeriods: 1,
             id: 'database-capacity',
-            metricName: 'ServerlessDatabaseCapacity',
+            metricNames: ['ReadThrottleEvents', 'WriteThrottleEvents'],
             name: `${name}-{stage}-database-capacity`,
-            namespace: 'AWS/RDS',
-            period: 300,
+            namespace: 'AWS/DynamoDB',
+            period: 60,
             runbook: 'docs/runbooks/database-recovery.md',
-            signal: 'metric',
-            statistic: 'Maximum',
-            threshold: 3.5,
-            treatMissingData: 'missing',
+            signal: 'metric-math',
+            statistic: 'Sum',
+            threshold: 1,
+            treatMissingData: 'notBreaching',
         },
         {
             ...primary,
@@ -635,7 +717,7 @@ function deployedAlarmResources(name) {
                 Slice: name,
                 Stage: '{stage}',
             },
-            evaluationPeriods: 1,
+            evaluationPeriods: 2,
             id: 'performance-load-cleanup-backlog',
             metricName: 'PerformanceLoadCleanupBacklog',
             name: `${name}-{stage}-performance-load-cleanup-backlog`,
@@ -838,6 +920,51 @@ function deployedAlarmResources(name) {
             threshold: 1,
             treatMissingData: 'notBreaching',
         },
+        {
+            ...primary,
+            dimensionNames: ['FunctionName'],
+            evaluationPeriods: 1,
+            id: 'key-value-stream-delivery-errors',
+            metricName: 'Errors',
+            name: `${name}-{stage}-key-value-stream-delivery-errors`,
+            namespace: 'AWS/Lambda',
+            period: 300,
+            runbook: 'docs/runbooks/async-recovery.md',
+            signal: 'metric',
+            statistic: 'Sum',
+            threshold: 1,
+            treatMissingData: 'notBreaching',
+        },
+        {
+            ...primary,
+            dimensionNames: ['FunctionName'],
+            evaluationPeriods: 1,
+            id: 'key-value-stream-iterator-age',
+            metricName: 'IteratorAge',
+            name: `${name}-{stage}-key-value-stream-iterator-age`,
+            namespace: 'AWS/Lambda',
+            period: 300,
+            runbook: 'docs/runbooks/async-recovery.md',
+            signal: 'metric',
+            statistic: 'Maximum',
+            threshold: 600_000,
+            treatMissingData: 'notBreaching',
+        },
+        {
+            ...primary,
+            dimensionNames: ['FunctionName'],
+            evaluationPeriods: 1,
+            id: 'key-value-reconciliation-errors',
+            metricName: 'Errors',
+            name: `${name}-{stage}-key-value-reconciliation-errors`,
+            namespace: 'AWS/Lambda',
+            period: 300,
+            runbook: 'docs/runbooks/async-recovery.md',
+            signal: 'metric',
+            statistic: 'Sum',
+            threshold: 1,
+            treatMissingData: 'notBreaching',
+        },
         {
             ...primary,
             dimensionNames: ['Slice', 'Stage'],
@@ -847,13 +974,96 @@ function deployedAlarmResources(name) {
             metricName: 'OutboxBacklog',
             name: `${name}-{stage}-outbox-backlog`,
             namespace: 'EBK/Async',
-            period: 300,
+            period: 3600,
             runbook: 'docs/runbooks/async-recovery.md',
             signal: 'metric',
             statistic: 'Maximum',
             threshold: 1,
             treatMissingData: 'notBreaching',
         },
+        {
+            ...primary,
+            dimensionNames: ['Slice', 'Stage'],
+            dimensionValues: { Slice: name, Stage: '{stage}' },
+            evaluationPeriods: 1,
+            id: 'event-intent-rejected',
+            metricName: 'EventReconciliationRejected',
+            name: `${name}-{stage}-event-intent-rejected`,
+            namespace: 'EBK/Async',
+            period: 3600,
+            runbook: 'docs/runbooks/async-recovery.md',
+            signal: 'metric',
+            statistic: 'Maximum',
+            threshold: 1,
+            treatMissingData: 'notBreaching',
+        },
+        {
+            ...primary,
+            dimensionNames: ['Slice', 'Stage'],
+            dimensionValues: { Slice: name, Stage: '{stage}' },
+            evaluationPeriods: 1,
+            id: 'event-delivery-attempts',
+            metricName: 'EventDeliveryMaxAttempts',
+            name: `${name}-{stage}-event-delivery-attempts`,
+            namespace: 'EBK/Async',
+            period: 3600,
+            runbook: 'docs/runbooks/async-recovery.md',
+            signal: 'metric',
+            statistic: 'Maximum',
+            threshold: 10,
+            treatMissingData: 'notBreaching',
+        },
+        ...(sql
+            ? [
+                {
+                    ...primary,
+                    dimensionNames: ['FunctionName'],
+                    evaluationPeriods: 1,
+                    id: 'sql-outbox-relay-errors',
+                    metricName: 'Errors',
+                    name: `${name}-{stage}-sql-outbox-relay-errors`,
+                    namespace: 'AWS/Lambda',
+                    period: 300,
+                    runbook: 'docs/runbooks/async-recovery.md',
+                    signal: 'metric',
+                    statistic: 'Sum',
+                    threshold: 1,
+                    treatMissingData: 'notBreaching',
+                },
+                {
+                    ...primary,
+                    dimensionNames: ['Slice', 'Stage'],
+                    dimensionValues: { Slice: name, Stage: '{stage}' },
+                    evaluationPeriods: 2,
+                    id: 'sql-outbox-backlog',
+                    metricName: 'SqlOutboxBacklog',
+                    name: `${name}-{stage}-sql-outbox-backlog`,
+                    namespace: 'EBK/Async',
+                    period: 3600,
+                    runbook: 'docs/runbooks/async-recovery.md',
+                    signal: 'metric',
+                    statistic: 'Minimum',
+                    threshold: 1,
+                    treatMissingData: 'notBreaching',
+                },
+                {
+                    ...primary,
+                    dimensionNames: ['Slice', 'Stage'],
+                    dimensionValues: { Slice: name, Stage: '{stage}' },
+                    evaluationPeriods: 1,
+                    id: 'sql-outbox-attempts',
+                    metricName: 'SqlOutboxMaxAttempts',
+                    name: `${name}-{stage}-sql-outbox-attempts`,
+                    namespace: 'EBK/Async',
+                    period: 3600,
+                    runbook: 'docs/runbooks/async-recovery.md',
+                    signal: 'metric',
+                    statistic: 'Maximum',
+                    threshold: 10,
+                    treatMissingData: 'notBreaching',
+                },
+            ]
+            : []),
         {
             ...primary,
             dimensionNames: ['Component', 'Feature', 'Operation', 'Slice', 'Stage'],
@@ -897,9 +1107,9 @@ function deployedAlarmResources(name) {
     ];
 }
 /** @internal */
-function renderProductReliabilityFiles(name, pascalName, reviewedAt) {
+function renderProductReliabilityFiles(name, pascalName, reviewedAt, sql = false) {
     const contract = reliabilityContract(name, reviewedAt);
-    const alarms = alarmRegistrations(name);
+    const alarms = alarmRegistrations(name, sql);
     const exercises = reliabilityExercises(name);
     return {
         'docs/reliability/slo.json': (0, built_in_plan_shared_js_1.json)(contract),
@@ -916,7 +1126,7 @@ CloudWatch resources do not rewrite it.
         'operations/reliability.json': (0, built_in_plan_shared_js_1.json)({
             alarmRegistrations: alarms,
             consoleIndependent: true,
-            deployedAlarmResources: deployedAlarmResources(name),
+            deployedAlarmResources: deployedAlarmResources(name, sql),
             diagnostic: {
                 metadataSources: [
                     'alarm-state',
@@ -1102,7 +1312,7 @@ export interface ReliabilityInfrastructureInput {
   alertTopicArn: any;
   alertTopicName: any;
   apiId: any;
-  databaseClusterIdentifier: any;
+${'  databaseTableName: any;\n'}\
   functionName: any;
   providerQueueName: any;
 }
@@ -1280,22 +1490,32 @@ export function create${pascalName}Reliability(
     {
       alarmActions: [input.alertTopicArn],
       alarmDescription: alarmDescription({
-        businessSymptom: 'The shared database is approaching its reviewed capacity bound.',
+        businessSymptom: ${JSON.stringify('The Slice table or event-reconciliation index is throttling operations.')},
         id: 'database-capacity',
         owner: '${name}-product',
         route: '${name}-on-call',
         runbook: 'docs/runbooks/database-recovery.md',
       }),
       comparisonOperator: 'GreaterThanOrEqualToThreshold',
-      dimensions: { DBClusterIdentifier: input.databaseClusterIdentifier },
-      evaluationPeriods: 3,
-      metricName: 'ServerlessDatabaseCapacity',
+${`      evaluationPeriods: 1,
       name: '${name}-' + $app.stage + '-database-capacity',
-      namespace: 'AWS/RDS',
-      period: 300,
-      statistic: 'Maximum',
-      threshold: 3.5,
-      treatMissingData: 'missing',
+      threshold: 1,
+      treatMissingData: 'notBreaching',
+      // AWS publishes throttle-event metrics for the table and each index,
+      // unlike ThrottledRequests, which also requires an Operation dimension.
+      metricQueries: [
+        { id: 'total', expression: 'SUM(METRICS())', returnData: true },
+        ...['ReadThrottleEvents', 'WriteThrottleEvents'].flatMap((metricName, kind) =>
+          [undefined, 'PendingEvents'].map((index, scope) => ({
+            id: 'throttle' + kind + scope, returnData: false,
+            metric: { namespace: 'AWS/DynamoDB', metricName, period: 60, stat: 'Sum',
+              dimensions: { TableName: input.databaseTableName,
+                ...(index ? { GlobalSecondaryIndexName: index } : {}) },
+            },
+          })),
+        ),
+      ],
+`}\
     },
   );
   const functionFailure = new aws.cloudwatch.MetricAlarm(
@@ -1303,7 +1523,8 @@ export function create${pascalName}Reliability(
     {
       alarmActions: [input.alertTopicArn],
       alarmDescription: alarmDescription({
-        businessSymptom: 'A generated Function has a terminal invocation error.',
+        businessSymptom:
+          'The generated confidential-export cleanup Function has a terminal invocation error.',
         id: 'function-failure',
         owner: '${name}-product',
         route: '${name}-on-call',
@@ -1358,19 +1579,34 @@ export function create${pascalName}Reliability(
 `,
         'docs/runbooks/reliability-diagnostic.md': `# Reliability diagnostic
 
-The stable-v1 generated adapter supports exactly three approval-free sources:
-alarm state, sanitized SLO metric signals, and bounded ECS or Step Functions
-resource metadata. It executes read-only \`DescribeAlarms\`,
+The Standard Slice database-capacity alarm measures DynamoDB
+ReadThrottleEvents and WriteThrottleEvents for its table and PendingEvents
+index (60-second Sum, threshold 1, one evaluation, missing data notBreaching).
+It does not measure Aurora ACUs, even when a user feature declares SQL.
+During a seeded upgrade, reconcile only this database-capacity registration
+against the generated infrastructure; preserve adopter-owned owners, routes
+and other operational decisions. Then run the reliability verification target
+with --deployed against the actual stage. A registry-only check cannot prove
+that an AWS alarm was updated.
+
+The generated diagnostic adapter is defined for exactly three approval-free
+sources: alarm state, sanitized SLO metric signals, and bounded ECS or Step
+Functions resource metadata, but no supported producer currently exists for
+its inputs. In a generated workspace \`reliability-diagnostic\` therefore fails
+closed before any AWS read; do not cite it as an executed diagnostic. Its
+definition follows. It executes read-only \`DescribeAlarms\`,
 \`GetMetricData\`, \`DescribeServices\`, and \`DescribeExecution\` calls
 against one exact target and immutable deployment binding. Dashboard, generic
 health, cost, and runbook lookups are not claimed as executable diagnostic
 sources. The adapter does not accept arbitrary metric expressions or resource
 scans, read application log contents, or unmask data.
 
-Before an AWS read, the adapter requires a protected GitHub Actions context and
-selects the requested workload only through
-\`.ebk/evidence/reliability/active-bindings.json\`. That protected
-deployed-smoke index must name the current checked-out source commit and exact
+Before an AWS read, the adapter requires a protected GitHub Actions context, a
+\`.ebk/evidence/reliability/active-bindings.json\` index whose authority names
+the \`Deployed smoke\` workflow, and matching
+\`EBK_RELIABILITY_AUTHORIZED_*\` run values. Only the retired Deployed smoke and
+Reliability read workflows produced these; the native Deploy workflow does not.
+That index must name the current checked-out source commit and exact
 repository. The adapter then recomputes the binding from the exact
 deployed-smoke, active protected-release, and release-manifest artifact bytes.
 The binding also carries the environment-reviewed alarm, API, ECS, and Step
@@ -1385,7 +1621,10 @@ binding; callers cannot label it.
 
 A bounded log query requires a human approval naming the approver, invoking
 human, incident, reason, account, region, stage, Slice, exact log groups, query
-class, time window, and expiry. The approved query window is at most one hour.
+class, time window, and expiry. The invoking human is the person whose session
+runs the query; an agent may prepare the request and run the query inside that
+human's session, and the approver must be a different human. Unmasking remains
+a separate break-glass authority. The approved query window is at most one hour.
 The script returns an authorization plan; the existing protected query runner
 retains attributable evidence. Empty results never prove recovery.
 `,

package/src/generators/slice/built-in-plan-secret-runtime-files.d.ts

@@ -1,2 +1,2 @@
 /** @internal */
-export declare function renderStandardSliceSecretRuntimeFiles(name: string): Readonly<Record<string, string>>;
+export declare function renderStandardSliceSecretRuntimeFiles(name: string, workspace?: string): Readonly<Record<string, string>>;