@empire-builder-kit/nx 1.0.0-rc.44 → 1.0.0-rc.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/blueprints/external-sandbox.d.ts +1 -1
- package/src/cli/main.d.ts +11 -5
- package/src/cli/main.js +88 -29
- package/src/container-images.d.ts +1 -1
- package/src/container-images.js +1 -1
- package/src/developer-agent/generator-planning.js +10 -3
- package/src/developer-agent/operation-registry.js +1 -1
- package/src/executors/deployment-plan/deployment-plan.js +5 -1
- package/src/executors/deployment-plan/schema.json +1 -1
- package/src/executors/fleet/fleet.js +4 -0
- package/src/executors/fleet/schema.json +1 -1
- package/src/executors/github-bootstrap/github-bootstrap.d.ts +3 -0
- package/src/executors/github-bootstrap/github-bootstrap.js +28 -0
- package/src/executors/github-bootstrap/schema.json +14 -0
- package/src/executors/local-deploy/federated-credential-transport.d.ts +4 -3
- package/src/executors/local-deploy/federated-credential-transport.js +15 -5
- package/src/foundation/cost-estimate.d.ts +8 -3
- package/src/foundation/cost-estimate.js +42 -19
- package/src/foundation/index.d.ts +1 -0
- package/src/foundation/index.js +1 -0
- package/src/foundation/local-private-smoke-access.js +6 -0
- package/src/foundation/logging.d.ts +3 -1
- package/src/foundation/logging.js +8 -1
- package/src/foundation/observability.d.ts +42 -0
- package/src/foundation/observability.js +181 -0
- package/src/foundation/preflight.js +1 -1
- package/src/generators/function/function.js +5 -2
- package/src/generators/observability-infrastructure.d.ts +2 -0
- package/src/generators/observability-infrastructure.js +31 -0
- package/src/generators/observability-policy.d.ts +4 -0
- package/src/generators/observability-policy.js +23 -0
- package/src/generators/preset/dependency-overrides.d.ts +17 -6
- package/src/generators/preset/dependency-overrides.js +17 -6
- package/src/generators/preset/tool-versions.d.ts +2 -2
- package/src/generators/preset/tool-versions.js +2 -2
- package/src/generators/preset/workspace-files-adopter-skills.js +28 -17
- package/src/generators/preset/workspace-files-agent-development.d.ts +9 -0
- package/src/generators/preset/workspace-files-agent-development.js +42 -0
- package/src/generators/preset/workspace-files-thin-deploy.js +17 -4
- package/src/generators/preset/workspace-files-thin-docs.js +10 -22
- package/src/generators/preset/workspace-files.d.ts +1 -0
- package/src/generators/preset/workspace-files.js +120 -35
- package/src/generators/slice/built-in-plan-alarm-identity.d.ts +173 -0
- package/src/generators/slice/built-in-plan-alarm-identity.js +202 -0
- package/src/generators/slice/built-in-plan-api-contract-files.js +136 -0
- package/src/generators/slice/built-in-plan-data-files.js +10 -3
- package/src/generators/slice/built-in-plan-database-lifecycle-files.js +1 -1
- package/src/generators/slice/built-in-plan-event-contract-literal.d.ts +6 -0
- package/src/generators/slice/built-in-plan-event-contract-literal.js +52 -0
- package/src/generators/slice/built-in-plan-infrastructure-files.js +304 -144
- package/src/generators/slice/built-in-plan-key-value-retention-infrastructure.js +2 -2
- package/src/generators/slice/built-in-plan-metadata-files.js +37 -2
- package/src/generators/slice/built-in-plan-operations-files.js +106 -6
- package/src/generators/slice/built-in-plan-project-files.js +1 -0
- package/src/generators/slice/built-in-plan-release-files.js +2 -44
- package/src/generators/slice/built-in-plan-reliability-files.js +222 -236
- package/src/generators/slice/built-in-plan-shared.d.ts +29 -3
- package/src/generators/slice/built-in-plan-shared.js +29 -2
- package/src/generators/slice/built-in-plan-verification-files.js +0 -1
- package/src/generators/slice/built-in-plan-web-files.js +22 -6
- package/src/generators/slice/built-in-plan.js +2 -2
- package/src/generators/slice/slice.js +2 -0
- package/src/generators/upgrade/upgrade.d.ts +4 -3
- package/src/generators/upgrade/upgrade.js +139 -29
- package/src/generators/workload/frozen-container.d.ts +1 -1
- package/src/generators/workload/reliability-registration.d.ts +1 -1
- package/src/generators/workload/reliability-registration.js +9 -9
- package/src/generators/workload/workload.d.ts +1 -1
- package/src/generators/workload/workload.js +32 -10
- package/src/git-policy/github-bootstrap.d.ts +18 -0
- package/src/git-policy/github-bootstrap.js +99 -6
- package/src/local-dev/dev-command-core.d.ts +24 -3
- package/src/local-dev/dev-command-core.js +124 -12
- package/src/local-dev/errors.d.ts +19 -1
- package/src/local-dev/errors.js +18 -1
- package/src/local-dev/fleet-client.d.ts +64 -6
- package/src/local-dev/fleet-client.js +189 -15
- package/src/local-dev/fleet-process.d.ts +47 -2
- package/src/local-dev/fleet-process.js +248 -31
- package/src/local-dev/fleet-runtime.d.ts +37 -0
- package/src/local-dev/fleet-runtime.js +127 -69
- package/src/local-dev/fleet-session.d.ts +1 -0
- package/src/local-dev/fleet-session.js +83 -7
- package/src/local-dev/fleet-state.d.ts +5 -0
- package/src/local-dev/fleet-state.js +11 -2
- package/src/local-dev/fleet-supervisor-daemon.js +1 -5
- package/src/local-dev/fleet-supervisor.d.ts +7 -0
- package/src/local-dev/fleet-supervisor.js +83 -12
- package/src/local-dev/slice-runner.d.ts +19 -1
- package/src/local-dev/slice-runner.js +290 -91
- package/src/repository-policy/deployment-plan.d.ts +1 -1
- package/src/repository-policy/deployment-plan.js +16 -1
- package/src/secrets/generated-aws-control-plane.d.ts +11 -1
- package/src/secrets/generated-aws-control-plane.js +92 -29
- package/src/secrets/operator.d.ts +3 -2
- package/src/secrets/operator.js +27 -5
- package/src/supply-chain/blueprint-verification-authority.js +24 -3
- package/src/supply-chain/generated-files.js +9 -0
- package/src/supply-chain/sbom.js +74 -12
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.renderProductReliabilityFiles = renderProductReliabilityFiles;
|
|
4
4
|
const reliability_1 = require("@empire-builder-kit/runtime/reliability");
|
|
5
|
+
const deterministic_order_js_1 = require("../../deterministic-order.js");
|
|
6
|
+
const built_in_plan_alarm_identity_js_1 = require("./built-in-plan-alarm-identity.js");
|
|
5
7
|
const built_in_plan_shared_js_1 = require("./built-in-plan-shared.js");
|
|
6
8
|
function reliabilityContract(name, reviewedAt) {
|
|
7
9
|
return {
|
|
@@ -128,295 +130,214 @@ function reliabilityContract(name, reviewedAt) {
|
|
|
128
130
|
};
|
|
129
131
|
}
|
|
130
132
|
function alarmRegistrations(name, sql) {
|
|
131
|
-
const owner = `${name}-product`;
|
|
132
|
-
const route = `${name}-on-call`;
|
|
133
133
|
const common = {
|
|
134
|
-
escalationRouteKey: route,
|
|
135
134
|
expectedFalseNegativeRisk: 'Low traffic or delayed CloudWatch data can delay detection.',
|
|
136
135
|
expectedFalsePositiveRisk: 'A dependency or synthetic client can fail while the application remains healthy.',
|
|
137
136
|
missingData: 'missing',
|
|
138
|
-
owner,
|
|
139
137
|
testRef: 'evidence:reliability/failure-recovery',
|
|
140
138
|
};
|
|
139
|
+
// Symptom, owner, route and runbook come from the shared alarm identity
|
|
140
|
+
// table, as every managed emitter's description does.
|
|
141
|
+
const registered = (id) => {
|
|
142
|
+
const identity = (0, built_in_plan_alarm_identity_js_1.sliceAlarmIdentity)(name, id);
|
|
143
|
+
return {
|
|
144
|
+
...common,
|
|
145
|
+
businessSymptom: identity.businessSymptom,
|
|
146
|
+
escalationRouteKey: identity.route,
|
|
147
|
+
id,
|
|
148
|
+
owner: identity.owner,
|
|
149
|
+
runbookRef: identity.runbookRef,
|
|
150
|
+
};
|
|
151
|
+
};
|
|
141
152
|
const registrations = [
|
|
142
153
|
{
|
|
143
|
-
...
|
|
144
|
-
businessSymptom: 'Critical API operations are failing.',
|
|
154
|
+
...registered('api-fast-burn'),
|
|
145
155
|
condition: 'The API availability objective is consuming budget rapidly.',
|
|
146
156
|
dependencyAlarmIds: ['database-capacity'],
|
|
147
|
-
id: 'api-fast-burn',
|
|
148
|
-
runbookRef: 'runbook:api-recovery',
|
|
149
157
|
severity: 'critical',
|
|
150
158
|
},
|
|
151
159
|
{
|
|
152
|
-
...
|
|
153
|
-
businessSymptom: 'Critical API operations have sustained failures.',
|
|
160
|
+
...registered('api-slow-burn'),
|
|
154
161
|
condition: 'The API availability objective is consuming budget slowly.',
|
|
155
162
|
dependencyAlarmIds: ['database-capacity'],
|
|
156
|
-
id: 'api-slow-burn',
|
|
157
|
-
runbookRef: 'runbook:api-recovery',
|
|
158
163
|
severity: 'high',
|
|
159
164
|
},
|
|
160
165
|
{
|
|
161
|
-
...
|
|
162
|
-
businessSymptom: 'Server-rendered routes exceed the latency objective.',
|
|
166
|
+
...registered('web-latency-fast-burn'),
|
|
163
167
|
condition: 'Web p95 latency exceeds the reviewed threshold.',
|
|
164
168
|
dependencyAlarmIds: ['database-capacity'],
|
|
165
|
-
id: 'web-latency-fast-burn',
|
|
166
|
-
runbookRef: 'runbook:web-recovery',
|
|
167
169
|
severity: 'high',
|
|
168
170
|
},
|
|
169
171
|
{
|
|
170
|
-
...
|
|
171
|
-
businessSymptom: 'Server-rendered routes have sustained latency degradation.',
|
|
172
|
+
...registered('web-latency-slow-burn'),
|
|
172
173
|
condition: 'Web p95 latency remains above the reviewed threshold for the slow window.',
|
|
173
174
|
dependencyAlarmIds: ['database-capacity'],
|
|
174
|
-
id: 'web-latency-slow-burn',
|
|
175
|
-
runbookRef: 'runbook:web-recovery',
|
|
176
175
|
severity: 'high',
|
|
177
176
|
},
|
|
178
177
|
{
|
|
179
|
-
...
|
|
180
|
-
businessSymptom: 'Accepted provider work is missing its deadline.',
|
|
178
|
+
...registered('provider-queue-age'),
|
|
181
179
|
condition: 'Oldest provider-queue work exceeds 900 seconds.',
|
|
182
180
|
dependencyAlarmIds: [],
|
|
183
|
-
id: 'provider-queue-age',
|
|
184
|
-
runbookRef: 'runbook:async-recovery',
|
|
185
181
|
severity: 'high',
|
|
186
182
|
},
|
|
187
183
|
{
|
|
188
|
-
...
|
|
189
|
-
businessSymptom: 'Slice data access or durable event delivery is being throttled.',
|
|
184
|
+
...registered('database-capacity'),
|
|
190
185
|
condition: 'The Slice DynamoDB table or PendingEvents index reports read or write throttling.',
|
|
191
186
|
dependencyAlarmIds: [],
|
|
192
|
-
id: 'database-capacity',
|
|
193
|
-
runbookRef: 'runbook:database-recovery',
|
|
194
187
|
severity: 'high',
|
|
195
188
|
},
|
|
196
189
|
{
|
|
197
|
-
...
|
|
198
|
-
businessSymptom: 'Expired protected performance-load control records are accumulating.',
|
|
190
|
+
...registered('performance-load-cleanup-backlog'),
|
|
199
191
|
condition: 'The bounded performance-load expiry cleanup reports a nonzero backlog.',
|
|
200
192
|
dependencyAlarmIds: [],
|
|
201
|
-
id: 'performance-load-cleanup-backlog',
|
|
202
|
-
runbookRef: 'runbook:performance-capacity-cost',
|
|
203
193
|
severity: 'high',
|
|
204
194
|
},
|
|
205
195
|
{
|
|
206
|
-
...
|
|
207
|
-
businessSymptom: 'Protected performance-load residue exceeded its cleanup deadline.',
|
|
196
|
+
...registered('performance-load-cleanup-deadline'),
|
|
208
197
|
condition: 'The bounded expiry worker reports residue at least 24 hours past eligible expiry.',
|
|
209
198
|
dependencyAlarmIds: ['performance-load-cleanup-backlog'],
|
|
210
|
-
id: 'performance-load-cleanup-deadline',
|
|
211
|
-
runbookRef: 'runbook:performance-capacity-cost',
|
|
212
199
|
severity: 'critical',
|
|
213
200
|
},
|
|
214
201
|
{
|
|
215
|
-
...
|
|
216
|
-
businessSymptom: 'The confidential-export cleanup invocation is failing.',
|
|
202
|
+
...registered('data-export-cleanup-failure'),
|
|
217
203
|
condition: 'AWS/Lambda reports an error for the exact generated confidential-export cleanup Function.',
|
|
218
204
|
dependencyAlarmIds: [],
|
|
219
|
-
id: 'data-export-cleanup-failure',
|
|
220
|
-
runbookRef: 'runbook:database-recovery',
|
|
221
205
|
severity: 'high',
|
|
222
206
|
},
|
|
223
207
|
{
|
|
224
|
-
...
|
|
225
|
-
businessSymptom: 'The scheduled confidential-export cleanup is not running.',
|
|
208
|
+
...registered('data-export-cleanup-invocation-missing'),
|
|
226
209
|
condition: 'AWS/Lambda reports fewer than one invocation across two consecutive hourly periods.',
|
|
227
210
|
dependencyAlarmIds: [],
|
|
228
|
-
id: 'data-export-cleanup-invocation-missing',
|
|
229
|
-
runbookRef: 'runbook:database-recovery',
|
|
230
211
|
severity: 'critical',
|
|
231
212
|
},
|
|
232
213
|
{
|
|
233
|
-
...
|
|
234
|
-
businessSymptom: 'Confidential-export residue exceeded its deletion deadline.',
|
|
214
|
+
...registered('data-export-cleanup-deadline'),
|
|
235
215
|
condition: 'The cleanup worker reports an open export at or after its immutable 24-hour cleanup deadline.',
|
|
236
216
|
dependencyAlarmIds: [
|
|
237
217
|
'data-export-cleanup-failure',
|
|
238
218
|
'data-export-cleanup-invocation-missing',
|
|
239
219
|
],
|
|
240
|
-
id: 'data-export-cleanup-deadline',
|
|
241
|
-
runbookRef: 'runbook:database-recovery',
|
|
242
220
|
severity: 'critical',
|
|
243
221
|
},
|
|
244
222
|
{
|
|
245
|
-
...
|
|
246
|
-
businessSymptom: 'The generated confidential-export cleanup Function has a terminal invocation error.',
|
|
223
|
+
...registered('function-failure'),
|
|
247
224
|
condition: 'AWS/Lambda reports one or more errors in one minute for the exact generated data-export cleanup Function. Event-delivery Functions have their own error alarms.',
|
|
248
225
|
dependencyAlarmIds: [],
|
|
249
|
-
id: 'function-failure',
|
|
250
|
-
runbookRef: 'runbook:api-recovery',
|
|
251
226
|
severity: 'high',
|
|
252
227
|
},
|
|
253
228
|
{
|
|
254
|
-
...
|
|
255
|
-
businessSymptom: 'The built-in retention Job terminated unsuccessfully.',
|
|
229
|
+
...registered('retention-job-failure'),
|
|
256
230
|
condition: 'AWS Step Functions reports a failed, timed-out, or aborted execution for the exact retention state machine.',
|
|
257
231
|
dependencyAlarmIds: [],
|
|
258
|
-
id: 'retention-job-failure',
|
|
259
|
-
runbookRef: 'runbook:database-recovery',
|
|
260
232
|
severity: 'high',
|
|
261
233
|
},
|
|
262
234
|
{
|
|
263
|
-
...
|
|
264
|
-
businessSymptom: 'The primary alert route is failing delivery.',
|
|
235
|
+
...registered('alert-delivery-degraded'),
|
|
265
236
|
condition: 'SNS reports one or more failed notifications.',
|
|
266
237
|
dependencyAlarmIds: [],
|
|
267
|
-
id: 'alert-delivery-degraded',
|
|
268
|
-
runbookRef: 'runbook:alert-delivery',
|
|
269
238
|
severity: 'critical',
|
|
270
239
|
},
|
|
271
240
|
{
|
|
272
|
-
...
|
|
273
|
-
businessSymptom: 'A managed log group contains a protected-data finding.',
|
|
241
|
+
...registered('data-protection-findings'),
|
|
274
242
|
condition: 'CloudWatch Logs reports one or more protected-data findings.',
|
|
275
243
|
dependencyAlarmIds: [],
|
|
276
|
-
id: 'data-protection-findings',
|
|
277
|
-
runbookRef: 'runbook:log-investigation',
|
|
278
244
|
severity: 'critical',
|
|
279
245
|
},
|
|
280
246
|
{
|
|
281
|
-
...
|
|
282
|
-
businessSymptom: 'Domain-event processing has terminal failures.',
|
|
247
|
+
...registered('event-dlq-depth'),
|
|
283
248
|
condition: 'The event dead-letter queue contains one or more messages.',
|
|
284
249
|
dependencyAlarmIds: [],
|
|
285
|
-
id: 'event-dlq-depth',
|
|
286
|
-
runbookRef: 'runbook:async-recovery',
|
|
287
250
|
severity: 'high',
|
|
288
251
|
},
|
|
289
252
|
{
|
|
290
|
-
...
|
|
291
|
-
businessSymptom: 'Provider deliveries have terminal failures.',
|
|
253
|
+
...registered('provider-dlq-depth'),
|
|
292
254
|
condition: 'The provider-delivery dead-letter queue contains one or more messages.',
|
|
293
255
|
dependencyAlarmIds: [],
|
|
294
|
-
id: 'provider-dlq-depth',
|
|
295
|
-
runbookRef: 'runbook:async-recovery',
|
|
296
256
|
severity: 'high',
|
|
297
257
|
},
|
|
298
258
|
{
|
|
299
|
-
...
|
|
300
|
-
businessSymptom: 'An asynchronous target has terminal failures.',
|
|
259
|
+
...registered('async-target-dlq-depth'),
|
|
301
260
|
condition: 'The asynchronous-target dead-letter queue contains one or more messages.',
|
|
302
261
|
dependencyAlarmIds: [],
|
|
303
|
-
id: 'async-target-dlq-depth',
|
|
304
|
-
runbookRef: 'runbook:async-recovery',
|
|
305
262
|
severity: 'high',
|
|
306
263
|
},
|
|
307
264
|
{
|
|
308
|
-
...
|
|
309
|
-
businessSymptom: 'The isolated performance-sink outbox relay is failing.',
|
|
265
|
+
...registered('outbox-relay-errors'),
|
|
310
266
|
condition: 'AWS/Lambda reports one or more errors in five minutes for the generated async/outbox-relay Function, which drains only the isolated performance sink. Domain-event delivery is the key-value stream and reconciliation.',
|
|
311
267
|
dependencyAlarmIds: [],
|
|
312
|
-
id: 'outbox-relay-errors',
|
|
313
|
-
runbookRef: 'runbook:async-recovery',
|
|
314
268
|
severity: 'high',
|
|
315
269
|
},
|
|
316
270
|
{
|
|
317
|
-
...
|
|
318
|
-
businessSymptom: 'The asynchronous inbox cleanup job is failing.',
|
|
271
|
+
...registered('inbox-cleanup-errors'),
|
|
319
272
|
condition: 'The generated inbox cleanup job reports one or more errors.',
|
|
320
273
|
dependencyAlarmIds: [],
|
|
321
|
-
id: 'inbox-cleanup-errors',
|
|
322
|
-
runbookRef: 'runbook:async-recovery',
|
|
323
274
|
severity: 'high',
|
|
324
275
|
},
|
|
325
276
|
{
|
|
326
|
-
...
|
|
327
|
-
businessSymptom: 'The key-value stream delivery Function failed an invocation.',
|
|
277
|
+
...registered('key-value-stream-delivery-errors'),
|
|
328
278
|
condition: 'AWS/Lambda reports one or more errors in five minutes for the exact key-value stream delivery Function: the stream identity or record shape was rejected, or the handler crashed. Per-record publish failures are partial batch failures, not Lambda errors; they surface through key-value-stream-iterator-age while retries hold the shard back, and through outbox-backlog and event-delivery-attempts after the mapping discards them.',
|
|
329
279
|
dependencyAlarmIds: [],
|
|
330
|
-
id: 'key-value-stream-delivery-errors',
|
|
331
|
-
runbookRef: 'runbook:async-recovery',
|
|
332
280
|
severity: 'high',
|
|
333
281
|
},
|
|
334
282
|
{
|
|
335
|
-
...
|
|
336
|
-
businessSymptom: 'Prompt domain-event delivery from the key-value stream is falling behind.',
|
|
283
|
+
...registered('key-value-stream-iterator-age'),
|
|
337
284
|
condition: 'AWS/Lambda reports a stream iterator age of at least ten minutes for the exact key-value stream delivery Function. The mapping discards records older than one hour; reconciliation then recovers their retained intents.',
|
|
338
285
|
dependencyAlarmIds: ['key-value-stream-delivery-errors'],
|
|
339
|
-
id: 'key-value-stream-iterator-age',
|
|
340
|
-
runbookRef: 'runbook:async-recovery',
|
|
341
286
|
severity: 'high',
|
|
342
287
|
},
|
|
343
288
|
{
|
|
344
|
-
...
|
|
345
|
-
businessSymptom: 'Durable domain-event recovery is failing.',
|
|
289
|
+
...registered('key-value-reconciliation-errors'),
|
|
346
290
|
condition: 'AWS/Lambda reports one or more errors in five minutes for the exact key-value reconciliation Function. Partial runs do not error; the backlog, rejected-intent, and attempts alarms report them.',
|
|
347
291
|
dependencyAlarmIds: [],
|
|
348
|
-
id: 'key-value-reconciliation-errors',
|
|
349
|
-
runbookRef: 'runbook:async-recovery',
|
|
350
292
|
severity: 'high',
|
|
351
293
|
},
|
|
352
294
|
{
|
|
353
|
-
...
|
|
354
|
-
businessSymptom: 'Durable domain-event intents remain undelivered.',
|
|
295
|
+
...registered('outbox-backlog'),
|
|
355
296
|
condition: 'In two consecutive hours, key-value reconciliation reported one or more due intents it could not deliver (retry-required plus rejected). Each run reads at most one 25-intent page per shard, so the value is a lower bound, not a global count, and excludes intents leased by another delivery owner.',
|
|
356
297
|
dependencyAlarmIds: [
|
|
357
298
|
'key-value-reconciliation-errors',
|
|
358
299
|
'key-value-stream-delivery-errors',
|
|
359
300
|
],
|
|
360
301
|
expectedFalseNegativeRisk: 'Intents beyond the sampled pages, or behind a paused outbox, are not counted until a later run reaches them. When the configured reconciliation cadence is slower than one hour, some hourly periods have no data and the two-period condition may never be met.',
|
|
361
|
-
id: 'outbox-backlog',
|
|
362
|
-
runbookRef: 'runbook:async-recovery',
|
|
363
302
|
severity: 'high',
|
|
364
303
|
},
|
|
365
304
|
{
|
|
366
|
-
...
|
|
367
|
-
businessSymptom: 'A retained domain-event intent cannot be delivered.',
|
|
305
|
+
...registered('event-intent-rejected'),
|
|
368
306
|
condition: 'Within an hour, key-value reconciliation reported one or more retained intents that fail validation. Rejected intents are never dispatched or deleted, so this recurs until an operator investigates.',
|
|
369
307
|
dependencyAlarmIds: [],
|
|
370
|
-
id: 'event-intent-rejected',
|
|
371
|
-
runbookRef: 'runbook:async-recovery',
|
|
372
308
|
severity: 'high',
|
|
373
309
|
},
|
|
374
310
|
{
|
|
375
|
-
...
|
|
376
|
-
businessSymptom: 'A domain-event intent is repeatedly failing delivery.',
|
|
311
|
+
...registered('event-delivery-attempts'),
|
|
377
312
|
condition: 'Within an hour, key-value reconciliation observed a due intent with at least ten delivery leases (attempts). Attempts are reported only; no intent is capped, dropped, or quarantined.',
|
|
378
313
|
dependencyAlarmIds: ['key-value-reconciliation-errors'],
|
|
379
|
-
id: 'event-delivery-attempts',
|
|
380
|
-
runbookRef: 'runbook:async-recovery',
|
|
381
314
|
severity: 'high',
|
|
382
315
|
},
|
|
383
316
|
{
|
|
384
|
-
...
|
|
385
|
-
businessSymptom: 'Terminal asynchronous inbox records are accumulating.',
|
|
317
|
+
...registered('inbox-cleanup-backlog'),
|
|
386
318
|
condition: 'The terminal inbox cleanup backlog remains nonempty.',
|
|
387
319
|
dependencyAlarmIds: ['inbox-cleanup-errors'],
|
|
388
|
-
id: 'inbox-cleanup-backlog',
|
|
389
|
-
runbookRef: 'runbook:async-recovery',
|
|
390
320
|
severity: 'high',
|
|
391
321
|
},
|
|
392
322
|
...(sql
|
|
393
323
|
? [
|
|
394
324
|
{
|
|
395
|
-
...
|
|
396
|
-
businessSymptom: 'Relational event delivery is failing.',
|
|
325
|
+
...registered('sql-outbox-relay-errors'),
|
|
397
326
|
condition: 'AWS/Lambda reports one or more errors in five minutes for the exact SQL outbox relay Function, which fails every run that retains a failed delivery.',
|
|
398
327
|
dependencyAlarmIds: [],
|
|
399
|
-
id: 'sql-outbox-relay-errors',
|
|
400
|
-
runbookRef: 'runbook:async-recovery',
|
|
401
328
|
severity: 'high',
|
|
402
329
|
},
|
|
403
330
|
{
|
|
404
|
-
...
|
|
405
|
-
businessSymptom: 'Relational event intents remain undelivered.',
|
|
331
|
+
...registered('sql-outbox-backlog'),
|
|
406
332
|
condition: 'Every SQL relay run in two consecutive hours ended with one or more unpublished intents (Minimum of a global count). A paused relay reports nothing.',
|
|
407
333
|
dependencyAlarmIds: ['sql-outbox-relay-errors'],
|
|
408
334
|
expectedFalseNegativeRisk: 'A paused relay reports nothing. When the configured sqlOutboxRecovery cadence is slower than one hour and no notification wakes the relay, some hourly periods have no data and the two-period condition may never be met.',
|
|
409
|
-
id: 'sql-outbox-backlog',
|
|
410
|
-
runbookRef: 'runbook:async-recovery',
|
|
411
335
|
severity: 'high',
|
|
412
336
|
},
|
|
413
337
|
{
|
|
414
|
-
...
|
|
415
|
-
businessSymptom: 'A relational event intent is repeatedly failing delivery.',
|
|
338
|
+
...registered('sql-outbox-attempts'),
|
|
416
339
|
condition: 'Within an hour, an unpublished SQL intent reached at least ten relay attempts. Attempts are reported only; no intent is capped or dropped.',
|
|
417
340
|
dependencyAlarmIds: ['sql-outbox-relay-errors'],
|
|
418
|
-
id: 'sql-outbox-attempts',
|
|
419
|
-
runbookRef: 'runbook:async-recovery',
|
|
420
341
|
severity: 'high',
|
|
421
342
|
},
|
|
422
343
|
]
|
|
@@ -595,14 +516,40 @@ function reliabilityExercises(name) {
|
|
|
595
516
|
},
|
|
596
517
|
];
|
|
597
518
|
}
|
|
519
|
+
const ROUTED_INVENTORY_LEADING_KEYS = [
|
|
520
|
+
'actions',
|
|
521
|
+
'comparisonOperator',
|
|
522
|
+
'owner',
|
|
523
|
+
'route',
|
|
524
|
+
];
|
|
525
|
+
/**
|
|
526
|
+
* One inventory entry with the shared identity's owner, route and runbook,
|
|
527
|
+
* the fields the generated verifier compares with the deployed alarm's
|
|
528
|
+
* description. Keys keep the seeded order: a routed alarm's actions,
|
|
529
|
+
* comparison operator, owner and route first, then the rest sorted.
|
|
530
|
+
*/
|
|
531
|
+
function inventoryAlarmResource(name, resource) {
|
|
532
|
+
const identity = (0, built_in_plan_alarm_identity_js_1.sliceAlarmIdentity)(name, resource.id);
|
|
533
|
+
const fields = {
|
|
534
|
+
...resource,
|
|
535
|
+
owner: identity.owner,
|
|
536
|
+
route: identity.route,
|
|
537
|
+
runbook: identity.runbook,
|
|
538
|
+
};
|
|
539
|
+
const leading = resource.actions === 'primary-route' ? ROUTED_INVENTORY_LEADING_KEYS : [];
|
|
540
|
+
return Object.fromEntries([
|
|
541
|
+
...leading,
|
|
542
|
+
...Object.keys(fields)
|
|
543
|
+
.filter((key) => !leading.includes(key))
|
|
544
|
+
.sort(deterministic_order_js_1.compareUtf16),
|
|
545
|
+
].map((key) => [key, fields[key]]));
|
|
546
|
+
}
|
|
598
547
|
function deployedAlarmResources(name, sql) {
|
|
599
548
|
const primary = {
|
|
600
549
|
actions: 'primary-route',
|
|
601
550
|
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
602
|
-
owner: `${name}-product`,
|
|
603
|
-
route: `${name}-on-call`,
|
|
604
551
|
};
|
|
605
|
-
|
|
552
|
+
const resources = [
|
|
606
553
|
{
|
|
607
554
|
...primary,
|
|
608
555
|
dimensionNames: ['ApiId'],
|
|
@@ -612,7 +559,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
612
559
|
name: `${name}-{stage}-api-fast-burn`,
|
|
613
560
|
namespace: 'AWS/ApiGateway',
|
|
614
561
|
objectiveId: 'api-availability',
|
|
615
|
-
runbook: 'docs/runbooks/api-recovery.md',
|
|
616
562
|
signal: 'metric-math',
|
|
617
563
|
threshold: 14.4,
|
|
618
564
|
treatMissingData: 'missing',
|
|
@@ -626,7 +572,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
626
572
|
name: `${name}-{stage}-api-slow-burn`,
|
|
627
573
|
namespace: 'AWS/ApiGateway',
|
|
628
574
|
objectiveId: 'api-availability',
|
|
629
|
-
runbook: 'docs/runbooks/api-recovery.md',
|
|
630
575
|
signal: 'metric-math',
|
|
631
576
|
threshold: 6,
|
|
632
577
|
treatMissingData: 'missing',
|
|
@@ -647,7 +592,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
647
592
|
name: `${name}-{stage}-web-latency-fast-burn`,
|
|
648
593
|
namespace: `EBK/${name}`,
|
|
649
594
|
objectiveId: 'web-latency',
|
|
650
|
-
runbook: 'docs/runbooks/web-recovery.md',
|
|
651
595
|
signal: 'metric-math',
|
|
652
596
|
threshold: 14.4,
|
|
653
597
|
treatMissingData: 'missing',
|
|
@@ -668,7 +612,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
668
612
|
name: `${name}-{stage}-web-latency-slow-burn`,
|
|
669
613
|
namespace: `EBK/${name}`,
|
|
670
614
|
objectiveId: 'web-latency',
|
|
671
|
-
runbook: 'docs/runbooks/web-recovery.md',
|
|
672
615
|
signal: 'metric-math',
|
|
673
616
|
threshold: 6,
|
|
674
617
|
treatMissingData: 'missing',
|
|
@@ -685,7 +628,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
685
628
|
name: `${name}-{stage}-provider-deadline`,
|
|
686
629
|
namespace: 'AWS/SQS',
|
|
687
630
|
period: 300,
|
|
688
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
689
631
|
signal: 'metric',
|
|
690
632
|
statistic: 'Maximum',
|
|
691
633
|
threshold: 900,
|
|
@@ -701,7 +643,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
701
643
|
name: `${name}-{stage}-database-capacity`,
|
|
702
644
|
namespace: 'AWS/DynamoDB',
|
|
703
645
|
period: 60,
|
|
704
|
-
runbook: 'docs/runbooks/database-recovery.md',
|
|
705
646
|
signal: 'metric-math',
|
|
706
647
|
statistic: 'Sum',
|
|
707
648
|
threshold: 1,
|
|
@@ -723,7 +664,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
723
664
|
name: `${name}-{stage}-performance-load-cleanup-backlog`,
|
|
724
665
|
namespace: 'EBK/Performance',
|
|
725
666
|
period: 3600,
|
|
726
|
-
runbook: 'docs/runbooks/performance-capacity-cost.md',
|
|
727
667
|
signal: 'metric',
|
|
728
668
|
statistic: 'Maximum',
|
|
729
669
|
threshold: 1,
|
|
@@ -745,7 +685,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
745
685
|
name: `${name}-{stage}-performance-load-cleanup-deadline`,
|
|
746
686
|
namespace: 'EBK/Performance',
|
|
747
687
|
period: 3600,
|
|
748
|
-
runbook: 'docs/runbooks/performance-capacity-cost.md',
|
|
749
688
|
signal: 'metric',
|
|
750
689
|
statistic: 'Maximum',
|
|
751
690
|
threshold: 1,
|
|
@@ -761,7 +700,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
761
700
|
name: `${name}-{stage}-data-export-cleanup-invocation-missing`,
|
|
762
701
|
namespace: 'AWS/Lambda',
|
|
763
702
|
period: 3600,
|
|
764
|
-
runbook: 'docs/runbooks/database-recovery.md',
|
|
765
703
|
signal: 'metric',
|
|
766
704
|
statistic: 'Sum',
|
|
767
705
|
threshold: 1,
|
|
@@ -776,7 +714,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
776
714
|
name: `${name}-{stage}-data-export-cleanup-failure`,
|
|
777
715
|
namespace: 'AWS/Lambda',
|
|
778
716
|
period: 300,
|
|
779
|
-
runbook: 'docs/runbooks/database-recovery.md',
|
|
780
717
|
signal: 'metric',
|
|
781
718
|
statistic: 'Sum',
|
|
782
719
|
threshold: 1,
|
|
@@ -798,7 +735,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
798
735
|
name: `${name}-{stage}-data-export-cleanup-deadline`,
|
|
799
736
|
namespace: 'EBK/DataLifecycle',
|
|
800
737
|
period: 3600,
|
|
801
|
-
runbook: 'docs/runbooks/database-recovery.md',
|
|
802
738
|
signal: 'metric',
|
|
803
739
|
statistic: 'Maximum',
|
|
804
740
|
threshold: 1,
|
|
@@ -813,7 +749,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
813
749
|
name: `${name}-{stage}-function-failure`,
|
|
814
750
|
namespace: 'AWS/Lambda',
|
|
815
751
|
period: 60,
|
|
816
|
-
runbook: 'docs/runbooks/api-recovery.md',
|
|
817
752
|
signal: 'metric',
|
|
818
753
|
statistic: 'Sum',
|
|
819
754
|
threshold: 1,
|
|
@@ -831,7 +766,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
831
766
|
],
|
|
832
767
|
name: `${name}-{stage}-retention-job-failure`,
|
|
833
768
|
namespace: 'AWS/States',
|
|
834
|
-
runbook: 'docs/runbooks/database-recovery.md',
|
|
835
769
|
signal: 'metric-math',
|
|
836
770
|
threshold: 1,
|
|
837
771
|
treatMissingData: 'notBreaching',
|
|
@@ -848,7 +782,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
848
782
|
name: `${name}-{stage}-event-dlq-depth`,
|
|
849
783
|
namespace: 'AWS/SQS',
|
|
850
784
|
period: 300,
|
|
851
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
852
785
|
signal: 'metric',
|
|
853
786
|
statistic: 'Maximum',
|
|
854
787
|
threshold: 1,
|
|
@@ -866,7 +799,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
866
799
|
name: `${name}-{stage}-provider-dlq-depth`,
|
|
867
800
|
namespace: 'AWS/SQS',
|
|
868
801
|
period: 300,
|
|
869
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
870
802
|
signal: 'metric',
|
|
871
803
|
statistic: 'Maximum',
|
|
872
804
|
threshold: 1,
|
|
@@ -884,7 +816,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
884
816
|
name: `${name}-{stage}-async-target-dlq-depth`,
|
|
885
817
|
namespace: 'AWS/SQS',
|
|
886
818
|
period: 300,
|
|
887
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
888
819
|
signal: 'metric',
|
|
889
820
|
statistic: 'Maximum',
|
|
890
821
|
threshold: 1,
|
|
@@ -899,7 +830,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
899
830
|
name: `${name}-{stage}-outbox-relay-errors`,
|
|
900
831
|
namespace: 'AWS/Lambda',
|
|
901
832
|
period: 300,
|
|
902
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
903
833
|
signal: 'metric',
|
|
904
834
|
statistic: 'Sum',
|
|
905
835
|
threshold: 1,
|
|
@@ -914,7 +844,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
914
844
|
name: `${name}-{stage}-inbox-cleanup-errors`,
|
|
915
845
|
namespace: 'AWS/Lambda',
|
|
916
846
|
period: 300,
|
|
917
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
918
847
|
signal: 'metric',
|
|
919
848
|
statistic: 'Sum',
|
|
920
849
|
threshold: 1,
|
|
@@ -929,7 +858,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
929
858
|
name: `${name}-{stage}-key-value-stream-delivery-errors`,
|
|
930
859
|
namespace: 'AWS/Lambda',
|
|
931
860
|
period: 300,
|
|
932
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
933
861
|
signal: 'metric',
|
|
934
862
|
statistic: 'Sum',
|
|
935
863
|
threshold: 1,
|
|
@@ -944,7 +872,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
944
872
|
name: `${name}-{stage}-key-value-stream-iterator-age`,
|
|
945
873
|
namespace: 'AWS/Lambda',
|
|
946
874
|
period: 300,
|
|
947
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
948
875
|
signal: 'metric',
|
|
949
876
|
statistic: 'Maximum',
|
|
950
877
|
threshold: 600_000,
|
|
@@ -959,7 +886,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
959
886
|
name: `${name}-{stage}-key-value-reconciliation-errors`,
|
|
960
887
|
namespace: 'AWS/Lambda',
|
|
961
888
|
period: 300,
|
|
962
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
963
889
|
signal: 'metric',
|
|
964
890
|
statistic: 'Sum',
|
|
965
891
|
threshold: 1,
|
|
@@ -975,7 +901,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
975
901
|
name: `${name}-{stage}-outbox-backlog`,
|
|
976
902
|
namespace: 'EBK/Async',
|
|
977
903
|
period: 3600,
|
|
978
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
979
904
|
signal: 'metric',
|
|
980
905
|
statistic: 'Maximum',
|
|
981
906
|
threshold: 1,
|
|
@@ -991,7 +916,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
991
916
|
name: `${name}-{stage}-event-intent-rejected`,
|
|
992
917
|
namespace: 'EBK/Async',
|
|
993
918
|
period: 3600,
|
|
994
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
995
919
|
signal: 'metric',
|
|
996
920
|
statistic: 'Maximum',
|
|
997
921
|
threshold: 1,
|
|
@@ -1007,7 +931,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
1007
931
|
name: `${name}-{stage}-event-delivery-attempts`,
|
|
1008
932
|
namespace: 'EBK/Async',
|
|
1009
933
|
period: 3600,
|
|
1010
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
1011
934
|
signal: 'metric',
|
|
1012
935
|
statistic: 'Maximum',
|
|
1013
936
|
threshold: 10,
|
|
@@ -1024,7 +947,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
1024
947
|
name: `${name}-{stage}-sql-outbox-relay-errors`,
|
|
1025
948
|
namespace: 'AWS/Lambda',
|
|
1026
949
|
period: 300,
|
|
1027
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
1028
950
|
signal: 'metric',
|
|
1029
951
|
statistic: 'Sum',
|
|
1030
952
|
threshold: 1,
|
|
@@ -1040,7 +962,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
1040
962
|
name: `${name}-{stage}-sql-outbox-backlog`,
|
|
1041
963
|
namespace: 'EBK/Async',
|
|
1042
964
|
period: 3600,
|
|
1043
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
1044
965
|
signal: 'metric',
|
|
1045
966
|
statistic: 'Minimum',
|
|
1046
967
|
threshold: 1,
|
|
@@ -1056,7 +977,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
1056
977
|
name: `${name}-{stage}-sql-outbox-attempts`,
|
|
1057
978
|
namespace: 'EBK/Async',
|
|
1058
979
|
period: 3600,
|
|
1059
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
1060
980
|
signal: 'metric',
|
|
1061
981
|
statistic: 'Maximum',
|
|
1062
982
|
threshold: 10,
|
|
@@ -1080,7 +1000,6 @@ function deployedAlarmResources(name, sql) {
|
|
|
1080
1000
|
name: `${name}-{stage}-inbox-cleanup-backlog`,
|
|
1081
1001
|
namespace: 'EBK/Async',
|
|
1082
1002
|
period: 3600,
|
|
1083
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
1084
1003
|
signal: 'metric',
|
|
1085
1004
|
statistic: 'Maximum',
|
|
1086
1005
|
threshold: 1,
|
|
@@ -1095,22 +1014,21 @@ function deployedAlarmResources(name, sql) {
|
|
|
1095
1014
|
metricName: 'NumberOfNotificationsFailed',
|
|
1096
1015
|
name: `${name}-{stage}-alert-delivery-degraded`,
|
|
1097
1016
|
namespace: 'AWS/SNS',
|
|
1098
|
-
owner: `${name}-product`,
|
|
1099
1017
|
period: 300,
|
|
1100
|
-
route: `${name}-on-call`,
|
|
1101
|
-
runbook: 'docs/runbooks/alert-delivery.md',
|
|
1102
1018
|
signal: 'metric',
|
|
1103
1019
|
statistic: 'Sum',
|
|
1104
1020
|
threshold: 1,
|
|
1105
1021
|
treatMissingData: 'notBreaching',
|
|
1106
1022
|
},
|
|
1107
1023
|
];
|
|
1024
|
+
return resources.map((resource) => inventoryAlarmResource(name, resource));
|
|
1108
1025
|
}
|
|
1109
1026
|
/** @internal */
|
|
1110
1027
|
function renderProductReliabilityFiles(name, pascalName, reviewedAt, sql = false) {
|
|
1111
1028
|
const contract = reliabilityContract(name, reviewedAt);
|
|
1112
1029
|
const alarms = alarmRegistrations(name, sql);
|
|
1113
1030
|
const exercises = reliabilityExercises(name);
|
|
1031
|
+
const ownership = (0, built_in_plan_alarm_identity_js_1.sliceAlarmOwnership)(name);
|
|
1114
1032
|
return {
|
|
1115
1033
|
'docs/reliability/slo.json': (0, built_in_plan_shared_js_1.json)(contract),
|
|
1116
1034
|
'docs/reliability/exclusions.md': `# SLO exclusions
|
|
@@ -1257,10 +1175,20 @@ import declaration from '../docs/reliability/slo.json';
|
|
|
1257
1175
|
export const Reliability = validateReliabilityContract(declaration);
|
|
1258
1176
|
`,
|
|
1259
1177
|
'reliability/metrics.ts': `import {
|
|
1178
|
+
createEmfMetricRecord,
|
|
1260
1179
|
createMetricEmitter,
|
|
1180
|
+
createMetricRejectionReporter,
|
|
1181
|
+
type BatchedMetricEmitter,
|
|
1182
|
+
type EmfMetricObservation,
|
|
1183
|
+
type MetricDimensions,
|
|
1261
1184
|
type MetricEmitter,
|
|
1185
|
+
type MetricEmitterOptions,
|
|
1262
1186
|
type MetricOutcome,
|
|
1263
1187
|
} from '@empire-builder-kit/runtime/observability';
|
|
1188
|
+
import {
|
|
1189
|
+
reliabilityMetricDeclarations,
|
|
1190
|
+
type ReliabilityMetricDeclaration,
|
|
1191
|
+
} from './metric-declarations.js';
|
|
1264
1192
|
|
|
1265
1193
|
export interface ReliabilityOperation {
|
|
1266
1194
|
component: string;
|
|
@@ -1271,37 +1199,146 @@ export interface ReliabilityOperation {
|
|
|
1271
1199
|
outcome: MetricOutcome;
|
|
1272
1200
|
}
|
|
1273
1201
|
|
|
1202
|
+
// Managed core declarations grow with generated callers on upgrade. Adopter
|
|
1203
|
+
// declarations stay in the seeded extension file, never in request input.
|
|
1204
|
+
const coreMetricDeclarations: readonly ReliabilityMetricDeclaration[] = [
|
|
1205
|
+
{ component: 'next-server', feature: 'web', operation: 'request' },
|
|
1206
|
+
];
|
|
1207
|
+
const metricDeclarations: readonly ReliabilityMetricDeclaration[] = Object.freeze(
|
|
1208
|
+
[...coreMetricDeclarations, ...reliabilityMetricDeclarations].map((entry) =>
|
|
1209
|
+
Object.freeze({ ...entry }),
|
|
1210
|
+
),
|
|
1211
|
+
);
|
|
1212
|
+
// Deployment identity is fixed when the Lambda/Next module starts, not per request.
|
|
1213
|
+
const reliabilityStage = process.env.EBK_STAGE ?? 'development';
|
|
1214
|
+
|
|
1215
|
+
export function createReliabilityMetricEmitter(
|
|
1216
|
+
options: Omit<MetricEmitterOptions, 'cardinalityPolicy'> = {},
|
|
1217
|
+
): BatchedMetricEmitter {
|
|
1218
|
+
const values = (dimension: keyof ReliabilityMetricDeclaration) =>
|
|
1219
|
+
[...new Set(metricDeclarations.map((entry) => entry[dimension]))];
|
|
1220
|
+
const reportRejected = createMetricRejectionReporter(options.onRejected);
|
|
1221
|
+
const emitter = createMetricEmitter({
|
|
1222
|
+
...options,
|
|
1223
|
+
onRejected: reportRejected,
|
|
1224
|
+
cardinalityPolicy: {
|
|
1225
|
+
dimensions: {
|
|
1226
|
+
component: values('component'),
|
|
1227
|
+
feature: values('feature'),
|
|
1228
|
+
operation: values('operation'),
|
|
1229
|
+
slice: ['${name}'],
|
|
1230
|
+
stage: [reliabilityStage],
|
|
1231
|
+
},
|
|
1232
|
+
metricNames: ['OperationValid', 'OperationGood', 'OperationDuration'],
|
|
1233
|
+
},
|
|
1234
|
+
});
|
|
1235
|
+
// A declaration owns one tuple, not every Cartesian combination of its labels.
|
|
1236
|
+
const tupleKey = (dimensions: MetricDimensions | undefined) => {
|
|
1237
|
+
const labels = [dimensions?.component, dimensions?.feature, dimensions?.operation];
|
|
1238
|
+
return labels.every((label) => typeof label === 'string')
|
|
1239
|
+
? JSON.stringify(labels.map((label) => label.trim()))
|
|
1240
|
+
: undefined;
|
|
1241
|
+
};
|
|
1242
|
+
const declaredTuples = new Set(metricDeclarations.map((entry) =>
|
|
1243
|
+
tupleKey({ ...entry, slice: '${name}', stage: reliabilityStage }),
|
|
1244
|
+
));
|
|
1245
|
+
return {
|
|
1246
|
+
reportRejected,
|
|
1247
|
+
emit(input) {
|
|
1248
|
+
if (!declaredTuples.has(tupleKey(input?.dimensions))) {
|
|
1249
|
+
reportRejected('undeclared-tuple');
|
|
1250
|
+
return undefined;
|
|
1251
|
+
}
|
|
1252
|
+
return emitter.emit(input);
|
|
1253
|
+
},
|
|
1254
|
+
emitBatch(input) {
|
|
1255
|
+
if (!declaredTuples.has(tupleKey(input?.dimensions))) {
|
|
1256
|
+
reportRejected('undeclared-tuple');
|
|
1257
|
+
return undefined;
|
|
1258
|
+
}
|
|
1259
|
+
return emitter.emitBatch(input);
|
|
1260
|
+
},
|
|
1261
|
+
};
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
// Invalid declarations fail closed during module initialization and prevent
|
|
1265
|
+
// startup. After successful initialization, emission failures stay observational.
|
|
1266
|
+
const defaultReliabilityMetricEmitter = createReliabilityMetricEmitter();
|
|
1267
|
+
|
|
1274
1268
|
export function recordReliabilityOperation(
|
|
1275
1269
|
input: ReliabilityOperation,
|
|
1276
|
-
metrics: MetricEmitter =
|
|
1270
|
+
metrics: MetricEmitter = defaultReliabilityMetricEmitter,
|
|
1277
1271
|
) {
|
|
1278
1272
|
const dimensions = {
|
|
1279
1273
|
component: input.component,
|
|
1280
1274
|
feature: input.feature,
|
|
1281
1275
|
operation: input.operation,
|
|
1282
1276
|
slice: '${name}',
|
|
1283
|
-
stage:
|
|
1277
|
+
stage: reliabilityStage,
|
|
1284
1278
|
};
|
|
1285
|
-
const
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
value: 1,
|
|
1290
|
-
});
|
|
1291
|
-
const good = metrics.emit({
|
|
1292
|
-
dimensions,
|
|
1279
|
+
const validObservation = {
|
|
1280
|
+
metricName: 'OperationValid', unit: 'Count', value: 1,
|
|
1281
|
+
} as const;
|
|
1282
|
+
const goodObservation = {
|
|
1293
1283
|
metricName: 'OperationGood',
|
|
1294
1284
|
unit: 'Count',
|
|
1295
1285
|
value: (input.good ?? input.outcome === 'success') ? 1 : 0,
|
|
1296
|
-
}
|
|
1297
|
-
const
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1286
|
+
} as const;
|
|
1287
|
+
const latencyObservation = Number.isFinite(input.durationMilliseconds)
|
|
1288
|
+
? {
|
|
1289
|
+
metricName: 'OperationDuration',
|
|
1290
|
+
unit: 'Milliseconds',
|
|
1291
|
+
value: input.durationMilliseconds,
|
|
1292
|
+
} as const
|
|
1293
|
+
: undefined;
|
|
1294
|
+
if (!latencyObservation) {
|
|
1295
|
+
try {
|
|
1296
|
+
const reportRejected = metrics.reportRejected ?? defaultReliabilityMetricEmitter.reportRejected;
|
|
1297
|
+
reportRejected?.('invalid-observation');
|
|
1298
|
+
} catch {
|
|
1299
|
+
// Custom diagnostic adapters cannot turn completed work into a failed response.
|
|
1300
|
+
}
|
|
1301
|
+
}
|
|
1302
|
+
const observations: readonly EmfMetricObservation[] = [
|
|
1303
|
+
validObservation,
|
|
1304
|
+
goodObservation,
|
|
1305
|
+
...(latencyObservation ? [latencyObservation] : []),
|
|
1306
|
+
];
|
|
1307
|
+
if (metrics.emitBatch) {
|
|
1308
|
+
const batch = metrics.emitBatch({ dimensions, metrics: observations });
|
|
1309
|
+
// Keep the public per-metric result records while delivering one envelope.
|
|
1310
|
+
const result = (observation: EmfMetricObservation | undefined) =>
|
|
1311
|
+
batch && observation && createEmfMetricRecord({
|
|
1312
|
+
...observation,
|
|
1313
|
+
dimensions,
|
|
1314
|
+
now: () => new Date(batch._aws.Timestamp),
|
|
1315
|
+
});
|
|
1316
|
+
return {
|
|
1317
|
+
good: result(goodObservation),
|
|
1318
|
+
latency: result(latencyObservation),
|
|
1319
|
+
valid: result(validObservation),
|
|
1320
|
+
};
|
|
1321
|
+
}
|
|
1322
|
+
// Existing custom emit-only adapters retain one call per valid observation.
|
|
1323
|
+
const valid = metrics.emit({ ...validObservation, dimensions });
|
|
1324
|
+
const good = metrics.emit({ ...goodObservation, dimensions });
|
|
1325
|
+
const latency = latencyObservation &&
|
|
1326
|
+
metrics.emit({ ...latencyObservation, dimensions });
|
|
1303
1327
|
return { good, latency, valid };
|
|
1304
1328
|
}
|
|
1329
|
+
`,
|
|
1330
|
+
'reliability/metric-declarations.ts': `import type {
|
|
1331
|
+
MetricDimensions,
|
|
1332
|
+
} from '@empire-builder-kit/runtime/observability';
|
|
1333
|
+
|
|
1334
|
+
export type ReliabilityMetricDeclaration = Required<
|
|
1335
|
+
Pick<MetricDimensions, 'component' | 'feature' | 'operation'>
|
|
1336
|
+
>;
|
|
1337
|
+
|
|
1338
|
+
// Seeded extension: declare finite deployment-owned labels for custom features.
|
|
1339
|
+
// Never derive declarations from request, user, resource, or correlation IDs.
|
|
1340
|
+
// Generated core declarations are added separately by the managed recorder.
|
|
1341
|
+
export const reliabilityMetricDeclarations: readonly ReliabilityMetricDeclaration[] = [];
|
|
1305
1342
|
`,
|
|
1306
1343
|
'infra/reliability.ts': `import {
|
|
1307
1344
|
createSloBurnAlarmPlan,
|
|
@@ -1320,15 +1357,13 @@ ${' databaseTableName: any;\n'}\
|
|
|
1320
1357
|
function alarmDescription(input: {
|
|
1321
1358
|
businessSymptom: string;
|
|
1322
1359
|
id: string;
|
|
1323
|
-
owner: string;
|
|
1324
|
-
route: string;
|
|
1325
1360
|
runbook: string;
|
|
1326
1361
|
}) {
|
|
1327
1362
|
return JSON.stringify({
|
|
1328
1363
|
businessSymptom: input.businessSymptom,
|
|
1329
1364
|
id: input.id,
|
|
1330
|
-
owner:
|
|
1331
|
-
route:
|
|
1365
|
+
owner: '${ownership.owner}',
|
|
1366
|
+
route: '${ownership.route}',
|
|
1332
1367
|
runbook: input.runbook,
|
|
1333
1368
|
schemaVersion: 1,
|
|
1334
1369
|
slice: '${name}',
|
|
@@ -1386,13 +1421,7 @@ export function create${pascalName}Reliability(
|
|
|
1386
1421
|
'${pascalName}ApiFastBurn',
|
|
1387
1422
|
{
|
|
1388
1423
|
alarmActions: [input.alertTopicArn],
|
|
1389
|
-
alarmDescription: alarmDescription({
|
|
1390
|
-
businessSymptom: 'Critical API operations are failing.',
|
|
1391
|
-
id: 'api-fast-burn',
|
|
1392
|
-
owner: '${name}-product',
|
|
1393
|
-
route: '${name}-on-call',
|
|
1394
|
-
runbook: 'docs/runbooks/api-recovery.md',
|
|
1395
|
-
}),
|
|
1424
|
+
alarmDescription: alarmDescription(${(0, built_in_plan_alarm_identity_js_1.renderSliceAlarmDescriptionInput)(name, 'api-fast-burn', ' ')}),
|
|
1396
1425
|
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1397
1426
|
datapointsToAlarm: 1,
|
|
1398
1427
|
evaluationPeriods: 1,
|
|
@@ -1406,13 +1435,7 @@ export function create${pascalName}Reliability(
|
|
|
1406
1435
|
'${pascalName}ApiSlowBurn',
|
|
1407
1436
|
{
|
|
1408
1437
|
alarmActions: [input.alertTopicArn],
|
|
1409
|
-
alarmDescription: alarmDescription({
|
|
1410
|
-
businessSymptom: 'Critical API operations have sustained failures.',
|
|
1411
|
-
id: 'api-slow-burn',
|
|
1412
|
-
owner: '${name}-product',
|
|
1413
|
-
route: '${name}-on-call',
|
|
1414
|
-
runbook: 'docs/runbooks/api-recovery.md',
|
|
1415
|
-
}),
|
|
1438
|
+
alarmDescription: alarmDescription(${(0, built_in_plan_alarm_identity_js_1.renderSliceAlarmDescriptionInput)(name, 'api-slow-burn', ' ')}),
|
|
1416
1439
|
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1417
1440
|
datapointsToAlarm: 1,
|
|
1418
1441
|
evaluationPeriods: 1,
|
|
@@ -1426,13 +1449,7 @@ export function create${pascalName}Reliability(
|
|
|
1426
1449
|
'${pascalName}WebLatencyBurn',
|
|
1427
1450
|
{
|
|
1428
1451
|
alarmActions: [input.alertTopicArn],
|
|
1429
|
-
alarmDescription: alarmDescription({
|
|
1430
|
-
businessSymptom: 'Server-rendered routes exceed the latency objective.',
|
|
1431
|
-
id: 'web-latency-fast-burn',
|
|
1432
|
-
owner: '${name}-product',
|
|
1433
|
-
route: '${name}-on-call',
|
|
1434
|
-
runbook: 'docs/runbooks/web-recovery.md',
|
|
1435
|
-
}),
|
|
1452
|
+
alarmDescription: alarmDescription(${(0, built_in_plan_alarm_identity_js_1.renderSliceAlarmDescriptionInput)(name, 'web-latency-fast-burn', ' ')}),
|
|
1436
1453
|
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1437
1454
|
datapointsToAlarm: 1,
|
|
1438
1455
|
evaluationPeriods: 1,
|
|
@@ -1446,13 +1463,7 @@ export function create${pascalName}Reliability(
|
|
|
1446
1463
|
'${pascalName}WebLatencySlowBurn',
|
|
1447
1464
|
{
|
|
1448
1465
|
alarmActions: [input.alertTopicArn],
|
|
1449
|
-
alarmDescription: alarmDescription({
|
|
1450
|
-
businessSymptom: 'Server-rendered routes have sustained latency degradation.',
|
|
1451
|
-
id: 'web-latency-slow-burn',
|
|
1452
|
-
owner: '${name}-product',
|
|
1453
|
-
route: '${name}-on-call',
|
|
1454
|
-
runbook: 'docs/runbooks/web-recovery.md',
|
|
1455
|
-
}),
|
|
1466
|
+
alarmDescription: alarmDescription(${(0, built_in_plan_alarm_identity_js_1.renderSliceAlarmDescriptionInput)(name, 'web-latency-slow-burn', ' ')}),
|
|
1456
1467
|
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1457
1468
|
datapointsToAlarm: 1,
|
|
1458
1469
|
evaluationPeriods: 1,
|
|
@@ -1466,13 +1477,7 @@ export function create${pascalName}Reliability(
|
|
|
1466
1477
|
'${pascalName}ProviderDeadline',
|
|
1467
1478
|
{
|
|
1468
1479
|
alarmActions: [input.alertTopicArn],
|
|
1469
|
-
alarmDescription: alarmDescription({
|
|
1470
|
-
businessSymptom: 'Accepted provider work is missing its deadline.',
|
|
1471
|
-
id: 'provider-queue-age',
|
|
1472
|
-
owner: '${name}-product',
|
|
1473
|
-
route: '${name}-on-call',
|
|
1474
|
-
runbook: 'docs/runbooks/async-recovery.md',
|
|
1475
|
-
}),
|
|
1480
|
+
alarmDescription: alarmDescription(${(0, built_in_plan_alarm_identity_js_1.renderSliceAlarmDescriptionInput)(name, 'provider-queue-age', ' ')}),
|
|
1476
1481
|
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1477
1482
|
dimensions: { QueueName: input.providerQueueName },
|
|
1478
1483
|
evaluationPeriods: 2,
|
|
@@ -1489,13 +1494,7 @@ export function create${pascalName}Reliability(
|
|
|
1489
1494
|
'${pascalName}DatabaseCapacity',
|
|
1490
1495
|
{
|
|
1491
1496
|
alarmActions: [input.alertTopicArn],
|
|
1492
|
-
alarmDescription: alarmDescription({
|
|
1493
|
-
businessSymptom: ${JSON.stringify('The Slice table or event-reconciliation index is throttling operations.')},
|
|
1494
|
-
id: 'database-capacity',
|
|
1495
|
-
owner: '${name}-product',
|
|
1496
|
-
route: '${name}-on-call',
|
|
1497
|
-
runbook: 'docs/runbooks/database-recovery.md',
|
|
1498
|
-
}),
|
|
1497
|
+
alarmDescription: alarmDescription(${(0, built_in_plan_alarm_identity_js_1.renderSliceAlarmDescriptionInput)(name, 'database-capacity', ' ')}),
|
|
1499
1498
|
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1500
1499
|
${` evaluationPeriods: 1,
|
|
1501
1500
|
name: '${name}-' + $app.stage + '-database-capacity',
|
|
@@ -1522,14 +1521,7 @@ ${` evaluationPeriods: 1,
|
|
|
1522
1521
|
'${pascalName}FunctionFailure',
|
|
1523
1522
|
{
|
|
1524
1523
|
alarmActions: [input.alertTopicArn],
|
|
1525
|
-
alarmDescription: alarmDescription({
|
|
1526
|
-
businessSymptom:
|
|
1527
|
-
'The generated confidential-export cleanup Function has a terminal invocation error.',
|
|
1528
|
-
id: 'function-failure',
|
|
1529
|
-
owner: '${name}-product',
|
|
1530
|
-
route: '${name}-on-call',
|
|
1531
|
-
runbook: 'docs/runbooks/api-recovery.md',
|
|
1532
|
-
}),
|
|
1524
|
+
alarmDescription: alarmDescription(${(0, built_in_plan_alarm_identity_js_1.renderSliceAlarmDescriptionInput)(name, 'function-failure', ' ')}),
|
|
1533
1525
|
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1534
1526
|
dimensions: { FunctionName: input.functionName },
|
|
1535
1527
|
evaluationPeriods: 1,
|
|
@@ -1545,13 +1537,7 @@ ${` evaluationPeriods: 1,
|
|
|
1545
1537
|
const alertDelivery = new aws.cloudwatch.MetricAlarm(
|
|
1546
1538
|
'${pascalName}AlertDelivery',
|
|
1547
1539
|
{
|
|
1548
|
-
alarmDescription: alarmDescription({
|
|
1549
|
-
businessSymptom: 'The primary alert route is failing delivery.',
|
|
1550
|
-
id: 'alert-delivery-degraded',
|
|
1551
|
-
owner: '${name}-product',
|
|
1552
|
-
route: '${name}-on-call',
|
|
1553
|
-
runbook: 'docs/runbooks/alert-delivery.md',
|
|
1554
|
-
}),
|
|
1540
|
+
alarmDescription: alarmDescription(${(0, built_in_plan_alarm_identity_js_1.renderSliceAlarmDescriptionInput)(name, 'alert-delivery-degraded', ' ')}),
|
|
1555
1541
|
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1556
1542
|
dimensions: { TopicName: input.alertTopicName },
|
|
1557
1543
|
evaluationPeriods: 1,
|