@empire-builder-kit/nx 1.0.0-beta.9 → 1.0.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +6 -0
- package/README.md +113 -3
- package/executors.json +30 -0
- package/package.json +84 -6
- package/src/api-contract/baseline-update.d.ts +82 -0
- package/src/api-contract/baseline-update.js +225 -0
- package/src/api-contract/baseline-update.js.map +1 -0
- package/src/api-contract/compatibility.d.ts +30 -0
- package/src/api-contract/compatibility.js +643 -0
- package/src/api-contract/compatibility.js.map +1 -0
- package/src/api-contract/contract-graph.d.ts +35 -0
- package/src/api-contract/contract-graph.js +141 -0
- package/src/api-contract/contract-graph.js.map +1 -0
- package/src/api-contract/evidence.d.ts +23 -0
- package/src/api-contract/evidence.js +87 -0
- package/src/api-contract/evidence.js.map +1 -0
- package/src/api-contract/index.d.ts +6 -0
- package/src/api-contract/index.js +10 -0
- package/src/api-contract/index.js.map +1 -0
- package/src/api-contract/model.d.ts +144 -0
- package/src/api-contract/model.fixtures.d.ts +2 -0
- package/src/api-contract/model.fixtures.js +81 -0
- package/src/api-contract/model.fixtures.js.map +1 -0
- package/src/api-contract/model.js +993 -0
- package/src/api-contract/model.js.map +1 -0
- package/src/api-contract/rollout.d.ts +32 -0
- package/src/api-contract/rollout.js +88 -0
- package/src/api-contract/rollout.js.map +1 -0
- package/src/authorization/compatibility.d.ts +62 -0
- package/src/authorization/compatibility.js +175 -0
- package/src/authorization/compatibility.js.map +1 -0
- package/src/authorization/index.d.ts +1 -0
- package/src/authorization/index.js +5 -0
- package/src/authorization/index.js.map +1 -0
- package/src/blueprints/execution.d.ts +25 -1
- package/src/blueprints/execution.js +246 -83
- package/src/blueprints/execution.js.map +1 -1
- package/src/blueprints/external-sandbox.d.ts +53 -0
- package/src/blueprints/external-sandbox.js +1182 -0
- package/src/blueprints/external-sandbox.js.map +1 -0
- package/src/blueprints/manifest.d.ts +42 -2
- package/src/blueprints/manifest.js +82 -27
- package/src/blueprints/manifest.js.map +1 -1
- package/src/blueprints/resolver.d.ts +14 -5
- package/src/blueprints/resolver.js +228 -68
- package/src/blueprints/resolver.js.map +1 -1
- package/src/cli/agent-command.d.ts +30 -0
- package/src/cli/agent-command.js +204 -0
- package/src/cli/agent-command.js.map +1 -0
- package/src/cli/agent-runtime.d.ts +25 -0
- package/src/cli/agent-runtime.js +488 -0
- package/src/cli/agent-runtime.js.map +1 -0
- package/src/cli/blueprint-authority.d.ts +9 -0
- package/src/cli/blueprint-authority.js +43 -0
- package/src/cli/blueprint-authority.js.map +1 -0
- package/src/cli/blueprint-command.d.ts +24 -0
- package/src/cli/blueprint-command.js +63 -0
- package/src/cli/blueprint-command.js.map +1 -0
- package/src/cli/bootstrap.d.ts +31 -0
- package/src/cli/bootstrap.js +165 -0
- package/src/cli/bootstrap.js.map +1 -0
- package/src/cli/dev-command.d.ts +92 -0
- package/src/cli/dev-command.js +561 -0
- package/src/cli/dev-command.js.map +1 -0
- package/src/cli/main.d.ts +131 -0
- package/src/cli/main.js +1199 -0
- package/src/cli/main.js.map +1 -0
- package/src/cli/output.d.ts +19 -0
- package/src/cli/output.js +51 -0
- package/src/cli/output.js.map +1 -0
- package/src/cli/personal-secret-runtime.d.ts +81 -0
- package/src/cli/personal-secret-runtime.js +567 -0
- package/src/cli/personal-secret-runtime.js.map +1 -0
- package/src/cli/secret-terminal.d.ts +12 -0
- package/src/cli/secret-terminal.js +128 -0
- package/src/cli/secret-terminal.js.map +1 -0
- package/src/container-images.d.ts +1 -0
- package/src/container-images.js +5 -0
- package/src/container-images.js.map +1 -0
- package/src/data-lifecycle/index.d.ts +3 -0
- package/src/data-lifecycle/index.js +7 -0
- package/src/data-lifecycle/index.js.map +1 -0
- package/src/data-lifecycle/model.d.ts +172 -0
- package/src/data-lifecycle/model.fixtures.d.ts +2 -0
- package/src/data-lifecycle/model.fixtures.js +244 -0
- package/src/data-lifecycle/model.fixtures.js.map +1 -0
- package/src/data-lifecycle/model.js +95 -0
- package/src/data-lifecycle/model.js.map +1 -0
- package/src/data-lifecycle/sensitive-artifacts.d.ts +28 -0
- package/src/data-lifecycle/sensitive-artifacts.js +146 -0
- package/src/data-lifecycle/sensitive-artifacts.js.map +1 -0
- package/src/data-lifecycle/validation.d.ts +32 -0
- package/src/data-lifecycle/validation.js +837 -0
- package/src/data-lifecycle/validation.js.map +1 -0
- package/src/database-retirement/contract.d.ts +68 -0
- package/src/database-retirement/contract.js +166 -0
- package/src/database-retirement/contract.js.map +1 -0
- package/src/database-retirement/index.d.ts +1 -0
- package/src/database-retirement/index.js +5 -0
- package/src/database-retirement/index.js.map +1 -0
- package/src/deterministic-order.d.ts +2 -0
- package/src/deterministic-order.js +7 -0
- package/src/deterministic-order.js.map +1 -0
- package/src/developer-agent/context.d.ts +27 -0
- package/src/developer-agent/context.js +497 -0
- package/src/developer-agent/context.js.map +1 -0
- package/src/developer-agent/generator-planning.d.ts +34 -0
- package/src/developer-agent/generator-planning.js +645 -0
- package/src/developer-agent/generator-planning.js.map +1 -0
- package/src/developer-agent/generator-preview.d.ts +8 -0
- package/src/developer-agent/generator-preview.js +650 -0
- package/src/developer-agent/generator-preview.js.map +1 -0
- package/src/developer-agent/git-inspection.d.ts +108 -0
- package/src/developer-agent/git-inspection.js +815 -0
- package/src/developer-agent/git-inspection.js.map +1 -0
- package/src/developer-agent/handoff.d.ts +43 -0
- package/src/developer-agent/handoff.js +698 -0
- package/src/developer-agent/handoff.js.map +1 -0
- package/src/developer-agent/index.d.ts +10 -0
- package/src/developer-agent/index.js +14 -0
- package/src/developer-agent/index.js.map +1 -0
- package/src/developer-agent/model.d.ts +481 -0
- package/src/developer-agent/model.js +143 -0
- package/src/developer-agent/model.js.map +1 -0
- package/src/developer-agent/operation-registry.d.ts +45 -0
- package/src/developer-agent/operation-registry.js +1255 -0
- package/src/developer-agent/operation-registry.js.map +1 -0
- package/src/developer-agent/ordering.d.ts +9 -0
- package/src/developer-agent/ordering.js +15 -0
- package/src/developer-agent/ordering.js.map +1 -0
- package/src/developer-agent/planning.d.ts +18 -0
- package/src/developer-agent/planning.js +447 -0
- package/src/developer-agent/planning.js.map +1 -0
- package/src/developer-agent/preview-changes.d.ts +26 -0
- package/src/developer-agent/preview-changes.js +203 -0
- package/src/developer-agent/preview-changes.js.map +1 -0
- package/src/developer-agent/preview-source.d.ts +2 -0
- package/src/developer-agent/preview-source.js +16 -0
- package/src/developer-agent/preview-source.js.map +1 -0
- package/src/developer-agent/validation.d.ts +18 -0
- package/src/developer-agent/validation.js +2617 -0
- package/src/developer-agent/validation.js.map +1 -0
- package/src/developer-agent/workspace-inspection.d.ts +20 -0
- package/src/developer-agent/workspace-inspection.js +1134 -0
- package/src/developer-agent/workspace-inspection.js.map +1 -0
- package/src/developer-agent/workspace-path.d.ts +6 -0
- package/src/developer-agent/workspace-path.js +22 -0
- package/src/developer-agent/workspace-path.js.map +1 -0
- package/src/executors/blueprint-trust/blueprint-trust.d.ts +29 -0
- package/src/executors/blueprint-trust/blueprint-trust.js +115 -0
- package/src/executors/blueprint-trust/blueprint-trust.js.map +1 -0
- package/src/executors/blueprint-trust/schema.json +27 -0
- package/src/executors/blueprint-verification/blueprint-verification.d.ts +13 -0
- package/src/executors/blueprint-verification/blueprint-verification.js +110 -0
- package/src/executors/blueprint-verification/blueprint-verification.js.map +1 -0
- package/src/executors/blueprint-verification/schema.json +44 -0
- package/src/executors/deployment-plan/deployment-plan.js +1 -0
- package/src/executors/deployment-plan/deployment-plan.js.map +1 -1
- package/src/executors/dev-doctor/dev-doctor.d.ts +19 -5
- package/src/executors/dev-doctor/dev-doctor.js +186 -32
- package/src/executors/dev-doctor/dev-doctor.js.map +1 -1
- package/src/executors/dev-doctor/schema.json +1 -6
- package/src/executors/dev-setup/dev-setup.d.ts +7 -1
- package/src/executors/dev-setup/dev-setup.js +14 -3
- package/src/executors/dev-setup/dev-setup.js.map +1 -1
- package/src/executors/developer-context/developer-context.d.ts +38 -0
- package/src/executors/developer-context/developer-context.js +78 -0
- package/src/executors/developer-context/developer-context.js.map +1 -0
- package/src/executors/developer-context/schema.json +87 -0
- package/src/executors/fleet/fleet.d.ts +12 -43
- package/src/executors/fleet/fleet.js +120 -500
- package/src/executors/fleet/fleet.js.map +1 -1
- package/src/executors/fleet/schema.json +64 -14
- package/src/executors/github-bootstrap/github-bootstrap.d.ts +1 -0
- package/src/executors/github-bootstrap/github-bootstrap.js +1 -0
- package/src/executors/github-bootstrap/github-bootstrap.js.map +1 -1
- package/src/executors/github-bootstrap/schema.json +5 -0
- package/src/executors/infrastructure-plan/infrastructure-plan.d.ts +144 -0
- package/src/executors/infrastructure-plan/infrastructure-plan.js +3320 -0
- package/src/executors/infrastructure-plan/infrastructure-plan.js.map +1 -0
- package/src/executors/infrastructure-plan/schema.json +78 -0
- package/src/executors/secret-guardrails/schema.json +6 -0
- package/src/executors/secret-guardrails/secret-guardrails.d.ts +3 -0
- package/src/executors/secret-guardrails/secret-guardrails.js +44 -0
- package/src/executors/secret-guardrails/secret-guardrails.js.map +1 -0
- package/src/executors/secret-metadata/schema.json +31 -0
- package/src/executors/secret-metadata/secret-metadata.d.ts +32 -0
- package/src/executors/secret-metadata/secret-metadata.js +131 -0
- package/src/executors/secret-metadata/secret-metadata.js.map +1 -0
- package/src/executors/sst-lifecycle/logging-preflight.d.ts +17 -0
- package/src/executors/sst-lifecycle/logging-preflight.js +196 -0
- package/src/executors/sst-lifecycle/logging-preflight.js.map +1 -0
- package/src/executors/sst-lifecycle/run-sst-command.d.ts +23 -3
- package/src/executors/sst-lifecycle/run-sst-command.js +113 -10
- package/src/executors/sst-lifecycle/run-sst-command.js.map +1 -1
- package/src/executors/sst-lifecycle/schema.json +9 -5
- package/src/executors/sst-lifecycle/secret-preflight.d.ts +25 -0
- package/src/executors/sst-lifecycle/secret-preflight.js +119 -0
- package/src/executors/sst-lifecycle/secret-preflight.js.map +1 -0
- package/src/foundation/discovery.d.ts +17 -1
- package/src/foundation/discovery.js +16 -1
- package/src/foundation/discovery.js.map +1 -1
- package/src/foundation/index.d.ts +3 -0
- package/src/foundation/index.js +3 -0
- package/src/foundation/index.js.map +1 -1
- package/src/foundation/local-database-contract.d.ts +1 -0
- package/src/foundation/local-database-contract.js +3 -0
- package/src/foundation/local-database-contract.js.map +1 -1
- package/src/foundation/logging-preflight.d.ts +30 -0
- package/src/foundation/logging-preflight.js +138 -0
- package/src/foundation/logging-preflight.js.map +1 -0
- package/src/foundation/logging.d.ts +70 -0
- package/src/foundation/logging.js +133 -0
- package/src/foundation/logging.js.map +1 -0
- package/src/foundation/preflight.d.ts +9 -0
- package/src/foundation/preflight.js +58 -2
- package/src/foundation/preflight.js.map +1 -1
- package/src/foundation/provision-database-bindings.d.ts +9 -1
- package/src/foundation/provision-database-bindings.js +25 -5
- package/src/foundation/provision-database-bindings.js.map +1 -1
- package/src/foundation/provision-logical-database.d.ts +10 -2
- package/src/foundation/provision-logical-database.js +68 -113
- package/src/foundation/provision-logical-database.js.map +1 -1
- package/src/foundation/router-origin-proof.d.ts +2 -0
- package/src/foundation/router-origin-proof.js +25 -0
- package/src/foundation/router-origin-proof.js.map +1 -0
- package/src/foundation/stages.d.ts +1 -0
- package/src/foundation/stages.js +9 -1
- package/src/foundation/stages.js.map +1 -1
- package/src/frontend-security/index.d.ts +1 -0
- package/src/frontend-security/index.js +5 -0
- package/src/frontend-security/index.js.map +1 -0
- package/src/frontend-security/security-headers.d.ts +50 -0
- package/src/frontend-security/security-headers.js +418 -0
- package/src/frontend-security/security-headers.js.map +1 -0
- package/src/generators/feature-registry.d.ts +13 -0
- package/src/generators/feature-registry.js +73 -0
- package/src/generators/feature-registry.js.map +1 -0
- package/src/generators/function/function.d.ts +1 -1
- package/src/generators/function/function.js +51 -6
- package/src/generators/function/function.js.map +1 -1
- package/src/generators/function/schema.d.ts +1 -0
- package/src/generators/function/schema.json +6 -1
- package/src/generators/logging-registry.d.ts +13 -0
- package/src/generators/logging-registry.js +62 -0
- package/src/generators/logging-registry.js.map +1 -0
- package/src/generators/post-generation-install.d.ts +67 -0
- package/src/generators/post-generation-install.js +3308 -0
- package/src/generators/post-generation-install.js.map +1 -0
- package/src/generators/preset/generator.js +3 -0
- package/src/generators/preset/generator.js.map +1 -1
- package/src/generators/preset/tool-versions.d.ts +18 -0
- package/src/generators/preset/tool-versions.js +22 -0
- package/src/generators/preset/tool-versions.js.map +1 -0
- package/src/generators/preset/workspace-files-developer-agent.d.ts +14 -0
- package/src/generators/preset/workspace-files-developer-agent.js +511 -0
- package/src/generators/preset/workspace-files-developer-agent.js.map +1 -0
- package/src/generators/preset/workspace-files-foundation-workload-roles.d.ts +6 -0
- package/src/generators/preset/workspace-files-foundation-workload-roles.js +406 -0
- package/src/generators/preset/workspace-files-foundation-workload-roles.js.map +1 -0
- package/src/generators/preset/workspace-files-performance-cost.d.ts +66 -0
- package/src/generators/preset/workspace-files-performance-cost.js +866 -0
- package/src/generators/preset/workspace-files-performance-cost.js.map +1 -0
- package/src/generators/preset/workspace-files-performance-load.d.ts +73 -0
- package/src/generators/preset/workspace-files-performance-load.js +801 -0
- package/src/generators/preset/workspace-files-performance-load.js.map +1 -0
- package/src/generators/preset/workspace-files-workload-role-baseline.d.ts +9 -0
- package/src/generators/preset/workspace-files-workload-role-baseline.js +1102 -0
- package/src/generators/preset/workspace-files-workload-role-baseline.js.map +1 -0
- package/src/generators/preset/workspace-files.d.ts +14 -1
- package/src/generators/preset/workspace-files.js +8189 -245
- package/src/generators/preset/workspace-files.js.map +1 -1
- package/src/generators/repository-policy/repository-policy.js +3 -47
- package/src/generators/repository-policy/repository-policy.js.map +1 -1
- package/src/generators/slice/built-in-plan-api-contract-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-api-contract-files.js +2656 -0
- package/src/generators/slice/built-in-plan-api-contract-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-api-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-api-files.js +1186 -0
- package/src/generators/slice/built-in-plan-api-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-async-work-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-async-work-files.js +7903 -0
- package/src/generators/slice/built-in-plan-async-work-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-authorization-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-authorization-files.js +2314 -0
- package/src/generators/slice/built-in-plan-authorization-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-authorization-infrastructure-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-authorization-infrastructure-files.js +559 -0
- package/src/generators/slice/built-in-plan-authorization-infrastructure-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-data-deletion-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-data-deletion-files.js +1101 -0
- package/src/generators/slice/built-in-plan-data-deletion-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-data-export-infrastructure-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-data-export-infrastructure-files.js +249 -0
- package/src/generators/slice/built-in-plan-data-export-infrastructure-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-data-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-data-files.js +1530 -0
- package/src/generators/slice/built-in-plan-data-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-data-lifecycle-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-data-lifecycle-files.js +750 -0
- package/src/generators/slice/built-in-plan-data-lifecycle-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-data-lifecycle-runtime-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-data-lifecycle-runtime-files.js +828 -0
- package/src/generators/slice/built-in-plan-data-lifecycle-runtime-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-data-retention-job-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-data-retention-job-files.js +2604 -0
- package/src/generators/slice/built-in-plan-data-retention-job-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-data-retention-launcher-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-data-retention-launcher-files.js +144 -0
- package/src/generators/slice/built-in-plan-data-retention-launcher-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-data-rights-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-data-rights-files.js +4628 -0
- package/src/generators/slice/built-in-plan-data-rights-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-database-lifecycle-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-database-lifecycle-files.js +1901 -0
- package/src/generators/slice/built-in-plan-database-lifecycle-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-database-retirement-rehearsal-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-database-retirement-rehearsal-files.js +485 -0
- package/src/generators/slice/built-in-plan-database-retirement-rehearsal-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-foundation-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-foundation-files.js +184 -0
- package/src/generators/slice/built-in-plan-foundation-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-frontend-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-frontend-files.js +2233 -0
- package/src/generators/slice/built-in-plan-frontend-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-infrastructure-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-infrastructure-files.js +3038 -0
- package/src/generators/slice/built-in-plan-infrastructure-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-infrastructure-safety-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-infrastructure-safety-files.js +312 -0
- package/src/generators/slice/built-in-plan-infrastructure-safety-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-metadata-files.d.ts +3 -0
- package/src/generators/slice/built-in-plan-metadata-files.js +219 -0
- package/src/generators/slice/built-in-plan-metadata-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-operations-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-operations-files.js +630 -0
- package/src/generators/slice/built-in-plan-operations-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-performance-files.d.ts +3 -0
- package/src/generators/slice/built-in-plan-performance-files.js +3293 -0
- package/src/generators/slice/built-in-plan-performance-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-project-files.d.ts +3 -0
- package/src/generators/slice/built-in-plan-project-files.js +979 -0
- package/src/generators/slice/built-in-plan-project-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-release-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-release-files.js +6387 -0
- package/src/generators/slice/built-in-plan-release-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-reliability-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-reliability-files.js +4499 -0
- package/src/generators/slice/built-in-plan-reliability-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-secret-runtime-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-secret-runtime-files.js +1004 -0
- package/src/generators/slice/built-in-plan-secret-runtime-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-shared.d.ts +53 -0
- package/src/generators/slice/built-in-plan-shared.js +113 -0
- package/src/generators/slice/built-in-plan-shared.js.map +1 -0
- package/src/generators/slice/built-in-plan-testing-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-testing-files.js +2356 -0
- package/src/generators/slice/built-in-plan-testing-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-verification-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-verification-files.js +358 -0
- package/src/generators/slice/built-in-plan-verification-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-web-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-web-files.js +430 -0
- package/src/generators/slice/built-in-plan-web-files.js.map +1 -0
- package/src/generators/slice/built-in-plan-workload-role-files.d.ts +2 -0
- package/src/generators/slice/built-in-plan-workload-role-files.js +2356 -0
- package/src/generators/slice/built-in-plan-workload-role-files.js.map +1 -0
- package/src/generators/slice/built-in-plan.d.ts +8 -0
- package/src/generators/slice/built-in-plan.js +264 -0
- package/src/generators/slice/built-in-plan.js.map +1 -0
- package/src/generators/slice/schema.json +1 -1
- package/src/generators/slice/slice.d.ts +1 -22
- package/src/generators/slice/slice.js +185 -1510
- package/src/generators/slice/slice.js.map +1 -1
- package/src/generators/upgrade/schema.d.ts +2 -0
- package/src/generators/upgrade/schema.json +11 -1
- package/src/generators/upgrade/upgrade.d.ts +51 -1
- package/src/generators/upgrade/upgrade.js +820 -82
- package/src/generators/upgrade/upgrade.js.map +1 -1
- package/src/generators/workload/frozen-container.d.ts +8 -0
- package/src/generators/workload/frozen-container.js +36 -0
- package/src/generators/workload/frozen-container.js.map +1 -0
- package/src/generators/workload/performance-registration.d.ts +32 -0
- package/src/generators/workload/performance-registration.js +226 -0
- package/src/generators/workload/performance-registration.js.map +1 -0
- package/src/generators/workload/reliability-registration.d.ts +16 -0
- package/src/generators/workload/reliability-registration.js +150 -0
- package/src/generators/workload/reliability-registration.js.map +1 -0
- package/src/generators/workload/schema.d.ts +1 -0
- package/src/generators/workload/schema.json +6 -1
- package/src/generators/workload/service-performance-files.d.ts +4 -0
- package/src/generators/workload/service-performance-files.js +1846 -0
- package/src/generators/workload/service-performance-files.js.map +1 -0
- package/src/generators/workload/workload.d.ts +3 -2
- package/src/generators/workload/workload.js +1083 -18
- package/src/generators/workload/workload.js.map +1 -1
- package/src/git-policy/action-pins.d.ts +10 -0
- package/src/git-policy/action-pins.js +10 -0
- package/src/git-policy/action-pins.js.map +1 -1
- package/src/git-policy/github-bootstrap.d.ts +14 -1
- package/src/git-policy/github-bootstrap.js +274 -46
- package/src/git-policy/github-bootstrap.js.map +1 -1
- package/src/index.d.ts +2 -0
- package/src/index.js +2 -0
- package/src/index.js.map +1 -1
- package/src/infrastructure-change/convergence.d.ts +13 -0
- package/src/infrastructure-change/convergence.js +85 -0
- package/src/infrastructure-change/convergence.js.map +1 -0
- package/src/infrastructure-change/drift.d.ts +13 -0
- package/src/infrastructure-change/drift.js +112 -0
- package/src/infrastructure-change/drift.js.map +1 -0
- package/src/infrastructure-change/index.d.ts +7 -0
- package/src/infrastructure-change/index.js +11 -0
- package/src/infrastructure-change/index.js.map +1 -0
- package/src/infrastructure-change/model.d.ts +253 -0
- package/src/infrastructure-change/model.js +28 -0
- package/src/infrastructure-change/model.js.map +1 -0
- package/src/infrastructure-change/normalize.d.ts +12 -0
- package/src/infrastructure-change/normalize.js +245 -0
- package/src/infrastructure-change/normalize.js.map +1 -0
- package/src/infrastructure-change/plan.d.ts +40 -0
- package/src/infrastructure-change/plan.js +361 -0
- package/src/infrastructure-change/plan.js.map +1 -0
- package/src/infrastructure-change/risk.d.ts +7 -0
- package/src/infrastructure-change/risk.js +269 -0
- package/src/infrastructure-change/risk.js.map +1 -0
- package/src/infrastructure-change/summary.d.ts +20 -0
- package/src/infrastructure-change/summary.js +130 -0
- package/src/infrastructure-change/summary.js.map +1 -0
- package/src/local-dev/context-discovery.d.ts +2 -0
- package/src/local-dev/context-discovery.js +59 -0
- package/src/local-dev/context-discovery.js.map +1 -0
- package/src/local-dev/contract-validation.d.ts +15 -0
- package/src/local-dev/contract-validation.js +84 -0
- package/src/local-dev/contract-validation.js.map +1 -0
- package/src/local-dev/contracts.d.ts +10 -1
- package/src/local-dev/contracts.js +57 -5
- package/src/local-dev/contracts.js.map +1 -1
- package/src/local-dev/control-contract.d.ts +67 -0
- package/src/local-dev/control-contract.js +180 -0
- package/src/local-dev/control-contract.js.map +1 -0
- package/src/local-dev/dev-command-core.d.ts +60 -0
- package/src/local-dev/dev-command-core.js +89 -0
- package/src/local-dev/dev-command-core.js.map +1 -0
- package/src/local-dev/effects.d.ts +16 -0
- package/src/local-dev/effects.js +63 -0
- package/src/local-dev/effects.js.map +1 -0
- package/src/local-dev/environment-inspector.d.ts +125 -0
- package/src/local-dev/environment-inspector.js +687 -0
- package/src/local-dev/environment-inspector.js.map +1 -0
- package/src/local-dev/environment-state.d.ts +113 -0
- package/src/local-dev/environment-state.js +463 -0
- package/src/local-dev/environment-state.js.map +1 -0
- package/src/local-dev/errors.d.ts +6 -0
- package/src/local-dev/errors.js +41 -0
- package/src/local-dev/errors.js.map +1 -0
- package/src/local-dev/fleet-client.d.ts +33 -0
- package/src/local-dev/fleet-client.js +156 -0
- package/src/local-dev/fleet-client.js.map +1 -0
- package/src/local-dev/fleet-configuration.d.ts +13 -0
- package/src/local-dev/fleet-configuration.js +71 -0
- package/src/local-dev/fleet-configuration.js.map +1 -0
- package/src/local-dev/fleet-inventory.d.ts +56 -0
- package/src/local-dev/fleet-inventory.js +469 -0
- package/src/local-dev/fleet-inventory.js.map +1 -0
- package/src/local-dev/fleet-ipc.d.ts +28 -0
- package/src/local-dev/fleet-ipc.js +267 -0
- package/src/local-dev/fleet-ipc.js.map +1 -0
- package/src/local-dev/fleet-launcher.d.ts +53 -0
- package/src/local-dev/fleet-launcher.js +379 -0
- package/src/local-dev/fleet-launcher.js.map +1 -0
- package/src/local-dev/fleet-lock.d.ts +41 -0
- package/src/local-dev/fleet-lock.js +135 -0
- package/src/local-dev/fleet-lock.js.map +1 -0
- package/src/local-dev/fleet-logs.d.ts +66 -0
- package/src/local-dev/fleet-logs.js +542 -0
- package/src/local-dev/fleet-logs.js.map +1 -0
- package/src/local-dev/fleet-process.d.ts +50 -0
- package/src/local-dev/fleet-process.js +289 -0
- package/src/local-dev/fleet-process.js.map +1 -0
- package/src/local-dev/fleet-runtime.d.ts +22 -0
- package/src/local-dev/fleet-runtime.js +368 -0
- package/src/local-dev/fleet-runtime.js.map +1 -0
- package/src/local-dev/fleet-session.d.ts +70 -0
- package/src/local-dev/fleet-session.js +301 -0
- package/src/local-dev/fleet-session.js.map +1 -0
- package/src/local-dev/fleet-slice-runner-daemon.d.ts +1 -0
- package/src/local-dev/fleet-slice-runner-daemon.js +21 -0
- package/src/local-dev/fleet-slice-runner-daemon.js.map +1 -0
- package/src/local-dev/fleet-startup.d.ts +22 -0
- package/src/local-dev/fleet-startup.js +114 -0
- package/src/local-dev/fleet-startup.js.map +1 -0
- package/src/local-dev/fleet-state.d.ts +62 -0
- package/src/local-dev/fleet-state.js +297 -0
- package/src/local-dev/fleet-state.js.map +1 -0
- package/src/local-dev/fleet-supervisor-daemon.d.ts +1 -0
- package/src/local-dev/fleet-supervisor-daemon.js +41 -0
- package/src/local-dev/fleet-supervisor-daemon.js.map +1 -0
- package/src/local-dev/fleet-supervisor.d.ts +52 -0
- package/src/local-dev/fleet-supervisor.js +384 -0
- package/src/local-dev/fleet-supervisor.js.map +1 -0
- package/src/local-dev/gateway.js +1 -1
- package/src/local-dev/gateway.js.map +1 -1
- package/src/local-dev/identity.js +4 -6
- package/src/local-dev/identity.js.map +1 -1
- package/src/local-dev/index.d.ts +1 -0
- package/src/local-dev/index.js +1 -0
- package/src/local-dev/index.js.map +1 -1
- package/src/local-dev/local-context.d.ts +62 -0
- package/src/local-dev/local-context.js +299 -0
- package/src/local-dev/local-context.js.map +1 -0
- package/src/local-dev/nx-workspace.d.ts +45 -0
- package/src/local-dev/nx-workspace.js +228 -0
- package/src/local-dev/nx-workspace.js.map +1 -0
- package/src/local-dev/personal-secret-ledger.d.ts +75 -0
- package/src/local-dev/personal-secret-ledger.js +391 -0
- package/src/local-dev/personal-secret-ledger.js.map +1 -0
- package/src/local-dev/personal-stage.d.ts +18 -0
- package/src/local-dev/personal-stage.js +129 -0
- package/src/local-dev/personal-stage.js.map +1 -0
- package/src/local-dev/private-state.d.ts +17 -0
- package/src/local-dev/private-state.js +254 -0
- package/src/local-dev/private-state.js.map +1 -0
- package/src/local-dev/process-identity.d.ts +9 -0
- package/src/local-dev/process-identity.js +79 -0
- package/src/local-dev/process-identity.js.map +1 -0
- package/src/local-dev/runtime-paths.d.ts +17 -0
- package/src/local-dev/runtime-paths.js +33 -0
- package/src/local-dev/runtime-paths.js.map +1 -0
- package/src/local-dev/secret-slice-claim.d.ts +17 -0
- package/src/local-dev/secret-slice-claim.js +75 -0
- package/src/local-dev/secret-slice-claim.js.map +1 -0
- package/src/local-dev/slice-claim.d.ts +36 -0
- package/src/local-dev/slice-claim.js +199 -0
- package/src/local-dev/slice-claim.js.map +1 -0
- package/src/local-dev/slice-runner.d.ts +121 -0
- package/src/local-dev/slice-runner.js +715 -0
- package/src/local-dev/slice-runner.js.map +1 -0
- package/src/local-dev/tls.js +51 -17
- package/src/local-dev/tls.js.map +1 -1
- package/src/ownership/composition.d.ts +5 -2
- package/src/ownership/composition.js +26 -3
- package/src/ownership/composition.js.map +1 -1
- package/src/ownership/hash.d.ts +1 -1
- package/src/ownership/hash.js +3 -1
- package/src/ownership/hash.js.map +1 -1
- package/src/ownership/record.d.ts +32 -1
- package/src/ownership/record.js +139 -2
- package/src/ownership/record.js.map +1 -1
- package/src/performance/cost-collection.d.ts +225 -0
- package/src/performance/cost-collection.js +825 -0
- package/src/performance/cost-collection.js.map +1 -0
- package/src/performance/cost-controls.d.ts +131 -0
- package/src/performance/cost-controls.js +336 -0
- package/src/performance/cost-controls.js.map +1 -0
- package/src/performance/evidence.d.ts +140 -0
- package/src/performance/evidence.js +599 -0
- package/src/performance/evidence.js.map +1 -0
- package/src/performance/index.d.ts +10 -0
- package/src/performance/index.js +14 -0
- package/src/performance/index.js.map +1 -0
- package/src/performance/load-safety.d.ts +84 -0
- package/src/performance/load-safety.js +346 -0
- package/src/performance/load-safety.js.map +1 -0
- package/src/performance/model.d.ts +129 -0
- package/src/performance/model.js +452 -0
- package/src/performance/model.js.map +1 -0
- package/src/performance/pr-artifact-budget.d.ts +42 -0
- package/src/performance/pr-artifact-budget.js +151 -0
- package/src/performance/pr-artifact-budget.js.map +1 -0
- package/src/performance/runner.d.ts +75 -0
- package/src/performance/runner.js +227 -0
- package/src/performance/runner.js.map +1 -0
- package/src/performance/scenario-endpoint.d.ts +1 -0
- package/src/performance/scenario-endpoint.js +5 -0
- package/src/performance/scenario-endpoint.js.map +1 -0
- package/src/performance/scenario.d.ts +193 -0
- package/src/performance/scenario.js +736 -0
- package/src/performance/scenario.js.map +1 -0
- package/src/performance/test-fixtures.d.ts +7 -0
- package/src/performance/test-fixtures.js +169 -0
- package/src/performance/test-fixtures.js.map +1 -0
- package/src/performance/unit-cost.d.ts +105 -0
- package/src/performance/unit-cost.js +364 -0
- package/src/performance/unit-cost.js.map +1 -0
- package/src/release/index.d.ts +4 -0
- package/src/release/index.js +8 -0
- package/src/release/index.js.map +1 -0
- package/src/release/lifecycle.d.ts +132 -0
- package/src/release/lifecycle.js +411 -0
- package/src/release/lifecycle.js.map +1 -0
- package/src/release/manifest.d.ts +148 -0
- package/src/release/manifest.js +503 -0
- package/src/release/manifest.js.map +1 -0
- package/src/release/rollout.d.ts +170 -0
- package/src/release/rollout.js +578 -0
- package/src/release/rollout.js.map +1 -0
- package/src/release/source-authority.d.ts +19 -0
- package/src/release/source-authority.js +195 -0
- package/src/release/source-authority.js.map +1 -0
- package/src/repository-policy/deployment-plan.d.ts +2 -1
- package/src/repository-policy/deployment-plan.js +2 -0
- package/src/repository-policy/deployment-plan.js.map +1 -1
- package/src/repository-policy/policy.d.ts +1 -1
- package/src/repository-policy/policy.js +29 -25
- package/src/repository-policy/policy.js.map +1 -1
- package/src/schemas/aggregate-count-evidence-v1.json +135 -0
- package/src/schemas/api-contract-baseline-v1.json +674 -0
- package/src/schemas/api-contract-compatibility-report-v1.json +126 -0
- package/src/schemas/aws-billing-actual-evidence-v1.json +407 -0
- package/src/schemas/blueprint-v1.json +18 -1
- package/src/schemas/data-lifecycle-policy-v1.json +517 -0
- package/src/schemas/generated-ownership-v2.json +22 -0
- package/src/schemas/performance-policy-v1.json +335 -0
- package/src/schemas/repository-policy-v1.json +1 -0
- package/src/schemas/testing-evidence-quarantine-v1.json +168 -0
- package/src/schemas/testing-evidence-result-v1.json +319 -0
- package/src/secrets/adapters.d.ts +49 -0
- package/src/secrets/adapters.js +184 -0
- package/src/secrets/adapters.js.map +1 -0
- package/src/secrets/aws-control-plane.d.ts +20 -0
- package/src/secrets/aws-control-plane.js +255 -0
- package/src/secrets/aws-control-plane.js.map +1 -0
- package/src/secrets/declaration-verification.d.ts +11 -0
- package/src/secrets/declaration-verification.js +68 -0
- package/src/secrets/declaration-verification.js.map +1 -0
- package/src/secrets/guardrails.d.ts +26 -0
- package/src/secrets/guardrails.js +328 -0
- package/src/secrets/guardrails.js.map +1 -0
- package/src/secrets/index.d.ts +12 -0
- package/src/secrets/index.js +16 -0
- package/src/secrets/index.js.map +1 -0
- package/src/secrets/metadata.d.ts +48 -0
- package/src/secrets/metadata.js +171 -0
- package/src/secrets/metadata.js.map +1 -0
- package/src/secrets/model.d.ts +49 -0
- package/src/secrets/model.js +125 -0
- package/src/secrets/model.js.map +1 -0
- package/src/secrets/operator.d.ts +101 -0
- package/src/secrets/operator.js +229 -0
- package/src/secrets/operator.js.map +1 -0
- package/src/secrets/permissions.d.ts +34 -0
- package/src/secrets/permissions.js +86 -0
- package/src/secrets/permissions.js.map +1 -0
- package/src/secrets/personal-command.d.ts +83 -0
- package/src/secrets/personal-command.js +197 -0
- package/src/secrets/personal-command.js.map +1 -0
- package/src/secrets/personal-ledger.d.ts +21 -0
- package/src/secrets/personal-ledger.js +135 -0
- package/src/secrets/personal-ledger.js.map +1 -0
- package/src/secrets/registry.d.ts +9 -0
- package/src/secrets/registry.js +86 -0
- package/src/secrets/registry.js.map +1 -0
- package/src/secrets/verification.d.ts +18 -0
- package/src/secrets/verification.js +139 -0
- package/src/secrets/verification.js.map +1 -0
- package/src/secrets/workload-access.d.ts +46 -0
- package/src/secrets/workload-access.js +163 -0
- package/src/secrets/workload-access.js.map +1 -0
- package/src/supply-chain/action-pins.d.ts +26 -0
- package/src/supply-chain/action-pins.js +208 -0
- package/src/supply-chain/action-pins.js.map +1 -0
- package/src/supply-chain/blueprint-trust-git.d.ts +130 -0
- package/src/supply-chain/blueprint-trust-git.js +1581 -0
- package/src/supply-chain/blueprint-trust-git.js.map +1 -0
- package/src/supply-chain/blueprint-trust-process.d.ts +4 -0
- package/src/supply-chain/blueprint-trust-process.js +88 -0
- package/src/supply-chain/blueprint-trust-process.js.map +1 -0
- package/src/supply-chain/blueprint-trust.d.ts +354 -0
- package/src/supply-chain/blueprint-trust.js +1558 -0
- package/src/supply-chain/blueprint-trust.js.map +1 -0
- package/src/supply-chain/blueprint-verification-authority.d.ts +62 -0
- package/src/supply-chain/blueprint-verification-authority.js +588 -0
- package/src/supply-chain/blueprint-verification-authority.js.map +1 -0
- package/src/supply-chain/blueprint-verification-execution.d.ts +68 -0
- package/src/supply-chain/blueprint-verification-execution.js +624 -0
- package/src/supply-chain/blueprint-verification-execution.js.map +1 -0
- package/src/supply-chain/build-output.d.ts +35 -0
- package/src/supply-chain/build-output.js +398 -0
- package/src/supply-chain/build-output.js.map +1 -0
- package/src/supply-chain/common.d.ts +21 -0
- package/src/supply-chain/common.js +107 -0
- package/src/supply-chain/common.js.map +1 -0
- package/src/supply-chain/container-artifact-contract.d.ts +23 -0
- package/src/supply-chain/container-artifact-contract.js +208 -0
- package/src/supply-chain/container-artifact-contract.js.map +1 -0
- package/src/supply-chain/container-release.d.ts +119 -0
- package/src/supply-chain/container-release.js +496 -0
- package/src/supply-chain/container-release.js.map +1 -0
- package/src/supply-chain/ecr-applicability.d.ts +19 -0
- package/src/supply-chain/ecr-applicability.js +103 -0
- package/src/supply-chain/ecr-applicability.js.map +1 -0
- package/src/supply-chain/ecr-scanning.d.ts +83 -0
- package/src/supply-chain/ecr-scanning.js +321 -0
- package/src/supply-chain/ecr-scanning.js.map +1 -0
- package/src/supply-chain/ecs-deployed-image.d.ts +85 -0
- package/src/supply-chain/ecs-deployed-image.js +300 -0
- package/src/supply-chain/ecs-deployed-image.js.map +1 -0
- package/src/supply-chain/errors.d.ts +6 -0
- package/src/supply-chain/errors.js +15 -0
- package/src/supply-chain/errors.js.map +1 -0
- package/src/supply-chain/generated-blueprint-trust-command.d.ts +4 -0
- package/src/supply-chain/generated-blueprint-trust-command.js +1362 -0
- package/src/supply-chain/generated-blueprint-trust-command.js.map +1 -0
- package/src/supply-chain/generated-command.d.ts +1 -0
- package/src/supply-chain/generated-command.js +687 -0
- package/src/supply-chain/generated-command.js.map +1 -0
- package/src/supply-chain/generated-container-release-command.d.ts +16 -0
- package/src/supply-chain/generated-container-release-command.js +2087 -0
- package/src/supply-chain/generated-container-release-command.js.map +1 -0
- package/src/supply-chain/generated-ecr-collector-command.d.ts +21 -0
- package/src/supply-chain/generated-ecr-collector-command.js +1338 -0
- package/src/supply-chain/generated-ecr-collector-command.js.map +1 -0
- package/src/supply-chain/generated-files.d.ts +13 -0
- package/src/supply-chain/generated-files.js +401 -0
- package/src/supply-chain/generated-files.js.map +1 -0
- package/src/supply-chain/generated-scanner-command.d.ts +1 -0
- package/src/supply-chain/generated-scanner-command.js +718 -0
- package/src/supply-chain/generated-scanner-command.js.map +1 -0
- package/src/supply-chain/index.d.ts +28 -0
- package/src/supply-chain/index.js +35 -0
- package/src/supply-chain/index.js.map +1 -0
- package/src/supply-chain/inventory.d.ts +79 -0
- package/src/supply-chain/inventory.js +717 -0
- package/src/supply-chain/inventory.js.map +1 -0
- package/src/supply-chain/license-expression.d.ts +7 -0
- package/src/supply-chain/license-expression.js +128 -0
- package/src/supply-chain/license-expression.js.map +1 -0
- package/src/supply-chain/pnpm-toolchain-authority.d.ts +31 -0
- package/src/supply-chain/pnpm-toolchain-authority.js +316 -0
- package/src/supply-chain/pnpm-toolchain-authority.js.map +1 -0
- package/src/supply-chain/policy.d.ts +105 -0
- package/src/supply-chain/policy.js +301 -0
- package/src/supply-chain/policy.js.map +1 -0
- package/src/supply-chain/publication.d.ts +48 -0
- package/src/supply-chain/publication.js +85 -0
- package/src/supply-chain/publication.js.map +1 -0
- package/src/supply-chain/sbom.d.ts +228 -0
- package/src/supply-chain/sbom.js +826 -0
- package/src/supply-chain/sbom.js.map +1 -0
- package/src/supply-chain/scanner.d.ts +77 -0
- package/src/supply-chain/scanner.js +502 -0
- package/src/supply-chain/scanner.js.map +1 -0
- package/src/supply-chain/vendored-source.d.ts +95 -0
- package/src/supply-chain/vendored-source.js +315 -0
- package/src/supply-chain/vendored-source.js.map +1 -0
- package/src/testing-evidence/fixture-scope.d.ts +52 -0
- package/src/testing-evidence/fixture-scope.js +140 -0
- package/src/testing-evidence/fixture-scope.js.map +1 -0
- package/src/testing-evidence/index.d.ts +5 -0
- package/src/testing-evidence/index.js +9 -0
- package/src/testing-evidence/index.js.map +1 -0
- package/src/testing-evidence/quarantine.d.ts +48 -0
- package/src/testing-evidence/quarantine.js +253 -0
- package/src/testing-evidence/quarantine.js.map +1 -0
- package/src/testing-evidence/result.d.ts +108 -0
- package/src/testing-evidence/result.js +571 -0
- package/src/testing-evidence/result.js.map +1 -0
- package/src/testing-evidence/runner-adapter.d.ts +51 -0
- package/src/testing-evidence/runner-adapter.js +288 -0
- package/src/testing-evidence/runner-adapter.js.map +1 -0
- package/src/testing-evidence/sensitive-artifacts.d.ts +13 -0
- package/src/testing-evidence/sensitive-artifacts.js +93 -0
- package/src/testing-evidence/sensitive-artifacts.js.map +1 -0
- package/src/work-markers/work-markers.js +4 -3
- package/src/work-markers/work-markers.js.map +1 -1
- package/src/workload-roles/account-baseline.d.ts +132 -0
- package/src/workload-roles/account-baseline.js +743 -0
- package/src/workload-roles/account-baseline.js.map +1 -0
- package/src/workload-roles/authority.d.ts +38 -0
- package/src/workload-roles/authority.js +156 -0
- package/src/workload-roles/authority.js.map +1 -0
- package/src/workload-roles/aws-adapter.d.ts +99 -0
- package/src/workload-roles/aws-adapter.js +416 -0
- package/src/workload-roles/aws-adapter.js.map +1 -0
- package/src/workload-roles/aws-internal-handler.d.ts +27 -0
- package/src/workload-roles/aws-internal-handler.js +58 -0
- package/src/workload-roles/aws-internal-handler.js.map +1 -0
- package/src/workload-roles/aws-s3-state.d.ts +52 -0
- package/src/workload-roles/aws-s3-state.js +164 -0
- package/src/workload-roles/aws-s3-state.js.map +1 -0
- package/src/workload-roles/aws-sdk-control-plane.d.ts +12 -0
- package/src/workload-roles/aws-sdk-control-plane.js +291 -0
- package/src/workload-roles/aws-sdk-control-plane.js.map +1 -0
- package/src/workload-roles/boundary.d.ts +33 -0
- package/src/workload-roles/boundary.js +130 -0
- package/src/workload-roles/boundary.js.map +1 -0
- package/src/workload-roles/convergence.d.ts +66 -0
- package/src/workload-roles/convergence.js +186 -0
- package/src/workload-roles/convergence.js.map +1 -0
- package/src/workload-roles/effective-policy.d.ts +51 -0
- package/src/workload-roles/effective-policy.js +183 -0
- package/src/workload-roles/effective-policy.js.map +1 -0
- package/src/workload-roles/handler.d.ts +43 -0
- package/src/workload-roles/handler.js +130 -0
- package/src/workload-roles/handler.js.map +1 -0
- package/src/workload-roles/history.d.ts +47 -0
- package/src/workload-roles/history.js +324 -0
- package/src/workload-roles/history.js.map +1 -0
- package/src/workload-roles/index.d.ts +17 -0
- package/src/workload-roles/index.js +21 -0
- package/src/workload-roles/index.js.map +1 -0
- package/src/workload-roles/injection.d.ts +119 -0
- package/src/workload-roles/injection.js +495 -0
- package/src/workload-roles/injection.js.map +1 -0
- package/src/workload-roles/manifest.d.ts +54 -0
- package/src/workload-roles/manifest.js +343 -0
- package/src/workload-roles/manifest.js.map +1 -0
- package/src/workload-roles/provisioner.d.ts +70 -0
- package/src/workload-roles/provisioner.js +231 -0
- package/src/workload-roles/provisioner.js.map +1 -0
- package/src/workload-roles/registration.d.ts +101 -0
- package/src/workload-roles/registration.js +457 -0
- package/src/workload-roles/registration.js.map +1 -0
- package/src/workload-roles/stage-boundary.d.ts +31 -0
- package/src/workload-roles/stage-boundary.js +208 -0
- package/src/workload-roles/stage-boundary.js.map +1 -0
- package/src/workload-roles/standard-slice.d.ts +54 -0
- package/src/workload-roles/standard-slice.js +252 -0
- package/src/workload-roles/standard-slice.js.map +1 -0
- package/migrations/standard-pre-v1-to-v1.md +0 -11
- package/src/foundation/aurora-logical-database.d.ts +0 -76
- package/src/foundation/aurora-logical-database.js +0 -419
- package/src/foundation/aurora-logical-database.js.map +0 -1
|
@@ -0,0 +1,4499 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.renderProductReliabilityFiles = renderProductReliabilityFiles;
|
|
4
|
+
const reliability_1 = require("@empire-builder-kit/runtime/reliability");
|
|
5
|
+
const built_in_plan_shared_js_1 = require("./built-in-plan-shared.js");
|
|
6
|
+
function reliabilityContract(name, reviewedAt) {
|
|
7
|
+
return {
|
|
8
|
+
escalationRouteKey: `${name}-on-call`,
|
|
9
|
+
objectives: [
|
|
10
|
+
{
|
|
11
|
+
allowedExclusionPolicyRef: 'docs:reliability/exclusions.md',
|
|
12
|
+
burn: { fast: 14.4, slow: 6, warning: 0.5 },
|
|
13
|
+
effectiveAt: reviewedAt,
|
|
14
|
+
feature: 'operations',
|
|
15
|
+
id: 'api-availability',
|
|
16
|
+
indicator: {
|
|
17
|
+
goodEventDefinition: 'Valid API requests complete without a server failure.',
|
|
18
|
+
metricRef: 'operations:metrics/api-availability',
|
|
19
|
+
type: 'availability',
|
|
20
|
+
validEventDefinition: 'Requests reaching a declared API operation, including authorization denials.',
|
|
21
|
+
},
|
|
22
|
+
journey: 'api-critical-operation',
|
|
23
|
+
lowTraffic: { mode: 'data-insufficient' },
|
|
24
|
+
minimumValidEvents: {
|
|
25
|
+
diagnosticSample: 100,
|
|
26
|
+
fastBurnWindow: 1,
|
|
27
|
+
objectiveWindow: 100,
|
|
28
|
+
slowBurnWindow: 5,
|
|
29
|
+
},
|
|
30
|
+
owner: `${name}-product`,
|
|
31
|
+
references: {
|
|
32
|
+
alarmIds: ['api-fast-burn', 'api-slow-burn'],
|
|
33
|
+
dashboardRef: 'dashboard:operations',
|
|
34
|
+
runbookRef: 'runbook:api-recovery',
|
|
35
|
+
},
|
|
36
|
+
target: 0.99,
|
|
37
|
+
tier: 'critical',
|
|
38
|
+
window: { days: 28, kind: 'rolling' },
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
allowedExclusionPolicyRef: 'docs:reliability/exclusions.md',
|
|
42
|
+
burn: { fast: 14.4, slow: 6, warning: 0.5 },
|
|
43
|
+
effectiveAt: reviewedAt,
|
|
44
|
+
feature: 'web',
|
|
45
|
+
id: 'web-latency',
|
|
46
|
+
indicator: {
|
|
47
|
+
goodEventDefinition: 'Valid server-rendered requests complete within 1000 milliseconds.',
|
|
48
|
+
metricRef: 'operations:metrics/web-latency',
|
|
49
|
+
threshold: 1000,
|
|
50
|
+
type: 'latency',
|
|
51
|
+
unit: 'Milliseconds',
|
|
52
|
+
validEventDefinition: 'Requests reaching the generated web application.',
|
|
53
|
+
},
|
|
54
|
+
journey: 'web-critical-route',
|
|
55
|
+
lowTraffic: { mode: 'data-insufficient' },
|
|
56
|
+
minimumValidEvents: {
|
|
57
|
+
diagnosticSample: 100,
|
|
58
|
+
fastBurnWindow: 1,
|
|
59
|
+
objectiveWindow: 100,
|
|
60
|
+
slowBurnWindow: 5,
|
|
61
|
+
},
|
|
62
|
+
owner: `${name}-product`,
|
|
63
|
+
references: {
|
|
64
|
+
alarmIds: ['web-latency-fast-burn', 'web-latency-slow-burn'],
|
|
65
|
+
dashboardRef: 'dashboard:operations',
|
|
66
|
+
runbookRef: 'runbook:web-recovery',
|
|
67
|
+
},
|
|
68
|
+
target: 0.95,
|
|
69
|
+
tier: 'important',
|
|
70
|
+
window: { days: 28, kind: 'rolling' },
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
allowedExclusionPolicyRef: 'docs:reliability/exclusions.md',
|
|
74
|
+
burn: { fast: 14.4, slow: 6, warning: 0.5 },
|
|
75
|
+
effectiveAt: reviewedAt,
|
|
76
|
+
feature: 'async-work',
|
|
77
|
+
id: 'async-delivery-deadline',
|
|
78
|
+
indicator: {
|
|
79
|
+
goodEventDefinition: 'Accepted provider deliveries leave the queue within 900 seconds.',
|
|
80
|
+
metricRef: 'operations:metrics/async-delivery-deadline',
|
|
81
|
+
threshold: 900,
|
|
82
|
+
type: 'deadline',
|
|
83
|
+
unit: 'Seconds',
|
|
84
|
+
validEventDefinition: 'Provider deliveries durably accepted into the generated queue.',
|
|
85
|
+
},
|
|
86
|
+
journey: 'provider-delivery',
|
|
87
|
+
lowTraffic: { mode: 'data-insufficient' },
|
|
88
|
+
minimumValidEvents: {
|
|
89
|
+
diagnosticSample: 10,
|
|
90
|
+
fastBurnWindow: 1,
|
|
91
|
+
objectiveWindow: 10,
|
|
92
|
+
slowBurnWindow: 5,
|
|
93
|
+
},
|
|
94
|
+
owner: `${name}-product`,
|
|
95
|
+
references: {
|
|
96
|
+
alarmIds: ['provider-queue-age'],
|
|
97
|
+
dashboardRef: 'dashboard:operations',
|
|
98
|
+
runbookRef: 'runbook:async-recovery',
|
|
99
|
+
},
|
|
100
|
+
target: 0.99,
|
|
101
|
+
tier: 'important',
|
|
102
|
+
window: { days: 28, kind: 'rolling' },
|
|
103
|
+
},
|
|
104
|
+
],
|
|
105
|
+
owner: `${name}-product`,
|
|
106
|
+
providerDecision: {
|
|
107
|
+
applicationSignalsCoverage: ['lambda', 'next'],
|
|
108
|
+
basis: 'Direct CloudWatch metric math covers the generated Lambda, Next.js, queue, Service, Job, and database surfaces without making Application Signals mandatory. Re-evaluate with retained exact-candidate coverage and cost evidence.',
|
|
109
|
+
cost: {
|
|
110
|
+
currency: 'USD',
|
|
111
|
+
monthly: 0,
|
|
112
|
+
provenance: 'estimate',
|
|
113
|
+
},
|
|
114
|
+
directMetricMathCoverage: [
|
|
115
|
+
'lambda',
|
|
116
|
+
'next',
|
|
117
|
+
'queue',
|
|
118
|
+
'service',
|
|
119
|
+
'job',
|
|
120
|
+
'database',
|
|
121
|
+
'alert-delivery',
|
|
122
|
+
],
|
|
123
|
+
reviewedAt,
|
|
124
|
+
selected: 'direct-cloudwatch-metric-math',
|
|
125
|
+
},
|
|
126
|
+
schemaVersion: 1,
|
|
127
|
+
slice: name,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
function alarmRegistrations(name) {
|
|
131
|
+
const owner = `${name}-product`;
|
|
132
|
+
const route = `${name}-on-call`;
|
|
133
|
+
const common = {
|
|
134
|
+
escalationRouteKey: route,
|
|
135
|
+
expectedFalseNegativeRisk: 'Low traffic or delayed CloudWatch data can delay detection.',
|
|
136
|
+
expectedFalsePositiveRisk: 'A dependency or synthetic client can fail while the application remains healthy.',
|
|
137
|
+
missingData: 'missing',
|
|
138
|
+
owner,
|
|
139
|
+
testRef: 'evidence:reliability/failure-recovery',
|
|
140
|
+
};
|
|
141
|
+
const registrations = [
|
|
142
|
+
{
|
|
143
|
+
...common,
|
|
144
|
+
businessSymptom: 'Critical API operations are failing.',
|
|
145
|
+
condition: 'The API availability objective is consuming budget rapidly.',
|
|
146
|
+
dependencyAlarmIds: ['database-capacity'],
|
|
147
|
+
id: 'api-fast-burn',
|
|
148
|
+
runbookRef: 'runbook:api-recovery',
|
|
149
|
+
severity: 'critical',
|
|
150
|
+
},
|
|
151
|
+
{
|
|
152
|
+
...common,
|
|
153
|
+
businessSymptom: 'Critical API operations have sustained failures.',
|
|
154
|
+
condition: 'The API availability objective is consuming budget slowly.',
|
|
155
|
+
dependencyAlarmIds: ['database-capacity'],
|
|
156
|
+
id: 'api-slow-burn',
|
|
157
|
+
runbookRef: 'runbook:api-recovery',
|
|
158
|
+
severity: 'high',
|
|
159
|
+
},
|
|
160
|
+
{
|
|
161
|
+
...common,
|
|
162
|
+
businessSymptom: 'Server-rendered routes exceed the latency objective.',
|
|
163
|
+
condition: 'Web p95 latency exceeds the reviewed threshold.',
|
|
164
|
+
dependencyAlarmIds: ['database-capacity'],
|
|
165
|
+
id: 'web-latency-fast-burn',
|
|
166
|
+
runbookRef: 'runbook:web-recovery',
|
|
167
|
+
severity: 'high',
|
|
168
|
+
},
|
|
169
|
+
{
|
|
170
|
+
...common,
|
|
171
|
+
businessSymptom: 'Server-rendered routes have sustained latency degradation.',
|
|
172
|
+
condition: 'Web p95 latency remains above the reviewed threshold for the slow window.',
|
|
173
|
+
dependencyAlarmIds: ['database-capacity'],
|
|
174
|
+
id: 'web-latency-slow-burn',
|
|
175
|
+
runbookRef: 'runbook:web-recovery',
|
|
176
|
+
severity: 'high',
|
|
177
|
+
},
|
|
178
|
+
{
|
|
179
|
+
...common,
|
|
180
|
+
businessSymptom: 'Accepted provider work is missing its deadline.',
|
|
181
|
+
condition: 'Oldest provider-queue work exceeds 900 seconds.',
|
|
182
|
+
dependencyAlarmIds: [],
|
|
183
|
+
id: 'provider-queue-age',
|
|
184
|
+
runbookRef: 'runbook:async-recovery',
|
|
185
|
+
severity: 'high',
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
...common,
|
|
189
|
+
businessSymptom: 'The shared database is approaching its capacity bound.',
|
|
190
|
+
condition: 'Aurora capacity exceeds the reviewed warning threshold.',
|
|
191
|
+
dependencyAlarmIds: [],
|
|
192
|
+
id: 'database-capacity',
|
|
193
|
+
runbookRef: 'runbook:database-recovery',
|
|
194
|
+
severity: 'high',
|
|
195
|
+
},
|
|
196
|
+
{
|
|
197
|
+
...common,
|
|
198
|
+
businessSymptom: 'Expired protected performance-load control records are accumulating.',
|
|
199
|
+
condition: 'The bounded performance-load expiry cleanup reports a nonzero backlog.',
|
|
200
|
+
dependencyAlarmIds: [],
|
|
201
|
+
id: 'performance-load-cleanup-backlog',
|
|
202
|
+
runbookRef: 'runbook:performance-capacity-cost',
|
|
203
|
+
severity: 'high',
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
...common,
|
|
207
|
+
businessSymptom: 'Protected performance-load residue exceeded its cleanup deadline.',
|
|
208
|
+
condition: 'The bounded expiry worker reports residue at least 24 hours past eligible expiry.',
|
|
209
|
+
dependencyAlarmIds: ['performance-load-cleanup-backlog'],
|
|
210
|
+
id: 'performance-load-cleanup-deadline',
|
|
211
|
+
runbookRef: 'runbook:performance-capacity-cost',
|
|
212
|
+
severity: 'critical',
|
|
213
|
+
},
|
|
214
|
+
{
|
|
215
|
+
...common,
|
|
216
|
+
businessSymptom: 'The confidential-export cleanup invocation is failing.',
|
|
217
|
+
condition: 'AWS/Lambda reports an error for the exact generated confidential-export cleanup Function.',
|
|
218
|
+
dependencyAlarmIds: [],
|
|
219
|
+
id: 'data-export-cleanup-failure',
|
|
220
|
+
runbookRef: 'runbook:database-recovery',
|
|
221
|
+
severity: 'high',
|
|
222
|
+
},
|
|
223
|
+
{
|
|
224
|
+
...common,
|
|
225
|
+
businessSymptom: 'The scheduled confidential-export cleanup is not running.',
|
|
226
|
+
condition: 'AWS/Lambda reports fewer than one invocation across two consecutive hourly periods.',
|
|
227
|
+
dependencyAlarmIds: [],
|
|
228
|
+
id: 'data-export-cleanup-invocation-missing',
|
|
229
|
+
runbookRef: 'runbook:database-recovery',
|
|
230
|
+
severity: 'critical',
|
|
231
|
+
},
|
|
232
|
+
{
|
|
233
|
+
...common,
|
|
234
|
+
businessSymptom: 'Confidential-export residue exceeded its deletion deadline.',
|
|
235
|
+
condition: 'The cleanup worker reports an open export at or after its immutable 24-hour cleanup deadline.',
|
|
236
|
+
dependencyAlarmIds: [
|
|
237
|
+
'data-export-cleanup-failure',
|
|
238
|
+
'data-export-cleanup-invocation-missing',
|
|
239
|
+
],
|
|
240
|
+
id: 'data-export-cleanup-deadline',
|
|
241
|
+
runbookRef: 'runbook:database-recovery',
|
|
242
|
+
severity: 'critical',
|
|
243
|
+
},
|
|
244
|
+
{
|
|
245
|
+
...common,
|
|
246
|
+
businessSymptom: 'A generated Function has a terminal invocation error.',
|
|
247
|
+
condition: 'AWS/Lambda reports one or more errors for the exact generated cleanup Function.',
|
|
248
|
+
dependencyAlarmIds: [],
|
|
249
|
+
id: 'function-failure',
|
|
250
|
+
runbookRef: 'runbook:api-recovery',
|
|
251
|
+
severity: 'high',
|
|
252
|
+
},
|
|
253
|
+
{
|
|
254
|
+
...common,
|
|
255
|
+
businessSymptom: 'The built-in retention Job terminated unsuccessfully.',
|
|
256
|
+
condition: 'AWS Step Functions reports a failed, timed-out, or aborted execution for the exact retention state machine.',
|
|
257
|
+
dependencyAlarmIds: [],
|
|
258
|
+
id: 'retention-job-failure',
|
|
259
|
+
runbookRef: 'runbook:database-recovery',
|
|
260
|
+
severity: 'high',
|
|
261
|
+
},
|
|
262
|
+
{
|
|
263
|
+
...common,
|
|
264
|
+
businessSymptom: 'The primary alert route is failing delivery.',
|
|
265
|
+
condition: 'SNS reports one or more failed notifications.',
|
|
266
|
+
dependencyAlarmIds: [],
|
|
267
|
+
id: 'alert-delivery-degraded',
|
|
268
|
+
runbookRef: 'runbook:alert-delivery',
|
|
269
|
+
severity: 'critical',
|
|
270
|
+
},
|
|
271
|
+
{
|
|
272
|
+
...common,
|
|
273
|
+
businessSymptom: 'A managed log group contains a protected-data finding.',
|
|
274
|
+
condition: 'CloudWatch Logs reports one or more protected-data findings.',
|
|
275
|
+
dependencyAlarmIds: [],
|
|
276
|
+
id: 'data-protection-findings',
|
|
277
|
+
runbookRef: 'runbook:log-investigation',
|
|
278
|
+
severity: 'critical',
|
|
279
|
+
},
|
|
280
|
+
{
|
|
281
|
+
...common,
|
|
282
|
+
businessSymptom: 'Domain-event processing has terminal failures.',
|
|
283
|
+
condition: 'The event dead-letter queue contains one or more messages.',
|
|
284
|
+
dependencyAlarmIds: [],
|
|
285
|
+
id: 'event-dlq-depth',
|
|
286
|
+
runbookRef: 'runbook:async-recovery',
|
|
287
|
+
severity: 'high',
|
|
288
|
+
},
|
|
289
|
+
{
|
|
290
|
+
...common,
|
|
291
|
+
businessSymptom: 'Provider deliveries have terminal failures.',
|
|
292
|
+
condition: 'The provider-delivery dead-letter queue contains one or more messages.',
|
|
293
|
+
dependencyAlarmIds: [],
|
|
294
|
+
id: 'provider-dlq-depth',
|
|
295
|
+
runbookRef: 'runbook:async-recovery',
|
|
296
|
+
severity: 'high',
|
|
297
|
+
},
|
|
298
|
+
{
|
|
299
|
+
...common,
|
|
300
|
+
businessSymptom: 'An asynchronous target has terminal failures.',
|
|
301
|
+
condition: 'The asynchronous-target dead-letter queue contains one or more messages.',
|
|
302
|
+
dependencyAlarmIds: [],
|
|
303
|
+
id: 'async-target-dlq-depth',
|
|
304
|
+
runbookRef: 'runbook:async-recovery',
|
|
305
|
+
severity: 'high',
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
...common,
|
|
309
|
+
businessSymptom: 'The transactional outbox relay is failing.',
|
|
310
|
+
condition: 'The generated outbox relay reports one or more errors.',
|
|
311
|
+
dependencyAlarmIds: [],
|
|
312
|
+
id: 'outbox-relay-errors',
|
|
313
|
+
runbookRef: 'runbook:async-recovery',
|
|
314
|
+
severity: 'high',
|
|
315
|
+
},
|
|
316
|
+
{
|
|
317
|
+
...common,
|
|
318
|
+
businessSymptom: 'The asynchronous inbox cleanup job is failing.',
|
|
319
|
+
condition: 'The generated inbox cleanup job reports one or more errors.',
|
|
320
|
+
dependencyAlarmIds: [],
|
|
321
|
+
id: 'inbox-cleanup-errors',
|
|
322
|
+
runbookRef: 'runbook:async-recovery',
|
|
323
|
+
severity: 'high',
|
|
324
|
+
},
|
|
325
|
+
{
|
|
326
|
+
...common,
|
|
327
|
+
businessSymptom: 'Transactional outbox work is accumulating.',
|
|
328
|
+
condition: 'The transactional outbox backlog remains nonempty.',
|
|
329
|
+
dependencyAlarmIds: ['outbox-relay-errors'],
|
|
330
|
+
id: 'outbox-backlog',
|
|
331
|
+
runbookRef: 'runbook:async-recovery',
|
|
332
|
+
severity: 'high',
|
|
333
|
+
},
|
|
334
|
+
{
|
|
335
|
+
...common,
|
|
336
|
+
businessSymptom: 'Terminal asynchronous inbox records are accumulating.',
|
|
337
|
+
condition: 'The terminal inbox cleanup backlog remains nonempty.',
|
|
338
|
+
dependencyAlarmIds: ['inbox-cleanup-errors'],
|
|
339
|
+
id: 'inbox-cleanup-backlog',
|
|
340
|
+
runbookRef: 'runbook:async-recovery',
|
|
341
|
+
severity: 'high',
|
|
342
|
+
},
|
|
343
|
+
];
|
|
344
|
+
return registrations.map((registration) => (0, reliability_1.validateAlarmRegistrationMetadata)(registration));
|
|
345
|
+
}
|
|
346
|
+
function reliabilityExercises(name) {
|
|
347
|
+
const shared = {
|
|
348
|
+
cleanupRequired: true,
|
|
349
|
+
owner: `${name}-product`,
|
|
350
|
+
protectedMutationRequiresApproval: true,
|
|
351
|
+
};
|
|
352
|
+
return [
|
|
353
|
+
{
|
|
354
|
+
...shared,
|
|
355
|
+
failure: 'Inject one bounded generated API handler failure.',
|
|
356
|
+
id: 'lambda-api-failure',
|
|
357
|
+
observation: {
|
|
358
|
+
adapter: 'aws-cloudwatch-alarm-history-v1',
|
|
359
|
+
alarmIds: ['api-fast-burn'],
|
|
360
|
+
dynamicAlarmKind: 'none',
|
|
361
|
+
workflowMetadata: 'none',
|
|
362
|
+
},
|
|
363
|
+
postconditions: [
|
|
364
|
+
'critical API journey succeeds',
|
|
365
|
+
'alarm returns to normal without being disabled',
|
|
366
|
+
'release and configuration identity remain attributable',
|
|
367
|
+
],
|
|
368
|
+
recovery: ['retry', 'roll-forward', 'protected-revert'],
|
|
369
|
+
workload: 'lambda-api',
|
|
370
|
+
},
|
|
371
|
+
{
|
|
372
|
+
...shared,
|
|
373
|
+
failure: 'Inject a bounded health or dependency failure in an EBK Service.',
|
|
374
|
+
id: 'service-failure',
|
|
375
|
+
observation: {
|
|
376
|
+
adapter: 'aws-cloudwatch-alarm-history-v1',
|
|
377
|
+
alarmIds: [],
|
|
378
|
+
dynamicAlarmKind: 'service',
|
|
379
|
+
workflowMetadata: 'none',
|
|
380
|
+
},
|
|
381
|
+
postconditions: [
|
|
382
|
+
'service readiness recovers',
|
|
383
|
+
'critical dependency remains protected',
|
|
384
|
+
],
|
|
385
|
+
recovery: ['retry', 'roll-forward', 'degraded-mode'],
|
|
386
|
+
workload: 'service',
|
|
387
|
+
},
|
|
388
|
+
{
|
|
389
|
+
...shared,
|
|
390
|
+
failure: 'Inject one bounded generated Job failure before completion.',
|
|
391
|
+
id: 'job-failure',
|
|
392
|
+
observation: {
|
|
393
|
+
adapter: 'aws-cloudwatch-alarm-history-v1',
|
|
394
|
+
alarmIds: [],
|
|
395
|
+
dynamicAlarmKind: 'job',
|
|
396
|
+
workflowMetadata: 'none',
|
|
397
|
+
},
|
|
398
|
+
postconditions: [
|
|
399
|
+
'checkpoint or idempotency state is retained',
|
|
400
|
+
'deadline outcome is verified',
|
|
401
|
+
],
|
|
402
|
+
recovery: ['retry', 'roll-forward'],
|
|
403
|
+
workload: 'job',
|
|
404
|
+
},
|
|
405
|
+
{
|
|
406
|
+
...shared,
|
|
407
|
+
failure: 'Run the built-in retention state machine with a bounded failing fixture.',
|
|
408
|
+
id: 'retention-job-failure',
|
|
409
|
+
observation: {
|
|
410
|
+
adapter: 'aws-cloudwatch-alarm-history-v1',
|
|
411
|
+
alarmIds: ['retention-job-failure'],
|
|
412
|
+
dynamicAlarmKind: 'none',
|
|
413
|
+
workflowMetadata: 'required',
|
|
414
|
+
},
|
|
415
|
+
postconditions: [
|
|
416
|
+
'the exact state machine execution reaches a terminal failure',
|
|
417
|
+
'the AWS-native execution alarm fires',
|
|
418
|
+
'the later recovery execution succeeds without weakening retention policy',
|
|
419
|
+
],
|
|
420
|
+
recovery: ['retry', 'roll-forward'],
|
|
421
|
+
workload: 'built-in-retention-job',
|
|
422
|
+
},
|
|
423
|
+
{
|
|
424
|
+
...shared,
|
|
425
|
+
failure: 'Route a deterministic poison fixture to the generated DLQ.',
|
|
426
|
+
id: 'queue-dlq-failure',
|
|
427
|
+
observation: {
|
|
428
|
+
adapter: 'aws-cloudwatch-alarm-history-v1',
|
|
429
|
+
alarmIds: [
|
|
430
|
+
'async-target-dlq-depth',
|
|
431
|
+
'event-dlq-depth',
|
|
432
|
+
'provider-dlq-depth',
|
|
433
|
+
],
|
|
434
|
+
dynamicAlarmKind: 'none',
|
|
435
|
+
workflowMetadata: 'optional',
|
|
436
|
+
},
|
|
437
|
+
postconditions: [
|
|
438
|
+
'DLQ alarm fires',
|
|
439
|
+
'bounded redrive converges',
|
|
440
|
+
'no duplicate provider effect occurs',
|
|
441
|
+
],
|
|
442
|
+
recovery: ['redrive', 'replay'],
|
|
443
|
+
workload: 'queue-dlq',
|
|
444
|
+
},
|
|
445
|
+
{
|
|
446
|
+
...shared,
|
|
447
|
+
failure: 'Inject bounded dependency latency and an availability failure.',
|
|
448
|
+
id: 'dependency-failure',
|
|
449
|
+
observation: {
|
|
450
|
+
adapter: 'aws-cloudwatch-alarm-history-v1',
|
|
451
|
+
alarmIds: ['api-fast-burn'],
|
|
452
|
+
dynamicAlarmKind: 'none',
|
|
453
|
+
workflowMetadata: 'none',
|
|
454
|
+
},
|
|
455
|
+
postconditions: [
|
|
456
|
+
'retry amplification stays bounded',
|
|
457
|
+
'critical work degrades or queues as declared',
|
|
458
|
+
],
|
|
459
|
+
recovery: ['retry', 'degraded-mode', 'roll-forward'],
|
|
460
|
+
workload: 'dependency',
|
|
461
|
+
},
|
|
462
|
+
{
|
|
463
|
+
...shared,
|
|
464
|
+
failure: 'Exercise database unavailability or cold-resume latency.',
|
|
465
|
+
id: 'database-failure',
|
|
466
|
+
observation: {
|
|
467
|
+
adapter: 'aws-cloudwatch-alarm-history-v1',
|
|
468
|
+
alarmIds: ['database-capacity'],
|
|
469
|
+
dynamicAlarmKind: 'none',
|
|
470
|
+
workflowMetadata: 'none',
|
|
471
|
+
},
|
|
472
|
+
postconditions: [
|
|
473
|
+
'data invariants pass',
|
|
474
|
+
'connection concurrency remains bounded',
|
|
475
|
+
'critical journey recovery is verified',
|
|
476
|
+
],
|
|
477
|
+
recovery: ['retry', 'restore', 'roll-forward'],
|
|
478
|
+
workload: 'database',
|
|
479
|
+
},
|
|
480
|
+
{
|
|
481
|
+
...shared,
|
|
482
|
+
failure: 'Fail the primary alert destination during a bounded proof.',
|
|
483
|
+
id: 'alert-delivery-failure',
|
|
484
|
+
observation: {
|
|
485
|
+
adapter: 'aws-cloudwatch-alarm-history-v1',
|
|
486
|
+
alarmIds: ['alert-delivery-degraded'],
|
|
487
|
+
dynamicAlarmKind: 'none',
|
|
488
|
+
workflowMetadata: 'none',
|
|
489
|
+
},
|
|
490
|
+
postconditions: [
|
|
491
|
+
'diagnostic degradation is recorded without using the failed route',
|
|
492
|
+
'delivery is unsupported unless an independent provider receipt exists',
|
|
493
|
+
],
|
|
494
|
+
recovery: ['roll-forward'],
|
|
495
|
+
workload: 'alert-delivery',
|
|
496
|
+
},
|
|
497
|
+
{
|
|
498
|
+
...shared,
|
|
499
|
+
failure: 'Retain one exact deployed-smoke failure artifact before the successful rerun.',
|
|
500
|
+
id: 'runtime-smoke-failure',
|
|
501
|
+
observation: {
|
|
502
|
+
adapter: 'testing-evidence-aggregate-v1',
|
|
503
|
+
alarmIds: [],
|
|
504
|
+
dynamicAlarmKind: 'none',
|
|
505
|
+
workflowMetadata: 'none',
|
|
506
|
+
},
|
|
507
|
+
postconditions: [
|
|
508
|
+
'the testing-evidence aggregate binds the exact deployment and candidate',
|
|
509
|
+
'the failure remains retained beside the successful rerun',
|
|
510
|
+
'cleanup evidence is complete',
|
|
511
|
+
],
|
|
512
|
+
recovery: ['retry', 'roll-forward'],
|
|
513
|
+
workload: 'deployed-smoke',
|
|
514
|
+
},
|
|
515
|
+
];
|
|
516
|
+
}
|
|
517
|
+
function deployedAlarmResources(name) {
|
|
518
|
+
const primary = {
|
|
519
|
+
actions: 'primary-route',
|
|
520
|
+
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
521
|
+
owner: `${name}-product`,
|
|
522
|
+
route: `${name}-on-call`,
|
|
523
|
+
};
|
|
524
|
+
return [
|
|
525
|
+
{
|
|
526
|
+
...primary,
|
|
527
|
+
dimensionNames: ['ApiId'],
|
|
528
|
+
evaluationPeriods: 1,
|
|
529
|
+
id: 'api-fast-burn',
|
|
530
|
+
metricNames: ['5xx', 'Count'],
|
|
531
|
+
name: `${name}-{stage}-api-fast-burn`,
|
|
532
|
+
namespace: 'AWS/ApiGateway',
|
|
533
|
+
objectiveId: 'api-availability',
|
|
534
|
+
runbook: 'docs/runbooks/api-recovery.md',
|
|
535
|
+
signal: 'metric-math',
|
|
536
|
+
threshold: 14.4,
|
|
537
|
+
treatMissingData: 'missing',
|
|
538
|
+
},
|
|
539
|
+
{
|
|
540
|
+
...primary,
|
|
541
|
+
dimensionNames: ['ApiId'],
|
|
542
|
+
evaluationPeriods: 1,
|
|
543
|
+
id: 'api-slow-burn',
|
|
544
|
+
metricNames: ['5xx', 'Count'],
|
|
545
|
+
name: `${name}-{stage}-api-slow-burn`,
|
|
546
|
+
namespace: 'AWS/ApiGateway',
|
|
547
|
+
objectiveId: 'api-availability',
|
|
548
|
+
runbook: 'docs/runbooks/api-recovery.md',
|
|
549
|
+
signal: 'metric-math',
|
|
550
|
+
threshold: 6,
|
|
551
|
+
treatMissingData: 'missing',
|
|
552
|
+
},
|
|
553
|
+
{
|
|
554
|
+
...primary,
|
|
555
|
+
dimensionNames: ['Component', 'Feature', 'Operation', 'Slice', 'Stage'],
|
|
556
|
+
dimensionValues: {
|
|
557
|
+
Component: 'next-server',
|
|
558
|
+
Feature: 'web',
|
|
559
|
+
Operation: 'request',
|
|
560
|
+
Slice: name,
|
|
561
|
+
Stage: '{stage}',
|
|
562
|
+
},
|
|
563
|
+
evaluationPeriods: 1,
|
|
564
|
+
id: 'web-latency-fast-burn',
|
|
565
|
+
metricNames: ['OperationGood', 'OperationValid'],
|
|
566
|
+
name: `${name}-{stage}-web-latency-fast-burn`,
|
|
567
|
+
namespace: `EBK/${name}`,
|
|
568
|
+
objectiveId: 'web-latency',
|
|
569
|
+
runbook: 'docs/runbooks/web-recovery.md',
|
|
570
|
+
signal: 'metric-math',
|
|
571
|
+
threshold: 14.4,
|
|
572
|
+
treatMissingData: 'missing',
|
|
573
|
+
},
|
|
574
|
+
{
|
|
575
|
+
...primary,
|
|
576
|
+
dimensionNames: ['Component', 'Feature', 'Operation', 'Slice', 'Stage'],
|
|
577
|
+
dimensionValues: {
|
|
578
|
+
Component: 'next-server',
|
|
579
|
+
Feature: 'web',
|
|
580
|
+
Operation: 'request',
|
|
581
|
+
Slice: name,
|
|
582
|
+
Stage: '{stage}',
|
|
583
|
+
},
|
|
584
|
+
evaluationPeriods: 1,
|
|
585
|
+
id: 'web-latency-slow-burn',
|
|
586
|
+
metricNames: ['OperationGood', 'OperationValid'],
|
|
587
|
+
name: `${name}-{stage}-web-latency-slow-burn`,
|
|
588
|
+
namespace: `EBK/${name}`,
|
|
589
|
+
objectiveId: 'web-latency',
|
|
590
|
+
runbook: 'docs/runbooks/web-recovery.md',
|
|
591
|
+
signal: 'metric-math',
|
|
592
|
+
threshold: 6,
|
|
593
|
+
treatMissingData: 'missing',
|
|
594
|
+
},
|
|
595
|
+
{
|
|
596
|
+
...primary,
|
|
597
|
+
dimensionNames: ['QueueName'],
|
|
598
|
+
dimensionValues: {
|
|
599
|
+
QueueName: `ebk-{stage}-${name}-provider-queue`,
|
|
600
|
+
},
|
|
601
|
+
evaluationPeriods: 2,
|
|
602
|
+
id: 'provider-queue-age',
|
|
603
|
+
metricName: 'ApproximateAgeOfOldestMessage',
|
|
604
|
+
name: `${name}-{stage}-provider-deadline`,
|
|
605
|
+
namespace: 'AWS/SQS',
|
|
606
|
+
period: 300,
|
|
607
|
+
runbook: 'docs/runbooks/async-recovery.md',
|
|
608
|
+
signal: 'metric',
|
|
609
|
+
statistic: 'Maximum',
|
|
610
|
+
threshold: 900,
|
|
611
|
+
treatMissingData: 'missing',
|
|
612
|
+
},
|
|
613
|
+
{
|
|
614
|
+
...primary,
|
|
615
|
+
dimensionNames: ['DBClusterIdentifier'],
|
|
616
|
+
evaluationPeriods: 3,
|
|
617
|
+
id: 'database-capacity',
|
|
618
|
+
metricName: 'ServerlessDatabaseCapacity',
|
|
619
|
+
name: `${name}-{stage}-database-capacity`,
|
|
620
|
+
namespace: 'AWS/RDS',
|
|
621
|
+
period: 300,
|
|
622
|
+
runbook: 'docs/runbooks/database-recovery.md',
|
|
623
|
+
signal: 'metric',
|
|
624
|
+
statistic: 'Maximum',
|
|
625
|
+
threshold: 3.5,
|
|
626
|
+
treatMissingData: 'missing',
|
|
627
|
+
},
|
|
628
|
+
{
|
|
629
|
+
...primary,
|
|
630
|
+
dimensionNames: ['Component', 'Feature', 'Operation', 'Slice', 'Stage'],
|
|
631
|
+
dimensionValues: {
|
|
632
|
+
Component: 'performance-load-cleanup',
|
|
633
|
+
Feature: 'performance',
|
|
634
|
+
Operation: 'expiry-cleanup',
|
|
635
|
+
Slice: name,
|
|
636
|
+
Stage: '{stage}',
|
|
637
|
+
},
|
|
638
|
+
evaluationPeriods: 1,
|
|
639
|
+
id: 'performance-load-cleanup-backlog',
|
|
640
|
+
metricName: 'PerformanceLoadCleanupBacklog',
|
|
641
|
+
name: `${name}-{stage}-performance-load-cleanup-backlog`,
|
|
642
|
+
namespace: 'EBK/Performance',
|
|
643
|
+
period: 3600,
|
|
644
|
+
runbook: 'docs/runbooks/performance-capacity-cost.md',
|
|
645
|
+
signal: 'metric',
|
|
646
|
+
statistic: 'Maximum',
|
|
647
|
+
threshold: 1,
|
|
648
|
+
treatMissingData: 'notBreaching',
|
|
649
|
+
},
|
|
650
|
+
{
|
|
651
|
+
...primary,
|
|
652
|
+
dimensionNames: ['Component', 'Feature', 'Operation', 'Slice', 'Stage'],
|
|
653
|
+
dimensionValues: {
|
|
654
|
+
Component: 'performance-load-cleanup',
|
|
655
|
+
Feature: 'performance',
|
|
656
|
+
Operation: 'expiry-cleanup',
|
|
657
|
+
Slice: name,
|
|
658
|
+
Stage: '{stage}',
|
|
659
|
+
},
|
|
660
|
+
evaluationPeriods: 1,
|
|
661
|
+
id: 'performance-load-cleanup-deadline',
|
|
662
|
+
metricName: 'PerformanceLoadCleanupDeadlineBreaches',
|
|
663
|
+
name: `${name}-{stage}-performance-load-cleanup-deadline`,
|
|
664
|
+
namespace: 'EBK/Performance',
|
|
665
|
+
period: 3600,
|
|
666
|
+
runbook: 'docs/runbooks/performance-capacity-cost.md',
|
|
667
|
+
signal: 'metric',
|
|
668
|
+
statistic: 'Maximum',
|
|
669
|
+
threshold: 1,
|
|
670
|
+
treatMissingData: 'notBreaching',
|
|
671
|
+
},
|
|
672
|
+
{
|
|
673
|
+
...primary,
|
|
674
|
+
comparisonOperator: 'LessThanThreshold',
|
|
675
|
+
dimensionNames: ['FunctionName'],
|
|
676
|
+
evaluationPeriods: 2,
|
|
677
|
+
id: 'data-export-cleanup-invocation-missing',
|
|
678
|
+
metricName: 'Invocations',
|
|
679
|
+
name: `${name}-{stage}-data-export-cleanup-invocation-missing`,
|
|
680
|
+
namespace: 'AWS/Lambda',
|
|
681
|
+
period: 3600,
|
|
682
|
+
runbook: 'docs/runbooks/database-recovery.md',
|
|
683
|
+
signal: 'metric',
|
|
684
|
+
statistic: 'Sum',
|
|
685
|
+
threshold: 1,
|
|
686
|
+
treatMissingData: 'breaching',
|
|
687
|
+
},
|
|
688
|
+
{
|
|
689
|
+
...primary,
|
|
690
|
+
dimensionNames: ['FunctionName'],
|
|
691
|
+
evaluationPeriods: 1,
|
|
692
|
+
id: 'data-export-cleanup-failure',
|
|
693
|
+
metricName: 'Errors',
|
|
694
|
+
name: `${name}-{stage}-data-export-cleanup-failure`,
|
|
695
|
+
namespace: 'AWS/Lambda',
|
|
696
|
+
period: 300,
|
|
697
|
+
runbook: 'docs/runbooks/database-recovery.md',
|
|
698
|
+
signal: 'metric',
|
|
699
|
+
statistic: 'Sum',
|
|
700
|
+
threshold: 1,
|
|
701
|
+
treatMissingData: 'notBreaching',
|
|
702
|
+
},
|
|
703
|
+
{
|
|
704
|
+
...primary,
|
|
705
|
+
dimensionNames: ['Component', 'Feature', 'Operation', 'Slice', 'Stage'],
|
|
706
|
+
dimensionValues: {
|
|
707
|
+
Component: 'data-export-cleanup',
|
|
708
|
+
Feature: 'data-lifecycle',
|
|
709
|
+
Operation: 'expiry-cleanup',
|
|
710
|
+
Slice: name,
|
|
711
|
+
Stage: '{stage}',
|
|
712
|
+
},
|
|
713
|
+
evaluationPeriods: 1,
|
|
714
|
+
id: 'data-export-cleanup-deadline',
|
|
715
|
+
metricName: 'DataExportCleanupDeadlineBreaches',
|
|
716
|
+
name: `${name}-{stage}-data-export-cleanup-deadline`,
|
|
717
|
+
namespace: 'EBK/DataLifecycle',
|
|
718
|
+
period: 3600,
|
|
719
|
+
runbook: 'docs/runbooks/database-recovery.md',
|
|
720
|
+
signal: 'metric',
|
|
721
|
+
statistic: 'Maximum',
|
|
722
|
+
threshold: 1,
|
|
723
|
+
treatMissingData: 'notBreaching',
|
|
724
|
+
},
|
|
725
|
+
{
|
|
726
|
+
...primary,
|
|
727
|
+
dimensionNames: ['FunctionName'],
|
|
728
|
+
evaluationPeriods: 1,
|
|
729
|
+
id: 'function-failure',
|
|
730
|
+
metricName: 'Errors',
|
|
731
|
+
name: `${name}-{stage}-function-failure`,
|
|
732
|
+
namespace: 'AWS/Lambda',
|
|
733
|
+
period: 60,
|
|
734
|
+
runbook: 'docs/runbooks/api-recovery.md',
|
|
735
|
+
signal: 'metric',
|
|
736
|
+
statistic: 'Sum',
|
|
737
|
+
threshold: 1,
|
|
738
|
+
treatMissingData: 'notBreaching',
|
|
739
|
+
},
|
|
740
|
+
{
|
|
741
|
+
...primary,
|
|
742
|
+
dimensionNames: ['StateMachineArn'],
|
|
743
|
+
evaluationPeriods: 1,
|
|
744
|
+
id: 'retention-job-failure',
|
|
745
|
+
metricNames: [
|
|
746
|
+
'ExecutionsAborted',
|
|
747
|
+
'ExecutionsFailed',
|
|
748
|
+
'ExecutionsTimedOut',
|
|
749
|
+
],
|
|
750
|
+
name: `${name}-{stage}-retention-job-failure`,
|
|
751
|
+
namespace: 'AWS/States',
|
|
752
|
+
runbook: 'docs/runbooks/database-recovery.md',
|
|
753
|
+
signal: 'metric-math',
|
|
754
|
+
threshold: 1,
|
|
755
|
+
treatMissingData: 'notBreaching',
|
|
756
|
+
},
|
|
757
|
+
{
|
|
758
|
+
...primary,
|
|
759
|
+
dimensionNames: ['QueueName'],
|
|
760
|
+
dimensionValues: {
|
|
761
|
+
QueueName: `ebk-{stage}-${name}-event-dlq`,
|
|
762
|
+
},
|
|
763
|
+
evaluationPeriods: 1,
|
|
764
|
+
id: 'event-dlq-depth',
|
|
765
|
+
metricName: 'ApproximateNumberOfMessagesVisible',
|
|
766
|
+
name: `${name}-{stage}-event-dlq-depth`,
|
|
767
|
+
namespace: 'AWS/SQS',
|
|
768
|
+
period: 300,
|
|
769
|
+
runbook: 'docs/runbooks/async-recovery.md',
|
|
770
|
+
signal: 'metric',
|
|
771
|
+
statistic: 'Maximum',
|
|
772
|
+
threshold: 1,
|
|
773
|
+
treatMissingData: 'notBreaching',
|
|
774
|
+
},
|
|
775
|
+
{
|
|
776
|
+
...primary,
|
|
777
|
+
dimensionNames: ['QueueName'],
|
|
778
|
+
dimensionValues: {
|
|
779
|
+
QueueName: `ebk-{stage}-${name}-provider-dlq`,
|
|
780
|
+
},
|
|
781
|
+
evaluationPeriods: 1,
|
|
782
|
+
id: 'provider-dlq-depth',
|
|
783
|
+
metricName: 'ApproximateNumberOfMessagesVisible',
|
|
784
|
+
name: `${name}-{stage}-provider-dlq-depth`,
|
|
785
|
+
namespace: 'AWS/SQS',
|
|
786
|
+
period: 300,
|
|
787
|
+
runbook: 'docs/runbooks/async-recovery.md',
|
|
788
|
+
signal: 'metric',
|
|
789
|
+
statistic: 'Maximum',
|
|
790
|
+
threshold: 1,
|
|
791
|
+
treatMissingData: 'notBreaching',
|
|
792
|
+
},
|
|
793
|
+
{
|
|
794
|
+
...primary,
|
|
795
|
+
dimensionNames: ['QueueName'],
|
|
796
|
+
dimensionValues: {
|
|
797
|
+
QueueName: `ebk-{stage}-${name}-async-target-dlq`,
|
|
798
|
+
},
|
|
799
|
+
evaluationPeriods: 1,
|
|
800
|
+
id: 'async-target-dlq-depth',
|
|
801
|
+
metricName: 'ApproximateNumberOfMessagesVisible',
|
|
802
|
+
name: `${name}-{stage}-async-target-dlq-depth`,
|
|
803
|
+
namespace: 'AWS/SQS',
|
|
804
|
+
period: 300,
|
|
805
|
+
runbook: 'docs/runbooks/async-recovery.md',
|
|
806
|
+
signal: 'metric',
|
|
807
|
+
statistic: 'Maximum',
|
|
808
|
+
threshold: 1,
|
|
809
|
+
treatMissingData: 'notBreaching',
|
|
810
|
+
},
|
|
811
|
+
{
|
|
812
|
+
...primary,
|
|
813
|
+
dimensionNames: ['FunctionName'],
|
|
814
|
+
evaluationPeriods: 1,
|
|
815
|
+
id: 'outbox-relay-errors',
|
|
816
|
+
metricName: 'Errors',
|
|
817
|
+
name: `${name}-{stage}-outbox-relay-errors`,
|
|
818
|
+
namespace: 'AWS/Lambda',
|
|
819
|
+
period: 300,
|
|
820
|
+
runbook: 'docs/runbooks/async-recovery.md',
|
|
821
|
+
signal: 'metric',
|
|
822
|
+
statistic: 'Sum',
|
|
823
|
+
threshold: 1,
|
|
824
|
+
treatMissingData: 'notBreaching',
|
|
825
|
+
},
|
|
826
|
+
{
|
|
827
|
+
...primary,
|
|
828
|
+
dimensionNames: ['FunctionName'],
|
|
829
|
+
evaluationPeriods: 1,
|
|
830
|
+
id: 'inbox-cleanup-errors',
|
|
831
|
+
metricName: 'Errors',
|
|
832
|
+
name: `${name}-{stage}-inbox-cleanup-errors`,
|
|
833
|
+
namespace: 'AWS/Lambda',
|
|
834
|
+
period: 300,
|
|
835
|
+
runbook: 'docs/runbooks/async-recovery.md',
|
|
836
|
+
signal: 'metric',
|
|
837
|
+
statistic: 'Sum',
|
|
838
|
+
threshold: 1,
|
|
839
|
+
treatMissingData: 'notBreaching',
|
|
840
|
+
},
|
|
841
|
+
{
|
|
842
|
+
...primary,
|
|
843
|
+
dimensionNames: ['Slice', 'Stage'],
|
|
844
|
+
dimensionValues: { Slice: name, Stage: '{stage}' },
|
|
845
|
+
evaluationPeriods: 2,
|
|
846
|
+
id: 'outbox-backlog',
|
|
847
|
+
metricName: 'OutboxBacklog',
|
|
848
|
+
name: `${name}-{stage}-outbox-backlog`,
|
|
849
|
+
namespace: 'EBK/Async',
|
|
850
|
+
period: 300,
|
|
851
|
+
runbook: 'docs/runbooks/async-recovery.md',
|
|
852
|
+
signal: 'metric',
|
|
853
|
+
statistic: 'Maximum',
|
|
854
|
+
threshold: 1,
|
|
855
|
+
treatMissingData: 'notBreaching',
|
|
856
|
+
},
|
|
857
|
+
{
|
|
858
|
+
...primary,
|
|
859
|
+
dimensionNames: ['Component', 'Feature', 'Operation', 'Slice', 'Stage'],
|
|
860
|
+
dimensionValues: {
|
|
861
|
+
Component: 'inbox-cleanup',
|
|
862
|
+
Feature: 'async-work',
|
|
863
|
+
Operation: 'inbox-cleanup',
|
|
864
|
+
Slice: name,
|
|
865
|
+
Stage: '{stage}',
|
|
866
|
+
},
|
|
867
|
+
evaluationPeriods: 2,
|
|
868
|
+
id: 'inbox-cleanup-backlog',
|
|
869
|
+
metricName: 'InboxCleanupBacklog',
|
|
870
|
+
name: `${name}-{stage}-inbox-cleanup-backlog`,
|
|
871
|
+
namespace: 'EBK/Async',
|
|
872
|
+
period: 3600,
|
|
873
|
+
runbook: 'docs/runbooks/async-recovery.md',
|
|
874
|
+
signal: 'metric',
|
|
875
|
+
statistic: 'Maximum',
|
|
876
|
+
threshold: 1,
|
|
877
|
+
treatMissingData: 'notBreaching',
|
|
878
|
+
},
|
|
879
|
+
{
|
|
880
|
+
actions: 'none',
|
|
881
|
+
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
882
|
+
dimensionNames: ['TopicName'],
|
|
883
|
+
evaluationPeriods: 1,
|
|
884
|
+
id: 'alert-delivery-degraded',
|
|
885
|
+
metricName: 'NumberOfNotificationsFailed',
|
|
886
|
+
name: `${name}-{stage}-alert-delivery-degraded`,
|
|
887
|
+
namespace: 'AWS/SNS',
|
|
888
|
+
owner: `${name}-product`,
|
|
889
|
+
period: 300,
|
|
890
|
+
route: `${name}-on-call`,
|
|
891
|
+
runbook: 'docs/runbooks/alert-delivery.md',
|
|
892
|
+
signal: 'metric',
|
|
893
|
+
statistic: 'Sum',
|
|
894
|
+
threshold: 1,
|
|
895
|
+
treatMissingData: 'notBreaching',
|
|
896
|
+
},
|
|
897
|
+
];
|
|
898
|
+
}
|
|
899
|
+
/** @internal */
|
|
900
|
+
function renderProductReliabilityFiles(name, pascalName, reviewedAt) {
|
|
901
|
+
const contract = reliabilityContract(name, reviewedAt);
|
|
902
|
+
const alarms = alarmRegistrations(name);
|
|
903
|
+
const exercises = reliabilityExercises(name);
|
|
904
|
+
return {
|
|
905
|
+
'docs/reliability/slo.json': (0, built_in_plan_shared_js_1.json)(contract),
|
|
906
|
+
'docs/reliability/exclusions.md': `# SLO exclusions
|
|
907
|
+
|
|
908
|
+
This adopter-owned record starts empty. An exclusion must name one generated
|
|
909
|
+
SLO ID, owner, reason, bounded UTC start and end, and review reference. Define
|
|
910
|
+
planned maintenance exclusions before the window where possible. Never add a
|
|
911
|
+
retroactive exclusion merely to make attainment green.
|
|
912
|
+
|
|
913
|
+
The generated verifier treats this document as the policy authority; managed
|
|
914
|
+
CloudWatch resources do not rewrite it.
|
|
915
|
+
`,
|
|
916
|
+
'operations/reliability.json': (0, built_in_plan_shared_js_1.json)({
|
|
917
|
+
alarmRegistrations: alarms,
|
|
918
|
+
consoleIndependent: true,
|
|
919
|
+
deployedAlarmResources: deployedAlarmResources(name),
|
|
920
|
+
diagnostic: {
|
|
921
|
+
metadataSources: [
|
|
922
|
+
'alarm-state',
|
|
923
|
+
'resource-metadata',
|
|
924
|
+
'sanitized-signal',
|
|
925
|
+
],
|
|
926
|
+
protectedSource: 'bounded-log-query',
|
|
927
|
+
runbookRef: 'runbook:reliability-diagnostic',
|
|
928
|
+
},
|
|
929
|
+
alertDelivery: {
|
|
930
|
+
confirmation: 'unsupported-until-independent-route-is-configured',
|
|
931
|
+
mode: 'diagnostic-only',
|
|
932
|
+
states: ['requested', 'confirmed', 'unsupported'],
|
|
933
|
+
},
|
|
934
|
+
dataLifecycleReferences: {
|
|
935
|
+
alarms: [
|
|
936
|
+
'alarm:data-export-cleanup-deadline',
|
|
937
|
+
'alarm:data-export-cleanup-failure',
|
|
938
|
+
'alarm:data-export-cleanup-invocation-missing',
|
|
939
|
+
],
|
|
940
|
+
status: 'requires-live-evidence',
|
|
941
|
+
},
|
|
942
|
+
incidentTemplateRef: 'operations:incident-template.json',
|
|
943
|
+
performanceReferences: {
|
|
944
|
+
alarms: [
|
|
945
|
+
'alarm:cost-anomaly',
|
|
946
|
+
'alarm:performance-load-cleanup-backlog',
|
|
947
|
+
'alarm:performance-load-cleanup-deadline',
|
|
948
|
+
],
|
|
949
|
+
capacityControls: [
|
|
950
|
+
'control:api-lambda-concurrency',
|
|
951
|
+
'control:async-worker-concurrency',
|
|
952
|
+
'control:database-connections',
|
|
953
|
+
'control:service-extension-concurrency',
|
|
954
|
+
],
|
|
955
|
+
capacitySignals: [
|
|
956
|
+
'adopter:service-saturation-percent',
|
|
957
|
+
'database:connection-utilization-percent',
|
|
958
|
+
'database:protected-async-backlog-utilization-percent',
|
|
959
|
+
'runtime:nodejs-heap-utilization-percent',
|
|
960
|
+
],
|
|
961
|
+
status: 'requires-adopter-values-and-live-evidence',
|
|
962
|
+
},
|
|
963
|
+
remediation: {
|
|
964
|
+
adapter: 'protected-async-recovery',
|
|
965
|
+
approvalRequired: true,
|
|
966
|
+
runbookRef: 'runbook:incident-remediation',
|
|
967
|
+
supportedApplyActions: ['redrive', 'replay'],
|
|
968
|
+
},
|
|
969
|
+
schemaVersion: 1,
|
|
970
|
+
slice: name,
|
|
971
|
+
sloRef: 'docs:reliability/slo.json',
|
|
972
|
+
runtimeSmoke: {
|
|
973
|
+
aggregateOwner: 'testing-evidence',
|
|
974
|
+
deployedResourceClass: 'testing-evidence-not-cloudwatch-alarm',
|
|
975
|
+
evidenceKind: 'testing-evidence-result',
|
|
976
|
+
metricOwner: 'not-provisioned-by-reliability',
|
|
977
|
+
reporter: 'testing-evidence-native-not-reliability-exercise-report',
|
|
978
|
+
target: 'smoke-deployed',
|
|
979
|
+
},
|
|
980
|
+
workloadCoverage: [
|
|
981
|
+
{ kind: 'lambda-api', registration: 'api-fast-burn' },
|
|
982
|
+
{ kind: 'function', registration: 'function-failure' },
|
|
983
|
+
{ kind: 'next', registration: 'web-latency-fast-burn' },
|
|
984
|
+
{
|
|
985
|
+
kind: 'service',
|
|
986
|
+
registration: 'required-when-generated-by-ebk-workload-service-health',
|
|
987
|
+
},
|
|
988
|
+
{
|
|
989
|
+
kind: 'job',
|
|
990
|
+
registration: 'required-when-generated-by-ebk-workload-job-failure',
|
|
991
|
+
},
|
|
992
|
+
{ kind: 'queue-dlq', registration: 'provider-queue-age' },
|
|
993
|
+
{ kind: 'database', registration: 'database-capacity' },
|
|
994
|
+
{
|
|
995
|
+
kind: 'built-in-retention-job',
|
|
996
|
+
registration: 'retention-job-failure',
|
|
997
|
+
},
|
|
998
|
+
{
|
|
999
|
+
kind: 'built-in-data-export-cleanup',
|
|
1000
|
+
registration: 'data-export-cleanup-failure',
|
|
1001
|
+
},
|
|
1002
|
+
{
|
|
1003
|
+
kind: 'deployed-smoke',
|
|
1004
|
+
registration: 'testing-evidence-aggregate',
|
|
1005
|
+
},
|
|
1006
|
+
{
|
|
1007
|
+
kind: 'alert-delivery',
|
|
1008
|
+
registration: 'alert-delivery-degraded',
|
|
1009
|
+
},
|
|
1010
|
+
],
|
|
1011
|
+
}),
|
|
1012
|
+
'testing/reliability-exercises.json': (0, built_in_plan_shared_js_1.json)({
|
|
1013
|
+
exercises,
|
|
1014
|
+
schemaVersion: 1,
|
|
1015
|
+
slice: name,
|
|
1016
|
+
}),
|
|
1017
|
+
'operations/incident-template.json': (0, built_in_plan_shared_js_1.json)({
|
|
1018
|
+
affectedJourneyIds: ['api-critical-operation'],
|
|
1019
|
+
declaredAt: reviewedAt,
|
|
1020
|
+
followUpRefs: ['replace-with-owned-work-reference'],
|
|
1021
|
+
id: `${name}-incident-template`,
|
|
1022
|
+
impact: 'Replace with confirmed user or operator impact.',
|
|
1023
|
+
roles: {
|
|
1024
|
+
communicationsOwner: 'unassigned-human',
|
|
1025
|
+
incidentCommander: 'unassigned-human',
|
|
1026
|
+
scribe: 'unassigned',
|
|
1027
|
+
technicalLead: 'unassigned-human',
|
|
1028
|
+
},
|
|
1029
|
+
severity: 'SEV-4',
|
|
1030
|
+
slice: name,
|
|
1031
|
+
state: 'declared',
|
|
1032
|
+
timeline: [
|
|
1033
|
+
{
|
|
1034
|
+
actor: 'unassigned-human',
|
|
1035
|
+
at: reviewedAt,
|
|
1036
|
+
evidenceRefs: ['replace-with-sanitized-evidence-reference'],
|
|
1037
|
+
message: 'Template only: confirm impact and explicitly declare the incident.',
|
|
1038
|
+
type: 'decision',
|
|
1039
|
+
},
|
|
1040
|
+
],
|
|
1041
|
+
}),
|
|
1042
|
+
'reliability/contract.ts': `import {
|
|
1043
|
+
validateReliabilityContract,
|
|
1044
|
+
} from '@empire-builder-kit/runtime/reliability';
|
|
1045
|
+
import declaration from '../docs/reliability/slo.json';
|
|
1046
|
+
|
|
1047
|
+
export const Reliability = validateReliabilityContract(declaration);
|
|
1048
|
+
`,
|
|
1049
|
+
'reliability/metrics.ts': `import {
|
|
1050
|
+
createMetricEmitter,
|
|
1051
|
+
type MetricEmitter,
|
|
1052
|
+
type MetricOutcome,
|
|
1053
|
+
} from '@empire-builder-kit/runtime/observability';
|
|
1054
|
+
|
|
1055
|
+
export interface ReliabilityOperation {
|
|
1056
|
+
component: string;
|
|
1057
|
+
durationMilliseconds: number;
|
|
1058
|
+
feature: string;
|
|
1059
|
+
good?: boolean;
|
|
1060
|
+
operation: string;
|
|
1061
|
+
outcome: MetricOutcome;
|
|
1062
|
+
}
|
|
1063
|
+
|
|
1064
|
+
export function recordReliabilityOperation(
|
|
1065
|
+
input: ReliabilityOperation,
|
|
1066
|
+
metrics: MetricEmitter = createMetricEmitter(),
|
|
1067
|
+
) {
|
|
1068
|
+
const dimensions = {
|
|
1069
|
+
component: input.component,
|
|
1070
|
+
feature: input.feature,
|
|
1071
|
+
operation: input.operation,
|
|
1072
|
+
slice: '${name}',
|
|
1073
|
+
stage: process.env.EBK_STAGE ?? 'development',
|
|
1074
|
+
};
|
|
1075
|
+
const valid = metrics.emit({
|
|
1076
|
+
dimensions,
|
|
1077
|
+
metricName: 'OperationValid',
|
|
1078
|
+
unit: 'Count',
|
|
1079
|
+
value: 1,
|
|
1080
|
+
});
|
|
1081
|
+
const good = metrics.emit({
|
|
1082
|
+
dimensions,
|
|
1083
|
+
metricName: 'OperationGood',
|
|
1084
|
+
unit: 'Count',
|
|
1085
|
+
value: (input.good ?? input.outcome === 'success') ? 1 : 0,
|
|
1086
|
+
});
|
|
1087
|
+
const latency = metrics.emit({
|
|
1088
|
+
dimensions,
|
|
1089
|
+
metricName: 'OperationDuration',
|
|
1090
|
+
unit: 'Milliseconds',
|
|
1091
|
+
value: input.durationMilliseconds,
|
|
1092
|
+
});
|
|
1093
|
+
return { good, latency, valid };
|
|
1094
|
+
}
|
|
1095
|
+
`,
|
|
1096
|
+
'infra/reliability.ts': `import {
|
|
1097
|
+
createSloBurnAlarmPlan,
|
|
1098
|
+
} from '@empire-builder-kit/runtime/reliability';
|
|
1099
|
+
import { Reliability } from '../reliability/contract.js';
|
|
1100
|
+
|
|
1101
|
+
export interface ReliabilityInfrastructureInput {
|
|
1102
|
+
alertTopicArn: any;
|
|
1103
|
+
alertTopicName: any;
|
|
1104
|
+
apiId: any;
|
|
1105
|
+
databaseClusterIdentifier: any;
|
|
1106
|
+
functionName: any;
|
|
1107
|
+
providerQueueName: any;
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
function alarmDescription(input: {
|
|
1111
|
+
businessSymptom: string;
|
|
1112
|
+
id: string;
|
|
1113
|
+
owner: string;
|
|
1114
|
+
route: string;
|
|
1115
|
+
runbook: string;
|
|
1116
|
+
}) {
|
|
1117
|
+
return JSON.stringify({
|
|
1118
|
+
businessSymptom: input.businessSymptom,
|
|
1119
|
+
id: input.id,
|
|
1120
|
+
owner: input.owner,
|
|
1121
|
+
route: input.route,
|
|
1122
|
+
runbook: input.runbook,
|
|
1123
|
+
schemaVersion: 1,
|
|
1124
|
+
slice: '${name}',
|
|
1125
|
+
});
|
|
1126
|
+
}
|
|
1127
|
+
|
|
1128
|
+
export function create${pascalName}Reliability(
|
|
1129
|
+
input: ReliabilityInfrastructureInput,
|
|
1130
|
+
) {
|
|
1131
|
+
const apiObjective = Reliability.objectives.find(
|
|
1132
|
+
({ id }) => id === 'api-availability',
|
|
1133
|
+
);
|
|
1134
|
+
const webObjective = Reliability.objectives.find(
|
|
1135
|
+
({ id }) => id === 'web-latency',
|
|
1136
|
+
);
|
|
1137
|
+
if (!apiObjective || !webObjective) {
|
|
1138
|
+
throw new Error('Generated reliability objectives are incomplete.');
|
|
1139
|
+
}
|
|
1140
|
+
const apiBurn = createSloBurnAlarmPlan(apiObjective, {
|
|
1141
|
+
bad: {
|
|
1142
|
+
dimensions: { ApiId: input.apiId },
|
|
1143
|
+
metricName: '5xx',
|
|
1144
|
+
namespace: 'AWS/ApiGateway',
|
|
1145
|
+
stat: 'Sum',
|
|
1146
|
+
},
|
|
1147
|
+
valid: {
|
|
1148
|
+
dimensions: { ApiId: input.apiId },
|
|
1149
|
+
metricName: 'Count',
|
|
1150
|
+
namespace: 'AWS/ApiGateway',
|
|
1151
|
+
stat: 'Sum',
|
|
1152
|
+
},
|
|
1153
|
+
});
|
|
1154
|
+
const webDimensions = {
|
|
1155
|
+
Component: 'next-server',
|
|
1156
|
+
Feature: 'web',
|
|
1157
|
+
Operation: 'request',
|
|
1158
|
+
Slice: '${name}',
|
|
1159
|
+
Stage: $app.stage,
|
|
1160
|
+
};
|
|
1161
|
+
const webBurn = createSloBurnAlarmPlan(webObjective, {
|
|
1162
|
+
good: {
|
|
1163
|
+
dimensions: webDimensions,
|
|
1164
|
+
metricName: 'OperationGood',
|
|
1165
|
+
namespace: 'EBK/${name}',
|
|
1166
|
+
stat: 'Sum',
|
|
1167
|
+
},
|
|
1168
|
+
valid: {
|
|
1169
|
+
dimensions: webDimensions,
|
|
1170
|
+
metricName: 'OperationValid',
|
|
1171
|
+
namespace: 'EBK/${name}',
|
|
1172
|
+
stat: 'Sum',
|
|
1173
|
+
},
|
|
1174
|
+
});
|
|
1175
|
+
const apiFastBurn = new aws.cloudwatch.MetricAlarm(
|
|
1176
|
+
'${pascalName}ApiFastBurn',
|
|
1177
|
+
{
|
|
1178
|
+
alarmActions: [input.alertTopicArn],
|
|
1179
|
+
alarmDescription: alarmDescription({
|
|
1180
|
+
businessSymptom: 'Critical API operations are failing.',
|
|
1181
|
+
id: 'api-fast-burn',
|
|
1182
|
+
owner: '${name}-product',
|
|
1183
|
+
route: '${name}-on-call',
|
|
1184
|
+
runbook: 'docs/runbooks/api-recovery.md',
|
|
1185
|
+
}),
|
|
1186
|
+
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1187
|
+
datapointsToAlarm: 1,
|
|
1188
|
+
evaluationPeriods: 1,
|
|
1189
|
+
metricQueries: [...apiBurn.fast.metricQueries],
|
|
1190
|
+
name: '${name}-' + $app.stage + '-api-fast-burn',
|
|
1191
|
+
threshold: apiBurn.fast.burnThreshold,
|
|
1192
|
+
treatMissingData: 'missing',
|
|
1193
|
+
},
|
|
1194
|
+
);
|
|
1195
|
+
const apiSlowBurn = new aws.cloudwatch.MetricAlarm(
|
|
1196
|
+
'${pascalName}ApiSlowBurn',
|
|
1197
|
+
{
|
|
1198
|
+
alarmActions: [input.alertTopicArn],
|
|
1199
|
+
alarmDescription: alarmDescription({
|
|
1200
|
+
businessSymptom: 'Critical API operations have sustained failures.',
|
|
1201
|
+
id: 'api-slow-burn',
|
|
1202
|
+
owner: '${name}-product',
|
|
1203
|
+
route: '${name}-on-call',
|
|
1204
|
+
runbook: 'docs/runbooks/api-recovery.md',
|
|
1205
|
+
}),
|
|
1206
|
+
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1207
|
+
datapointsToAlarm: 1,
|
|
1208
|
+
evaluationPeriods: 1,
|
|
1209
|
+
metricQueries: [...apiBurn.slow.metricQueries],
|
|
1210
|
+
name: '${name}-' + $app.stage + '-api-slow-burn',
|
|
1211
|
+
threshold: apiBurn.slow.burnThreshold,
|
|
1212
|
+
treatMissingData: 'missing',
|
|
1213
|
+
},
|
|
1214
|
+
);
|
|
1215
|
+
const webLatency = new aws.cloudwatch.MetricAlarm(
|
|
1216
|
+
'${pascalName}WebLatencyBurn',
|
|
1217
|
+
{
|
|
1218
|
+
alarmActions: [input.alertTopicArn],
|
|
1219
|
+
alarmDescription: alarmDescription({
|
|
1220
|
+
businessSymptom: 'Server-rendered routes exceed the latency objective.',
|
|
1221
|
+
id: 'web-latency-fast-burn',
|
|
1222
|
+
owner: '${name}-product',
|
|
1223
|
+
route: '${name}-on-call',
|
|
1224
|
+
runbook: 'docs/runbooks/web-recovery.md',
|
|
1225
|
+
}),
|
|
1226
|
+
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1227
|
+
datapointsToAlarm: 1,
|
|
1228
|
+
evaluationPeriods: 1,
|
|
1229
|
+
metricQueries: [...webBurn.fast.metricQueries],
|
|
1230
|
+
name: '${name}-' + $app.stage + '-web-latency-fast-burn',
|
|
1231
|
+
threshold: webBurn.fast.burnThreshold,
|
|
1232
|
+
treatMissingData: 'missing',
|
|
1233
|
+
},
|
|
1234
|
+
);
|
|
1235
|
+
const webLatencySlow = new aws.cloudwatch.MetricAlarm(
|
|
1236
|
+
'${pascalName}WebLatencySlowBurn',
|
|
1237
|
+
{
|
|
1238
|
+
alarmActions: [input.alertTopicArn],
|
|
1239
|
+
alarmDescription: alarmDescription({
|
|
1240
|
+
businessSymptom: 'Server-rendered routes have sustained latency degradation.',
|
|
1241
|
+
id: 'web-latency-slow-burn',
|
|
1242
|
+
owner: '${name}-product',
|
|
1243
|
+
route: '${name}-on-call',
|
|
1244
|
+
runbook: 'docs/runbooks/web-recovery.md',
|
|
1245
|
+
}),
|
|
1246
|
+
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1247
|
+
datapointsToAlarm: 1,
|
|
1248
|
+
evaluationPeriods: 1,
|
|
1249
|
+
metricQueries: [...webBurn.slow.metricQueries],
|
|
1250
|
+
name: '${name}-' + $app.stage + '-web-latency-slow-burn',
|
|
1251
|
+
threshold: webBurn.slow.burnThreshold,
|
|
1252
|
+
treatMissingData: 'missing',
|
|
1253
|
+
},
|
|
1254
|
+
);
|
|
1255
|
+
const providerDeadline = new aws.cloudwatch.MetricAlarm(
|
|
1256
|
+
'${pascalName}ProviderDeadline',
|
|
1257
|
+
{
|
|
1258
|
+
alarmActions: [input.alertTopicArn],
|
|
1259
|
+
alarmDescription: alarmDescription({
|
|
1260
|
+
businessSymptom: 'Accepted provider work is missing its deadline.',
|
|
1261
|
+
id: 'provider-queue-age',
|
|
1262
|
+
owner: '${name}-product',
|
|
1263
|
+
route: '${name}-on-call',
|
|
1264
|
+
runbook: 'docs/runbooks/async-recovery.md',
|
|
1265
|
+
}),
|
|
1266
|
+
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1267
|
+
dimensions: { QueueName: input.providerQueueName },
|
|
1268
|
+
evaluationPeriods: 2,
|
|
1269
|
+
metricName: 'ApproximateAgeOfOldestMessage',
|
|
1270
|
+
name: '${name}-' + $app.stage + '-provider-deadline',
|
|
1271
|
+
namespace: 'AWS/SQS',
|
|
1272
|
+
period: 300,
|
|
1273
|
+
statistic: 'Maximum',
|
|
1274
|
+
threshold: 900,
|
|
1275
|
+
treatMissingData: 'missing',
|
|
1276
|
+
},
|
|
1277
|
+
);
|
|
1278
|
+
const databaseCapacity = new aws.cloudwatch.MetricAlarm(
|
|
1279
|
+
'${pascalName}DatabaseCapacity',
|
|
1280
|
+
{
|
|
1281
|
+
alarmActions: [input.alertTopicArn],
|
|
1282
|
+
alarmDescription: alarmDescription({
|
|
1283
|
+
businessSymptom: 'The shared database is approaching its reviewed capacity bound.',
|
|
1284
|
+
id: 'database-capacity',
|
|
1285
|
+
owner: '${name}-product',
|
|
1286
|
+
route: '${name}-on-call',
|
|
1287
|
+
runbook: 'docs/runbooks/database-recovery.md',
|
|
1288
|
+
}),
|
|
1289
|
+
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1290
|
+
dimensions: { DBClusterIdentifier: input.databaseClusterIdentifier },
|
|
1291
|
+
evaluationPeriods: 3,
|
|
1292
|
+
metricName: 'ServerlessDatabaseCapacity',
|
|
1293
|
+
name: '${name}-' + $app.stage + '-database-capacity',
|
|
1294
|
+
namespace: 'AWS/RDS',
|
|
1295
|
+
period: 300,
|
|
1296
|
+
statistic: 'Maximum',
|
|
1297
|
+
threshold: 3.5,
|
|
1298
|
+
treatMissingData: 'missing',
|
|
1299
|
+
},
|
|
1300
|
+
);
|
|
1301
|
+
const functionFailure = new aws.cloudwatch.MetricAlarm(
|
|
1302
|
+
'${pascalName}FunctionFailure',
|
|
1303
|
+
{
|
|
1304
|
+
alarmActions: [input.alertTopicArn],
|
|
1305
|
+
alarmDescription: alarmDescription({
|
|
1306
|
+
businessSymptom: 'A generated Function has a terminal invocation error.',
|
|
1307
|
+
id: 'function-failure',
|
|
1308
|
+
owner: '${name}-product',
|
|
1309
|
+
route: '${name}-on-call',
|
|
1310
|
+
runbook: 'docs/runbooks/api-recovery.md',
|
|
1311
|
+
}),
|
|
1312
|
+
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1313
|
+
dimensions: { FunctionName: input.functionName },
|
|
1314
|
+
evaluationPeriods: 1,
|
|
1315
|
+
metricName: 'Errors',
|
|
1316
|
+
name: '${name}-' + $app.stage + '-function-failure',
|
|
1317
|
+
namespace: 'AWS/Lambda',
|
|
1318
|
+
period: 60,
|
|
1319
|
+
statistic: 'Sum',
|
|
1320
|
+
threshold: 1,
|
|
1321
|
+
treatMissingData: 'notBreaching',
|
|
1322
|
+
},
|
|
1323
|
+
);
|
|
1324
|
+
const alertDelivery = new aws.cloudwatch.MetricAlarm(
|
|
1325
|
+
'${pascalName}AlertDelivery',
|
|
1326
|
+
{
|
|
1327
|
+
alarmDescription: alarmDescription({
|
|
1328
|
+
businessSymptom: 'The primary alert route is failing delivery.',
|
|
1329
|
+
id: 'alert-delivery-degraded',
|
|
1330
|
+
owner: '${name}-product',
|
|
1331
|
+
route: '${name}-on-call',
|
|
1332
|
+
runbook: 'docs/runbooks/alert-delivery.md',
|
|
1333
|
+
}),
|
|
1334
|
+
comparisonOperator: 'GreaterThanOrEqualToThreshold',
|
|
1335
|
+
dimensions: { TopicName: input.alertTopicName },
|
|
1336
|
+
evaluationPeriods: 1,
|
|
1337
|
+
metricName: 'NumberOfNotificationsFailed',
|
|
1338
|
+
name: '${name}-' + $app.stage + '-alert-delivery-degraded',
|
|
1339
|
+
namespace: 'AWS/SNS',
|
|
1340
|
+
okActions: [],
|
|
1341
|
+
period: 300,
|
|
1342
|
+
statistic: 'Sum',
|
|
1343
|
+
threshold: 1,
|
|
1344
|
+
treatMissingData: 'notBreaching',
|
|
1345
|
+
},
|
|
1346
|
+
);
|
|
1347
|
+
return {
|
|
1348
|
+
alertDelivery,
|
|
1349
|
+
apiFastBurn,
|
|
1350
|
+
apiSlowBurn,
|
|
1351
|
+
databaseCapacity,
|
|
1352
|
+
functionFailure,
|
|
1353
|
+
providerDeadline,
|
|
1354
|
+
webLatency,
|
|
1355
|
+
webLatencySlow,
|
|
1356
|
+
};
|
|
1357
|
+
}
|
|
1358
|
+
`,
|
|
1359
|
+
'docs/runbooks/reliability-diagnostic.md': `# Reliability diagnostic
|
|
1360
|
+
|
|
1361
|
+
The stable-v1 generated adapter supports exactly three approval-free sources:
|
|
1362
|
+
alarm state, sanitized SLO metric signals, and bounded ECS or Step Functions
|
|
1363
|
+
resource metadata. It executes read-only \`DescribeAlarms\`,
|
|
1364
|
+
\`GetMetricData\`, \`DescribeServices\`, and \`DescribeExecution\` calls
|
|
1365
|
+
against one exact target and immutable deployment binding. Dashboard, generic
|
|
1366
|
+
health, cost, and runbook lookups are not claimed as executable diagnostic
|
|
1367
|
+
sources. The adapter does not accept arbitrary metric expressions or resource
|
|
1368
|
+
scans, read application log contents, or unmask data.
|
|
1369
|
+
|
|
1370
|
+
Before an AWS read, the adapter requires a protected GitHub Actions context and
|
|
1371
|
+
selects the requested workload only through
|
|
1372
|
+
\`.ebk/evidence/reliability/active-bindings.json\`. That protected
|
|
1373
|
+
deployed-smoke index must name the current checked-out source commit and exact
|
|
1374
|
+
repository. The adapter then recomputes the binding from the exact
|
|
1375
|
+
deployed-smoke, active protected-release, and release-manifest artifact bytes.
|
|
1376
|
+
The binding also carries the environment-reviewed alarm, API, ECS, and Step
|
|
1377
|
+
Functions resource allowlist. A stale binding or substituted resource fails
|
|
1378
|
+
before its service query.
|
|
1379
|
+
|
|
1380
|
+
Sanitized metric capture is limited to six hours and produces
|
|
1381
|
+
\`reliability-slo-diagnostic-sample\`. It reports the declared objective window
|
|
1382
|
+
beside the sampled window but never claims that the sample proves the rolling
|
|
1383
|
+
28-day SLO. The report derives staging or production provenance from the active
|
|
1384
|
+
binding; callers cannot label it.
|
|
1385
|
+
|
|
1386
|
+
A bounded log query requires a human approval naming the approver, invoking
|
|
1387
|
+
human, incident, reason, account, region, stage, Slice, exact log groups, query
|
|
1388
|
+
class, time window, and expiry. The approved query window is at most one hour.
|
|
1389
|
+
The script returns an authorization plan; the existing protected query runner
|
|
1390
|
+
retains attributable evidence. Empty results never prove recovery.
|
|
1391
|
+
`,
|
|
1392
|
+
'docs/runbooks/incident-remediation.md': `# Approval-gated incident remediation
|
|
1393
|
+
|
|
1394
|
+
Use \`scripts/incident-remediation.mjs\` in report mode first. The report binds
|
|
1395
|
+
one action, account, region, stage, Slice, preconditions, stop conditions,
|
|
1396
|
+
verification, cleanup, adapter choice, and any adapter evidence through an
|
|
1397
|
+
internally computed SHA-256 remediation-plan digest. That digest is distinct
|
|
1398
|
+
from the async-recovery report digest consumed by redrive or replay.
|
|
1399
|
+
|
|
1400
|
+
Apply requires an unexpired human approval bound to that exact report, action,
|
|
1401
|
+
and target plus protected GitHub OIDC on the approved commit. The stable-v1
|
|
1402
|
+
adapter supports bounded async redrive or replay through the generated
|
|
1403
|
+
protected async-recovery path; its own resource, rate, idempotency, alarm,
|
|
1404
|
+
cleanup, and evidence controls still apply. Other remediation paths remain
|
|
1405
|
+
report-only until separately implemented. Stop when the active target or
|
|
1406
|
+
approved report changes.
|
|
1407
|
+
`,
|
|
1408
|
+
'docs/runbooks/api-recovery.md': `# API recovery
|
|
1409
|
+
|
|
1410
|
+
Confirm customer impact, API validity classification, recent release and
|
|
1411
|
+
configuration identity, database/dependency health, and correlated sanitized
|
|
1412
|
+
signals. Prefer a verified roll-forward. A source revert must traverse the
|
|
1413
|
+
normal develop-to-staging evidence path before protected production promotion.
|
|
1414
|
+
Verify the critical API journey and data invariants before resolution.
|
|
1415
|
+
`,
|
|
1416
|
+
'docs/runbooks/web-recovery.md': `# Web recovery
|
|
1417
|
+
|
|
1418
|
+
Confirm the server-rendered route, deployed release, configuration digest,
|
|
1419
|
+
origin, cache class, and dependency state. Compare cold and warm paths without
|
|
1420
|
+
combining their populations. Verify headers, authentication, keyboard/focus,
|
|
1421
|
+
responsive layout, reduced motion, and the user journey after recovery.
|
|
1422
|
+
`,
|
|
1423
|
+
'docs/runbooks/alert-delivery.md': `# Alert delivery recovery
|
|
1424
|
+
|
|
1425
|
+
The generated SNS delivery-failure alarm is diagnostic-only: it deliberately
|
|
1426
|
+
has no action on the route whose failure it observes. Record delivery as
|
|
1427
|
+
\`requested\` or \`unsupported\`; never promote it to \`confirmed\` without a
|
|
1428
|
+
receipt captured from an independently configured provider or route. Do not
|
|
1429
|
+
disable customer-impact alarms. Repair or select an approved independent route,
|
|
1430
|
+
send one bounded proof, and retain its provider receipt before resolving.
|
|
1431
|
+
`,
|
|
1432
|
+
'scripts/reliability-evidence-lib.mjs': `import { createHash } from 'node:crypto';
|
|
1433
|
+
import { isBoundedSafeReference } from '@empire-builder-kit/runtime/safe-reference';
|
|
1434
|
+
import {
|
|
1435
|
+
lstatSync,
|
|
1436
|
+
mkdirSync,
|
|
1437
|
+
readFileSync,
|
|
1438
|
+
writeFileSync,
|
|
1439
|
+
} from 'node:fs';
|
|
1440
|
+
import { relative, resolve, sep } from 'node:path';
|
|
1441
|
+
|
|
1442
|
+
export const MAX_RELIABILITY_ARTIFACT_BYTES = 64 * 1024;
|
|
1443
|
+
const DIGEST = /^[a-f0-9]{64}$/;
|
|
1444
|
+
|
|
1445
|
+
export function sha256(bytes) {
|
|
1446
|
+
return createHash('sha256').update(bytes).digest('hex');
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
export function exactKeys(value, allowed, label) {
|
|
1450
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
1451
|
+
throw new Error(label + ' must be an object.');
|
|
1452
|
+
}
|
|
1453
|
+
const unexpected = Object.keys(value).filter((key) => !allowed.includes(key));
|
|
1454
|
+
if (unexpected.length > 0) {
|
|
1455
|
+
throw new Error(label + ' contains unsupported fields: ' + unexpected.sort().join(', ') + '.');
|
|
1456
|
+
}
|
|
1457
|
+
return value;
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1460
|
+
export function safeToken(value, label, maximum = 512) {
|
|
1461
|
+
if (
|
|
1462
|
+
typeof value !== 'string' ||
|
|
1463
|
+
value.length < 1 ||
|
|
1464
|
+
value.length > maximum ||
|
|
1465
|
+
!isBoundedSafeReference(value, 'repository-evidence') ||
|
|
1466
|
+
value.includes('..') ||
|
|
1467
|
+
value.includes('\\\\')
|
|
1468
|
+
) {
|
|
1469
|
+
throw new Error(label + ' is not a bounded sanitized token.');
|
|
1470
|
+
}
|
|
1471
|
+
return value;
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1474
|
+
export function digest(value, label) {
|
|
1475
|
+
if (typeof value !== 'string' || !DIGEST.test(value)) {
|
|
1476
|
+
throw new Error(label + ' must be an exact SHA-256 digest.');
|
|
1477
|
+
}
|
|
1478
|
+
return value;
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
export function utc(value, label) {
|
|
1482
|
+
if (
|
|
1483
|
+
typeof value !== 'string' ||
|
|
1484
|
+
value.length > 32 ||
|
|
1485
|
+
!value.endsWith('Z') ||
|
|
1486
|
+
!Number.isFinite(Date.parse(value))
|
|
1487
|
+
) {
|
|
1488
|
+
throw new Error(label + ' must be a bounded UTC timestamp.');
|
|
1489
|
+
}
|
|
1490
|
+
return value;
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1493
|
+
export function readBoundedJson(path, label) {
|
|
1494
|
+
const bytes = readFileSync(path);
|
|
1495
|
+
if (bytes.length === 0 || bytes.length > MAX_RELIABILITY_ARTIFACT_BYTES) {
|
|
1496
|
+
throw new Error(label + ' exceeds the reliability artifact size bound.');
|
|
1497
|
+
}
|
|
1498
|
+
let value;
|
|
1499
|
+
try {
|
|
1500
|
+
value = JSON.parse(bytes.toString('utf8'));
|
|
1501
|
+
} catch {
|
|
1502
|
+
throw new Error(label + ' is not valid JSON.');
|
|
1503
|
+
}
|
|
1504
|
+
return { bytes, value };
|
|
1505
|
+
}
|
|
1506
|
+
|
|
1507
|
+
export function binding(value, label) {
|
|
1508
|
+
const record = exactKeys(
|
|
1509
|
+
value,
|
|
1510
|
+
[
|
|
1511
|
+
'candidateDigest',
|
|
1512
|
+
'configurationDigest',
|
|
1513
|
+
'deploymentDigest',
|
|
1514
|
+
'releaseDigest',
|
|
1515
|
+
'sourceCommit',
|
|
1516
|
+
'workload',
|
|
1517
|
+
],
|
|
1518
|
+
label,
|
|
1519
|
+
);
|
|
1520
|
+
const sourceCommit = safeToken(
|
|
1521
|
+
record.sourceCommit,
|
|
1522
|
+
label + '.sourceCommit',
|
|
1523
|
+
40,
|
|
1524
|
+
);
|
|
1525
|
+
if (!/^[a-f0-9]{40}$/.test(sourceCommit)) {
|
|
1526
|
+
throw new Error(label + '.sourceCommit must be an exact git commit.');
|
|
1527
|
+
}
|
|
1528
|
+
return {
|
|
1529
|
+
candidateDigest: digest(record.candidateDigest, label + '.candidateDigest'),
|
|
1530
|
+
configurationDigest: digest(
|
|
1531
|
+
record.configurationDigest,
|
|
1532
|
+
label + '.configurationDigest',
|
|
1533
|
+
),
|
|
1534
|
+
deploymentDigest: digest(
|
|
1535
|
+
record.deploymentDigest,
|
|
1536
|
+
label + '.deploymentDigest',
|
|
1537
|
+
),
|
|
1538
|
+
releaseDigest: digest(record.releaseDigest, label + '.releaseDigest'),
|
|
1539
|
+
sourceCommit,
|
|
1540
|
+
workload: safeToken(record.workload, label + '.workload', 128),
|
|
1541
|
+
};
|
|
1542
|
+
}
|
|
1543
|
+
|
|
1544
|
+
export function sameBinding(left, right) {
|
|
1545
|
+
return (
|
|
1546
|
+
left.candidateDigest === right.candidateDigest &&
|
|
1547
|
+
left.configurationDigest === right.configurationDigest &&
|
|
1548
|
+
left.deploymentDigest === right.deploymentDigest &&
|
|
1549
|
+
left.releaseDigest === right.releaseDigest &&
|
|
1550
|
+
left.sourceCommit === right.sourceCommit &&
|
|
1551
|
+
left.workload === right.workload
|
|
1552
|
+
);
|
|
1553
|
+
}
|
|
1554
|
+
|
|
1555
|
+
function artifactReferencePath(value, label) {
|
|
1556
|
+
const path = value;
|
|
1557
|
+
if (
|
|
1558
|
+
typeof path !== 'string' ||
|
|
1559
|
+
path.length < 1 ||
|
|
1560
|
+
path.length > 1024 ||
|
|
1561
|
+
!/^(?:[A-Za-z0-9._-]+\\/)*[A-Za-z0-9._-]+$/.test(path) ||
|
|
1562
|
+
path.startsWith('/') ||
|
|
1563
|
+
path.split('/').includes('..')
|
|
1564
|
+
) {
|
|
1565
|
+
throw new Error(label + ' must remain inside the Slice package.');
|
|
1566
|
+
}
|
|
1567
|
+
const root = resolve(process.cwd());
|
|
1568
|
+
const absolute = resolve(root, path);
|
|
1569
|
+
const within = relative(root, absolute);
|
|
1570
|
+
if (within.startsWith('..' + sep) || within === '..') {
|
|
1571
|
+
throw new Error(label + ' escapes the Slice package.');
|
|
1572
|
+
}
|
|
1573
|
+
return path;
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
export function artifactReference(value, label) {
|
|
1577
|
+
const record = exactKeys(value, ['digest', 'path'], label);
|
|
1578
|
+
const expectedDigest = digest(record.digest, label + '.digest');
|
|
1579
|
+
const path = artifactReferencePath(record.path, label + '.path');
|
|
1580
|
+
const absolute = resolve(process.cwd(), path);
|
|
1581
|
+
const artifact = readBoundedJson(absolute, label);
|
|
1582
|
+
const metadata = lstatSync(absolute);
|
|
1583
|
+
if (!metadata.isFile() || metadata.isSymbolicLink()) {
|
|
1584
|
+
throw new Error(label + ' must be a regular non-symlink artifact.');
|
|
1585
|
+
}
|
|
1586
|
+
if (sha256(artifact.bytes) !== expectedDigest) {
|
|
1587
|
+
throw new Error(label + ' digest does not match its exact artifact bytes.');
|
|
1588
|
+
}
|
|
1589
|
+
return { ...artifact, digest: expectedDigest, path };
|
|
1590
|
+
}
|
|
1591
|
+
|
|
1592
|
+
function reliabilityTarget(value, label) {
|
|
1593
|
+
const target = exactKeys(
|
|
1594
|
+
value,
|
|
1595
|
+
['account', 'region', 'slice', 'stage'],
|
|
1596
|
+
label,
|
|
1597
|
+
);
|
|
1598
|
+
const account = safeToken(target.account, label + '.account', 12);
|
|
1599
|
+
if (!/^\\d{12}$/.test(account)) {
|
|
1600
|
+
throw new Error(label + '.account must be an exact AWS account ID.');
|
|
1601
|
+
}
|
|
1602
|
+
const stage = safeToken(target.stage, label + '.stage', 16);
|
|
1603
|
+
if (!['staging', 'production'].includes(stage)) {
|
|
1604
|
+
throw new Error(label + '.stage must be a protected stage.');
|
|
1605
|
+
}
|
|
1606
|
+
const slice = safeToken(target.slice, label + '.slice', 128);
|
|
1607
|
+
if (slice !== '${name}') {
|
|
1608
|
+
throw new Error(label + '.slice escaped the generated Slice.');
|
|
1609
|
+
}
|
|
1610
|
+
const region = safeToken(target.region, label + '.region', 64);
|
|
1611
|
+
if (!/^[a-z]{2}(?:-[a-z]+)+-\\d$/.test(region)) {
|
|
1612
|
+
throw new Error(label + '.region must be an exact AWS Region.');
|
|
1613
|
+
}
|
|
1614
|
+
return {
|
|
1615
|
+
account,
|
|
1616
|
+
region,
|
|
1617
|
+
slice,
|
|
1618
|
+
stage,
|
|
1619
|
+
};
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1622
|
+
export function awsPartition(region) {
|
|
1623
|
+
if (region.startsWith('cn-')) return 'aws-cn';
|
|
1624
|
+
if (region.startsWith('us-gov-')) return 'aws-us-gov';
|
|
1625
|
+
if (region.startsWith('us-isob-')) return 'aws-iso-b';
|
|
1626
|
+
if (region.startsWith('us-isoe-')) return 'aws-iso-e';
|
|
1627
|
+
if (region.startsWith('us-isof-')) return 'aws-iso-f';
|
|
1628
|
+
if (region.startsWith('us-iso-')) return 'aws-iso';
|
|
1629
|
+
if (region.startsWith('eusc-')) return 'aws-eusc';
|
|
1630
|
+
return 'aws';
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
function reliabilityResources(value, target, label) {
|
|
1634
|
+
const resources = exactKeys(
|
|
1635
|
+
value,
|
|
1636
|
+
['alarmNames', 'apiId', 'ecsServices', 'stateMachineArns'],
|
|
1637
|
+
label,
|
|
1638
|
+
);
|
|
1639
|
+
if (
|
|
1640
|
+
!Array.isArray(resources.alarmNames) ||
|
|
1641
|
+
resources.alarmNames.length < 1 ||
|
|
1642
|
+
resources.alarmNames.length > 100 ||
|
|
1643
|
+
new Set(resources.alarmNames).size !== resources.alarmNames.length
|
|
1644
|
+
) {
|
|
1645
|
+
throw new Error(label + '.alarmNames must contain exact unique alarms.');
|
|
1646
|
+
}
|
|
1647
|
+
const alarmNames = resources.alarmNames
|
|
1648
|
+
.map((value, index) =>
|
|
1649
|
+
safeToken(value, label + '.alarmNames[' + index + ']', 255),
|
|
1650
|
+
)
|
|
1651
|
+
.sort();
|
|
1652
|
+
if (
|
|
1653
|
+
alarmNames.some(
|
|
1654
|
+
(value) => !value.startsWith(target.slice + '-' + target.stage + '-'),
|
|
1655
|
+
)
|
|
1656
|
+
) {
|
|
1657
|
+
throw new Error(label + '.alarmNames escaped the exact Slice and stage.');
|
|
1658
|
+
}
|
|
1659
|
+
const apiId = safeToken(resources.apiId, label + '.apiId', 128);
|
|
1660
|
+
if (!/^[a-z0-9]{1,128}$/.test(apiId)) {
|
|
1661
|
+
throw new Error(label + '.apiId must be one exact API Gateway ID.');
|
|
1662
|
+
}
|
|
1663
|
+
if (
|
|
1664
|
+
!Array.isArray(resources.ecsServices) ||
|
|
1665
|
+
resources.ecsServices.length > 8
|
|
1666
|
+
) {
|
|
1667
|
+
throw new Error(label + '.ecsServices exceeds its eight-cluster bound.');
|
|
1668
|
+
}
|
|
1669
|
+
const ecsServices = resources.ecsServices
|
|
1670
|
+
.map((value, index) => {
|
|
1671
|
+
const entry = exactKeys(
|
|
1672
|
+
value,
|
|
1673
|
+
['cluster', 'services'],
|
|
1674
|
+
label + '.ecsServices[' + index + ']',
|
|
1675
|
+
);
|
|
1676
|
+
const cluster = safeToken(
|
|
1677
|
+
entry.cluster,
|
|
1678
|
+
label + '.ecsServices[' + index + '].cluster',
|
|
1679
|
+
512,
|
|
1680
|
+
);
|
|
1681
|
+
if (
|
|
1682
|
+
!Array.isArray(entry.services) ||
|
|
1683
|
+
entry.services.length < 1 ||
|
|
1684
|
+
entry.services.length > 10 ||
|
|
1685
|
+
new Set(entry.services).size !== entry.services.length
|
|
1686
|
+
) {
|
|
1687
|
+
throw new Error(
|
|
1688
|
+
label + '.ecsServices[' + index + '] has invalid services.',
|
|
1689
|
+
);
|
|
1690
|
+
}
|
|
1691
|
+
const services = entry.services
|
|
1692
|
+
.map((service, serviceIndex) =>
|
|
1693
|
+
safeToken(
|
|
1694
|
+
service,
|
|
1695
|
+
label +
|
|
1696
|
+
'.ecsServices[' +
|
|
1697
|
+
index +
|
|
1698
|
+
'].services[' +
|
|
1699
|
+
serviceIndex +
|
|
1700
|
+
']',
|
|
1701
|
+
255,
|
|
1702
|
+
),
|
|
1703
|
+
)
|
|
1704
|
+
.sort();
|
|
1705
|
+
const clusterArnPrefix =
|
|
1706
|
+
'arn:' +
|
|
1707
|
+
awsPartition(target.region) +
|
|
1708
|
+
':ecs:' +
|
|
1709
|
+
target.region +
|
|
1710
|
+
':' +
|
|
1711
|
+
target.account +
|
|
1712
|
+
':cluster/';
|
|
1713
|
+
if (
|
|
1714
|
+
cluster.startsWith('arn:') &&
|
|
1715
|
+
!cluster.startsWith(clusterArnPrefix)
|
|
1716
|
+
) {
|
|
1717
|
+
throw new Error(
|
|
1718
|
+
label + '.ecsServices[' + index + '] escaped the exact target.',
|
|
1719
|
+
);
|
|
1720
|
+
}
|
|
1721
|
+
return { cluster, services };
|
|
1722
|
+
})
|
|
1723
|
+
.sort((left, right) =>
|
|
1724
|
+
left.cluster < right.cluster
|
|
1725
|
+
? -1
|
|
1726
|
+
: left.cluster > right.cluster
|
|
1727
|
+
? 1
|
|
1728
|
+
: 0,
|
|
1729
|
+
);
|
|
1730
|
+
if (
|
|
1731
|
+
new Set(ecsServices.map(({ cluster }) => cluster)).size !==
|
|
1732
|
+
ecsServices.length
|
|
1733
|
+
) {
|
|
1734
|
+
throw new Error(label + '.ecsServices repeats an exact cluster.');
|
|
1735
|
+
}
|
|
1736
|
+
if (
|
|
1737
|
+
!Array.isArray(resources.stateMachineArns) ||
|
|
1738
|
+
resources.stateMachineArns.length > 8 ||
|
|
1739
|
+
new Set(resources.stateMachineArns).size !==
|
|
1740
|
+
resources.stateMachineArns.length
|
|
1741
|
+
) {
|
|
1742
|
+
throw new Error(
|
|
1743
|
+
label + '.stateMachineArns exceeds its exact unique bound.',
|
|
1744
|
+
);
|
|
1745
|
+
}
|
|
1746
|
+
const stateMachinePrefix =
|
|
1747
|
+
'arn:' +
|
|
1748
|
+
awsPartition(target.region) +
|
|
1749
|
+
':states:' +
|
|
1750
|
+
target.region +
|
|
1751
|
+
':' +
|
|
1752
|
+
target.account +
|
|
1753
|
+
':stateMachine:';
|
|
1754
|
+
const stateMachineArns = resources.stateMachineArns
|
|
1755
|
+
.map((value, index) =>
|
|
1756
|
+
safeToken(value, label + '.stateMachineArns[' + index + ']', 512),
|
|
1757
|
+
)
|
|
1758
|
+
.sort();
|
|
1759
|
+
if (
|
|
1760
|
+
stateMachineArns.some(
|
|
1761
|
+
(value) =>
|
|
1762
|
+
!value.startsWith(stateMachinePrefix) ||
|
|
1763
|
+
value.length === stateMachinePrefix.length,
|
|
1764
|
+
)
|
|
1765
|
+
) {
|
|
1766
|
+
throw new Error(label + '.stateMachineArns escaped the exact target.');
|
|
1767
|
+
}
|
|
1768
|
+
return {
|
|
1769
|
+
alarmNames,
|
|
1770
|
+
apiId,
|
|
1771
|
+
ecsServices,
|
|
1772
|
+
stateMachineArns,
|
|
1773
|
+
};
|
|
1774
|
+
}
|
|
1775
|
+
|
|
1776
|
+
export function authorizedExecutionArn(resources, executionArn) {
|
|
1777
|
+
return resources.stateMachineArns.some((stateMachineArn) => {
|
|
1778
|
+
const separator = ':stateMachine:';
|
|
1779
|
+
const position = stateMachineArn.indexOf(separator);
|
|
1780
|
+
if (position < 0) return false;
|
|
1781
|
+
const executionPrefix =
|
|
1782
|
+
stateMachineArn.slice(0, position) +
|
|
1783
|
+
':execution:' +
|
|
1784
|
+
stateMachineArn.slice(position + separator.length) +
|
|
1785
|
+
':';
|
|
1786
|
+
return executionArn.startsWith(executionPrefix);
|
|
1787
|
+
});
|
|
1788
|
+
}
|
|
1789
|
+
|
|
1790
|
+
function protectedDeploymentSource(reference, target, expectedSmoke) {
|
|
1791
|
+
const deploymentReference = artifactReference(
|
|
1792
|
+
reference,
|
|
1793
|
+
'sources.deployment',
|
|
1794
|
+
);
|
|
1795
|
+
const deployment = exactKeys(
|
|
1796
|
+
deploymentReference.value,
|
|
1797
|
+
[
|
|
1798
|
+
'deployment',
|
|
1799
|
+
'kind',
|
|
1800
|
+
'project',
|
|
1801
|
+
'resources',
|
|
1802
|
+
'schemaVersion',
|
|
1803
|
+
'smoke',
|
|
1804
|
+
'sourceCommit',
|
|
1805
|
+
'stage',
|
|
1806
|
+
'trigger',
|
|
1807
|
+
],
|
|
1808
|
+
'Protected deployment binding',
|
|
1809
|
+
);
|
|
1810
|
+
const identity = exactKeys(
|
|
1811
|
+
deployment.deployment,
|
|
1812
|
+
[
|
|
1813
|
+
'conclusion',
|
|
1814
|
+
'event',
|
|
1815
|
+
'headBranch',
|
|
1816
|
+
'headSha',
|
|
1817
|
+
'runAttempt',
|
|
1818
|
+
'runId',
|
|
1819
|
+
'workflow',
|
|
1820
|
+
],
|
|
1821
|
+
'Protected deployment identity',
|
|
1822
|
+
);
|
|
1823
|
+
const smoke = exactKeys(
|
|
1824
|
+
deployment.smoke,
|
|
1825
|
+
['runAttempt', 'runId', 'workflow'],
|
|
1826
|
+
'Protected smoke identity',
|
|
1827
|
+
);
|
|
1828
|
+
const sourceCommit = safeToken(
|
|
1829
|
+
deployment.sourceCommit,
|
|
1830
|
+
'Protected deployment sourceCommit',
|
|
1831
|
+
40,
|
|
1832
|
+
);
|
|
1833
|
+
const expectedBranch = target.stage === 'production' ? 'main' : 'develop';
|
|
1834
|
+
if (
|
|
1835
|
+
deployment.kind !== 'ebk-deployed-smoke-binding' ||
|
|
1836
|
+
deployment.schemaVersion !== 1 ||
|
|
1837
|
+
deployment.project !== '${name}' ||
|
|
1838
|
+
deployment.stage !== target.stage ||
|
|
1839
|
+
deployment.trigger !== 'deployment' ||
|
|
1840
|
+
identity.conclusion !== 'success' ||
|
|
1841
|
+
!['push', 'workflow_dispatch'].includes(identity.event) ||
|
|
1842
|
+
identity.headBranch !== expectedBranch ||
|
|
1843
|
+
identity.headSha !== sourceCommit ||
|
|
1844
|
+
identity.workflow !== 'Deploy' ||
|
|
1845
|
+
smoke.workflow !== 'Deployed smoke' ||
|
|
1846
|
+
String(smoke.runId) !== expectedSmoke.runId ||
|
|
1847
|
+
String(smoke.runAttempt) !== expectedSmoke.runAttempt ||
|
|
1848
|
+
!/^\\d+$/.test(String(identity.runId ?? '')) ||
|
|
1849
|
+
!/^\\d+$/.test(String(identity.runAttempt ?? '')) ||
|
|
1850
|
+
!/^[a-f0-9]{40}$/.test(sourceCommit)
|
|
1851
|
+
) {
|
|
1852
|
+
throw new Error(
|
|
1853
|
+
'Protected deployment source does not prove the active Slice deployment.',
|
|
1854
|
+
);
|
|
1855
|
+
}
|
|
1856
|
+
return {
|
|
1857
|
+
reference: deploymentReference,
|
|
1858
|
+
resources: reliabilityResources(
|
|
1859
|
+
deployment.resources,
|
|
1860
|
+
target,
|
|
1861
|
+
'Protected deployment resources',
|
|
1862
|
+
),
|
|
1863
|
+
runAttempt: String(identity.runAttempt),
|
|
1864
|
+
runId: String(identity.runId),
|
|
1865
|
+
sourceCommit,
|
|
1866
|
+
};
|
|
1867
|
+
}
|
|
1868
|
+
|
|
1869
|
+
function protectedReleaseSource(reference, target, deployment) {
|
|
1870
|
+
const releaseReference = artifactReference(
|
|
1871
|
+
reference,
|
|
1872
|
+
'sources.release',
|
|
1873
|
+
);
|
|
1874
|
+
const release = exactKeys(
|
|
1875
|
+
releaseReference.value,
|
|
1876
|
+
[
|
|
1877
|
+
'candidateDigest',
|
|
1878
|
+
'deploymentCommit',
|
|
1879
|
+
'kind',
|
|
1880
|
+
'manifestDigest',
|
|
1881
|
+
'releaseId',
|
|
1882
|
+
'releaseMaterialDigest',
|
|
1883
|
+
'releaseSourceCommit',
|
|
1884
|
+
'runAttempt',
|
|
1885
|
+
'runId',
|
|
1886
|
+
'schemaVersion',
|
|
1887
|
+
'stage',
|
|
1888
|
+
'workflow',
|
|
1889
|
+
],
|
|
1890
|
+
'Active protected release',
|
|
1891
|
+
);
|
|
1892
|
+
const releaseSourceCommit = safeToken(
|
|
1893
|
+
release.releaseSourceCommit,
|
|
1894
|
+
'Active protected release.releaseSourceCommit',
|
|
1895
|
+
40,
|
|
1896
|
+
);
|
|
1897
|
+
if (
|
|
1898
|
+
release.kind !== 'ebk-active-protected-release-binding' ||
|
|
1899
|
+
release.schemaVersion !== 1 ||
|
|
1900
|
+
release.stage !== target.stage ||
|
|
1901
|
+
release.deploymentCommit !== deployment.sourceCommit ||
|
|
1902
|
+
String(release.runId) !== deployment.runId ||
|
|
1903
|
+
String(release.runAttempt) !== deployment.runAttempt ||
|
|
1904
|
+
release.workflow !== 'Deploy' ||
|
|
1905
|
+
!/^[a-f0-9]{40}$/.test(releaseSourceCommit)
|
|
1906
|
+
) {
|
|
1907
|
+
throw new Error(
|
|
1908
|
+
'Active release does not match the protected deployment identity.',
|
|
1909
|
+
);
|
|
1910
|
+
}
|
|
1911
|
+
return {
|
|
1912
|
+
candidateDigest: digest(
|
|
1913
|
+
release.candidateDigest,
|
|
1914
|
+
'Active protected release.candidateDigest',
|
|
1915
|
+
),
|
|
1916
|
+
manifestDigest: digest(
|
|
1917
|
+
release.manifestDigest,
|
|
1918
|
+
'Active protected release.manifestDigest',
|
|
1919
|
+
),
|
|
1920
|
+
reference: releaseReference,
|
|
1921
|
+
releaseMaterialDigest: digest(
|
|
1922
|
+
release.releaseMaterialDigest,
|
|
1923
|
+
'Active protected release.releaseMaterialDigest',
|
|
1924
|
+
),
|
|
1925
|
+
releaseSourceCommit,
|
|
1926
|
+
};
|
|
1927
|
+
}
|
|
1928
|
+
|
|
1929
|
+
function releaseManifestSource(reference, target, sourceCommit) {
|
|
1930
|
+
const manifestReference = artifactReference(
|
|
1931
|
+
reference,
|
|
1932
|
+
'sources.manifest',
|
|
1933
|
+
);
|
|
1934
|
+
const manifest = exactKeys(
|
|
1935
|
+
manifestReference.value,
|
|
1936
|
+
[
|
|
1937
|
+
'artifacts',
|
|
1938
|
+
'blueprints',
|
|
1939
|
+
'build',
|
|
1940
|
+
'builtAt',
|
|
1941
|
+
'configuration',
|
|
1942
|
+
'contracts',
|
|
1943
|
+
'controls',
|
|
1944
|
+
'frameworkCandidate',
|
|
1945
|
+
'infrastructure',
|
|
1946
|
+
'initialLifecycle',
|
|
1947
|
+
'kind',
|
|
1948
|
+
'materialDigest',
|
|
1949
|
+
'migrations',
|
|
1950
|
+
'packageLockDigest',
|
|
1951
|
+
'projects',
|
|
1952
|
+
'releaseId',
|
|
1953
|
+
'requiredTests',
|
|
1954
|
+
'schemaVersion',
|
|
1955
|
+
'secrets',
|
|
1956
|
+
'source',
|
|
1957
|
+
'supplyChain',
|
|
1958
|
+
'toolchain',
|
|
1959
|
+
],
|
|
1960
|
+
'Release manifest',
|
|
1961
|
+
);
|
|
1962
|
+
const source = exactKeys(
|
|
1963
|
+
manifest.source,
|
|
1964
|
+
['commit', 'repository', 'treeDigest'],
|
|
1965
|
+
'Release manifest source',
|
|
1966
|
+
);
|
|
1967
|
+
const infrastructure = exactKeys(
|
|
1968
|
+
manifest.infrastructure,
|
|
1969
|
+
[
|
|
1970
|
+
'aggregateRisk',
|
|
1971
|
+
'configurationMetadataDigest',
|
|
1972
|
+
'materialDigest',
|
|
1973
|
+
'policyProfileDigest',
|
|
1974
|
+
'riskClasses',
|
|
1975
|
+
'stage',
|
|
1976
|
+
'tool',
|
|
1977
|
+
],
|
|
1978
|
+
'Release manifest infrastructure',
|
|
1979
|
+
);
|
|
1980
|
+
if (
|
|
1981
|
+
manifest.kind !== 'ebk-release-manifest' ||
|
|
1982
|
+
manifest.schemaVersion !== 1 ||
|
|
1983
|
+
source.commit !== sourceCommit ||
|
|
1984
|
+
infrastructure.stage !== target.stage ||
|
|
1985
|
+
!Array.isArray(manifest.projects) ||
|
|
1986
|
+
!manifest.projects.includes('${name}') ||
|
|
1987
|
+
!Array.isArray(manifest.configuration)
|
|
1988
|
+
) {
|
|
1989
|
+
throw new Error(
|
|
1990
|
+
'Release manifest does not match the protected deployment source.',
|
|
1991
|
+
);
|
|
1992
|
+
}
|
|
1993
|
+
digest(manifest.materialDigest, 'Release manifest.materialDigest');
|
|
1994
|
+
const configuration = manifest.configuration.filter(
|
|
1995
|
+
(entry) => entry?.stage === target.stage,
|
|
1996
|
+
);
|
|
1997
|
+
if (configuration.length !== 1) {
|
|
1998
|
+
throw new Error(
|
|
1999
|
+
'Release manifest must contain one exact protected-stage configuration.',
|
|
2000
|
+
);
|
|
2001
|
+
}
|
|
2002
|
+
const configured = exactKeys(
|
|
2003
|
+
configuration[0],
|
|
2004
|
+
['digest', 'stage'],
|
|
2005
|
+
'Release manifest configuration',
|
|
2006
|
+
);
|
|
2007
|
+
return {
|
|
2008
|
+
configurationDigest: digest(
|
|
2009
|
+
configured.digest,
|
|
2010
|
+
'Release manifest configuration.digest',
|
|
2011
|
+
),
|
|
2012
|
+
reference: manifestReference,
|
|
2013
|
+
};
|
|
2014
|
+
}
|
|
2015
|
+
|
|
2016
|
+
export function createReliabilityDeploymentBinding(
|
|
2017
|
+
input,
|
|
2018
|
+
expectedTarget,
|
|
2019
|
+
expectedSmoke,
|
|
2020
|
+
) {
|
|
2021
|
+
const request = exactKeys(
|
|
2022
|
+
input,
|
|
2023
|
+
['schemaVersion', 'sources', 'workload'],
|
|
2024
|
+
'Reliability deployment binding input',
|
|
2025
|
+
);
|
|
2026
|
+
if (request.schemaVersion !== 1) {
|
|
2027
|
+
throw new Error('Reliability deployment binding input is incompatible.');
|
|
2028
|
+
}
|
|
2029
|
+
const target = reliabilityTarget(expectedTarget, 'target');
|
|
2030
|
+
const sources = exactKeys(
|
|
2031
|
+
request.sources,
|
|
2032
|
+
['deployment', 'manifest', 'release'],
|
|
2033
|
+
'Reliability deployment sources',
|
|
2034
|
+
);
|
|
2035
|
+
const smoke = exactKeys(
|
|
2036
|
+
expectedSmoke,
|
|
2037
|
+
['runAttempt', 'runId'],
|
|
2038
|
+
'expectedSmoke',
|
|
2039
|
+
);
|
|
2040
|
+
for (const [name, value] of Object.entries(smoke)) {
|
|
2041
|
+
if (!/^\\d+$/.test(String(value ?? ''))) {
|
|
2042
|
+
throw new Error('expectedSmoke.' + name + ' must be an exact run number.');
|
|
2043
|
+
}
|
|
2044
|
+
}
|
|
2045
|
+
const deployment = protectedDeploymentSource(sources.deployment, target, {
|
|
2046
|
+
runAttempt: String(smoke.runAttempt),
|
|
2047
|
+
runId: String(smoke.runId),
|
|
2048
|
+
});
|
|
2049
|
+
const release = protectedReleaseSource(
|
|
2050
|
+
sources.release,
|
|
2051
|
+
target,
|
|
2052
|
+
deployment,
|
|
2053
|
+
);
|
|
2054
|
+
const manifest = releaseManifestSource(
|
|
2055
|
+
sources.manifest,
|
|
2056
|
+
target,
|
|
2057
|
+
release.releaseSourceCommit,
|
|
2058
|
+
);
|
|
2059
|
+
if (
|
|
2060
|
+
release.manifestDigest !== manifest.reference.digest ||
|
|
2061
|
+
release.releaseMaterialDigest !==
|
|
2062
|
+
manifest.reference.value.materialDigest
|
|
2063
|
+
) {
|
|
2064
|
+
throw new Error(
|
|
2065
|
+
'Active release does not bind the exact release manifest bytes.',
|
|
2066
|
+
);
|
|
2067
|
+
}
|
|
2068
|
+
return {
|
|
2069
|
+
binding: {
|
|
2070
|
+
candidateDigest: release.candidateDigest,
|
|
2071
|
+
configurationDigest: manifest.configurationDigest,
|
|
2072
|
+
deploymentDigest: deployment.reference.digest,
|
|
2073
|
+
releaseDigest: manifest.reference.digest,
|
|
2074
|
+
sourceCommit: release.releaseSourceCommit,
|
|
2075
|
+
workload: safeToken(request.workload, 'workload', 128),
|
|
2076
|
+
},
|
|
2077
|
+
kind: 'reliability-active-deployment-binding',
|
|
2078
|
+
resources: deployment.resources,
|
|
2079
|
+
schemaVersion: 1,
|
|
2080
|
+
slice: '${name}',
|
|
2081
|
+
smoke: {
|
|
2082
|
+
runAttempt: String(smoke.runAttempt),
|
|
2083
|
+
runId: String(smoke.runId),
|
|
2084
|
+
},
|
|
2085
|
+
sources: {
|
|
2086
|
+
deployment: {
|
|
2087
|
+
digest: deployment.reference.digest,
|
|
2088
|
+
path: deployment.reference.path,
|
|
2089
|
+
},
|
|
2090
|
+
manifest: {
|
|
2091
|
+
digest: manifest.reference.digest,
|
|
2092
|
+
path: manifest.reference.path,
|
|
2093
|
+
},
|
|
2094
|
+
release: {
|
|
2095
|
+
digest: release.reference.digest,
|
|
2096
|
+
path: release.reference.path,
|
|
2097
|
+
},
|
|
2098
|
+
},
|
|
2099
|
+
target,
|
|
2100
|
+
};
|
|
2101
|
+
}
|
|
2102
|
+
|
|
2103
|
+
export function activeDeploymentBinding(reference, label) {
|
|
2104
|
+
if (
|
|
2105
|
+
process.env.GITHUB_ACTIONS !== 'true' ||
|
|
2106
|
+
process.env.GITHUB_REF_PROTECTED !== 'true'
|
|
2107
|
+
) {
|
|
2108
|
+
throw new Error(
|
|
2109
|
+
'AWS reliability access requires one protected GitHub Actions context.',
|
|
2110
|
+
);
|
|
2111
|
+
}
|
|
2112
|
+
const requested = exactKeys(reference, ['digest', 'path'], label);
|
|
2113
|
+
const indexPath = '.ebk/evidence/reliability/active-bindings.json';
|
|
2114
|
+
const indexAbsolute = resolve(process.cwd(), indexPath);
|
|
2115
|
+
const indexArtifact = readBoundedJson(
|
|
2116
|
+
indexAbsolute,
|
|
2117
|
+
'Protected reliability binding index',
|
|
2118
|
+
);
|
|
2119
|
+
const indexMetadata = lstatSync(indexAbsolute);
|
|
2120
|
+
if (!indexMetadata.isFile() || indexMetadata.isSymbolicLink()) {
|
|
2121
|
+
throw new Error(
|
|
2122
|
+
'Protected reliability binding index must be a regular non-symlink artifact.',
|
|
2123
|
+
);
|
|
2124
|
+
}
|
|
2125
|
+
const index = exactKeys(
|
|
2126
|
+
indexArtifact.value,
|
|
2127
|
+
[
|
|
2128
|
+
'authority',
|
|
2129
|
+
'bindings',
|
|
2130
|
+
'kind',
|
|
2131
|
+
'schemaVersion',
|
|
2132
|
+
'slice',
|
|
2133
|
+
'stage',
|
|
2134
|
+
'target',
|
|
2135
|
+
],
|
|
2136
|
+
'Protected reliability binding index',
|
|
2137
|
+
);
|
|
2138
|
+
const authority = exactKeys(
|
|
2139
|
+
index.authority,
|
|
2140
|
+
[
|
|
2141
|
+
'repository',
|
|
2142
|
+
'runAttempt',
|
|
2143
|
+
'runId',
|
|
2144
|
+
'sourceCommit',
|
|
2145
|
+
'workflow',
|
|
2146
|
+
],
|
|
2147
|
+
'Protected reliability binding authority',
|
|
2148
|
+
);
|
|
2149
|
+
const sourceCommit = safeToken(
|
|
2150
|
+
authority.sourceCommit,
|
|
2151
|
+
'Protected reliability binding authority.sourceCommit',
|
|
2152
|
+
40,
|
|
2153
|
+
);
|
|
2154
|
+
const repository = safeToken(
|
|
2155
|
+
authority.repository,
|
|
2156
|
+
'Protected reliability binding authority.repository',
|
|
2157
|
+
256,
|
|
2158
|
+
);
|
|
2159
|
+
if (
|
|
2160
|
+
index.kind !== 'reliability-active-binding-index' ||
|
|
2161
|
+
index.schemaVersion !== 1 ||
|
|
2162
|
+
index.slice !== '${name}' ||
|
|
2163
|
+
!['staging', 'production'].includes(index.stage) ||
|
|
2164
|
+
authority.workflow !== 'Deployed smoke' ||
|
|
2165
|
+
!/^\\d+$/.test(String(authority.runId ?? '')) ||
|
|
2166
|
+
!/^\\d+$/.test(String(authority.runAttempt ?? '')) ||
|
|
2167
|
+
!/^[a-f0-9]{40}$/.test(sourceCommit) ||
|
|
2168
|
+
!/^[A-Za-z0-9_.-]+\\/[A-Za-z0-9_.-]+$/.test(repository) ||
|
|
2169
|
+
process.env.EBK_RELIABILITY_AUTHORIZED_SOURCE_COMMIT !== sourceCommit ||
|
|
2170
|
+
process.env.EBK_RELIABILITY_AUTHORIZED_SMOKE_RUN_ID !==
|
|
2171
|
+
String(authority.runId) ||
|
|
2172
|
+
process.env.EBK_RELIABILITY_AUTHORIZED_SMOKE_RUN_ATTEMPT !==
|
|
2173
|
+
String(authority.runAttempt) ||
|
|
2174
|
+
process.env.GITHUB_REPOSITORY !== repository
|
|
2175
|
+
) {
|
|
2176
|
+
throw new Error(
|
|
2177
|
+
'Protected reliability binding index is stale or lacks current workflow authority.',
|
|
2178
|
+
);
|
|
2179
|
+
}
|
|
2180
|
+
const indexTarget = reliabilityTarget(
|
|
2181
|
+
{
|
|
2182
|
+
...index.target,
|
|
2183
|
+
slice: index.slice,
|
|
2184
|
+
stage: index.stage,
|
|
2185
|
+
},
|
|
2186
|
+
'Protected reliability binding index target',
|
|
2187
|
+
);
|
|
2188
|
+
if (
|
|
2189
|
+
!Array.isArray(index.bindings) ||
|
|
2190
|
+
index.bindings.length < 1 ||
|
|
2191
|
+
index.bindings.length > 128
|
|
2192
|
+
) {
|
|
2193
|
+
throw new Error(
|
|
2194
|
+
'Protected reliability binding index has invalid workload coverage.',
|
|
2195
|
+
);
|
|
2196
|
+
}
|
|
2197
|
+
const indexedBindings = index.bindings.map((value, entryIndex) => {
|
|
2198
|
+
const entry = exactKeys(
|
|
2199
|
+
value,
|
|
2200
|
+
['digest', 'path', 'workload'],
|
|
2201
|
+
'Protected reliability binding index.bindings[' +
|
|
2202
|
+
entryIndex +
|
|
2203
|
+
']',
|
|
2204
|
+
);
|
|
2205
|
+
return {
|
|
2206
|
+
digest: digest(
|
|
2207
|
+
entry.digest,
|
|
2208
|
+
'Protected reliability binding index.bindings.digest',
|
|
2209
|
+
),
|
|
2210
|
+
path: artifactReferencePath(
|
|
2211
|
+
entry.path,
|
|
2212
|
+
'Protected reliability binding index.bindings.path',
|
|
2213
|
+
),
|
|
2214
|
+
workload: safeToken(
|
|
2215
|
+
entry.workload,
|
|
2216
|
+
'Protected reliability binding index.bindings.workload',
|
|
2217
|
+
128,
|
|
2218
|
+
),
|
|
2219
|
+
};
|
|
2220
|
+
});
|
|
2221
|
+
if (
|
|
2222
|
+
new Set(
|
|
2223
|
+
indexedBindings.map(
|
|
2224
|
+
({ digest: artifactDigest, path }) => artifactDigest + ':' + path,
|
|
2225
|
+
),
|
|
2226
|
+
).size !== indexedBindings.length ||
|
|
2227
|
+
new Set(indexedBindings.map(({ workload }) => workload)).size !==
|
|
2228
|
+
indexedBindings.length
|
|
2229
|
+
) {
|
|
2230
|
+
throw new Error(
|
|
2231
|
+
'Protected reliability binding index contains ambiguous bindings.',
|
|
2232
|
+
);
|
|
2233
|
+
}
|
|
2234
|
+
const indexed = indexedBindings.filter(
|
|
2235
|
+
({ digest: artifactDigest, path }) =>
|
|
2236
|
+
artifactDigest === requested.digest && path === requested.path,
|
|
2237
|
+
);
|
|
2238
|
+
if (indexed.length !== 1) {
|
|
2239
|
+
throw new Error(
|
|
2240
|
+
'Requested reliability binding is not selected by the protected current index.',
|
|
2241
|
+
);
|
|
2242
|
+
}
|
|
2243
|
+
const active = artifactReference(requested, label);
|
|
2244
|
+
const document = exactKeys(
|
|
2245
|
+
active.value,
|
|
2246
|
+
[
|
|
2247
|
+
'binding',
|
|
2248
|
+
'kind',
|
|
2249
|
+
'resources',
|
|
2250
|
+
'schemaVersion',
|
|
2251
|
+
'slice',
|
|
2252
|
+
'smoke',
|
|
2253
|
+
'sources',
|
|
2254
|
+
'target',
|
|
2255
|
+
],
|
|
2256
|
+
'Active reliability deployment binding',
|
|
2257
|
+
);
|
|
2258
|
+
const target = reliabilityTarget(
|
|
2259
|
+
document.target,
|
|
2260
|
+
'Active reliability deployment binding target',
|
|
2261
|
+
);
|
|
2262
|
+
const capturedBinding = binding(
|
|
2263
|
+
document.binding,
|
|
2264
|
+
'Active reliability deployment binding.binding',
|
|
2265
|
+
);
|
|
2266
|
+
const expected = createReliabilityDeploymentBinding(
|
|
2267
|
+
{
|
|
2268
|
+
schemaVersion: 1,
|
|
2269
|
+
sources: document.sources,
|
|
2270
|
+
workload: capturedBinding.workload,
|
|
2271
|
+
},
|
|
2272
|
+
target,
|
|
2273
|
+
document.smoke,
|
|
2274
|
+
);
|
|
2275
|
+
if (
|
|
2276
|
+
document.kind !== expected.kind ||
|
|
2277
|
+
document.schemaVersion !== 1 ||
|
|
2278
|
+
document.slice !== '${name}' ||
|
|
2279
|
+
capturedBinding.sourceCommit !== sourceCommit ||
|
|
2280
|
+
capturedBinding.workload !== indexed[0].workload ||
|
|
2281
|
+
target.account !== indexTarget.account ||
|
|
2282
|
+
target.region !== indexTarget.region ||
|
|
2283
|
+
target.stage !== indexTarget.stage ||
|
|
2284
|
+
JSON.stringify(document) !== JSON.stringify(expected)
|
|
2285
|
+
) {
|
|
2286
|
+
throw new Error(
|
|
2287
|
+
'Active reliability deployment binding is stale or was substituted.',
|
|
2288
|
+
);
|
|
2289
|
+
}
|
|
2290
|
+
return {
|
|
2291
|
+
binding: capturedBinding,
|
|
2292
|
+
authorityDigest: sha256(indexArtifact.bytes),
|
|
2293
|
+
digest: active.digest,
|
|
2294
|
+
document,
|
|
2295
|
+
path: active.path,
|
|
2296
|
+
resources: reliabilityResources(
|
|
2297
|
+
document.resources,
|
|
2298
|
+
target,
|
|
2299
|
+
'Active reliability deployment binding.resources',
|
|
2300
|
+
),
|
|
2301
|
+
target,
|
|
2302
|
+
};
|
|
2303
|
+
}
|
|
2304
|
+
|
|
2305
|
+
export function writeImmutableArtifact(directory, value) {
|
|
2306
|
+
const bytes = Buffer.from(JSON.stringify(value, null, 2) + '\\n');
|
|
2307
|
+
if (bytes.length > MAX_RELIABILITY_ARTIFACT_BYTES) {
|
|
2308
|
+
throw new Error('Reliability artifact exceeds its output size bound.');
|
|
2309
|
+
}
|
|
2310
|
+
const artifactDigest = sha256(bytes);
|
|
2311
|
+
const relativePath = directory + '/' + artifactDigest + '.json';
|
|
2312
|
+
const absolute = resolve(process.cwd(), relativePath);
|
|
2313
|
+
mkdirSync(resolve(process.cwd(), directory), {
|
|
2314
|
+
mode: 0o700,
|
|
2315
|
+
recursive: true,
|
|
2316
|
+
});
|
|
2317
|
+
try {
|
|
2318
|
+
writeFileSync(absolute, bytes, { flag: 'wx', mode: 0o600 });
|
|
2319
|
+
} catch (error) {
|
|
2320
|
+
if (!readFileSync(absolute).equals(bytes)) throw error;
|
|
2321
|
+
}
|
|
2322
|
+
return { digest: artifactDigest, path: relativePath };
|
|
2323
|
+
}
|
|
2324
|
+
`,
|
|
2325
|
+
'scripts/create-reliability-deployment-binding.mjs': `import {
|
|
2326
|
+
createReliabilityDeploymentBinding,
|
|
2327
|
+
exactKeys,
|
|
2328
|
+
readBoundedJson,
|
|
2329
|
+
writeImmutableArtifact,
|
|
2330
|
+
} from './reliability-evidence-lib.mjs';
|
|
2331
|
+
|
|
2332
|
+
const inputPath =
|
|
2333
|
+
process.env.EBK_RELIABILITY_BINDING_INPUT_PATH?.trim();
|
|
2334
|
+
if (!inputPath) {
|
|
2335
|
+
throw new Error('EBK_RELIABILITY_BINDING_INPUT_PATH is required.');
|
|
2336
|
+
}
|
|
2337
|
+
const required = (name, pattern) => {
|
|
2338
|
+
const value = process.env[name]?.trim();
|
|
2339
|
+
if (!value || !pattern.test(value)) {
|
|
2340
|
+
throw new Error(name + ' is required and outside its exact bound.');
|
|
2341
|
+
}
|
|
2342
|
+
return value;
|
|
2343
|
+
};
|
|
2344
|
+
const input = exactKeys(
|
|
2345
|
+
readBoundedJson(inputPath, 'Reliability deployment binding input').value,
|
|
2346
|
+
['schemaVersion', 'sources', 'workload'],
|
|
2347
|
+
'Reliability deployment binding input',
|
|
2348
|
+
);
|
|
2349
|
+
const document = createReliabilityDeploymentBinding(input, {
|
|
2350
|
+
account: required('AWS_EXPECTED_ACCOUNT_ID', /^\\d{12}$/),
|
|
2351
|
+
region: required('AWS_REGION', /^[a-z]{2}(?:-[a-z]+)+-\\d$/),
|
|
2352
|
+
slice: '${name}',
|
|
2353
|
+
stage: required('EBK_STAGE', /^(?:staging|production)$/),
|
|
2354
|
+
}, {
|
|
2355
|
+
runAttempt: required('EBK_SMOKE_RUN_ATTEMPT', /^\\d+$/),
|
|
2356
|
+
runId: required('EBK_SMOKE_RUN_ID', /^\\d+$/),
|
|
2357
|
+
});
|
|
2358
|
+
const reference = writeImmutableArtifact(
|
|
2359
|
+
'.ebk/evidence/reliability/bindings',
|
|
2360
|
+
document,
|
|
2361
|
+
);
|
|
2362
|
+
process.stdout.write(
|
|
2363
|
+
JSON.stringify({ ...reference, ...document }) + '\\n',
|
|
2364
|
+
);
|
|
2365
|
+
`,
|
|
2366
|
+
'scripts/verify-reliability.mjs': `import {
|
|
2367
|
+
CloudWatchClient,
|
|
2368
|
+
DescribeAlarmsCommand,
|
|
2369
|
+
} from '@aws-sdk/client-cloudwatch';
|
|
2370
|
+
import {
|
|
2371
|
+
GetCallerIdentityCommand,
|
|
2372
|
+
STSClient,
|
|
2373
|
+
} from '@aws-sdk/client-sts';
|
|
2374
|
+
import { readFileSync } from 'node:fs';
|
|
2375
|
+
import { awsPartition } from './reliability-evidence-lib.mjs';
|
|
2376
|
+
import {
|
|
2377
|
+
validateAlarmRegistrationMetadata,
|
|
2378
|
+
validateIncidentRecord,
|
|
2379
|
+
validateReliabilityContract,
|
|
2380
|
+
} from '@empire-builder-kit/runtime/reliability';
|
|
2381
|
+
|
|
2382
|
+
const slo = JSON.parse(
|
|
2383
|
+
readFileSync(new URL('../docs/reliability/slo.json', import.meta.url), 'utf8'),
|
|
2384
|
+
);
|
|
2385
|
+
const operations = JSON.parse(
|
|
2386
|
+
readFileSync(new URL('../operations/reliability.json', import.meta.url), 'utf8'),
|
|
2387
|
+
);
|
|
2388
|
+
const incident = JSON.parse(
|
|
2389
|
+
readFileSync(new URL('../operations/incident-template.json', import.meta.url), 'utf8'),
|
|
2390
|
+
);
|
|
2391
|
+
const contract = validateReliabilityContract(slo);
|
|
2392
|
+
validateIncidentRecord(incident);
|
|
2393
|
+
if (
|
|
2394
|
+
operations?.schemaVersion !== 1 ||
|
|
2395
|
+
operations?.slice !== '${name}' ||
|
|
2396
|
+
operations?.sloRef !== 'docs:reliability/slo.json' ||
|
|
2397
|
+
operations?.consoleIndependent !== true
|
|
2398
|
+
) {
|
|
2399
|
+
throw new Error('Reliability operations registration is incompatible.');
|
|
2400
|
+
}
|
|
2401
|
+
if (!Array.isArray(operations.alarmRegistrations)) {
|
|
2402
|
+
throw new Error('Reliability alarm registrations are absent.');
|
|
2403
|
+
}
|
|
2404
|
+
const alarmRegistrations = operations.alarmRegistrations.map(
|
|
2405
|
+
validateAlarmRegistrationMetadata,
|
|
2406
|
+
);
|
|
2407
|
+
const registered = new Set(alarmRegistrations.map((alarm) => alarm.id));
|
|
2408
|
+
if (registered.size !== alarmRegistrations.length) {
|
|
2409
|
+
throw new Error('Reliability alarm registrations contain duplicate IDs.');
|
|
2410
|
+
}
|
|
2411
|
+
for (const objective of contract.objectives) {
|
|
2412
|
+
for (const alarmId of objective.references.alarmIds) {
|
|
2413
|
+
if (!registered.has(alarmId)) {
|
|
2414
|
+
throw new Error(
|
|
2415
|
+
'SLO objective ' + objective.id + ' references an unregistered alarm.',
|
|
2416
|
+
);
|
|
2417
|
+
}
|
|
2418
|
+
}
|
|
2419
|
+
}
|
|
2420
|
+
const requiredCoverage = [
|
|
2421
|
+
'alert-delivery',
|
|
2422
|
+
'built-in-retention-job',
|
|
2423
|
+
'deployed-smoke',
|
|
2424
|
+
'function',
|
|
2425
|
+
'job',
|
|
2426
|
+
'service',
|
|
2427
|
+
];
|
|
2428
|
+
if (requiredCoverage.some(
|
|
2429
|
+
(kind) => !operations.workloadCoverage?.some((entry) => entry.kind === kind),
|
|
2430
|
+
)) {
|
|
2431
|
+
throw new Error('Reliability workload coverage is incomplete.');
|
|
2432
|
+
}
|
|
2433
|
+
if (
|
|
2434
|
+
operations.alertDelivery?.mode !== 'diagnostic-only' ||
|
|
2435
|
+
operations.alertDelivery?.confirmation !==
|
|
2436
|
+
'unsupported-until-independent-route-is-configured' ||
|
|
2437
|
+
operations.runtimeSmoke?.aggregateOwner !== 'testing-evidence' ||
|
|
2438
|
+
operations.runtimeSmoke?.deployedResourceClass !==
|
|
2439
|
+
'testing-evidence-not-cloudwatch-alarm' ||
|
|
2440
|
+
operations.runtimeSmoke?.reporter !==
|
|
2441
|
+
'testing-evidence-native-not-reliability-exercise-report' ||
|
|
2442
|
+
operations.runtimeSmoke?.target !== 'smoke-deployed'
|
|
2443
|
+
) {
|
|
2444
|
+
throw new Error(
|
|
2445
|
+
'Alert-delivery or deployed-smoke ownership is overstated.',
|
|
2446
|
+
);
|
|
2447
|
+
}
|
|
2448
|
+
|
|
2449
|
+
let deployed = false;
|
|
2450
|
+
if (process.argv.includes('--deployed')) {
|
|
2451
|
+
const required = (key) => {
|
|
2452
|
+
const value = process.env[key]?.trim();
|
|
2453
|
+
if (!value || !/^[A-Za-z0-9._:/-]{1,128}$/.test(value)) {
|
|
2454
|
+
throw new Error(key + ' is required for deployed verification.');
|
|
2455
|
+
}
|
|
2456
|
+
return value;
|
|
2457
|
+
};
|
|
2458
|
+
const account = required('AWS_EXPECTED_ACCOUNT_ID');
|
|
2459
|
+
const region = required('AWS_REGION');
|
|
2460
|
+
const stage = required('EBK_STAGE');
|
|
2461
|
+
const partition = awsPartition(region);
|
|
2462
|
+
const identity = await new STSClient({ region }).send(
|
|
2463
|
+
new GetCallerIdentityCommand({}),
|
|
2464
|
+
);
|
|
2465
|
+
if (
|
|
2466
|
+
identity.Account !== account ||
|
|
2467
|
+
!identity.Arn?.startsWith('arn:' + partition + ':sts::' + account + ':')
|
|
2468
|
+
) {
|
|
2469
|
+
throw new Error(
|
|
2470
|
+
'AWS caller identity does not match deployed reliability verification.',
|
|
2471
|
+
);
|
|
2472
|
+
}
|
|
2473
|
+
const expected = operations.deployedAlarmResources?.map((resource) => ({
|
|
2474
|
+
...resource,
|
|
2475
|
+
name: resource.name.replace('{stage}', stage),
|
|
2476
|
+
}));
|
|
2477
|
+
if (
|
|
2478
|
+
!Array.isArray(expected) ||
|
|
2479
|
+
expected.length < 1 ||
|
|
2480
|
+
expected.length > 100 ||
|
|
2481
|
+
new Set(expected.map(({ name }) => name)).size !== expected.length ||
|
|
2482
|
+
new Set(expected.map(({ id }) => id)).size !== expected.length ||
|
|
2483
|
+
expected.some(({ id }) => !registered.has(id))
|
|
2484
|
+
) {
|
|
2485
|
+
throw new Error('Deployed alarm resource inventory is invalid.');
|
|
2486
|
+
}
|
|
2487
|
+
const response = await new CloudWatchClient({ region }).send(
|
|
2488
|
+
new DescribeAlarmsCommand({
|
|
2489
|
+
AlarmNames: expected.map(({ name }) => name),
|
|
2490
|
+
MaxRecords: expected.length,
|
|
2491
|
+
}),
|
|
2492
|
+
);
|
|
2493
|
+
if (response.NextToken) {
|
|
2494
|
+
throw new Error('Deployed alarm verification exceeded its one-page bound.');
|
|
2495
|
+
}
|
|
2496
|
+
const alarms = response.MetricAlarms ?? [];
|
|
2497
|
+
if (
|
|
2498
|
+
alarms.length !== expected.length ||
|
|
2499
|
+
new Set(alarms.map(({ AlarmName }) => AlarmName)).size !== alarms.length
|
|
2500
|
+
) {
|
|
2501
|
+
throw new Error('Deployed alarm resource identity is absent or ambiguous.');
|
|
2502
|
+
}
|
|
2503
|
+
const primaryRouteArns = new Set();
|
|
2504
|
+
for (const resource of expected) {
|
|
2505
|
+
const alarm = alarms.find(({ AlarmName }) => AlarmName === resource.name);
|
|
2506
|
+
const description = JSON.parse(alarm?.AlarmDescription ?? 'null');
|
|
2507
|
+
if (
|
|
2508
|
+
!alarm ||
|
|
2509
|
+
alarm.AlarmArn !==
|
|
2510
|
+
'arn:' +
|
|
2511
|
+
partition +
|
|
2512
|
+
':cloudwatch:' +
|
|
2513
|
+
region +
|
|
2514
|
+
':' +
|
|
2515
|
+
account +
|
|
2516
|
+
':alarm:' +
|
|
2517
|
+
resource.name ||
|
|
2518
|
+
description?.id !== resource.id ||
|
|
2519
|
+
description?.slice !== '${name}' ||
|
|
2520
|
+
description?.schemaVersion !== 1 ||
|
|
2521
|
+
description?.owner !== resource.owner ||
|
|
2522
|
+
description?.route !== resource.route ||
|
|
2523
|
+
description?.runbook !== resource.runbook
|
|
2524
|
+
) {
|
|
2525
|
+
throw new Error(
|
|
2526
|
+
'Deployed alarm does not match its exact AWS resource identity.',
|
|
2527
|
+
);
|
|
2528
|
+
}
|
|
2529
|
+
if (
|
|
2530
|
+
(resource.actions === 'none' && (alarm.AlarmActions?.length ?? 0) !== 0) ||
|
|
2531
|
+
(resource.actions === 'primary-route' &&
|
|
2532
|
+
(alarm.AlarmActions?.length ?? 0) !== 1)
|
|
2533
|
+
) {
|
|
2534
|
+
throw new Error('Deployed alarm action route does not match policy.');
|
|
2535
|
+
}
|
|
2536
|
+
if (resource.actions === 'primary-route') {
|
|
2537
|
+
const action = alarm.AlarmActions[0];
|
|
2538
|
+
if (
|
|
2539
|
+
typeof action !== 'string' ||
|
|
2540
|
+
!action.startsWith(
|
|
2541
|
+
'arn:' + partition + ':sns:' + region + ':' + account + ':',
|
|
2542
|
+
)
|
|
2543
|
+
) {
|
|
2544
|
+
throw new Error('Deployed alarm action escaped the exact AWS target.');
|
|
2545
|
+
}
|
|
2546
|
+
primaryRouteArns.add(action);
|
|
2547
|
+
}
|
|
2548
|
+
const metricDefinitions = [
|
|
2549
|
+
...(alarm.MetricName
|
|
2550
|
+
? [{
|
|
2551
|
+
dimensions: alarm.Dimensions ?? [],
|
|
2552
|
+
metricName: alarm.MetricName,
|
|
2553
|
+
namespace: alarm.Namespace,
|
|
2554
|
+
period: alarm.Period,
|
|
2555
|
+
statistic: alarm.Statistic,
|
|
2556
|
+
}]
|
|
2557
|
+
: []),
|
|
2558
|
+
...(alarm.Metrics ?? [])
|
|
2559
|
+
.filter(({ MetricStat }) => MetricStat?.Metric)
|
|
2560
|
+
.map(({ MetricStat }) => ({
|
|
2561
|
+
dimensions: MetricStat.Metric.Dimensions ?? [],
|
|
2562
|
+
metricName: MetricStat.Metric.MetricName,
|
|
2563
|
+
namespace: MetricStat.Metric.Namespace,
|
|
2564
|
+
period: MetricStat.Period,
|
|
2565
|
+
statistic: MetricStat.Stat,
|
|
2566
|
+
})),
|
|
2567
|
+
];
|
|
2568
|
+
const actualDimensionNames = [
|
|
2569
|
+
...new Set(
|
|
2570
|
+
metricDefinitions.flatMap(({ dimensions }) =>
|
|
2571
|
+
dimensions.map(({ Name }) => Name),
|
|
2572
|
+
),
|
|
2573
|
+
),
|
|
2574
|
+
].sort();
|
|
2575
|
+
const dimensionValues = new Map();
|
|
2576
|
+
let incompatibleDimensionValue = false;
|
|
2577
|
+
for (const { dimensions } of metricDefinitions) {
|
|
2578
|
+
for (const { Name, Value } of dimensions) {
|
|
2579
|
+
if (
|
|
2580
|
+
typeof Name !== 'string' ||
|
|
2581
|
+
typeof Value !== 'string' ||
|
|
2582
|
+
Value.length < 1 ||
|
|
2583
|
+
Value.length > 1024 ||
|
|
2584
|
+
(dimensionValues.has(Name) &&
|
|
2585
|
+
dimensionValues.get(Name) !== Value)
|
|
2586
|
+
) {
|
|
2587
|
+
incompatibleDimensionValue = true;
|
|
2588
|
+
}
|
|
2589
|
+
dimensionValues.set(Name, Value);
|
|
2590
|
+
}
|
|
2591
|
+
}
|
|
2592
|
+
const expectedDimensionValues = Object.fromEntries(
|
|
2593
|
+
Object.entries(resource.dimensionValues ?? {}).map(([key, value]) => [
|
|
2594
|
+
key,
|
|
2595
|
+
value.replace('{stage}', stage),
|
|
2596
|
+
]),
|
|
2597
|
+
);
|
|
2598
|
+
const actualMetricNames = [
|
|
2599
|
+
...new Set(metricDefinitions.map(({ metricName }) => metricName)),
|
|
2600
|
+
].sort();
|
|
2601
|
+
const actualNamespaces = [
|
|
2602
|
+
...new Set(metricDefinitions.map(({ namespace }) => namespace)),
|
|
2603
|
+
];
|
|
2604
|
+
const expectedMetricNames = (
|
|
2605
|
+
resource.metricNames ?? [resource.metricName]
|
|
2606
|
+
).slice().sort();
|
|
2607
|
+
if (
|
|
2608
|
+
JSON.stringify(actualDimensionNames) !==
|
|
2609
|
+
JSON.stringify(resource.dimensionNames.slice().sort()) ||
|
|
2610
|
+
incompatibleDimensionValue ||
|
|
2611
|
+
Object.entries(expectedDimensionValues).some(
|
|
2612
|
+
([key, value]) => dimensionValues.get(key) !== value,
|
|
2613
|
+
) ||
|
|
2614
|
+
JSON.stringify(actualMetricNames) !==
|
|
2615
|
+
JSON.stringify(expectedMetricNames) ||
|
|
2616
|
+
actualNamespaces.length !== 1 ||
|
|
2617
|
+
actualNamespaces[0] !== resource.namespace ||
|
|
2618
|
+
alarm.ComparisonOperator !== resource.comparisonOperator ||
|
|
2619
|
+
alarm.EvaluationPeriods !== resource.evaluationPeriods ||
|
|
2620
|
+
alarm.Threshold !== resource.threshold ||
|
|
2621
|
+
alarm.TreatMissingData !== resource.treatMissingData ||
|
|
2622
|
+
(resource.period !== undefined &&
|
|
2623
|
+
metricDefinitions.some(({ period }) => period !== resource.period)) ||
|
|
2624
|
+
(resource.statistic !== undefined &&
|
|
2625
|
+
metricDefinitions.some(
|
|
2626
|
+
({ statistic }) => statistic !== resource.statistic,
|
|
2627
|
+
))
|
|
2628
|
+
) {
|
|
2629
|
+
throw new Error(
|
|
2630
|
+
'Deployed alarm dimensions or AWS signal do not match inventory.',
|
|
2631
|
+
);
|
|
2632
|
+
}
|
|
2633
|
+
if (
|
|
2634
|
+
resource.signal === 'metric-math' &&
|
|
2635
|
+
(
|
|
2636
|
+
!Array.isArray(alarm.Metrics) ||
|
|
2637
|
+
alarm.Metrics.filter(({ ReturnData }) => ReturnData).length !== 1 ||
|
|
2638
|
+
(resource.objectiveId !== undefined &&
|
|
2639
|
+
!alarm.Metrics.some(({ Expression }) => {
|
|
2640
|
+
const objective = contract.objectives.find(
|
|
2641
|
+
({ id }) => id === resource.objectiveId,
|
|
2642
|
+
);
|
|
2643
|
+
const allowedBadFraction = objective
|
|
2644
|
+
? Number((1 - objective.target).toFixed(6)).toString()
|
|
2645
|
+
: '';
|
|
2646
|
+
return (
|
|
2647
|
+
objective &&
|
|
2648
|
+
Expression?.includes('IF(v>=') &&
|
|
2649
|
+
Expression.includes('/' + allowedBadFraction)
|
|
2650
|
+
);
|
|
2651
|
+
}))
|
|
2652
|
+
)
|
|
2653
|
+
) {
|
|
2654
|
+
throw new Error('Deployed metric-math alarm is not executable.');
|
|
2655
|
+
}
|
|
2656
|
+
}
|
|
2657
|
+
if (primaryRouteArns.size !== 1) {
|
|
2658
|
+
throw new Error('Deployed alarms do not share one exact primary route.');
|
|
2659
|
+
}
|
|
2660
|
+
deployed = true;
|
|
2661
|
+
}
|
|
2662
|
+
process.stdout.write(JSON.stringify({
|
|
2663
|
+
deployedAlarmResourcesVerified: deployed,
|
|
2664
|
+
kind: 'reliability-contract-verification',
|
|
2665
|
+
schemaVersion: 1,
|
|
2666
|
+
slice: '${name}',
|
|
2667
|
+
}) + '\\n');
|
|
2668
|
+
`,
|
|
2669
|
+
'scripts/reliability-report.mjs': `import { readFileSync } from 'node:fs';
|
|
2670
|
+
import {
|
|
2671
|
+
evaluateSlo,
|
|
2672
|
+
validateReliabilityContract,
|
|
2673
|
+
} from '@empire-builder-kit/runtime/reliability';
|
|
2674
|
+
import {
|
|
2675
|
+
activeDeploymentBinding,
|
|
2676
|
+
artifactReference,
|
|
2677
|
+
binding,
|
|
2678
|
+
digest,
|
|
2679
|
+
exactKeys,
|
|
2680
|
+
readBoundedJson,
|
|
2681
|
+
safeToken,
|
|
2682
|
+
sameBinding,
|
|
2683
|
+
utc,
|
|
2684
|
+
writeImmutableArtifact,
|
|
2685
|
+
} from './reliability-evidence-lib.mjs';
|
|
2686
|
+
|
|
2687
|
+
const inputPath = process.env.EBK_RELIABILITY_EVIDENCE_INPUT_PATH?.trim();
|
|
2688
|
+
if (!inputPath) {
|
|
2689
|
+
throw new Error('EBK_RELIABILITY_EVIDENCE_INPUT_PATH is required.');
|
|
2690
|
+
}
|
|
2691
|
+
const inputArtifact = readBoundedJson(inputPath, 'Reliability report input');
|
|
2692
|
+
const input = exactKeys(
|
|
2693
|
+
inputArtifact.value,
|
|
2694
|
+
['capture', 'objectiveId', 'schemaVersion'],
|
|
2695
|
+
'Reliability report input',
|
|
2696
|
+
);
|
|
2697
|
+
if (input.schemaVersion !== 1) {
|
|
2698
|
+
throw new Error('Reliability report input is incompatible.');
|
|
2699
|
+
}
|
|
2700
|
+
const objectiveId = safeToken(input.objectiveId, 'objectiveId', 128);
|
|
2701
|
+
const captured = artifactReference(input.capture, 'capture');
|
|
2702
|
+
const capture = exactKeys(
|
|
2703
|
+
captured.value,
|
|
2704
|
+
[
|
|
2705
|
+
'binding',
|
|
2706
|
+
'bindingArtifact',
|
|
2707
|
+
'capturedAt',
|
|
2708
|
+
'kind',
|
|
2709
|
+
'metricResults',
|
|
2710
|
+
'objectiveId',
|
|
2711
|
+
'queryDigest',
|
|
2712
|
+
'sampleWindow',
|
|
2713
|
+
'schemaVersion',
|
|
2714
|
+
'slice',
|
|
2715
|
+
'target',
|
|
2716
|
+
],
|
|
2717
|
+
'Captured metric artifact',
|
|
2718
|
+
);
|
|
2719
|
+
if (
|
|
2720
|
+
capture.kind !== 'reliability-metric-capture' ||
|
|
2721
|
+
capture.schemaVersion !== 1 ||
|
|
2722
|
+
capture.slice !== '${name}' ||
|
|
2723
|
+
capture.objectiveId !== objectiveId ||
|
|
2724
|
+
!Array.isArray(capture.metricResults)
|
|
2725
|
+
) {
|
|
2726
|
+
throw new Error(
|
|
2727
|
+
'Reliability report requires the exact captured AWS metric artifact.',
|
|
2728
|
+
);
|
|
2729
|
+
}
|
|
2730
|
+
const capturedBinding = binding(capture.binding, 'capture.binding');
|
|
2731
|
+
const activeBinding = activeDeploymentBinding(
|
|
2732
|
+
capture.bindingArtifact,
|
|
2733
|
+
'capture.bindingArtifact',
|
|
2734
|
+
);
|
|
2735
|
+
const captureTarget = exactKeys(
|
|
2736
|
+
capture.target,
|
|
2737
|
+
['account', 'region', 'slice', 'stage'],
|
|
2738
|
+
'capture.target',
|
|
2739
|
+
);
|
|
2740
|
+
const sampleWindow = exactKeys(
|
|
2741
|
+
capture.sampleWindow,
|
|
2742
|
+
['durationSeconds', 'end', 'start'],
|
|
2743
|
+
'capture.sampleWindow',
|
|
2744
|
+
);
|
|
2745
|
+
const sampleStart = utc(sampleWindow.start, 'capture.sampleWindow.start');
|
|
2746
|
+
const sampleEnd = utc(sampleWindow.end, 'capture.sampleWindow.end');
|
|
2747
|
+
const sampleDurationSeconds =
|
|
2748
|
+
(Date.parse(sampleEnd) - Date.parse(sampleStart)) / 1_000;
|
|
2749
|
+
digest(capture.queryDigest, 'capture.queryDigest');
|
|
2750
|
+
if (
|
|
2751
|
+
!sameBinding(capturedBinding, activeBinding.binding) ||
|
|
2752
|
+
['account', 'region', 'slice', 'stage'].some(
|
|
2753
|
+
(key) => captureTarget[key] !== activeBinding.target[key],
|
|
2754
|
+
) ||
|
|
2755
|
+
!Number.isSafeInteger(sampleWindow.durationSeconds) ||
|
|
2756
|
+
sampleWindow.durationSeconds !== sampleDurationSeconds ||
|
|
2757
|
+
sampleDurationSeconds <= 0 ||
|
|
2758
|
+
sampleDurationSeconds > 6 * 60 * 60
|
|
2759
|
+
) {
|
|
2760
|
+
throw new Error(
|
|
2761
|
+
'Captured metric sample is not bound to the exact active deployment and bounded window.',
|
|
2762
|
+
);
|
|
2763
|
+
}
|
|
2764
|
+
const result = (id, optional = false) => {
|
|
2765
|
+
const matches = capture.metricResults.filter((entry) => entry?.id === id);
|
|
2766
|
+
if (matches.length === 0 && optional) return 0;
|
|
2767
|
+
if (
|
|
2768
|
+
matches.length !== 1 ||
|
|
2769
|
+
!Array.isArray(matches[0].timestamps) ||
|
|
2770
|
+
!Array.isArray(matches[0].values) ||
|
|
2771
|
+
matches[0].timestamps.length !== matches[0].values.length ||
|
|
2772
|
+
matches[0].values.length > 1008 ||
|
|
2773
|
+
matches[0].timestamps.some(
|
|
2774
|
+
(value) =>
|
|
2775
|
+
typeof value !== 'string' ||
|
|
2776
|
+
!Number.isFinite(Date.parse(value)) ||
|
|
2777
|
+
Date.parse(value) < Date.parse(sampleStart) ||
|
|
2778
|
+
Date.parse(value) > Date.parse(sampleEnd),
|
|
2779
|
+
) ||
|
|
2780
|
+
matches[0].values.some(
|
|
2781
|
+
(value) => typeof value !== 'number' || !Number.isFinite(value) || value < 0,
|
|
2782
|
+
)
|
|
2783
|
+
) {
|
|
2784
|
+
throw new Error('Captured metric series ' + id + ' is absent or ambiguous.');
|
|
2785
|
+
}
|
|
2786
|
+
return matches[0].values.reduce((total, value) => total + value, 0);
|
|
2787
|
+
};
|
|
2788
|
+
const contract = validateReliabilityContract(
|
|
2789
|
+
JSON.parse(
|
|
2790
|
+
readFileSync(
|
|
2791
|
+
new URL('../docs/reliability/slo.json', import.meta.url),
|
|
2792
|
+
'utf8',
|
|
2793
|
+
),
|
|
2794
|
+
),
|
|
2795
|
+
);
|
|
2796
|
+
const objective = contract.objectives.find(
|
|
2797
|
+
({ id }) => id === objectiveId,
|
|
2798
|
+
);
|
|
2799
|
+
if (!objective) throw new Error('Reliability objective is not declared.');
|
|
2800
|
+
if (capturedBinding.workload !== objective.journey) {
|
|
2801
|
+
throw new Error(
|
|
2802
|
+
'Captured metric workload does not match the declared objective journey.',
|
|
2803
|
+
);
|
|
2804
|
+
}
|
|
2805
|
+
const validEvents = result('valid');
|
|
2806
|
+
const goodEvents = result('good');
|
|
2807
|
+
const syntheticValid = result(
|
|
2808
|
+
'synthetic-valid',
|
|
2809
|
+
objective.lowTraffic.mode === 'data-insufficient',
|
|
2810
|
+
);
|
|
2811
|
+
const syntheticGood = result(
|
|
2812
|
+
'synthetic-good',
|
|
2813
|
+
objective.lowTraffic.mode === 'data-insufficient',
|
|
2814
|
+
);
|
|
2815
|
+
if (
|
|
2816
|
+
!Number.isSafeInteger(validEvents) ||
|
|
2817
|
+
!Number.isSafeInteger(goodEvents) ||
|
|
2818
|
+
!Number.isSafeInteger(syntheticValid) ||
|
|
2819
|
+
!Number.isSafeInteger(syntheticGood) ||
|
|
2820
|
+
goodEvents > validEvents ||
|
|
2821
|
+
syntheticGood > syntheticValid
|
|
2822
|
+
) {
|
|
2823
|
+
throw new Error('Captured metric counts are incompatible.');
|
|
2824
|
+
}
|
|
2825
|
+
const evaluation = evaluateSlo({
|
|
2826
|
+
goodEvents,
|
|
2827
|
+
minimumValidEvents: objective.minimumValidEvents.diagnosticSample,
|
|
2828
|
+
objective,
|
|
2829
|
+
...(validEvents >= objective.minimumValidEvents.diagnosticSample
|
|
2830
|
+
? {}
|
|
2831
|
+
: {
|
|
2832
|
+
syntheticSuccess:
|
|
2833
|
+
syntheticValid > 0 ? syntheticGood === syntheticValid : undefined,
|
|
2834
|
+
}),
|
|
2835
|
+
validEvents,
|
|
2836
|
+
});
|
|
2837
|
+
const report = {
|
|
2838
|
+
binding: capturedBinding,
|
|
2839
|
+
capturedMetricDigest: captured.digest,
|
|
2840
|
+
declaredObjectiveWindow: objective.window,
|
|
2841
|
+
kind: 'reliability-slo-diagnostic-sample',
|
|
2842
|
+
objectiveId: objective.id,
|
|
2843
|
+
provenance: activeBinding.target.stage,
|
|
2844
|
+
sampleEvaluation: evaluation,
|
|
2845
|
+
sampleWindow: {
|
|
2846
|
+
durationSeconds: sampleDurationSeconds,
|
|
2847
|
+
end: sampleEnd,
|
|
2848
|
+
start: sampleStart,
|
|
2849
|
+
},
|
|
2850
|
+
schemaVersion: 1,
|
|
2851
|
+
scope: 'bounded-diagnostic-sample-not-slo-window-evidence',
|
|
2852
|
+
slice: '${name}',
|
|
2853
|
+
};
|
|
2854
|
+
const reference = writeImmutableArtifact(
|
|
2855
|
+
'.ebk/evidence/reliability/slo',
|
|
2856
|
+
report,
|
|
2857
|
+
);
|
|
2858
|
+
process.stdout.write(JSON.stringify({ ...reference, ...report }) + '\\n');
|
|
2859
|
+
`,
|
|
2860
|
+
'scripts/collect-reliability-exercise.mjs': `import {
|
|
2861
|
+
CloudWatchClient,
|
|
2862
|
+
DescribeAlarmHistoryCommand,
|
|
2863
|
+
DescribeAlarmsCommand,
|
|
2864
|
+
} from '@aws-sdk/client-cloudwatch';
|
|
2865
|
+
import {
|
|
2866
|
+
DescribeExecutionCommand,
|
|
2867
|
+
SFNClient,
|
|
2868
|
+
} from '@aws-sdk/client-sfn';
|
|
2869
|
+
import {
|
|
2870
|
+
GetCallerIdentityCommand,
|
|
2871
|
+
STSClient,
|
|
2872
|
+
} from '@aws-sdk/client-sts';
|
|
2873
|
+
import { readFileSync } from 'node:fs';
|
|
2874
|
+
import {
|
|
2875
|
+
activeDeploymentBinding,
|
|
2876
|
+
awsPartition,
|
|
2877
|
+
authorizedExecutionArn,
|
|
2878
|
+
exactKeys,
|
|
2879
|
+
readBoundedJson,
|
|
2880
|
+
safeToken,
|
|
2881
|
+
utc,
|
|
2882
|
+
writeImmutableArtifact,
|
|
2883
|
+
} from './reliability-evidence-lib.mjs';
|
|
2884
|
+
|
|
2885
|
+
const inputPath =
|
|
2886
|
+
process.env.EBK_RELIABILITY_EXERCISE_COLLECTION_INPUT_PATH?.trim();
|
|
2887
|
+
if (!inputPath) {
|
|
2888
|
+
throw new Error(
|
|
2889
|
+
'EBK_RELIABILITY_EXERCISE_COLLECTION_INPUT_PATH is required.',
|
|
2890
|
+
);
|
|
2891
|
+
}
|
|
2892
|
+
const input = exactKeys(
|
|
2893
|
+
readBoundedJson(
|
|
2894
|
+
inputPath,
|
|
2895
|
+
'Reliability exercise collection input',
|
|
2896
|
+
).value,
|
|
2897
|
+
[
|
|
2898
|
+
'alarmResourceId',
|
|
2899
|
+
'bindingArtifact',
|
|
2900
|
+
'exerciseId',
|
|
2901
|
+
'schemaVersion',
|
|
2902
|
+
'target',
|
|
2903
|
+
'windowEnd',
|
|
2904
|
+
'windowStart',
|
|
2905
|
+
'workflowExecutions',
|
|
2906
|
+
],
|
|
2907
|
+
'Reliability exercise collection input',
|
|
2908
|
+
);
|
|
2909
|
+
if (input.schemaVersion !== 1) {
|
|
2910
|
+
throw new Error('Reliability exercise collection input is incompatible.');
|
|
2911
|
+
}
|
|
2912
|
+
const exerciseId = safeToken(input.exerciseId, 'exerciseId', 128);
|
|
2913
|
+
const declaration = JSON.parse(
|
|
2914
|
+
readFileSync(
|
|
2915
|
+
new URL('../testing/reliability-exercises.json', import.meta.url),
|
|
2916
|
+
'utf8',
|
|
2917
|
+
),
|
|
2918
|
+
);
|
|
2919
|
+
const operations = JSON.parse(
|
|
2920
|
+
readFileSync(
|
|
2921
|
+
new URL('../operations/reliability.json', import.meta.url),
|
|
2922
|
+
'utf8',
|
|
2923
|
+
),
|
|
2924
|
+
);
|
|
2925
|
+
const exercise = declaration.exercises?.find(({ id }) => id === exerciseId);
|
|
2926
|
+
if (
|
|
2927
|
+
!exercise ||
|
|
2928
|
+
exercise.observation?.adapter !== 'aws-cloudwatch-alarm-history-v1'
|
|
2929
|
+
) {
|
|
2930
|
+
throw new Error(
|
|
2931
|
+
'Reliability exercise has no stable-v1 AWS collection adapter.',
|
|
2932
|
+
);
|
|
2933
|
+
}
|
|
2934
|
+
const alarmResourceId = safeToken(
|
|
2935
|
+
input.alarmResourceId,
|
|
2936
|
+
'alarmResourceId',
|
|
2937
|
+
128,
|
|
2938
|
+
);
|
|
2939
|
+
const alarmResource = operations.deployedAlarmResources?.find(
|
|
2940
|
+
({ id }) => id === alarmResourceId,
|
|
2941
|
+
);
|
|
2942
|
+
const dynamicMatch =
|
|
2943
|
+
(exercise.observation.dynamicAlarmKind === 'service' &&
|
|
2944
|
+
/^service-[a-z][a-z0-9-]*-health$/.test(alarmResourceId)) ||
|
|
2945
|
+
(exercise.observation.dynamicAlarmKind === 'job' &&
|
|
2946
|
+
/^job-[a-z][a-z0-9-]*-failure$/.test(alarmResourceId));
|
|
2947
|
+
if (
|
|
2948
|
+
!alarmResource ||
|
|
2949
|
+
(!exercise.observation.alarmIds.includes(alarmResourceId) && !dynamicMatch)
|
|
2950
|
+
) {
|
|
2951
|
+
throw new Error(
|
|
2952
|
+
'Reliability exercise alarm is absent or outside its declared workload.',
|
|
2953
|
+
);
|
|
2954
|
+
}
|
|
2955
|
+
const target = exactKeys(
|
|
2956
|
+
input.target,
|
|
2957
|
+
['account', 'region', 'slice', 'stage'],
|
|
2958
|
+
'target',
|
|
2959
|
+
);
|
|
2960
|
+
for (const key of ['account', 'region', 'slice', 'stage']) {
|
|
2961
|
+
safeToken(target[key], 'target.' + key, 128);
|
|
2962
|
+
}
|
|
2963
|
+
if (target.slice !== '${name}') {
|
|
2964
|
+
throw new Error('Reliability collection target escaped the exact Slice.');
|
|
2965
|
+
}
|
|
2966
|
+
const activeBinding = activeDeploymentBinding(
|
|
2967
|
+
input.bindingArtifact,
|
|
2968
|
+
'bindingArtifact',
|
|
2969
|
+
);
|
|
2970
|
+
const collectedBinding = activeBinding.binding;
|
|
2971
|
+
const activeResources = activeBinding.resources;
|
|
2972
|
+
const bindingTarget = activeBinding.target;
|
|
2973
|
+
if (
|
|
2974
|
+
collectedBinding.workload !== exercise.workload ||
|
|
2975
|
+
['account', 'region', 'slice', 'stage'].some(
|
|
2976
|
+
(key) => bindingTarget[key] !== target[key],
|
|
2977
|
+
)
|
|
2978
|
+
) {
|
|
2979
|
+
throw new Error(
|
|
2980
|
+
'Reliability collection deployment binding does not match the exercise target.',
|
|
2981
|
+
);
|
|
2982
|
+
}
|
|
2983
|
+
const identity = await new STSClient({ region: target.region }).send(
|
|
2984
|
+
new GetCallerIdentityCommand({}),
|
|
2985
|
+
);
|
|
2986
|
+
const partition = awsPartition(target.region);
|
|
2987
|
+
if (
|
|
2988
|
+
identity.Account !== target.account ||
|
|
2989
|
+
!identity.Arn?.startsWith(
|
|
2990
|
+
'arn:' + partition + ':sts::' + target.account + ':',
|
|
2991
|
+
)
|
|
2992
|
+
) {
|
|
2993
|
+
throw new Error(
|
|
2994
|
+
'AWS caller identity does not match the active reliability target.',
|
|
2995
|
+
);
|
|
2996
|
+
}
|
|
2997
|
+
const windowStart = utc(input.windowStart, 'windowStart');
|
|
2998
|
+
const windowEnd = utc(input.windowEnd, 'windowEnd');
|
|
2999
|
+
const windowMilliseconds = Date.parse(windowEnd) - Date.parse(windowStart);
|
|
3000
|
+
if (
|
|
3001
|
+
windowMilliseconds <= 0 ||
|
|
3002
|
+
windowMilliseconds > 6 * 60 * 60 * 1000
|
|
3003
|
+
) {
|
|
3004
|
+
throw new Error(
|
|
3005
|
+
'Reliability collection window must be positive and at most six hours.',
|
|
3006
|
+
);
|
|
3007
|
+
}
|
|
3008
|
+
const alarmName = alarmResource.name.replace('{stage}', target.stage);
|
|
3009
|
+
if (
|
|
3010
|
+
!alarmName.startsWith('${name}-' + target.stage + '-') ||
|
|
3011
|
+
alarmName.includes('{stage}') ||
|
|
3012
|
+
!activeResources.alarmNames.includes(alarmName)
|
|
3013
|
+
) {
|
|
3014
|
+
throw new Error(
|
|
3015
|
+
'Reliability alarm is not authorized by the protected deployment.',
|
|
3016
|
+
);
|
|
3017
|
+
}
|
|
3018
|
+
const historyRequest = {
|
|
3019
|
+
AlarmName: alarmName,
|
|
3020
|
+
EndDate: new Date(windowEnd),
|
|
3021
|
+
HistoryItemType: 'StateUpdate',
|
|
3022
|
+
MaxRecords: 32,
|
|
3023
|
+
ScanBy: 'TimestampAscending',
|
|
3024
|
+
StartDate: new Date(windowStart),
|
|
3025
|
+
};
|
|
3026
|
+
const stateRequest = {
|
|
3027
|
+
AlarmNames: [alarmName],
|
|
3028
|
+
MaxRecords: 1,
|
|
3029
|
+
};
|
|
3030
|
+
const cloudwatch = new CloudWatchClient({ region: target.region });
|
|
3031
|
+
const historyResponse = await cloudwatch.send(
|
|
3032
|
+
new DescribeAlarmHistoryCommand(historyRequest),
|
|
3033
|
+
);
|
|
3034
|
+
const stateResponse = await cloudwatch.send(
|
|
3035
|
+
new DescribeAlarmsCommand(stateRequest),
|
|
3036
|
+
);
|
|
3037
|
+
if (historyResponse.NextToken || stateResponse.NextToken) {
|
|
3038
|
+
throw new Error('Reliability alarm collection exceeded its page bound.');
|
|
3039
|
+
}
|
|
3040
|
+
const boundedText = (value, label, maximum = 2048) => {
|
|
3041
|
+
if (
|
|
3042
|
+
typeof value !== 'string' ||
|
|
3043
|
+
value.length < 1 ||
|
|
3044
|
+
value.length > maximum ||
|
|
3045
|
+
[...value].some((character) => {
|
|
3046
|
+
const code = character.charCodeAt(0);
|
|
3047
|
+
return code < 9 || (code > 13 && code < 32) || code === 127;
|
|
3048
|
+
})
|
|
3049
|
+
) {
|
|
3050
|
+
throw new Error(label + ' is not bounded provider metadata.');
|
|
3051
|
+
}
|
|
3052
|
+
return value;
|
|
3053
|
+
};
|
|
3054
|
+
const requestId = safeToken(
|
|
3055
|
+
historyResponse.$metadata?.requestId,
|
|
3056
|
+
'alarmHistory.requestId',
|
|
3057
|
+
128,
|
|
3058
|
+
);
|
|
3059
|
+
const stateRequestId = safeToken(
|
|
3060
|
+
stateResponse.$metadata?.requestId,
|
|
3061
|
+
'alarmState.requestId',
|
|
3062
|
+
128,
|
|
3063
|
+
);
|
|
3064
|
+
const historyItems = (historyResponse.AlarmHistoryItems ?? []).map(
|
|
3065
|
+
(item, index) => {
|
|
3066
|
+
const historyData = boundedText(
|
|
3067
|
+
item.HistoryData,
|
|
3068
|
+
'alarmHistory[' + index + '].historyData',
|
|
3069
|
+
4096,
|
|
3070
|
+
);
|
|
3071
|
+
let parsed;
|
|
3072
|
+
try {
|
|
3073
|
+
parsed = JSON.parse(historyData);
|
|
3074
|
+
} catch {
|
|
3075
|
+
throw new Error('Alarm history state data is not valid JSON.');
|
|
3076
|
+
}
|
|
3077
|
+
const state = parsed?.newState?.stateValue;
|
|
3078
|
+
if (!['ALARM', 'INSUFFICIENT_DATA', 'OK'].includes(state)) {
|
|
3079
|
+
throw new Error('Alarm history contains an unsupported state.');
|
|
3080
|
+
}
|
|
3081
|
+
const timestamp = utc(
|
|
3082
|
+
item.Timestamp?.toISOString?.(),
|
|
3083
|
+
'alarmHistory[' + index + '].timestamp',
|
|
3084
|
+
);
|
|
3085
|
+
if (
|
|
3086
|
+
Date.parse(timestamp) < Date.parse(windowStart) ||
|
|
3087
|
+
Date.parse(timestamp) > Date.parse(windowEnd) ||
|
|
3088
|
+
item.AlarmName !== alarmName ||
|
|
3089
|
+
item.HistoryItemType !== 'StateUpdate'
|
|
3090
|
+
) {
|
|
3091
|
+
throw new Error('Alarm history escaped the exact collection request.');
|
|
3092
|
+
}
|
|
3093
|
+
return {
|
|
3094
|
+
alarmName,
|
|
3095
|
+
historyData,
|
|
3096
|
+
historyItemType: item.HistoryItemType,
|
|
3097
|
+
historySummary: boundedText(
|
|
3098
|
+
item.HistorySummary,
|
|
3099
|
+
'alarmHistory[' + index + '].historySummary',
|
|
3100
|
+
),
|
|
3101
|
+
state,
|
|
3102
|
+
timestamp,
|
|
3103
|
+
};
|
|
3104
|
+
},
|
|
3105
|
+
);
|
|
3106
|
+
const failureIndex = historyItems.findIndex(({ state }) => state === 'ALARM');
|
|
3107
|
+
const recoveryIndex = historyItems.findIndex(
|
|
3108
|
+
({ state }, index) => index > failureIndex && state === 'OK',
|
|
3109
|
+
);
|
|
3110
|
+
if (failureIndex < 0 || recoveryIndex < 0) {
|
|
3111
|
+
throw new Error(
|
|
3112
|
+
'Alarm history does not independently prove failure followed by recovery.',
|
|
3113
|
+
);
|
|
3114
|
+
}
|
|
3115
|
+
const alarm = stateResponse.MetricAlarms?.[0];
|
|
3116
|
+
const alarmArn =
|
|
3117
|
+
'arn:' +
|
|
3118
|
+
partition +
|
|
3119
|
+
':cloudwatch:' +
|
|
3120
|
+
target.region +
|
|
3121
|
+
':' +
|
|
3122
|
+
target.account +
|
|
3123
|
+
':alarm:' +
|
|
3124
|
+
alarmName;
|
|
3125
|
+
let description;
|
|
3126
|
+
try {
|
|
3127
|
+
description = JSON.parse(alarm?.AlarmDescription ?? 'null');
|
|
3128
|
+
} catch {
|
|
3129
|
+
throw new Error('Reliability alarm description is not valid metadata.');
|
|
3130
|
+
}
|
|
3131
|
+
if (
|
|
3132
|
+
stateResponse.MetricAlarms?.length !== 1 ||
|
|
3133
|
+
alarm?.AlarmArn !== alarmArn ||
|
|
3134
|
+
alarm?.AlarmName !== alarmName ||
|
|
3135
|
+
alarm?.StateValue !== 'OK' ||
|
|
3136
|
+
description?.id !== alarmResource.id ||
|
|
3137
|
+
description?.slice !== '${name}' ||
|
|
3138
|
+
description?.schemaVersion !== 1
|
|
3139
|
+
) {
|
|
3140
|
+
throw new Error(
|
|
3141
|
+
'Reliability alarm response does not match the exact declared resource.',
|
|
3142
|
+
);
|
|
3143
|
+
}
|
|
3144
|
+
const workflowInputs = input.workflowExecutions ?? [];
|
|
3145
|
+
if (
|
|
3146
|
+
!Array.isArray(workflowInputs) ||
|
|
3147
|
+
workflowInputs.length > 2 ||
|
|
3148
|
+
(exercise.observation.workflowMetadata === 'required' &&
|
|
3149
|
+
workflowInputs.length !== 2) ||
|
|
3150
|
+
(exercise.observation.workflowMetadata === 'none' &&
|
|
3151
|
+
workflowInputs.length !== 0)
|
|
3152
|
+
) {
|
|
3153
|
+
throw new Error(
|
|
3154
|
+
'Reliability workflow provenance is absent or outside its declared bound.',
|
|
3155
|
+
);
|
|
3156
|
+
}
|
|
3157
|
+
const workflowExecutions = [];
|
|
3158
|
+
for (const [index, workflowInput] of workflowInputs.entries()) {
|
|
3159
|
+
const workflow = exactKeys(
|
|
3160
|
+
workflowInput,
|
|
3161
|
+
['executionArn', 'role'],
|
|
3162
|
+
'workflowExecutions[' + index + ']',
|
|
3163
|
+
);
|
|
3164
|
+
if (
|
|
3165
|
+
!['failure', 'recovery'].includes(workflow.role) ||
|
|
3166
|
+
workflowInputs.filter(({ role }) => role === workflow.role).length !== 1
|
|
3167
|
+
) {
|
|
3168
|
+
throw new Error('Reliability workflow roles must be exact and unique.');
|
|
3169
|
+
}
|
|
3170
|
+
const executionArn = safeToken(
|
|
3171
|
+
workflow.executionArn,
|
|
3172
|
+
'workflowExecutions[' + index + '].executionArn',
|
|
3173
|
+
512,
|
|
3174
|
+
);
|
|
3175
|
+
if (!authorizedExecutionArn(activeResources, executionArn)) {
|
|
3176
|
+
throw new Error(
|
|
3177
|
+
'Reliability workflow is not authorized by the protected deployment.',
|
|
3178
|
+
);
|
|
3179
|
+
}
|
|
3180
|
+
const workflowRequest = {
|
|
3181
|
+
executionArn,
|
|
3182
|
+
includedData: 'METADATA_ONLY',
|
|
3183
|
+
};
|
|
3184
|
+
const response = await new SFNClient({ region: target.region }).send(
|
|
3185
|
+
new DescribeExecutionCommand(workflowRequest),
|
|
3186
|
+
);
|
|
3187
|
+
const expectedStatuses =
|
|
3188
|
+
workflow.role === 'failure'
|
|
3189
|
+
? ['ABORTED', 'FAILED', 'TIMED_OUT']
|
|
3190
|
+
: ['SUCCEEDED'];
|
|
3191
|
+
if (
|
|
3192
|
+
response.executionArn !== executionArn ||
|
|
3193
|
+
!activeResources.stateMachineArns.includes(response.stateMachineArn) ||
|
|
3194
|
+
!expectedStatuses.includes(response.status)
|
|
3195
|
+
) {
|
|
3196
|
+
throw new Error(
|
|
3197
|
+
'Reliability workflow response does not prove the declared outcome.',
|
|
3198
|
+
);
|
|
3199
|
+
}
|
|
3200
|
+
workflowExecutions.push({
|
|
3201
|
+
request: workflowRequest,
|
|
3202
|
+
response: {
|
|
3203
|
+
executionArn: response.executionArn,
|
|
3204
|
+
requestId: safeToken(
|
|
3205
|
+
response.$metadata?.requestId,
|
|
3206
|
+
'workflow.requestId',
|
|
3207
|
+
128,
|
|
3208
|
+
),
|
|
3209
|
+
startDate: utc(
|
|
3210
|
+
response.startDate?.toISOString?.(),
|
|
3211
|
+
'workflow.startDate',
|
|
3212
|
+
),
|
|
3213
|
+
stateMachineArn: response.stateMachineArn,
|
|
3214
|
+
status: response.status,
|
|
3215
|
+
stopDate: utc(
|
|
3216
|
+
response.stopDate?.toISOString?.(),
|
|
3217
|
+
'workflow.stopDate',
|
|
3218
|
+
),
|
|
3219
|
+
},
|
|
3220
|
+
role: workflow.role,
|
|
3221
|
+
});
|
|
3222
|
+
}
|
|
3223
|
+
const failure = historyItems[failureIndex];
|
|
3224
|
+
const recovery = historyItems[recoveryIndex];
|
|
3225
|
+
const collection = {
|
|
3226
|
+
alarm: {
|
|
3227
|
+
alarmArn,
|
|
3228
|
+
alarmName,
|
|
3229
|
+
currentState: alarm.StateValue,
|
|
3230
|
+
description,
|
|
3231
|
+
},
|
|
3232
|
+
binding: collectedBinding,
|
|
3233
|
+
bindingArtifact: {
|
|
3234
|
+
digest: activeBinding.digest,
|
|
3235
|
+
path: activeBinding.path,
|
|
3236
|
+
},
|
|
3237
|
+
exerciseId,
|
|
3238
|
+
kind: 'reliability-exercise-aws-collection',
|
|
3239
|
+
requests: {
|
|
3240
|
+
alarmHistory: {
|
|
3241
|
+
...historyRequest,
|
|
3242
|
+
EndDate: windowEnd,
|
|
3243
|
+
StartDate: windowStart,
|
|
3244
|
+
},
|
|
3245
|
+
alarmState: stateRequest,
|
|
3246
|
+
},
|
|
3247
|
+
responses: {
|
|
3248
|
+
alarmHistory: {
|
|
3249
|
+
items: historyItems,
|
|
3250
|
+
requestId,
|
|
3251
|
+
},
|
|
3252
|
+
alarmState: {
|
|
3253
|
+
requestId: stateRequestId,
|
|
3254
|
+
state: alarm.StateValue,
|
|
3255
|
+
},
|
|
3256
|
+
workflowExecutions,
|
|
3257
|
+
},
|
|
3258
|
+
schemaVersion: 1,
|
|
3259
|
+
slice: '${name}',
|
|
3260
|
+
target,
|
|
3261
|
+
transitions: {
|
|
3262
|
+
failure: { state: failure.state, timestamp: failure.timestamp },
|
|
3263
|
+
recovery: { state: recovery.state, timestamp: recovery.timestamp },
|
|
3264
|
+
},
|
|
3265
|
+
workload: exercise.workload,
|
|
3266
|
+
};
|
|
3267
|
+
const collectionArtifact = writeImmutableArtifact(
|
|
3268
|
+
'.ebk/evidence/reliability/collections',
|
|
3269
|
+
collection,
|
|
3270
|
+
);
|
|
3271
|
+
const observation = (role, transition, status) => ({
|
|
3272
|
+
binding: collectedBinding,
|
|
3273
|
+
capturedAt: transition.timestamp,
|
|
3274
|
+
evidenceId:
|
|
3275
|
+
exerciseId + '-' + role + '-' + collectionArtifact.digest.slice(0, 16),
|
|
3276
|
+
exerciseId,
|
|
3277
|
+
kind: 'reliability-exercise-' + role,
|
|
3278
|
+
producer: {
|
|
3279
|
+
adapter: 'aws-cloudwatch-alarm-history-v1',
|
|
3280
|
+
collectionArtifact,
|
|
3281
|
+
requestId,
|
|
3282
|
+
resource: alarmArn,
|
|
3283
|
+
},
|
|
3284
|
+
schemaVersion: 1,
|
|
3285
|
+
slice: '${name}',
|
|
3286
|
+
status,
|
|
3287
|
+
workload: exercise.workload,
|
|
3288
|
+
});
|
|
3289
|
+
const failureArtifact = writeImmutableArtifact(
|
|
3290
|
+
'.ebk/evidence/reliability/exercise-inputs',
|
|
3291
|
+
observation('failure', failure, 'observed'),
|
|
3292
|
+
);
|
|
3293
|
+
const recoveryArtifact = writeImmutableArtifact(
|
|
3294
|
+
'.ebk/evidence/reliability/exercise-inputs',
|
|
3295
|
+
observation(
|
|
3296
|
+
'recovery',
|
|
3297
|
+
recovery,
|
|
3298
|
+
exercise.workload === 'alert-delivery' ? 'unsupported' : 'verified',
|
|
3299
|
+
),
|
|
3300
|
+
);
|
|
3301
|
+
process.stdout.write(
|
|
3302
|
+
JSON.stringify({
|
|
3303
|
+
collection: collectionArtifact,
|
|
3304
|
+
failure: failureArtifact,
|
|
3305
|
+
kind: 'reliability-exercise-collection-result',
|
|
3306
|
+
recovery: recoveryArtifact,
|
|
3307
|
+
schemaVersion: 1,
|
|
3308
|
+
}) + '\\n',
|
|
3309
|
+
);
|
|
3310
|
+
`,
|
|
3311
|
+
'scripts/reliability-exercise-report.mjs': `import { readFileSync } from 'node:fs';
|
|
3312
|
+
import {
|
|
3313
|
+
activeDeploymentBinding,
|
|
3314
|
+
artifactReference,
|
|
3315
|
+
binding,
|
|
3316
|
+
exactKeys,
|
|
3317
|
+
readBoundedJson,
|
|
3318
|
+
safeToken,
|
|
3319
|
+
sameBinding,
|
|
3320
|
+
utc,
|
|
3321
|
+
writeImmutableArtifact,
|
|
3322
|
+
} from './reliability-evidence-lib.mjs';
|
|
3323
|
+
|
|
3324
|
+
const inputPath = process.env.EBK_RELIABILITY_EXERCISE_INPUT_PATH?.trim();
|
|
3325
|
+
if (!inputPath) {
|
|
3326
|
+
throw new Error('EBK_RELIABILITY_EXERCISE_INPUT_PATH is required.');
|
|
3327
|
+
}
|
|
3328
|
+
const inputArtifact = readBoundedJson(
|
|
3329
|
+
inputPath,
|
|
3330
|
+
'Reliability exercise report input',
|
|
3331
|
+
);
|
|
3332
|
+
const input = exactKeys(
|
|
3333
|
+
inputArtifact.value,
|
|
3334
|
+
['artifacts', 'binding', 'exerciseId', 'schemaVersion'],
|
|
3335
|
+
'Reliability exercise report input',
|
|
3336
|
+
);
|
|
3337
|
+
const declaration = JSON.parse(
|
|
3338
|
+
readFileSync(
|
|
3339
|
+
new URL('../testing/reliability-exercises.json', import.meta.url),
|
|
3340
|
+
'utf8',
|
|
3341
|
+
),
|
|
3342
|
+
);
|
|
3343
|
+
if (input.schemaVersion !== 1) {
|
|
3344
|
+
throw new Error('Reliability exercise report input is incompatible.');
|
|
3345
|
+
}
|
|
3346
|
+
const exerciseId = safeToken(input.exerciseId, 'exerciseId', 128);
|
|
3347
|
+
const exercise = declaration.exercises?.find(
|
|
3348
|
+
({ id }) => id === exerciseId,
|
|
3349
|
+
);
|
|
3350
|
+
if (!exercise) {
|
|
3351
|
+
throw new Error('Reliability exercise is undeclared or incomplete.');
|
|
3352
|
+
}
|
|
3353
|
+
const reportBinding = binding(input.binding, 'binding');
|
|
3354
|
+
if (reportBinding.workload !== exercise.workload) {
|
|
3355
|
+
throw new Error('Reliability exercise workload binding does not match.');
|
|
3356
|
+
}
|
|
3357
|
+
const references = exactKeys(
|
|
3358
|
+
input.artifacts,
|
|
3359
|
+
['cleanup', 'failure', 'postconditions', 'recovery'],
|
|
3360
|
+
'artifacts',
|
|
3361
|
+
);
|
|
3362
|
+
const roles = ['failure', 'recovery', 'postconditions', 'cleanup'];
|
|
3363
|
+
const statuses = {
|
|
3364
|
+
cleanup: 'complete',
|
|
3365
|
+
failure: 'observed',
|
|
3366
|
+
postconditions: 'passed',
|
|
3367
|
+
recovery:
|
|
3368
|
+
exercise.workload === 'alert-delivery' ? 'unsupported' : 'verified',
|
|
3369
|
+
};
|
|
3370
|
+
const evidence = {};
|
|
3371
|
+
const collectionDigests = [];
|
|
3372
|
+
let protectedStage;
|
|
3373
|
+
let previous = 0;
|
|
3374
|
+
for (const role of roles) {
|
|
3375
|
+
const reference = artifactReference(references[role], 'artifacts.' + role);
|
|
3376
|
+
const artifact = exactKeys(
|
|
3377
|
+
reference.value,
|
|
3378
|
+
[
|
|
3379
|
+
'binding',
|
|
3380
|
+
'capturedAt',
|
|
3381
|
+
'checks',
|
|
3382
|
+
'evidenceId',
|
|
3383
|
+
'exerciseId',
|
|
3384
|
+
'kind',
|
|
3385
|
+
'producer',
|
|
3386
|
+
'schemaVersion',
|
|
3387
|
+
'slice',
|
|
3388
|
+
'status',
|
|
3389
|
+
'workload',
|
|
3390
|
+
],
|
|
3391
|
+
role + ' artifact',
|
|
3392
|
+
);
|
|
3393
|
+
const artifactBinding = binding(artifact.binding, role + '.binding');
|
|
3394
|
+
const producer = exactKeys(
|
|
3395
|
+
artifact.producer,
|
|
3396
|
+
role === 'failure' || role === 'recovery'
|
|
3397
|
+
? ['adapter', 'collectionArtifact', 'requestId', 'resource']
|
|
3398
|
+
: ['kind', 'requestId', 'resource'],
|
|
3399
|
+
role + '.producer',
|
|
3400
|
+
);
|
|
3401
|
+
const capturedAt = Date.parse(utc(artifact.capturedAt, role + '.capturedAt'));
|
|
3402
|
+
if (
|
|
3403
|
+
artifact.schemaVersion !== 1 ||
|
|
3404
|
+
artifact.kind !== 'reliability-exercise-' + role ||
|
|
3405
|
+
artifact.exerciseId !== exercise.id ||
|
|
3406
|
+
artifact.slice !== '${name}' ||
|
|
3407
|
+
artifact.workload !== exercise.workload ||
|
|
3408
|
+
!sameBinding(reportBinding, artifactBinding) ||
|
|
3409
|
+
capturedAt < previous ||
|
|
3410
|
+
!safeToken(artifact.evidenceId, role + '.evidenceId', 128) ||
|
|
3411
|
+
!safeToken(producer.requestId, role + '.producer.requestId', 128) ||
|
|
3412
|
+
!safeToken(producer.resource, role + '.producer.resource', 512)
|
|
3413
|
+
) {
|
|
3414
|
+
throw new Error(
|
|
3415
|
+
'Reliability exercise artifact is fabricated, cross-paired, or unsanitized.',
|
|
3416
|
+
);
|
|
3417
|
+
}
|
|
3418
|
+
previous = capturedAt;
|
|
3419
|
+
if (role === 'failure' || role === 'recovery') {
|
|
3420
|
+
if (producer.adapter !== 'aws-cloudwatch-alarm-history-v1') {
|
|
3421
|
+
throw new Error(
|
|
3422
|
+
'Failure and recovery require the generated AWS collection adapter.',
|
|
3423
|
+
);
|
|
3424
|
+
}
|
|
3425
|
+
const collectionReference = artifactReference(
|
|
3426
|
+
producer.collectionArtifact,
|
|
3427
|
+
role + '.producer.collectionArtifact',
|
|
3428
|
+
);
|
|
3429
|
+
const collection = exactKeys(
|
|
3430
|
+
collectionReference.value,
|
|
3431
|
+
[
|
|
3432
|
+
'alarm',
|
|
3433
|
+
'binding',
|
|
3434
|
+
'bindingArtifact',
|
|
3435
|
+
'exerciseId',
|
|
3436
|
+
'kind',
|
|
3437
|
+
'requests',
|
|
3438
|
+
'responses',
|
|
3439
|
+
'schemaVersion',
|
|
3440
|
+
'slice',
|
|
3441
|
+
'target',
|
|
3442
|
+
'transitions',
|
|
3443
|
+
'workload',
|
|
3444
|
+
],
|
|
3445
|
+
role + ' collection',
|
|
3446
|
+
);
|
|
3447
|
+
const collectionBinding = binding(
|
|
3448
|
+
collection.binding,
|
|
3449
|
+
role + '.collection.binding',
|
|
3450
|
+
);
|
|
3451
|
+
const collectionActiveBinding = activeDeploymentBinding(
|
|
3452
|
+
collection.bindingArtifact,
|
|
3453
|
+
role + '.collection.bindingArtifact',
|
|
3454
|
+
);
|
|
3455
|
+
const alarm = exactKeys(
|
|
3456
|
+
collection.alarm,
|
|
3457
|
+
['alarmArn', 'alarmName', 'currentState', 'description'],
|
|
3458
|
+
role + '.collection.alarm',
|
|
3459
|
+
);
|
|
3460
|
+
const requests = exactKeys(
|
|
3461
|
+
collection.requests,
|
|
3462
|
+
['alarmHistory', 'alarmState'],
|
|
3463
|
+
role + '.collection.requests',
|
|
3464
|
+
);
|
|
3465
|
+
const responses = exactKeys(
|
|
3466
|
+
collection.responses,
|
|
3467
|
+
['alarmHistory', 'alarmState', 'workflowExecutions'],
|
|
3468
|
+
role + '.collection.responses',
|
|
3469
|
+
);
|
|
3470
|
+
const alarmHistoryRequest = exactKeys(
|
|
3471
|
+
requests.alarmHistory,
|
|
3472
|
+
[
|
|
3473
|
+
'AlarmName',
|
|
3474
|
+
'EndDate',
|
|
3475
|
+
'HistoryItemType',
|
|
3476
|
+
'MaxRecords',
|
|
3477
|
+
'ScanBy',
|
|
3478
|
+
'StartDate',
|
|
3479
|
+
],
|
|
3480
|
+
role + '.collection.requests.alarmHistory',
|
|
3481
|
+
);
|
|
3482
|
+
const alarmStateRequest = exactKeys(
|
|
3483
|
+
requests.alarmState,
|
|
3484
|
+
['AlarmNames', 'MaxRecords'],
|
|
3485
|
+
role + '.collection.requests.alarmState',
|
|
3486
|
+
);
|
|
3487
|
+
const alarmHistoryResponse = exactKeys(
|
|
3488
|
+
responses.alarmHistory,
|
|
3489
|
+
['items', 'requestId'],
|
|
3490
|
+
role + '.collection.responses.alarmHistory',
|
|
3491
|
+
);
|
|
3492
|
+
const alarmStateResponse = exactKeys(
|
|
3493
|
+
responses.alarmState,
|
|
3494
|
+
['requestId', 'state'],
|
|
3495
|
+
role + '.collection.responses.alarmState',
|
|
3496
|
+
);
|
|
3497
|
+
const collectionTarget = exactKeys(
|
|
3498
|
+
collection.target,
|
|
3499
|
+
['account', 'region', 'slice', 'stage'],
|
|
3500
|
+
role + '.collection.target',
|
|
3501
|
+
);
|
|
3502
|
+
const transitions = exactKeys(
|
|
3503
|
+
collection.transitions,
|
|
3504
|
+
['failure', 'recovery'],
|
|
3505
|
+
role + '.collection.transitions',
|
|
3506
|
+
);
|
|
3507
|
+
const transition = exactKeys(
|
|
3508
|
+
transitions[role],
|
|
3509
|
+
['state', 'timestamp'],
|
|
3510
|
+
role + '.collection.transitions.' + role,
|
|
3511
|
+
);
|
|
3512
|
+
if (
|
|
3513
|
+
!Array.isArray(alarmHistoryResponse.items) ||
|
|
3514
|
+
!Array.isArray(responses.workflowExecutions)
|
|
3515
|
+
) {
|
|
3516
|
+
throw new Error(
|
|
3517
|
+
'Reliability collection responses are outside their bounded shape.',
|
|
3518
|
+
);
|
|
3519
|
+
}
|
|
3520
|
+
const historyStates = alarmHistoryResponse.items.map((item, index) => {
|
|
3521
|
+
const historyItem = exactKeys(
|
|
3522
|
+
item,
|
|
3523
|
+
[
|
|
3524
|
+
'alarmName',
|
|
3525
|
+
'historyData',
|
|
3526
|
+
'historyItemType',
|
|
3527
|
+
'historySummary',
|
|
3528
|
+
'state',
|
|
3529
|
+
'timestamp',
|
|
3530
|
+
],
|
|
3531
|
+
role + '.collection.responses.alarmHistory.items[' + index + ']',
|
|
3532
|
+
);
|
|
3533
|
+
if (
|
|
3534
|
+
historyItem.alarmName !== alarm.alarmName ||
|
|
3535
|
+
historyItem.historyItemType !== 'StateUpdate' ||
|
|
3536
|
+
!['ALARM', 'INSUFFICIENT_DATA', 'OK'].includes(historyItem.state) ||
|
|
3537
|
+
!utc(
|
|
3538
|
+
historyItem.timestamp,
|
|
3539
|
+
role + '.collection.responses.alarmHistory.items.timestamp',
|
|
3540
|
+
) ||
|
|
3541
|
+
typeof historyItem.historyData !== 'string' ||
|
|
3542
|
+
historyItem.historyData.length > 4096 ||
|
|
3543
|
+
typeof historyItem.historySummary !== 'string' ||
|
|
3544
|
+
historyItem.historySummary.length > 2048
|
|
3545
|
+
) {
|
|
3546
|
+
throw new Error(
|
|
3547
|
+
'Reliability collection contains incompatible alarm history.',
|
|
3548
|
+
);
|
|
3549
|
+
}
|
|
3550
|
+
return {
|
|
3551
|
+
state: historyItem.state,
|
|
3552
|
+
timestamp: historyItem.timestamp,
|
|
3553
|
+
};
|
|
3554
|
+
});
|
|
3555
|
+
const failureHistoryIndex = historyStates.findIndex(
|
|
3556
|
+
({ state }) => state === 'ALARM',
|
|
3557
|
+
);
|
|
3558
|
+
const recoveryHistoryIndex = historyStates.findIndex(
|
|
3559
|
+
({ state }, index) =>
|
|
3560
|
+
index > failureHistoryIndex && state === 'OK',
|
|
3561
|
+
);
|
|
3562
|
+
for (const [index, workflowEntry] of
|
|
3563
|
+
responses.workflowExecutions.entries()) {
|
|
3564
|
+
const workflow = exactKeys(
|
|
3565
|
+
workflowEntry,
|
|
3566
|
+
['request', 'response', 'role'],
|
|
3567
|
+
role + '.collection.responses.workflowExecutions[' + index + ']',
|
|
3568
|
+
);
|
|
3569
|
+
const workflowRequest = exactKeys(
|
|
3570
|
+
workflow.request,
|
|
3571
|
+
['executionArn', 'includedData'],
|
|
3572
|
+
role + '.collection.responses.workflowExecutions.request',
|
|
3573
|
+
);
|
|
3574
|
+
const workflowResponse = exactKeys(
|
|
3575
|
+
workflow.response,
|
|
3576
|
+
[
|
|
3577
|
+
'executionArn',
|
|
3578
|
+
'requestId',
|
|
3579
|
+
'startDate',
|
|
3580
|
+
'stateMachineArn',
|
|
3581
|
+
'status',
|
|
3582
|
+
'stopDate',
|
|
3583
|
+
],
|
|
3584
|
+
role + '.collection.responses.workflowExecutions.response',
|
|
3585
|
+
);
|
|
3586
|
+
if (
|
|
3587
|
+
!['failure', 'recovery'].includes(workflow.role) ||
|
|
3588
|
+
workflowRequest.includedData !== 'METADATA_ONLY' ||
|
|
3589
|
+
workflowRequest.executionArn !== workflowResponse.executionArn ||
|
|
3590
|
+
!safeToken(
|
|
3591
|
+
workflowResponse.requestId,
|
|
3592
|
+
role + '.collection.responses.workflow.requestId',
|
|
3593
|
+
128,
|
|
3594
|
+
) ||
|
|
3595
|
+
(workflow.role === 'failure'
|
|
3596
|
+
? !['ABORTED', 'FAILED', 'TIMED_OUT'].includes(
|
|
3597
|
+
workflowResponse.status,
|
|
3598
|
+
)
|
|
3599
|
+
: workflowResponse.status !== 'SUCCEEDED')
|
|
3600
|
+
) {
|
|
3601
|
+
throw new Error(
|
|
3602
|
+
'Reliability collection workflow provenance is incompatible.',
|
|
3603
|
+
);
|
|
3604
|
+
}
|
|
3605
|
+
}
|
|
3606
|
+
if (
|
|
3607
|
+
collection.kind !== 'reliability-exercise-aws-collection' ||
|
|
3608
|
+
collection.schemaVersion !== 1 ||
|
|
3609
|
+
collection.exerciseId !== exercise.id ||
|
|
3610
|
+
collection.slice !== '${name}' ||
|
|
3611
|
+
collection.workload !== exercise.workload ||
|
|
3612
|
+
!sameBinding(reportBinding, collectionBinding) ||
|
|
3613
|
+
!sameBinding(collectionBinding, collectionActiveBinding.binding) ||
|
|
3614
|
+
alarm.currentState !== 'OK' ||
|
|
3615
|
+
alarmStateResponse.state !== 'OK' ||
|
|
3616
|
+
!safeToken(
|
|
3617
|
+
alarmStateResponse.requestId,
|
|
3618
|
+
role + '.collection.responses.alarmState.requestId',
|
|
3619
|
+
128,
|
|
3620
|
+
) ||
|
|
3621
|
+
alarm.alarmArn !== producer.resource ||
|
|
3622
|
+
collectionTarget.slice !== '${name}' ||
|
|
3623
|
+
['account', 'region', 'slice', 'stage'].some(
|
|
3624
|
+
(key) =>
|
|
3625
|
+
collectionTarget[key] !==
|
|
3626
|
+
collectionActiveBinding.target[key],
|
|
3627
|
+
) ||
|
|
3628
|
+
!safeToken(
|
|
3629
|
+
collectionTarget.account,
|
|
3630
|
+
role + '.collection.target.account',
|
|
3631
|
+
128,
|
|
3632
|
+
) ||
|
|
3633
|
+
!safeToken(
|
|
3634
|
+
collectionTarget.region,
|
|
3635
|
+
role + '.collection.target.region',
|
|
3636
|
+
128,
|
|
3637
|
+
) ||
|
|
3638
|
+
!safeToken(
|
|
3639
|
+
collectionTarget.stage,
|
|
3640
|
+
role + '.collection.target.stage',
|
|
3641
|
+
128,
|
|
3642
|
+
) ||
|
|
3643
|
+
alarmHistoryRequest.AlarmName !== alarm.alarmName ||
|
|
3644
|
+
alarmHistoryRequest.HistoryItemType !== 'StateUpdate' ||
|
|
3645
|
+
alarmHistoryRequest.MaxRecords !== 32 ||
|
|
3646
|
+
alarmHistoryRequest.ScanBy !== 'TimestampAscending' ||
|
|
3647
|
+
!Array.isArray(alarmStateRequest.AlarmNames) ||
|
|
3648
|
+
alarmStateRequest.AlarmNames.length !== 1 ||
|
|
3649
|
+
alarmStateRequest.AlarmNames[0] !== alarm.alarmName ||
|
|
3650
|
+
alarmStateRequest.MaxRecords !== 1 ||
|
|
3651
|
+
alarmHistoryResponse.requestId !== producer.requestId ||
|
|
3652
|
+
alarmHistoryResponse.items.length < 2 ||
|
|
3653
|
+
failureHistoryIndex < 0 ||
|
|
3654
|
+
recoveryHistoryIndex < 0 ||
|
|
3655
|
+
transitions.failure.timestamp !==
|
|
3656
|
+
historyStates[failureHistoryIndex].timestamp ||
|
|
3657
|
+
transitions.recovery.timestamp !==
|
|
3658
|
+
historyStates[recoveryHistoryIndex].timestamp ||
|
|
3659
|
+
transition.timestamp !== artifact.capturedAt ||
|
|
3660
|
+
transition.state !== (role === 'failure' ? 'ALARM' : 'OK')
|
|
3661
|
+
) {
|
|
3662
|
+
throw new Error(
|
|
3663
|
+
'Reliability collection request, response, resource, or transition is cross-paired.',
|
|
3664
|
+
);
|
|
3665
|
+
}
|
|
3666
|
+
if (
|
|
3667
|
+
protectedStage !== undefined &&
|
|
3668
|
+
protectedStage !== collectionActiveBinding.target.stage
|
|
3669
|
+
) {
|
|
3670
|
+
throw new Error(
|
|
3671
|
+
'Reliability exercise collections crossed protected stages.',
|
|
3672
|
+
);
|
|
3673
|
+
}
|
|
3674
|
+
protectedStage = collectionActiveBinding.target.stage;
|
|
3675
|
+
collectionDigests.push(collectionReference.digest);
|
|
3676
|
+
} else if (producer.kind !== 'testing-evidence') {
|
|
3677
|
+
throw new Error(
|
|
3678
|
+
'Postconditions and cleanup require testing-evidence provenance.',
|
|
3679
|
+
);
|
|
3680
|
+
}
|
|
3681
|
+
if (artifact.status !== statuses[role]) {
|
|
3682
|
+
throw new Error('Reliability exercise artifact status is incompatible.');
|
|
3683
|
+
}
|
|
3684
|
+
if (role === 'postconditions') {
|
|
3685
|
+
if (
|
|
3686
|
+
!Array.isArray(artifact.checks) ||
|
|
3687
|
+
artifact.checks.length !== exercise.postconditions.length ||
|
|
3688
|
+
artifact.checks.some(
|
|
3689
|
+
(check, index) =>
|
|
3690
|
+
check !== 'postcondition-' + String(index + 1) + '-passed',
|
|
3691
|
+
)
|
|
3692
|
+
) {
|
|
3693
|
+
throw new Error(
|
|
3694
|
+
'Reliability exercise postcondition artifact is incomplete.',
|
|
3695
|
+
);
|
|
3696
|
+
}
|
|
3697
|
+
} else if (artifact.checks !== undefined) {
|
|
3698
|
+
throw new Error('Only the postcondition artifact may contain checks.');
|
|
3699
|
+
}
|
|
3700
|
+
evidence[role] = reference.digest;
|
|
3701
|
+
}
|
|
3702
|
+
if (
|
|
3703
|
+
collectionDigests.length !== 2 ||
|
|
3704
|
+
new Set(collectionDigests).size !== 1 ||
|
|
3705
|
+
!['staging', 'production'].includes(protectedStage)
|
|
3706
|
+
) {
|
|
3707
|
+
throw new Error(
|
|
3708
|
+
'Failure and recovery must come from one independently collected AWS transition chain.',
|
|
3709
|
+
);
|
|
3710
|
+
}
|
|
3711
|
+
if (new Set(Object.values(evidence)).size !== roles.length) {
|
|
3712
|
+
throw new Error('Reliability exercise artifacts must be independently captured.');
|
|
3713
|
+
}
|
|
3714
|
+
const report = {
|
|
3715
|
+
artifacts: evidence,
|
|
3716
|
+
binding: reportBinding,
|
|
3717
|
+
...(exercise.workload === 'alert-delivery'
|
|
3718
|
+
? { alertDelivery: { state: artifactReference(
|
|
3719
|
+
references.recovery,
|
|
3720
|
+
'artifacts.recovery',
|
|
3721
|
+
).value.status } }
|
|
3722
|
+
: {}),
|
|
3723
|
+
exerciseId: exercise.id,
|
|
3724
|
+
kind: 'reliability-failure-recovery-evidence',
|
|
3725
|
+
provenance: protectedStage,
|
|
3726
|
+
schemaVersion: 1,
|
|
3727
|
+
slice: '${name}',
|
|
3728
|
+
workload: exercise.workload,
|
|
3729
|
+
};
|
|
3730
|
+
const reference = writeImmutableArtifact(
|
|
3731
|
+
'.ebk/evidence/reliability/exercises',
|
|
3732
|
+
report,
|
|
3733
|
+
);
|
|
3734
|
+
process.stdout.write(JSON.stringify({ ...reference, ...report }) + '\\n');
|
|
3735
|
+
`,
|
|
3736
|
+
'scripts/reliability-diagnostic.mjs': `import {
|
|
3737
|
+
CloudWatchClient,
|
|
3738
|
+
DescribeAlarmsCommand,
|
|
3739
|
+
GetMetricDataCommand,
|
|
3740
|
+
} from '@aws-sdk/client-cloudwatch';
|
|
3741
|
+
import {
|
|
3742
|
+
DescribeServicesCommand,
|
|
3743
|
+
ECSClient,
|
|
3744
|
+
} from '@aws-sdk/client-ecs';
|
|
3745
|
+
import {
|
|
3746
|
+
DescribeExecutionCommand,
|
|
3747
|
+
SFNClient,
|
|
3748
|
+
} from '@aws-sdk/client-sfn';
|
|
3749
|
+
import {
|
|
3750
|
+
GetCallerIdentityCommand,
|
|
3751
|
+
STSClient,
|
|
3752
|
+
} from '@aws-sdk/client-sts';
|
|
3753
|
+
import { readFileSync } from 'node:fs';
|
|
3754
|
+
import {
|
|
3755
|
+
authorizeDiagnostic,
|
|
3756
|
+
validateReliabilityContract,
|
|
3757
|
+
} from '@empire-builder-kit/runtime/reliability';
|
|
3758
|
+
import {
|
|
3759
|
+
activeDeploymentBinding,
|
|
3760
|
+
awsPartition,
|
|
3761
|
+
authorizedExecutionArn,
|
|
3762
|
+
exactKeys,
|
|
3763
|
+
readBoundedJson,
|
|
3764
|
+
safeToken,
|
|
3765
|
+
sha256,
|
|
3766
|
+
utc,
|
|
3767
|
+
writeImmutableArtifact,
|
|
3768
|
+
} from './reliability-evidence-lib.mjs';
|
|
3769
|
+
|
|
3770
|
+
const inputPath = process.env.EBK_RELIABILITY_DIAGNOSTIC_INPUT_PATH?.trim();
|
|
3771
|
+
if (!inputPath) {
|
|
3772
|
+
throw new Error('EBK_RELIABILITY_DIAGNOSTIC_INPUT_PATH is required.');
|
|
3773
|
+
}
|
|
3774
|
+
const requestInput = readBoundedJson(inputPath, 'Reliability diagnostic input');
|
|
3775
|
+
const request = exactKeys(
|
|
3776
|
+
requestInput.value,
|
|
3777
|
+
['approval', 'bindingArtifact', 'query', 'source', 'target'],
|
|
3778
|
+
'Reliability diagnostic input',
|
|
3779
|
+
);
|
|
3780
|
+
const plan = authorizeDiagnostic({
|
|
3781
|
+
...(request.approval === undefined ? {} : { approval: request.approval }),
|
|
3782
|
+
source: request.source,
|
|
3783
|
+
target: request.target,
|
|
3784
|
+
});
|
|
3785
|
+
if (plan.approvalRequired) {
|
|
3786
|
+
process.stdout.write(JSON.stringify({
|
|
3787
|
+
approvalRequired: true,
|
|
3788
|
+
expiresAt: plan.expiresAt,
|
|
3789
|
+
queryClass: plan.queryClass,
|
|
3790
|
+
source: plan.source,
|
|
3791
|
+
target: plan.target,
|
|
3792
|
+
}) + '\\n');
|
|
3793
|
+
process.exit(0);
|
|
3794
|
+
}
|
|
3795
|
+
const activeBinding = activeDeploymentBinding(
|
|
3796
|
+
request.bindingArtifact,
|
|
3797
|
+
'bindingArtifact',
|
|
3798
|
+
);
|
|
3799
|
+
const capturedBinding = activeBinding.binding;
|
|
3800
|
+
const activeResources = activeBinding.resources;
|
|
3801
|
+
const bindingTarget = activeBinding.target;
|
|
3802
|
+
if (
|
|
3803
|
+
bindingTarget.account !== plan.target.account ||
|
|
3804
|
+
bindingTarget.region !== plan.target.region ||
|
|
3805
|
+
bindingTarget.slice !== plan.target.slice ||
|
|
3806
|
+
bindingTarget.stage !== plan.target.stage
|
|
3807
|
+
) {
|
|
3808
|
+
throw new Error(
|
|
3809
|
+
'Reliability diagnostic deployment binding does not match the exact target.',
|
|
3810
|
+
);
|
|
3811
|
+
}
|
|
3812
|
+
if (plan.source === 'alarm-state') {
|
|
3813
|
+
const query = exactKeys(request.query, ['alarmNames'], 'alarm-state query');
|
|
3814
|
+
if (
|
|
3815
|
+
!Array.isArray(query.alarmNames) ||
|
|
3816
|
+
query.alarmNames.some(
|
|
3817
|
+
(value) =>
|
|
3818
|
+
!activeResources.alarmNames.includes(
|
|
3819
|
+
safeToken(value, 'alarm-state alarm', 255),
|
|
3820
|
+
),
|
|
3821
|
+
)
|
|
3822
|
+
) {
|
|
3823
|
+
throw new Error(
|
|
3824
|
+
'Alarm-state query is not authorized by the protected deployment.',
|
|
3825
|
+
);
|
|
3826
|
+
}
|
|
3827
|
+
}
|
|
3828
|
+
if (
|
|
3829
|
+
plan.source === 'sanitized-signal' &&
|
|
3830
|
+
request.query?.apiId !== undefined &&
|
|
3831
|
+
safeToken(request.query.apiId, 'apiId', 128) !== activeResources.apiId
|
|
3832
|
+
) {
|
|
3833
|
+
throw new Error(
|
|
3834
|
+
'Metric query API is not authorized by the protected deployment.',
|
|
3835
|
+
);
|
|
3836
|
+
}
|
|
3837
|
+
if (plan.source === 'resource-metadata') {
|
|
3838
|
+
const query = exactKeys(
|
|
3839
|
+
request.query,
|
|
3840
|
+
['ecsServices', 'executions'],
|
|
3841
|
+
'resource-metadata query',
|
|
3842
|
+
);
|
|
3843
|
+
if (!Array.isArray(query.ecsServices) || !Array.isArray(query.executions)) {
|
|
3844
|
+
throw new Error(
|
|
3845
|
+
'Resource query is not authorized by the protected deployment.',
|
|
3846
|
+
);
|
|
3847
|
+
}
|
|
3848
|
+
for (const [index, value] of query.ecsServices.entries()) {
|
|
3849
|
+
const requested = exactKeys(
|
|
3850
|
+
value,
|
|
3851
|
+
['cluster', 'services'],
|
|
3852
|
+
'ecsServices[' + index + ']',
|
|
3853
|
+
);
|
|
3854
|
+
const cluster = safeToken(
|
|
3855
|
+
requested.cluster,
|
|
3856
|
+
'ecsServices[' + index + '].cluster',
|
|
3857
|
+
512,
|
|
3858
|
+
);
|
|
3859
|
+
const authorized = activeResources.ecsServices.find(
|
|
3860
|
+
(entry) => entry.cluster === cluster,
|
|
3861
|
+
);
|
|
3862
|
+
if (
|
|
3863
|
+
!authorized ||
|
|
3864
|
+
!Array.isArray(requested.services) ||
|
|
3865
|
+
requested.services.some(
|
|
3866
|
+
(service) =>
|
|
3867
|
+
!authorized.services.includes(
|
|
3868
|
+
safeToken(service, 'ecsServices.service', 255),
|
|
3869
|
+
),
|
|
3870
|
+
)
|
|
3871
|
+
) {
|
|
3872
|
+
throw new Error(
|
|
3873
|
+
'ECS query is not authorized by the protected deployment.',
|
|
3874
|
+
);
|
|
3875
|
+
}
|
|
3876
|
+
}
|
|
3877
|
+
for (const [index, value] of query.executions.entries()) {
|
|
3878
|
+
const executionArn = safeToken(value, 'executions[' + index + ']', 512);
|
|
3879
|
+
if (!authorizedExecutionArn(activeResources, executionArn)) {
|
|
3880
|
+
throw new Error(
|
|
3881
|
+
'Step Functions query is not authorized by the protected deployment.',
|
|
3882
|
+
);
|
|
3883
|
+
}
|
|
3884
|
+
}
|
|
3885
|
+
}
|
|
3886
|
+
const region = plan.target.region;
|
|
3887
|
+
const partition = awsPartition(region);
|
|
3888
|
+
const identity = await new STSClient({ region }).send(
|
|
3889
|
+
new GetCallerIdentityCommand({}),
|
|
3890
|
+
);
|
|
3891
|
+
if (
|
|
3892
|
+
identity.Account !== plan.target.account ||
|
|
3893
|
+
!identity.Arn?.startsWith(
|
|
3894
|
+
'arn:' + partition + ':sts::' + plan.target.account + ':',
|
|
3895
|
+
)
|
|
3896
|
+
) {
|
|
3897
|
+
throw new Error(
|
|
3898
|
+
'AWS caller identity does not match the active reliability target.',
|
|
3899
|
+
);
|
|
3900
|
+
}
|
|
3901
|
+
const capture = async (value, directory) => {
|
|
3902
|
+
const reference = writeImmutableArtifact(directory, value);
|
|
3903
|
+
process.stdout.write(JSON.stringify({ ...reference, ...value }) + '\\n');
|
|
3904
|
+
};
|
|
3905
|
+
if (plan.source === 'alarm-state') {
|
|
3906
|
+
const query = exactKeys(request.query, ['alarmNames'], 'alarm-state query');
|
|
3907
|
+
if (
|
|
3908
|
+
!Array.isArray(query.alarmNames) ||
|
|
3909
|
+
query.alarmNames.length < 1 ||
|
|
3910
|
+
query.alarmNames.length > 32 ||
|
|
3911
|
+
new Set(query.alarmNames).size !== query.alarmNames.length
|
|
3912
|
+
) {
|
|
3913
|
+
throw new Error('Alarm-state query must name one to 32 exact alarms.');
|
|
3914
|
+
}
|
|
3915
|
+
const alarmNames = query.alarmNames.map((value, index) =>
|
|
3916
|
+
safeToken(value, 'alarmNames[' + index + ']', 255),
|
|
3917
|
+
);
|
|
3918
|
+
if (
|
|
3919
|
+
alarmNames.some(
|
|
3920
|
+
(value) => !value.startsWith('${name}-' + plan.target.stage + '-'),
|
|
3921
|
+
)
|
|
3922
|
+
) {
|
|
3923
|
+
throw new Error('Alarm-state query escaped the exact Slice and stage.');
|
|
3924
|
+
}
|
|
3925
|
+
const response = await new CloudWatchClient({ region }).send(
|
|
3926
|
+
new DescribeAlarmsCommand({
|
|
3927
|
+
AlarmNames: alarmNames,
|
|
3928
|
+
MaxRecords: alarmNames.length,
|
|
3929
|
+
}),
|
|
3930
|
+
);
|
|
3931
|
+
if (response.NextToken) {
|
|
3932
|
+
throw new Error('Alarm-state query exceeded its one-page bound.');
|
|
3933
|
+
}
|
|
3934
|
+
const alarms = (response.MetricAlarms ?? []).map((alarm) => ({
|
|
3935
|
+
actionsEnabled: alarm.ActionsEnabled ?? null,
|
|
3936
|
+
alarmArn: alarm.AlarmArn ?? null,
|
|
3937
|
+
alarmName: alarm.AlarmName ?? null,
|
|
3938
|
+
stateUpdatedTimestamp:
|
|
3939
|
+
alarm.StateUpdatedTimestamp?.toISOString?.() ?? null,
|
|
3940
|
+
stateValue: alarm.StateValue ?? null,
|
|
3941
|
+
}));
|
|
3942
|
+
if (
|
|
3943
|
+
alarms.length !== alarmNames.length ||
|
|
3944
|
+
new Set(alarms.map(({ alarmName }) => alarmName)).size !== alarms.length ||
|
|
3945
|
+
alarms.some(
|
|
3946
|
+
({ alarmArn, alarmName }) =>
|
|
3947
|
+
!alarmNames.includes(alarmName) ||
|
|
3948
|
+
!alarmArn?.startsWith(
|
|
3949
|
+
'arn:' +
|
|
3950
|
+
partition +
|
|
3951
|
+
':cloudwatch:' +
|
|
3952
|
+
region +
|
|
3953
|
+
':' +
|
|
3954
|
+
plan.target.account +
|
|
3955
|
+
':alarm:',
|
|
3956
|
+
),
|
|
3957
|
+
)
|
|
3958
|
+
) {
|
|
3959
|
+
throw new Error('AWS alarm response did not match the exact query target.');
|
|
3960
|
+
}
|
|
3961
|
+
await capture(
|
|
3962
|
+
{
|
|
3963
|
+
alarms,
|
|
3964
|
+
binding: capturedBinding,
|
|
3965
|
+
bindingDigest: activeBinding.digest,
|
|
3966
|
+
capturedAt: new Date().toISOString(),
|
|
3967
|
+
kind: 'reliability-alarm-capture',
|
|
3968
|
+
schemaVersion: 1,
|
|
3969
|
+
slice: '${name}',
|
|
3970
|
+
target: plan.target,
|
|
3971
|
+
},
|
|
3972
|
+
'.ebk/evidence/reliability/diagnostics',
|
|
3973
|
+
);
|
|
3974
|
+
process.exit(0);
|
|
3975
|
+
}
|
|
3976
|
+
if (plan.source === 'sanitized-signal') {
|
|
3977
|
+
const query = exactKeys(
|
|
3978
|
+
request.query,
|
|
3979
|
+
['apiId', 'endTime', 'objectiveId', 'startTime'],
|
|
3980
|
+
'sanitized-signal query',
|
|
3981
|
+
);
|
|
3982
|
+
const objectiveId = safeToken(query.objectiveId, 'objectiveId', 128);
|
|
3983
|
+
const contract = validateReliabilityContract(
|
|
3984
|
+
JSON.parse(
|
|
3985
|
+
readFileSync(
|
|
3986
|
+
new URL('../docs/reliability/slo.json', import.meta.url),
|
|
3987
|
+
'utf8',
|
|
3988
|
+
),
|
|
3989
|
+
),
|
|
3990
|
+
);
|
|
3991
|
+
const objective = contract.objectives.find(({ id }) => id === objectiveId);
|
|
3992
|
+
if (!objective || capturedBinding.workload !== objective.journey) {
|
|
3993
|
+
throw new Error(
|
|
3994
|
+
'Metric query workload does not match the exact objective journey.',
|
|
3995
|
+
);
|
|
3996
|
+
}
|
|
3997
|
+
const startTime = utc(query.startTime, 'startTime');
|
|
3998
|
+
const endTime = utc(query.endTime, 'endTime');
|
|
3999
|
+
const elapsed = Date.parse(endTime) - Date.parse(startTime);
|
|
4000
|
+
if (elapsed <= 0 || elapsed > 6 * 60 * 60 * 1000) {
|
|
4001
|
+
throw new Error('Metric query window must be positive and at most six hours.');
|
|
4002
|
+
}
|
|
4003
|
+
const dimensions = [
|
|
4004
|
+
{ Name: 'Component', Value: 'next-server' },
|
|
4005
|
+
{ Name: 'Feature', Value: objective.feature },
|
|
4006
|
+
{ Name: 'Operation', Value: 'request' },
|
|
4007
|
+
{ Name: 'Slice', Value: '${name}' },
|
|
4008
|
+
{ Name: 'Stage', Value: plan.target.stage },
|
|
4009
|
+
];
|
|
4010
|
+
const metric = (id, metricName, metricDimensions = dimensions) => ({
|
|
4011
|
+
Id: id,
|
|
4012
|
+
MetricStat: {
|
|
4013
|
+
Metric: {
|
|
4014
|
+
Dimensions: metricDimensions,
|
|
4015
|
+
MetricName: metricName,
|
|
4016
|
+
Namespace:
|
|
4017
|
+
objective.id === 'api-availability' &&
|
|
4018
|
+
metricName !== 'SyntheticGood' &&
|
|
4019
|
+
metricName !== 'SyntheticValid'
|
|
4020
|
+
? 'AWS/ApiGateway'
|
|
4021
|
+
: 'EBK/${name}',
|
|
4022
|
+
},
|
|
4023
|
+
Period: 60,
|
|
4024
|
+
Stat: 'Sum',
|
|
4025
|
+
},
|
|
4026
|
+
ReturnData: true,
|
|
4027
|
+
});
|
|
4028
|
+
const syntheticDimensions = [
|
|
4029
|
+
{ Name: 'Component', Value: 'synthetic' },
|
|
4030
|
+
{ Name: 'Feature', Value: 'operations' },
|
|
4031
|
+
{
|
|
4032
|
+
Name: 'Operation',
|
|
4033
|
+
Value:
|
|
4034
|
+
objective.id === 'api-availability'
|
|
4035
|
+
? 'api-health'
|
|
4036
|
+
: 'web-critical-route',
|
|
4037
|
+
},
|
|
4038
|
+
{ Name: 'Slice', Value: '${name}' },
|
|
4039
|
+
{ Name: 'Stage', Value: plan.target.stage },
|
|
4040
|
+
];
|
|
4041
|
+
const syntheticQueries =
|
|
4042
|
+
objective.lowTraffic.mode === 'synthetic'
|
|
4043
|
+
? [
|
|
4044
|
+
metric('synthetic_valid', 'SyntheticValid', syntheticDimensions),
|
|
4045
|
+
metric('synthetic_good', 'SyntheticGood', syntheticDimensions),
|
|
4046
|
+
]
|
|
4047
|
+
: [];
|
|
4048
|
+
let metricDataQueries;
|
|
4049
|
+
if (objective.id === 'api-availability') {
|
|
4050
|
+
const apiId = safeToken(query.apiId, 'apiId', 128);
|
|
4051
|
+
if (apiId !== activeResources.apiId) {
|
|
4052
|
+
throw new Error(
|
|
4053
|
+
'Metric query API is not authorized by the protected deployment.',
|
|
4054
|
+
);
|
|
4055
|
+
}
|
|
4056
|
+
const apiDimensions = [{ Name: 'ApiId', Value: apiId }];
|
|
4057
|
+
metricDataQueries = [
|
|
4058
|
+
metric('valid', 'Count', apiDimensions),
|
|
4059
|
+
{ ...metric('bad', '5xx', apiDimensions), ReturnData: false },
|
|
4060
|
+
{
|
|
4061
|
+
Expression: 'valid-bad',
|
|
4062
|
+
Id: 'good',
|
|
4063
|
+
Label: 'Good API events',
|
|
4064
|
+
ReturnData: true,
|
|
4065
|
+
},
|
|
4066
|
+
...syntheticQueries,
|
|
4067
|
+
];
|
|
4068
|
+
} else if (objective.id === 'web-latency') {
|
|
4069
|
+
if (query.apiId !== undefined) {
|
|
4070
|
+
throw new Error('Web metric capture must not accept an API identity.');
|
|
4071
|
+
}
|
|
4072
|
+
metricDataQueries = [
|
|
4073
|
+
metric('valid', 'OperationValid'),
|
|
4074
|
+
metric('good', 'OperationGood'),
|
|
4075
|
+
...syntheticQueries,
|
|
4076
|
+
];
|
|
4077
|
+
} else {
|
|
4078
|
+
throw new Error(
|
|
4079
|
+
'This objective has no stable-v1 AWS metric capture adapter.',
|
|
4080
|
+
);
|
|
4081
|
+
}
|
|
4082
|
+
const response = await new CloudWatchClient({ region }).send(
|
|
4083
|
+
new GetMetricDataCommand({
|
|
4084
|
+
EndTime: new Date(endTime),
|
|
4085
|
+
MaxDatapoints: 1008,
|
|
4086
|
+
MetricDataQueries: metricDataQueries,
|
|
4087
|
+
ScanBy: 'TimestampDescending',
|
|
4088
|
+
StartTime: new Date(startTime),
|
|
4089
|
+
}),
|
|
4090
|
+
);
|
|
4091
|
+
if (response.NextToken) {
|
|
4092
|
+
throw new Error('Metric query exceeded its one-page data-point bound.');
|
|
4093
|
+
}
|
|
4094
|
+
const idMap = {
|
|
4095
|
+
good: 'good',
|
|
4096
|
+
synthetic_good: 'synthetic-good',
|
|
4097
|
+
synthetic_valid: 'synthetic-valid',
|
|
4098
|
+
valid: 'valid',
|
|
4099
|
+
};
|
|
4100
|
+
const metricResults = (response.MetricDataResults ?? [])
|
|
4101
|
+
.filter(({ Id }) => idMap[Id])
|
|
4102
|
+
.map((result) => ({
|
|
4103
|
+
id: idMap[result.Id],
|
|
4104
|
+
status: result.StatusCode ?? null,
|
|
4105
|
+
timestamps: (result.Timestamps ?? []).map((value) => value.toISOString()),
|
|
4106
|
+
values: result.Values ?? [],
|
|
4107
|
+
}));
|
|
4108
|
+
if (
|
|
4109
|
+
metricResults.length !==
|
|
4110
|
+
(objective.lowTraffic.mode === 'synthetic' ? 4 : 2) ||
|
|
4111
|
+
new Set(metricResults.map(({ id }) => id)).size !== metricResults.length ||
|
|
4112
|
+
!metricResults.some(({ id }) => id === 'valid') ||
|
|
4113
|
+
!metricResults.some(({ id }) => id === 'good') ||
|
|
4114
|
+
metricResults.some(
|
|
4115
|
+
({ status, timestamps, values }) =>
|
|
4116
|
+
status !== 'Complete' ||
|
|
4117
|
+
timestamps.length !== values.length ||
|
|
4118
|
+
values.length > 1008 ||
|
|
4119
|
+
timestamps.some(
|
|
4120
|
+
(value) =>
|
|
4121
|
+
!Number.isFinite(Date.parse(value)) ||
|
|
4122
|
+
Date.parse(value) < Date.parse(startTime) ||
|
|
4123
|
+
Date.parse(value) > Date.parse(endTime),
|
|
4124
|
+
) ||
|
|
4125
|
+
values.some((value) => !Number.isFinite(value) || value < 0),
|
|
4126
|
+
)
|
|
4127
|
+
) {
|
|
4128
|
+
throw new Error('AWS metric response is incomplete or outside its bound.');
|
|
4129
|
+
}
|
|
4130
|
+
const queryDigest = sha256(
|
|
4131
|
+
Buffer.from(JSON.stringify({
|
|
4132
|
+
endTime,
|
|
4133
|
+
metricDataQueries,
|
|
4134
|
+
startTime,
|
|
4135
|
+
target: plan.target,
|
|
4136
|
+
})),
|
|
4137
|
+
);
|
|
4138
|
+
await capture(
|
|
4139
|
+
{
|
|
4140
|
+
binding: capturedBinding,
|
|
4141
|
+
bindingArtifact: {
|
|
4142
|
+
digest: activeBinding.digest,
|
|
4143
|
+
path: activeBinding.path,
|
|
4144
|
+
},
|
|
4145
|
+
capturedAt: new Date().toISOString(),
|
|
4146
|
+
kind: 'reliability-metric-capture',
|
|
4147
|
+
metricResults,
|
|
4148
|
+
objectiveId,
|
|
4149
|
+
queryDigest,
|
|
4150
|
+
sampleWindow: {
|
|
4151
|
+
durationSeconds: elapsed / 1_000,
|
|
4152
|
+
end: endTime,
|
|
4153
|
+
start: startTime,
|
|
4154
|
+
},
|
|
4155
|
+
schemaVersion: 1,
|
|
4156
|
+
slice: '${name}',
|
|
4157
|
+
target: plan.target,
|
|
4158
|
+
},
|
|
4159
|
+
'.ebk/evidence/reliability/captures',
|
|
4160
|
+
);
|
|
4161
|
+
process.exit(0);
|
|
4162
|
+
}
|
|
4163
|
+
if (plan.source === 'resource-metadata') {
|
|
4164
|
+
const query = exactKeys(
|
|
4165
|
+
request.query,
|
|
4166
|
+
['ecsServices', 'executions'],
|
|
4167
|
+
'resource-metadata query',
|
|
4168
|
+
);
|
|
4169
|
+
const ecsServices = query.ecsServices ?? [];
|
|
4170
|
+
const executions = query.executions ?? [];
|
|
4171
|
+
if (
|
|
4172
|
+
!Array.isArray(ecsServices) ||
|
|
4173
|
+
!Array.isArray(executions) ||
|
|
4174
|
+
ecsServices.length > 8 ||
|
|
4175
|
+
executions.length > 8 ||
|
|
4176
|
+
ecsServices.length + executions.length < 1
|
|
4177
|
+
) {
|
|
4178
|
+
throw new Error('Resource query is empty or exceeds eight targets per kind.');
|
|
4179
|
+
}
|
|
4180
|
+
const resources = [];
|
|
4181
|
+
for (const [index, input] of ecsServices.entries()) {
|
|
4182
|
+
const target = exactKeys(input, ['cluster', 'services'], 'ecsServices[' + index + ']');
|
|
4183
|
+
const cluster = safeToken(target.cluster, 'ecsServices.cluster', 512);
|
|
4184
|
+
if (
|
|
4185
|
+
!Array.isArray(target.services) ||
|
|
4186
|
+
target.services.length < 1 ||
|
|
4187
|
+
target.services.length > 10
|
|
4188
|
+
) {
|
|
4189
|
+
throw new Error('ECS service query is outside its ten-service bound.');
|
|
4190
|
+
}
|
|
4191
|
+
const services = target.services.map((value) =>
|
|
4192
|
+
safeToken(value, 'ecsServices.service', 255),
|
|
4193
|
+
);
|
|
4194
|
+
const response = await new ECSClient({ region }).send(
|
|
4195
|
+
new DescribeServicesCommand({ cluster, services }),
|
|
4196
|
+
);
|
|
4197
|
+
if ((response.failures?.length ?? 0) > 0) {
|
|
4198
|
+
throw new Error('ECS returned an unresolved reliability target.');
|
|
4199
|
+
}
|
|
4200
|
+
if (
|
|
4201
|
+
response.services?.length !== services.length ||
|
|
4202
|
+
new Set(
|
|
4203
|
+
(response.services ?? []).map(({ serviceArn }) =>
|
|
4204
|
+
serviceArn?.slice(serviceArn.lastIndexOf('/') + 1),
|
|
4205
|
+
),
|
|
4206
|
+
).size !== services.length ||
|
|
4207
|
+
response.services.some(
|
|
4208
|
+
({ serviceArn }) =>
|
|
4209
|
+
!services.includes(serviceArn?.slice(serviceArn.lastIndexOf('/') + 1)),
|
|
4210
|
+
)
|
|
4211
|
+
) {
|
|
4212
|
+
throw new Error('ECS response escaped the exact diagnostic target.');
|
|
4213
|
+
}
|
|
4214
|
+
resources.push(...response.services.map((service) => ({
|
|
4215
|
+
desiredCount: service.desiredCount ?? null,
|
|
4216
|
+
pendingCount: service.pendingCount ?? null,
|
|
4217
|
+
runningCount: service.runningCount ?? null,
|
|
4218
|
+
serviceArn: service.serviceArn ?? null,
|
|
4219
|
+
status: service.status ?? null,
|
|
4220
|
+
})));
|
|
4221
|
+
}
|
|
4222
|
+
for (const [index, value] of executions.entries()) {
|
|
4223
|
+
const executionArn = safeToken(value, 'executions[' + index + ']', 512);
|
|
4224
|
+
const response = await new SFNClient({ region }).send(
|
|
4225
|
+
new DescribeExecutionCommand({
|
|
4226
|
+
executionArn,
|
|
4227
|
+
includedData: 'METADATA_ONLY',
|
|
4228
|
+
}),
|
|
4229
|
+
);
|
|
4230
|
+
if (
|
|
4231
|
+
response.executionArn !== executionArn ||
|
|
4232
|
+
!activeResources.stateMachineArns.includes(response.stateMachineArn)
|
|
4233
|
+
) {
|
|
4234
|
+
throw new Error(
|
|
4235
|
+
'Step Functions response escaped the exact diagnostic target.',
|
|
4236
|
+
);
|
|
4237
|
+
}
|
|
4238
|
+
resources.push({
|
|
4239
|
+
executionArn: response.executionArn ?? null,
|
|
4240
|
+
stateMachineArn: response.stateMachineArn ?? null,
|
|
4241
|
+
status: response.status ?? null,
|
|
4242
|
+
stopDate: response.stopDate?.toISOString?.() ?? null,
|
|
4243
|
+
});
|
|
4244
|
+
}
|
|
4245
|
+
await capture(
|
|
4246
|
+
{
|
|
4247
|
+
binding: capturedBinding,
|
|
4248
|
+
bindingDigest: activeBinding.digest,
|
|
4249
|
+
capturedAt: new Date().toISOString(),
|
|
4250
|
+
kind: 'reliability-resource-capture',
|
|
4251
|
+
resources,
|
|
4252
|
+
schemaVersion: 1,
|
|
4253
|
+
slice: '${name}',
|
|
4254
|
+
target: plan.target,
|
|
4255
|
+
},
|
|
4256
|
+
'.ebk/evidence/reliability/diagnostics',
|
|
4257
|
+
);
|
|
4258
|
+
process.exit(0);
|
|
4259
|
+
}
|
|
4260
|
+
throw new Error(
|
|
4261
|
+
'This diagnostic source has no stable-v1 read-only AWS query adapter.',
|
|
4262
|
+
);
|
|
4263
|
+
`,
|
|
4264
|
+
'scripts/incident-remediation.mjs': `import { spawnSync } from 'node:child_process';
|
|
4265
|
+
import {
|
|
4266
|
+
planRemediation,
|
|
4267
|
+
} from '@empire-builder-kit/runtime/reliability';
|
|
4268
|
+
import {
|
|
4269
|
+
readBoundedJson,
|
|
4270
|
+
} from './reliability-evidence-lib.mjs';
|
|
4271
|
+
|
|
4272
|
+
const inputPath = process.env.EBK_INCIDENT_REMEDIATION_INPUT_PATH?.trim();
|
|
4273
|
+
if (!inputPath) {
|
|
4274
|
+
throw new Error('EBK_INCIDENT_REMEDIATION_INPUT_PATH is required.');
|
|
4275
|
+
}
|
|
4276
|
+
const request = readBoundedJson(
|
|
4277
|
+
inputPath,
|
|
4278
|
+
'Incident remediation input',
|
|
4279
|
+
).value;
|
|
4280
|
+
const plan = planRemediation(request);
|
|
4281
|
+
if (plan.mode === 'report') {
|
|
4282
|
+
process.stdout.write(JSON.stringify({
|
|
4283
|
+
action: plan.action,
|
|
4284
|
+
approvalRequired: true,
|
|
4285
|
+
authorized: false,
|
|
4286
|
+
execution: plan.execution,
|
|
4287
|
+
remediationPlanDigest: plan.remediationPlanDigest,
|
|
4288
|
+
target: plan.target,
|
|
4289
|
+
}) + '\\n');
|
|
4290
|
+
process.exit(0);
|
|
4291
|
+
}
|
|
4292
|
+
const required = (name) => {
|
|
4293
|
+
const value = process.env[name]?.trim();
|
|
4294
|
+
if (!value) throw new Error(name + ' is required.');
|
|
4295
|
+
return value;
|
|
4296
|
+
};
|
|
4297
|
+
if (
|
|
4298
|
+
process.env.GITHUB_ACTIONS !== 'true' ||
|
|
4299
|
+
process.env.GITHUB_REF_PROTECTED !== 'true' ||
|
|
4300
|
+
process.env.GITHUB_SHA !== process.env.EBK_APPROVED_COMMIT ||
|
|
4301
|
+
!process.env.ACTIONS_ID_TOKEN_REQUEST_URL
|
|
4302
|
+
) {
|
|
4303
|
+
throw new Error(
|
|
4304
|
+
'Incident remediation apply requires protected GitHub OIDC on the approved commit.',
|
|
4305
|
+
);
|
|
4306
|
+
}
|
|
4307
|
+
if (
|
|
4308
|
+
plan.target.account !== required('AWS_EXPECTED_ACCOUNT_ID') ||
|
|
4309
|
+
plan.target.region !== required('AWS_REGION') ||
|
|
4310
|
+
plan.target.stage !== required('EBK_STAGE') ||
|
|
4311
|
+
plan.target.slice !== '${name}'
|
|
4312
|
+
) {
|
|
4313
|
+
throw new Error(
|
|
4314
|
+
'Incident remediation approval does not match the effective target.',
|
|
4315
|
+
);
|
|
4316
|
+
}
|
|
4317
|
+
if (
|
|
4318
|
+
!['redrive', 'replay'].includes(plan.action) ||
|
|
4319
|
+
plan.execution.adapter !== 'protected-async-recovery'
|
|
4320
|
+
) {
|
|
4321
|
+
throw new Error(
|
|
4322
|
+
'This remediation action has no stable-v1 executable adapter.',
|
|
4323
|
+
);
|
|
4324
|
+
}
|
|
4325
|
+
if (
|
|
4326
|
+
plan.execution.asyncRecoveryReportDigest !==
|
|
4327
|
+
required('EBK_ASYNC_REPORT_DIGEST')
|
|
4328
|
+
) {
|
|
4329
|
+
throw new Error(
|
|
4330
|
+
'Incident remediation does not match the distinct async-recovery report.',
|
|
4331
|
+
);
|
|
4332
|
+
}
|
|
4333
|
+
const capability =
|
|
4334
|
+
plan.action === 'redrive'
|
|
4335
|
+
? 'bounded-selective-redrive'
|
|
4336
|
+
: 'event-replay';
|
|
4337
|
+
const result = spawnSync(
|
|
4338
|
+
process.execPath,
|
|
4339
|
+
['scripts/protected-async-recovery.mjs'],
|
|
4340
|
+
{
|
|
4341
|
+
env: {
|
|
4342
|
+
...process.env,
|
|
4343
|
+
EBK_ASYNC_CAPABILITY: capability,
|
|
4344
|
+
EBK_RECOVERY_ACTION: 'apply',
|
|
4345
|
+
},
|
|
4346
|
+
stdio: 'inherit',
|
|
4347
|
+
},
|
|
4348
|
+
);
|
|
4349
|
+
if (result.error) throw result.error;
|
|
4350
|
+
if (result.status !== 0) process.exit(result.status ?? 1);
|
|
4351
|
+
`,
|
|
4352
|
+
'reliability/contract.spec.ts': `import { describe, expect, it } from 'vitest';
|
|
4353
|
+
import { createAuthenticatedSession } from '@empire-builder-kit/runtime/auth';
|
|
4354
|
+
import {
|
|
4355
|
+
createApprovalRequest,
|
|
4356
|
+
createDirectAuthorizationPrincipal,
|
|
4357
|
+
decideApprovalRequest,
|
|
4358
|
+
} from '@empire-builder-kit/runtime/authorization';
|
|
4359
|
+
import {
|
|
4360
|
+
authorizeDiagnostic,
|
|
4361
|
+
candidateIncidentSeverity,
|
|
4362
|
+
diagnosticApprovalDigestInput,
|
|
4363
|
+
evaluateSlo,
|
|
4364
|
+
planRemediation,
|
|
4365
|
+
RELIABILITY_APPROVAL_POLICY_VERSION,
|
|
4366
|
+
remediationApprovalDigestInput,
|
|
4367
|
+
} from '@empire-builder-kit/runtime/reliability';
|
|
4368
|
+
import { Reliability } from './contract.js';
|
|
4369
|
+
|
|
4370
|
+
const human = (id: string) =>
|
|
4371
|
+
createDirectAuthorizationPrincipal({
|
|
4372
|
+
id,
|
|
4373
|
+
kind: 'admin',
|
|
4374
|
+
roles: [],
|
|
4375
|
+
scopes: [],
|
|
4376
|
+
session: createAuthenticatedSession({
|
|
4377
|
+
subject: { id, type: 'user' },
|
|
4378
|
+
}),
|
|
4379
|
+
});
|
|
4380
|
+
const approvedAuthorization = <Action extends string>(
|
|
4381
|
+
input: {
|
|
4382
|
+
action: Action;
|
|
4383
|
+
command: unknown;
|
|
4384
|
+
resource: { id?: string; type: string };
|
|
4385
|
+
},
|
|
4386
|
+
) => {
|
|
4387
|
+
const now = Date.now();
|
|
4388
|
+
const pending = createApprovalRequest({
|
|
4389
|
+
...input,
|
|
4390
|
+
author: human('human-operator'),
|
|
4391
|
+
expiresAt: new Date(now + 30 * 60 * 1_000).toISOString(),
|
|
4392
|
+
now: new Date(now - 60 * 1_000).toISOString(),
|
|
4393
|
+
policyVersion: RELIABILITY_APPROVAL_POLICY_VERSION,
|
|
4394
|
+
slice: '${name}',
|
|
4395
|
+
});
|
|
4396
|
+
return decideApprovalRequest(pending, {
|
|
4397
|
+
approver: human('human-reviewer'),
|
|
4398
|
+
expectedRequestDigest: pending.requestDigest,
|
|
4399
|
+
now: new Date(now - 30 * 1_000).toISOString(),
|
|
4400
|
+
outcome: 'approved',
|
|
4401
|
+
reasonReference: 'incident-1-review',
|
|
4402
|
+
requiredPolicyVersion: RELIABILITY_APPROVAL_POLICY_VERSION,
|
|
4403
|
+
}).request;
|
|
4404
|
+
};
|
|
4405
|
+
|
|
4406
|
+
describe('${name} reliability contract', () => {
|
|
4407
|
+
it('binds owned objectives and never auto-declares SEV-1', () => {
|
|
4408
|
+
expect(Reliability.slice).toBe('${name}');
|
|
4409
|
+
expect(Reliability.objectives.map(({ id }) => id)).toEqual([
|
|
4410
|
+
'api-availability',
|
|
4411
|
+
'web-latency',
|
|
4412
|
+
'async-delivery-deadline',
|
|
4413
|
+
]);
|
|
4414
|
+
expect(candidateIncidentSeverity('critical')).toBe('SEV-2');
|
|
4415
|
+
});
|
|
4416
|
+
|
|
4417
|
+
it('makes low-traffic data sufficiency explicit', () => {
|
|
4418
|
+
const objective = Reliability.objectives[0]!;
|
|
4419
|
+
expect(
|
|
4420
|
+
evaluateSlo({
|
|
4421
|
+
goodEvents: 1,
|
|
4422
|
+
objective,
|
|
4423
|
+
validEvents: 1,
|
|
4424
|
+
}).state,
|
|
4425
|
+
).toBe('data-insufficient');
|
|
4426
|
+
});
|
|
4427
|
+
|
|
4428
|
+
it('rejects cross-target diagnostic and remediation approvals', () => {
|
|
4429
|
+
const target = {
|
|
4430
|
+
account: '123456789012',
|
|
4431
|
+
region: 'us-east-1',
|
|
4432
|
+
slice: '${name}',
|
|
4433
|
+
stage: 'production',
|
|
4434
|
+
};
|
|
4435
|
+
const approvalScope = {
|
|
4436
|
+
allowedLogGroups: ['/ebk/production/${name}/api-health'],
|
|
4437
|
+
incidentRef: 'incident-1',
|
|
4438
|
+
queryClass: 'scoped-errors',
|
|
4439
|
+
reason: 'Confirm the bounded failure signature.',
|
|
4440
|
+
target: { ...target, slice: 'other-slice' },
|
|
4441
|
+
windowEnd: '2026-07-29T12:00:00.000Z',
|
|
4442
|
+
windowStart: '2026-07-29T11:30:00.000Z',
|
|
4443
|
+
};
|
|
4444
|
+
const approval = {
|
|
4445
|
+
...approvalScope,
|
|
4446
|
+
authorization: approvedAuthorization(
|
|
4447
|
+
diagnosticApprovalDigestInput(approvalScope),
|
|
4448
|
+
),
|
|
4449
|
+
};
|
|
4450
|
+
expect(() =>
|
|
4451
|
+
authorizeDiagnostic({
|
|
4452
|
+
approval,
|
|
4453
|
+
source: 'bounded-log-query',
|
|
4454
|
+
target,
|
|
4455
|
+
}),
|
|
4456
|
+
).toThrow('does not match');
|
|
4457
|
+
const report = planRemediation({
|
|
4458
|
+
action: 'replay',
|
|
4459
|
+
cleanup: ['Remove test resources.'],
|
|
4460
|
+
execution: {
|
|
4461
|
+
adapter: 'protected-async-recovery',
|
|
4462
|
+
asyncRecoveryReportDigest: 'a'.repeat(64),
|
|
4463
|
+
},
|
|
4464
|
+
mode: 'report',
|
|
4465
|
+
preconditions: ['The target is still active.'],
|
|
4466
|
+
stopConditions: ['Stop on target change.'],
|
|
4467
|
+
target,
|
|
4468
|
+
verification: ['Verify the critical journey.'],
|
|
4469
|
+
});
|
|
4470
|
+
expect(() =>
|
|
4471
|
+
planRemediation({
|
|
4472
|
+
action: 'replay',
|
|
4473
|
+
approval: {
|
|
4474
|
+
authorization: approvedAuthorization(
|
|
4475
|
+
remediationApprovalDigestInput({
|
|
4476
|
+
incidentRef: 'incident-1',
|
|
4477
|
+
plan: report,
|
|
4478
|
+
}),
|
|
4479
|
+
),
|
|
4480
|
+
incidentRef: 'incident-1',
|
|
4481
|
+
},
|
|
4482
|
+
cleanup: ['Remove test resources.'],
|
|
4483
|
+
execution: {
|
|
4484
|
+
adapter: 'protected-async-recovery',
|
|
4485
|
+
asyncRecoveryReportDigest: 'a'.repeat(64),
|
|
4486
|
+
},
|
|
4487
|
+
mode: 'apply',
|
|
4488
|
+
preconditions: ['The target is still active.'],
|
|
4489
|
+
stopConditions: ['Stop on target change.'],
|
|
4490
|
+
target,
|
|
4491
|
+
verification: ['Verify a substituted journey.'],
|
|
4492
|
+
}),
|
|
4493
|
+
).toThrow('does not match');
|
|
4494
|
+
});
|
|
4495
|
+
});
|
|
4496
|
+
`,
|
|
4497
|
+
};
|
|
4498
|
+
}
|
|
4499
|
+
//# sourceMappingURL=built-in-plan-reliability-files.js.map
|