@empire-builder-kit/nx 1.0.0-rc.33 → 1.0.0-rc.35

package/src/executors/router-waf-rehearsal/router-waf-rehearsal.js

@@ -5,6 +5,7 @@ const client_cloudwatch_1 = require("@aws-sdk/client-cloudwatch");
 const client_cloudwatch_logs_1 = require("@aws-sdk/client-cloudwatch-logs");
 const client_cloudfront_1 = require("@aws-sdk/client-cloudfront");
 const client_sts_1 = require("@aws-sdk/client-sts");
+const node_https_1 = require("node:https");
 const client_wafv2_1 = require("@aws-sdk/client-wafv2");
 const credential_providers_1 = require("@aws-sdk/credential-providers");
 const node_crypto_1 = require("node:crypto");
@@ -14,6 +15,7 @@ const devkit_1 = require("@nx/devkit");
 const index_js_1 = require("../../foundation/index.js");
 const router_waf_rate_rehearsal_js_1 = require("../../foundation/router-waf-rate-rehearsal.js");
 const index_js_2 = require("../../aws-authority/index.js");
+const private_smoke_access_js_1 = require("../../foundation/private-smoke-access.js");
 const identity_js_1 = require("../../aws/identity.js");
 const index_js_3 = require("../../repository-policy/index.js");
 const deterministic_order_js_1 = require("../../deterministic-order.js");
@@ -772,6 +774,83 @@ function mergeRateEvidence(evidence, allowedRequestStatus, rateLimit, completedA
         probes: { ...probes, allowedRequestStatus, rateLimit },
     };
 }
+const SEED_ATTESTATION_RETRIES = 12;
+const SEED_ATTESTATION_INTERVAL_MS = 10_000;
+function rawGet(url, headers) {
+    return new Promise((resolve, reject) => {
+        const request = (0, node_https_1.request)(url, { headers, method: 'GET', timeout: REQUEST_TIMEOUT_MS }, (response) => {
+            response.resume();
+            resolve({ status: response.statusCode ?? 0 });
+        });
+        request.on('timeout', () => request.destroy(new Error('timeout')));
+        request.on('error', reject);
+        request.end();
+    });
+}
+async function attestOperatorBlocks(input) {
+    // AWS WAF publishes sampled requests with a short delay; a seeded run
+    // probed moments ago, so its attestation waits a bounded number of times.
+    for (let attempt = 0;; attempt += 1) {
+        try {
+            await Promise.all([
+                attestOperatorBlock({
+                    evidence: input.seed.probes
+                        .managedRule,
+                    stage: input.stage,
+                    type: 'managed',
+                    waf: input.waf,
+                    webAclArn: input.seed.webAclArn,
+                }),
+                attestOperatorBlock({
+                    evidence: input.seed.probes
+                        .nonAllowlisted,
+                    stage: input.stage,
+                    type: 'nonallowlisted',
+                    waf: input.waf,
+                    webAclArn: input.seed.webAclArn,
+                }),
+            ]);
+            return;
+        }
+        catch (error) {
+            if (attempt >= input.retries)
+                throw error;
+            await input.sleep(SEED_ATTESTATION_INTERVAL_MS);
+        }
+    }
+}
+async function discoverRouterDistribution(input) {
+    const items = [];
+    let marker;
+    const seen = new Set();
+    do {
+        const page = await input.cloudFront.send(new client_cloudfront_1.ListDistributionsCommand({ Marker: marker }));
+        items.push(...(page.DistributionList?.Items ?? []));
+        marker = page.DistributionList?.NextMarker;
+        if (marker && seen.has(marker)) {
+            throw new Error('Router inventory pagination did not advance.');
+        }
+        if (marker)
+            seen.add(marker);
+        if (seen.size > 100) {
+            throw new Error('Router inventory exceeds bounded inspection.');
+        }
+    } while (marker);
+    const routers = items.filter((router) => router.DomainName === input.probeHost ||
+        router.Aliases?.Items?.includes(input.probeHost));
+    const router = routers[0];
+    if (routers.length !== 1 ||
+        !router?.Enabled ||
+        typeof router.Id !== 'string' ||
+        router.ARN !==
+            `arn:${input.partition}:cloudfront::${input.accountId}:distribution/${router.Id}` ||
+        typeof router.WebACLId !== 'string' ||
+        !WAF_ARN.test(router.WebACLId) ||
+        WAF_ARN.exec(router.WebACLId)?.[2] !== input.accountId) {
+        throw new Error('Managed rehearsal probe host does not resolve to one exact enabled Router distribution with an account-owned Web ACL.');
+    }
+    return { distributionId: router.Id, webAclArn: router.WebACLId };
+}
 function createRouterWafRehearsalExecutor(dependencies = {}) {
     return async (options, context) => {
         try {
@@ -812,160 +891,267 @@ function createRouterWafRehearsalExecutor(dependencies = {}) {
                 stage,
             });
             const evidencePath = (0, node_path_1.join)(root, EVIDENCE_PATH);
-            const retainedEvidence = readBoundedJson(evidencePath, 'Router WAF live evidence');
-            const seed = (0, index_js_1.validateRouterWafLiveEvidenceSeed)({
-                evidence: retainedEvidence,
-                policy,
-            });
-            if (seed.stage !== stage ||
-                seed.accountId !== accountId ||
-                seed.region !== region ||
-                seed.deploymentRoleArn !== expectedRoleArn ||
-                seed.workspace !== workspace ||
-                seed.wafContractDigest !== wafContractDigest) {
-                throw new Error('Router WAF live evidence stage, account, Region, deployment role, workspace, or WAF contract differs from committed authority.');
-            }
-            const invocationStartedAt = (dependencies.now ?? (() => new Date()))();
-            if (!Number.isFinite(invocationStartedAt.getTime()) ||
-                seed.startedAt > invocationStartedAt.getTime() ||
-                seed.completedAt > invocationStartedAt.getTime() ||
-                invocationStartedAt.getTime() - seed.completedAt >
-                    MAX_ROUTER_WAF_SEED_AGE_MS) {
-                throw new Error('Router WAF live evidence seed is stale or future-dated; no live request was made.');
-            }
-            const receiptPath = (0, node_path_1.join)(root, RECEIPT_PATH);
-            const retainedReceipt = (0, index_js_1.normalizeRouterWafBlockingRehearsal)(readBoundedJson(receiptPath, 'Router WAF blocking rehearsal receipt'));
-            if (retainedReceipt) {
-                throw new Error('Router WAF blocking rehearsal receipt must be reset to its exact pending seed before another live probe.');
-            }
             const url = normalizedProbeUrl(options.probeUrl);
             const profile = selectedProfile(options, environment);
             const aws = clients(profile, region, dependencies);
+            const seedProbes = options.seedProbes === true;
+            const sleep = dependencies.sleep ??
+                ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)));
+            const seal = async () => {
+                const retainedEvidence = readBoundedJson(evidencePath, 'Router WAF live evidence');
+                const seed = (0, index_js_1.validateRouterWafLiveEvidenceSeed)({
+                    evidence: retainedEvidence,
+                    policy,
+                });
+                if (seed.stage !== stage ||
+                    seed.accountId !== accountId ||
+                    seed.region !== region ||
+                    seed.deploymentRoleArn !== expectedRoleArn ||
+                    seed.workspace !== workspace ||
+                    seed.wafContractDigest !== wafContractDigest) {
+                    throw new Error('Router WAF live evidence stage, account, Region, deployment role, workspace, or WAF contract differs from committed authority.');
+                }
+                const invocationStartedAt = (dependencies.now ?? (() => new Date()))();
+                if (!Number.isFinite(invocationStartedAt.getTime()) ||
+                    seed.startedAt > invocationStartedAt.getTime() ||
+                    seed.completedAt > invocationStartedAt.getTime() ||
+                    invocationStartedAt.getTime() - seed.completedAt >
+                        MAX_ROUTER_WAF_SEED_AGE_MS) {
+                    throw new Error('Router WAF live evidence seed is stale or future-dated; no live request was made.');
+                }
+                const receiptPath = (0, node_path_1.join)(root, RECEIPT_PATH);
+                const retainedReceipt = (0, index_js_1.normalizeRouterWafBlockingRehearsal)(readBoundedJson(receiptPath, 'Router WAF blocking rehearsal receipt'));
+                if (retainedReceipt) {
+                    throw new Error('Router WAF blocking rehearsal receipt must be reset to its exact pending seed before another live probe.');
+                }
+                const identity = await aws.sts.send(new client_sts_1.GetCallerIdentityCommand({}));
+                if (identity.Account !== accountId ||
+                    typeof identity.Arn !== 'string' ||
+                    !(0, identity_js_1.matchesExpectedAssumedRoleArn)(accountId, identity.Arn, expectedRoleArn)) {
+                    throw new Error('Router WAF rehearsal caller does not match the committed account and exact stage deployment role.');
+                }
+                await (dependencies.ownership ?? index_js_4.assertWorkspaceStageOwnershipForWorkspace)(context.root, stage, profile ? { profile } : {});
+                const webAclVisibilityMetricName = (0, router_waf_rate_rehearsal_js_1.routerWafWebAclMetricName)(stage);
+                const [, webAclMetricName] = await Promise.all([
+                    attestCloudFrontDistribution({
+                        accountId,
+                        cloudFront: aws.cloudFront,
+                        distributionId: seed.distributionId,
+                        partition,
+                        probeHost: url.hostname,
+                        stage,
+                        webAclArn: seed.webAclArn,
+                        workspace,
+                    }),
+                    attestRouterWafBaseline({
+                        accountId,
+                        cloudWatchLogs: aws.cloudWatchLogs,
+                        plan,
+                        policyDigest: (0, index_js_1.routerWafPolicyDigest)(policy),
+                        stage,
+                        waf: aws.waf,
+                        webAclArn: seed.webAclArn,
+                        webAclMetricName: webAclVisibilityMetricName,
+                        workspace,
+                    }),
+                ]);
+                await attestOperatorBlocks({
+                    retries: seedProbes ? SEED_ATTESTATION_RETRIES : 0,
+                    seed,
+                    sleep,
+                    stage,
+                    waf: aws.waf,
+                });
+                const fetchRequest = dependencies.fetch ?? fetch;
+                const baselineResponse = await fetchRequest(url, {
+                    cache: 'no-store',
+                    headers: { 'cache-control': 'no-cache' },
+                    redirect: 'manual',
+                    signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+                });
+                const allowedRequestStatus = baselineResponse.status;
+                await baselineResponse.body?.cancel();
+                if (allowedRequestStatus < 200 || allowedRequestStatus > 399) {
+                    throw new Error(`Router WAF rehearsal health preflight returned unexpected HTTP ${allowedRequestStatus}; rate traffic was not sent.`);
+                }
+                const nonce = (dependencies.createNonce ?? (() => (0, node_crypto_1.randomBytes)(32).toString('hex')))();
+                if (!RATE_MARKER_NONCE_PATTERN.test(nonce)) {
+                    throw new Error('Router WAF rehearsal invocation nonce is not one opaque 256-bit lowercase hexadecimal value.');
+                }
+                const rateLimit = await (dependencies.runRateRehearsal ?? router_waf_rate_rehearsal_js_1.runRouterWafRateRehearsal)({
+                    dependencies: {
+                        ...(dependencies.now ? { now: dependencies.now } : {}),
+                        observe: ({ endTime, markerDigest: expectedMarkerDigest, ruleMetricName, sequence, startTime, webAclMetricName: expectedWebAclMetricName, }) => {
+                            const marker = rateMarker(nonce, sequence);
+                            if (expectedWebAclMetricName !== webAclMetricName) {
+                                throw new Error('Router WAF rate observation metric dimension differs from the attested physical Web ACL name.');
+                            }
+                            if (markerDigest(marker) !== expectedMarkerDigest) {
+                                throw new Error('Router WAF rate observation marker differs from the exact blocked request.');
+                            }
+                            return observeRateLimit({
+                                cloudWatch: aws.cloudWatch,
+                                endTime,
+                                marker,
+                                markerDigest: expectedMarkerDigest,
+                                probePath: url.pathname,
+                                ruleMetricName,
+                                startTime,
+                                waf: aws.waf,
+                                webAclArn: seed.webAclArn,
+                                webAclMetricName: expectedWebAclMetricName,
+                            });
+                        },
+                        request: async (sequence) => {
+                            const marker = rateMarker(nonce, sequence);
+                            const response = await fetchRequest(url, {
+                                cache: 'no-store',
+                                headers: {
+                                    'cache-control': 'no-cache',
+                                    [RATE_MARKER_HEADER]: marker,
+                                },
+                                redirect: 'manual',
+                                signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+                            });
+                            await response.body?.cancel();
+                            return {
+                                markerDigest: markerDigest(marker),
+                                status: response.status,
+                            };
+                        },
+                        ...(dependencies.sleep ? { sleep: dependencies.sleep } : {}),
+                    },
+                    rateLimitPerIp: policy.rateLimitPerIp,
+                    stage,
+                    webAclMetricName,
+                });
+                const completedAt = (dependencies.now ?? (() => new Date()))().toISOString();
+                const evidence = mergeRateEvidence(retainedEvidence, allowedRequestStatus, rateLimit, completedAt);
+                const receipt = (0, index_js_1.createRouterWafBlockingRehearsal)({ evidence, policy });
+                writeBoundedJson(evidencePath, evidence, 'Router WAF live evidence');
+                writeBoundedJson(receiptPath, receipt, 'Router WAF blocking rehearsal receipt');
+                devkit_1.logger.info(JSON.stringify({
+                    evidenceDigest: receipt.evidenceDigest,
+                    accountId: receipt.accountId,
+                    policyDigest: receipt.policyDigest,
+                    rateRequests: rateLimit.requestsSent,
+                    region: receipt.region,
+                    sealed: true,
+                    stage: receipt.stage,
+                }));
+            };
+            if (!seedProbes) {
+                await seal();
+                return { success: true };
+            }
+            // Managed rehearsal: the protected dispatch performs the operator probes
+            // itself. The non-allowlisted probe runs before the runner is admitted;
+            // the managed-rule probe and the rest of the rehearsal run inside the
+            // temporary allowlist admission that private smoke already governs.
+            if (environment.GITHUB_ACTIONS !== 'true' || stage !== 'staging') {
+                throw new Error('Managed Router WAF rehearsal seeding runs only from the protected GitHub staging dispatch.');
+            }
+            const pendingEvidence = readBoundedJson(evidencePath, 'Router WAF live evidence');
+            if (pendingEvidence?.status !== 'pending') {
+                throw new Error('Managed Router WAF rehearsal seeding requires the exact pending live evidence seed; reset both seeds before another rehearsal.');
+            }
+            const sourceCommit = environment.EBK_SOURCE_COMMIT?.trim() ||
+                environment.GITHUB_SHA?.trim() ||
+                '';
+            if (!/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(sourceCommit)) {
+                throw new Error('Managed Router WAF rehearsal seeding requires the exact protected source commit.');
+            }
             const identity = await aws.sts.send(new client_sts_1.GetCallerIdentityCommand({}));
             if (identity.Account !== accountId ||
                 typeof identity.Arn !== 'string' ||
                 !(0, identity_js_1.matchesExpectedAssumedRoleArn)(accountId, identity.Arn, expectedRoleArn)) {
                 throw new Error('Router WAF rehearsal caller does not match the committed account and exact stage deployment role.');
             }
-            await (dependencies.ownership ?? index_js_4.assertWorkspaceStageOwnershipForWorkspace)(context.root, stage, profile ? { profile } : {});
-            const webAclVisibilityMetricName = (0, router_waf_rate_rehearsal_js_1.routerWafWebAclMetricName)(stage);
-            const [, webAclMetricName] = await Promise.all([
-                attestCloudFrontDistribution({
-                    accountId,
-                    cloudFront: aws.cloudFront,
-                    distributionId: seed.distributionId,
-                    partition,
-                    probeHost: url.hostname,
-                    stage,
-                    webAclArn: seed.webAclArn,
-                    workspace,
-                }),
-                attestRouterWafBaseline({
-                    accountId,
-                    cloudWatchLogs: aws.cloudWatchLogs,
-                    plan,
-                    policyDigest: (0, index_js_1.routerWafPolicyDigest)(policy),
-                    stage,
-                    waf: aws.waf,
-                    webAclArn: seed.webAclArn,
-                    webAclMetricName: webAclVisibilityMetricName,
-                    workspace,
-                }),
-            ]);
-            await Promise.all([
-                attestOperatorBlock({
-                    evidence: seed.probes.managedRule,
-                    stage,
-                    type: 'managed',
-                    waf: aws.waf,
-                    webAclArn: seed.webAclArn,
-                }),
-                attestOperatorBlock({
-                    evidence: seed.probes
-                        .nonAllowlisted,
-                    stage,
-                    type: 'nonallowlisted',
-                    waf: aws.waf,
-                    webAclArn: seed.webAclArn,
-                }),
-            ]);
-            const fetchRequest = dependencies.fetch ?? fetch;
-            const baselineResponse = await fetchRequest(url, {
-                cache: 'no-store',
-                headers: { 'cache-control': 'no-cache' },
-                redirect: 'manual',
-                signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+            const discovered = await discoverRouterDistribution({
+                accountId,
+                cloudFront: aws.cloudFront,
+                partition,
+                probeHost: url.hostname,
             });
-            const allowedRequestStatus = baselineResponse.status;
-            await baselineResponse.body?.cancel();
-            if (allowedRequestStatus < 200 || allowedRequestStatus > 399) {
-                throw new Error(`Router WAF rehearsal health preflight returned unexpected HTTP ${allowedRequestStatus}; rate traffic was not sent.`);
-            }
+            const now = dependencies.now ?? (() => new Date());
+            const get = dependencies.rawGet ?? rawGet;
             const nonce = (dependencies.createNonce ?? (() => (0, node_crypto_1.randomBytes)(32).toString('hex')))();
             if (!RATE_MARKER_NONCE_PATTERN.test(nonce)) {
                 throw new Error('Router WAF rehearsal invocation nonce is not one opaque 256-bit lowercase hexadecimal value.');
             }
-            const rateLimit = await (dependencies.runRateRehearsal ?? router_waf_rate_rehearsal_js_1.runRouterWafRateRehearsal)({
-                dependencies: {
-                    ...(dependencies.now ? { now: dependencies.now } : {}),
-                    observe: ({ endTime, markerDigest: expectedMarkerDigest, ruleMetricName, sequence, startTime, webAclMetricName: expectedWebAclMetricName, }) => {
-                        const marker = rateMarker(nonce, sequence);
-                        if (expectedWebAclMetricName !== webAclMetricName) {
-                            throw new Error('Router WAF rate observation metric dimension differs from the attested physical Web ACL name.');
-                        }
-                        if (markerDigest(marker) !== expectedMarkerDigest) {
-                            throw new Error('Router WAF rate observation marker differs from the exact blocked request.');
-                        }
-                        return observeRateLimit({
-                            cloudWatch: aws.cloudWatch,
-                            endTime,
-                            marker,
-                            markerDigest: expectedMarkerDigest,
-                            probePath: url.pathname,
-                            ruleMetricName,
-                            startTime,
-                            waf: aws.waf,
-                            webAclArn: seed.webAclArn,
-                            webAclMetricName: expectedWebAclMetricName,
-                        });
-                    },
-                    request: async (sequence) => {
-                        const marker = rateMarker(nonce, sequence);
-                        const response = await fetchRequest(url, {
-                            cache: 'no-store',
-                            headers: {
-                                'cache-control': 'no-cache',
-                                [RATE_MARKER_HEADER]: marker,
-                            },
-                            redirect: 'manual',
-                            signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
-                        });
-                        await response.body?.cancel();
-                        return {
-                            markerDigest: markerDigest(marker),
-                            status: response.status,
-                        };
-                    },
-                    ...(dependencies.sleep ? { sleep: dependencies.sleep } : {}),
+            const startedAt = now().toISOString();
+            const operatorProbe = async (type, ruleName) => {
+                const marker = `${nonce}:${type}`;
+                const response = await get(url, {
+                    'cache-control': 'no-cache',
+                    [OPERATOR_MARKER_HEADER]: marker,
+                });
+                const blockedAt = now().toISOString();
+                if (response.status !== 403) {
+                    throw new Error(`Managed Router WAF ${type} probe returned HTTP ${response.status} instead of the expected 403 BLOCK.`);
+                }
+                return {
+                    blockedAt,
+                    marker,
+                    method: 'GET',
+                    path: url.pathname,
+                    ruleName,
+                    status: 403,
+                };
+            };
+            const nonAllowlisted = await operatorProbe('nonallowlisted', 'StageIpAllowlist');
+            await (dependencies.privateSmoke ?? private_smoke_access_js_1.runGithubPrivateSmoke)({
+                account: accountId,
+                deploymentProfile: operatingPolicy.deploymentProfile,
+                root: context.root,
+                run: async () => {
+                    const managedRule = await operatorProbe('managed', 'AWSManagedRulesCommonRuleSet#NoUserAgent_HEADER');
+                    const seedEvidence = {
+                        accountId,
+                        completedAt: now().toISOString(),
+                        deploymentRoleArn: expectedRoleArn,
+                        distributionId: discovered.distributionId,
+                        kind: 'ebk-router-waf-live-evidence',
+                        mode: 'block',
+                        policyDigest: (0, index_js_1.routerWafPolicyDigest)(policy),
+                        probes: {
+                            allowedRequestStatus: 200,
+                            cloudWatchMetricsObserved: true,
+                            countModeTuningObserved: false,
+                            managedRule,
+                            nonAllowlisted,
+                            rateLimit: null,
+                            sampledRequestsObserved: true,
+                        },
+                        region,
+                        requiredRules: [
+                            'StageIpAllowlist',
+                            'RateLimitPerIP',
+                            ...index_js_1.ROUTER_WAF_REQUIRED_MANAGED_RULES,
+                        ],
+                        result: 'passed',
+                        schemaVersion: index_js_1.ROUTER_WAF_LIVE_EVIDENCE_SCHEMA_VERSION,
+                        sourceCommit,
+                        stage,
+                        startedAt,
+                        wafContractDigest,
+                        webAclArn: discovered.webAclArn,
+                        webAclScope: 'CLOUDFRONT',
+                        workspace,
+                    };
+                    writeBoundedJson(evidencePath, seedEvidence, 'Router WAF live evidence');
+                    await seal();
                 },
-                rateLimitPerIp: policy.rateLimitPerIp,
+                source: sourceCommit,
                 stage,
-                webAclMetricName,
+                url: url.href,
+                workspace,
+            }, {
+                ...(dependencies.environment
+                    ? { environment: dependencies.environment }
+                    : {}),
+                ...(dependencies.fetch ? { fetch: dependencies.fetch } : {}),
             });
-            const completedAt = (dependencies.now ?? (() => new Date()))().toISOString();
-            const evidence = mergeRateEvidence(retainedEvidence, allowedRequestStatus, rateLimit, completedAt);
-            const receipt = (0, index_js_1.createRouterWafBlockingRehearsal)({ evidence, policy });
-            writeBoundedJson(evidencePath, evidence, 'Router WAF live evidence');
-            writeBoundedJson(receiptPath, receipt, 'Router WAF blocking rehearsal receipt');
-            devkit_1.logger.info(JSON.stringify({
-                evidenceDigest: receipt.evidenceDigest,
-                accountId: receipt.accountId,
-                policyDigest: receipt.policyDigest,
-                rateRequests: rateLimit.requestsSent,
-                region: receipt.region,
-                sealed: true,
-                stage: receipt.stage,
-            }));
             return { success: true };
         }
         catch (error) {