@empire-builder-kit/nx 1.0.0-beta.9 → 1.0.0-rc.2

package/src/performance/scenario.js

@@ -0,0 +1,736 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PROTECTED_LOAD_SCENARIO_ADAPTERS = void 0;
+exports.validateProtectedLoadScenarioCatalog = validateProtectedLoadScenarioCatalog;
+exports.createProtectedLoadScenarioDriver = createProtectedLoadScenarioDriver;
+exports.cleanupProtectedLoadScenarioEffects = cleanupProtectedLoadScenarioEffects;
+const model_js_1 = require("./model.js");
+const performance_1 = require("@empire-builder-kit/runtime/performance");
+exports.PROTECTED_LOAD_SCENARIO_ADAPTERS = performance_1.PROTECTED_LOAD_ENDPOINT_ADAPTERS;
+const IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._:/#@+-]{0,511}$/;
+const DIGEST = /^[a-f0-9]{64}$/;
+const FULL_COMMIT = /^(?:[a-f0-9]{40}|[a-f0-9]{64})$/;
+const CLEANUP_MAXIMUM_REQUEST_MILLISECONDS = 15_000;
+const CLEANUP_MAXIMUM_RUNTIME_MILLISECONDS = 30_000;
+function record(value) {
+    return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+function onlyKeys(value, keys, pointer) {
+    if (!record(value)) {
+        throw invalid(pointer, 'must be an object');
+    }
+    for (const key of Object.keys(value)) {
+        if (!keys.includes(key)) {
+            throw invalid(`${pointer}/${key}`, 'is not part of the v1 contract');
+        }
+    }
+}
+function invalid(pointer, message) {
+    return new model_js_1.PerformanceContractError('performance-load-plan-invalid', pointer, message);
+}
+function requireIdentifier(value, pointer) {
+    if (typeof value !== 'string' || !IDENTIFIER.test(value)) {
+        throw invalid(pointer, 'must be one bounded opaque identifier');
+    }
+    return value;
+}
+function validHmacKey(value) {
+    const secret = record(value) && typeof value.secret === 'string' ? value.secret : '';
+    const decoded = /^[A-Za-z0-9_-]{43}$/.test(secret)
+        ? Buffer.from(secret, 'base64url')
+        : Buffer.alloc(0);
+    return (record(value) &&
+        Object.keys(value).length === 2 &&
+        typeof value.id === 'string' &&
+        /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/.test(value.id) &&
+        decoded.length === 32 &&
+        decoded.toString('base64url') === secret);
+}
+function exactAuthorityReference(value, reference, pointer) {
+    if (!Array.isArray(value) ||
+        value.length === 0 ||
+        value.length > 512 ||
+        value.some((entry) => typeof entry !== 'string' || !IDENTIFIER.test(entry)) ||
+        value.filter((entry) => entry === reference).length !== 1) {
+        throw invalid(pointer, `must contain exactly one authoritative ${reference} reference`);
+    }
+}
+function validateOperationsBinding(value, policyAlarmRef, capacity) {
+    if (!record(value) || !record(value.performanceReferences)) {
+        throw invalid('operationsAuthority/performanceReferences', 'must expose reliability-owned performance references');
+    }
+    const references = value.performanceReferences;
+    exactAuthorityReference(references.capacityControls, capacity.protectionRef, 'operationsAuthority/performanceReferences/capacityControls');
+    exactAuthorityReference(references.capacitySignals, capacity.saturationSignalRef, 'operationsAuthority/performanceReferences/capacitySignals');
+    exactAuthorityReference(references.alarms, policyAlarmRef, 'operationsAuthority/performanceReferences/alarms');
+    if (capacity.saturationSignalRef.startsWith('alarm:')) {
+        exactAuthorityReference(references.alarms, capacity.saturationSignalRef, 'operationsAuthority/performanceReferences/alarms');
+    }
+    return {
+        alarmRef: policyAlarmRef,
+        capacityControlRef: capacity.protectionRef,
+        capacitySignalRef: capacity.saturationSignalRef,
+    };
+}
+function adapterForScenario(scenarioId) {
+    return exports.PROTECTED_LOAD_SCENARIO_ADAPTERS.find((adapter) => adapter.scenarioId === scenarioId);
+}
+function adapterForId(adapterId) {
+    return exports.PROTECTED_LOAD_SCENARIO_ADAPTERS.find((adapter) => adapter.id === adapterId);
+}
+function costStopThreshold(plan) {
+    const condition = plan.stopConditions.find(({ metric }) => metric === 'estimated-cost-usd');
+    if (!condition) {
+        throw invalid('plan/stopConditions', 'must include the emergency estimated-cost-usd threshold');
+    }
+    return condition.threshold;
+}
+function releaseEquals(left, right) {
+    return (left.candidateDigest === right.candidateDigest &&
+        left.releaseId === right.releaseId &&
+        left.releaseMaterialDigest === right.releaseMaterialDigest &&
+        left.sourceCommit === right.sourceCommit);
+}
+function validateRelease(value, expected, pointer) {
+    onlyKeys(value, ['candidateDigest', 'releaseId', 'releaseMaterialDigest', 'sourceCommit'], pointer);
+    if (typeof value.candidateDigest !== 'string' ||
+        !DIGEST.test(value.candidateDigest) ||
+        typeof value.releaseMaterialDigest !== 'string' ||
+        !DIGEST.test(value.releaseMaterialDigest) ||
+        typeof value.sourceCommit !== 'string' ||
+        !FULL_COMMIT.test(value.sourceCommit) ||
+        typeof value.releaseId !== 'string' ||
+        !IDENTIFIER.test(value.releaseId) ||
+        !releaseEquals(value, expected)) {
+        throw invalid(pointer, 'must bind the exact protected release');
+    }
+}
+function validateProtectedLoadScenarioCatalog(value, plan, policyValue, operationsAuthorityValue) {
+    const policy = (0, model_js_1.validatePerformancePolicy)(policyValue);
+    if ((0, model_js_1.performanceDigest)(policy) !== plan.policyDigest ||
+        policy.slice !== plan.slice) {
+        throw invalid('catalog/policy', 'must be the exact performance policy bound by the protected plan');
+    }
+    onlyKeys(value, ['kind', 'scenarios', 'schemaVersion', 'slice'], 'catalog');
+    if (value.kind !== 'ebk-protected-load-scenario-catalog' ||
+        value.schemaVersion !== 1 ||
+        value.slice !== plan.slice ||
+        !Array.isArray(value.scenarios)) {
+        throw invalid('catalog', 'must be the exact v1 scenario catalog for the planned Slice');
+    }
+    const ids = new Set();
+    const scenarios = value.scenarios.map((raw, index) => {
+        onlyKeys(raw, [
+            'adapterId',
+            'adapterStatus',
+            'demandShape',
+            'disabledUntilAdopted',
+            'estimatedCostUsdPerRequest',
+            'id',
+            'saturationSignalRef',
+            'workload',
+        ], `catalog/scenarios/${index}`);
+        const id = requireIdentifier(raw.id, `catalog/scenarios/${index}/id`);
+        if (ids.has(id)) {
+            throw invalid(`catalog/scenarios/${index}/id`, 'must be unique');
+        }
+        ids.add(id);
+        const adapter = adapterForScenario(id);
+        if (!adapter ||
+            raw.adapterId !== adapter.id ||
+            raw.workload !== adapter.workload ||
+            (adapter.saturationSignalRef !== null &&
+                raw.saturationSignalRef !== adapter.saturationSignalRef) ||
+            (raw.adapterStatus !== 'managed' &&
+                raw.adapterStatus !== 'requires-adopter-service-endpoint') ||
+            typeof raw.disabledUntilAdopted !== 'boolean' ||
+            typeof raw.demandShape !== 'string' ||
+            !IDENTIFIER.test(raw.demandShape) ||
+            typeof raw.saturationSignalRef !== 'string' ||
+            !IDENTIFIER.test(raw.saturationSignalRef) ||
+            typeof raw.estimatedCostUsdPerRequest !== 'number' ||
+            !Number.isFinite(raw.estimatedCostUsdPerRequest) ||
+            raw.estimatedCostUsdPerRequest <= 0) {
+            throw invalid(`catalog/scenarios/${index}`, 'must use the fixed workload adapter and positive reviewed cost estimate');
+        }
+        return raw;
+    });
+    const matches = scenarios.filter(({ id }) => id === plan.scenarioId);
+    if (matches.length !== 1) {
+        throw invalid('catalog/scenarios', 'must contain exactly one planned scenario');
+    }
+    const scenario = matches[0];
+    const adapter = adapterForId(scenario.adapterId);
+    if (!adapter ||
+        scenario.disabledUntilAdopted ||
+        scenario.adapterStatus !== 'managed' ||
+        scenario.workload !== plan.workload ||
+        adapter.workload !== plan.workload) {
+        throw invalid('catalog/scenarios', scenario.adapterStatus === 'requires-adopter-service-endpoint'
+            ? 'Service load requires a real adopter-supplied Service runtime endpoint; the framework does not relabel Next/Lambda evidence'
+            : 'planned scenario must be deliberately adopted for its fixed workload adapter');
+    }
+    const matchingCapacity = policy.capacity.filter((resource) => resource.kind === scenario.workload &&
+        resource.saturationSignalRef === scenario.saturationSignalRef);
+    if (matchingCapacity.length !== 1) {
+        throw invalid('catalog/scenarios/saturationSignalRef', 'must resolve exactly one workload saturation signal in the bound policy');
+    }
+    const selectedCapacity = matchingCapacity[0];
+    if (!selectedCapacity ||
+        selectedCapacity.status !== 'declared' ||
+        typeof selectedCapacity.configuredMaximum !== 'number' ||
+        !Number.isFinite(selectedCapacity.configuredMaximum) ||
+        selectedCapacity.configuredMaximum <= 0 ||
+        typeof selectedCapacity.downstreamMaximum !== 'number' ||
+        !Number.isFinite(selectedCapacity.downstreamMaximum) ||
+        selectedCapacity.downstreamMaximum <= 0 ||
+        typeof selectedCapacity.quotaMaximum !== 'number' ||
+        !Number.isFinite(selectedCapacity.quotaMaximum) ||
+        selectedCapacity.quotaMaximum <= 0 ||
+        selectedCapacity.configuredMaximum > selectedCapacity.downstreamMaximum ||
+        selectedCapacity.configuredMaximum > selectedCapacity.quotaMaximum) {
+        throw invalid('catalog/scenarios/capacity', 'selected capacity must be declared with positive finite configured, downstream, and quota maxima, and configured capacity must not exceed either bound');
+    }
+    const capacity = selectedCapacity;
+    const plannedCapacity = adapter.capacityBound === 'rate'
+        ? plan.maximumRatePerSecond
+        : plan.maximumConcurrency;
+    if (plannedCapacity > capacity.configuredMaximum) {
+        throw invalid(adapter.capacityBound === 'rate'
+            ? 'plan/maximumRatePerSecond'
+            : 'plan/maximumConcurrency', `must not exceed selected configured capacity ${capacity.configuredMaximum}`);
+    }
+    const operationsBinding = validateOperationsBinding(operationsAuthorityValue, policy.cost.anomalyRouteRef, capacity);
+    const scenarioMaximumCost = plan.maximumRequests * scenario.estimatedCostUsdPerRequest;
+    const scenarioCostStopThreshold = costStopThreshold(plan);
+    if (scenarioMaximumCost > plan.estimatedMaximumCostUsd + Number.EPSILON ||
+        scenario.estimatedCostUsdPerRequest >
+            scenarioCostStopThreshold + Number.EPSILON) {
+        throw invalid('catalog/scenarios/estimatedCostUsdPerRequest', 'reviewed per-request cost can exceed the plan or emergency cost envelope');
+    }
+    return {
+        adapter,
+        capacity,
+        catalog: {
+            kind: 'ebk-protected-load-scenario-catalog',
+            scenarios,
+            schemaVersion: 1,
+            slice: value.slice,
+        },
+        operationsBinding,
+        scenario,
+    };
+}
+function scenarioUrl(plan, path) {
+    const endpoint = new URL(plan.target.endpoint);
+    return new URL(`/${plan.slice}${path}`, endpoint.origin).href;
+}
+function scenarioRequestUrl(plan, adapter) {
+    const endpoint = new URL(plan.target.endpoint);
+    const expectedPath = `/${plan.slice}${adapter.requestPath}`;
+    if (endpoint.pathname !== expectedPath || endpoint.search || endpoint.hash) {
+        throw invalid('plan/target/endpoint', `must be the fixed exact ${adapter.id} request endpoint ${expectedPath}`);
+    }
+    return endpoint.href;
+}
+function cleanupPath(adapter) {
+    if (adapter.cleanupPath === null) {
+        throw invalid('cleanup/adapter', 'adapter must define its fixed cleanup path');
+    }
+    return adapter.cleanupPath;
+}
+function effectForRequest(adapter, plan, requestOrdinal) {
+    if (adapter.workload !== 'async-worker' && adapter.workload !== 'database') {
+        throw invalid('driver/effect', 'read-only adapters cannot reserve effects');
+    }
+    return {
+        adapterId: adapter.id,
+        effectId: (0, model_js_1.performanceDigest)({
+            adapterId: adapter.id,
+            planMaterialDigest: plan.materialDigest,
+            requestOrdinal,
+            runId: plan.runId,
+        }),
+        kind: 'ebk-protected-load-effect',
+        planMaterialDigest: plan.materialDigest,
+        requestOrdinal,
+        runId: plan.runId,
+        scenarioId: plan.scenarioId,
+        schemaVersion: 1,
+        sourceCommit: plan.sourceCommit,
+        state: 'reserved',
+        workload: adapter.workload,
+    };
+}
+function requestAuthorization(input) {
+    return {
+        adapterId: input.adapter.id,
+        configuredMaximum: input.capacity.configuredMaximum,
+        createdAt: input.plan.createdAt,
+        durationSeconds: input.plan.durationSeconds,
+        effectId: input.effect?.effectId ?? null,
+        expiresAt: input.plan.expiresAt,
+        hmacKeyId: input.hmacKeyId,
+        maximumRequests: input.plan.maximumRequests,
+        operation: input.operation,
+        planMaterialDigest: input.plan.materialDigest,
+        release: input.plan.release,
+        requestOrdinal: input.requestOrdinal,
+        runId: input.plan.runId,
+        saturationSignalRef: input.scenario.saturationSignalRef,
+        scenarioId: input.plan.scenarioId,
+        stage: input.plan.target.stage,
+        target: {
+            accountFingerprint: input.plan.target.accountFingerprint,
+            endpoint: input.operation === 'cleanup'
+                ? scenarioUrl(input.plan, cleanupPath(input.adapter))
+                : input.plan.target.endpoint,
+            region: input.plan.target.region,
+            slice: input.plan.slice,
+        },
+        workload: input.adapter
+            .workload,
+    };
+}
+function authorizationHeaders(input) {
+    const { authorization } = input;
+    return {
+        'x-ebk-performance-account-fingerprint': authorization.target.accountFingerprint,
+        'x-ebk-performance-candidate': authorization.release.candidateDigest,
+        'x-ebk-performance-configured-maximum': String(authorization.configuredMaximum),
+        'x-ebk-performance-created-at': authorization.createdAt,
+        'x-ebk-performance-duration-seconds': String(authorization.durationSeconds),
+        'x-ebk-performance-expires-at': authorization.expiresAt,
+        'x-ebk-performance-hmac-key-id': authorization.hmacKeyId,
+        'x-ebk-performance-maximum-requests': String(authorization.maximumRequests),
+        'x-ebk-performance-plan': authorization.planMaterialDigest,
+        'x-ebk-performance-release-id': authorization.release.releaseId,
+        'x-ebk-performance-release-material': authorization.release.releaseMaterialDigest,
+        'x-ebk-performance-request': String(authorization.requestOrdinal),
+        'x-ebk-performance-region': authorization.target.region,
+        'x-ebk-performance-run': authorization.runId,
+        'x-ebk-performance-saturation-signal': authorization.saturationSignalRef,
+        'x-ebk-performance-scenario': authorization.scenarioId,
+        'x-ebk-performance-slice': authorization.target.slice,
+        'x-ebk-performance-signature': (0, performance_1.createProtectedLoadRequestSignature)(input),
+        'x-ebk-performance-source': authorization.release.sourceCommit,
+        'x-ebk-performance-stage': authorization.stage,
+    };
+}
+function validateTelemetry(value, input) {
+    onlyKeys(value, [
+        'effect',
+        'hmacKeyId',
+        'kind',
+        'planMaterialDigest',
+        'release',
+        'requestOrdinal',
+        'runId',
+        'saturation',
+        'scenarioId',
+        'schemaVersion',
+        'target',
+        'workload',
+    ], 'driver/telemetry');
+    onlyKeys(value.saturation, ['percent', 'signalRef'], 'driver/telemetry/saturation');
+    onlyKeys(value.target, ['accountFingerprint', 'endpoint', 'region', 'slice'], 'driver/telemetry/target');
+    validateRelease(value.release, input.plan.release, 'driver/telemetry/release');
+    if (value.effect !== undefined) {
+        onlyKeys(value.effect, ['id', 'state'], 'driver/telemetry/effect');
+    }
+    if (value.kind !== 'ebk-protected-load-telemetry' ||
+        value.hmacKeyId !== input.hmacKeyId ||
+        value.schemaVersion !== 1 ||
+        value.planMaterialDigest !== input.plan.materialDigest ||
+        value.requestOrdinal !== input.requestOrdinal ||
+        value.runId !== input.plan.runId ||
+        value.scenarioId !== input.plan.scenarioId ||
+        JSON.stringify(value.target) !==
+            JSON.stringify({
+                accountFingerprint: input.plan.target.accountFingerprint,
+                endpoint: input.plan.target.endpoint,
+                region: input.plan.target.region,
+                slice: input.plan.slice,
+            }) ||
+        value.workload !== input.plan.workload ||
+        value.saturation.signalRef !== input.scenario.saturationSignalRef ||
+        typeof value.saturation.percent !== 'number' ||
+        !Number.isFinite(value.saturation.percent) ||
+        value.saturation.percent < 0 ||
+        value.saturation.percent > 100) {
+        throw invalid('driver/telemetry', 'omitted or mismatched authoritative saturation and release telemetry');
+    }
+    if (input.adapter.mutation &&
+        (!input.effect ||
+            !record(value.effect) ||
+            value.effect.id !== input.effect.effectId ||
+            !['already-existed', 'created'].includes(String(value.effect.state)))) {
+        throw invalid('driver/telemetry/effect', 'mutating adapter must confirm its pre-reserved cleanup identity');
+    }
+    if (!input.adapter.mutation && value.effect !== undefined) {
+        throw invalid('driver/telemetry/effect', 'read-only adapter cannot report a cleanup effect');
+    }
+    return value;
+}
+function createProtectedLoadScenarioDriver(input) {
+    const { adapter, capacity, catalog, operationsBinding, scenario } = validateProtectedLoadScenarioCatalog(input.catalog, input.plan, input.policy, input.operationsAuthority);
+    if (!validHmacKey(input.hmacKey)) {
+        throw invalid('driver/hmacKey', 'must be one bounded protected HMAC key generation');
+    }
+    if (adapter.mutation && !input.recordEffect) {
+        throw invalid('driver/recordEffect', 'mutating adapter requires durable pre-request effect reservation');
+    }
+    const fetchImplementation = input.fetch ??
+        ((url, init) => fetch(url, init));
+    const wait = input.wait ??
+        ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)));
+    let terminated = false;
+    let nextOrdinal = 1;
+    let cumulativeEstimatedCostUsd = 0;
+    const scenarioCostStopThreshold = costStopThreshold(input.plan);
+    const scenarioContractDigest = (0, model_js_1.performanceDigest)({
+        adapter,
+        capacity,
+        catalog,
+        operationsBinding,
+        scenario,
+    });
+    const requestUrl = scenarioRequestUrl(input.plan, adapter);
+    const driver = {
+        hmacKeyId: input.hmacKey.id,
+        id: adapter.id,
+        scenarioContractDigest,
+        version: adapter.version,
+        workload: adapter.workload,
+        async executeStep(context) {
+            if (terminated) {
+                throw invalid('driver', 'is already terminated');
+            }
+            const startedAt = Date.now();
+            const deadline = startedAt + context.stepDurationSeconds * 1_000;
+            let nextRequestIndex = 0;
+            let attemptedRequests = 0;
+            let completedRequests = 0;
+            let downstreamSaturation = 0;
+            let errors = 0;
+            const latencies = [];
+            let fatal;
+            let costCeilingReached = false;
+            const worker = async () => {
+                while (!terminated && fatal === undefined) {
+                    const requestIndex = nextRequestIndex++;
+                    if (requestIndex >= context.maximumRequests)
+                        return;
+                    const scheduledAt = startedAt + (requestIndex / context.maximumRatePerSecond) * 1_000;
+                    const delay = scheduledAt - Date.now();
+                    if (delay > 0)
+                        await wait(delay);
+                    if (Date.now() >= deadline || terminated || fatal !== undefined) {
+                        return;
+                    }
+                    if (cumulativeEstimatedCostUsd + scenario.estimatedCostUsdPerRequest >
+                        scenarioCostStopThreshold + Number.EPSILON) {
+                        costCeilingReached = true;
+                        return;
+                    }
+                    cumulativeEstimatedCostUsd += scenario.estimatedCostUsdPerRequest;
+                    const requestOrdinal = nextOrdinal++;
+                    const effect = adapter.mutation
+                        ? effectForRequest(adapter, input.plan, requestOrdinal)
+                        : undefined;
+                    if (effect) {
+                        await input.recordEffect?.(effect);
+                    }
+                    attemptedRequests += 1;
+                    const requestStartedAt = Date.now();
+                    const controller = new AbortController();
+                    const timeout = setTimeout(() => controller.abort(), Math.max(1, deadline - Date.now()));
+                    try {
+                        const body = {
+                            ...(effect ? { effectId: effect.effectId } : {}),
+                            planMaterialDigest: input.plan.materialDigest,
+                            release: input.plan.release,
+                            requestOrdinal,
+                            runId: input.plan.runId,
+                            scenarioId: input.plan.scenarioId,
+                            workload: input.plan.workload,
+                        };
+                        const authorization = requestAuthorization({
+                            adapter,
+                            capacity,
+                            ...(effect ? { effect } : {}),
+                            hmacKeyId: input.hmacKey.id,
+                            operation: 'execute',
+                            plan: input.plan,
+                            requestOrdinal,
+                            scenario,
+                        });
+                        const response = await fetchImplementation(requestUrl, {
+                            ...(adapter.requestMethod === 'POST'
+                                ? { body: JSON.stringify(body) }
+                                : {}),
+                            headers: {
+                                accept: 'application/json',
+                                ...(adapter.requestMethod === 'POST'
+                                    ? { 'content-type': 'application/json' }
+                                    : {}),
+                                ...authorizationHeaders({
+                                    authorization,
+                                    hmacKey: input.hmacKey,
+                                }),
+                                'user-agent': 'empire-builder-kit-protected-load-scenario/1.0.0',
+                            },
+                            method: adapter.requestMethod,
+                            redirect: 'manual',
+                            signal: controller.signal,
+                        });
+                        completedRequests += 1;
+                        if (!response.ok)
+                            errors += 1;
+                        let telemetry;
+                        try {
+                            telemetry = await response.json();
+                        }
+                        catch {
+                            throw invalid('driver/telemetry', 'omitted the authoritative JSON telemetry envelope');
+                        }
+                        const envelope = validateTelemetry(telemetry, {
+                            adapter,
+                            effect,
+                            hmacKeyId: input.hmacKey.id,
+                            plan: input.plan,
+                            requestOrdinal,
+                            scenario,
+                        });
+                        downstreamSaturation = Math.max(downstreamSaturation, envelope.saturation.percent);
+                    }
+                    catch (error) {
+                        if (error instanceof model_js_1.PerformanceContractError) {
+                            fatal = error;
+                            terminated = true;
+                            return;
+                        }
+                        errors += 1;
+                    }
+                    finally {
+                        clearTimeout(timeout);
+                        latencies.push(Date.now() - requestStartedAt);
+                    }
+                }
+            };
+            await Promise.all(Array.from({
+                length: Math.min(context.maximumConcurrency, context.maximumRequests),
+            }, () => worker()));
+            if (fatal !== undefined)
+                throw fatal;
+            const orderedLatencies = [...latencies].sort((left, right) => left - right);
+            return {
+                attemptedRequests,
+                completedRequests,
+                downstreamSaturation,
+                elapsedSeconds: Math.max(0.001, Math.min(context.stepDurationSeconds, (Date.now() - startedAt) / 1_000)),
+                errors,
+                estimatedCostUsd: attemptedRequests * scenario.estimatedCostUsdPerRequest,
+                latencyP95Milliseconds: orderedLatencies[Math.max(0, Math.ceil(orderedLatencies.length * 0.95) - 1)] ?? 0,
+                ...(costCeilingReached
+                    ? { stopReason: 'estimated-cost' }
+                    : {}),
+            };
+        },
+        async terminate() {
+            terminated = true;
+        },
+    };
+    return {
+        allowedDriver: {
+            hmacKeyId: input.hmacKey.id,
+            id: adapter.id,
+            scenarioContractDigest,
+            version: adapter.version,
+            workload: adapter.workload,
+        },
+        driver,
+    };
+}
+function validateEffect(value, adapter, plan, pointer) {
+    onlyKeys(value, [
+        'adapterId',
+        'effectId',
+        'kind',
+        'planMaterialDigest',
+        'requestOrdinal',
+        'runId',
+        'scenarioId',
+        'schemaVersion',
+        'sourceCommit',
+        'state',
+        'workload',
+    ], pointer);
+    if (value.adapterId !== adapter.id ||
+        typeof value.effectId !== 'string' ||
+        !DIGEST.test(value.effectId) ||
+        value.kind !== 'ebk-protected-load-effect' ||
+        value.planMaterialDigest !== plan.materialDigest ||
+        typeof value.requestOrdinal !== 'number' ||
+        !Number.isSafeInteger(value.requestOrdinal) ||
+        value.requestOrdinal <= 0 ||
+        value.runId !== plan.runId ||
+        value.scenarioId !== plan.scenarioId ||
+        value.schemaVersion !== 1 ||
+        value.sourceCommit !== plan.sourceCommit ||
+        value.state !== 'reserved' ||
+        value.workload !== adapter.workload) {
+        throw invalid(pointer, 'does not belong to the exact protected load run');
+    }
+    return value;
+}
+async function cleanupProtectedLoadScenarioEffects(input) {
+    const cleanupDeadline = Date.now() + CLEANUP_MAXIMUM_RUNTIME_MILLISECONDS;
+    const { adapter, capacity, catalog, operationsBinding, scenario } = validateProtectedLoadScenarioCatalog(input.catalog, input.plan, input.policy, input.operationsAuthority);
+    if (!validHmacKey(input.hmacKey)) {
+        throw invalid('cleanup/hmacKey', 'must be one bounded protected HMAC key generation');
+    }
+    scenarioRequestUrl(input.plan, adapter);
+    if (!Array.isArray(input.effects)) {
+        throw invalid('cleanup/effects', 'must be an array');
+    }
+    if (input.effects.length > input.plan.maximumRequests) {
+        throw invalid('cleanup/effects', 'cannot exceed the exact protected run request budget');
+    }
+    if (!adapter.mutation && input.effects.length > 0) {
+        throw invalid('cleanup/effects', 'read-only adapter cannot own cleanup effects');
+    }
+    const fetchImplementation = input.fetch ??
+        ((url, init) => fetch(url, init));
+    const seen = new Set();
+    const effects = input.effects.map((effect, index) => {
+        const validated = validateEffect(effect, adapter, input.plan, `cleanup/effects/${index}`);
+        if (seen.has(validated.effectId)) {
+            throw invalid(`cleanup/effects/${index}`, 'duplicates an effect identity');
+        }
+        seen.add(validated.effectId);
+        return validated;
+    });
+    let remoteCleanup = 'not-required';
+    if (adapter.cleanupPath !== null) {
+        const controller = new AbortController();
+        let timeout;
+        try {
+            const authorization = requestAuthorization({
+                adapter,
+                capacity,
+                hmacKeyId: input.hmacKey.id,
+                operation: 'cleanup',
+                plan: input.plan,
+                requestOrdinal: 1,
+                scenario,
+            });
+            const remainingRuntime = cleanupDeadline - Date.now();
+            if (remainingRuntime <= 0) {
+                throw invalid('cleanup/deadline', 'exhausted the bounded cleanup runtime before the request');
+            }
+            const requestTimeout = Math.min(CLEANUP_MAXIMUM_REQUEST_MILLISECONDS, remainingRuntime);
+            const request = (async () => {
+                const response = await fetchImplementation(scenarioUrl(input.plan, cleanupPath(adapter)), {
+                    body: JSON.stringify({
+                        planMaterialDigest: input.plan.materialDigest,
+                        release: input.plan.release,
+                        requestOrdinal: 1,
+                        runId: input.plan.runId,
+                        scenarioId: input.plan.scenarioId,
+                        workload: input.plan.workload,
+                    }),
+                    headers: {
+                        accept: 'application/json',
+                        'content-type': 'application/json',
+                        ...authorizationHeaders({
+                            authorization,
+                            hmacKey: input.hmacKey,
+                        }),
+                        'user-agent': 'empire-builder-kit-protected-load-cleanup/1.0.0',
+                    },
+                    method: 'POST',
+                    redirect: 'manual',
+                    signal: controller.signal,
+                });
+                return { body: await response.json(), response };
+            })();
+            const expired = new Promise((_resolve, reject) => {
+                timeout = setTimeout(() => {
+                    controller.abort();
+                    reject(invalid('cleanup/deadline', 'exceeded the bounded cleanup request deadline'));
+                }, requestTimeout);
+            });
+            const { body, response } = await Promise.race([request, expired]);
+            onlyKeys(body, [
+                'kind',
+                'hmacKeyId',
+                'planMaterialDigest',
+                'release',
+                'remainingEffectCount',
+                'removedEffectCount',
+                'runId',
+                'scenarioId',
+                'schemaVersion',
+                'status',
+                'target',
+            ], 'cleanup/response');
+            validateRelease(body.release, input.plan.release, 'cleanup/response/release');
+            onlyKeys(body.target, ['accountFingerprint', 'endpoint', 'region', 'slice'], 'cleanup/response/target');
+            if (!response.ok ||
+                body.kind !== 'ebk-protected-load-cleanup-result' ||
+                body.hmacKeyId !== input.hmacKey.id ||
+                body.planMaterialDigest !== input.plan.materialDigest ||
+                typeof body.remainingEffectCount !== 'number' ||
+                !Number.isSafeInteger(body.remainingEffectCount) ||
+                body.remainingEffectCount !== 0 ||
+                typeof body.removedEffectCount !== 'number' ||
+                !Number.isSafeInteger(body.removedEffectCount) ||
+                body.removedEffectCount < 0 ||
+                body.runId !== input.plan.runId ||
+                body.scenarioId !== input.plan.scenarioId ||
+                body.schemaVersion !== 1 ||
+                body.status !== 'run-absent' ||
+                JSON.stringify(body.target) !==
+                    JSON.stringify({
+                        accountFingerprint: input.plan.target.accountFingerprint,
+                        endpoint: scenarioUrl(input.plan, cleanupPath(adapter)),
+                        region: input.plan.target.region,
+                        slice: input.plan.slice,
+                    })) {
+                throw invalid('cleanup/response', 'must prove every protected effect for the exact run is absent');
+            }
+            remoteCleanup = 'run-proved-absent';
+        }
+        catch (error) {
+            throw invalid('cleanup', `did not prove the exact protected run absent (${error instanceof Error ? error.message : String(error)})`);
+        }
+        finally {
+            if (timeout !== undefined)
+                clearTimeout(timeout);
+        }
+    }
+    const unsigned = {
+        adapterId: adapter.id,
+        cleanupScope: 'run',
+        hmacKeyId: input.hmacKey.id,
+        kind: 'ebk-protected-load-cleanup',
+        observedEffectIds: effects.map(({ effectId }) => effectId),
+        planMaterialDigest: input.plan.materialDigest,
+        remainingEffectCount: 0,
+        remoteCleanup,
+        result: 'run-effects-absent',
+        runId: input.plan.runId,
+        scenarioContractDigest: (0, model_js_1.performanceDigest)({
+            adapter,
+            capacity,
+            catalog,
+            operationsBinding,
+            scenario,
+        }),
+        schemaVersion: 1,
+        sourceCommit: input.plan.sourceCommit,
+    };
+    return { ...unsigned, materialDigest: (0, model_js_1.performanceDigest)(unsigned) };
+}
+//# sourceMappingURL=scenario.js.map