firedrill-run 2.2.5 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +178 -22
- package/bin/firedrill.mjs +29 -5
- package/dist/assembly.json +13 -0
- package/dist/cli/runtime/candidate_copy.py +395 -0
- package/dist/cli/runtime/source_snapshot.py +391 -0
- package/dist/cli/src/administration.js +295 -0
- package/dist/cli/src/agent-changes.js +496 -0
- package/dist/cli/src/assessment-requests.js +67 -0
- package/dist/cli/src/auth.js +465 -0
- package/dist/cli/src/billing-commerce-contracts.js +1 -0
- package/dist/cli/src/billing-usage.js +227 -0
- package/dist/cli/src/billing-wallet-contracts.js +1 -0
- package/dist/cli/src/billing.js +170 -0
- package/dist/cli/src/bootstrap-process-lock.js +136 -0
- package/dist/cli/src/browser-links.js +14 -0
- package/dist/cli/src/campaign-controls.js +122 -0
- package/dist/cli/src/campaign-diagnoses.js +118 -0
- package/dist/cli/src/campaign-reports.js +1215 -0
- package/dist/cli/src/campaign-world.js +156 -0
- package/dist/cli/src/candidate-copy.js +376 -0
- package/dist/cli/src/catalog-contracts.js +1 -0
- package/dist/cli/src/ci-access-contracts.js +1 -0
- package/dist/cli/src/ci-configuration-contracts.js +1 -0
- package/dist/cli/src/ci-coverage.js +64 -0
- package/dist/cli/src/ci-delivery-contracts.js +1 -0
- package/dist/cli/src/ci-result-contracts.js +1 -0
- package/dist/cli/src/ci-run-contracts.js +1 -0
- package/dist/cli/src/ci-setup-proposals.js +52 -0
- package/dist/cli/src/company-policies.js +229 -0
- package/dist/cli/src/completion.js +79 -0
- package/dist/cli/src/configuration-intents.js +277 -0
- package/dist/cli/src/configuration.js +369 -0
- package/dist/cli/src/continuous-simulation.js +277 -0
- package/dist/cli/src/coordinated-checkpoints.js +133 -0
- package/dist/cli/src/credential-mode.js +139 -0
- package/dist/cli/src/credentials.js +331 -0
- package/dist/cli/src/credit-format.js +8 -0
- package/dist/cli/src/dataset-contracts.js +1 -0
- package/dist/cli/src/dataset-export.js +154 -0
- package/dist/cli/src/dataset-reads.js +37 -0
- package/dist/cli/src/dependency-preparation.js +239 -0
- package/dist/cli/src/director-tool-setup.js +104 -0
- package/dist/cli/src/driver-review.js +225 -0
- package/dist/cli/src/evaluation-authoring.js +315 -0
- package/dist/cli/src/evidence-bundles.js +367 -0
- package/dist/cli/src/execution-proposals.js +36 -0
- package/dist/cli/src/http.js +302 -0
- package/dist/cli/src/local-container-boundary.js +38 -0
- package/dist/cli/src/local-container-policy.js +201 -0
- package/dist/cli/src/local-execution-setup.js +81 -0
- package/dist/cli/src/local-source-basis.js +73 -0
- package/dist/cli/src/local-source.js +66 -0
- package/dist/cli/src/local-value-store.js +91 -0
- package/dist/cli/src/local-values.js +163 -0
- package/dist/cli/src/login-organizations.js +70 -0
- package/dist/cli/src/main.js +1622 -0
- package/dist/cli/src/mcp-connection.js +109 -0
- package/dist/cli/src/native/bindings.js +137 -0
- package/dist/cli/src/native/canonical.js +370 -0
- package/dist/cli/src/native/cloud-resources.js +942 -0
- package/dist/cli/src/native/companion-channel.js +421 -0
- package/dist/cli/src/native/companion-ipc.js +473 -0
- package/dist/cli/src/native/contracts.js +218 -0
- package/dist/cli/src/native/control-plane.js +1354 -0
- package/dist/cli/src/native/credit-approval.js +15 -0
- package/dist/cli/src/native/dependency-proxy.js +1006 -0
- package/dist/cli/src/native/drill-cleanup.js +115 -0
- package/dist/cli/src/native/drill-runner.js +1562 -0
- package/dist/cli/src/native/drivers/action-trace.js +210 -0
- package/dist/cli/src/native/drivers/function-wrapper.js +152 -0
- package/dist/cli/src/native/drivers/http.js +442 -0
- package/dist/cli/src/native/drivers/index.js +47 -0
- package/dist/cli/src/native/drivers/jsonl-process.js +538 -0
- package/dist/cli/src/native/drivers/playwright.js +600 -0
- package/dist/cli/src/native/drivers/signed-event.js +244 -0
- package/dist/cli/src/native/drivers/types.js +578 -0
- package/dist/cli/src/native/evidence-filter.js +255 -0
- package/dist/cli/src/native/isolated-access.js +241 -0
- package/dist/cli/src/native/library-sharing.js +220 -0
- package/dist/cli/src/native/local-run.js +667 -0
- package/dist/cli/src/native/local-secrets.js +19 -0
- package/dist/cli/src/native/mcp-config.js +123 -0
- package/dist/cli/src/native/metadata-requests.js +120 -0
- package/dist/cli/src/native/observations.js +1159 -0
- package/dist/cli/src/native/owned-groups.js +46 -0
- package/dist/cli/src/native/plan.js +594 -0
- package/dist/cli/src/native/png.js +159 -0
- package/dist/cli/src/native/ports.js +43 -0
- package/dist/cli/src/native/readiness.js +139 -0
- package/dist/cli/src/native/reassessments.js +206 -0
- package/dist/cli/src/native/redaction.js +235 -0
- package/dist/cli/src/native/registration.js +155 -0
- package/dist/cli/src/native/roots.js +110 -0
- package/dist/cli/src/native/run-cleanup.js +199 -0
- package/dist/cli/src/native/scratch.js +117 -0
- package/dist/cli/src/native/screenshots.js +60 -0
- package/dist/cli/src/native/session.js +556 -0
- package/dist/cli/src/native/standalone-proxy.js +580 -0
- package/dist/cli/src/native/standalone-renewals.js +183 -0
- package/dist/cli/src/native/standalone-state.js +193 -0
- package/dist/cli/src/native/standalone-tools.js +187 -0
- package/dist/cli/src/native/supervisor.js +831 -0
- package/dist/cli/src/native/target-auth.js +161 -0
- package/dist/cli/src/native/target-state.js +330 -0
- package/dist/cli/src/native/tool-reuse.js +364 -0
- package/dist/cli/src/native/tool-setups.js +336 -0
- package/dist/cli/src/native/world-reuse.js +288 -0
- package/dist/cli/src/onboarding.js +23 -0
- package/dist/cli/src/priority-requests.js +70 -0
- package/dist/cli/src/priority.js +223 -0
- package/dist/cli/src/prompts.js +184 -0
- package/dist/cli/src/real-operations.js +221 -0
- package/dist/cli/src/recipe-intents.js +58 -0
- package/dist/cli/src/recorded-context-intents.js +59 -0
- package/dist/cli/src/recorded-context.js +248 -0
- package/dist/cli/src/recordings.js +69 -0
- package/dist/cli/src/registration-guide.js +39 -0
- package/dist/cli/src/report-exports.js +213 -0
- package/dist/cli/src/request-input.js +47 -0
- package/dist/cli/src/resource-lifecycle.js +141 -0
- package/dist/cli/src/revision-requests.js +55 -0
- package/dist/cli/src/runner-abandon.js +53 -0
- package/dist/cli/src/runner-campaign-controls.js +262 -0
- package/dist/cli/src/runner-claim-reads.js +36 -0
- package/dist/cli/src/runner-completion.js +65 -0
- package/dist/cli/src/runner-configuration.js +546 -0
- package/dist/cli/src/runner-container.js +794 -0
- package/dist/cli/src/runner-evidence-erasure.js +351 -0
- package/dist/cli/src/runner-evidence-privacy.js +37 -0
- package/dist/cli/src/runner-evidence-retention.js +161 -0
- package/dist/cli/src/runner-evidence.js +326 -0
- package/dist/cli/src/runner-execute.js +92 -0
- package/dist/cli/src/runner-execution-deadline.js +25 -0
- package/dist/cli/src/runner-grants.js +307 -0
- package/dist/cli/src/runner-identity.js +615 -0
- package/dist/cli/src/runner-inputs.js +308 -0
- package/dist/cli/src/runner-jobs.js +49 -0
- package/dist/cli/src/runner-model-transport.js +392 -0
- package/dist/cli/src/runner-ownership.js +332 -0
- package/dist/cli/src/runner-process-lock.js +175 -0
- package/dist/cli/src/runner-runtime.js +396 -0
- package/dist/cli/src/runner-session-channel.js +377 -0
- package/dist/cli/src/runner-trial-completion.js +34 -0
- package/dist/cli/src/runner-trial-journal.js +358 -0
- package/dist/cli/src/runner-trial-live.js +393 -0
- package/dist/cli/src/runner-trial-protocol.js +289 -0
- package/dist/cli/src/runner-trial-sync.js +110 -0
- package/dist/cli/src/runner-trial-transfer.js +107 -0
- package/dist/cli/src/runner-voice.js +40 -0
- package/dist/cli/src/runner-world-claim.js +21 -0
- package/dist/cli/src/runner-world-collector.js +159 -0
- package/dist/cli/src/runner-world-command-evidence.js +487 -0
- package/dist/cli/src/runner-world-commands.js +268 -0
- package/dist/cli/src/runner-world-completion.js +76 -0
- package/dist/cli/src/runner-world-erasure.js +95 -0
- package/dist/cli/src/runner-world-evidence-sync.js +91 -0
- package/dist/cli/src/runner-world-journal.js +510 -0
- package/dist/cli/src/runner-world-native-filters.js +134 -0
- package/dist/cli/src/runner-world-observations.js +50 -0
- package/dist/cli/src/runner-world-session-protocol.js +134 -0
- package/dist/cli/src/saved-records.js +40 -0
- package/dist/cli/src/scenario-authoring.js +212 -0
- package/dist/cli/src/scenario-discovery.js +216 -0
- package/dist/cli/src/scenario-paths.js +68 -0
- package/dist/cli/src/scenario-requests.js +4 -0
- package/dist/cli/src/service-receipts.js +105 -0
- package/dist/cli/src/simple/authoring.js +270 -0
- package/dist/cli/src/simple/credit-approval.js +16 -0
- package/dist/cli/src/simple/help.js +98 -0
- package/dist/cli/src/simple/init.js +61 -0
- package/dist/cli/src/simple/library-sharing.js +414 -0
- package/dist/cli/src/simple/metadata-requests.js +46 -0
- package/dist/cli/src/simple/native-report.js +841 -0
- package/dist/cli/src/simple/native-tools.js +780 -0
- package/dist/cli/src/simple/plan-apply.js +506 -0
- package/dist/cli/src/simple/reassessment.js +323 -0
- package/dist/cli/src/simple/register.js +123 -0
- package/dist/cli/src/simple/run.js +122 -0
- package/dist/cli/src/simple/status.js +49 -0
- package/dist/cli/src/simple/tool-renewal.js +308 -0
- package/dist/cli/src/simple/tool-reuse.js +526 -0
- package/dist/cli/src/simple/tool-start-catalog.js +100 -0
- package/dist/cli/src/simple/tool-state.js +426 -0
- package/dist/cli/src/simple/tools-author.js +222 -0
- package/dist/cli/src/simple/watch.js +28 -0
- package/dist/cli/src/simple/world-reuse.js +586 -0
- package/dist/cli/src/simulation-intents.js +406 -0
- package/dist/cli/src/simulation-run.js +1058 -0
- package/dist/cli/src/source-batch.js +322 -0
- package/dist/cli/src/source-connection.js +451 -0
- package/dist/cli/src/source-runtime.js +346 -0
- package/dist/cli/src/source-selection-intents.js +75 -0
- package/dist/cli/src/source-sets.js +214 -0
- package/dist/cli/src/target-models.js +109 -0
- package/dist/cli/src/tool-catalog.js +224 -0
- package/dist/cli/src/tool-command-contracts.js +1 -0
- package/dist/cli/src/tool-command-export.js +245 -0
- package/dist/cli/src/tool-connection-contracts.js +1 -0
- package/dist/cli/src/tool-connection-reads.js +23 -0
- package/dist/cli/src/tool-contributions.js +58 -0
- package/dist/cli/src/tool-data.js +116 -0
- package/dist/cli/src/tool-datasets.js +176 -0
- package/dist/cli/src/tool-group-contracts.js +1 -0
- package/dist/cli/src/tool-group-reads.js +26 -0
- package/dist/cli/src/tool-group-state.js +147 -0
- package/dist/cli/src/tool-groups.js +311 -0
- package/dist/cli/src/tool-instances.js +186 -0
- package/dist/cli/src/tool-lifecycle-contracts.js +1 -0
- package/dist/cli/src/tool-lifecycle-reads.js +30 -0
- package/dist/cli/src/tool-lifecycle.js +323 -0
- package/dist/cli/src/tool-package-contracts.js +1 -0
- package/dist/cli/src/tool-package-reads.js +41 -0
- package/dist/cli/src/tool-records.js +53 -0
- package/dist/cli/src/tool-runtime.js +96 -0
- package/dist/cli/src/tool-state-contracts.js +1 -0
- package/dist/cli/src/tool-state-reads.js +31 -0
- package/dist/cli/src/tools.js +306 -0
- package/dist/cli/src/trial-actions.js +93 -0
- package/dist/cli/src/trial-checkpoints.js +133 -0
- package/dist/cli/src/trial-checks.js +70 -0
- package/dist/cli/src/trial-state.js +197 -0
- package/dist/cli/src/vault.js +266 -0
- package/dist/cli/src/workspace-registration.js +383 -0
- package/dist/cli/src/workspace-team-contracts.js +1 -0
- package/dist/cli/src/world-access.js +231 -0
- package/dist/cli/src/world-agent.js +4 -0
- package/dist/cli/src/world-checkpoint-scopes.js +15 -0
- package/dist/cli/src/world-contracts.js +6 -0
- package/dist/cli/src/world-creation.js +201 -0
- package/dist/cli/src/world-datasets.js +207 -0
- package/dist/cli/src/world-inspection.js +257 -0
- package/dist/cli/src/world-recipes.js +103 -0
- package/dist/cli/src/world-requests.js +241 -0
- package/dist/cli/src/world-session-checkpoints.js +196 -0
- package/dist/cli/src/world-session-lifecycle.js +207 -0
- package/dist/cli/src/world-session-requests.js +215 -0
- package/dist/cli/src/world-session-state.js +145 -0
- package/dist/cli/src/world-session-voice.js +66 -0
- package/dist/cli/src/world-sessions.js +288 -0
- package/dist/cli/src/world-stream.js +99 -0
- package/dist/cli/src/world-time-report.js +58 -0
- package/dist/contracts/agent-change-contracts.js +75 -0
- package/dist/contracts/agent-execution-contracts.js +38 -0
- package/dist/contracts/application-state-contracts.js +26 -0
- package/dist/contracts/assessment-aggregate-contracts.js +15 -0
- package/dist/contracts/assessment-contracts.js +74 -0
- package/dist/contracts/billing-commerce-contracts.js +107 -0
- package/dist/contracts/billing-wallet-contracts.js +43 -0
- package/dist/contracts/campaign-lifecycle.js +1 -0
- package/dist/contracts/campaign-world-contracts.js +80 -0
- package/dist/contracts/catalog-contracts.js +73 -0
- package/dist/contracts/ci-access-contracts.js +68 -0
- package/dist/contracts/ci-configuration-contracts.js +88 -0
- package/dist/contracts/ci-coverage-contracts.js +65 -0
- package/dist/contracts/ci-coverage-labels.js +25 -0
- package/dist/contracts/ci-delivery-contracts.js +61 -0
- package/dist/contracts/ci-result-contracts.js +103 -0
- package/dist/contracts/ci-run-contracts.js +51 -0
- package/dist/contracts/ci-setup-contracts.js +88 -0
- package/dist/contracts/clip-contracts.js +32 -0
- package/dist/contracts/comparison-contracts.js +53 -0
- package/dist/contracts/composite-contracts.js +34 -0
- package/dist/contracts/context-contracts.js +76 -0
- package/dist/contracts/continuation-report-contracts.js +1 -0
- package/dist/contracts/continuity-requirements.js +8 -0
- package/dist/contracts/coordinated-checkpoint-contracts.js +34 -0
- package/dist/contracts/dataset-contracts.js +88 -0
- package/dist/contracts/dependency-decision-contracts.js +36 -0
- package/dist/contracts/dependency-protocol-contracts.js +43 -0
- package/dist/contracts/director-execution.js +51 -0
- package/dist/contracts/driver-review-contracts.js +99 -0
- package/dist/contracts/evidence-contracts.js +65 -0
- package/dist/contracts/evidence-coverage-contracts.js +10 -0
- package/dist/contracts/export-contracts.js +44 -0
- package/dist/contracts/github-directory-contracts.js +96 -0
- package/dist/contracts/insight-contracts.js +11 -0
- package/dist/contracts/inspection-contracts.js +1 -0
- package/dist/contracts/local-claim-contracts.js +39 -0
- package/dist/contracts/local-configuration-contracts.js +7 -0
- package/dist/contracts/local-evidence-contracts.js +32 -0
- package/dist/contracts/local-grant-contracts.js +83 -0
- package/dist/contracts/local-input-contracts.js +19 -0
- package/dist/contracts/local-runner-contracts.js +42 -0
- package/dist/contracts/local-trial-live-contracts.js +16 -0
- package/dist/contracts/local-value-source-contracts.js +92 -0
- package/dist/contracts/local-world-native-contracts.js +8 -0
- package/dist/contracts/mcp-contracts.js +42 -0
- package/dist/contracts/media-contracts.js +28 -0
- package/dist/contracts/native-check-contracts.js +31 -0
- package/dist/contracts/native-identity-contracts.js +52 -0
- package/dist/contracts/onboarding-contracts.js +32 -0
- package/dist/contracts/performance-contracts.js +85 -0
- package/dist/contracts/planning-contracts.js +81 -0
- package/dist/contracts/real-operation-contracts.js +30 -0
- package/dist/contracts/recommendation-contracts.js +54 -0
- package/dist/contracts/report-contracts.js +1 -0
- package/dist/contracts/resource-lifecycle-contracts.js +57 -0
- package/dist/contracts/revision-contracts.js +12 -0
- package/dist/contracts/revision-request-contracts.js +39 -0
- package/dist/contracts/saved-record-contracts.js +76 -0
- package/dist/contracts/scenario-draft-comparison.js +91 -0
- package/dist/contracts/scenario-world-contracts.js +14 -0
- package/dist/contracts/service-contracts.js +1 -0
- package/dist/contracts/session-command-contracts.js +56 -0
- package/dist/contracts/session-contracts.js +65 -0
- package/dist/contracts/session-state-contracts.js +67 -0
- package/dist/contracts/session-voice-contracts.js +45 -0
- package/dist/contracts/source-set-contracts.js +26 -0
- package/dist/contracts/source-upload-contracts.js +22 -0
- package/dist/contracts/tool-command-contracts.js +57 -0
- package/dist/contracts/tool-connection-contracts.js +87 -0
- package/dist/contracts/tool-contribution-contracts.js +35 -0
- package/dist/contracts/tool-group-contracts.js +53 -0
- package/dist/contracts/tool-group-state-contracts.js +63 -0
- package/dist/contracts/tool-lifecycle-contracts.js +94 -0
- package/dist/contracts/tool-package-contracts.js +91 -0
- package/dist/contracts/tool-protocol-contracts.js +71 -0
- package/dist/contracts/tool-runtime-billing-contracts.js +28 -0
- package/dist/contracts/tool-setup-proposal.js +59 -0
- package/dist/contracts/tool-state-contracts.js +79 -0
- package/dist/contracts/trial-state-contracts.js +30 -0
- package/dist/contracts/typescript/agent-change.js +1 -0
- package/dist/contracts/typescript/assessment-aggregate.js +1 -0
- package/dist/contracts/typescript/assessment.js +1 -0
- package/dist/contracts/typescript/billing-commerce.js +1 -0
- package/dist/contracts/typescript/campaigns.js +1 -0
- package/dist/contracts/typescript/catalog.js +1 -0
- package/dist/contracts/typescript/ci-access.js +1 -0
- package/dist/contracts/typescript/ci-configuration.js +1 -0
- package/dist/contracts/typescript/ci-delivery.js +1 -0
- package/dist/contracts/typescript/ci-result.js +1 -0
- package/dist/contracts/typescript/ci-runs.js +1 -0
- package/dist/contracts/typescript/clips.js +1 -0
- package/dist/contracts/typescript/comparison.js +1 -0
- package/dist/contracts/typescript/composite.js +1 -0
- package/dist/contracts/typescript/connections.js +1 -0
- package/dist/contracts/typescript/context.js +1 -0
- package/dist/contracts/typescript/datasets.js +1 -0
- package/dist/contracts/typescript/evidence-coverage.js +1 -0
- package/dist/contracts/typescript/export.js +1 -0
- package/dist/contracts/typescript/github-directory.js +1 -0
- package/dist/contracts/typescript/insight.js +1 -0
- package/dist/contracts/typescript/inspection.js +1 -0
- package/dist/contracts/typescript/intake.js +1 -0
- package/dist/contracts/typescript/local-claim.js +1 -0
- package/dist/contracts/typescript/local-configuration.js +1 -0
- package/dist/contracts/typescript/local-evidence.js +1 -0
- package/dist/contracts/typescript/local-grant.js +1 -0
- package/dist/contracts/typescript/local-input.js +1 -0
- package/dist/contracts/typescript/local-runner.js +1 -0
- package/dist/contracts/typescript/local-trial-live.js +1 -0
- package/dist/contracts/typescript/local-value-source.js +1 -0
- package/dist/contracts/typescript/local-world-native.js +1 -0
- package/dist/contracts/typescript/mcp.js +1 -0
- package/dist/contracts/typescript/media.js +1 -0
- package/dist/contracts/typescript/native-check.js +1 -0
- package/dist/contracts/typescript/native-identity.js +1 -0
- package/dist/contracts/typescript/onboarding.js +1 -0
- package/dist/contracts/typescript/performance.js +1 -0
- package/dist/contracts/typescript/planning.js +1 -0
- package/dist/contracts/typescript/preparation.js +1 -0
- package/dist/contracts/typescript/real-operation.js +1 -0
- package/dist/contracts/typescript/recommendation.js +1 -0
- package/dist/contracts/typescript/reports.js +1 -0
- package/dist/contracts/typescript/resource-lifecycle.js +1 -0
- package/dist/contracts/typescript/saved-record.js +1 -0
- package/dist/contracts/typescript/scenarios.js +1 -0
- package/dist/contracts/typescript/services.js +1 -0
- package/dist/contracts/typescript/session-command.js +1 -0
- package/dist/contracts/typescript/session-state.js +1 -0
- package/dist/contracts/typescript/session-voice.js +1 -0
- package/dist/contracts/typescript/session.js +1 -0
- package/dist/contracts/typescript/tool-command.js +1 -0
- package/dist/contracts/typescript/tool-connection.js +1 -0
- package/dist/contracts/typescript/tool-contribution.js +1 -0
- package/dist/contracts/typescript/tool-groups.js +1 -0
- package/dist/contracts/typescript/tool-lifecycle.js +1 -0
- package/dist/contracts/typescript/tool-package.js +1 -0
- package/dist/contracts/typescript/tool-state.js +1 -0
- package/dist/contracts/typescript/trial-state.js +1 -0
- package/dist/contracts/typescript/vault.js +1 -0
- package/dist/contracts/typescript/workspace-team.js +1 -0
- package/dist/contracts/typescript/world-stream.js +1 -0
- package/dist/contracts/workspace-team-contracts.js +55 -0
- package/dist/contracts/world-command.js +65 -0
- package/dist/contracts/world-lifetime-contracts.js +74 -0
- package/dist/contracts/world-recipe-contracts.js +27 -0
- package/dist/contracts/world-stream-contracts.js +27 -0
- package/dist/package.json +1 -0
- package/package.json +16 -16
- package/LICENSE.md +0 -23
- package/PACKAGING.json +0 -50
|
@@ -0,0 +1,1562 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* One native drill attempt (Plan20 W09, spec 3.5).
|
|
3
|
+
*
|
|
4
|
+
* `runDrill(assignment, deps)`:
|
|
5
|
+
* 1. verify the attempt (ids, lease epoch, state generation, expiry, descriptor,
|
|
6
|
+
* plan and adapter digests, budgets);
|
|
7
|
+
* 2. check that the approved plan routes every declared dependency (before
|
|
8
|
+
* waiting for root); once root prepared the drill, start the per-drill
|
|
9
|
+
* loopback dependency proxy, validate the synthetic identity, and build
|
|
10
|
+
* per-service dependency env (and the drill-scoped MCP client configuration
|
|
11
|
+
* when the plan names its variable);
|
|
12
|
+
* 3. apply target state resets (approved hooks) through target-state;
|
|
13
|
+
* 4. ensure shared services, start per-drill service copies;
|
|
14
|
+
* 5. prepare the driver; loop `nextSteps` (root's offered actions) within budgets. For
|
|
15
|
+
* each step: lease/expiry check, `intent_id` required for non-idempotent
|
|
16
|
+
* steps, `stepStarted` (the claim) before invoke, invoke only the dispatch
|
|
17
|
+
* the claim returned, with a bounded timeout; buffer observations; for a
|
|
18
|
+
* message step, record the reply as a v2 target message and wait for its
|
|
19
|
+
* acknowledgement; then `completeStep`. A recovered claim is never invoked,
|
|
20
|
+
* and an `unknown` outcome is never replayed: both are marked handled by id.
|
|
21
|
+
* A claimed input the driver itself says never reached the target (refused
|
|
22
|
+
* here, or nothing written) is withdrawn with a signed `abandon`, and the
|
|
23
|
+
* drill ends; unknown delivery is never declared;
|
|
24
|
+
* 6. stop on cloud cancel, local stop, lease expiry, budget exhaustion or
|
|
25
|
+
* revocation; close driver sessions; stop and reap owned per-drill services;
|
|
26
|
+
* flush observations; then close the execution with a signed close that
|
|
27
|
+
* states, for every declared service, whether its owned process group was
|
|
28
|
+
* reaped (with its exit code) or an attached process was only detached.
|
|
29
|
+
*
|
|
30
|
+
* Commands, argv and paths come only from the locally approved plan; nothing in
|
|
31
|
+
* the assignment or a step is executed. `runDrill` never throws: every failure
|
|
32
|
+
* becomes a reported outcome with a stable reason code.
|
|
33
|
+
*/
|
|
34
|
+
import { CliError } from '../http.js';
|
|
35
|
+
import { COMPLETE_ENDINGS, ENDING_TEXT, NATIVE_ERRORS, NATIVE_ID, SHA256_HEX, SecretValue, activeTime, nativeError } from './contracts.js';
|
|
36
|
+
import { sha256Hex } from './canonical.js';
|
|
37
|
+
import { dependencyEnv, mcpServers, validateDependencyRoutes } from './bindings.js';
|
|
38
|
+
import { DEPENDENCY_ERRORS, DependencyProxy } from './dependency-proxy.js';
|
|
39
|
+
import { writeMcpConfig } from './mcp-config.js';
|
|
40
|
+
import { applyDrillState, planStateActions } from './target-state.js';
|
|
41
|
+
import { READINESS_ERRORS, readinessDomains, readinessServices, resetEvidence } from './readiness.js';
|
|
42
|
+
import { identitySecrets, validateIdentityGrant } from './target-auth.js';
|
|
43
|
+
import { buildServiceEnv, exitCode } from './supervisor.js';
|
|
44
|
+
import { localSecretBindings } from './local-secrets.js';
|
|
45
|
+
import { allocateLoopbackPort } from './ports.js';
|
|
46
|
+
import { driverFor } from './drivers/index.js';
|
|
47
|
+
import { DRIVER_ERRORS } from './drivers/types.js';
|
|
48
|
+
import { ObservationBuffer } from './observations.js';
|
|
49
|
+
import { Wakeup } from './companion-channel.js';
|
|
50
|
+
import { backoffMs, errorCode, isFenced, isRevocation, isTransient, retryTransient, sleep } from './control-plane.js';
|
|
51
|
+
/** Reason codes for drill outcomes that are not thrown errors (proposed additions to NATIVE_ERRORS). */
|
|
52
|
+
export const DRILL_REASONS = {
|
|
53
|
+
companion_stopped: 'native_companion_stopped',
|
|
54
|
+
intent_missing: 'native_intent_missing',
|
|
55
|
+
step_invalid: 'native_step_invalid',
|
|
56
|
+
assignment_invalid: 'native_assignment_invalid',
|
|
57
|
+
state_reset_failed: 'native_state_reset_failed',
|
|
58
|
+
drill_failed: 'native_drill_failed',
|
|
59
|
+
step_report_failed: 'native_step_report_failed',
|
|
60
|
+
/** A message step finished but its reply was not acknowledged as evidence (or there was none). */
|
|
61
|
+
reply_undelivered: 'native_target_reply_undelivered',
|
|
62
|
+
/** A task or event step finished but its result was not acknowledged as evidence. */
|
|
63
|
+
result_undelivered: 'native_target_result_undelivered',
|
|
64
|
+
/** The target's retained task or event result said it failed; root then ends the input. */
|
|
65
|
+
target_failed: 'native_target_failed',
|
|
66
|
+
/** The target's retained task or event result said it was cancelled; root then ends the input. */
|
|
67
|
+
target_cancelled: 'native_target_cancelled',
|
|
68
|
+
/**
|
|
69
|
+
* Root ended the input because it could not generate the simulated participant's next message in time
|
|
70
|
+
* (`participant_unavailable`). An infrastructure ending: the drill closes incomplete, never as a result.
|
|
71
|
+
*/
|
|
72
|
+
participant_unavailable: 'native_participant_unavailable',
|
|
73
|
+
/** The service never reported the accepted drill as prepared before it had to stop. */
|
|
74
|
+
not_prepared: 'native_drill_not_prepared',
|
|
75
|
+
/** The service began draining the drill (cancelled, or capacity or startup refused) before it started. */
|
|
76
|
+
draining: 'native_assignment_draining',
|
|
77
|
+
/** The service did not confirm the stop, so the drill has no exact cutoff to close on. */
|
|
78
|
+
stop_unconfirmed: 'native_stop_unconfirmed',
|
|
79
|
+
};
|
|
80
|
+
/** Root execution states in which the local target may start (after capacity and dependency bindings are prepared). */
|
|
81
|
+
const PREPARED_STATES = new Set(['starting', 'ready', 'executing']);
|
|
82
|
+
/** States after root admitted the drill's readiness: target actions may follow. */
|
|
83
|
+
const ADMITTED_STATES = new Set(['ready', 'executing']);
|
|
84
|
+
/** States in which an accepted drill is still being prepared by the service. */
|
|
85
|
+
const PREPARING_STATES = new Set(['accepted', 'provisioning']);
|
|
86
|
+
const ROOT_TERMINAL_STATES = new Set(['rejected', 'expired', 'completed', 'failed', 'cancelled']);
|
|
87
|
+
const PREPARE_CHECK_MS = 1_000;
|
|
88
|
+
/** Lease renewal (which also reports root's state) backs off from 2s to 15s while the drill is being prepared. */
|
|
89
|
+
const PREPARE_RENEW_MIN_MS = 2_000;
|
|
90
|
+
const PREPARE_RENEW_MAX_MS = 15_000;
|
|
91
|
+
const LEASE_MARGIN_MS = 2_000;
|
|
92
|
+
/**
|
|
93
|
+
* Root refuses a close while a participant response it already dispatched is
|
|
94
|
+
* unfinished (`native_execution_close_pending`) or its ending projection is
|
|
95
|
+
* still pending. Those end within root's
|
|
96
|
+
* participant deadline (at most 90 s), so the same close is resubmitted for
|
|
97
|
+
* this long while the lease holds.
|
|
98
|
+
*/
|
|
99
|
+
const CLOSE_PENDING_WAIT_MS = 120_000;
|
|
100
|
+
const MAX_DRIVER_SESSIONS = 64;
|
|
101
|
+
const EVENT_SESSION_KEY = 'events';
|
|
102
|
+
// ---------------------------------------------------------------------------
|
|
103
|
+
// Service pool: one Supervisor per companion run. Shared services (attach, or
|
|
104
|
+
// launch with per_drill false) start once; per_drill launch services start a
|
|
105
|
+
// fresh copy per drill with that drill's binding env.
|
|
106
|
+
function isPerDrill(service) {
|
|
107
|
+
return service.mode === 'launch' && service.per_drill;
|
|
108
|
+
}
|
|
109
|
+
/** Topological order of plan services following descriptor depends_on (stable by plan order). */
|
|
110
|
+
function startupOrder(plan, descriptor) {
|
|
111
|
+
const depends = new Map(descriptor.services.map((service) => [service.alias, service.depends_on]));
|
|
112
|
+
const order = [];
|
|
113
|
+
const state = new Map();
|
|
114
|
+
const visit = (alias) => {
|
|
115
|
+
const mark = state.get(alias);
|
|
116
|
+
if (mark === 'done')
|
|
117
|
+
return;
|
|
118
|
+
if (mark === 'visiting')
|
|
119
|
+
throw nativeError('plan_changed', 'Services were not started: the descriptor service dependencies form a cycle.');
|
|
120
|
+
state.set(alias, 'visiting');
|
|
121
|
+
for (const dependency of depends.get(alias) ?? [])
|
|
122
|
+
visit(dependency);
|
|
123
|
+
state.set(alias, 'done');
|
|
124
|
+
order.push(alias);
|
|
125
|
+
};
|
|
126
|
+
for (const service of plan.services)
|
|
127
|
+
visit(service.alias);
|
|
128
|
+
const planned = new Set(plan.services.map((service) => service.alias));
|
|
129
|
+
return order.filter((alias) => planned.has(alias));
|
|
130
|
+
}
|
|
131
|
+
export class ServicePool {
|
|
132
|
+
supervisor;
|
|
133
|
+
#plan;
|
|
134
|
+
#order;
|
|
135
|
+
#signal;
|
|
136
|
+
constructor(options) {
|
|
137
|
+
this.supervisor = options.supervisor;
|
|
138
|
+
this.#plan = options.plan;
|
|
139
|
+
this.#signal = options.signal;
|
|
140
|
+
this.#order = startupOrder(options.plan, options.descriptor);
|
|
141
|
+
const perDrill = new Set(options.plan.services.filter(isPerDrill).map((service) => service.alias));
|
|
142
|
+
const depends = new Map(options.descriptor.services.map((service) => [service.alias, service.depends_on]));
|
|
143
|
+
for (const service of options.plan.services) {
|
|
144
|
+
if (perDrill.has(service.alias))
|
|
145
|
+
continue;
|
|
146
|
+
if ((depends.get(service.alias) ?? []).some((dependency) => perDrill.has(dependency))) {
|
|
147
|
+
throw nativeError('plan_changed', `Services were not started: shared service "${service.alias}" depends on a per-drill service. Make it per-drill or change the plan.`);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
get perDrillAliases() {
|
|
152
|
+
const perDrill = new Set(this.#plan.services.filter(isPerDrill).map((service) => service.alias));
|
|
153
|
+
return this.#order.filter((alias) => perDrill.has(alias));
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Start or reuse every shared service. Not cached here: the supervisor reuses
|
|
157
|
+
* live shared entries and dedupes concurrent starts, so each drill gets current
|
|
158
|
+
* handles rather than a dead one. A shared service of the campaign's local run
|
|
159
|
+
* starts once under its registered launch id and is never restarted (an ended
|
|
160
|
+
* one fails the drill with `native_local_run_service_ended`).
|
|
161
|
+
*/
|
|
162
|
+
async shared() {
|
|
163
|
+
const perDrill = new Set(this.perDrillAliases);
|
|
164
|
+
const aliases = this.#order.filter((alias) => !perDrill.has(alias));
|
|
165
|
+
if (aliases.length === 0)
|
|
166
|
+
return new Map();
|
|
167
|
+
const handles = await this.supervisor.start(aliases, { drillKey: 'shared', env: {}, secrets: [], signal: this.#signal });
|
|
168
|
+
return new Map(handles.map((handle) => [handle.alias, handle]));
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Start one fresh copy of each per-drill service, in dependency order, one
|
|
172
|
+
* supervisor call per service so each receives only its own binding env
|
|
173
|
+
* (dependencies already started for this drill key are reused by the
|
|
174
|
+
* supervisor). The jsonl-process driver's own service is skipped by the
|
|
175
|
+
* supervisor; the driver starts it with the driver env.
|
|
176
|
+
*/
|
|
177
|
+
async startPerDrill(drillKey, envFor, signal, onStarted) {
|
|
178
|
+
for (const alias of this.perDrillAliases) {
|
|
179
|
+
if (signal.aborted)
|
|
180
|
+
throw nativeError('disconnected', 'Per-drill services were not all started: the drill was stopped.');
|
|
181
|
+
const { env, secrets } = envFor(alias);
|
|
182
|
+
const handles = await this.supervisor.start([alias], { drillKey, env, secrets, signal });
|
|
183
|
+
for (const handle of handles)
|
|
184
|
+
onStarted(handle);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
/** Stop every per-drill copy this drill key owns (reverse start order). Shared services keep running. */
|
|
188
|
+
async stopDrill(drillKey, reason) {
|
|
189
|
+
await this.supervisor.stopDrill(drillKey, reason.slice(0, 64)).catch(() => undefined);
|
|
190
|
+
}
|
|
191
|
+
/** Every per-drill process launched for this drill key, ready or not (once, after stopDrill), for the signed close. */
|
|
192
|
+
takeLaunches(drillKey) {
|
|
193
|
+
return this.supervisor.takeDrillLaunches(drillKey);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
// ---------------------------------------------------------------------------
|
|
197
|
+
// Attempt verification.
|
|
198
|
+
function leaseRemainingMs(assignment) {
|
|
199
|
+
const expires = Date.parse(assignment.lease_expires_at);
|
|
200
|
+
return Number.isFinite(expires) ? expires - Date.now() : -1;
|
|
201
|
+
}
|
|
202
|
+
function verifyAssignment(assignment, deps) {
|
|
203
|
+
const invalid = (what) => new CliError(DRILL_REASONS.assignment_invalid, `The drill was not started: the assignment ${what}.`);
|
|
204
|
+
const int = (value, min) => typeof value === 'number' && Number.isSafeInteger(value) && value >= min;
|
|
205
|
+
if (!NATIVE_ID.test(assignment.assignment_id ?? '') || !NATIVE_ID.test(assignment.attempt_id ?? ''))
|
|
206
|
+
throw invalid('identity is malformed');
|
|
207
|
+
if (!int(assignment.lease_epoch, 1))
|
|
208
|
+
throw invalid('lease epoch is malformed');
|
|
209
|
+
if (!int(assignment.state_generation, 1))
|
|
210
|
+
throw invalid('state generation is malformed');
|
|
211
|
+
if (!int(assignment.step_cursor, 0) || !int(assignment.observation_ack, 0))
|
|
212
|
+
throw invalid('cursor is malformed');
|
|
213
|
+
if (leaseRemainingMs(assignment) <= LEASE_MARGIN_MS)
|
|
214
|
+
throw nativeError('lease_expired', 'The drill was not started: its lease has expired.');
|
|
215
|
+
if (!SHA256_HEX.test(assignment.descriptor_sha256 ?? '') || assignment.descriptor_sha256 !== deps.plan.descriptor_sha256) {
|
|
216
|
+
throw nativeError('plan_changed', 'The drill was not started: it targets a different descriptor than the approved plan.');
|
|
217
|
+
}
|
|
218
|
+
const driver = assignment.driver;
|
|
219
|
+
if (!driver || driver.profile !== deps.descriptor.driver.profile || driver.profile !== deps.plan.driver.profile
|
|
220
|
+
|| driver.adapter_sha256 !== deps.descriptor.driver.adapter_sha256) {
|
|
221
|
+
throw nativeError('plan_changed', 'The drill was not started: its driver or adapter digest differs from the approved plan.');
|
|
222
|
+
}
|
|
223
|
+
const budgets = assignment.budgets;
|
|
224
|
+
if (!budgets || !int(budgets.max_steps, 1) || !int(budgets.max_elapsed_ms, 1) || !int(budgets.step_timeout_ms, 1)
|
|
225
|
+
|| !int(budgets.max_observation_bytes, 1))
|
|
226
|
+
throw invalid('budgets are malformed');
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* A service whose process starts afresh for every drill, so it can receive the
|
|
230
|
+
* drill's dependency URLs: a per-drill launch, or the service the jsonl-process
|
|
231
|
+
* driver starts itself for each drill. Shared and attached processes cannot.
|
|
232
|
+
*/
|
|
233
|
+
export function freshForDrill(plan) {
|
|
234
|
+
return (service) => isPerDrill(service)
|
|
235
|
+
|| (plan.driver.profile === 'jsonl-process.v1' && plan.driver.service_alias === service.alias);
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Env for driver-owned child processes (jsonl-process service, function wrapper),
|
|
239
|
+
* built with the supervisor's rules (empty base, passthrough names, env_static,
|
|
240
|
+
* binding env; undeclared secret configuration stripped).
|
|
241
|
+
*/
|
|
242
|
+
async function driverEnv(service, descriptor, binding, plan, services, signal) {
|
|
243
|
+
if (!service) {
|
|
244
|
+
const base = process.platform === 'win32'
|
|
245
|
+
? ['PATH', 'SystemRoot', 'TEMP', 'TMP', 'USERPROFILE', 'PATHEXT', 'ComSpec', 'LANG']
|
|
246
|
+
: ['PATH', 'HOME', 'LANG', 'TMPDIR'];
|
|
247
|
+
const env = {};
|
|
248
|
+
for (const name of base) {
|
|
249
|
+
const value = process.env[name];
|
|
250
|
+
if (typeof value === 'string')
|
|
251
|
+
env[name] = value;
|
|
252
|
+
}
|
|
253
|
+
return { ...env, ...binding };
|
|
254
|
+
}
|
|
255
|
+
const declared = descriptor.services.find((entry) => entry.alias === service.alias);
|
|
256
|
+
const configuration = declared?.configuration ?? [];
|
|
257
|
+
const secretNames = new Set(configuration.filter((item) => item.secret).map((item) => item.name));
|
|
258
|
+
// Same port rules as supervisor-launched services: an allocated loopback port
|
|
259
|
+
// under port_env, and each started dependency's port under its own port_env.
|
|
260
|
+
const port = service.port_env ? await allocateLoopbackPort(signal) : null;
|
|
261
|
+
const dependencyPorts = [];
|
|
262
|
+
for (const dependency of declared?.depends_on ?? []) {
|
|
263
|
+
const handle = services.get(dependency), dependencyPlan = plan.services.find((entry) => entry.alias === dependency);
|
|
264
|
+
if (handle?.port && dependencyPlan?.port_env)
|
|
265
|
+
dependencyPorts.push([dependencyPlan.port_env, handle.port]);
|
|
266
|
+
}
|
|
267
|
+
return buildServiceEnv({ service, secretNames, bindingEnv: binding, port, dependencyPorts });
|
|
268
|
+
}
|
|
269
|
+
/** Step shape checks before anything is sent. Returns a reason code or null. */
|
|
270
|
+
function stepProblem(step) {
|
|
271
|
+
if (!step || typeof step !== 'object' || typeof step.step_id !== 'string' || !NATIVE_ID.test(step.step_id))
|
|
272
|
+
return DRILL_REASONS.step_invalid;
|
|
273
|
+
if (!Number.isSafeInteger(step.sequence) || step.sequence < 1)
|
|
274
|
+
return DRILL_REASONS.step_invalid;
|
|
275
|
+
if (!step.payload || step.payload.kind !== step.kind || !['message', 'task', 'event'].includes(step.kind))
|
|
276
|
+
return DRILL_REASONS.step_invalid;
|
|
277
|
+
if (typeof step.idempotent !== 'boolean')
|
|
278
|
+
return DRILL_REASONS.step_invalid;
|
|
279
|
+
if (!step.idempotent && (typeof step.intent_id !== 'string' || !NATIVE_ID.test(step.intent_id)))
|
|
280
|
+
return DRILL_REASONS.intent_missing;
|
|
281
|
+
if (step.payload.kind !== 'event' && (typeof step.payload.session_key !== 'string' || !NATIVE_ID.test(step.payload.session_key))) {
|
|
282
|
+
return DRILL_REASONS.step_invalid;
|
|
283
|
+
}
|
|
284
|
+
return null;
|
|
285
|
+
}
|
|
286
|
+
// ---------------------------------------------------------------------------
|
|
287
|
+
// The drill.
|
|
288
|
+
/** Driver codes for an input the client declined to send (versus one it tried and could not deliver). */
|
|
289
|
+
const REFUSAL_CODES = new Set([DRIVER_ERRORS.intent_missing, DRIVER_ERRORS.step_unsupported,
|
|
290
|
+
DRIVER_ERRORS.session_mismatch, DRIVER_ERRORS.session_closed, DRIVER_ERRORS.event_invalid, DRIVER_ERRORS.not_loopback,
|
|
291
|
+
DRIVER_ERRORS.driver_plan_invalid]);
|
|
292
|
+
export async function runDrill(assignment, deps) {
|
|
293
|
+
const startedAt = Date.now();
|
|
294
|
+
const { controlPlane, plan, descriptor, output } = deps;
|
|
295
|
+
const drillKey = `d${sha256Hex(`${assignment.assignment_id}\0${assignment.attempt_id}`).slice(0, 24)}`;
|
|
296
|
+
const label = `drill ${typeof assignment.drill_id === 'string' && NATIVE_ID.test(assignment.drill_id) ? assignment.drill_id : 'assignment'}`;
|
|
297
|
+
const transport = deps.abortSignal;
|
|
298
|
+
const retry = { signal: transport, attempts: 5, onTransient: () => { deps.gate.disconnected(); } };
|
|
299
|
+
const secrets = [];
|
|
300
|
+
const sessions = new Map();
|
|
301
|
+
const drillController = new AbortController();
|
|
302
|
+
const onStop = () => drillController.abort();
|
|
303
|
+
deps.abortSignal.addEventListener('abort', onStop, { once: true });
|
|
304
|
+
let stop = null;
|
|
305
|
+
/** Why root said the participant's input ended, when it did. */
|
|
306
|
+
let endingReason = null;
|
|
307
|
+
let schedule = null;
|
|
308
|
+
const stopWith = (outcome, reason) => { if (!stop)
|
|
309
|
+
stop = { outcome, reason }; };
|
|
310
|
+
/** Set when lease renewal was refused (fence or revocation); checked before every step. */
|
|
311
|
+
let leaseStop = null;
|
|
312
|
+
let leaseDeadlineTimer;
|
|
313
|
+
const watchLeaseDeadline = () => {
|
|
314
|
+
if (drillController.signal.aborted)
|
|
315
|
+
return;
|
|
316
|
+
const remaining = leaseRemainingMs(assignment) - LEASE_MARGIN_MS;
|
|
317
|
+
if (!Number.isFinite(remaining) || remaining <= 0) {
|
|
318
|
+
leaseStop ??= { outcome: 'incomplete', reason: NATIVE_ERRORS.lease_expired };
|
|
319
|
+
drillController.abort();
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
// Renewal mutates the original assignment expiry. Re-read it when this
|
|
323
|
+
// deadline wakes rather than treating the first short lease as a turn limit.
|
|
324
|
+
leaseDeadlineTimer = setTimeout(watchLeaseDeadline, Math.min(remaining, 2_147_483_647));
|
|
325
|
+
};
|
|
326
|
+
drillController.signal.addEventListener('abort', () => {
|
|
327
|
+
if (leaseDeadlineTimer !== undefined)
|
|
328
|
+
clearTimeout(leaseDeadlineTimer);
|
|
329
|
+
}, { once: true });
|
|
330
|
+
const leaseControl = new AbortController();
|
|
331
|
+
let leaseKeeper = Promise.resolve();
|
|
332
|
+
/**
|
|
333
|
+
* The companion's campaign channel (one per device), when this drill is in its campaign. Nothing is read, stepped
|
|
334
|
+
* or started while it waits for a valid resync; without it (no channel) the drill re-reads on bounded timers.
|
|
335
|
+
*/
|
|
336
|
+
const channel = controlPlane.wakes && assignment.campaign_id === controlPlane.wakes.campaign_id ? controlPlane.wakes : null;
|
|
337
|
+
/** Root's wake-ups for this assignment from the channel. */
|
|
338
|
+
let assignmentWake = null;
|
|
339
|
+
let unwatchChannel = () => { };
|
|
340
|
+
/**
|
|
341
|
+
* The drill's local waits: woken by the channel (this assignment, a resync, a state change) and by local news (root's
|
|
342
|
+
* state from a lease renewal, a lost lease, a failed dependency), so a wait never polls root.
|
|
343
|
+
*/
|
|
344
|
+
const progress = new Wakeup();
|
|
345
|
+
/** Until the next time-based halt (the lease margin or the drill's elapsed budget), so a local wait re-checks it on time. */
|
|
346
|
+
const haltDue = () => AbortSignal.timeout(Math.max(1, Math.min(2_147_483_647, leaseRemainingMs(assignment) - LEASE_MARGIN_MS, assignment.budgets.max_elapsed_ms - (Date.now() - startedAt))));
|
|
347
|
+
let buffer = null;
|
|
348
|
+
let tracker = null;
|
|
349
|
+
let driver = null;
|
|
350
|
+
let mcpConfigs = [];
|
|
351
|
+
/** The per-drill loopback proxy to root's tool gateway, when the target declares dependencies. */
|
|
352
|
+
let proxy = null;
|
|
353
|
+
/** Set when root refused dependency access for good (fence, revocation, closed, changed binding); checked before every step. */
|
|
354
|
+
let dependencyStop = null;
|
|
355
|
+
let scratch = null;
|
|
356
|
+
let steps = 0;
|
|
357
|
+
/** Invoked steps whose invocation completed, for the close's client summary. */
|
|
358
|
+
let completedSteps = 0;
|
|
359
|
+
let unknownSteps = 0;
|
|
360
|
+
let cursor = assignment.step_cursor;
|
|
361
|
+
/**
|
|
362
|
+
* Actions this attempt claimed (invoked or recovered): never claimed or invoked again. Root numbers participant
|
|
363
|
+
* turns and scripted, task and event actions separately, so two actions can share a sequence; ids are the cursor.
|
|
364
|
+
*/
|
|
365
|
+
const handled = new Set();
|
|
366
|
+
/** A step whose claim got no answer: root hides a claimed action from the offered list, so it is replayed with the same key. */
|
|
367
|
+
let unconfirmed = null;
|
|
368
|
+
/** A step result the cloud has not confirmed yet (transient failure); re-sent before the next step, never re-invoked. */
|
|
369
|
+
let pendingReport = null;
|
|
370
|
+
/** This client claimed a target action that root has not seen completed; root refuses a close until it is resolved. */
|
|
371
|
+
let claimOpen = false;
|
|
372
|
+
/** Every service handle this drill used (shared and per-drill), for the signed close. */
|
|
373
|
+
let drillServices = new Map();
|
|
374
|
+
/** Root's execution state as last reported by a lease renewal; an accepted offer starts as `accepted`. */
|
|
375
|
+
let rootState = 'accepted';
|
|
376
|
+
/** Root reported the drill prepared, so it accepts evidence and a close. */
|
|
377
|
+
let prepared = false;
|
|
378
|
+
const requireLaunchAuthority = async () => {
|
|
379
|
+
for (;;) {
|
|
380
|
+
const halt = leaseStop ?? dependencyStop ?? preStepHalt(assignment, deps, startedAt, steps) ?? prepareHalt(rootState);
|
|
381
|
+
if (halt)
|
|
382
|
+
throw new CliError(halt.reason ?? DRILL_REASONS.not_prepared, 'The drill can no longer start local work.');
|
|
383
|
+
if (drillController.signal.aborted || channel?.state === 'closed') {
|
|
384
|
+
throw new CliError('native_stream_disconnected', 'The runner connection closed before launch. No new process was started.');
|
|
385
|
+
}
|
|
386
|
+
if (deps.gate.connected && (!channel || channel.state === 'live'))
|
|
387
|
+
return;
|
|
388
|
+
// A wake timer is a snapshot of the current lease, not a revocation.
|
|
389
|
+
// The server may renew while the notification channel is reconnecting.
|
|
390
|
+
// Re-read the live lease and budget before waiting again or launching.
|
|
391
|
+
const signal = AbortSignal.any([deps.stopSignal, transport, drillController.signal, haltDue()]);
|
|
392
|
+
if (await deps.gate.wait(signal) && channel)
|
|
393
|
+
await channel.live(signal);
|
|
394
|
+
}
|
|
395
|
+
};
|
|
396
|
+
const observe = (observation) => { buffer?.add(observation); };
|
|
397
|
+
/**
|
|
398
|
+
* Withdraw this client's claim of a step whose input never reached the target, with the device's signed
|
|
399
|
+
* declaration (refused here, or not delivered). Root cancels the action and drains the drill, so the close
|
|
400
|
+
* no longer waits on it. It is never proof of a target failure, and a refused withdrawal leaves the claim
|
|
401
|
+
* for root's normal close and cleanup to settle.
|
|
402
|
+
*/
|
|
403
|
+
const withdrawClaim = async (step, code) => {
|
|
404
|
+
const reason = code !== null && REFUSAL_CODES.has(code) ? 'client_refused' : 'not_delivered';
|
|
405
|
+
try {
|
|
406
|
+
const withdrawal = await retryTransient(() => controlPlane.abandonStep(assignment, step, reason, transport), retry);
|
|
407
|
+
claimOpen = false;
|
|
408
|
+
observe({ kind: 'action.observed', at_ms: Date.now(), content: { type: 'target.withdrawn', step_id: step.step_id, reason } });
|
|
409
|
+
// The code says why (a driver refusal or delivery failure, or a runner-side halt such as a reconnecting
|
|
410
|
+
// stream, lease, budget or dependency stop), since `reason` alone cannot tell them apart.
|
|
411
|
+
output('native_step_withdrawn', { assignment_id: assignment.assignment_id, step_id: step.step_id, sequence: step.sequence,
|
|
412
|
+
reason, error_code: code, withdrawn_at: withdrawal.withdrawn_at }, `${label}: step ${step.sequence} never reached the target (${reason === 'client_refused' ? 'refused on this machine' : 'not delivered'}`
|
|
413
|
+
+ `${code ? `, ${code}` : ''}); its claim was withdrawn and the drill is ending.`);
|
|
414
|
+
}
|
|
415
|
+
catch (error) {
|
|
416
|
+
output('native_step_withdraw_failed', { assignment_id: assignment.assignment_id, step_id: step.step_id, error_code: errorCode(error) }, `${label}: step ${step.sequence}'s claim could not be withdrawn (${errorCode(error) ?? 'unknown error'}); inspect its original cleanup status. Do not repeat the target action.`);
|
|
417
|
+
}
|
|
418
|
+
};
|
|
419
|
+
try {
|
|
420
|
+
verifyAssignment(assignment, deps);
|
|
421
|
+
watchLeaseDeadline();
|
|
422
|
+
if (channel) {
|
|
423
|
+
assignmentWake = channel.subscribe({ kind: 'assignment', id: assignment.assignment_id });
|
|
424
|
+
assignmentWake.onWake(() => progress.wake());
|
|
425
|
+
unwatchChannel = channel.onState(() => progress.wake());
|
|
426
|
+
}
|
|
427
|
+
// Root leases last about a minute; keep this one live until the evidence is flushed.
|
|
428
|
+
leaseKeeper = keepLease(assignment, controlPlane, AbortSignal.any([leaseControl.signal, transport]), (state) => { leaseStop = state; drillController.abort(); progress.wake(); }, (state) => { if (state !== rootState) {
|
|
429
|
+
rootState = state;
|
|
430
|
+
progress.wake();
|
|
431
|
+
} }, () => !ADMITTED_STATES.has(rootState), assignmentWake);
|
|
432
|
+
buffer = new ObservationBuffer({
|
|
433
|
+
controlPlane, assignment, startedAtMs: startedAt, signal: transport,
|
|
434
|
+
maxBufferedBytes: deps.observationBufferBytes, onTransient: () => { deps.gate.disconnected(); },
|
|
435
|
+
});
|
|
436
|
+
// Root admits a drill only with reset evidence for every local state domain:
|
|
437
|
+
// one this client cannot reset is refused before anything is waited for or started.
|
|
438
|
+
const statePlan = planStateActions(descriptor, plan);
|
|
439
|
+
const resetDomains = readinessDomains(descriptor, statePlan);
|
|
440
|
+
// Every declared dependency must reach the target only through this drill's
|
|
441
|
+
// proxy; an unrouted one is refused now, never left to reach a live service.
|
|
442
|
+
const routes = validateDependencyRoutes(descriptor, plan, { freshForDrill: freshForDrill(plan) });
|
|
443
|
+
// Accepting does not start the drill: root's controller reserves capacity and
|
|
444
|
+
// prepares bindings, then reports it as starting. Nothing starts locally before that,
|
|
445
|
+
// nor while the campaign channel waits for a valid resync.
|
|
446
|
+
if (!PREPARED_STATES.has(rootState)) {
|
|
447
|
+
output('native_drill_waiting', { assignment_id: assignment.assignment_id, state: rootState }, `Waiting for the service to prepare ${label}.`);
|
|
448
|
+
}
|
|
449
|
+
for (;;) {
|
|
450
|
+
const seen = progress.generation;
|
|
451
|
+
if (PREPARED_STATES.has(rootState) && (!channel || channel.state === 'live'))
|
|
452
|
+
break;
|
|
453
|
+
const halt = leaseStop ?? dependencyStop ?? preStepHalt(assignment, deps, startedAt, steps) ?? prepareHalt(rootState);
|
|
454
|
+
if (halt) {
|
|
455
|
+
stopWith(halt.outcome, halt.reason ?? DRILL_REASONS.not_prepared);
|
|
456
|
+
throw new CliError(halt.reason ?? DRILL_REASONS.not_prepared, `${label} stopped before the service prepared it. Nothing started locally.`);
|
|
457
|
+
}
|
|
458
|
+
// Local only: a renewal reporting root's state, the channel or a time-based halt wakes it.
|
|
459
|
+
await progress.wait(seen, AbortSignal.any([deps.stopSignal, transport, channel ? haltDue() : AbortSignal.timeout(PREPARE_CHECK_MS)]));
|
|
460
|
+
}
|
|
461
|
+
prepared = true;
|
|
462
|
+
output('native_drill_started', { assignment_id: assignment.assignment_id, attempt_id: assignment.attempt_id,
|
|
463
|
+
drill_id: assignment.drill_id ?? null }, `Started ${label}.`);
|
|
464
|
+
// Identity, validated before anything starts.
|
|
465
|
+
const identity = validateIdentityGrant(assignment.identity ?? null, descriptor);
|
|
466
|
+
secrets.push(...identitySecrets(identity));
|
|
467
|
+
scratch = await deps.scratch.drill(drillKey);
|
|
468
|
+
// Dependencies: stable loopback URLs now; root's tokens are fetched lazily once
|
|
469
|
+
// the drill is admitted and never leave this process.
|
|
470
|
+
if (routes.length > 0) {
|
|
471
|
+
proxy = await DependencyProxy.start({
|
|
472
|
+
controlPlane, assignment, descriptor, signal: drillController.signal,
|
|
473
|
+
onExchange: (exchange) => {
|
|
474
|
+
if (exchange)
|
|
475
|
+
buffer?.add({ kind: 'action.observed', at_ms: Date.now(), content: {}, dependency_exchange: exchange });
|
|
476
|
+
else
|
|
477
|
+
buffer?.dependencyCaptureUnavailable();
|
|
478
|
+
},
|
|
479
|
+
// Root's 503 "no gateway" is permanent for the run, not a disconnection.
|
|
480
|
+
onFatal: (error) => {
|
|
481
|
+
dependencyStop ??= errorCode(error) === DEPENDENCY_ERRORS.unavailable
|
|
482
|
+
? { outcome: 'failed', reason: DEPENDENCY_ERRORS.unavailable } : classify(error);
|
|
483
|
+
progress.wake();
|
|
484
|
+
},
|
|
485
|
+
});
|
|
486
|
+
secrets.push(proxy.secret);
|
|
487
|
+
}
|
|
488
|
+
const urls = proxy ?? { url: () => { throw nativeError('binding_invalid', 'No dependency proxy runs for this drill.'); } };
|
|
489
|
+
const serviceEnv = new Map();
|
|
490
|
+
for (const service of plan.services) {
|
|
491
|
+
const mapped = dependencyEnv(routes, service.alias, urls);
|
|
492
|
+
const env = { ...mapped.env };
|
|
493
|
+
const own = [...mapped.secrets];
|
|
494
|
+
const mcp = await writeServiceMcpConfig(routes, service, urls, scratch.path);
|
|
495
|
+
if (mcp) {
|
|
496
|
+
mcpConfigs.push(mcp);
|
|
497
|
+
Object.assign(env, mcp.env);
|
|
498
|
+
own.push(...mcp.secrets);
|
|
499
|
+
}
|
|
500
|
+
const local = localSecretBindings(service, env);
|
|
501
|
+
own.push(...local.secrets);
|
|
502
|
+
serviceEnv.set(service.alias, { env: local.env, secrets: own });
|
|
503
|
+
secrets.push(...own);
|
|
504
|
+
}
|
|
505
|
+
// Target state resets (approved hooks) before the target starts.
|
|
506
|
+
await requireLaunchAuthority();
|
|
507
|
+
tracker = await applyDrillState(statePlan, {
|
|
508
|
+
plan, signal: AbortSignal.any([deps.stopSignal, transport]), drillKey,
|
|
509
|
+
});
|
|
510
|
+
for (const run of tracker.hookRuns) {
|
|
511
|
+
if (run.result === 'applied')
|
|
512
|
+
continue; // applied hooks are recorded as reset evidence once the drill is ready
|
|
513
|
+
observe({ kind: 'state.observed', at_ms: Date.now(), content: { type: 'state.reset', domain: run.domain_alias, result: run.result,
|
|
514
|
+
error_code: run.error_code, exit_code: run.exit_code, duration_ms: run.duration_ms } });
|
|
515
|
+
}
|
|
516
|
+
if (tracker.planChanged())
|
|
517
|
+
throw nativeError('plan_changed', 'The drill was not continued: an approved reset hook changed on disk. Approve the plan again.');
|
|
518
|
+
if (tracker.failures().length > 0)
|
|
519
|
+
throw new CliError(DRILL_REASONS.state_reset_failed, 'The drill was not started: a declared state reset failed.');
|
|
520
|
+
// Services: shared once per run, per-drill copies for this drill.
|
|
521
|
+
const startSignal = AbortSignal.any([deps.stopSignal, transport]);
|
|
522
|
+
await requireLaunchAuthority();
|
|
523
|
+
const services = new Map(await deps.services.shared());
|
|
524
|
+
drillServices = services;
|
|
525
|
+
// Root admits run-owned services only once it knows them running: report a
|
|
526
|
+
// first launch (or any change) and have it confirmed before readiness.
|
|
527
|
+
const localRun = deps.localRun ?? null;
|
|
528
|
+
if (localRun)
|
|
529
|
+
await retryTransient(() => localRun.sync(transport), retry);
|
|
530
|
+
const activeTracker = tracker;
|
|
531
|
+
await requireLaunchAuthority();
|
|
532
|
+
await deps.services.startPerDrill(drillKey, (alias) => serviceEnv.get(alias) ?? { env: {}, secrets: [] }, startSignal, (handle) => {
|
|
533
|
+
services.set(handle.alias, handle);
|
|
534
|
+
if (handle.drillKey === drillKey)
|
|
535
|
+
activeTracker.markFreshProcess(handle.alias);
|
|
536
|
+
});
|
|
537
|
+
// Driver.
|
|
538
|
+
driver = driverFor(plan.driver.profile, { plan });
|
|
539
|
+
const availability = await driver.describe();
|
|
540
|
+
if (!availability.available)
|
|
541
|
+
throw nativeError('driver_unavailable', `The drill was not started: the ${plan.driver.profile} driver is unavailable locally.`);
|
|
542
|
+
const driverService = 'service_alias' in plan.driver ? plan.driver.service_alias : descriptor.entry_service;
|
|
543
|
+
// The event receiver's signing secret: read here, by the name the approved plan gives, never from the cloud.
|
|
544
|
+
const signingName = plan.driver.profile === 'signed-http-event.v1' ? plan.driver.signing_secret_env ?? null : null;
|
|
545
|
+
const signingValue = signingName === null ? undefined : process.env[signingName];
|
|
546
|
+
if (signingName !== null && (typeof signingValue !== 'string' || signingValue.length === 0 || signingValue.includes('\0'))) {
|
|
547
|
+
throw nativeError('driver_unavailable', `The drill was not started: the approved event signing secret ${signingName} is not set in this environment.`);
|
|
548
|
+
}
|
|
549
|
+
const eventSigningKey = signingName === null ? null : new SecretValue(signingValue);
|
|
550
|
+
if (eventSigningKey)
|
|
551
|
+
secrets.push(eventSigningKey);
|
|
552
|
+
const ctx = {
|
|
553
|
+
plan, driver: plan.driver, adapterSha256: descriptor.driver.adapter_sha256, services, roots: deps.roots,
|
|
554
|
+
env: await driverEnv(plan.services.find((service) => service.alias === driverService), descriptor, serviceEnv.get(driverService)?.env ?? {}, plan, services, startSignal),
|
|
555
|
+
identity, eventSigningKey, secrets, scratch: scratch.path, signal: drillController.signal, observe,
|
|
556
|
+
};
|
|
557
|
+
await requireLaunchAuthority();
|
|
558
|
+
await driver.prepare(ctx);
|
|
559
|
+
// The jsonl-process driver starts its own service afresh for every drill
|
|
560
|
+
// (the supervisor skips it), so that service's fresh-process reset applied.
|
|
561
|
+
if (plan.driver.profile === 'jsonl-process.v1')
|
|
562
|
+
activeTracker.markFreshProcess(plan.driver.service_alias);
|
|
563
|
+
// Readiness. Each local domain's reset is committed as evidence; the signed
|
|
564
|
+
// readiness record cites those records, every service and the entry probe,
|
|
565
|
+
// and goes to root with the drill's only driver.started observation. Root then
|
|
566
|
+
// admits the drill (ready) on its own; this client never sets it. A resumed
|
|
567
|
+
// drill root already admitted does not send a second one.
|
|
568
|
+
const activeBuffer = buffer;
|
|
569
|
+
const activeDriver = driver;
|
|
570
|
+
const ownedLaunches = new Map((driver.ownedProcesses?.() ?? []).map((item) => [item.alias, item.launch_id]));
|
|
571
|
+
const freshLaunch = (alias) => {
|
|
572
|
+
const handle = services.get(alias);
|
|
573
|
+
return handle?.mode === 'launch' && handle.drillKey === drillKey ? handle.launchId : ownedLaunches.get(alias) ?? null;
|
|
574
|
+
};
|
|
575
|
+
const resetRecords = resetDomains.map((action) => ({ action, event_id: activeBuffer.add({ kind: 'state.observed', at_ms: Date.now(),
|
|
576
|
+
content: resetEvidence(action, activeTracker, freshLaunch) }, true) }));
|
|
577
|
+
if (!ADMITTED_STATES.has(rootState)) {
|
|
578
|
+
const entryProbe = await activeDriver.readinessProbe(ctx);
|
|
579
|
+
const readyServices = readinessServices(descriptor, plan, services, activeDriver, entryProbe, drillKey, deps.localRun?.runService ?? (() => null));
|
|
580
|
+
const undelivered = await activeBuffer.submitReadiness(async (sequence) => {
|
|
581
|
+
const stateResets = [];
|
|
582
|
+
for (const { action, event_id: eventId } of resetRecords) {
|
|
583
|
+
const record = eventId ? activeBuffer.record(eventId) : null;
|
|
584
|
+
if (!record || action.action === 'unavailable' || action.action === 'not-applicable')
|
|
585
|
+
return READINESS_ERRORS.evidence_missing;
|
|
586
|
+
stateResets.push({ alias: action.alias, reset: action.action, hook_sha256: action.descriptor_hook_sha256,
|
|
587
|
+
evidence_artifact_id: record.artifact_id, evidence_sha256: record.sha256 });
|
|
588
|
+
}
|
|
589
|
+
return retryTransient(() => controlPlane.recordReadiness(assignment, { observation_sequence: sequence,
|
|
590
|
+
descriptor_digest: assignment.descriptor_sha256, services: readyServices, state_resets: stateResets }, transport), retry);
|
|
591
|
+
});
|
|
592
|
+
if (undelivered !== null)
|
|
593
|
+
throw new CliError(undelivered, `${label} did not report readiness to the service, so no step was taken.`);
|
|
594
|
+
output('native_drill_admitting', { assignment_id: assignment.assignment_id, state: rootState }, `Ready locally; waiting for the service to admit ${label}.`);
|
|
595
|
+
for (;;) {
|
|
596
|
+
const seen = progress.generation;
|
|
597
|
+
if (ADMITTED_STATES.has(rootState))
|
|
598
|
+
break;
|
|
599
|
+
const halt = leaseStop ?? dependencyStop ?? preStepHalt(assignment, deps, startedAt, steps) ?? admissionHalt(rootState);
|
|
600
|
+
if (halt) {
|
|
601
|
+
stopWith(halt.outcome, halt.reason);
|
|
602
|
+
throw new CliError(halt.reason ?? DRILL_REASONS.draining, `${label} stopped before the service admitted it. No step was taken.`);
|
|
603
|
+
}
|
|
604
|
+
await progress.wait(seen, AbortSignal.any([deps.stopSignal, transport, channel ? haltDue() : AbortSignal.timeout(PREPARE_CHECK_MS)]));
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
const openSession = async (key) => {
|
|
608
|
+
const existing = sessions.get(key);
|
|
609
|
+
if (existing)
|
|
610
|
+
return existing;
|
|
611
|
+
if (sessions.size >= MAX_DRIVER_SESSIONS)
|
|
612
|
+
throw new CliError(DRILL_REASONS.step_invalid, 'The step was not sent: the drill opened too many target sessions.');
|
|
613
|
+
const opened = await driver.openSession(ctx, key);
|
|
614
|
+
sessions.set(key, opened);
|
|
615
|
+
activeTracker.markFreshSession();
|
|
616
|
+
return opened;
|
|
617
|
+
};
|
|
618
|
+
// Step loop.
|
|
619
|
+
let idle = 0;
|
|
620
|
+
while (!stop) {
|
|
621
|
+
const halt = leaseStop ?? dependencyStop ?? preStepHalt(assignment, deps, startedAt, steps) ?? executionHalt(rootState);
|
|
622
|
+
if (halt) {
|
|
623
|
+
stopWith(halt.outcome, halt.reason);
|
|
624
|
+
break;
|
|
625
|
+
}
|
|
626
|
+
const wake = AbortSignal.any([deps.stopSignal, transport, AbortSignal.timeout(Math.max(1, leaseRemainingMs(assignment) - LEASE_MARGIN_MS))]);
|
|
627
|
+
if (!await deps.gate.wait(wake))
|
|
628
|
+
continue;
|
|
629
|
+
// Nothing is read or stepped while the campaign channel waits for a valid resync.
|
|
630
|
+
if (channel && !await channel.live(wake))
|
|
631
|
+
continue;
|
|
632
|
+
if (buffer)
|
|
633
|
+
await buffer.room(wake);
|
|
634
|
+
if (pendingReport) {
|
|
635
|
+
try {
|
|
636
|
+
await reportStep(controlPlane, assignment, pendingReport, retry);
|
|
637
|
+
pendingReport = null;
|
|
638
|
+
claimOpen = false;
|
|
639
|
+
}
|
|
640
|
+
catch (error) {
|
|
641
|
+
if (isTransient(error)) {
|
|
642
|
+
deps.gate.disconnected();
|
|
643
|
+
continue;
|
|
644
|
+
}
|
|
645
|
+
const halted = classify(error);
|
|
646
|
+
stopWith(halted.outcome === 'failed' ? 'incomplete' : halted.outcome, halted.reason ?? DRILL_REASONS.step_report_failed);
|
|
647
|
+
break;
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
// Read before the canonical read of root's actions, so a wake-up that lands during it is not lost.
|
|
651
|
+
const seen = progress.generation;
|
|
652
|
+
let batch;
|
|
653
|
+
try {
|
|
654
|
+
batch = await retryTransient(() => controlPlane.nextSteps(assignment, cursor, transport), retry);
|
|
655
|
+
}
|
|
656
|
+
catch (error) {
|
|
657
|
+
if (isTransient(error)) {
|
|
658
|
+
deps.gate.disconnected();
|
|
659
|
+
continue;
|
|
660
|
+
}
|
|
661
|
+
throw error;
|
|
662
|
+
}
|
|
663
|
+
if (batch.cancelled) {
|
|
664
|
+
stopWith('cancelled', null);
|
|
665
|
+
break;
|
|
666
|
+
}
|
|
667
|
+
// Root's latest coverage (sent only while nothing is in flight, so a page with steps clears it): a v3 scenario
|
|
668
|
+
// may be waiting on a timed event, and a drill that stops before its input ends still says what was never applied.
|
|
669
|
+
schedule = batch.schedule ?? null;
|
|
670
|
+
// Steps this attempt already claimed (including `unknown` ones) are never re-invoked.
|
|
671
|
+
if (batch.reconciliation_required === 'participant_outcome_unknown') {
|
|
672
|
+
// Do not replay an unconfirmed claim or infer that cloud inference did
|
|
673
|
+
// not run. Cleanup is local; cloud close/accounting may remain pending.
|
|
674
|
+
stopWith('incomplete', 'native_participant_outcome_unknown');
|
|
675
|
+
break;
|
|
676
|
+
}
|
|
677
|
+
const ordered = (Array.isArray(batch.steps) ? batch.steps : [])
|
|
678
|
+
.filter((step) => !(typeof step?.step_id === 'string' && handled.has(step.step_id)))
|
|
679
|
+
.sort((a, b) => a.sequence - b.sequence);
|
|
680
|
+
// A claim whose answer was lost: if it landed, root lists nothing offered (the action is claimed) and the
|
|
681
|
+
// input cannot end, so the same claim is replayed (root answers `recovered`). If root offers another action
|
|
682
|
+
// or says the input ended, that claim never landed and is dropped.
|
|
683
|
+
if (unconfirmed && !ordered.some((step) => step.step_id === unconfirmed?.step_id)) {
|
|
684
|
+
if (ordered.length === 0 && !batch.done)
|
|
685
|
+
ordered.push(unconfirmed);
|
|
686
|
+
else
|
|
687
|
+
unconfirmed = null;
|
|
688
|
+
}
|
|
689
|
+
if (ordered.length === 0) {
|
|
690
|
+
// Root says the drill's input ended. The drill ran to the end of its input; that is not a pass,
|
|
691
|
+
// and root's frozen completion evidence, not this client, decides coverage. Root refuses a
|
|
692
|
+
// completed close over a retained target failure or cancellation, so those close as the target ended.
|
|
693
|
+
if (batch.done) {
|
|
694
|
+
endingReason = batch.ending_reason;
|
|
695
|
+
schedule = batch.schedule ?? null;
|
|
696
|
+
if (endingReason === 'target_failed')
|
|
697
|
+
stopWith('failed', DRILL_REASONS.target_failed);
|
|
698
|
+
else if (endingReason === 'target_cancelled')
|
|
699
|
+
stopWith('cancelled', DRILL_REASONS.target_cancelled);
|
|
700
|
+
// Root's own infrastructure ending (root refuses it as completed): cleaned up and closed incomplete.
|
|
701
|
+
else if (endingReason === 'participant_unavailable')
|
|
702
|
+
stopWith('incomplete', DRILL_REASONS.participant_unavailable);
|
|
703
|
+
else
|
|
704
|
+
stopWith('completed', null);
|
|
705
|
+
break;
|
|
706
|
+
}
|
|
707
|
+
// Nothing offered yet: root's next change for this assignment (an action, a timed event, a cancellation)
|
|
708
|
+
// or a resync wakes the drill; it does not poll.
|
|
709
|
+
if (channel)
|
|
710
|
+
await progress.wait(seen, AbortSignal.any([deps.stopSignal, transport, haltDue()]));
|
|
711
|
+
else
|
|
712
|
+
await sleep(Math.min(250 * 2 ** Math.min(idle++, 3), 2_000), AbortSignal.any([deps.stopSignal, transport]));
|
|
713
|
+
continue;
|
|
714
|
+
}
|
|
715
|
+
idle = 0;
|
|
716
|
+
for (const step of ordered) {
|
|
717
|
+
if (handled.has(step?.step_id))
|
|
718
|
+
continue;
|
|
719
|
+
const stepHalt = leaseStop ?? dependencyStop ?? preStepHalt(assignment, deps, startedAt, steps) ?? executionHalt(rootState);
|
|
720
|
+
if (stepHalt) {
|
|
721
|
+
stopWith(stepHalt.outcome, stepHalt.reason);
|
|
722
|
+
break;
|
|
723
|
+
}
|
|
724
|
+
if (!deps.gate.connected || (channel && channel.state !== 'live'))
|
|
725
|
+
break; // paused: re-enter the outer loop and wait
|
|
726
|
+
const problem = stepProblem(step);
|
|
727
|
+
if (problem) {
|
|
728
|
+
// Never invoke a malformed step or a non-idempotent step without a persisted intent.
|
|
729
|
+
await reportStep(controlPlane, assignment, failedResult(step, problem), retry).catch(() => undefined);
|
|
730
|
+
stopWith('failed', problem);
|
|
731
|
+
break;
|
|
732
|
+
}
|
|
733
|
+
if (stepTimeout(step, assignment, startedAt) <= 0) {
|
|
734
|
+
stopWith('incomplete', NATIVE_ERRORS.budget_exhausted);
|
|
735
|
+
break;
|
|
736
|
+
}
|
|
737
|
+
// Open (or reuse) the target session first: a driver failure here records no start.
|
|
738
|
+
const session = await openSession(step.payload.kind === 'event' ? EVENT_SESSION_KEY : step.payload.session_key);
|
|
739
|
+
// Session setup can span a channel loss. Do not claim new work until a
|
|
740
|
+
// current resync, and leave this still-unclaimed step to the outer loop.
|
|
741
|
+
if (!deps.gate.connected || (channel && channel.state !== 'live'))
|
|
742
|
+
break;
|
|
743
|
+
const setupHalt = leaseStop ?? dependencyStop ?? preStepHalt(assignment, deps, startedAt, steps) ?? executionHalt(rootState);
|
|
744
|
+
if (setupHalt) {
|
|
745
|
+
stopWith(setupHalt.outcome, setupHalt.reason);
|
|
746
|
+
break;
|
|
747
|
+
}
|
|
748
|
+
let claimed;
|
|
749
|
+
try {
|
|
750
|
+
// The same claim is replayed on a transient failure (one request key per step).
|
|
751
|
+
claimed = await retryTransient(() => controlPlane.stepStarted(assignment, step, transport), retry);
|
|
752
|
+
}
|
|
753
|
+
catch (error) {
|
|
754
|
+
// No invocation without a recorded start. A transient failure pauses; the same
|
|
755
|
+
// claim is re-sent with its key, whether root still offers the step or hides it as claimed.
|
|
756
|
+
if (isTransient(error)) {
|
|
757
|
+
unconfirmed = step;
|
|
758
|
+
deps.gate.disconnected();
|
|
759
|
+
break;
|
|
760
|
+
}
|
|
761
|
+
const halted = classify(error);
|
|
762
|
+
stopWith(halted.outcome, halted.reason);
|
|
763
|
+
break;
|
|
764
|
+
}
|
|
765
|
+
unconfirmed = null;
|
|
766
|
+
handled.add(step.step_id);
|
|
767
|
+
claimOpen = true;
|
|
768
|
+
if (claimed?.recovered === true) {
|
|
769
|
+
// Root kept an earlier claim of this step (its first answer was lost), so the
|
|
770
|
+
// step may have reached the target. It is never invoked again.
|
|
771
|
+
cursor = Math.max(cursor, step.sequence);
|
|
772
|
+
unknownSteps++;
|
|
773
|
+
observe({ kind: 'action.observed', at_ms: Date.now(), content: { type: 'target.claim_recovered', step_id: step.step_id, outcome: 'unknown' } });
|
|
774
|
+
stopWith('incomplete', NATIVE_ERRORS.unknown_outcome);
|
|
775
|
+
break;
|
|
776
|
+
}
|
|
777
|
+
// Invoke exactly the dispatch root froze for this claim, when it returned one.
|
|
778
|
+
const dispatched = claimed?.payload ? { ...step, payload: claimed.payload } : step;
|
|
779
|
+
if (dispatched.payload.kind !== step.kind) {
|
|
780
|
+
stopWith('failed', DRILL_REASONS.step_invalid);
|
|
781
|
+
break;
|
|
782
|
+
}
|
|
783
|
+
// Session setup and claim recovery consume elapsed budget too. A renewing
|
|
784
|
+
// lease is not a fixed turn deadline; lease loss aborts drillController.
|
|
785
|
+
const halts = () => leaseStop ?? dependencyStop ?? preStepHalt(assignment, deps, startedAt, steps) ?? executionHalt(rootState);
|
|
786
|
+
const confirmed = () => deps.gate.connected && (!channel || channel.state === 'live');
|
|
787
|
+
let timeoutMs = stepTimeout(dispatched, assignment, startedAt);
|
|
788
|
+
let dispatchHalt = halts();
|
|
789
|
+
if (!dispatchHalt && timeoutMs > 0 && !confirmed()) {
|
|
790
|
+
// The channel rotates about once a minute and is briefly not live while it reconnects; a claim answered in
|
|
791
|
+
// that window waits for the next valid resync (or a transport reconnect), bounded by the next lease or
|
|
792
|
+
// budget halt, instead of being withdrawn. Nothing is invoked until the connection is confirmed again.
|
|
793
|
+
while (!confirmed()) {
|
|
794
|
+
dispatchHalt = halts();
|
|
795
|
+
timeoutMs = stepTimeout(dispatched, assignment, startedAt);
|
|
796
|
+
if (dispatchHalt || timeoutMs <= 0 || deps.stopSignal.aborted || transport.aborted
|
|
797
|
+
|| drillController.signal.aborted || channel?.state === 'closed')
|
|
798
|
+
break;
|
|
799
|
+
// This timer snapshots the current lease. A renewal can move its
|
|
800
|
+
// deadline while resync is pending, so expiry only wakes this loop.
|
|
801
|
+
// Keep the same acknowledged claim and re-check live authority;
|
|
802
|
+
// never withdraw or invoke solely because an old timer elapsed.
|
|
803
|
+
const waitFor = AbortSignal.any([deps.stopSignal, transport, drillController.signal, haltDue()]);
|
|
804
|
+
if (await deps.gate.wait(waitFor) && channel)
|
|
805
|
+
await channel.live(waitFor);
|
|
806
|
+
}
|
|
807
|
+
timeoutMs = stepTimeout(dispatched, assignment, startedAt);
|
|
808
|
+
dispatchHalt = halts();
|
|
809
|
+
}
|
|
810
|
+
if (dispatchHalt || timeoutMs <= 0 || !confirmed()) {
|
|
811
|
+
const reason = dispatchHalt?.reason ?? (timeoutMs <= 0 ? NATIVE_ERRORS.budget_exhausted : 'native_stream_disconnected');
|
|
812
|
+
// Only this freshly acknowledged, never-invoked claim can be withdrawn.
|
|
813
|
+
// Recovered/uncertain claims took the earlier unknown-outcome path.
|
|
814
|
+
if (controlPlane.abandonStep)
|
|
815
|
+
await withdrawClaim(step, reason);
|
|
816
|
+
stopWith(dispatchHalt?.outcome ?? 'incomplete', reason);
|
|
817
|
+
break;
|
|
818
|
+
}
|
|
819
|
+
const { result, evidenceLost } = await invokeStep(session, { ...dispatched, deadline_ms: timeoutMs }, timeoutMs, drillController.signal);
|
|
820
|
+
// The driver itself returned and says nothing was written: the client refused the input or could not
|
|
821
|
+
// deliver it. A step the runner had to interrupt, or any unknown outcome, never counts as undelivered.
|
|
822
|
+
const neverDelivered = !evidenceLost && result.delivered !== true && (result.outcome === 'failed' || result.outcome === 'cancelled');
|
|
823
|
+
steps++;
|
|
824
|
+
cursor = Math.max(cursor, step.sequence);
|
|
825
|
+
if (result.outcome === 'unknown')
|
|
826
|
+
unknownSteps++;
|
|
827
|
+
if (result.outcome === 'completed')
|
|
828
|
+
completedSteps++;
|
|
829
|
+
for (const observation of result.observations ?? [])
|
|
830
|
+
observe(observation);
|
|
831
|
+
if (evidenceLost)
|
|
832
|
+
buffer?.noteLostStepEvidence();
|
|
833
|
+
// The evidence root completes the action with: the target's reply to a message, as cloud
|
|
834
|
+
// participants read it (v2: names the claimed action, which is this step), or the retained
|
|
835
|
+
// result of a task or event. An unknown outcome has neither and never completes the action.
|
|
836
|
+
let evidenceId = null;
|
|
837
|
+
if (buffer && step.kind === 'message') {
|
|
838
|
+
const reply = result.outcome === 'completed' ? replyText(result.output) : null;
|
|
839
|
+
evidenceId = reply ? buffer.add({ kind: 'conversation.observed', at_ms: result.finished_ms, content: {},
|
|
840
|
+
target_message: { schema: 'firedrill.native-target-message.v2', action_id: step.step_id, role: 'assistant', content: reply } }) : null;
|
|
841
|
+
if (!reply && result.outcome === 'failed' && result.delivered === true && claimed?.claim_key
|
|
842
|
+
&& typeof step.input_digest === 'string' && (step.action_kind === 'participant_message' || step.action_kind === 'scenario_message')) {
|
|
843
|
+
evidenceId = buffer.add({ kind: 'action.observed', at_ms: result.finished_ms, content: {},
|
|
844
|
+
target_result: { schema: 'firedrill.native-target-result.v1', action_id: step.step_id,
|
|
845
|
+
input_digest: step.input_digest, kind: step.action_kind, claim_key: claimed.claim_key,
|
|
846
|
+
outcome: 'failed', output: { error_code: 'native_target_error' }, output_coverage: 'complete',
|
|
847
|
+
completion: { kind: 'target_returned', delivery_id: null } } });
|
|
848
|
+
}
|
|
849
|
+
}
|
|
850
|
+
else if (buffer && result.outcome !== 'unknown' && (result.outcome === 'completed' || result.delivered === true)
|
|
851
|
+
&& typeof step.input_digest === 'string') {
|
|
852
|
+
// Only an input that reached the target has a target result. A step the client refused or could not
|
|
853
|
+
// deliver is never recorded as the target's own failure: root would end the input as target_failed.
|
|
854
|
+
evidenceId = buffer.add({ kind: 'action.observed', at_ms: result.finished_ms, content: {},
|
|
855
|
+
target_result: targetResult(step, result.outcome, result.output, result.output_coverage) });
|
|
856
|
+
}
|
|
857
|
+
// Root completes a target action only with an acknowledged evidence event.
|
|
858
|
+
const evidenceAcknowledged = evidenceId && buffer ? await buffer.delivered(evidenceId, AbortSignal.any([transport, AbortSignal.timeout(Math.max(1, leaseRemainingMs(assignment) - LEASE_MARGIN_MS))])) : false;
|
|
859
|
+
result.reply_event_id = evidenceAcknowledged ? evidenceId : null;
|
|
860
|
+
output('native_step', { assignment_id: assignment.assignment_id, step_id: step.step_id, sequence: step.sequence,
|
|
861
|
+
outcome: result.outcome, error_code: result.error_code, duration_ms: Math.max(0, result.finished_ms - result.started_ms),
|
|
862
|
+
reply_event_id: result.reply_event_id }, `${label}: step ${step.sequence} ${result.outcome}${result.error_code ? ` (${result.error_code})` : ''}.`);
|
|
863
|
+
if (!result.reply_event_id) {
|
|
864
|
+
// Without acknowledged evidence root cannot complete the action, and offers
|
|
865
|
+
// no next input while it is claimed: the drill ends here.
|
|
866
|
+
// Evidence was lost only for a completed step or a task or event that reached the target; any other
|
|
867
|
+
// step ended on its own error (refused, not delivered, or a failed message), which is the reason.
|
|
868
|
+
const undelivered = step.kind === 'message' ? DRILL_REASONS.reply_undelivered : DRILL_REASONS.result_undelivered;
|
|
869
|
+
const evidenceLost = result.outcome === 'completed' || (step.kind !== 'message' && result.delivered === true);
|
|
870
|
+
if (neverDelivered && controlPlane.abandonStep)
|
|
871
|
+
await withdrawClaim(step, result.error_code);
|
|
872
|
+
stopWith('incomplete', result.outcome === 'unknown' ? NATIVE_ERRORS.unknown_outcome
|
|
873
|
+
: evidenceLost ? undelivered : result.error_code ?? undelivered);
|
|
874
|
+
break;
|
|
875
|
+
}
|
|
876
|
+
try {
|
|
877
|
+
await reportStep(controlPlane, assignment, result, retry);
|
|
878
|
+
claimOpen = false;
|
|
879
|
+
if (step.kind === 'message' && result.outcome === 'failed') {
|
|
880
|
+
stopWith('failed', result.error_code ?? 'native_target_error');
|
|
881
|
+
break;
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
catch (error) {
|
|
885
|
+
if (isTransient(error)) {
|
|
886
|
+
pendingReport = result;
|
|
887
|
+
deps.gate.disconnected();
|
|
888
|
+
break;
|
|
889
|
+
}
|
|
890
|
+
const halted = classify(error);
|
|
891
|
+
stopWith(halted.outcome === 'failed' ? 'incomplete' : halted.outcome, halted.reason ?? DRILL_REASONS.step_report_failed);
|
|
892
|
+
break;
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
catch (error) {
|
|
898
|
+
const halted = classify(error);
|
|
899
|
+
stopWith(halted.outcome, halted.reason);
|
|
900
|
+
}
|
|
901
|
+
finally {
|
|
902
|
+
deps.abortSignal.removeEventListener('abort', onStop);
|
|
903
|
+
// Teardown needs no wake-ups: the signed stop, close and cleanup carry their own receipts.
|
|
904
|
+
unwatchChannel();
|
|
905
|
+
assignmentWake?.close();
|
|
906
|
+
}
|
|
907
|
+
// Teardown: sessions, driver, owned services, observations, scratch, then the signed close.
|
|
908
|
+
const final = leaseStop ?? stop ?? { outcome: 'failed', reason: DRILL_REASONS.drill_failed };
|
|
909
|
+
const driverOutcome = final.outcome === 'completed' ? 'completed'
|
|
910
|
+
: final.outcome === 'cancelled' ? 'cancelled' : 'failed';
|
|
911
|
+
// Root stops dispatching first (withdrawing only offers never claimed) while local
|
|
912
|
+
// work stops; its receipt names the cutoff the close must carry. Root refuses a stop
|
|
913
|
+
// before it prepared the drill, and a revoked device cannot sign one.
|
|
914
|
+
const stopping = prepared && !deps.isRevoked()
|
|
915
|
+
? retryTransient(() => controlPlane.stopExecution(assignment, transport), retry)
|
|
916
|
+
.then((receipt) => ({ receipt, error: null }), (error) => ({ receipt: null, error }))
|
|
917
|
+
: null;
|
|
918
|
+
drillController.abort();
|
|
919
|
+
let driverCleanupUnknown = false;
|
|
920
|
+
for (const session of sessions.values())
|
|
921
|
+
await session.close().catch(() => { driverCleanupUnknown = true; });
|
|
922
|
+
if (driver)
|
|
923
|
+
await driver.dispose().catch(() => { driverCleanupUnknown = true; });
|
|
924
|
+
// Driver-owned helper processes need not be descriptor services. They still
|
|
925
|
+
// must end before claiming that the target is closed.
|
|
926
|
+
if (driver?.ownedProcesses?.().some(process => process.exit === null))
|
|
927
|
+
driverCleanupUnknown = true;
|
|
928
|
+
await deps.services.stopDrill(drillKey, final.reason ?? final.outcome);
|
|
929
|
+
// The target is stopped, so nothing may reach root's gateway for this drill any more;
|
|
930
|
+
// the proxy's tokens are dropped with it, before the close is recorded.
|
|
931
|
+
if (proxy)
|
|
932
|
+
await proxy.close().catch(() => undefined);
|
|
933
|
+
proxy = null;
|
|
934
|
+
// After every owned process was stopped: how each declared service ended, or why that cannot be attested.
|
|
935
|
+
// A shared run service that ended during the drill is reported to root first, so the close names what root retains.
|
|
936
|
+
if (prepared && deps.localRun && !deps.isRevoked()) {
|
|
937
|
+
const localRun = deps.localRun;
|
|
938
|
+
await retryTransient(() => localRun.sync(transport), retry).catch(() => undefined);
|
|
939
|
+
}
|
|
940
|
+
const ends = serviceEnds(plan, descriptor, drillServices, deps.services.takeLaunches(drillKey), driver, driverOutcome, deps.localRun ?? null);
|
|
941
|
+
if (driverCleanupUnknown) {
|
|
942
|
+
const entry = ends.findIndex(end => ('known' in end ? end.known.service : end.unknown.service) === descriptor.entry_service);
|
|
943
|
+
const previous = entry >= 0 ? ends[entry] : null;
|
|
944
|
+
const process = previous && ('known' in previous ? previous.known : previous.unknown);
|
|
945
|
+
const unknown = { problem: 'native_driver_cleanup_unconfirmed', unknown: {
|
|
946
|
+
service: descriptor.entry_service, launch_id: process && 'launch_id' in process ? process.launch_id : null,
|
|
947
|
+
ownership: 'unknown', close_kind: 'unknown',
|
|
948
|
+
} };
|
|
949
|
+
if (entry >= 0)
|
|
950
|
+
ends[entry] = unknown;
|
|
951
|
+
else
|
|
952
|
+
ends.push(unknown);
|
|
953
|
+
}
|
|
954
|
+
const closed = processCloses(ends);
|
|
955
|
+
let evidence = buffer ? await buffer.close() : null;
|
|
956
|
+
for (const config of mcpConfigs)
|
|
957
|
+
await config.cleanup().catch(() => undefined);
|
|
958
|
+
mcpConfigs = [];
|
|
959
|
+
if (scratch)
|
|
960
|
+
await scratch.cleanup();
|
|
961
|
+
const stateCoverage = tracker ? tracker.coverage() : descriptor.state_domains.map((domain) => ({
|
|
962
|
+
alias: domain.alias, reset: domain.reset, result: 'unavailable',
|
|
963
|
+
}));
|
|
964
|
+
let closeId = null;
|
|
965
|
+
let completionError = null;
|
|
966
|
+
let reported = false;
|
|
967
|
+
let withdrawn = false;
|
|
968
|
+
let withdrawCode = null;
|
|
969
|
+
let stepReportLost = false;
|
|
970
|
+
const stopped = stopping ? await stopping : null;
|
|
971
|
+
if (deps.isRevoked()) {
|
|
972
|
+
completionError = NATIVE_ERRORS.revoked;
|
|
973
|
+
}
|
|
974
|
+
else {
|
|
975
|
+
if (pendingReport) {
|
|
976
|
+
// Last attempt to record an invoked step's result; the step itself is never re-invoked.
|
|
977
|
+
try {
|
|
978
|
+
await reportStep(controlPlane, assignment, pendingReport, retry);
|
|
979
|
+
pendingReport = null;
|
|
980
|
+
claimOpen = false;
|
|
981
|
+
}
|
|
982
|
+
catch {
|
|
983
|
+
stepReportLost = true;
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
if (!prepared) {
|
|
987
|
+
// Before root prepared the drill it accepts no evidence, and nothing ran locally to close.
|
|
988
|
+
completionError = final.reason ?? DRILL_REASONS.not_prepared;
|
|
989
|
+
// Withdraw it with a signed cancel while root still holds it accepted or provisioning, so it keeps no
|
|
990
|
+
// capacity and the campaign's local run can close; how its services ended goes on the cleanup-only path.
|
|
991
|
+
if (PREPARING_STATES.has(rootState)) {
|
|
992
|
+
try {
|
|
993
|
+
await retryTransient(() => controlPlane.cancelExecution(assignment, transport), retry);
|
|
994
|
+
withdrawn = true;
|
|
995
|
+
output('native_drill_withdrawn', { assignment_id: assignment.assignment_id, attempt_id: assignment.attempt_id }, `${label} was withdrawn before the service prepared it. Nothing started locally.`);
|
|
996
|
+
}
|
|
997
|
+
catch (error) {
|
|
998
|
+
withdrawCode = errorCode(error);
|
|
999
|
+
output('native_drill_withdraw_failed', { assignment_id: assignment.assignment_id, error_code: errorCode(error) }, `${label} could not be withdrawn; the service releases it when its lease ends.`);
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
}
|
|
1003
|
+
else if (closed.problem !== null) {
|
|
1004
|
+
// Never attest a process ending that did not happen; root keeps the drill until it is reconciled.
|
|
1005
|
+
completionError = closed.problem;
|
|
1006
|
+
}
|
|
1007
|
+
else if (pendingReport) {
|
|
1008
|
+
// Root refuses a close while a claimed action is unfinished; the drill is left to root's reconciliation.
|
|
1009
|
+
completionError = DRILL_REASONS.step_report_failed;
|
|
1010
|
+
}
|
|
1011
|
+
else if (!stopped?.receipt) {
|
|
1012
|
+
// Without root's stop receipt there is no exact cutoff to close on; none is guessed.
|
|
1013
|
+
completionError = errorCode(stopped?.error) ?? DRILL_REASONS.stop_unconfirmed;
|
|
1014
|
+
}
|
|
1015
|
+
else if (!buffer || !evidence || evidence.unacknowledged > 0 || evidence.failure_code !== null) {
|
|
1016
|
+
// The final outcome event must directly follow everything root acknowledged.
|
|
1017
|
+
completionError = evidence?.failure_code ?? 'native_observation_delivery_failed';
|
|
1018
|
+
}
|
|
1019
|
+
else {
|
|
1020
|
+
// Root's order: commit the signed close as the driver.<outcome> record, send the final
|
|
1021
|
+
// outcome observation referencing it as the next sequence, and once root acknowledged
|
|
1022
|
+
// that event, submit the close. Nothing is observed after it.
|
|
1023
|
+
const close = {
|
|
1024
|
+
observation_sequence: evidence.last_sequence + 1,
|
|
1025
|
+
last_action_id: stopped.receipt.last_action_id,
|
|
1026
|
+
driver_outcome: driverOutcome,
|
|
1027
|
+
processes: closed.processes,
|
|
1028
|
+
};
|
|
1029
|
+
const reason = stopReason(final, steps, assignment, endingReason);
|
|
1030
|
+
if (reason)
|
|
1031
|
+
close.client_summary = { stop_reason: reason, steps_started: steps, steps_completed: completedSteps,
|
|
1032
|
+
...(descriptor.dependencies.some(dependency => dependency.mode === 'synthetic')
|
|
1033
|
+
? { dependency_capture_complete: evidence.evidence_complete,
|
|
1034
|
+
dependency_capture_records_complete: evidence.dependency_capture_records_complete } : {}) };
|
|
1035
|
+
let closeStage = 'prepare_record';
|
|
1036
|
+
try {
|
|
1037
|
+
// Retry only the cached signed close and its original upload keys.
|
|
1038
|
+
// Five fast attempts can end before an authority refresh finishes.
|
|
1039
|
+
// This finite window never repeats a target or the outcome observation.
|
|
1040
|
+
// The control plane caps each renewed lease at the original deadline.
|
|
1041
|
+
const preparationDeadline = Date.now() + CLOSE_PENDING_WAIT_MS;
|
|
1042
|
+
let record;
|
|
1043
|
+
for (let attempt = 0;; attempt++) {
|
|
1044
|
+
const remaining = Math.min(preparationDeadline - Date.now(), leaseRemainingMs(assignment) - LEASE_MARGIN_MS * 2);
|
|
1045
|
+
if (deps.isRevoked())
|
|
1046
|
+
throw new CliError(NATIVE_ERRORS.revoked, 'The executor authority has ended.');
|
|
1047
|
+
if (leaseStop || leaseRemainingMs(assignment) <= LEASE_MARGIN_MS * 2) {
|
|
1048
|
+
throw new CliError(leaseStop?.reason ?? NATIVE_ERRORS.lease_expired, 'The original execution lease has ended.');
|
|
1049
|
+
}
|
|
1050
|
+
if (transport.aborted || remaining <= 0) {
|
|
1051
|
+
throw new CliError('native_execution_close_pending', 'The original close remains unconfirmed. Do not run the target again.');
|
|
1052
|
+
}
|
|
1053
|
+
const closeSignal = AbortSignal.any([transport, AbortSignal.timeout(remaining)]);
|
|
1054
|
+
try {
|
|
1055
|
+
record = await controlPlane.recordClose(assignment, close, closeSignal);
|
|
1056
|
+
break;
|
|
1057
|
+
}
|
|
1058
|
+
catch (error) {
|
|
1059
|
+
if (!isTransient(error) || transport.aborted || deps.isRevoked() || leaseStop
|
|
1060
|
+
|| Date.now() >= preparationDeadline || leaseRemainingMs(assignment) <= LEASE_MARGIN_MS * 2)
|
|
1061
|
+
throw error;
|
|
1062
|
+
deps.gate.disconnected();
|
|
1063
|
+
const delay = Math.min(backoffMs(attempt, 1_000, 10_000), preparationDeadline - Date.now(), leaseRemainingMs(assignment) - LEASE_MARGIN_MS * 2);
|
|
1064
|
+
if (delay <= 0 || !await sleep(delay, transport))
|
|
1065
|
+
throw error;
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
closeStage = 'outcome_observation';
|
|
1069
|
+
const undelivered = await buffer.submitOutcome(record.kind, record);
|
|
1070
|
+
evidence = buffer.summary();
|
|
1071
|
+
if (undelivered !== null)
|
|
1072
|
+
throw new CliError(undelivered, 'The final outcome observation was not acknowledged, so the drill was not closed.');
|
|
1073
|
+
// Root returns the original close for a repeated submit, so waiting out its pending participant work is safe.
|
|
1074
|
+
// A target action this client left claimed (or whose claim got no answer) never finishes by waiting, so
|
|
1075
|
+
// then root's refusal stands at once.
|
|
1076
|
+
const waitable = !claimOpen && unconfirmed === null && final.reason !== 'native_participant_outcome_unknown';
|
|
1077
|
+
const closeDeadline = Date.now() + CLOSE_PENDING_WAIT_MS;
|
|
1078
|
+
closeStage = 'submit_close';
|
|
1079
|
+
let receipt;
|
|
1080
|
+
for (let attempt = 0;; attempt++) {
|
|
1081
|
+
const remainingClose = closeDeadline - Date.now();
|
|
1082
|
+
if (remainingClose <= 0)
|
|
1083
|
+
throw new CliError('native_execution_close_pending', 'The original close remains unconfirmed. Its evidence is retained; do not run the target again.');
|
|
1084
|
+
const closeSignal = AbortSignal.any([transport, AbortSignal.timeout(remainingClose)]);
|
|
1085
|
+
try {
|
|
1086
|
+
receipt = await controlPlane.submitClose(assignment, record, closeSignal);
|
|
1087
|
+
break;
|
|
1088
|
+
}
|
|
1089
|
+
catch (error) {
|
|
1090
|
+
const pending = ['native_execution_close_pending', 'native_execution_ending_pending'].includes(errorCode(error) ?? '');
|
|
1091
|
+
// Same retained artifact only. A lost submit reply is recoverable;
|
|
1092
|
+
// no claim, target invocation, outcome or observation is repeated.
|
|
1093
|
+
if ((!isTransient(error) && !(pending && waitable)) || transport.aborted || leaseStop
|
|
1094
|
+
|| Date.now() >= closeDeadline || leaseRemainingMs(assignment) <= LEASE_MARGIN_MS * 2)
|
|
1095
|
+
throw error;
|
|
1096
|
+
if (isTransient(error))
|
|
1097
|
+
deps.gate.disconnected();
|
|
1098
|
+
if (!await sleep(Math.min(backoffMs(attempt, 1_000, 10_000), closeDeadline - Date.now()), transport))
|
|
1099
|
+
throw error;
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
closeId = receipt.close_id;
|
|
1103
|
+
reported = true;
|
|
1104
|
+
}
|
|
1105
|
+
catch (error) {
|
|
1106
|
+
completionError = errorCode(error) ?? DRILL_REASONS.drill_failed;
|
|
1107
|
+
output('native_close_stage_unconfirmed', { assignment_id: assignment.assignment_id,
|
|
1108
|
+
stage: closeStage, error_code: completionError }, `${label}: the original close remains unconfirmed at ${closeStage}. Do not repeat the target.`);
|
|
1109
|
+
}
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
// Root fenced this execution (its lease expired or it was revoked), so no close is possible:
|
|
1113
|
+
// how each service ended goes on root's cleanup-only path instead. It completes nothing.
|
|
1114
|
+
let cleanupOutcome = null;
|
|
1115
|
+
const fenceCodes = new Set([NATIVE_ERRORS.lease_expired, NATIVE_ERRORS.stale_generation]);
|
|
1116
|
+
// Before root prepared the drill it may also have fenced it (lease lost, so the withdrawal was refused) or revoked
|
|
1117
|
+
// it (capacity refused): root seals it only with this device's cleanup report, even though nothing started.
|
|
1118
|
+
const fenced = [final.reason, completionError, errorCode(stopped?.error), withdrawCode].some((code) => fenceCodes.has(code));
|
|
1119
|
+
const endedUnprepared = !prepared && (rootState === 'draining' || rootState === 'paused' || final.reason === DRILL_REASONS.draining);
|
|
1120
|
+
if (!reported && !deps.isRevoked() && (withdrawn || fenced || endedUnprepared)) {
|
|
1121
|
+
try {
|
|
1122
|
+
const receipt = await reportCleanup(controlPlane, assignment, ends, retry);
|
|
1123
|
+
if (receipt) {
|
|
1124
|
+
cleanupOutcome = receipt.outcome;
|
|
1125
|
+
output('native_cleanup_reported', { assignment_id: assignment.assignment_id, cleanup_id: receipt.cleanup_id, outcome: receipt.outcome,
|
|
1126
|
+
unresolved_actions: receipt.unresolved_actions, unresolved_participants: receipt.unresolved_participants }, `${label}: ${withdrawn ? 'the drill was withdrawn' : 'the service fenced this drill'}, so how its services ended was reported for cleanup only`
|
|
1127
|
+
+ `${receipt.outcome === 'unknown' ? ' (some ends could not be attested and stay unknown)' : ''}. Cleanup itself is the service's.`);
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
catch (error) {
|
|
1131
|
+
output('native_cleanup_unreported', { assignment_id: assignment.assignment_id, error_code: errorCode(error) }, `${label}: the service fenced this drill and the cleanup report was not confirmed.`);
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
const completion = {
|
|
1135
|
+
outcome: final.outcome,
|
|
1136
|
+
reason_code: final.reason ?? (final.outcome === 'completed' && unknownSteps > 0 ? NATIVE_ERRORS.unknown_outcome : null),
|
|
1137
|
+
state_coverage: stateCoverage,
|
|
1138
|
+
last_sequence: evidence?.last_sequence ?? assignment.observation_ack,
|
|
1139
|
+
evidence_complete: (evidence?.evidence_complete ?? false) && !stepReportLost,
|
|
1140
|
+
};
|
|
1141
|
+
// End the lease keeper only after an accepted close/cleanup, or after the
|
|
1142
|
+
// bounded reconciliation attempt explicitly leaves the original uncertain.
|
|
1143
|
+
// Never describe deadline/shutdown/authority refusal as successful closure.
|
|
1144
|
+
if (!reported && cleanupOutcome === null && prepared) {
|
|
1145
|
+
output('native_close_unconfirmed', { assignment_id: assignment.assignment_id,
|
|
1146
|
+
attempt_id: assignment.attempt_id, error_code: completionError }, `${label}: original closing evidence remains unconfirmed. Do not repeat the target; cloud cleanup and accounting are still pending.`);
|
|
1147
|
+
}
|
|
1148
|
+
leaseControl.abort();
|
|
1149
|
+
await leaseKeeper;
|
|
1150
|
+
const report = {
|
|
1151
|
+
assignment_id: assignment.assignment_id, attempt_id: assignment.attempt_id,
|
|
1152
|
+
outcome: completion.outcome, reason_code: completion.reason_code, ending_reason: endingReason, schedule, steps, unknown_steps: unknownSteps,
|
|
1153
|
+
evidence, state_coverage: stateCoverage, completion: reported ? 'reported' : 'not-reported',
|
|
1154
|
+
completion_error: completionError, close_id: closeId, cleanup_outcome: cleanupOutcome,
|
|
1155
|
+
};
|
|
1156
|
+
const evidenceText = evidence?.evidence_complete ? 'evidence complete' : 'evidence incomplete';
|
|
1157
|
+
output('native_drill_finished', report, `Finished ${label}: ${completion.outcome}${completion.reason_code ? ` (${completion.reason_code})` : ''}`
|
|
1158
|
+
+ `${endingReason ? `; input ended because ${ENDING_TEXT[endingReason]} (not a result)` : ''}${coverageText(endingReason, schedule, prepared)}, ${steps} step(s), `
|
|
1159
|
+
+ `${unknownSteps} unknown, ${evidenceText}${reported ? '; closed with the service' : `; not closed with the service (${completionError})`}.`);
|
|
1160
|
+
return report;
|
|
1161
|
+
}
|
|
1162
|
+
// ---------------------------------------------------------------------------
|
|
1163
|
+
// Helpers.
|
|
1164
|
+
const LEASE_RENEW_MS = 20_000;
|
|
1165
|
+
/** Renewals prompted by the channel are at least this far apart. */
|
|
1166
|
+
const WAKE_RENEW_SPACING_MS = 1_000;
|
|
1167
|
+
/**
|
|
1168
|
+
* Renew the assignment lease (signed control `heartbeat`) until `signal`
|
|
1169
|
+
* aborts, updating `assignment.lease_expires_at` in place so every lease check
|
|
1170
|
+
* sees the renewed expiry. A fence or revocation stops renewal and reports why
|
|
1171
|
+
* the drill must stop; a pending contract stops renewal quietly (the lease then
|
|
1172
|
+
* simply runs out); transient failures are retried on the next tick. Never throws.
|
|
1173
|
+
*
|
|
1174
|
+
* The renewal also reports root's execution state. While the drill is being
|
|
1175
|
+
* prepared or admitted, a channel wake-up for this assignment prompts a renewal
|
|
1176
|
+
* at once (root's heartbeat control raises no wake-up itself); otherwise renewal
|
|
1177
|
+
* keeps its lease interval. Without a channel it backs off from 2 s to 15 s then.
|
|
1178
|
+
*/
|
|
1179
|
+
async function keepLease(assignment, controlPlane, signal, onLost, onState, preparing, wake) {
|
|
1180
|
+
let preparingRenewals = 0;
|
|
1181
|
+
let seen = wake?.generation ?? 0;
|
|
1182
|
+
let renewedAt = 0;
|
|
1183
|
+
while (!signal.aborted) {
|
|
1184
|
+
const remaining = leaseRemainingMs(assignment);
|
|
1185
|
+
const prompted = wake !== null && !wake.closed && preparing() ? wake : null;
|
|
1186
|
+
const interval = preparing() && !prompted
|
|
1187
|
+
? Math.min(PREPARE_RENEW_MAX_MS, PREPARE_RENEW_MIN_MS * 2 ** Math.min(preparingRenewals++, 3))
|
|
1188
|
+
: LEASE_RENEW_MS;
|
|
1189
|
+
const due = Math.max(1_000, Math.min(interval, Math.floor(remaining / 3)));
|
|
1190
|
+
if (prompted) {
|
|
1191
|
+
await prompted.wait(seen, AbortSignal.any([signal, AbortSignal.timeout(due)]));
|
|
1192
|
+
const spacing = WAKE_RENEW_SPACING_MS - (Date.now() - renewedAt);
|
|
1193
|
+
if (spacing > 0 && !await sleep(spacing, signal))
|
|
1194
|
+
return;
|
|
1195
|
+
if (signal.aborted)
|
|
1196
|
+
return;
|
|
1197
|
+
}
|
|
1198
|
+
else if (!await sleep(due, signal))
|
|
1199
|
+
return;
|
|
1200
|
+
// Read before the renewal, so a wake-up that lands during it prompts the next one.
|
|
1201
|
+
seen = wake?.generation ?? 0;
|
|
1202
|
+
renewedAt = Date.now();
|
|
1203
|
+
try {
|
|
1204
|
+
const renewed = await controlPlane.renewLease(assignment, signal);
|
|
1205
|
+
const expires = Date.parse(renewed?.lease_expires_at);
|
|
1206
|
+
if (Number.isFinite(expires) && expires > Date.parse(assignment.lease_expires_at))
|
|
1207
|
+
assignment.lease_expires_at = renewed.lease_expires_at;
|
|
1208
|
+
if (typeof renewed?.state === 'string')
|
|
1209
|
+
onState(renewed.state);
|
|
1210
|
+
}
|
|
1211
|
+
catch (error) {
|
|
1212
|
+
if (signal.aborted)
|
|
1213
|
+
return;
|
|
1214
|
+
if (isTransient(error))
|
|
1215
|
+
continue;
|
|
1216
|
+
// The cloud ended this assignment (completed, cancelled or failed there): stop, without blaming the target.
|
|
1217
|
+
if (errorCode(error) === 'native_assignment_terminal') {
|
|
1218
|
+
onLost({ outcome: 'incomplete', reason: 'native_assignment_terminal' });
|
|
1219
|
+
return;
|
|
1220
|
+
}
|
|
1221
|
+
onLost(classify(error));
|
|
1222
|
+
return;
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
/** The target's reply as message text, or null when there is none. */
|
|
1227
|
+
function replyText(output) {
|
|
1228
|
+
if (typeof output === 'string')
|
|
1229
|
+
return output.trim() ? output : null;
|
|
1230
|
+
if (output === null || output === undefined)
|
|
1231
|
+
return null;
|
|
1232
|
+
try {
|
|
1233
|
+
const text = JSON.stringify(output);
|
|
1234
|
+
return typeof text === 'string' && text.length > 0 ? text : null;
|
|
1235
|
+
}
|
|
1236
|
+
catch {
|
|
1237
|
+
return null;
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1240
|
+
/**
|
|
1241
|
+
* How every declared service ended for this drill, or why it cannot be stated
|
|
1242
|
+
* truthfully: a process still running after stop, a shared launched service
|
|
1243
|
+
* outside a local run (neither reaped nor attached), a run service root keeps
|
|
1244
|
+
* uncertain, or, for a drill that completed, a service it never reached. A
|
|
1245
|
+
* failed or cancelled drill reports a service it never launched or attached as
|
|
1246
|
+
* not started. `launches` holds every per-drill process the supervisor launched
|
|
1247
|
+
* for this drill, including one stopped because it never became ready.
|
|
1248
|
+
*/
|
|
1249
|
+
function serviceEnds(plan, descriptor, handles, launches, driver, outcome, localRun) {
|
|
1250
|
+
const driverOwned = new Map((driver?.ownedProcesses?.() ?? []).map((item) => [item.alias, item]));
|
|
1251
|
+
const driverAlias = plan.driver.profile === 'jsonl-process.v1' ? plan.driver.service_alias : null;
|
|
1252
|
+
const planned = new Map(plan.services.map((service) => [service.alias, service]));
|
|
1253
|
+
const launched = new Map();
|
|
1254
|
+
for (const handle of launches)
|
|
1255
|
+
launched.set(handle.alias, [...(launched.get(handle.alias) ?? []), handle]);
|
|
1256
|
+
const ends = [];
|
|
1257
|
+
for (const { alias } of descriptor.services) {
|
|
1258
|
+
const owned = driverOwned.get(alias);
|
|
1259
|
+
const runs = launched.get(alias) ?? [];
|
|
1260
|
+
const handle = handles.get(alias);
|
|
1261
|
+
const service = planned.get(alias);
|
|
1262
|
+
// A run-owned service: reported by what root retains for the campaign's local run, never as drill-owned.
|
|
1263
|
+
const runEntry = localRun?.closeEntry(alias, outcome) ?? null;
|
|
1264
|
+
if (typeof runEntry === 'string') {
|
|
1265
|
+
ends.push({ problem: runEntry, unknown: { service: alias, launch_id: localRun?.launchId(alias) ?? null, ownership: 'owned_run', close_kind: 'unknown' } });
|
|
1266
|
+
continue;
|
|
1267
|
+
}
|
|
1268
|
+
if (runEntry) {
|
|
1269
|
+
ends.push({ known: runEntry });
|
|
1270
|
+
continue;
|
|
1271
|
+
}
|
|
1272
|
+
if (owned) {
|
|
1273
|
+
const code = exitCode(owned.exit);
|
|
1274
|
+
ends.push(code === null
|
|
1275
|
+
? { problem: 'native_close_process_running', unknown: { service: alias, launch_id: owned.launch_id, ownership: 'owned', close_kind: 'unknown' } }
|
|
1276
|
+
: { known: { service: alias, launch_id: owned.launch_id, ownership: 'owned', close_kind: 'owned_group_reaped', exit_code: code } });
|
|
1277
|
+
}
|
|
1278
|
+
else if (runs.length) {
|
|
1279
|
+
// A per-drill service relaunched within the drill reports its latest launch; every launch must have ended.
|
|
1280
|
+
const codes = runs.map((run) => exitCode(run.exit()));
|
|
1281
|
+
const last = codes[codes.length - 1];
|
|
1282
|
+
const launchId = runs[runs.length - 1].launchId;
|
|
1283
|
+
ends.push(codes.some((code) => code === null) || last === null
|
|
1284
|
+
? { problem: 'native_close_process_running', unknown: { service: alias, launch_id: launchId, ownership: 'owned', close_kind: 'unknown' } }
|
|
1285
|
+
: { known: { service: alias, launch_id: launchId, ownership: 'owned', close_kind: 'owned_group_reaped', exit_code: last } });
|
|
1286
|
+
}
|
|
1287
|
+
else if (handle?.mode === 'attach') {
|
|
1288
|
+
ends.push({ known: { service: alias, launch_id: handle.launchId, ownership: 'attached', close_kind: 'transport_detached' } });
|
|
1289
|
+
}
|
|
1290
|
+
else if (alias !== driverAlias && (handle?.mode === 'launch' || (service?.mode === 'launch' && !service.per_drill))) {
|
|
1291
|
+
ends.push({ problem: 'native_close_shared_service', unknown: { service: alias, launch_id: handle?.launchId ?? null, ownership: 'unknown', close_kind: 'unknown' } });
|
|
1292
|
+
}
|
|
1293
|
+
else if (outcome !== 'completed') {
|
|
1294
|
+
ends.push({ known: { service: alias, ownership: 'none', close_kind: 'not_started' } });
|
|
1295
|
+
}
|
|
1296
|
+
else {
|
|
1297
|
+
ends.push({ problem: 'native_close_service_not_launched', unknown: { service: alias, launch_id: null, ownership: 'unknown', close_kind: 'unknown' } });
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
1300
|
+
return ends;
|
|
1301
|
+
}
|
|
1302
|
+
/** The signed close's processes, or the first reason they cannot be stated truthfully. */
|
|
1303
|
+
function processCloses(ends) {
|
|
1304
|
+
const processes = [];
|
|
1305
|
+
for (const end of ends) {
|
|
1306
|
+
if ('problem' in end)
|
|
1307
|
+
return { processes: null, problem: end.problem };
|
|
1308
|
+
processes.push(end.known);
|
|
1309
|
+
}
|
|
1310
|
+
return { processes, problem: null };
|
|
1311
|
+
}
|
|
1312
|
+
/** How long teardown waits for root to freeze a fenced execution (root expires a lease on its own schedule). */
|
|
1313
|
+
const CLEANUP_WAIT_MS = 90_000;
|
|
1314
|
+
/**
|
|
1315
|
+
* Root fenced this execution, so it accepts no close: report how each service
|
|
1316
|
+
* ended on root's cleanup-only path, scoped exactly to the execution root froze.
|
|
1317
|
+
* An end that cannot be attested is reported `unknown`, never as stopped. Waits
|
|
1318
|
+
* (bounded) for root to freeze the execution; null when root holds none for it.
|
|
1319
|
+
*/
|
|
1320
|
+
async function reportCleanup(controlPlane, assignment, ends, retry) {
|
|
1321
|
+
const until = AbortSignal.any([retry.signal, AbortSignal.timeout(CLEANUP_WAIT_MS)]);
|
|
1322
|
+
for (let attempt = 0;; attempt++) {
|
|
1323
|
+
const challenge = await retryTransient(() => controlPlane.readCleanup(assignment, until), { ...retry, signal: until });
|
|
1324
|
+
if (challenge) {
|
|
1325
|
+
const processes = ends.map((end) => 'known' in end ? end.known : end.unknown);
|
|
1326
|
+
return await retryTransient(() => controlPlane.submitCleanup(assignment, challenge, processes, retry.signal), retry);
|
|
1327
|
+
}
|
|
1328
|
+
if (!await sleep(backoffMs(attempt, 2_000, 10_000), until))
|
|
1329
|
+
return null;
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
/** While waiting for the service to prepare an accepted drill: why it must stop instead, or null to keep waiting. */
|
|
1333
|
+
function prepareHalt(state) {
|
|
1334
|
+
if (PREPARING_STATES.has(state) || PREPARED_STATES.has(state))
|
|
1335
|
+
return null;
|
|
1336
|
+
if (ROOT_TERMINAL_STATES.has(state))
|
|
1337
|
+
return { outcome: 'incomplete', reason: 'native_assignment_terminal' };
|
|
1338
|
+
return { outcome: 'incomplete', reason: DRILL_REASONS.draining };
|
|
1339
|
+
}
|
|
1340
|
+
/** While waiting for root to admit a drill that reported readiness: why it must stop instead, or null to keep waiting. */
|
|
1341
|
+
function admissionHalt(state) {
|
|
1342
|
+
if (state === 'starting' || ADMITTED_STATES.has(state))
|
|
1343
|
+
return null;
|
|
1344
|
+
if (ROOT_TERMINAL_STATES.has(state))
|
|
1345
|
+
return { outcome: 'incomplete', reason: 'native_assignment_terminal' };
|
|
1346
|
+
// A lease lost while starting pauses the drill under a new epoch; this attempt cannot continue.
|
|
1347
|
+
if (state === 'paused')
|
|
1348
|
+
return { outcome: 'incomplete', reason: NATIVE_ERRORS.lease_expired };
|
|
1349
|
+
return { outcome: 'incomplete', reason: DRILL_REASONS.draining };
|
|
1350
|
+
}
|
|
1351
|
+
/**
|
|
1352
|
+
* While executing: the service began draining (a stop or cancel on its side) or
|
|
1353
|
+
* ended the drill, so no further step is taken. Paused keeps waiting.
|
|
1354
|
+
*/
|
|
1355
|
+
function executionHalt(state) {
|
|
1356
|
+
if (ROOT_TERMINAL_STATES.has(state))
|
|
1357
|
+
return { outcome: 'incomplete', reason: 'native_assignment_terminal' };
|
|
1358
|
+
if (state === 'draining' || state === 'reconciling')
|
|
1359
|
+
return { outcome: 'incomplete', reason: DRILL_REASONS.draining };
|
|
1360
|
+
return null;
|
|
1361
|
+
}
|
|
1362
|
+
/** Codes that name a local process, driver or target failure (anything not listed is reported as unknown). */
|
|
1363
|
+
const DRIVER_ERROR_CODES = new Set([NATIVE_ERRORS.process_exited, NATIVE_ERRORS.process_start_failed,
|
|
1364
|
+
NATIVE_ERRORS.process_limit, NATIVE_ERRORS.port_unavailable, NATIVE_ERRORS.readiness_timeout, NATIVE_ERRORS.driver_unavailable,
|
|
1365
|
+
NATIVE_ERRORS.unknown_outcome, DRILL_REASONS.state_reset_failed, DRILL_REASONS.drill_failed, READINESS_ERRORS.service_missing,
|
|
1366
|
+
'native_readiness_unattested', 'native_local_run_service_ended']);
|
|
1367
|
+
/** Local approval and policy refusals. */
|
|
1368
|
+
const POLICY_CODES = new Set([NATIVE_ERRORS.plan_changed, NATIVE_ERRORS.plan_unapproved, NATIVE_ERRORS.root_outside,
|
|
1369
|
+
NATIVE_ERRORS.auth_unsupported, DRILL_REASONS.intent_missing]);
|
|
1370
|
+
/**
|
|
1371
|
+
* The close's client stop reason, or null when how the drill stopped does not
|
|
1372
|
+
* map truthfully onto root's reasons (then no client summary is sent). Step
|
|
1373
|
+
* failures carry the driver's own `native_target_*`/`native_step_*`/
|
|
1374
|
+
* `native_session_*`/`native_response_*` codes.
|
|
1375
|
+
*/
|
|
1376
|
+
function stopReason(final, steps, assignment, ending) {
|
|
1377
|
+
// Every root input ending is its own stop reason. A retained target failure or cancellation
|
|
1378
|
+
// closes as failed or cancelled (root refuses it as completed); any other ending as completed,
|
|
1379
|
+
// except root's `participant_unavailable`, which closes incomplete under its own stop reason.
|
|
1380
|
+
if (final.outcome === 'completed') {
|
|
1381
|
+
if (ending === null)
|
|
1382
|
+
return 'completed';
|
|
1383
|
+
return ending === 'target_failed' || ending === 'target_cancelled' || ending === 'participant_unavailable' ? null : ending;
|
|
1384
|
+
}
|
|
1385
|
+
if (ending === 'participant_unavailable' && final.reason === DRILL_REASONS.participant_unavailable)
|
|
1386
|
+
return 'participant_unavailable';
|
|
1387
|
+
if (ending === 'target_failed' && final.reason === DRILL_REASONS.target_failed)
|
|
1388
|
+
return 'target_failed';
|
|
1389
|
+
if (ending === 'target_cancelled' && final.reason === DRILL_REASONS.target_cancelled)
|
|
1390
|
+
return 'target_cancelled';
|
|
1391
|
+
if (final.outcome === 'cancelled')
|
|
1392
|
+
return 'user_cancelled';
|
|
1393
|
+
const reason = final.reason ?? '';
|
|
1394
|
+
if (reason === NATIVE_ERRORS.budget_exhausted)
|
|
1395
|
+
return steps >= assignment.budgets.max_steps ? 'step_limit' : 'deadline';
|
|
1396
|
+
if (reason === NATIVE_ERRORS.disconnected || reason === NATIVE_ERRORS.lease_expired)
|
|
1397
|
+
return 'connection_lost';
|
|
1398
|
+
if (POLICY_CODES.has(reason))
|
|
1399
|
+
return 'policy_denied';
|
|
1400
|
+
if (DRIVER_ERROR_CODES.has(reason))
|
|
1401
|
+
return 'driver_error';
|
|
1402
|
+
if (!ROOT_SIDE_CODES.has(reason) && /^native_(target|step|session|response|driver)_/.test(reason))
|
|
1403
|
+
return 'driver_error';
|
|
1404
|
+
return null;
|
|
1405
|
+
}
|
|
1406
|
+
/**
|
|
1407
|
+
* What root's ending already says about coverage, in words. Root's frozen
|
|
1408
|
+
* completion evidence decides it; this only names what is certainly partial:
|
|
1409
|
+
* any ending other than a finished participant or delivered inputs, or a
|
|
1410
|
+
* scheduled scenario event that was due, never reached or still timed for later.
|
|
1411
|
+
*/
|
|
1412
|
+
function coverageText(ending, schedule, ran) {
|
|
1413
|
+
const unapplied = schedule ? schedule.due_remaining + schedule.future_conditions + (schedule.future_timers ?? 0) : 0;
|
|
1414
|
+
const counted = schedule && unapplied > 0 ? `${unapplied} of ${schedule.scheduled} scheduled scenario event(s) not applied`
|
|
1415
|
+
+ (typeof schedule.next_after_active_ms === 'number' && schedule.active_elapsed_ms !== undefined
|
|
1416
|
+
? `; the next timed event was set for ${activeTime(schedule.next_after_active_ms)} of active time, ${activeTime(schedule.active_elapsed_ms)} had run` : '')
|
|
1417
|
+
: '';
|
|
1418
|
+
// Root freezes a drill whose input had not ended as partial coverage.
|
|
1419
|
+
if (ending === null)
|
|
1420
|
+
return ran ? `; coverage is partial (the input had not ended when the drill stopped${counted ? `; ${counted}` : ''})` : '';
|
|
1421
|
+
if (COMPLETE_ENDINGS.has(ending) && unapplied === 0)
|
|
1422
|
+
return '';
|
|
1423
|
+
return `; coverage is partial${counted ? ` (${counted})` : ''}`;
|
|
1424
|
+
}
|
|
1425
|
+
/** Step-related codes about the service's steps or reports, not the local target. */
|
|
1426
|
+
const ROOT_SIDE_CODES = new Set([DRILL_REASONS.step_invalid, DRILL_REASONS.step_report_failed, DRILL_REASONS.reply_undelivered,
|
|
1427
|
+
DRILL_REASONS.result_undelivered]);
|
|
1428
|
+
/**
|
|
1429
|
+
* Root's retained result for a task or event step: the target's own answer, never
|
|
1430
|
+
* an assessment. A task returned; an event's receiver acknowledged the delivery
|
|
1431
|
+
* (this client observes no later correlated result).
|
|
1432
|
+
*/
|
|
1433
|
+
function targetResult(step, outcome, output, coverage) {
|
|
1434
|
+
return { schema: 'firedrill.native-target-result.v1', action_id: step.step_id, input_digest: step.input_digest,
|
|
1435
|
+
kind: step.kind === 'event' ? 'event' : 'task', outcome, output: output ?? null, output_coverage: coverage,
|
|
1436
|
+
completion: step.kind === 'event' ? { kind: 'delivery_acknowledged', delivery_id: step.step_id } : { kind: 'target_returned', delivery_id: null } };
|
|
1437
|
+
}
|
|
1438
|
+
/** Checks before admitting the next step. Returns why the drill must stop, or null. */
|
|
1439
|
+
function preStepHalt(assignment, deps, startedAt, steps) {
|
|
1440
|
+
if (deps.isRevoked())
|
|
1441
|
+
return { outcome: 'incomplete', reason: NATIVE_ERRORS.revoked };
|
|
1442
|
+
if (deps.abortSignal.aborted || deps.stopSignal.aborted)
|
|
1443
|
+
return { outcome: 'cancelled', reason: DRILL_REASONS.companion_stopped };
|
|
1444
|
+
if (leaseRemainingMs(assignment) <= LEASE_MARGIN_MS)
|
|
1445
|
+
return { outcome: 'incomplete', reason: NATIVE_ERRORS.lease_expired };
|
|
1446
|
+
if (steps >= assignment.budgets.max_steps)
|
|
1447
|
+
return { outcome: 'incomplete', reason: NATIVE_ERRORS.budget_exhausted };
|
|
1448
|
+
if (Date.now() - startedAt >= assignment.budgets.max_elapsed_ms)
|
|
1449
|
+
return { outcome: 'incomplete', reason: NATIVE_ERRORS.budget_exhausted };
|
|
1450
|
+
return null;
|
|
1451
|
+
}
|
|
1452
|
+
/** The live lease signal fences work; the renewed lease is not a fixed step timeout. */
|
|
1453
|
+
function stepTimeout(step, assignment, startedAt) {
|
|
1454
|
+
const limits = [assignment.budgets.step_timeout_ms,
|
|
1455
|
+
assignment.budgets.max_elapsed_ms - (Date.now() - startedAt)];
|
|
1456
|
+
if (Number.isSafeInteger(step.deadline_ms) && step.deadline_ms > 0)
|
|
1457
|
+
limits.push(step.deadline_ms);
|
|
1458
|
+
return Math.floor(Math.min(...limits));
|
|
1459
|
+
}
|
|
1460
|
+
function classify(error) {
|
|
1461
|
+
if (isRevocation(error))
|
|
1462
|
+
return { outcome: 'incomplete', reason: NATIVE_ERRORS.revoked };
|
|
1463
|
+
if (isFenced(error))
|
|
1464
|
+
return { outcome: 'incomplete', reason: errorCode(error) };
|
|
1465
|
+
if (isTransient(error))
|
|
1466
|
+
return { outcome: 'incomplete', reason: NATIVE_ERRORS.disconnected };
|
|
1467
|
+
return { outcome: 'failed', reason: errorCode(error) ?? DRILL_REASONS.drill_failed };
|
|
1468
|
+
}
|
|
1469
|
+
function failedResult(step, code) {
|
|
1470
|
+
const now = Date.now();
|
|
1471
|
+
return {
|
|
1472
|
+
step_id: typeof step?.step_id === 'string' ? step.step_id : '', intent_id: typeof step?.intent_id === 'string' ? step.intent_id : '',
|
|
1473
|
+
outcome: 'failed', started_ms: now, finished_ms: now, output: null, error_code: code, observations: [],
|
|
1474
|
+
};
|
|
1475
|
+
}
|
|
1476
|
+
async function reportStep(controlPlane, assignment, result, retry) {
|
|
1477
|
+
const { observations: _observations, ...record } = result;
|
|
1478
|
+
await retryTransient(() => controlPlane.completeStep(assignment, record, retry.signal), retry);
|
|
1479
|
+
}
|
|
1480
|
+
/**
|
|
1481
|
+
* Invoke one step with a hard bound. A driver that ignores the abort is not
|
|
1482
|
+
* awaited past the bound. Interrupted or thrown non-idempotent steps are
|
|
1483
|
+
* `unknown` (the request may have reached the target) and are never replayed.
|
|
1484
|
+
*
|
|
1485
|
+
* Drivers return a step's own observations with its result, so when the driver
|
|
1486
|
+
* never returns one (bound reached, or it threw) those observations are lost:
|
|
1487
|
+
* the runner records a terminal `driver.failed` observation for the step and
|
|
1488
|
+
* reports `evidenceLost` so the drill's evidence is marked incomplete.
|
|
1489
|
+
*/
|
|
1490
|
+
async function invokeStep(session, step, timeoutMs, drillSignal) {
|
|
1491
|
+
const startedMs = Date.now();
|
|
1492
|
+
const controller = new AbortController();
|
|
1493
|
+
let timedOut = false;
|
|
1494
|
+
const timer = setTimeout(() => { timedOut = true; controller.abort(new DOMException('The step budget ran out.', 'TimeoutError')); }, timeoutMs);
|
|
1495
|
+
const onDrillAbort = () => controller.abort();
|
|
1496
|
+
drillSignal.addEventListener('abort', onDrillAbort, { once: true });
|
|
1497
|
+
const interrupted = (code, cause) => {
|
|
1498
|
+
const outcome = step.idempotent ? 'failed' : 'unknown';
|
|
1499
|
+
const errorCodeValue = step.idempotent ? code : NATIVE_ERRORS.unknown_outcome;
|
|
1500
|
+
const finishedMs = Date.now();
|
|
1501
|
+
const result = {
|
|
1502
|
+
step_id: step.step_id, intent_id: step.intent_id, outcome, started_ms: startedMs, finished_ms: finishedMs, output: null,
|
|
1503
|
+
error_code: errorCodeValue,
|
|
1504
|
+
observations: [{ kind: 'driver.failed', at_ms: finishedMs, content: {
|
|
1505
|
+
step_id: step.step_id, type: 'step.interrupted', cause, interrupt_code: code, outcome, error_code: errorCodeValue,
|
|
1506
|
+
} }],
|
|
1507
|
+
};
|
|
1508
|
+
return { result, evidenceLost: true };
|
|
1509
|
+
};
|
|
1510
|
+
try {
|
|
1511
|
+
// Grace beyond the bound lets a cooperating driver return its own (more precise) result.
|
|
1512
|
+
const graceMs = 2_000;
|
|
1513
|
+
const bound = new Promise((resolve) => {
|
|
1514
|
+
const onAbort = () => setTimeout(() => resolve('bound'), graceMs).unref?.();
|
|
1515
|
+
if (controller.signal.aborted)
|
|
1516
|
+
onAbort();
|
|
1517
|
+
else
|
|
1518
|
+
controller.signal.addEventListener('abort', onAbort, { once: true });
|
|
1519
|
+
});
|
|
1520
|
+
const raced = await Promise.race([session.invoke(step, controller.signal), bound]);
|
|
1521
|
+
if (raced === 'bound')
|
|
1522
|
+
return interrupted(timedOut ? 'native_step_timeout' : 'native_step_cancelled', 'not_returned');
|
|
1523
|
+
return { result: normalizeResult(raced, step, startedMs), evidenceLost: false };
|
|
1524
|
+
}
|
|
1525
|
+
catch (error) {
|
|
1526
|
+
return interrupted(errorCode(error) ?? 'native_target_error', 'thrown');
|
|
1527
|
+
}
|
|
1528
|
+
finally {
|
|
1529
|
+
clearTimeout(timer);
|
|
1530
|
+
drillSignal.removeEventListener('abort', onDrillAbort);
|
|
1531
|
+
}
|
|
1532
|
+
}
|
|
1533
|
+
const OUTCOMES = new Set(['completed', 'failed', 'cancelled', 'unknown']);
|
|
1534
|
+
/** Keep the driver's result but bind it to this step; an unrecognised outcome on a non-idempotent step is `unknown`. */
|
|
1535
|
+
function normalizeResult(result, step, startedMs) {
|
|
1536
|
+
const outcome = result && OUTCOMES.has(result.outcome) ? result.outcome : (step.idempotent ? 'failed' : 'unknown');
|
|
1537
|
+
return {
|
|
1538
|
+
step_id: step.step_id,
|
|
1539
|
+
intent_id: step.intent_id,
|
|
1540
|
+
outcome,
|
|
1541
|
+
started_ms: Number.isFinite(result?.started_ms) ? result.started_ms : startedMs,
|
|
1542
|
+
finished_ms: Number.isFinite(result?.finished_ms) ? result.finished_ms : Date.now(),
|
|
1543
|
+
output: result?.output ?? null,
|
|
1544
|
+
output_coverage: result?.output_coverage === 'complete' || result?.output_coverage === 'redacted'
|
|
1545
|
+
|| result?.output_coverage === 'omitted' ? result.output_coverage : undefined,
|
|
1546
|
+
error_code: typeof result?.error_code === 'string' ? result.error_code
|
|
1547
|
+
: outcome === 'unknown' ? NATIVE_ERRORS.unknown_outcome : null,
|
|
1548
|
+
observations: Array.isArray(result?.observations) ? result.observations : [],
|
|
1549
|
+
delivered: result?.delivered === true,
|
|
1550
|
+
};
|
|
1551
|
+
}
|
|
1552
|
+
/**
|
|
1553
|
+
* Write the MCP client configuration for one per-drill service whose MCP
|
|
1554
|
+
* dependencies the plan routes through it (`mcp_config_env` names the variable
|
|
1555
|
+
* that receives its path). Returns null when the service has none.
|
|
1556
|
+
*/
|
|
1557
|
+
async function writeServiceMcpConfig(routes, service, urls, scratchDir) {
|
|
1558
|
+
const servers = mcpServers(routes, service.alias, urls);
|
|
1559
|
+
if (servers.length === 0)
|
|
1560
|
+
return null;
|
|
1561
|
+
return writeMcpConfig(servers, scratchDir, { envName: service.mcp_config_env });
|
|
1562
|
+
}
|