@opengeni/worker-bundle 2.0.1 → 2.0.2-canary.38789554071001
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/activities/agent-run-admission.d.ts +1 -12
- package/dist/activities/agent-turn/admission.d.ts +88 -10
- package/dist/activities/agent-turn/agent-build.d.ts +14 -3
- package/dist/activities/agent-turn/claim.d.ts +4 -1
- package/dist/activities/agent-turn/claude-core.d.ts +195 -0
- package/dist/activities/agent-turn/code-search.d.ts +60 -0
- package/dist/activities/agent-turn/codex-capacity.d.ts +3 -0
- package/dist/activities/agent-turn/codex-core-capacity.d.ts +37 -0
- package/dist/activities/agent-turn/codex-core-claim.d.ts +40 -0
- package/dist/activities/agent-turn/codex-core-errors.d.ts +102 -0
- package/dist/activities/agent-turn/codex-core-failover.d.ts +15 -0
- package/dist/activities/agent-turn/codex-core-requests.d.ts +28 -0
- package/dist/activities/agent-turn/codex-core-settlement.d.ts +77 -0
- package/dist/activities/agent-turn/codex-credit-policy.d.ts +30 -0
- package/dist/activities/agent-turn/codex-plan-entitlement.d.ts +105 -0
- package/dist/activities/agent-turn/codex-usage-policy.d.ts +2 -0
- package/dist/activities/agent-turn/codex.d.ts +2 -19
- package/dist/activities/agent-turn/compaction-interruption.d.ts +32 -0
- package/dist/activities/agent-turn/compaction-prep.d.ts +30 -1
- package/dist/activities/agent-turn/credential-leases.d.ts +64 -37
- package/dist/activities/agent-turn/credential-restriction.d.ts +3 -0
- package/dist/activities/agent-turn/errors.d.ts +185 -26
- package/dist/activities/agent-turn/failure-settlement.d.ts +10 -43
- package/dist/activities/agent-turn/final-reply.d.ts +18 -0
- package/dist/activities/agent-turn/finalization-monitor.d.ts +7 -2
- package/dist/activities/agent-turn/finalization.d.ts +2 -0
- package/dist/activities/agent-turn/github-repository-bindings.d.ts +2 -0
- package/dist/activities/agent-turn/governance-model.d.ts +13 -1
- package/dist/activities/agent-turn/history-sink.d.ts +4 -1
- package/dist/activities/agent-turn/history.d.ts +2 -0
- package/dist/activities/agent-turn/input-wait-reply.d.ts +76 -0
- package/dist/activities/agent-turn/meaningful-output.d.ts +9 -0
- package/dist/activities/agent-turn/media-artifacts.d.ts +5 -3
- package/dist/activities/agent-turn/model-call-admission.d.ts +15 -0
- package/dist/activities/agent-turn/model-usage.d.ts +56 -14
- package/dist/activities/agent-turn/model-wire-dispatch.d.ts +15 -0
- package/dist/activities/agent-turn/optional-repositories.d.ts +59 -0
- package/dist/activities/agent-turn/prompt-tool-availability.d.ts +17 -0
- package/dist/activities/agent-turn/provider-dispatch-barrier.d.ts +12 -1
- package/dist/activities/agent-turn/provider-recovery-copy.d.ts +24 -0
- package/dist/activities/agent-turn/provider-recovery-metrics.d.ts +18 -0
- package/dist/activities/agent-turn/provider-recovery-policy.d.ts +6 -0
- package/dist/activities/agent-turn/quiescence.d.ts +6 -7
- package/dist/activities/agent-turn/recovery-warning.d.ts +6 -5
- package/dist/activities/agent-turn/refresh-credentials.d.ts +6 -0
- package/dist/activities/agent-turn/run-credentials.d.ts +9 -5
- package/dist/activities/agent-turn/run.d.ts +27 -2
- package/dist/activities/agent-turn/sandbox-establish.d.ts +1 -0
- package/dist/activities/agent-turn/sandbox-provision.d.ts +25 -1
- package/dist/activities/agent-turn/sandbox-route.d.ts +3 -0
- package/dist/activities/agent-turn/sandbox-runtime.d.ts +1 -1
- package/dist/activities/agent-turn/secret-resolver.d.ts +64 -0
- package/dist/activities/agent-turn/session-title.d.ts +60 -3
- package/dist/activities/agent-turn/session-usage-custody.d.ts +66 -0
- package/dist/activities/agent-turn/skill-checkout.d.ts +25 -6
- package/dist/activities/agent-turn/skill-read.d.ts +70 -2
- package/dist/activities/agent-turn/skill-search.d.ts +2 -0
- package/dist/activities/agent-turn/skill-tools.d.ts +41 -4
- package/dist/activities/agent-turn/skill-transfer.d.ts +18 -6
- package/dist/activities/agent-turn/stream-attempt.d.ts +6 -3
- package/dist/activities/agent-turn/subscription-capacity-arming.d.ts +41 -0
- package/dist/activities/agent-turn/subscription-core-capacity.d.ts +86 -0
- package/dist/activities/agent-turn/subscription-core-failover.d.ts +69 -0
- package/dist/activities/agent-turn/subscription-core-placed.d.ts +14 -0
- package/dist/activities/agent-turn/subscription-core-requests.d.ts +109 -0
- package/dist/activities/agent-turn/subscription-core-settlement.d.ts +112 -0
- package/dist/activities/agent-turn/subscription-lease.d.ts +44 -0
- package/dist/activities/agent-turn/subscription-serving-capacity.d.ts +6 -0
- package/dist/activities/agent-turn/subscription-serving-catalog.d.ts +13 -0
- package/dist/activities/agent-turn/subscription-serving-claim.d.ts +30 -0
- package/dist/activities/agent-turn/subscription-serving-providers.d.ts +2 -0
- package/dist/activities/agent-turn/tool-environment.d.ts +31 -6
- package/dist/activities/agent-turn/tool-policy.d.ts +55 -3
- package/dist/activities/agent-turn/turn-context.d.ts +104 -5
- package/dist/activities/agent-turn/web-search.d.ts +57 -0
- package/dist/activities/agent-turn/xai-core.d.ts +96 -0
- package/dist/activities/automations.d.ts +3 -1
- package/dist/activities/browser-deadline-checkpoint.d.ts +34 -0
- package/dist/activities/capabilities.d.ts +8 -20
- package/dist/activities/codemode-dispatcher.d.ts +20 -3
- package/dist/activities/codex-capacity.d.ts +1 -56
- package/dist/activities/codex-image-generation.d.ts +49 -3
- package/dist/activities/common.d.ts +3 -1
- package/dist/activities/context-compaction.d.ts +41 -2
- package/dist/activities/environment.d.ts +1 -19
- package/dist/activities/gmail-files.d.ts +50 -0
- package/dist/activities/goals.d.ts +1 -9
- package/dist/activities/incident-telemetry-authority.d.ts +5 -2
- package/dist/activities/incident-telemetry-preflight.d.ts +10 -3
- package/dist/activities/knowledge-indexing.d.ts +19 -0
- package/dist/activities/mcp-account-routes.d.ts +2 -21
- package/dist/activities/model-call-fact-reconciler.d.ts +34 -0
- package/dist/activities/open-suffix-resume.d.ts +15 -0
- package/dist/activities/parent-wake.d.ts +17 -12
- package/dist/activities/programmatic-approvals.d.ts +10 -0
- package/dist/activities/requester-unavailable-receipt.d.ts +8 -0
- package/dist/activities/retained-screenshots.d.ts +7 -2
- package/dist/activities/run-credential-renewal.d.ts +1 -0
- package/dist/activities/run-credentials.d.ts +16 -4
- package/dist/activities/run-input.d.ts +17 -0
- package/dist/activities/sandbox-lease.d.ts +132 -12
- package/dist/activities/scheduled-tasks.d.ts +4 -35
- package/dist/activities/session-archive.d.ts +58 -0
- package/dist/activities/session-state.d.ts +15 -3
- package/dist/activities/session-storage.d.ts +44 -0
- package/dist/activities/subscription-core-claude-waits.d.ts +2 -0
- package/dist/activities/subscription-core-codex-quota.d.ts +25 -0
- package/dist/activities/subscription-core-codex-waits.d.ts +153 -0
- package/dist/activities/subscription-core-image-operation.d.ts +29 -0
- package/dist/activities/subscription-core-usage-recheck.d.ts +49 -0
- package/dist/activities/subscription-core-wait-bindings.d.ts +3 -0
- package/dist/activities/subscription-core-xai-quota.d.ts +30 -0
- package/dist/activities/subscription-core-xai-waits.d.ts +2 -0
- package/dist/activities/types.d.ts +68 -5
- package/dist/activities/video-generation-admission.d.ts +2 -8
- package/dist/activities/video-generation-credential.d.ts +15 -12
- package/dist/activities/video-generation-subscription-core.d.ts +70 -0
- package/dist/activities/workspace-capture.d.ts +2 -0
- package/dist/activities/workspace-credential-provider.d.ts +47 -0
- package/dist/activities/xai-auth.d.ts +6 -18
- package/dist/activities/xai-image-generation.d.ts +22 -6
- package/dist/{activities-control-QA6OLRQY.js → activities-control-5IIVRXLA.js} +1569 -2883
- package/dist/activities-control-5IIVRXLA.js.map +1 -0
- package/dist/activities-control.d.ts +64 -4
- package/dist/{activities-turn-YVVKCGAK.js → activities-turn-FLYJOM54.js} +9300 -4403
- package/dist/activities-turn-FLYJOM54.js.map +1 -0
- package/dist/activities.d.ts +64 -4
- package/dist/artifact-materializer-entry.js +107 -36
- package/dist/artifact-materializer-entry.js.map +1 -1
- package/dist/artifact-outbox-entry.js +132 -61
- package/dist/artifact-outbox-entry.js.map +1 -1
- package/dist/browser-deadline-checkpoint-contract.d.ts +2 -0
- package/dist/{chunk-GDCLDKSH.js → chunk-24GP3BX5.js} +79 -72
- package/dist/chunk-24GP3BX5.js.map +1 -0
- package/dist/{chunk-RAMNYSAW.js → chunk-4ER3YXOT.js} +22 -4
- package/dist/chunk-4ER3YXOT.js.map +1 -0
- package/dist/{chunk-I7HKKMUJ.js → chunk-5NNS4327.js} +734 -58
- package/dist/chunk-5NNS4327.js.map +1 -0
- package/dist/chunk-MVESOYFC.js +3746 -0
- package/dist/chunk-MVESOYFC.js.map +1 -0
- package/dist/{chunk-M3NVMF2V.js → chunk-N57B3PJ5.js} +2307 -3951
- package/dist/chunk-N57B3PJ5.js.map +1 -0
- package/dist/chunk-NQDVJ6RI.js +2823 -0
- package/dist/chunk-NQDVJ6RI.js.map +1 -0
- package/dist/chunk-O7UPBWXK.js +79 -0
- package/dist/chunk-O7UPBWXK.js.map +1 -0
- package/dist/chunk-W2QSM5HR.js +414 -0
- package/dist/chunk-W2QSM5HR.js.map +1 -0
- package/dist/chunk-ZN3XJR33.js +227 -0
- package/dist/chunk-ZN3XJR33.js.map +1 -0
- package/dist/editable-artifact-outbox-dispatcher.d.ts +12 -1
- package/dist/editable-artifact-outbox-service.d.ts +3 -1
- package/dist/host-export-pump.d.ts +3 -1
- package/dist/host-export.d.ts +2 -2
- package/dist/host-export.js +1 -1
- package/dist/http.d.ts +1 -1
- package/dist/index.d.ts +49 -48
- package/dist/index.js +835 -256
- package/dist/index.js.map +1 -1
- package/dist/internal-maintenance.d.ts +63 -0
- package/dist/maintenance-cadence.d.ts +43 -0
- package/dist/observability-metrics.d.ts +293 -24
- package/dist/onepassword-vault-provider.d.ts +7 -0
- package/dist/parent-wake-PCLH7A3Y.js +24 -0
- package/dist/parent-wake-PCLH7A3Y.js.map +1 -0
- package/dist/personal-github-git-credentials.d.ts +1 -1
- package/dist/readiness-checks.d.ts +39 -0
- package/dist/retained-process-retry.d.ts +7 -0
- package/dist/sandbox-lease-BJYU72N3.js +84 -0
- package/dist/sandbox-lease-BJYU72N3.js.map +1 -0
- package/dist/sandbox-reaper-contract.d.ts +25 -0
- package/dist/sandbox-resume.d.ts +72 -4
- package/dist/sandbox-routing.d.ts +4 -2
- package/dist/session-event-fanout.d.ts +6 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js +12 -0
- package/dist/site-auth-maintenance-WK3MR2PV.js.map +1 -0
- package/dist/telemetry-scheduler.d.ts +14 -0
- package/dist/temporal-runtime-logger.d.ts +22 -0
- package/dist/turn-task-queue-reader.d.ts +18 -0
- package/dist/turn-worker-activity-telemetry.d.ts +47 -0
- package/dist/worker-service-lifecycle.d.ts +12 -0
- package/dist/workflow-bundle.js +468 -71
- package/dist/workflows/activities.d.ts +1 -1
- package/dist/workflows/browser-deadline-checkpoint.d.ts +10 -0
- package/dist/workflows/sandbox-reaper.d.ts +10 -1
- package/dist/workflows/scheduled-tasks.d.ts +4 -0
- package/dist/workflows/session-storage.d.ts +2 -0
- package/dist/workflows/session.d.ts +36 -0
- package/dist/workflows/site-auth-maintenance.d.ts +2 -1
- package/dist/workflows/workflow-wake.d.ts +14 -3
- package/dist/workflows.d.ts +3 -1
- package/package.json +26 -20
- package/src/activities/agent-run-admission.ts +1 -82
- package/src/activities/agent-turn/admission.ts +175 -24
- package/src/activities/agent-turn/agent-build.ts +226 -145
- package/src/activities/agent-turn/claim.ts +363 -64
- package/src/activities/agent-turn/claude-core.ts +587 -0
- package/src/activities/agent-turn/code-search.ts +275 -0
- package/src/activities/agent-turn/codex-capacity.ts +30 -757
- package/src/activities/agent-turn/codex-core-capacity.ts +190 -0
- package/src/activities/agent-turn/codex-core-claim.ts +83 -0
- package/src/activities/agent-turn/codex-core-errors.ts +308 -0
- package/src/activities/agent-turn/codex-core-failover.ts +73 -0
- package/src/activities/agent-turn/codex-core-requests.ts +86 -0
- package/src/activities/agent-turn/codex-core-settlement.ts +263 -0
- package/src/activities/agent-turn/codex-credit-policy.ts +247 -0
- package/src/activities/agent-turn/codex-plan-entitlement.ts +254 -0
- package/src/activities/agent-turn/codex-usage-policy.ts +2 -0
- package/src/activities/agent-turn/codex.ts +8 -76
- package/src/activities/agent-turn/compaction-interruption.ts +56 -0
- package/src/activities/agent-turn/compaction-prep.ts +249 -36
- package/src/activities/agent-turn/credential-leases.ts +269 -214
- package/src/activities/agent-turn/credential-restriction.ts +14 -0
- package/src/activities/agent-turn/errors.ts +1152 -82
- package/src/activities/agent-turn/failure-settlement.ts +879 -886
- package/src/activities/agent-turn/final-reply.ts +46 -0
- package/src/activities/agent-turn/finalization-monitor.ts +17 -5
- package/src/activities/agent-turn/finalization.ts +107 -88
- package/src/activities/agent-turn/github-repository-bindings.ts +3 -1
- package/src/activities/agent-turn/governance-model.ts +90 -22
- package/src/activities/agent-turn/history-sink.ts +42 -12
- package/src/activities/agent-turn/history.ts +25 -5
- package/src/activities/agent-turn/input-wait-reply.ts +139 -0
- package/src/activities/agent-turn/knowledge-source-tools.ts +2 -0
- package/src/activities/agent-turn/meaningful-output.ts +44 -0
- package/src/activities/agent-turn/media-artifacts.ts +3 -1
- package/src/activities/agent-turn/model-call-admission.ts +114 -0
- package/src/activities/agent-turn/model-usage.ts +299 -233
- package/src/activities/agent-turn/model-wire-dispatch.ts +27 -0
- package/src/activities/agent-turn/optional-repositories.ts +224 -0
- package/src/activities/agent-turn/prompt-tool-availability.ts +27 -0
- package/src/activities/agent-turn/provider-dispatch-barrier.ts +22 -0
- package/src/activities/agent-turn/provider-recovery-copy.ts +76 -0
- package/src/activities/agent-turn/provider-recovery-metrics.ts +64 -0
- package/src/activities/agent-turn/provider-recovery-policy.ts +25 -0
- package/src/activities/agent-turn/quiescence.ts +7 -7
- package/src/activities/agent-turn/recovery-warning.ts +6 -8
- package/src/activities/agent-turn/refresh-credentials.ts +44 -0
- package/src/activities/agent-turn/run-credentials.ts +124 -76
- package/src/activities/agent-turn/run.ts +501 -112
- package/src/activities/agent-turn/sandbox-establish.ts +152 -75
- package/src/activities/agent-turn/sandbox-provision.ts +54 -1
- package/src/activities/agent-turn/sandbox-route.ts +4 -1
- package/src/activities/agent-turn/sandbox-runtime.ts +319 -11
- package/src/activities/agent-turn/secret-resolver.ts +155 -0
- package/src/activities/agent-turn/session-title.ts +177 -12
- package/src/activities/agent-turn/session-usage-custody.ts +170 -0
- package/src/activities/agent-turn/skill-checkout.ts +111 -24
- package/src/activities/agent-turn/skill-install.ts +1 -1
- package/src/activities/agent-turn/skill-read.ts +374 -28
- package/src/activities/agent-turn/skill-save.ts +35 -7
- package/src/activities/agent-turn/skill-search.ts +9 -0
- package/src/activities/agent-turn/skill-selection.ts +4 -0
- package/src/activities/agent-turn/skill-tools.ts +208 -21
- package/src/activities/agent-turn/skill-transfer.ts +68 -23
- package/src/activities/agent-turn/stream-attempt.ts +634 -111
- package/src/activities/agent-turn/subscription-capacity-arming.ts +81 -0
- package/src/activities/agent-turn/subscription-core-capacity.ts +439 -0
- package/src/activities/agent-turn/subscription-core-failover.ts +229 -0
- package/src/activities/agent-turn/subscription-core-placed.ts +37 -0
- package/src/activities/agent-turn/subscription-core-requests.ts +271 -0
- package/src/activities/agent-turn/subscription-core-settlement.ts +234 -0
- package/src/activities/agent-turn/subscription-lease.ts +140 -0
- package/src/activities/agent-turn/subscription-serving-capacity.ts +195 -0
- package/src/activities/agent-turn/subscription-serving-catalog.ts +71 -0
- package/src/activities/agent-turn/subscription-serving-claim.ts +178 -0
- package/src/activities/agent-turn/subscription-serving-providers.ts +15 -0
- package/src/activities/agent-turn/tool-environment.ts +384 -292
- package/src/activities/agent-turn/tool-policy.ts +104 -4
- package/src/activities/agent-turn/tool-result-spill.ts +1 -0
- package/src/activities/agent-turn/turn-context.ts +124 -8
- package/src/activities/agent-turn/web-search.ts +431 -0
- package/src/activities/agent-turn/xai-core.ts +301 -0
- package/src/activities/automations.ts +81 -6
- package/src/activities/browser-deadline-checkpoint.ts +297 -0
- package/src/activities/capabilities.ts +53 -213
- package/src/activities/codemode-dispatcher.ts +236 -14
- package/src/activities/codex-capacity.ts +27 -380
- package/src/activities/codex-image-generation.ts +252 -42
- package/src/activities/common.ts +12 -0
- package/src/activities/connector-attachments.ts +3 -1
- package/src/activities/context-compaction.ts +182 -21
- package/src/activities/environment.ts +18 -106
- package/src/activities/gmail-files.ts +108 -0
- package/src/activities/goals.ts +120 -204
- package/src/activities/image-generation-operation.ts +1 -1
- package/src/activities/incident-telemetry-authority.ts +80 -50
- package/src/activities/incident-telemetry-preflight.ts +27 -15
- package/src/activities/knowledge-indexing.ts +287 -17
- package/src/activities/knowledge-source-sync.ts +10 -0
- package/src/activities/mcp-account-routes.ts +2 -124
- package/src/activities/model-call-fact-reconciler.ts +112 -0
- package/src/activities/open-suffix-resume.ts +122 -88
- package/src/activities/parent-wake.ts +89 -21
- package/src/activities/programmatic-approvals.ts +26 -0
- package/src/activities/requester-unavailable-receipt.ts +14 -0
- package/src/activities/retained-screenshots.ts +135 -24
- package/src/activities/run-credential-renewal.ts +13 -1
- package/src/activities/run-credentials.ts +61 -19
- package/src/activities/run-input.ts +124 -9
- package/src/activities/sandbox-lease.ts +1615 -73
- package/src/activities/scheduled-tasks.ts +780 -147
- package/src/activities/session-archive.ts +382 -0
- package/src/activities/session-state.ts +219 -10
- package/src/activities/session-storage.ts +124 -0
- package/src/activities/site-auth-maintenance.ts +5 -2
- package/src/activities/subscription-core-claude-waits.ts +49 -0
- package/src/activities/subscription-core-codex-quota.ts +46 -0
- package/src/activities/subscription-core-codex-waits.ts +488 -0
- package/src/activities/subscription-core-image-operation.ts +126 -0
- package/src/activities/subscription-core-usage-recheck.ts +146 -0
- package/src/activities/subscription-core-wait-bindings.ts +12 -0
- package/src/activities/subscription-core-xai-quota.ts +85 -0
- package/src/activities/subscription-core-xai-waits.ts +48 -0
- package/src/activities/types.ts +85 -9
- package/src/activities/video-generation-admission.ts +1 -36
- package/src/activities/video-generation-credential.ts +38 -56
- package/src/activities/video-generation-reconciliation.ts +150 -131
- package/src/activities/video-generation-subscription-core.ts +180 -0
- package/src/activities/workspace-capture.ts +13 -2
- package/src/activities/workspace-credential-provider.ts +302 -0
- package/src/activities/xai-auth.ts +34 -112
- package/src/activities/xai-image-generation.ts +84 -36
- package/src/activities-control.ts +8 -0
- package/src/activities-turn.ts +5 -0
- package/src/activities.ts +17 -0
- package/src/activity-services.ts +29 -4
- package/src/browser-deadline-checkpoint-contract.ts +5 -0
- package/src/editable-artifact-hint-broker.ts +3 -0
- package/src/editable-artifact-materializer-service.ts +85 -39
- package/src/editable-artifact-materializer-subprocess.ts +47 -2
- package/src/editable-artifact-outbox-dispatcher.ts +107 -46
- package/src/editable-artifact-outbox-service.ts +55 -25
- package/src/github-rest-mcp.ts +1 -1
- package/src/host-export-pump.ts +26 -4
- package/src/host-export.ts +4 -1
- package/src/http.ts +4 -2
- package/src/index.ts +222 -282
- package/src/internal-maintenance.ts +347 -0
- package/src/maintenance-cadence.ts +156 -0
- package/src/observability-metrics.ts +1173 -73
- package/src/onepassword-vault-provider.ts +11 -0
- package/src/personal-github-git-credentials.ts +3 -1
- package/src/pr-review-credentials.ts +1 -1
- package/src/readiness-checks.ts +113 -0
- package/src/retained-process-retry.ts +36 -0
- package/src/sandbox-reaper-contract.ts +33 -0
- package/src/sandbox-resume.ts +444 -73
- package/src/sandbox-routing.ts +230 -75
- package/src/session-event-fanout.ts +23 -0
- package/src/telemetry-scheduler.ts +53 -0
- package/src/temporal-runtime-logger.ts +44 -0
- package/src/turn-task-queue-reader.ts +110 -0
- package/src/turn-worker-activity-telemetry.ts +209 -0
- package/src/worker-service-lifecycle.ts +54 -1
- package/src/workflows/activities.ts +16 -1
- package/src/workflows/browser-deadline-checkpoint.ts +55 -0
- package/src/workflows/file-upload-reaper.ts +2 -0
- package/src/workflows/sandbox-reaper.ts +88 -0
- package/src/workflows/scheduled-tasks.ts +26 -13
- package/src/workflows/session-storage.ts +19 -0
- package/src/workflows/session.ts +238 -33
- package/src/workflows/site-auth-maintenance.ts +2 -1
- package/src/workflows/workflow-wake.ts +26 -4
- package/src/workflows.ts +7 -0
- package/dist/activities/agent-turn/xai-capacity.d.ts +0 -2
- package/dist/activities/codex-fleet-shadow.d.ts +0 -74
- package/dist/activities/codex-rotation.d.ts +0 -200
- package/dist/activities/xai-quota.d.ts +0 -3
- package/dist/activities-control-QA6OLRQY.js.map +0 -1
- package/dist/activities-turn-YVVKCGAK.js.map +0 -1
- package/dist/chunk-GDCLDKSH.js.map +0 -1
- package/dist/chunk-I7HKKMUJ.js.map +0 -1
- package/dist/chunk-M3NVMF2V.js.map +0 -1
- package/dist/chunk-RAMNYSAW.js.map +0 -1
- package/src/activities/agent-turn/xai-capacity.ts +0 -257
- package/src/activities/codex-fleet-shadow.ts +0 -306
- package/src/activities/codex-rotation.ts +0 -685
- package/src/activities/xai-quota.ts +0 -35
|
@@ -1,8 +1,21 @@
|
|
|
1
1
|
import {
|
|
2
|
+
agentToolCallOutcome,
|
|
3
|
+
agentToolMetricFamily,
|
|
4
|
+
interactionInterventionMetricObserver,
|
|
5
|
+
measureMcpPhase,
|
|
6
|
+
recordAgentToolCall,
|
|
7
|
+
withMcpCallIdentity,
|
|
8
|
+
type AgentToolMetricFamily,
|
|
9
|
+
} from "@opengeni/observability";
|
|
10
|
+
import {
|
|
11
|
+
appendSessionHistoryItems,
|
|
12
|
+
sessionTurnFinalReplyFacts,
|
|
13
|
+
sessionTurnHasFinalReplyNudge,
|
|
2
14
|
getSessionEvent,
|
|
3
15
|
getHumanInputResumeForEvent,
|
|
4
16
|
getSessionHumanInputRequest,
|
|
5
17
|
getWorkspace,
|
|
18
|
+
listTurnCodemodeApprovals,
|
|
6
19
|
recordUsageEvent,
|
|
7
20
|
registerPendingSessionToolCall,
|
|
8
21
|
recordPendingSessionToolCallResult,
|
|
@@ -14,20 +27,28 @@ import {
|
|
|
14
27
|
updateSessionTitleWithEvent,
|
|
15
28
|
} from "@opengeni/db";
|
|
16
29
|
import { publishDurableSessionEvents } from "@opengeni/events";
|
|
30
|
+
import { recordProviderRecoveryOutcome } from "./provider-recovery-metrics";
|
|
17
31
|
import {
|
|
32
|
+
AssistantMessagePhaseTracker,
|
|
18
33
|
normalizeModelCallUsage,
|
|
19
34
|
normalizeSdkEvent,
|
|
20
35
|
withMcpToolDisplayMetadata,
|
|
36
|
+
withToolCallFamily,
|
|
21
37
|
extractOpenSuffixFromRunState,
|
|
22
38
|
assertOpenSuffixResumable,
|
|
23
39
|
interruptionKindForCallItem,
|
|
24
40
|
releaseMcpResultCustomDataFromSdkEvent,
|
|
25
41
|
findCompactionNeededError,
|
|
42
|
+
findAnthropicRequestSizeError,
|
|
43
|
+
type AnthropicRequestSizeError,
|
|
44
|
+
ANTHROPIC_REQUEST_MAX_BYTES,
|
|
26
45
|
compactionProviderRejection,
|
|
27
46
|
withRunCredentialsSession,
|
|
28
47
|
runOwnedSandboxSetup,
|
|
48
|
+
modelResponseHasVisibleText,
|
|
29
49
|
type SandboxFileDownload,
|
|
30
50
|
type OpenGeniRuntime,
|
|
51
|
+
type GeneratedSessionTitle,
|
|
31
52
|
type HistoryProviderApi,
|
|
32
53
|
type GitCredentialTokenWriterSession,
|
|
33
54
|
type NormalizedRunCredentialMaterial,
|
|
@@ -36,6 +57,10 @@ import {
|
|
|
36
57
|
} from "@opengeni/runtime";
|
|
37
58
|
import { serviceTierForLatencyMode, type Settings } from "@opengeni/config";
|
|
38
59
|
import { maybeCompactContext, settleFailedContextCompactionLandmark } from "../context-compaction";
|
|
60
|
+
import {
|
|
61
|
+
isContextCompactionInterruption,
|
|
62
|
+
settleContextCompactionInterruption,
|
|
63
|
+
} from "./compaction-interruption";
|
|
39
64
|
import { TurnAttemptFencedError } from "../turn-attempt-fenced";
|
|
40
65
|
import { type MintedRunGitCredentials } from "../environment";
|
|
41
66
|
import { withFirstPartyTools } from "../goals";
|
|
@@ -61,7 +86,7 @@ import {
|
|
|
61
86
|
import {
|
|
62
87
|
compactRetainedScreenshotHistory,
|
|
63
88
|
sdkEventContainsInlineImage,
|
|
64
|
-
|
|
89
|
+
retainSessionScreenshot,
|
|
65
90
|
typedScreenshotFromSdkEvent,
|
|
66
91
|
unavailableRetainedSessionImage,
|
|
67
92
|
} from "../retained-screenshots";
|
|
@@ -70,6 +95,8 @@ import {
|
|
|
70
95
|
generatedImageFromSdkEvent,
|
|
71
96
|
isCompletedGeneratedImageSdkEvent,
|
|
72
97
|
} from "../generated-images";
|
|
98
|
+
import { programmaticApproval, programmaticContinuationNote } from "../programmatic-approvals";
|
|
99
|
+
import { CodemodeAttemptDispatcher } from "../codemode-dispatcher";
|
|
73
100
|
import { ToolResultSpill } from "./tool-result-spill";
|
|
74
101
|
import { ownedTurnSandboxForAgent } from "./turn-sandbox-access";
|
|
75
102
|
import { createTurnCredentialLeases } from "./credential-leases";
|
|
@@ -82,24 +109,31 @@ import {
|
|
|
82
109
|
import {
|
|
83
110
|
OPEN_SUFFIX_RUN_STATE_BLOB,
|
|
84
111
|
resolveWorkspaceAgentHumanInputEnabled,
|
|
112
|
+
withPublicApprovalFields,
|
|
85
113
|
type RetainedArtifactMetadata,
|
|
86
114
|
type SessionEvent,
|
|
115
|
+
type SessionTurn,
|
|
87
116
|
} from "@opengeni/contracts";
|
|
88
117
|
import { createModelCheckpointMemoryCollector } from "../../model-checkpoint-memory-collector";
|
|
118
|
+
import { createModelCallAdmission } from "./model-call-admission";
|
|
119
|
+
import { isCoreSourceDisconnectedError } from "./subscription-core-capacity";
|
|
120
|
+
import { revalidatePlacedSubscriptionCore } from "./subscription-core-placed";
|
|
89
121
|
|
|
90
122
|
import {
|
|
91
123
|
assertWorkspaceHumanInputAllowed,
|
|
92
124
|
stableHumanInputRequestId,
|
|
93
125
|
stableInteractionInterventionId,
|
|
94
126
|
stableInteractionInterventionOperationId,
|
|
95
|
-
|
|
96
|
-
ensureRunAllowed,
|
|
127
|
+
ensureRunAllowedBetweenModelCalls,
|
|
97
128
|
} from "./admission";
|
|
98
129
|
import {
|
|
130
|
+
assertProviderOverloadRecoveryActive,
|
|
99
131
|
compactionFailureReason,
|
|
100
132
|
safeErrorDiagnostic,
|
|
101
133
|
compactionFailureReasonFromError,
|
|
102
134
|
compactionFailureTurnEventPayload,
|
|
135
|
+
compactionFirstPreparationEventPayload,
|
|
136
|
+
modelPreparationFailureEventPayload,
|
|
103
137
|
isCompactionSummaryFailure,
|
|
104
138
|
PostCompactionContinuationEmptyError,
|
|
105
139
|
shouldRecoverCompactionProviderFailure,
|
|
@@ -108,10 +142,11 @@ import {
|
|
|
108
142
|
import {
|
|
109
143
|
pendingToolCallFromSdkEvent,
|
|
110
144
|
toolCallProducesRetainableSessionImage,
|
|
145
|
+
retainableBrowserScreenshotToolCall,
|
|
111
146
|
completedToolCallFromSdkEvent,
|
|
112
147
|
} from "./history";
|
|
113
|
-
import { checkpointHistoryBeforeProviderDispatch } from "./provider-dispatch-barrier";
|
|
114
148
|
import {
|
|
149
|
+
aggregateCreditPolicyRevision,
|
|
115
150
|
modelUsageSourceKey,
|
|
116
151
|
recordCompletedModelCallBeforeOwnershipFences,
|
|
117
152
|
TurnEventPublisher,
|
|
@@ -124,15 +159,27 @@ import {
|
|
|
124
159
|
emitModelCallUsage,
|
|
125
160
|
recordModelUsageAndDebitCredits,
|
|
126
161
|
recordAuthoritativeModelCallFact,
|
|
162
|
+
recordAuthoritativeModelUsageMetrics,
|
|
127
163
|
} from "./model-usage";
|
|
128
|
-
import {
|
|
164
|
+
import { flushHeldSessionUsageReceipts } from "./session-usage-custody";
|
|
165
|
+
import {
|
|
166
|
+
sessionTitleGenerationOptions,
|
|
167
|
+
sessionTitleRoute,
|
|
168
|
+
startParallelSessionTitleGeneration,
|
|
169
|
+
} from "./session-title";
|
|
129
170
|
import {
|
|
130
171
|
assertAgentStreamNotCancelled,
|
|
131
172
|
assertSuccessfulAgentStreamCompletion,
|
|
132
173
|
requireAgentStreamFinalOutput,
|
|
133
174
|
} from "./quiescence";
|
|
175
|
+
import {
|
|
176
|
+
inputWaitReply,
|
|
177
|
+
inputWaitReplyRefusal,
|
|
178
|
+
latestDurableTurnMessageText,
|
|
179
|
+
} from "./input-wait-reply";
|
|
134
180
|
import { waitForTurnOperation } from "./sandbox-provision";
|
|
135
181
|
import { createSharedRigSetupCoordinator } from "./sandbox-shared-preparation";
|
|
182
|
+
import { finalReplyNudge, needsFinalReply } from "./final-reply";
|
|
136
183
|
|
|
137
184
|
import type { CompactionSummarizer } from "../context-compaction";
|
|
138
185
|
import type { TurnExecutionPolicyV1 } from "@opengeni/contracts";
|
|
@@ -147,6 +194,8 @@ import type {
|
|
|
147
194
|
SandboxRuntimeState,
|
|
148
195
|
TurnControlState,
|
|
149
196
|
} from "./turn-context";
|
|
197
|
+
import { sdkResponseHasMeaningfulOutput } from "./meaningful-output";
|
|
198
|
+
import { subscriptionCoreTurnOf } from "./turn-context";
|
|
150
199
|
|
|
151
200
|
export type TurnStreamAttemptDeps = {
|
|
152
201
|
input: RunAgentTurnInput;
|
|
@@ -173,11 +222,12 @@ export type TurnStreamAttemptDeps = {
|
|
|
173
222
|
providerTurn: ProviderTurnState;
|
|
174
223
|
leases: ReturnType<typeof createTurnCredentialLeases>;
|
|
175
224
|
historySink: ReturnType<typeof createTurnHistorySink>;
|
|
225
|
+
checkpointBeforeProviderDispatch: () => Promise<void>;
|
|
176
226
|
media: ReturnType<typeof createTurnMediaArtifacts>;
|
|
177
227
|
toolResultSpill: ToolResultSpill;
|
|
178
228
|
claimedResult: ClaimedResult;
|
|
179
229
|
flushRuntimeBatcher: () => Promise<void>;
|
|
180
|
-
finalizeTurnOpStreamOps: () => Promise<void>;
|
|
230
|
+
finalizeTurnOpStreamOps: (toolCallIds?: readonly string[]) => Promise<void>;
|
|
181
231
|
runWorkspaceMutationForSandbox: <T>(
|
|
182
232
|
sandbox: ResumedTurnSandbox,
|
|
183
233
|
operation: string,
|
|
@@ -225,7 +275,13 @@ export type TurnStreamAttemptDeps = {
|
|
|
225
275
|
activeSandboxBackend: Settings["sandboxBackend"] | undefined;
|
|
226
276
|
groupBoxBackend: Settings["sandboxBackend"];
|
|
227
277
|
turnExecutionPolicy: TurnExecutionPolicyV1;
|
|
228
|
-
turn:
|
|
278
|
+
turn: Pick<SessionTurn, "initiator" | "initiatorContext"> & {
|
|
279
|
+
initiatingHumanSubjectId: string | null;
|
|
280
|
+
id: string;
|
|
281
|
+
executionGeneration: number;
|
|
282
|
+
model: string;
|
|
283
|
+
source?: string;
|
|
284
|
+
};
|
|
229
285
|
trigger: NonNullable<Awaited<ReturnType<typeof getSessionEvent>>>;
|
|
230
286
|
humanInputResume: Awaited<ReturnType<typeof getHumanInputResumeForEvent>>;
|
|
231
287
|
attachPendingUpdatesAfterOpenSuffix: () => Promise<boolean>;
|
|
@@ -291,6 +347,7 @@ export async function runTurnStreamAttempt(
|
|
|
291
347
|
providerTurn,
|
|
292
348
|
leases,
|
|
293
349
|
historySink,
|
|
350
|
+
checkpointBeforeProviderDispatch,
|
|
294
351
|
media,
|
|
295
352
|
toolResultSpill,
|
|
296
353
|
claimedResult,
|
|
@@ -361,15 +418,24 @@ export async function runTurnStreamAttempt(
|
|
|
361
418
|
);
|
|
362
419
|
let parallelSessionTitle: ReturnType<typeof startParallelSessionTitleGeneration> | null = null;
|
|
363
420
|
let parallelSessionTitleFinished = false;
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
421
|
+
let creditPolicyRevision: number | undefined;
|
|
422
|
+
let titleCreditPolicyRevision: number | undefined;
|
|
423
|
+
let titleUsageModelId: string | null = null;
|
|
424
|
+
const persistParallelSessionTitle = async (
|
|
425
|
+
generated: GeneratedSessionTitle | null,
|
|
426
|
+
): Promise<void> => {
|
|
427
|
+
if (!generated) {
|
|
428
|
+
// A cancelled title may have received a complete provider response before
|
|
429
|
+
// its result was discarded. Checkpoint only that positive observation;
|
|
430
|
+
// pending or unknown requests keep their replay fence.
|
|
431
|
+
await subscriptionCoreTurnOf(providerTurn)?.titleRequests?.checkpoint();
|
|
432
|
+
return;
|
|
433
|
+
}
|
|
369
434
|
|
|
370
435
|
if (generated.usage) {
|
|
371
436
|
await processSessionTitleModelUsageEvent({
|
|
372
437
|
usage: generated.usage,
|
|
438
|
+
creditPolicyRevision: titleCreditPolicyRevision,
|
|
373
439
|
state: sessionTitleUsageState,
|
|
374
440
|
dispatchId: modelUsageDispatchId,
|
|
375
441
|
settings,
|
|
@@ -383,19 +449,23 @@ export async function runTurnStreamAttempt(
|
|
|
383
449
|
turnAttemptId: input.attemptId,
|
|
384
450
|
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
385
451
|
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
386
|
-
model: resolvedModel?.configured.id ?? turn.model,
|
|
452
|
+
model: titleUsageModelId ?? resolvedModel?.configured.id ?? turn.model,
|
|
387
453
|
externallyBilled: billingState.isExternallyBilledTurn,
|
|
388
454
|
chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
|
|
389
455
|
countsTowardTokenCap: billingState.countsTowardTokenCap,
|
|
390
456
|
servingCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
391
457
|
priorSessionCredentialId: providerTurn.priorSessionCodexCredentialId,
|
|
458
|
+
subscriptionConnectionId: subscriptionCoreTurnOf(providerTurn)?.connectionId ?? null,
|
|
392
459
|
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
393
460
|
renewLease: () => leases.renewServing("model_usage"),
|
|
394
461
|
leaseLost: leases.servingLost,
|
|
395
462
|
leaseLostMessage: "Provider credential lease expired during session title generation",
|
|
396
463
|
});
|
|
397
464
|
}
|
|
398
|
-
if (!generated.title)
|
|
465
|
+
if (!generated.title) {
|
|
466
|
+
await subscriptionCoreTurnOf(providerTurn)?.titleRequests?.checkpoint();
|
|
467
|
+
return;
|
|
468
|
+
}
|
|
399
469
|
|
|
400
470
|
try {
|
|
401
471
|
const result = await updateSessionTitleWithEvent(db, {
|
|
@@ -404,6 +474,7 @@ export async function runTurnStreamAttempt(
|
|
|
404
474
|
title: generated.title,
|
|
405
475
|
source: "agent",
|
|
406
476
|
});
|
|
477
|
+
await subscriptionCoreTurnOf(providerTurn)?.titleRequests?.checkpoint();
|
|
407
478
|
if (result.events.length > 0) {
|
|
408
479
|
await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, result.events);
|
|
409
480
|
}
|
|
@@ -416,12 +487,19 @@ export async function runTurnStreamAttempt(
|
|
|
416
487
|
});
|
|
417
488
|
}
|
|
418
489
|
};
|
|
490
|
+
const finishParallelSessionTitle = async (): Promise<void> => {
|
|
491
|
+
if (parallelSessionTitleFinished) return;
|
|
492
|
+
parallelSessionTitleFinished = true;
|
|
493
|
+
await parallelSessionTitle?.finish();
|
|
494
|
+
};
|
|
419
495
|
const cancelParallelSessionTitle = async (): Promise<void> => {
|
|
420
496
|
if (parallelSessionTitleFinished) return;
|
|
421
497
|
parallelSessionTitleFinished = true;
|
|
422
498
|
await parallelSessionTitle?.cancel();
|
|
423
499
|
};
|
|
424
500
|
let runInput: Awaited<ReturnType<typeof turnInput>>["input"] | null = null;
|
|
501
|
+
let codemodeContinuationNote: string | undefined;
|
|
502
|
+
let programmaticApprovalAcknowledged = false;
|
|
425
503
|
const prepareRunAttemptInput = async () => {
|
|
426
504
|
const historyPreparationStartedAt = performance.now();
|
|
427
505
|
let historyPreparationOutcome: "completed" | "failed" = "completed";
|
|
@@ -434,11 +512,18 @@ export async function runTurnStreamAttempt(
|
|
|
434
512
|
subjectId: fileAuthoritySubjectId,
|
|
435
513
|
},
|
|
436
514
|
recovering: turn.executionGeneration > 1,
|
|
515
|
+
programmaticApprovalAcknowledged,
|
|
516
|
+
...(codemodeContinuationNote ? { codemodeContinuationNote } : {}),
|
|
437
517
|
...(unavailableSandboxFilesNote ? { unavailableSandboxFilesNote } : {}),
|
|
438
518
|
...(runCredentialsNote ? { runCredentialsNote } : {}),
|
|
439
519
|
...(mcpAvailabilityNote ? { mcpAvailabilityNote } : {}),
|
|
440
520
|
...(knowledgeSourcePreparationNote ? { knowledgeSourcePreparationNote } : {}),
|
|
441
521
|
providerApi,
|
|
522
|
+
...(settings.subscriptionCrossProviderFailoverEnabled &&
|
|
523
|
+
compactionModeOptions.codexCompactionMode === "portable" &&
|
|
524
|
+
turnExecutionPolicy.credentialRestriction === undefined
|
|
525
|
+
? { servingProviderId: turnExecutionPolicy.providerId }
|
|
526
|
+
: {}),
|
|
442
527
|
projectCanonicalHistory: generatedImageHistoryProjector,
|
|
443
528
|
materializeModelHistory: media.materializeScreenshotHistory,
|
|
444
529
|
projectModelHistory: modelHistoryProjector,
|
|
@@ -518,6 +603,7 @@ export async function runTurnStreamAttempt(
|
|
|
518
603
|
const forceContextCompaction = async (
|
|
519
604
|
triggerLabel: "overflow" | "proactive" | "operator",
|
|
520
605
|
recoverySignalTokens: number | null,
|
|
606
|
+
requestSizeError?: AnthropicRequestSizeError | null,
|
|
521
607
|
) => {
|
|
522
608
|
const outcome = await waitForTurnOperation(
|
|
523
609
|
maybeCompactContext(
|
|
@@ -542,6 +628,19 @@ export async function runTurnStreamAttempt(
|
|
|
542
628
|
force: true,
|
|
543
629
|
...(triggerLabel === "operator" ? { clearRequestedCompaction: true } : {}),
|
|
544
630
|
trigger: triggerLabel,
|
|
631
|
+
...(requestSizeError
|
|
632
|
+
? {
|
|
633
|
+
requestSizeRecovery: {
|
|
634
|
+
// A proxy may reject below the documented API maximum. The
|
|
635
|
+
// checkpoint uses at most half the rejected payload size.
|
|
636
|
+
maxBytes: Math.min(
|
|
637
|
+
ANTHROPIC_REQUEST_MAX_BYTES,
|
|
638
|
+
Math.floor(requestSizeError.requestSize.requestBytes / 2),
|
|
639
|
+
),
|
|
640
|
+
size: requestSizeError.requestSize,
|
|
641
|
+
},
|
|
642
|
+
}
|
|
643
|
+
: {}),
|
|
545
644
|
materializeHistory: media.materializeScreenshotHistory,
|
|
546
645
|
projectModelInput: compactionModelHistoryProjector,
|
|
547
646
|
...compactionModeOptions,
|
|
@@ -564,15 +663,62 @@ export async function runTurnStreamAttempt(
|
|
|
564
663
|
// calling runStreamAttempt again; resetting this state there would reuse
|
|
565
664
|
// the first no-response-ID fallback key and suppress a real model call.
|
|
566
665
|
const modelResponseState = createModelResponseEventState(claimedModelUsageSourceKeys);
|
|
666
|
+
// Text of the newest assistant message any stream of this activity completed
|
|
667
|
+
// durably: the reply a wait-ended human turn records on turn.completed.
|
|
668
|
+
let latestAssistantMessageText: string | null = null;
|
|
669
|
+
// Waits refused because they would leave a person's message unanswered.
|
|
670
|
+
let inputWaitReplyRefusals = 0;
|
|
567
671
|
let workerPreparationTotalRecorded = false;
|
|
672
|
+
let toolsExecuted = false;
|
|
673
|
+
let finalReplyNudged = false;
|
|
674
|
+
const revalidateModelCallAdmission = async () => {
|
|
675
|
+
// The three checks are independent, so they run together; this sits on the
|
|
676
|
+
// critical path of every model call. All of them settle before any outcome
|
|
677
|
+
// is reported, and failures surface in the original order (durable
|
|
678
|
+
// history, subscription source, then usage), so a refusal is the same one
|
|
679
|
+
// the sequential checks would have produced.
|
|
680
|
+
const [history, source, usage] = await Promise.allSettled([
|
|
681
|
+
historySink.reconcileConversationTruth({ requireDurable: true }),
|
|
682
|
+
revalidatePlacedSubscriptionCore(db, providerTurn),
|
|
683
|
+
ensureRunAllowedBetweenModelCalls({
|
|
684
|
+
modelId: resolvedModel?.configured.id ?? turn.model,
|
|
685
|
+
settings,
|
|
686
|
+
db,
|
|
687
|
+
accountId: input.accountId,
|
|
688
|
+
workspaceId: input.workspaceId,
|
|
689
|
+
isExternallyBilledTurn: billingState.isExternallyBilledTurn,
|
|
690
|
+
entitlements,
|
|
691
|
+
chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
|
|
692
|
+
countsTowardTokenCap: billingState.countsTowardTokenCap,
|
|
693
|
+
initiatingHumanSubjectId: turn.initiatingHumanSubjectId,
|
|
694
|
+
sessionId: input.sessionId,
|
|
695
|
+
}),
|
|
696
|
+
]);
|
|
697
|
+
if (history.status === "rejected") throw history.reason;
|
|
698
|
+
if (source.status === "rejected") throw source.reason;
|
|
699
|
+
if (usage.status === "rejected") throw usage.reason;
|
|
700
|
+
creditPolicyRevision = usage.value;
|
|
701
|
+
};
|
|
568
702
|
const runStreamAttempt = async (options: {
|
|
569
703
|
requireTerminalModelResponse: boolean;
|
|
570
|
-
}): Promise<RunAgentTurnResult> => {
|
|
704
|
+
}): Promise<RunAgentTurnResult | "reply_nudge"> => {
|
|
571
705
|
if (!runInput) {
|
|
572
706
|
throw new Error("Run input was not prepared");
|
|
573
707
|
}
|
|
574
|
-
|
|
708
|
+
// The previous stream was persisted before compaction; the sink is now
|
|
709
|
+
// seeded from its durable replacement. Do not reconcile the old prefix
|
|
710
|
+
// against that replacement while checking the next stream's admission.
|
|
575
711
|
eventing.stream = undefined;
|
|
712
|
+
// Compaction commits its paid usage and replacement history before this
|
|
713
|
+
// boundary, both during preparation and in-activity recovery. Revalidate
|
|
714
|
+
// the accepted turn's frozen human before another stream can dispatch.
|
|
715
|
+
if (options.requireTerminalModelResponse) await revalidateModelCallAdmission();
|
|
716
|
+
const modelCallAdmission = createModelCallAdmission({
|
|
717
|
+
signal: runtimeCancellationSignal,
|
|
718
|
+
admit: revalidateModelCallAdmission,
|
|
719
|
+
});
|
|
720
|
+
const responseCreditPolicyRevisions = new Set<number | undefined>();
|
|
721
|
+
const responseCountBeforeStream = modelResponseState.responseCount;
|
|
576
722
|
eventing.batcher = null;
|
|
577
723
|
// The SDK emits every processed call item for one model response before
|
|
578
724
|
// it emits any result for that response. Keep that response-local batch
|
|
@@ -580,8 +726,19 @@ export async function runTurnStreamAttempt(
|
|
|
580
726
|
// calls. Durable recovery remains call/result based and needs no batch
|
|
581
727
|
// schema or compatibility state.
|
|
582
728
|
let currentToolBatchCallIds = new Set<string>();
|
|
729
|
+
// Content-free per-call family + start time for opengeni_agent_tool_calls_total.
|
|
730
|
+
const toolCallMetricStarts = new Map<
|
|
731
|
+
string,
|
|
732
|
+
{ family: AgentToolMetricFamily; startedAtMs: number }
|
|
733
|
+
>();
|
|
583
734
|
let currentToolBatchCompletedCallIds = new Set<string>();
|
|
584
735
|
let streamSawPerResponseUsage = false;
|
|
736
|
+
// Deltas learn the phase a provider declares when it announces a message;
|
|
737
|
+
// undeclared messages the SDK runs past (same response asks for tools)
|
|
738
|
+
// are commentary. Every SDK event of this stream is normalized once.
|
|
739
|
+
const messagePhases = new AssistantMessagePhaseTracker();
|
|
740
|
+
// Text of the newest assistant message this stream completed durably.
|
|
741
|
+
let latestStreamedAssistantText: string | null = null;
|
|
585
742
|
// Actual input tokens of the most recent model response this turn; the
|
|
586
743
|
// pre-read trigger for the NEXT turn. Persisted at every turn-end path.
|
|
587
744
|
throwIfWorkerShuttingDown();
|
|
@@ -590,10 +747,20 @@ export async function runTurnStreamAttempt(
|
|
|
590
747
|
const providerPublishesNativeRequestEvents =
|
|
591
748
|
resolvedModel?.provider.kind === "codex-subscription" ||
|
|
592
749
|
resolvedModel?.provider.kind === "xai-subscription";
|
|
750
|
+
// Bounded provider label for the streaming SLIs — the resolved registry
|
|
751
|
+
// provider id (or the built-in OpenAI/Azure provider), never a raw
|
|
752
|
+
// user-supplied model string. Created before the stream starts so the
|
|
753
|
+
// first request's model entry and provider dispatch are observed.
|
|
754
|
+
const streamTiming = new StreamTimingMetrics(observability, {
|
|
755
|
+
provider: streamProvider,
|
|
756
|
+
});
|
|
757
|
+
// Native Codex/SuperGrok transports report their literal dispatch through
|
|
758
|
+
// the shared turn context; generic transports use the wire-dispatch clock.
|
|
759
|
+
eventing.providerDispatchObserver = () => streamTiming.onProviderDispatch();
|
|
593
760
|
let fallbackProviderRequestStartedAt: number | null = null;
|
|
594
761
|
let fallbackProviderRequestLifecycleStartedAt: number | null = null;
|
|
595
762
|
const recordFallbackProviderDispatchAtWire = async (): Promise<void> => {
|
|
596
|
-
await
|
|
763
|
+
await checkpointBeforeProviderDispatch();
|
|
597
764
|
if (
|
|
598
765
|
providerPublishesNativeRequestEvents ||
|
|
599
766
|
eventing.firstModelRequestPreparationRecorded ||
|
|
@@ -653,6 +820,13 @@ export async function runTurnStreamAttempt(
|
|
|
653
820
|
count: turnTools.length,
|
|
654
821
|
});
|
|
655
822
|
}
|
|
823
|
+
// The required audit itself may have consumed the remaining window.
|
|
824
|
+
// This awaited observer still owns the literal pre-fetch boundary.
|
|
825
|
+
assertProviderOverloadRecoveryActive({
|
|
826
|
+
failureCode: attempt.providerRecoveryPolicyCode,
|
|
827
|
+
providerRecoveryCount: attempt.providerRecoveryCount,
|
|
828
|
+
recoveryStartedAt: attempt.providerRecoveryStartedAt,
|
|
829
|
+
});
|
|
656
830
|
};
|
|
657
831
|
const ownedEstablished = ownedTurnSandboxForAgent(sandboxState);
|
|
658
832
|
const runStreamOnce = async (): ReturnType<OpenGeniRuntime["runStream"]> => {
|
|
@@ -782,6 +956,14 @@ export async function runTurnStreamAttempt(
|
|
|
782
956
|
}
|
|
783
957
|
attempt.modelRequestStarted = true;
|
|
784
958
|
return await runtime.runStream(agent, runInput!, eventing.modelRunSettings, {
|
|
959
|
+
beforeModelRequest: (point) => {
|
|
960
|
+
streamTiming.onModelRequestEntry();
|
|
961
|
+
return modelCallAdmission.beforeModelRequest(point);
|
|
962
|
+
},
|
|
963
|
+
onModelResponse: (event) => {
|
|
964
|
+
responseCreditPolicyRevisions.add(creditPolicyRevision);
|
|
965
|
+
return modelCallAdmission.onModelResponse(event);
|
|
966
|
+
},
|
|
785
967
|
signal: runtimeCancellationSignal,
|
|
786
968
|
sandboxEnvironment,
|
|
787
969
|
onModelVisibleContext: async (snapshot) => {
|
|
@@ -888,6 +1070,13 @@ export async function runTurnStreamAttempt(
|
|
|
888
1070
|
...(!providerPublishesNativeRequestEvents
|
|
889
1071
|
? {
|
|
890
1072
|
onModelTransportStarted: recordFallbackProviderDispatchAtWire,
|
|
1073
|
+
onModelTransportDispatched: (clock) => {
|
|
1074
|
+
streamTiming.onProviderDispatch();
|
|
1075
|
+
eventing.initialModelWireDispatch.record(
|
|
1076
|
+
{ provider: streamProvider, dispatchId },
|
|
1077
|
+
clock,
|
|
1078
|
+
);
|
|
1079
|
+
},
|
|
891
1080
|
}
|
|
892
1081
|
: {}),
|
|
893
1082
|
...(eventing.toolCancellationFenceRef.current
|
|
@@ -917,13 +1106,36 @@ export async function runTurnStreamAttempt(
|
|
|
917
1106
|
if (leases.xai.lost) {
|
|
918
1107
|
throw new Error("xAI credential lease expired before the model run");
|
|
919
1108
|
}
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
1109
|
+
try {
|
|
1110
|
+
eventing.stream = await withProviderRequestContext(runStreamOnce);
|
|
1111
|
+
const stream = eventing.stream;
|
|
1112
|
+
eventing.inputWaitReplyGuard = async () => {
|
|
1113
|
+
const refusal = await inputWaitReplyRefusal({
|
|
1114
|
+
turn,
|
|
1115
|
+
refusalsSoFar: inputWaitReplyRefusals,
|
|
1116
|
+
hasVisibleReply: async () =>
|
|
1117
|
+
Boolean(latestAssistantMessageText?.trim()) ||
|
|
1118
|
+
modelResponseHasVisibleText(
|
|
1119
|
+
(stream.state as { _lastTurnResponse?: unknown })._lastTurnResponse,
|
|
1120
|
+
) ||
|
|
1121
|
+
Boolean(
|
|
1122
|
+
(
|
|
1123
|
+
await latestDurableTurnMessageText(db, {
|
|
1124
|
+
workspaceId: input.workspaceId,
|
|
1125
|
+
sessionId: input.sessionId,
|
|
1126
|
+
turnId: turn.id,
|
|
1127
|
+
})
|
|
1128
|
+
)?.trim(),
|
|
1129
|
+
),
|
|
1130
|
+
});
|
|
1131
|
+
if (refusal) inputWaitReplyRefusals += 1;
|
|
1132
|
+
return refusal;
|
|
1133
|
+
};
|
|
1134
|
+
} catch (error) {
|
|
1135
|
+
modelCallAdmission.fail(error);
|
|
1136
|
+
modelCallAdmission.close();
|
|
1137
|
+
throw error;
|
|
1138
|
+
}
|
|
927
1139
|
eventing.batcher = createRuntimeBatcher(
|
|
928
1140
|
async (events) => {
|
|
929
1141
|
await eventing.publish!(events);
|
|
@@ -963,6 +1175,7 @@ export async function runTurnStreamAttempt(
|
|
|
963
1175
|
attemptId: input.attemptId,
|
|
964
1176
|
dispatchId,
|
|
965
1177
|
executionGeneration: attempt.executionGeneration,
|
|
1178
|
+
...eventing.initialModelWireDispatch.payload({ provider: streamProvider, dispatchId }),
|
|
966
1179
|
},
|
|
967
1180
|
},
|
|
968
1181
|
]);
|
|
@@ -980,6 +1193,7 @@ export async function runTurnStreamAttempt(
|
|
|
980
1193
|
}
|
|
981
1194
|
await settleFallbackProviderFirstByte();
|
|
982
1195
|
let stableToolCallIdsToClear: string[] | null = null;
|
|
1196
|
+
let newlyDurableToolCallId: string | null = null;
|
|
983
1197
|
let completedCurrentToolBatch = false;
|
|
984
1198
|
let retainedScreenshotMetadata: RetainedArtifactMetadata | null = null;
|
|
985
1199
|
let normalizedSdkEvents: ReturnType<typeof normalizeSdkEvent> | null = null;
|
|
@@ -993,6 +1207,7 @@ export async function runTurnStreamAttempt(
|
|
|
993
1207
|
? await media.retainNativeGeneratedImage(generatedImage)
|
|
994
1208
|
: null;
|
|
995
1209
|
const responseResult = await processModelResponseTerminalEvent({
|
|
1210
|
+
creditPolicyRevision,
|
|
996
1211
|
event: next.value,
|
|
997
1212
|
state: modelResponseState,
|
|
998
1213
|
dispatchId: modelUsageDispatchId,
|
|
@@ -1007,7 +1222,7 @@ export async function runTurnStreamAttempt(
|
|
|
1007
1222
|
turnAttemptId: input.attemptId,
|
|
1008
1223
|
provider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
1009
1224
|
providerApi: resolvedModel?.provider.api ?? "responses",
|
|
1010
|
-
model:
|
|
1225
|
+
model: turnExecutionPolicy.productModelId,
|
|
1011
1226
|
latencyMode: turnExecutionPolicy.latencyMode,
|
|
1012
1227
|
metricProvider: streamProvider,
|
|
1013
1228
|
externallyBilled: billingState.isExternallyBilledTurn,
|
|
@@ -1015,6 +1230,7 @@ export async function runTurnStreamAttempt(
|
|
|
1015
1230
|
countsTowardTokenCap: billingState.countsTowardTokenCap,
|
|
1016
1231
|
servingCredentialId: providerTurn.effectiveCodexCredentialId,
|
|
1017
1232
|
priorSessionCredentialId: providerTurn.priorSessionCodexCredentialId,
|
|
1233
|
+
subscriptionConnectionId: subscriptionCoreTurnOf(providerTurn)?.connectionId ?? null,
|
|
1018
1234
|
emittedSourceKeys: emittedModelUsageSourceKeys,
|
|
1019
1235
|
renewLease: () => leases.renewServing("model_usage"),
|
|
1020
1236
|
leaseLost: leases.servingLost,
|
|
@@ -1025,7 +1241,7 @@ export async function runTurnStreamAttempt(
|
|
|
1025
1241
|
assertModelResponseLatencyMode({
|
|
1026
1242
|
event: next.value,
|
|
1027
1243
|
requested: turnExecutionPolicy.latencyMode,
|
|
1028
|
-
model:
|
|
1244
|
+
model: turnExecutionPolicy.productModelId,
|
|
1029
1245
|
...(resolvedModel?.provider.id ? { providerId: resolvedModel.provider.id } : {}),
|
|
1030
1246
|
});
|
|
1031
1247
|
if (responseResult.status === "processed") {
|
|
@@ -1049,11 +1265,30 @@ export async function runTurnStreamAttempt(
|
|
|
1049
1265
|
attemptId: input.attemptId,
|
|
1050
1266
|
dispatchId,
|
|
1051
1267
|
executionGeneration: attempt.executionGeneration,
|
|
1268
|
+
...eventing.initialModelWireDispatch.payload({
|
|
1269
|
+
provider: streamProvider,
|
|
1270
|
+
dispatchId,
|
|
1271
|
+
}),
|
|
1272
|
+
// A core-placed turn reports real work for the shared
|
|
1273
|
+
// capacity breaker (providers with their own request events
|
|
1274
|
+
// set it there).
|
|
1275
|
+
...(subscriptionCoreTurnOf(providerTurn)
|
|
1276
|
+
? { meaningfulOutput: sdkResponseHasMeaningfulOutput(next.value) }
|
|
1277
|
+
: {}),
|
|
1052
1278
|
},
|
|
1053
1279
|
},
|
|
1054
1280
|
]);
|
|
1055
1281
|
attempt.providerRecoveryCount = 0;
|
|
1056
1282
|
}
|
|
1283
|
+
if (attempt.providerRecoveryObservation) {
|
|
1284
|
+
recordProviderRecoveryOutcome(observability, {
|
|
1285
|
+
route: attempt.modelMetricRoute,
|
|
1286
|
+
cause: attempt.providerRecoveryObservation.cause,
|
|
1287
|
+
outcome: "recovered",
|
|
1288
|
+
elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt,
|
|
1289
|
+
});
|
|
1290
|
+
attempt.providerRecoveryObservation = undefined;
|
|
1291
|
+
}
|
|
1057
1292
|
const rawStreamHistory = (eventing.stream.state as { history?: unknown[] }).history;
|
|
1058
1293
|
if (Array.isArray(rawStreamHistory)) {
|
|
1059
1294
|
// The completed image item is normally retained from its own
|
|
@@ -1070,35 +1305,10 @@ export async function runTurnStreamAttempt(
|
|
|
1070
1305
|
await historySink.reconcileConversationTruth();
|
|
1071
1306
|
turnLifecycleMetricsFor(observability).progress({ attemptId: input.attemptId });
|
|
1072
1307
|
modelCheckpointMemoryCollector.schedule(observability);
|
|
1073
|
-
try {
|
|
1074
|
-
await ensureRunAllowed(
|
|
1075
|
-
settings,
|
|
1076
|
-
db,
|
|
1077
|
-
input.accountId,
|
|
1078
|
-
input.workspaceId,
|
|
1079
|
-
billingState.isExternallyBilledTurn,
|
|
1080
|
-
entitlements,
|
|
1081
|
-
billingState.chargesOpenGeniCredits,
|
|
1082
|
-
billingState.countsTowardTokenCap,
|
|
1083
|
-
);
|
|
1084
|
-
} catch (limitError) {
|
|
1085
|
-
// Capture the run state at the boundary so the budget valve in
|
|
1086
|
-
// the outer catch can end this segment gracefully with full
|
|
1087
|
-
// conversation context preserved for the post-top-up resume.
|
|
1088
|
-
let serializedRunState: string | null = null;
|
|
1089
|
-
try {
|
|
1090
|
-
serializedRunState = media.compactMediaRunState(
|
|
1091
|
-
String(eventing.stream.state.toString()),
|
|
1092
|
-
);
|
|
1093
|
-
} catch {
|
|
1094
|
-
serializedRunState = null;
|
|
1095
|
-
}
|
|
1096
|
-
throw new BudgetExhaustedError(
|
|
1097
|
-
limitError instanceof Error ? limitError.message : String(limitError),
|
|
1098
|
-
serializedRunState,
|
|
1099
|
-
);
|
|
1100
|
-
}
|
|
1101
1308
|
}
|
|
1309
|
+
// Release after settlement. The producer checks fresh admission before
|
|
1310
|
+
// its next request; a final response keeps the revision it ran under.
|
|
1311
|
+
modelCallAdmission.settle(next.value);
|
|
1102
1312
|
const durableSdkEvent = generatedImageReceipt
|
|
1103
1313
|
? compactGeneratedImageSdkEvent(next.value, generatedImageReceipt)
|
|
1104
1314
|
: next.value;
|
|
@@ -1125,10 +1335,17 @@ export async function runTurnStreamAttempt(
|
|
|
1125
1335
|
currentToolBatchCallIds.add(pendingToolCall.callId);
|
|
1126
1336
|
if (toolCallProducesRetainableSessionImage(pendingToolCall.callName)) {
|
|
1127
1337
|
media.retainedSessionImageCallIds.add(pendingToolCall.callId);
|
|
1338
|
+
if (retainableBrowserScreenshotToolCall(pendingToolCall.callName)) {
|
|
1339
|
+
media.retainedSessionImageKindsByCallId.set(
|
|
1340
|
+
pendingToolCall.callId,
|
|
1341
|
+
"browser_screenshot",
|
|
1342
|
+
);
|
|
1343
|
+
}
|
|
1128
1344
|
}
|
|
1129
1345
|
}
|
|
1130
1346
|
const completedToolCall = completedToolCallFromSdkEvent(durableSdkEvent);
|
|
1131
1347
|
if (completedToolCall) {
|
|
1348
|
+
toolsExecuted = true;
|
|
1132
1349
|
retainedScreenshotMetadata =
|
|
1133
1350
|
media.retainedScreenshotReceiptsByCallId.get(completedToolCall.callId) ?? null;
|
|
1134
1351
|
const typedScreenshot = retainedScreenshotMetadata
|
|
@@ -1147,6 +1364,7 @@ export async function runTurnStreamAttempt(
|
|
|
1147
1364
|
toolCallId: completedToolCall.callId,
|
|
1148
1365
|
toolOutputId: completedToolCall.callId,
|
|
1149
1366
|
reason: "unsupported",
|
|
1367
|
+
kind: media.retainedSessionImageKindsByCallId.get(completedToolCall.callId),
|
|
1150
1368
|
});
|
|
1151
1369
|
media.retainedScreenshotReceiptsByCallId.set(
|
|
1152
1370
|
completedToolCall.callId,
|
|
@@ -1169,12 +1387,13 @@ export async function runTurnStreamAttempt(
|
|
|
1169
1387
|
toolCallId: typedScreenshot.callId,
|
|
1170
1388
|
toolOutputId: typedScreenshot.toolOutputId,
|
|
1171
1389
|
reason: "pending",
|
|
1390
|
+
kind: media.retainedSessionImageKindsByCallId.get(completedToolCall.callId),
|
|
1172
1391
|
});
|
|
1173
1392
|
media.retainedScreenshotReceiptsByCallId.set(
|
|
1174
1393
|
completedToolCall.callId,
|
|
1175
1394
|
retainedScreenshotMetadata,
|
|
1176
1395
|
);
|
|
1177
|
-
retainedScreenshotMetadata = await
|
|
1396
|
+
retainedScreenshotMetadata = await retainSessionScreenshot({
|
|
1178
1397
|
db,
|
|
1179
1398
|
objectStorage,
|
|
1180
1399
|
accountId: input.accountId,
|
|
@@ -1183,6 +1402,7 @@ export async function runTurnStreamAttempt(
|
|
|
1183
1402
|
turnId: activeTurnId,
|
|
1184
1403
|
attemptId: input.attemptId,
|
|
1185
1404
|
output: typedScreenshot,
|
|
1405
|
+
kind: media.retainedSessionImageKindsByCallId.get(completedToolCall.callId),
|
|
1186
1406
|
});
|
|
1187
1407
|
media.retainedScreenshotReceiptsByCallId.set(
|
|
1188
1408
|
completedToolCall.callId,
|
|
@@ -1193,6 +1413,7 @@ export async function runTurnStreamAttempt(
|
|
|
1193
1413
|
durableSdkEvent as typeof next.value,
|
|
1194
1414
|
retainedScreenshotMetadata
|
|
1195
1415
|
? {
|
|
1416
|
+
messagePhases,
|
|
1196
1417
|
toolOutputOverride: retainedScreenshotMetadata,
|
|
1197
1418
|
retainedOutputEvidence: retainedScreenshotMetadata.available
|
|
1198
1419
|
? retainedScreenshotMetadata
|
|
@@ -1201,7 +1422,7 @@ export async function runTurnStreamAttempt(
|
|
|
1201
1422
|
reason: retainedScreenshotMetadata.reason,
|
|
1202
1423
|
},
|
|
1203
1424
|
}
|
|
1204
|
-
: {},
|
|
1425
|
+
: { messagePhases },
|
|
1205
1426
|
);
|
|
1206
1427
|
const normalizedToolOutput = normalizedSdkEvents.find(
|
|
1207
1428
|
(event) =>
|
|
@@ -1249,6 +1470,7 @@ export async function runTurnStreamAttempt(
|
|
|
1249
1470
|
"turn attempt ended while recording a tool-call result",
|
|
1250
1471
|
);
|
|
1251
1472
|
}
|
|
1473
|
+
if (recorded.recorded) newlyDurableToolCallId = completedToolCall.callId;
|
|
1252
1474
|
const videoAcceptance = videoGenerationAcceptancesByCallId.get(completedToolCall.callId);
|
|
1253
1475
|
if (videoAcceptance && startVideoGenerationWorkflow) {
|
|
1254
1476
|
try {
|
|
@@ -1294,6 +1516,7 @@ export async function runTurnStreamAttempt(
|
|
|
1294
1516
|
durableSdkEvent as typeof next.value,
|
|
1295
1517
|
retainedScreenshotMetadata
|
|
1296
1518
|
? {
|
|
1519
|
+
messagePhases,
|
|
1297
1520
|
toolOutputOverride: retainedScreenshotMetadata,
|
|
1298
1521
|
retainedOutputEvidence: retainedScreenshotMetadata.available
|
|
1299
1522
|
? retainedScreenshotMetadata
|
|
@@ -1302,16 +1525,62 @@ export async function runTurnStreamAttempt(
|
|
|
1302
1525
|
reason: retainedScreenshotMetadata.reason,
|
|
1303
1526
|
},
|
|
1304
1527
|
}
|
|
1305
|
-
: {},
|
|
1528
|
+
: { messagePhases },
|
|
1306
1529
|
);
|
|
1307
1530
|
for (const event of normalized) {
|
|
1308
|
-
if (event.type === "agent.toolCall.created")
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1531
|
+
if (event.type === "agent.toolCall.created") {
|
|
1532
|
+
const preparedServers = eventing.preparedTools?.mcpServers ?? [];
|
|
1533
|
+
// Display metadata and the content-free analytics family are
|
|
1534
|
+
// event-copy enrichments only; the executable call is unchanged.
|
|
1535
|
+
event.payload = withToolCallFamily(
|
|
1536
|
+
preparedServers,
|
|
1537
|
+
runSettings.mcpServers,
|
|
1538
|
+
withMcpToolDisplayMetadata(preparedServers, event.payload),
|
|
1312
1539
|
);
|
|
1540
|
+
const created = event.payload as {
|
|
1541
|
+
id?: unknown;
|
|
1542
|
+
name?: unknown;
|
|
1543
|
+
toolName?: unknown;
|
|
1544
|
+
toolFamily?: unknown;
|
|
1545
|
+
};
|
|
1546
|
+
if (typeof created.id === "string") {
|
|
1547
|
+
toolCallMetricStarts.set(created.id, {
|
|
1548
|
+
family: agentToolMetricFamily(
|
|
1549
|
+
typeof created.toolFamily === "string" ? created.toolFamily : null,
|
|
1550
|
+
typeof created.name === "string"
|
|
1551
|
+
? created.name
|
|
1552
|
+
: typeof created.toolName === "string"
|
|
1553
|
+
? created.toolName
|
|
1554
|
+
: null,
|
|
1555
|
+
),
|
|
1556
|
+
startedAtMs: performance.now(),
|
|
1557
|
+
});
|
|
1558
|
+
}
|
|
1559
|
+
}
|
|
1560
|
+
if (event.type === "agent.toolCall.output") {
|
|
1561
|
+
const id = (event.payload as { id?: unknown }).id;
|
|
1562
|
+
const started = typeof id === "string" ? toolCallMetricStarts.get(id) : undefined;
|
|
1563
|
+
if (typeof id === "string") toolCallMetricStarts.delete(id);
|
|
1564
|
+
recordAgentToolCall(observability, {
|
|
1565
|
+
family: started?.family ?? "other",
|
|
1566
|
+
outcome: agentToolCallOutcome(event.payload),
|
|
1567
|
+
...(started ? { durationMs: performance.now() - started.startedAtMs } : {}),
|
|
1568
|
+
});
|
|
1569
|
+
}
|
|
1313
1570
|
streamTiming.onEvent(event.type);
|
|
1314
|
-
|
|
1571
|
+
if (event.type === "agent.toolCall.output") {
|
|
1572
|
+
const batcher = eventing.batcher;
|
|
1573
|
+
await withMcpCallIdentity((event.payload as { id: string }).id, () =>
|
|
1574
|
+
measureMcpPhase("event_persistence", () => batcher.push(event)),
|
|
1575
|
+
);
|
|
1576
|
+
} else {
|
|
1577
|
+
await eventing.batcher.push(event);
|
|
1578
|
+
}
|
|
1579
|
+
if (event.type === "agent.message.completed") {
|
|
1580
|
+
// Completed messages are structural: push returns once durable.
|
|
1581
|
+
latestStreamedAssistantText = (event.payload as { text: string }).text;
|
|
1582
|
+
latestAssistantMessageText = latestStreamedAssistantText;
|
|
1583
|
+
}
|
|
1315
1584
|
}
|
|
1316
1585
|
// Structural tool-output events await their durable append before
|
|
1317
1586
|
// push returns. The complete result is now retained in the event
|
|
@@ -1339,10 +1608,20 @@ export async function runTurnStreamAttempt(
|
|
|
1339
1608
|
}
|
|
1340
1609
|
for (const callId of stableToolCallIdsToClear) {
|
|
1341
1610
|
media.retainedSessionImageCallIds.delete(callId);
|
|
1611
|
+
media.retainedSessionImageKindsByCallId.delete(callId);
|
|
1342
1612
|
}
|
|
1343
1613
|
}
|
|
1614
|
+
if (newlyDurableToolCallId) {
|
|
1615
|
+
// The exact call/result receipt and structural output event are now
|
|
1616
|
+
// durable even when parallel SDK history is still non-monotonic.
|
|
1617
|
+
// Release only this result owner's completed foreground ops. A
|
|
1618
|
+
// missing/duplicate receipt cannot license new output collection;
|
|
1619
|
+
// the complete-turn hook remains its later durability boundary.
|
|
1620
|
+
await finalizeTurnOpStreamOps([newlyDurableToolCallId]);
|
|
1621
|
+
}
|
|
1344
1622
|
}
|
|
1345
1623
|
} catch (error) {
|
|
1624
|
+
modelCallAdmission.fail(error);
|
|
1346
1625
|
// Event processing can fail while SDK completion is still pending.
|
|
1347
1626
|
// Close this stream before any failure publication; a legitimate
|
|
1348
1627
|
// compaction retry may start a new, independently fenced generation.
|
|
@@ -1365,13 +1644,21 @@ export async function runTurnStreamAttempt(
|
|
|
1365
1644
|
attemptId: input.attemptId,
|
|
1366
1645
|
dispatchId,
|
|
1367
1646
|
executionGeneration: attempt.executionGeneration,
|
|
1647
|
+
...eventing.initialModelWireDispatch.payload({
|
|
1648
|
+
provider: streamProvider,
|
|
1649
|
+
dispatchId,
|
|
1650
|
+
}),
|
|
1368
1651
|
},
|
|
1369
1652
|
},
|
|
1370
1653
|
]);
|
|
1371
1654
|
}
|
|
1372
1655
|
if (
|
|
1373
1656
|
!eventing.firstModelRequestPreparationRecorded &&
|
|
1374
|
-
eventing.firstModelRequestPreparationStartedAt !== null
|
|
1657
|
+
eventing.firstModelRequestPreparationStartedAt !== null &&
|
|
1658
|
+
// Compaction needed before the first request is an expected step the
|
|
1659
|
+
// in-activity recovery below settles and then reopens; it is not a
|
|
1660
|
+
// preparation failure.
|
|
1661
|
+
!(findCompactionNeededError(error) && eventing.turnStartedPublished)
|
|
1375
1662
|
) {
|
|
1376
1663
|
eventing.firstModelRequestPreparationRecorded = true;
|
|
1377
1664
|
const durationMs = Math.max(
|
|
@@ -1389,15 +1676,13 @@ export async function runTurnStreamAttempt(
|
|
|
1389
1676
|
await eventing.publish!([
|
|
1390
1677
|
{
|
|
1391
1678
|
type: "turn.startup.phase.failed",
|
|
1392
|
-
payload:
|
|
1393
|
-
phase: "model_preparation",
|
|
1394
|
-
durationMs: Math.round(durationMs),
|
|
1395
|
-
},
|
|
1679
|
+
payload: modelPreparationFailureEventPayload(error, durationMs),
|
|
1396
1680
|
},
|
|
1397
1681
|
]);
|
|
1398
1682
|
}
|
|
1399
1683
|
throw error;
|
|
1400
1684
|
} finally {
|
|
1685
|
+
modelCallAdmission.close();
|
|
1401
1686
|
if (!streamDone) {
|
|
1402
1687
|
// ReadableStream cancellation synchronously trips the Agents SDK's
|
|
1403
1688
|
// abort controller, but its returned promise may wait for an
|
|
@@ -1419,7 +1704,10 @@ export async function runTurnStreamAttempt(
|
|
|
1419
1704
|
// External Codemode stays reachable until finalization. Close wait
|
|
1420
1705
|
// admission before any terminal output/history decision, and drain an
|
|
1421
1706
|
// already-admitted wait before consulting the actual runner-yield latch.
|
|
1422
|
-
|
|
1707
|
+
// Drain trusted receipts before deciding whether an empty stream may need
|
|
1708
|
+
// a handoff. Do not terminally seal the attempt until that decision: the
|
|
1709
|
+
// one same-turn stream still uses the ordinary input-wait admission gate.
|
|
1710
|
+
await eventing.preparedTools?.inputWaitYield?.drainForHandoff(runtimeCancellationSignal);
|
|
1423
1711
|
if (
|
|
1424
1712
|
options.requireTerminalModelResponse &&
|
|
1425
1713
|
!eventing.preparedTools?.inputWaitYield?.yielded &&
|
|
@@ -1441,6 +1729,12 @@ export async function runTurnStreamAttempt(
|
|
|
1441
1729
|
if (!streamSawPerResponseUsage) {
|
|
1442
1730
|
const aggregateUsage = eventing.stream.state.usage;
|
|
1443
1731
|
const normalizedAggregateUsage = normalizeModelCallUsage(aggregateUsage);
|
|
1732
|
+
const aggregatePolicyRevision = aggregateCreditPolicyRevision({
|
|
1733
|
+
responseRevisions: responseCreditPolicyRevisions,
|
|
1734
|
+
lastAdmittedRevision: creditPolicyRevision,
|
|
1735
|
+
chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
|
|
1736
|
+
totalTokens: normalizedAggregateUsage.totalTokens,
|
|
1737
|
+
});
|
|
1444
1738
|
const aggregateInput = normalizedAggregateUsage.telemetry.inputTokens;
|
|
1445
1739
|
const aggregateSourceKey = modelUsageSourceKey({
|
|
1446
1740
|
responseId: null,
|
|
@@ -1463,12 +1757,13 @@ export async function runTurnStreamAttempt(
|
|
|
1463
1757
|
leaseLostMessage: "Provider credential lease expired during the active turn",
|
|
1464
1758
|
recordUsage: async () => {
|
|
1465
1759
|
const billing = await recordModelUsageAndDebitCredits(settings, db, {
|
|
1760
|
+
creditPolicyRevision: aggregatePolicyRevision,
|
|
1466
1761
|
accountId: input.accountId,
|
|
1467
1762
|
workspaceId: input.workspaceId,
|
|
1468
1763
|
sessionId: input.sessionId,
|
|
1469
1764
|
turnId: activeTurnId,
|
|
1470
1765
|
turnAttemptId: input.attemptId,
|
|
1471
|
-
model:
|
|
1766
|
+
model: turnExecutionPolicy.productModelId,
|
|
1472
1767
|
externallyBilled: billingState.isExternallyBilledTurn,
|
|
1473
1768
|
chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
|
|
1474
1769
|
countsTowardTokenCap: billingState.countsTowardTokenCap,
|
|
@@ -1477,6 +1772,7 @@ export async function runTurnStreamAttempt(
|
|
|
1477
1772
|
sourceKey: aggregateSourceKey,
|
|
1478
1773
|
latencyMode: turnExecutionPolicy.latencyMode,
|
|
1479
1774
|
observability,
|
|
1775
|
+
metricProvider: resolvedModel?.provider.id ?? settings.openaiProvider,
|
|
1480
1776
|
});
|
|
1481
1777
|
const aggregateProvider = resolvedModel?.provider.id ?? settings.openaiProvider;
|
|
1482
1778
|
const aggregateProviderApi = resolvedModel?.provider.api ?? "responses";
|
|
@@ -1489,7 +1785,8 @@ export async function runTurnStreamAttempt(
|
|
|
1489
1785
|
turnId: activeTurnId,
|
|
1490
1786
|
provider: aggregateProvider,
|
|
1491
1787
|
providerApi: aggregateProviderApi,
|
|
1492
|
-
model:
|
|
1788
|
+
model: turnExecutionPolicy.productModelId,
|
|
1789
|
+
purpose: "main",
|
|
1493
1790
|
sourceKey: aggregateSourceKey,
|
|
1494
1791
|
usage: { usage: aggregateUsage },
|
|
1495
1792
|
normalizedUsage: normalizedAggregateUsage,
|
|
@@ -1511,10 +1808,20 @@ export async function runTurnStreamAttempt(
|
|
|
1511
1808
|
sourceKey: aggregateSourceKey,
|
|
1512
1809
|
provider: aggregateProvider,
|
|
1513
1810
|
providerApi: aggregateProviderApi,
|
|
1514
|
-
model:
|
|
1811
|
+
model: turnExecutionPolicy.productModelId,
|
|
1515
1812
|
billing,
|
|
1813
|
+
subscriptionConnectionId:
|
|
1814
|
+
subscriptionCoreTurnOf(providerTurn)?.connectionId ?? null,
|
|
1516
1815
|
contextContributions: eventing.companyBrainContextContributions,
|
|
1517
1816
|
});
|
|
1817
|
+
recordAuthoritativeModelUsageMetrics({
|
|
1818
|
+
observability,
|
|
1819
|
+
settings,
|
|
1820
|
+
provider: aggregateProvider,
|
|
1821
|
+
model: turnExecutionPolicy.productModelId,
|
|
1822
|
+
externallyBilled: billingState.isExternallyBilledTurn,
|
|
1823
|
+
billing,
|
|
1824
|
+
});
|
|
1518
1825
|
}
|
|
1519
1826
|
if (aggregateAuthoritative && aggregateInput !== null && aggregateInput > 0) {
|
|
1520
1827
|
recordModelInputTokens(observability, streamProvider, aggregateInput);
|
|
@@ -1531,9 +1838,21 @@ export async function runTurnStreamAttempt(
|
|
|
1531
1838
|
});
|
|
1532
1839
|
}
|
|
1533
1840
|
}
|
|
1534
|
-
|
|
1841
|
+
const programmaticPending = (
|
|
1842
|
+
await listTurnCodemodeApprovals(db, {
|
|
1843
|
+
accountId: input.accountId,
|
|
1844
|
+
workspaceId: input.workspaceId,
|
|
1845
|
+
sessionId: input.sessionId,
|
|
1846
|
+
turnId: activeTurnId,
|
|
1847
|
+
})
|
|
1848
|
+
)
|
|
1849
|
+
.filter((operation) => operation.state === "waiting_for_approval")
|
|
1850
|
+
.map(programmaticApproval);
|
|
1851
|
+
if (eventing.stream.interruptions.length > 0 || programmaticPending.length > 0) {
|
|
1852
|
+
await eventing.preparedTools?.inputWaitYield?.sealForSettlement(runtimeCancellationSignal);
|
|
1535
1853
|
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
1536
|
-
const
|
|
1854
|
+
const sdkApprovals = runtime.serializeApprovals(eventing.stream.interruptions);
|
|
1855
|
+
const approvals = [...sdkApprovals, ...programmaticPending];
|
|
1537
1856
|
const humanInputInterruptions =
|
|
1538
1857
|
runtime.serializeHumanInputRequests?.(eventing.stream.interruptions) ?? [];
|
|
1539
1858
|
const interactionInterventionInterruptions =
|
|
@@ -1613,32 +1932,43 @@ export async function runTurnStreamAttempt(
|
|
|
1613
1932
|
const interruptionCallIds = interruptionCallIdsFromPause({
|
|
1614
1933
|
humanInputRequests,
|
|
1615
1934
|
interactionInterventionRequests,
|
|
1616
|
-
pendingApprovals
|
|
1935
|
+
pendingApprovals: [
|
|
1936
|
+
...sdkApprovals,
|
|
1937
|
+
...interactionInterventionInterruptions.map((interruption) => interruption.approval),
|
|
1938
|
+
],
|
|
1617
1939
|
});
|
|
1618
1940
|
assertOpenSuffixResumable(suffixMembers, interruptionCallIds);
|
|
1619
1941
|
const suffixByCallId = new Map(suffixMembers.map((member) => [member.callId, member]));
|
|
1620
|
-
const attached =
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1942
|
+
const attached =
|
|
1943
|
+
interruptionCallIds.length === 0
|
|
1944
|
+
? { accepted: true }
|
|
1945
|
+
: await attachOpenSuffixToPendingToolCalls(db, {
|
|
1946
|
+
accountId: input.accountId,
|
|
1947
|
+
workspaceId: input.workspaceId,
|
|
1948
|
+
sessionId: input.sessionId,
|
|
1949
|
+
turnId: activeTurnId,
|
|
1950
|
+
executionGeneration: attempt.executionGeneration,
|
|
1951
|
+
attemptId: input.attemptId,
|
|
1952
|
+
members: interruptionCallIds.map((callId) => {
|
|
1953
|
+
const member = suffixByCallId.get(callId)!;
|
|
1954
|
+
return {
|
|
1955
|
+
callId,
|
|
1956
|
+
interruptionKind: interruptionKindForCallItem(
|
|
1957
|
+
member.callItem as Record<string, unknown>,
|
|
1958
|
+
),
|
|
1959
|
+
reasoningItems: member.reasoningItems as Array<Record<string, unknown>>,
|
|
1960
|
+
};
|
|
1961
|
+
}),
|
|
1962
|
+
});
|
|
1638
1963
|
if (!attached.accepted) {
|
|
1639
1964
|
return claimedResult({ status: "cancelled" });
|
|
1640
1965
|
}
|
|
1966
|
+
// History plus the exact approval suffix now own the complete response.
|
|
1967
|
+
await subscriptionCoreTurnOf(providerTurn)?.requests?.checkpoint();
|
|
1641
1968
|
await finishParallelSessionTitle();
|
|
1969
|
+
// Held task-usage receipts are durable before the attempt waits on a
|
|
1970
|
+
// person (the resume may run in another worker process).
|
|
1971
|
+
await flushHeldSessionUsageReceipts(db, input.sessionId);
|
|
1642
1972
|
if (
|
|
1643
1973
|
!(await eventing.settle!({
|
|
1644
1974
|
events: [
|
|
@@ -1649,9 +1979,11 @@ export async function runTurnStreamAttempt(
|
|
|
1649
1979
|
type: "session.requiresAction" as const,
|
|
1650
1980
|
payload: {
|
|
1651
1981
|
approvals: approvals.map((approval) =>
|
|
1652
|
-
|
|
1653
|
-
|
|
1654
|
-
|
|
1982
|
+
withPublicApprovalFields(
|
|
1983
|
+
withMcpToolDisplayMetadata(
|
|
1984
|
+
eventing.preparedTools?.mcpServers ?? [],
|
|
1985
|
+
approval,
|
|
1986
|
+
),
|
|
1655
1987
|
),
|
|
1656
1988
|
),
|
|
1657
1989
|
},
|
|
@@ -1676,6 +2008,13 @@ export async function runTurnStreamAttempt(
|
|
|
1676
2008
|
) {
|
|
1677
2009
|
return claimedResult({ status: "cancelled" });
|
|
1678
2010
|
}
|
|
2011
|
+
// Agent-requested human handoffs open here, in the worker, never through
|
|
2012
|
+
// the API intervention route, so they are counted at their durable boundary.
|
|
2013
|
+
const observeIntervention = interactionInterventionMetricObserver(observability);
|
|
2014
|
+
for (const interruption of interactionInterventionInterruptions) {
|
|
2015
|
+
if (interruption.input.operation !== "request") continue;
|
|
2016
|
+
observeIntervention({ kind: interruption.input.kind, outcome: "opened" });
|
|
2017
|
+
}
|
|
1679
2018
|
// The interruption and its preceding tool results are now durable.
|
|
1680
2019
|
await finalizeTurnOpStreamOps();
|
|
1681
2020
|
control.activityStatus = "requires_action";
|
|
@@ -1686,6 +2025,78 @@ export async function runTurnStreamAttempt(
|
|
|
1686
2025
|
const finalOutput = String(
|
|
1687
2026
|
requireAgentStreamFinalOutput(eventing.stream.finalOutput, inputWaitYielded),
|
|
1688
2027
|
);
|
|
2028
|
+
const durableReplyFacts =
|
|
2029
|
+
!inputWaitYielded && finalOutput.trim().length === 0
|
|
2030
|
+
? await sessionTurnFinalReplyFacts(db, input.workspaceId, input.sessionId, activeTurnId)
|
|
2031
|
+
: { toolsExecuted: false, completedGoal: false };
|
|
2032
|
+
let emptyFinalReply = false;
|
|
2033
|
+
if (
|
|
2034
|
+
needsFinalReply({
|
|
2035
|
+
output: finalOutput,
|
|
2036
|
+
inputWaitYielded:
|
|
2037
|
+
inputWaitYielded || eventing.preparedTools?.inputWaitYield?.requested === true,
|
|
2038
|
+
interrupted: false, // interruption settlement returned above
|
|
2039
|
+
maintenance: turn.source === "compaction",
|
|
2040
|
+
toolsExecuted: toolsExecuted || finalReplyNudged || durableReplyFacts.toolsExecuted,
|
|
2041
|
+
completedGoal: durableReplyFacts.completedGoal,
|
|
2042
|
+
})
|
|
2043
|
+
) {
|
|
2044
|
+
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
2045
|
+
await subscriptionCoreTurnOf(providerTurn)?.requests?.checkpoint();
|
|
2046
|
+
// Consult retained truth, including inactive compacted rows, so an
|
|
2047
|
+
// attempt replacement or compaction cannot spend this bound again.
|
|
2048
|
+
if (
|
|
2049
|
+
finalReplyNudged ||
|
|
2050
|
+
(await sessionTurnHasFinalReplyNudge(
|
|
2051
|
+
db,
|
|
2052
|
+
input.workspaceId,
|
|
2053
|
+
input.sessionId,
|
|
2054
|
+
activeTurnId,
|
|
2055
|
+
finalReplyNudge(activeTurnId).content[0]!.text,
|
|
2056
|
+
))
|
|
2057
|
+
) {
|
|
2058
|
+
// A second empty response is a delivery-quality notice, not failed
|
|
2059
|
+
// execution: preserve goal continuation and later machine-input wakes.
|
|
2060
|
+
emptyFinalReply = true;
|
|
2061
|
+
} else {
|
|
2062
|
+
const appended = await appendSessionHistoryItems(db, {
|
|
2063
|
+
accountId: input.accountId,
|
|
2064
|
+
workspaceId: input.workspaceId,
|
|
2065
|
+
sessionId: input.sessionId,
|
|
2066
|
+
turnId: activeTurnId,
|
|
2067
|
+
expectedExecutionGeneration: attempt.executionGeneration,
|
|
2068
|
+
expectedAttemptId: input.attemptId,
|
|
2069
|
+
items: [
|
|
2070
|
+
{
|
|
2071
|
+
position: await nextSessionHistoryPosition(db, input.workspaceId, input.sessionId),
|
|
2072
|
+
item: finalReplyNudge(activeTurnId),
|
|
2073
|
+
},
|
|
2074
|
+
],
|
|
2075
|
+
});
|
|
2076
|
+
if (!appended) throw new TurnAttemptFencedError("turn ended before final reply handoff");
|
|
2077
|
+
finalReplyNudged = true;
|
|
2078
|
+
await prepareRunAttemptInput();
|
|
2079
|
+
return "reply_nudge";
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
await eventing.preparedTools?.inputWaitYield?.sealForSettlement(runtimeCancellationSignal);
|
|
2083
|
+
// The final output is the newest message this stream completed, already
|
|
2084
|
+
// durable with its provider identity and phase. A phase-less settlement
|
|
2085
|
+
// copy is published only when this stream did not complete that text.
|
|
2086
|
+
const finalOutputAlreadyCompleted = latestStreamedAssistantText === finalOutput;
|
|
2087
|
+
// A wait ends the turn with empty output; a human's message still gets
|
|
2088
|
+
// its answer recorded for unread attention and Slack.
|
|
2089
|
+
const reply = await inputWaitReply({
|
|
2090
|
+
inputWaitYielded,
|
|
2091
|
+
turn,
|
|
2092
|
+
latestAssistantMessageText,
|
|
2093
|
+
readLatestDurableTurnMessage: async () =>
|
|
2094
|
+
await latestDurableTurnMessageText(db, {
|
|
2095
|
+
workspaceId: input.workspaceId,
|
|
2096
|
+
sessionId: input.sessionId,
|
|
2097
|
+
turnId: turn.id,
|
|
2098
|
+
}),
|
|
2099
|
+
});
|
|
1689
2100
|
await historySink.reconcileConversationTruth({ requireDurable: true });
|
|
1690
2101
|
// Op-stream durability fence: the tool outputs are now durably in the
|
|
1691
2102
|
// history store (a redispatch would NOT re-execute them), so this
|
|
@@ -1693,15 +2104,26 @@ export async function runTurnStreamAttempt(
|
|
|
1693
2104
|
// then wire final ack (licensing the runner to GC its retained
|
|
1694
2105
|
// frames). Best-effort: a miss leaves the runner's retention TTL to
|
|
1695
2106
|
// reap, never fails a completed turn.
|
|
2107
|
+
await subscriptionCoreTurnOf(providerTurn)?.requests?.checkpoint();
|
|
1696
2108
|
await finalizeTurnOpStreamOps();
|
|
1697
2109
|
await finishParallelSessionTitle();
|
|
2110
|
+
// Every task-usage receipt held after a failed write is durable before
|
|
2111
|
+
// the turn completes (a persistent failure enters DB-only recovery).
|
|
2112
|
+
await flushHeldSessionUsageReceipts(db, input.sessionId);
|
|
1698
2113
|
if (
|
|
1699
2114
|
!(await eventing.settle!({
|
|
1700
2115
|
events: [
|
|
1701
|
-
...(inputWaitYielded
|
|
2116
|
+
...(inputWaitYielded || finalOutputAlreadyCompleted
|
|
1702
2117
|
? []
|
|
1703
2118
|
: [{ type: "agent.message.completed" as const, payload: { text: finalOutput } }]),
|
|
1704
|
-
{
|
|
2119
|
+
{
|
|
2120
|
+
type: "turn.completed",
|
|
2121
|
+
payload: {
|
|
2122
|
+
output: finalOutput,
|
|
2123
|
+
...(reply === null ? {} : { reply }),
|
|
2124
|
+
...(emptyFinalReply ? { emptyFinalReply: true } : {}),
|
|
2125
|
+
},
|
|
2126
|
+
},
|
|
1705
2127
|
{ type: "session.status.changed", payload: { status: "idle" } },
|
|
1706
2128
|
],
|
|
1707
2129
|
turnStatus: "completed",
|
|
@@ -1729,7 +2151,41 @@ export async function runTurnStreamAttempt(
|
|
|
1729
2151
|
return claimedResult({ status: "idle" });
|
|
1730
2152
|
};
|
|
1731
2153
|
|
|
2154
|
+
// Recover committed waits/results before any model dispatch, including crash replacement.
|
|
2155
|
+
// A new attempt uses its own signed caller; origin provenance never changes.
|
|
2156
|
+
// Join-on-use: lazy MCP preparation is awaited only when stored work must resume.
|
|
2157
|
+
const codemodeOperations = await CodemodeAttemptDispatcher.resumeApproved(
|
|
2158
|
+
db,
|
|
2159
|
+
{
|
|
2160
|
+
accountId: input.accountId,
|
|
2161
|
+
workspaceId: input.workspaceId,
|
|
2162
|
+
sessionId: input.sessionId,
|
|
2163
|
+
turnId: activeTurnId,
|
|
2164
|
+
attemptId: input.attemptId,
|
|
2165
|
+
executionGeneration: attempt.executionGeneration,
|
|
2166
|
+
},
|
|
2167
|
+
`sandbox:${input.attemptId}`,
|
|
2168
|
+
trigger.type === "user.approvalDecision"
|
|
2169
|
+
? (trigger.payload as { approvalId: string }).approvalId
|
|
2170
|
+
: undefined,
|
|
2171
|
+
async () => {
|
|
2172
|
+
await eventing.toolPreparationReady;
|
|
2173
|
+
return eventing.codemodeDispatcher;
|
|
2174
|
+
},
|
|
2175
|
+
);
|
|
2176
|
+
codemodeContinuationNote = programmaticContinuationNote(codemodeOperations);
|
|
2177
|
+
programmaticApprovalAcknowledged =
|
|
2178
|
+
trigger.type === "user.approvalDecision" &&
|
|
2179
|
+
codemodeOperations.some(
|
|
2180
|
+
(operation) =>
|
|
2181
|
+
operation.operationId === (trigger.payload as { approvalId?: unknown }).approvalId,
|
|
2182
|
+
);
|
|
2183
|
+
const pendingProgrammaticApprovals = codemodeOperations
|
|
2184
|
+
.filter((operation) => operation.state === "waiting_for_approval")
|
|
2185
|
+
.map(programmaticApproval);
|
|
1732
2186
|
const openSuffixResume = await settleOpenSuffixResumeIfNeeded({
|
|
2187
|
+
additionalApprovals: pendingProgrammaticApprovals,
|
|
2188
|
+
programmaticApprovalAcknowledged,
|
|
1733
2189
|
db,
|
|
1734
2190
|
agent,
|
|
1735
2191
|
accountId: input.accountId,
|
|
@@ -1761,6 +2217,9 @@ export async function runTurnStreamAttempt(
|
|
|
1761
2217
|
) {
|
|
1762
2218
|
return claimedResult({ status: "cancelled" });
|
|
1763
2219
|
}
|
|
2220
|
+
// Preparation may have spent the last allowance on a completed summary.
|
|
2221
|
+
// Neither the title sidecar nor ordinary inference may dispatch afterward.
|
|
2222
|
+
await revalidateModelCallAdmission();
|
|
1764
2223
|
if (
|
|
1765
2224
|
turn.source !== "compaction" &&
|
|
1766
2225
|
generateSessionTitleInParallel &&
|
|
@@ -1771,22 +2230,28 @@ export async function runTurnStreamAttempt(
|
|
|
1771
2230
|
turnExecutionPolicy.providerId,
|
|
1772
2231
|
turnExecutionPolicy.latencyMode,
|
|
1773
2232
|
);
|
|
2233
|
+
titleCreditPolicyRevision = creditPolicyRevision;
|
|
2234
|
+
const titleRoute = sessionTitleRoute({
|
|
2235
|
+
resolvedModel,
|
|
2236
|
+
modelName: turnExecutionPolicy.upstreamModelId,
|
|
2237
|
+
resolveTurnModel: (modelId) => runtime.resolveTurnModel(runSettings, modelId),
|
|
2238
|
+
});
|
|
2239
|
+
titleUsageModelId = titleRoute.usageModelId;
|
|
1774
2240
|
parallelSessionTitle = startParallelSessionTitleGeneration({
|
|
1775
2241
|
signal: runtimeCancellationSignal,
|
|
2242
|
+
onFinish: persistParallelSessionTitle,
|
|
1776
2243
|
generate: async (signal) =>
|
|
1777
2244
|
await withSessionTitleProviderRequestContext(() =>
|
|
1778
|
-
runtime.generateSessionTitle!(
|
|
1779
|
-
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
|
|
1788
|
-
signal,
|
|
1789
|
-
}),
|
|
2245
|
+
runtime.generateSessionTitle!(
|
|
2246
|
+
runSettings,
|
|
2247
|
+
sessionTitlePrompt,
|
|
2248
|
+
sessionTitleGenerationOptions({
|
|
2249
|
+
resolvedModel: titleRoute.resolvedModel,
|
|
2250
|
+
modelName: titleRoute.modelName,
|
|
2251
|
+
serviceTier,
|
|
2252
|
+
signal,
|
|
2253
|
+
}),
|
|
2254
|
+
),
|
|
1790
2255
|
),
|
|
1791
2256
|
onError: (error) => {
|
|
1792
2257
|
observability.warn("parallel session title generation failed", {
|
|
@@ -1800,11 +2265,19 @@ export async function runTurnStreamAttempt(
|
|
|
1800
2265
|
}
|
|
1801
2266
|
try {
|
|
1802
2267
|
let retriedAfterCompaction = false;
|
|
2268
|
+
finalReplyNudged = await sessionTurnHasFinalReplyNudge(
|
|
2269
|
+
db,
|
|
2270
|
+
input.workspaceId,
|
|
2271
|
+
input.sessionId,
|
|
2272
|
+
activeTurnId,
|
|
2273
|
+
finalReplyNudge(activeTurnId).content[0]!.text,
|
|
2274
|
+
);
|
|
1803
2275
|
while (true) {
|
|
1804
2276
|
try {
|
|
1805
2277
|
const result = await runStreamAttempt({
|
|
1806
2278
|
requireTerminalModelResponse: retriedAfterCompaction,
|
|
1807
2279
|
});
|
|
2280
|
+
if (result === "reply_nudge") continue;
|
|
1808
2281
|
if (retriedAfterCompaction) {
|
|
1809
2282
|
observability.info("context compaction recovery succeeded after in-activity retry", {
|
|
1810
2283
|
sessionId: input.sessionId,
|
|
@@ -1813,13 +2286,14 @@ export async function runTurnStreamAttempt(
|
|
|
1813
2286
|
}
|
|
1814
2287
|
return result;
|
|
1815
2288
|
} catch (attemptError) {
|
|
2289
|
+
const requestSizeError = findAnthropicRequestSizeError(attemptError);
|
|
1816
2290
|
const overflow = classifyContextWindowOverflowError(attemptError);
|
|
1817
2291
|
const compactionNeeded = findCompactionNeededError(attemptError);
|
|
1818
2292
|
const recoveryKind = compactionNeeded
|
|
1819
2293
|
? compactionNeeded.trigger === "operator"
|
|
1820
2294
|
? "operator"
|
|
1821
2295
|
: "proactive"
|
|
1822
|
-
: overflow
|
|
2296
|
+
: overflow || requestSizeError
|
|
1823
2297
|
? "overflow"
|
|
1824
2298
|
: null;
|
|
1825
2299
|
if (!recoveryKind || !eventing.publish || !eventing.turnStartedPublished) {
|
|
@@ -1834,7 +2308,26 @@ export async function runTurnStreamAttempt(
|
|
|
1834
2308
|
...safeErrorDiagnostic(attemptError),
|
|
1835
2309
|
signalTokens: compactionNeeded?.signalTokens,
|
|
1836
2310
|
thresholdTokens: compactionNeeded?.thresholdTokens,
|
|
2311
|
+
...(requestSizeError ? { ...requestSizeError.requestSize } : {}),
|
|
1837
2312
|
});
|
|
2313
|
+
// When compaction is needed before the turn's first model request,
|
|
2314
|
+
// preparation is still open. Settle it as an expected transition and
|
|
2315
|
+
// reopen it for the retried request, whose real dispatch then records
|
|
2316
|
+
// the preparation landmark and its startup sample.
|
|
2317
|
+
if (
|
|
2318
|
+
!eventing.firstModelRequestPreparationRecorded &&
|
|
2319
|
+
eventing.firstModelRequestPreparationStartedAt !== null
|
|
2320
|
+
) {
|
|
2321
|
+
const durationMs = performance.now() - eventing.firstModelRequestPreparationStartedAt;
|
|
2322
|
+
eventing.firstModelRequestPreparationStartedAt = null;
|
|
2323
|
+
eventing.firstModelRequestCheckpointAt = null;
|
|
2324
|
+
await eventing.publish([
|
|
2325
|
+
{
|
|
2326
|
+
type: "turn.startup.phase.failed",
|
|
2327
|
+
payload: compactionFirstPreparationEventPayload(durationMs),
|
|
2328
|
+
},
|
|
2329
|
+
]);
|
|
2330
|
+
}
|
|
1838
2331
|
let compacted = false;
|
|
1839
2332
|
let compactionHandled = false;
|
|
1840
2333
|
let compactionFailureMessage: string | null = null;
|
|
@@ -1844,6 +2337,7 @@ export async function runTurnStreamAttempt(
|
|
|
1844
2337
|
const outcome = await forceContextCompaction(
|
|
1845
2338
|
recoveryKind,
|
|
1846
2339
|
compactionNeeded?.signalTokens ?? null,
|
|
2340
|
+
requestSizeError,
|
|
1847
2341
|
);
|
|
1848
2342
|
compacted = outcome.compacted;
|
|
1849
2343
|
if (outcome.compacted) {
|
|
@@ -1858,8 +2352,31 @@ export async function runTurnStreamAttempt(
|
|
|
1858
2352
|
// Transient checkpoint-provider failures recover this same accepted
|
|
1859
2353
|
// turn through the normal provider/capacity path. They are not an
|
|
1860
2354
|
// empty summary and must not create a new goal continuation.
|
|
1861
|
-
if (
|
|
2355
|
+
if (
|
|
2356
|
+
shouldRecoverCompactionProviderFailure(compactError) ||
|
|
2357
|
+
isCoreSourceDisconnectedError(compactError)
|
|
2358
|
+
)
|
|
2359
|
+
throw compactError;
|
|
1862
2360
|
if (compactError instanceof TurnAttemptFencedError) throw compactError;
|
|
2361
|
+
if (isContextCompactionInterruption(compactError, cancellationSignal)) {
|
|
2362
|
+
// Graceful worker shutdown (or a control's activity cancellation)
|
|
2363
|
+
// aborted the in-flight checkpoint request. That is not a
|
|
2364
|
+
// summarization failure: record it truthfully, keep any operator
|
|
2365
|
+
// request pending, and let the same-turn recovery path run.
|
|
2366
|
+
await settleContextCompactionInterruption(
|
|
2367
|
+
db,
|
|
2368
|
+
{
|
|
2369
|
+
accountId: input.accountId,
|
|
2370
|
+
workspaceId: input.workspaceId,
|
|
2371
|
+
sessionId: input.sessionId,
|
|
2372
|
+
turnId: activeTurnId!,
|
|
2373
|
+
executionGeneration: attempt.executionGeneration,
|
|
2374
|
+
attemptId: input.attemptId,
|
|
2375
|
+
},
|
|
2376
|
+
{ publishLiveEvents: publishCompactionLiveEvents, observability },
|
|
2377
|
+
);
|
|
2378
|
+
throw compactError;
|
|
2379
|
+
}
|
|
1863
2380
|
const landmark = await settleFailedContextCompactionLandmark(
|
|
1864
2381
|
db,
|
|
1865
2382
|
{
|
|
@@ -1935,6 +2452,7 @@ export async function runTurnStreamAttempt(
|
|
|
1935
2452
|
const deferredSteer = await settleDeferredSteerAfterCompaction();
|
|
1936
2453
|
if (deferredSteer) return deferredSteer;
|
|
1937
2454
|
if (turn.source === "compaction") {
|
|
2455
|
+
await flushHeldSessionUsageReceipts(db, input.sessionId);
|
|
1938
2456
|
const settled = await eventing.settle!({
|
|
1939
2457
|
events: [
|
|
1940
2458
|
{
|
|
@@ -1969,6 +2487,11 @@ export async function runTurnStreamAttempt(
|
|
|
1969
2487
|
await prepareRunAttemptInput();
|
|
1970
2488
|
}
|
|
1971
2489
|
}
|
|
2490
|
+
} catch (error) {
|
|
2491
|
+
// A lifecycle disconnect fences the next request, not the already admitted
|
|
2492
|
+
// title stream. Join and persist its bounded response before recovery.
|
|
2493
|
+
if (isCoreSourceDisconnectedError(error)) await finishParallelSessionTitle();
|
|
2494
|
+
throw error;
|
|
1972
2495
|
} finally {
|
|
1973
2496
|
await cancelParallelSessionTitle();
|
|
1974
2497
|
}
|