@caupulican/pi-adaptative 0.86.3 → 0.86.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/dist/core/agent-paths.d.ts +13 -0
- package/dist/core/agent-paths.d.ts.map +1 -1
- package/dist/core/agent-paths.js +23 -0
- package/dist/core/agent-paths.js.map +1 -1
- package/dist/core/autonomy/bounded-completion.d.ts +7 -0
- package/dist/core/autonomy/bounded-completion.d.ts.map +1 -1
- package/dist/core/autonomy/bounded-completion.js +12 -4
- package/dist/core/autonomy/bounded-completion.js.map +1 -1
- package/dist/core/autonomy/lane-tool-surface.d.ts +2 -0
- package/dist/core/autonomy/lane-tool-surface.d.ts.map +1 -1
- package/dist/core/autonomy/lane-tool-surface.js +6 -3
- package/dist/core/autonomy/lane-tool-surface.js.map +1 -1
- package/dist/core/autonomy/subagent-prompt.js +1 -1
- package/dist/core/autonomy/subagent-prompt.js.map +1 -1
- package/dist/core/background-lane-controller.d.ts +25 -2
- package/dist/core/background-lane-controller.d.ts.map +1 -1
- package/dist/core/background-lane-controller.js +67 -0
- package/dist/core/background-lane-controller.js.map +1 -1
- package/dist/core/delegation/replay-safe-mailbox-bounds.d.ts +4 -0
- package/dist/core/delegation/replay-safe-mailbox-bounds.d.ts.map +1 -0
- package/dist/core/delegation/replay-safe-mailbox-bounds.js +20 -0
- package/dist/core/delegation/replay-safe-mailbox-bounds.js.map +1 -0
- package/dist/core/delegation/sanitized-context-fork.d.ts +25 -0
- package/dist/core/delegation/sanitized-context-fork.d.ts.map +1 -0
- package/dist/core/delegation/sanitized-context-fork.js +299 -0
- package/dist/core/delegation/sanitized-context-fork.js.map +1 -0
- package/dist/core/delegation/session-root-mailbox.d.ts +75 -0
- package/dist/core/delegation/session-root-mailbox.d.ts.map +1 -0
- package/dist/core/delegation/session-root-mailbox.js +627 -0
- package/dist/core/delegation/session-root-mailbox.js.map +1 -0
- package/dist/core/delegation/worker-agent-control-coordinator.d.ts +102 -9
- package/dist/core/delegation/worker-agent-control-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-agent-control-coordinator.js +1307 -125
- package/dist/core/delegation/worker-agent-control-coordinator.js.map +1 -1
- package/dist/core/delegation/worker-agent-control.d.ts +183 -8
- package/dist/core/delegation/worker-agent-control.d.ts.map +1 -1
- package/dist/core/delegation/worker-agent-control.js +778 -48
- package/dist/core/delegation/worker-agent-control.js.map +1 -1
- package/dist/core/delegation/worker-attempt-executor.d.ts +9 -0
- package/dist/core/delegation/worker-attempt-executor.d.ts.map +1 -1
- package/dist/core/delegation/worker-attempt-executor.js +479 -214
- package/dist/core/delegation/worker-attempt-executor.js.map +1 -1
- package/dist/core/delegation/worker-authority-resolver.d.ts +9 -0
- package/dist/core/delegation/worker-authority-resolver.d.ts.map +1 -1
- package/dist/core/delegation/worker-authority-resolver.js +21 -2
- package/dist/core/delegation/worker-authority-resolver.js.map +1 -1
- package/dist/core/delegation/worker-context-fork-store.d.ts +71 -0
- package/dist/core/delegation/worker-context-fork-store.d.ts.map +1 -0
- package/dist/core/delegation/worker-context-fork-store.js +437 -0
- package/dist/core/delegation/worker-context-fork-store.js.map +1 -0
- package/dist/core/delegation/worker-context-inheritance-policy.d.ts +18 -0
- package/dist/core/delegation/worker-context-inheritance-policy.d.ts.map +1 -0
- package/dist/core/delegation/worker-context-inheritance-policy.js +28 -0
- package/dist/core/delegation/worker-context-inheritance-policy.js.map +1 -0
- package/dist/core/delegation/worker-conversation-revision.d.ts +31 -0
- package/dist/core/delegation/worker-conversation-revision.d.ts.map +1 -0
- package/dist/core/delegation/worker-conversation-revision.js +203 -0
- package/dist/core/delegation/worker-conversation-revision.js.map +1 -0
- package/dist/core/delegation/worker-conversation-store.d.ts +130 -18
- package/dist/core/delegation/worker-conversation-store.d.ts.map +1 -1
- package/dist/core/delegation/worker-conversation-store.js +1263 -79
- package/dist/core/delegation/worker-conversation-store.js.map +1 -1
- package/dist/core/delegation/worker-delegation-controller.d.ts +30 -2
- package/dist/core/delegation/worker-delegation-controller.d.ts.map +1 -1
- package/dist/core/delegation/worker-delegation-controller.js +675 -111
- package/dist/core/delegation/worker-delegation-controller.js.map +1 -1
- package/dist/core/delegation/worker-delegation-request.d.ts +2 -0
- package/dist/core/delegation/worker-delegation-request.d.ts.map +1 -1
- package/dist/core/delegation/worker-delegation-request.js.map +1 -1
- package/dist/core/delegation/worker-dispatch-scheduler.d.ts +20 -1
- package/dist/core/delegation/worker-dispatch-scheduler.d.ts.map +1 -1
- package/dist/core/delegation/worker-dispatch-scheduler.js +210 -37
- package/dist/core/delegation/worker-dispatch-scheduler.js.map +1 -1
- package/dist/core/delegation/worker-execution-policy.d.ts.map +1 -1
- package/dist/core/delegation/worker-execution-policy.js +14 -2
- package/dist/core/delegation/worker-execution-policy.js.map +1 -1
- package/dist/core/delegation/worker-fleet-limits.d.ts +38 -0
- package/dist/core/delegation/worker-fleet-limits.d.ts.map +1 -0
- package/dist/core/delegation/worker-fleet-limits.js +113 -0
- package/dist/core/delegation/worker-fleet-limits.js.map +1 -0
- package/dist/core/delegation/worker-lane-projection.d.ts +1 -0
- package/dist/core/delegation/worker-lane-projection.d.ts.map +1 -1
- package/dist/core/delegation/worker-lane-projection.js +6 -2
- package/dist/core/delegation/worker-lane-projection.js.map +1 -1
- package/dist/core/delegation/worker-lifecycle.d.ts +20 -3
- package/dist/core/delegation/worker-lifecycle.d.ts.map +1 -1
- package/dist/core/delegation/worker-lifecycle.js +42 -18
- package/dist/core/delegation/worker-lifecycle.js.map +1 -1
- package/dist/core/delegation/worker-notification-coordinator.d.ts +4 -0
- package/dist/core/delegation/worker-notification-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-notification-coordinator.js +45 -4
- package/dist/core/delegation/worker-notification-coordinator.js.map +1 -1
- package/dist/core/delegation/worker-provider-turn-protocol.d.ts +50 -0
- package/dist/core/delegation/worker-provider-turn-protocol.d.ts.map +1 -0
- package/dist/core/delegation/worker-provider-turn-protocol.js +185 -0
- package/dist/core/delegation/worker-provider-turn-protocol.js.map +1 -0
- package/dist/core/delegation/worker-recovery-coordinator.d.ts +45 -4
- package/dist/core/delegation/worker-recovery-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-recovery-coordinator.js +223 -17
- package/dist/core/delegation/worker-recovery-coordinator.js.map +1 -1
- package/dist/core/delegation/worker-retry-policy.d.ts +36 -0
- package/dist/core/delegation/worker-retry-policy.d.ts.map +1 -0
- package/dist/core/delegation/worker-retry-policy.js +41 -0
- package/dist/core/delegation/worker-retry-policy.js.map +1 -0
- package/dist/core/delegation/worker-runner.d.ts +3 -1
- package/dist/core/delegation/worker-runner.d.ts.map +1 -1
- package/dist/core/delegation/worker-runner.js +4 -2
- package/dist/core/delegation/worker-runner.js.map +1 -1
- package/dist/core/delegation/worker-task-view.d.ts +43 -0
- package/dist/core/delegation/worker-task-view.d.ts.map +1 -0
- package/dist/core/delegation/worker-task-view.js +194 -0
- package/dist/core/delegation/worker-task-view.js.map +1 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.d.ts +55 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.d.ts.map +1 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.js +179 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.js.map +1 -0
- package/dist/core/delegation/worker-tree-budget-coordinator.d.ts +2 -2
- package/dist/core/delegation/worker-tree-budget-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-tree-budget-coordinator.js +12 -46
- package/dist/core/delegation/worker-tree-budget-coordinator.js.map +1 -1
- package/dist/core/goals/goal-continuation-controller.d.ts.map +1 -1
- package/dist/core/goals/goal-continuation-controller.js +15 -0
- package/dist/core/goals/goal-continuation-controller.js.map +1 -1
- package/dist/core/goals/goal-state.d.ts +6 -0
- package/dist/core/goals/goal-state.d.ts.map +1 -1
- package/dist/core/goals/goal-state.js +1 -0
- package/dist/core/goals/goal-state.js.map +1 -1
- package/dist/core/goals/goal-tool-core.d.ts +1 -0
- package/dist/core/goals/goal-tool-core.d.ts.map +1 -1
- package/dist/core/goals/goal-tool-core.js +25 -3
- package/dist/core/goals/goal-tool-core.js.map +1 -1
- package/dist/core/keybindings.d.ts +10 -0
- package/dist/core/keybindings.d.ts.map +1 -1
- package/dist/core/keybindings.js +2 -0
- package/dist/core/keybindings.js.map +1 -1
- package/dist/core/orchestration/attempt-ordering.d.ts +9 -0
- package/dist/core/orchestration/attempt-ordering.d.ts.map +1 -0
- package/dist/core/orchestration/attempt-ordering.js +29 -0
- package/dist/core/orchestration/attempt-ordering.js.map +1 -0
- package/dist/core/orchestration/capability-gateway.d.ts +19 -0
- package/dist/core/orchestration/capability-gateway.d.ts.map +1 -1
- package/dist/core/orchestration/capability-gateway.js +19 -2
- package/dist/core/orchestration/capability-gateway.js.map +1 -1
- package/dist/core/orchestration/contracts.d.ts +43 -3
- package/dist/core/orchestration/contracts.d.ts.map +1 -1
- package/dist/core/orchestration/contracts.js +24 -0
- package/dist/core/orchestration/contracts.js.map +1 -1
- package/dist/core/orchestration/delegation-ledger.d.ts +8 -3
- package/dist/core/orchestration/delegation-ledger.d.ts.map +1 -1
- package/dist/core/orchestration/delegation-ledger.js +148 -55
- package/dist/core/orchestration/delegation-ledger.js.map +1 -1
- package/dist/core/orchestration/event-store.d.ts +6 -3
- package/dist/core/orchestration/event-store.d.ts.map +1 -1
- package/dist/core/orchestration/event-store.js +228 -30
- package/dist/core/orchestration/event-store.js.map +1 -1
- package/dist/core/orchestration/task-runtime-codecs.d.ts +31 -0
- package/dist/core/orchestration/task-runtime-codecs.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-codecs.js +1471 -0
- package/dist/core/orchestration/task-runtime-codecs.js.map +1 -0
- package/dist/core/orchestration/task-runtime-projection.d.ts +30 -0
- package/dist/core/orchestration/task-runtime-projection.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-projection.js +340 -0
- package/dist/core/orchestration/task-runtime-projection.js.map +1 -0
- package/dist/core/orchestration/task-runtime-reducer.d.ts +38 -0
- package/dist/core/orchestration/task-runtime-reducer.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-reducer.js +1270 -0
- package/dist/core/orchestration/task-runtime-reducer.js.map +1 -0
- package/dist/core/orchestration/task-runtime-state.d.ts +142 -0
- package/dist/core/orchestration/task-runtime-state.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-state.js +14 -0
- package/dist/core/orchestration/task-runtime-state.js.map +1 -0
- package/dist/core/orchestration/task-runtime.d.ts +44 -98
- package/dist/core/orchestration/task-runtime.d.ts.map +1 -1
- package/dist/core/orchestration/task-runtime.js +357 -1113
- package/dist/core/orchestration/task-runtime.js.map +1 -1
- package/dist/core/orchestration/worker-context-fork-reference.d.ts +20 -0
- package/dist/core/orchestration/worker-context-fork-reference.d.ts.map +1 -0
- package/dist/core/orchestration/worker-context-fork-reference.js +53 -0
- package/dist/core/orchestration/worker-context-fork-reference.js.map +1 -0
- package/dist/core/orchestration/worker-execution-contract.d.ts.map +1 -1
- package/dist/core/orchestration/worker-execution-contract.js +4 -2
- package/dist/core/orchestration/worker-execution-contract.js.map +1 -1
- package/dist/core/research/research-runner.d.ts +2 -0
- package/dist/core/research/research-runner.d.ts.map +1 -1
- package/dist/core/research/research-runner.js +1 -0
- package/dist/core/research/research-runner.js.map +1 -1
- package/dist/core/runtime-builder.d.ts.map +1 -1
- package/dist/core/runtime-builder.js +8 -0
- package/dist/core/runtime-builder.js.map +1 -1
- package/dist/core/secrets/credential-project.d.ts.map +1 -1
- package/dist/core/secrets/credential-project.js +24 -2
- package/dist/core/secrets/credential-project.js.map +1 -1
- package/dist/core/security/secret-text.d.ts +3 -0
- package/dist/core/security/secret-text.d.ts.map +1 -1
- package/dist/core/security/secret-text.js +15 -0
- package/dist/core/security/secret-text.js.map +1 -1
- package/dist/core/settings-manager.d.ts +3 -0
- package/dist/core/settings-manager.d.ts.map +1 -1
- package/dist/core/settings-manager.js +160 -8
- package/dist/core/settings-manager.js.map +1 -1
- package/dist/core/tools/delegate.d.ts +33 -15
- package/dist/core/tools/delegate.d.ts.map +1 -1
- package/dist/core/tools/delegate.js +763 -115
- package/dist/core/tools/delegate.js.map +1 -1
- package/dist/core/tools/goal.d.ts +1 -0
- package/dist/core/tools/goal.d.ts.map +1 -1
- package/dist/core/tools/goal.js +9 -1
- package/dist/core/tools/goal.js.map +1 -1
- package/dist/main.d.ts.map +1 -1
- package/dist/main.js +1 -0
- package/dist/main.js.map +1 -1
- package/dist/modes/interactive/components/activity-lane.d.ts +5 -0
- package/dist/modes/interactive/components/activity-lane.d.ts.map +1 -1
- package/dist/modes/interactive/components/activity-lane.js +133 -49
- package/dist/modes/interactive/components/activity-lane.js.map +1 -1
- package/dist/modes/interactive/components/agents-overlay.d.ts +40 -0
- package/dist/modes/interactive/components/agents-overlay.d.ts.map +1 -0
- package/dist/modes/interactive/components/agents-overlay.js +153 -0
- package/dist/modes/interactive/components/agents-overlay.js.map +1 -0
- package/dist/modes/interactive/components/settings-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/settings-selector.js +18 -13
- package/dist/modes/interactive/components/settings-selector.js.map +1 -1
- package/dist/modes/interactive/interactive-event-controller.d.ts +1 -1
- package/dist/modes/interactive/interactive-event-controller.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-event-controller.js +6 -3
- package/dist/modes/interactive/interactive-event-controller.js.map +1 -1
- package/dist/modes/interactive/interactive-mode.d.ts +5 -0
- package/dist/modes/interactive/interactive-mode.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-mode.js +51 -0
- package/dist/modes/interactive/interactive-mode.js.map +1 -1
- package/dist/modes/interactive/key-handlers.d.ts +1 -0
- package/dist/modes/interactive/key-handlers.d.ts.map +1 -1
- package/dist/modes/interactive/key-handlers.js +1 -0
- package/dist/modes/interactive/key-handlers.js.map +1 -1
- package/docs/harness-architecture.md +61 -17
- package/docs/settings.md +17 -9
- package/docs/worker-profiles.md +6 -2
- package/examples/extensions/custom-provider-anthropic/package-lock.json +2 -2
- package/examples/extensions/custom-provider-anthropic/package.json +1 -1
- package/examples/extensions/custom-provider-gitlab-duo/package.json +1 -1
- package/examples/extensions/sandbox/package-lock.json +2 -2
- package/examples/extensions/sandbox/package.json +1 -1
- package/examples/extensions/with-deps/package-lock.json +2 -2
- package/examples/extensions/with-deps/package.json +1 -1
- package/npm-shrinkwrap.json +12 -12
- package/package.json +4 -4
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { isDeepStrictEqual } from "node:util";
|
|
3
|
+
import { getProcessWorkRun } from "../agent-paths.js";
|
|
2
4
|
import { getPrivateLaneDeniedPaths } from "../autonomy/lane-private-paths.js";
|
|
3
5
|
import { createLaneToolSurface } from "../autonomy/lane-tool-surface.js";
|
|
6
|
+
import { isLaneTerminalStatus } from "../autonomy/lane-tracker.js";
|
|
4
7
|
import { appendLaneRecordSnapshot } from "../autonomy/session-lane-record.js";
|
|
5
8
|
import { AUTONOMY_TELEMETRY_EVENT_TYPES } from "../autonomy/telemetry-events.js";
|
|
6
9
|
import { STABLE_SHELL_TOOL_NAME } from "../default-tool-surface.js";
|
|
7
10
|
import { deriveModelCapabilityProfile } from "../model-capability.js";
|
|
8
11
|
import { isLoopbackModelEndpoint } from "../models/model-endpoint.js";
|
|
9
12
|
import { providerUsageFromAttemptUsage } from "../orchestration/attempt-usage.js";
|
|
13
|
+
import { MAX_ORCHESTRATION_DISPATCH_INSTRUCTIONS_LENGTH, } from "../orchestration/contracts.js";
|
|
10
14
|
import { SessionTaskProfileStore } from "../orchestration/session-task-profile-store.js";
|
|
11
15
|
import { TaskProfileWriter, } from "../orchestration/task-profile-writer.js";
|
|
12
16
|
import { createWorkerExecutionContract, verifierWorkerExecutionContract, } from "../orchestration/worker-execution-contract.js";
|
|
@@ -14,20 +18,25 @@ import { registerInFlightWork } from "../reload-blockers.js";
|
|
|
14
18
|
import { createDelegateToolDefinition } from "../tools/delegate.js";
|
|
15
19
|
import { disposePersistentShellSession } from "../tools/shell-session.js";
|
|
16
20
|
import { wrapToolDefinition } from "../tools/tool-definition-wrapper.js";
|
|
21
|
+
import { selectSanitizedContextFork } from "./sanitized-context-fork.js";
|
|
17
22
|
import { applyWorkerActions } from "./worker-actions.js";
|
|
18
23
|
import { WorkerAgentControlCoordinator } from "./worker-agent-control-coordinator.js";
|
|
19
24
|
import { createWorkerAttemptExecutor } from "./worker-attempt-executor.js";
|
|
20
25
|
import { resolveWorkerAuthority } from "./worker-authority-resolver.js";
|
|
26
|
+
import { WorkerContextForkStore, WorkerContextForkStoreError } from "./worker-context-fork-store.js";
|
|
27
|
+
import { resolveWorkerContextInheritanceMode } from "./worker-context-inheritance-policy.js";
|
|
21
28
|
import { WorkerConversationStore, } from "./worker-conversation-store.js";
|
|
22
29
|
import { parseWorkerDelegationAuthorityRequest } from "./worker-delegation-request.js";
|
|
23
30
|
import { WorkerDispatchScheduler } from "./worker-dispatch-scheduler.js";
|
|
24
31
|
import { buildWorkerExecutionPlan, compileWorkerExecutionGrant, narrowWorkerExecutionPlan, workerExecutionAuthorityFromPlan, } from "./worker-execution-policy.js";
|
|
32
|
+
import { DEFAULT_WORKER_FLEET_LIMITS, evaluateNewWorkerAdmission, pendingVerifierSubjectTaskIds, } from "./worker-fleet-limits.js";
|
|
25
33
|
import { createLocalWorkerProcessOwnerId } from "./worker-process-owner.js";
|
|
26
34
|
import { WorkerProfileResolver } from "./worker-profile-resolver.js";
|
|
27
35
|
import { WorkerRecoveryCoordinator } from "./worker-recovery-coordinator.js";
|
|
28
36
|
import { selectWorkerResourcePointers } from "./worker-resource-catalog.js";
|
|
29
37
|
import { materializeWorkerResourceBundle } from "./worker-resource-materializer.js";
|
|
30
38
|
import { finalizeWorkerClaim } from "./worker-terminal-finalizer.js";
|
|
39
|
+
import { WorkerTerminalHandoffCoordinator, } from "./worker-terminal-handoff-coordinator.js";
|
|
31
40
|
import { collectWorkerTreeBudgetSeeds, WorkerTreeBudgetCoordinator } from "./worker-tree-budget-coordinator.js";
|
|
32
41
|
import { WorkerWriteReservationCoordinator } from "./worker-write-reservation-coordinator.js";
|
|
33
42
|
export function isLocalExecutionModel(model) {
|
|
@@ -57,36 +66,39 @@ function workerConversationRetentionPolicy(model, settingsManager) {
|
|
|
57
66
|
keepRecentTokens: Math.max(1, Math.min(settings.keepRecentTokens, Math.floor(contextWindow * 0.5), maxContextTokens - 1)),
|
|
58
67
|
};
|
|
59
68
|
}
|
|
69
|
+
function workerContextModelIdentity(modelRef) {
|
|
70
|
+
if (!modelRef)
|
|
71
|
+
throw new Error("Worker parent resume model is missing.");
|
|
72
|
+
const separator = modelRef.indexOf("/");
|
|
73
|
+
if (separator <= 0 || separator === modelRef.length - 1) {
|
|
74
|
+
throw new Error("Worker parent resume model is invalid.");
|
|
75
|
+
}
|
|
76
|
+
return { provider: modelRef.slice(0, separator), model: modelRef.slice(separator + 1) };
|
|
77
|
+
}
|
|
60
78
|
export class WorkerDelegationController {
|
|
61
79
|
constructor(deps, notifications, lifecycle) {
|
|
62
80
|
this.workerAbort = new AbortController();
|
|
63
81
|
this.laneAbortControllers = new Map();
|
|
64
82
|
this.shellSessionKeys = new Set();
|
|
65
83
|
this.publishedTerminalAttemptIds = new Set();
|
|
66
|
-
this.yieldedCapacityAttemptIds = new
|
|
84
|
+
this.yieldedCapacityAttemptIds = new Map();
|
|
67
85
|
this.conversations = new WorkerConversationStore();
|
|
68
86
|
this.treeBudgets = new WorkerTreeBudgetCoordinator();
|
|
69
87
|
this.inFlightLedgers = new Map();
|
|
70
88
|
this.deps = deps;
|
|
71
89
|
this.notifications = notifications;
|
|
72
90
|
this.lifecycle = lifecycle;
|
|
91
|
+
this.contextForks = new WorkerContextForkStore({
|
|
92
|
+
agentDir: this.deps.getAgentDir(),
|
|
93
|
+
parentSessionId: this.deps.getSessionId(),
|
|
94
|
+
});
|
|
73
95
|
this.scheduler = new WorkerDispatchScheduler({
|
|
74
96
|
agentDir: this.deps.getAgentDir?.() ?? "",
|
|
75
97
|
isDisposed: () => this.deps.isDisposed(),
|
|
76
98
|
admit: (request, record) => this.workerDispatchAdmission(request, record),
|
|
77
99
|
getRecord: (laneId) => this.getWorkerLifecycle().getRecord(laneId),
|
|
78
100
|
run: (request, record) => this.runOnce(request, undefined, record),
|
|
79
|
-
cancel: (laneId, reasonCode) =>
|
|
80
|
-
try {
|
|
81
|
-
this.writeReservations.release(laneId);
|
|
82
|
-
const terminal = this.getWorkerLifecycle().cancel(laneId, reasonCode);
|
|
83
|
-
if (terminal)
|
|
84
|
-
this.publishTerminalRecord(terminal);
|
|
85
|
-
}
|
|
86
|
-
catch (error) {
|
|
87
|
-
this.safeWarn(`Failed to cancel durable worker ${laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
88
|
-
}
|
|
89
|
-
},
|
|
101
|
+
cancel: (laneId, reasonCode) => this.cancelScheduledWorker(laneId, reasonCode),
|
|
90
102
|
warn: (message) => this.safeWarn(message),
|
|
91
103
|
});
|
|
92
104
|
this.recovery = new WorkerRecoveryCoordinator({
|
|
@@ -94,19 +106,33 @@ export class WorkerDelegationController {
|
|
|
94
106
|
scheduler: this.scheduler,
|
|
95
107
|
recoverWriteReservations: () => this.writeReservations.recoverProvenStale(),
|
|
96
108
|
publishTerminalRecord: (record) => this.publishTerminalRecord(record),
|
|
109
|
+
publishTerminalRecords: (records) => this.publishRecoveredTerminalRecords(records),
|
|
97
110
|
dispatchVerification: (recovery) => this.dispatchRecoveredVerification(recovery),
|
|
111
|
+
recoverTaskBearingMailboxTurns: () => this.agentControl.reconcileTaskBearingMailboxTurns(),
|
|
112
|
+
recoverSessionRootReplies: () => this.agentControl.reconcileSessionRootReplies(),
|
|
113
|
+
retryReady: () => {
|
|
114
|
+
if (this.deps.isDisposed())
|
|
115
|
+
return;
|
|
116
|
+
this.scheduler.drain();
|
|
117
|
+
this.terminalHandoffs.signal();
|
|
118
|
+
this.notifications.statusChanged();
|
|
119
|
+
},
|
|
98
120
|
warn: (message) => this.safeWarn(message),
|
|
99
121
|
});
|
|
100
122
|
this.agentControl = new WorkerAgentControlCoordinator({
|
|
101
123
|
agentDir: this.deps.getAgentDir(),
|
|
102
124
|
parentSessionId: this.deps.getSessionId(),
|
|
103
125
|
processOwnerId: createLocalWorkerProcessOwnerId(process.pid, randomUUID()),
|
|
126
|
+
conversationStore: this.conversations,
|
|
104
127
|
isControlAvailable: () => this.deps.isDelegateToolActive(),
|
|
105
128
|
getLifecycle: () => this.getWorkerLifecycle(),
|
|
106
129
|
recoveredRequest: (attempt) => this.recovery.recoveredRequest(attempt),
|
|
107
130
|
run: (request, record) => this.runOnce(request, undefined, record),
|
|
108
131
|
scheduler: this.scheduler,
|
|
109
|
-
statusChanged: () =>
|
|
132
|
+
statusChanged: () => {
|
|
133
|
+
this.terminalHandoffs.signal();
|
|
134
|
+
this.notifications.statusChanged();
|
|
135
|
+
},
|
|
110
136
|
abortLane: (laneId, reasonCode) => this.laneAbortControllers.get(laneId)?.abort(reasonCode),
|
|
111
137
|
cancelLane: (laneId, reasonCode) => {
|
|
112
138
|
this.scheduler.dropQueued(laneId);
|
|
@@ -114,10 +140,26 @@ export class WorkerDelegationController {
|
|
|
114
140
|
const terminal = this.getWorkerLifecycle().cancel(laneId, reasonCode);
|
|
115
141
|
if (terminal)
|
|
116
142
|
this.publishTerminalRecord(terminal);
|
|
143
|
+
if (terminal && !this.deps.isDisposed())
|
|
144
|
+
this.scheduler.drain();
|
|
117
145
|
return terminal;
|
|
118
146
|
},
|
|
147
|
+
taskStartHeadroomSkipReason: (agent) => {
|
|
148
|
+
const contract = this.lifecycle.getLatestAgentAttempt(agent.agentId)?.dispatch.executionContract;
|
|
149
|
+
return contract
|
|
150
|
+
? this.workerProjectionHeadroomSkipReason(contract)
|
|
151
|
+
: "orchestration_execution_contract_missing";
|
|
152
|
+
},
|
|
119
153
|
yieldCapacity: (callerAgentId, targetAgentId) => this.yieldWorkerCapacity(callerAgentId, targetAgentId),
|
|
154
|
+
warn: (message) => this.safeWarn(message),
|
|
155
|
+
});
|
|
156
|
+
this.terminalHandoffs = new WorkerTerminalHandoffCoordinator({
|
|
157
|
+
deliver: (handoff) => this.deliverTerminalHandoff(handoff),
|
|
158
|
+
onDeliveryError: (handoff, error) => {
|
|
159
|
+
this.safeWarn(`Worker terminal handoff for ${handoff.record.laneId} failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
160
|
+
},
|
|
120
161
|
});
|
|
162
|
+
this.agentControl.reconcileSessionRootReplies();
|
|
121
163
|
this.writeReservations = new WorkerWriteReservationCoordinator({
|
|
122
164
|
agentDir: this.deps.getAgentDir(),
|
|
123
165
|
getCwd: () => this.deps.getCwd(),
|
|
@@ -138,6 +180,33 @@ export class WorkerDelegationController {
|
|
|
138
180
|
// Disposal and recovery diagnostics must never throw.
|
|
139
181
|
}
|
|
140
182
|
}
|
|
183
|
+
cancelScheduledWorker(laneId, reasonCode) {
|
|
184
|
+
try {
|
|
185
|
+
this.writeReservations.release(laneId);
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
this.safeWarn(`Failed to release worker ${laneId} before cancellation: ${error instanceof Error ? error.message : String(error)}`);
|
|
189
|
+
throw error;
|
|
190
|
+
}
|
|
191
|
+
let terminal;
|
|
192
|
+
try {
|
|
193
|
+
terminal = this.getWorkerLifecycle().cancel(laneId, reasonCode);
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
this.safeWarn(`Failed to cancel durable worker ${laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
197
|
+
throw error;
|
|
198
|
+
}
|
|
199
|
+
if (!terminal)
|
|
200
|
+
return;
|
|
201
|
+
try {
|
|
202
|
+
this.publishTerminalRecord(terminal);
|
|
203
|
+
}
|
|
204
|
+
catch (error) {
|
|
205
|
+
// The durable cancellation is authoritative. Publication remains recoverable from its
|
|
206
|
+
// pending notification and must not make the scheduler requeue a terminal attempt.
|
|
207
|
+
this.safeWarn(`Failed to publish canceled worker ${laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
141
210
|
/** Narrow model-facing control port. It owns control state; this controller only composes it. */
|
|
142
211
|
getAgentControl() {
|
|
143
212
|
return this.agentControl;
|
|
@@ -173,12 +242,27 @@ export class WorkerDelegationController {
|
|
|
173
242
|
this.lifecycle.markNotificationsDelivered(notificationIds);
|
|
174
243
|
}
|
|
175
244
|
abort() {
|
|
176
|
-
this.workerAbort.abort();
|
|
245
|
+
this.runTeardownStep("abort worker execution", () => this.workerAbort.abort());
|
|
177
246
|
// Bound attempts have an authoritative transcript and agent identity. A normal owner-session
|
|
178
247
|
// shutdown is an execution interruption, not an explicit worker cancellation: fence it into
|
|
179
248
|
// suspended state before the abort continuation can observe the signal.
|
|
180
|
-
const suspendedAttemptIds = new Set(
|
|
181
|
-
|
|
249
|
+
const suspendedAttemptIds = new Set();
|
|
250
|
+
try {
|
|
251
|
+
for (const attemptId of this.lifecycle.suspendBoundInProcessAttemptsForRestart(this.agentControl.getProcessOwnerId())) {
|
|
252
|
+
suspendedAttemptIds.add(attemptId);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
catch (error) {
|
|
256
|
+
this.safeWarn(`Failed to persist worker restart suspension during teardown: ${error instanceof Error ? error.message : String(error)}`);
|
|
257
|
+
}
|
|
258
|
+
let records = [];
|
|
259
|
+
try {
|
|
260
|
+
records = this.lifecycle.getRecords();
|
|
261
|
+
}
|
|
262
|
+
catch (error) {
|
|
263
|
+
this.safeWarn(`Failed to inspect durable worker records during teardown: ${error instanceof Error ? error.message : String(error)}`);
|
|
264
|
+
}
|
|
265
|
+
for (const record of records) {
|
|
182
266
|
if (record.status !== "queued" && record.status !== "running")
|
|
183
267
|
continue;
|
|
184
268
|
const ledger = this.inFlightLedgers.get(record.laneId);
|
|
@@ -235,11 +319,24 @@ export class WorkerDelegationController {
|
|
|
235
319
|
this.safeWarn(`Failed to persist canceled worker claim ${record.laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
236
320
|
}
|
|
237
321
|
}
|
|
238
|
-
this.
|
|
239
|
-
this.
|
|
240
|
-
|
|
241
|
-
|
|
322
|
+
this.inFlightLedgers.clear();
|
|
323
|
+
this.runTeardownStep("cancel queued worker dispatches", () => this.scheduler.cancelQueued());
|
|
324
|
+
this.runTeardownStep("dispose worker recovery", () => this.recovery.dispose());
|
|
325
|
+
this.runTeardownStep("dispose worker terminal handoffs", () => this.terminalHandoffs.dispose());
|
|
326
|
+
this.runTeardownStep("dispose worker write reservations", () => this.writeReservations.dispose());
|
|
327
|
+
for (const shellSessionKey of this.shellSessionKeys) {
|
|
328
|
+
this.runTeardownStep(`dispose worker shell ${shellSessionKey}`, () => disposePersistentShellSession(shellSessionKey));
|
|
329
|
+
}
|
|
242
330
|
this.shellSessionKeys.clear();
|
|
331
|
+
this.runTeardownStep("clear worker conversation cache", () => this.conversations.clearCache());
|
|
332
|
+
}
|
|
333
|
+
runTeardownStep(label, step) {
|
|
334
|
+
try {
|
|
335
|
+
step();
|
|
336
|
+
}
|
|
337
|
+
catch (error) {
|
|
338
|
+
this.safeWarn(`Failed to ${label}: ${error instanceof Error ? error.message : String(error)}`);
|
|
339
|
+
}
|
|
243
340
|
}
|
|
244
341
|
getWorkerLifecycle() {
|
|
245
342
|
return this.lifecycle;
|
|
@@ -285,6 +382,8 @@ export class WorkerDelegationController {
|
|
|
285
382
|
/** Reconcile the session goal into an already-loaded worker runtime without defeating lazy UAC. */
|
|
286
383
|
synchronizeGoalState(goal) {
|
|
287
384
|
this.publishGoalTerminalRecords(this.lifecycle.synchronizeGoalState(goal));
|
|
385
|
+
if (goal.status === "active" && !this.deps.isDisposed())
|
|
386
|
+
this.scheduler.drain();
|
|
288
387
|
}
|
|
289
388
|
publishGoalTerminalRecords(records) {
|
|
290
389
|
for (const record of records) {
|
|
@@ -292,6 +391,8 @@ export class WorkerDelegationController {
|
|
|
292
391
|
this.laneAbortControllers.get(record.laneId)?.abort("goal_terminal");
|
|
293
392
|
this.publishTerminalRecord(record);
|
|
294
393
|
}
|
|
394
|
+
if (records.length > 0 && !this.deps.isDisposed())
|
|
395
|
+
this.scheduler.drain();
|
|
295
396
|
}
|
|
296
397
|
dispatchRecoveredVerification(recovery) {
|
|
297
398
|
const legacyImplementation = recovery.verifierExecutionContract
|
|
@@ -334,9 +435,45 @@ export class WorkerDelegationController {
|
|
|
334
435
|
if (!profile.requireIndependentVerification)
|
|
335
436
|
return { ok: true };
|
|
336
437
|
const resolved = this.getWorkerProfileResolver().resolveVerifier(profile);
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
438
|
+
if (!resolved.ok)
|
|
439
|
+
return { ok: false, skipReason: `independent_verifier_unavailable:${resolved.reason}` };
|
|
440
|
+
// Fresh verifier profiles pass through the same shipment admission owner as fresh workers.
|
|
441
|
+
// Recovery-pinned verifier contracts bypass this path and preserve their immutable grant.
|
|
442
|
+
const admitted = resolveWorkerAuthority({
|
|
443
|
+
base: resolved.resolved,
|
|
444
|
+
modelRegistry: this.deps.getModelRegistry(),
|
|
445
|
+
isModelExhausted: (model) => this.deps.isModelExhausted(model),
|
|
446
|
+
});
|
|
447
|
+
return admitted.ok
|
|
448
|
+
? { ok: true, shipment: admitted.shipment }
|
|
449
|
+
: { ok: false, skipReason: `independent_verifier_unavailable:${admitted.reason}` };
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* A descendant may select a different routing profile, but that preset cannot introduce context
|
|
453
|
+
* its ancestor never admitted. Pointer ids alone are insufficient authority: retain only pointers
|
|
454
|
+
* whose complete immutable metadata matches the ancestral contract, and require exact soul text.
|
|
455
|
+
*/
|
|
456
|
+
narrowWorkerShipmentContext(shipment, boundary) {
|
|
457
|
+
const boundaryPointers = new Map(boundary.resourcePointers.map((pointer) => [pointer.id, pointer]));
|
|
458
|
+
const resourcePointers = [];
|
|
459
|
+
for (const pointer of shipment.resourcePointers) {
|
|
460
|
+
const inherited = boundaryPointers.get(pointer.id);
|
|
461
|
+
if (!inherited || !isDeepStrictEqual(inherited, pointer)) {
|
|
462
|
+
return { ok: false, skipReason: "orchestration_context_authority_exceeded" };
|
|
463
|
+
}
|
|
464
|
+
resourcePointers.push(structuredClone(inherited));
|
|
465
|
+
}
|
|
466
|
+
if (shipment.soul !== undefined && shipment.soul !== boundary.soul) {
|
|
467
|
+
return { ok: false, skipReason: "orchestration_context_authority_exceeded" };
|
|
468
|
+
}
|
|
469
|
+
return {
|
|
470
|
+
ok: true,
|
|
471
|
+
shipment: {
|
|
472
|
+
...shipment,
|
|
473
|
+
resourcePointers,
|
|
474
|
+
...(shipment.soul ? { soul: shipment.soul } : {}),
|
|
475
|
+
},
|
|
476
|
+
};
|
|
340
477
|
}
|
|
341
478
|
hasExactRecursiveCycle(parentAgentId, instructions, profileId) {
|
|
342
479
|
const normalizedInstructions = instructions.trim();
|
|
@@ -355,6 +492,17 @@ export class WorkerDelegationController {
|
|
|
355
492
|
}
|
|
356
493
|
return false;
|
|
357
494
|
}
|
|
495
|
+
/** Reject a new logical identity before creating its task, transcript, agent, or queue entry. */
|
|
496
|
+
newWorkerFleetSkipReason(request, requiredAgentSlots = 1) {
|
|
497
|
+
const decision = evaluateNewWorkerAdmission(this.lifecycle.getTaskRuntimeSnapshot().agents, request.parentAgentId, DEFAULT_WORKER_FLEET_LIMITS, requiredAgentSlots);
|
|
498
|
+
return decision.ok ? undefined : decision.reasonCode;
|
|
499
|
+
}
|
|
500
|
+
requiredAgentSlotsForAdmission(request, admission) {
|
|
501
|
+
const pendingVerifierSubjects = pendingVerifierSubjectTaskIds(this.lifecycle.getTaskRuntimeSnapshot());
|
|
502
|
+
if (request.verificationOfTaskId)
|
|
503
|
+
pendingVerifierSubjects.delete(request.verificationOfTaskId);
|
|
504
|
+
return 1 + pendingVerifierSubjects.size + (admission.verifierShipment ? 1 : 0);
|
|
505
|
+
}
|
|
358
506
|
/** Single admission contract shared by enqueue, scheduler revalidation, and execution. */
|
|
359
507
|
resolveWorkerAdmission(request, pinnedContract) {
|
|
360
508
|
if (this.deps.isDisposed())
|
|
@@ -421,7 +569,13 @@ export class WorkerDelegationController {
|
|
|
421
569
|
});
|
|
422
570
|
if (!adaptive.ok)
|
|
423
571
|
return { ok: false, skipReason: adaptive.reason };
|
|
424
|
-
|
|
572
|
+
let shipment = adaptive.shipment;
|
|
573
|
+
if (parentContract) {
|
|
574
|
+
const narrowed = this.narrowWorkerShipmentContext(shipment, parentContract.worker);
|
|
575
|
+
if (!narrowed.ok)
|
|
576
|
+
return narrowed;
|
|
577
|
+
shipment = narrowed.shipment;
|
|
578
|
+
}
|
|
425
579
|
if (parentAgent && this.hasExactRecursiveCycle(parentAgent.agentId, instructions, shipment.profile.profileId)) {
|
|
426
580
|
return { ok: false, skipReason: "recursive_delegation_cycle" };
|
|
427
581
|
}
|
|
@@ -445,6 +599,13 @@ export class WorkerDelegationController {
|
|
|
445
599
|
return verifier;
|
|
446
600
|
verifierShipment = verifier.shipment;
|
|
447
601
|
}
|
|
602
|
+
if (verifierShipment && parentContract) {
|
|
603
|
+
const verifierBoundary = parentContract.verifier ?? parentContract.worker;
|
|
604
|
+
const narrowed = this.narrowWorkerShipmentContext(verifierShipment, verifierBoundary);
|
|
605
|
+
if (!narrowed.ok)
|
|
606
|
+
return narrowed;
|
|
607
|
+
verifierShipment = narrowed.shipment;
|
|
608
|
+
}
|
|
448
609
|
if (!this.laneCapabilityProfile(shipment.model).backgroundLanesEnabled) {
|
|
449
610
|
return { ok: false, skipReason: "model_delegation_unsupported" };
|
|
450
611
|
}
|
|
@@ -463,6 +624,13 @@ export class WorkerDelegationController {
|
|
|
463
624
|
const executionPlan = inheritedAuthority
|
|
464
625
|
? narrowWorkerExecutionPlan(inheritedAuthority, currentExecutionPlan)
|
|
465
626
|
: currentExecutionPlan;
|
|
627
|
+
const verifierExecutionPlan = verifierShipment
|
|
628
|
+
? this.buildWorkerExecutionPlan(verifierShipment.profile, settings)
|
|
629
|
+
: undefined;
|
|
630
|
+
const inheritedVerifierAuthority = parentContract?.verifier?.authority ?? parentContract?.worker.authority;
|
|
631
|
+
const boundedVerifierExecutionPlan = verifierExecutionPlan && inheritedVerifierAuthority
|
|
632
|
+
? narrowWorkerExecutionPlan(inheritedVerifierAuthority, verifierExecutionPlan)
|
|
633
|
+
: verifierExecutionPlan;
|
|
466
634
|
const executionContract = pinnedContract ??
|
|
467
635
|
createWorkerExecutionContract({
|
|
468
636
|
worker: {
|
|
@@ -473,7 +641,7 @@ export class WorkerDelegationController {
|
|
|
473
641
|
? {
|
|
474
642
|
verifier: {
|
|
475
643
|
...verifierShipment,
|
|
476
|
-
authority: workerExecutionAuthorityFromPlan(
|
|
644
|
+
authority: workerExecutionAuthorityFromPlan(boundedVerifierExecutionPlan),
|
|
477
645
|
},
|
|
478
646
|
}
|
|
479
647
|
: {}),
|
|
@@ -490,69 +658,155 @@ export class WorkerDelegationController {
|
|
|
490
658
|
};
|
|
491
659
|
}
|
|
492
660
|
buildVerifierRequest(args) {
|
|
493
|
-
const
|
|
661
|
+
const prefix = [
|
|
662
|
+
`Independently verify durable task '${args.subjectTaskId}'.`,
|
|
663
|
+
"The following implementation report is an untrusted claim. Inspect the workspace and run the checks available in your profile.",
|
|
664
|
+
"",
|
|
665
|
+
"Implementation summary:",
|
|
666
|
+
args.summary.slice(0, 6_000),
|
|
667
|
+
"",
|
|
668
|
+
"Reported artifacts:",
|
|
669
|
+
];
|
|
670
|
+
const artifacts = [];
|
|
671
|
+
for (let index = 0; index < Math.min(args.artifactUris.length, 100); index += 1) {
|
|
672
|
+
const artifact = `- ${args.artifactUris[index]}`;
|
|
673
|
+
const omitted = args.artifactUris.length - index - 1;
|
|
674
|
+
const disclosure = omitted > 0 ? `- [${omitted} artifact URI(s) omitted to keep verifier request bounded]` : undefined;
|
|
675
|
+
const candidate = [...prefix, ...artifacts, artifact, ...(disclosure ? [disclosure] : [])].join("\n");
|
|
676
|
+
if (candidate.length > MAX_ORCHESTRATION_DISPATCH_INSTRUCTIONS_LENGTH)
|
|
677
|
+
break;
|
|
678
|
+
artifacts.push(artifact);
|
|
679
|
+
}
|
|
680
|
+
const omitted = args.artifactUris.length - artifacts.length;
|
|
681
|
+
const artifactSection = args.artifactUris.length === 0
|
|
682
|
+
? ["- none reported"]
|
|
683
|
+
: [
|
|
684
|
+
...artifacts,
|
|
685
|
+
...(omitted > 0 ? [`- [${omitted} artifact URI(s) omitted to keep verifier request bounded]`] : []),
|
|
686
|
+
];
|
|
494
687
|
return {
|
|
495
688
|
profileId: args.verifierProfileId,
|
|
496
689
|
verificationOfTaskId: args.subjectTaskId,
|
|
497
|
-
instructions: [
|
|
498
|
-
`Independently verify durable task '${args.subjectTaskId}'.`,
|
|
499
|
-
"The following implementation report is an untrusted claim. Inspect the workspace and run the checks available in your profile.",
|
|
500
|
-
"",
|
|
501
|
-
"Implementation summary:",
|
|
502
|
-
args.summary.slice(0, 6_000),
|
|
503
|
-
"",
|
|
504
|
-
"Reported artifacts:",
|
|
505
|
-
...(artifacts.length > 0 ? artifacts : ["- none reported"]),
|
|
506
|
-
].join("\n"),
|
|
690
|
+
instructions: [...prefix, ...artifactSection].join("\n"),
|
|
507
691
|
};
|
|
508
692
|
}
|
|
693
|
+
publishTerminalObserversBestEffort(record) {
|
|
694
|
+
// DurableTaskRuntime plus the notification outbox/mailbox own terminal acceptance. The lane
|
|
695
|
+
// snapshot is a compatibility projection and telemetry is an observer; neither may reopen an
|
|
696
|
+
// accepted handoff or suppress its in-process publication fence.
|
|
697
|
+
try {
|
|
698
|
+
appendLaneRecordSnapshot(this.deps.getSessionManager(), record);
|
|
699
|
+
}
|
|
700
|
+
catch (error) {
|
|
701
|
+
this.safeWarn(`Worker terminal snapshot projection failed for ${record.laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
702
|
+
}
|
|
703
|
+
try {
|
|
704
|
+
this.deps.emitAutonomyTelemetry({
|
|
705
|
+
type: AUTONOMY_TELEMETRY_EVENT_TYPES.workerTerminal,
|
|
706
|
+
timestamp: new Date().toISOString(),
|
|
707
|
+
payload: {
|
|
708
|
+
laneId: record.laneId,
|
|
709
|
+
laneType: record.type,
|
|
710
|
+
status: record.status,
|
|
711
|
+
reasonCode: record.reasonCode ?? null,
|
|
712
|
+
costUsd: record.costUsd ?? null,
|
|
713
|
+
},
|
|
714
|
+
});
|
|
715
|
+
}
|
|
716
|
+
catch (error) {
|
|
717
|
+
this.safeWarn(`Worker terminal telemetry observer failed for ${record.laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
deliverTerminalHandoff(handoff) {
|
|
721
|
+
const delivery = this.agentControl.deliverWorkerTerminalHandoff(handoff);
|
|
722
|
+
if (!delivery.accepted) {
|
|
723
|
+
this.safeWarn(`Worker terminal handoff for ${handoff.record.laneId} was not accepted${delivery.skipReason ? `: ${delivery.skipReason}` : "."}`);
|
|
724
|
+
return "retained";
|
|
725
|
+
}
|
|
726
|
+
const notification = this.lifecycle.getTerminalNotification(handoff.record.laneId);
|
|
727
|
+
if (notification?.status === "pending") {
|
|
728
|
+
this.lifecycle.markNotificationsDelivered([notification.notificationId]);
|
|
729
|
+
}
|
|
730
|
+
this.publishedTerminalAttemptIds.add(handoff.terminalAttemptId);
|
|
731
|
+
this.publishTerminalObserversBestEffort(handoff.record);
|
|
732
|
+
try {
|
|
733
|
+
this.notifications.statusChanged();
|
|
734
|
+
}
|
|
735
|
+
catch (error) {
|
|
736
|
+
this.safeWarn(`Worker terminal status observer failed for ${handoff.record.laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
737
|
+
}
|
|
738
|
+
try {
|
|
739
|
+
this.agentControl.signalStateChanged();
|
|
740
|
+
}
|
|
741
|
+
catch (error) {
|
|
742
|
+
this.safeWarn(`Worker terminal state observer failed for ${handoff.record.laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
743
|
+
}
|
|
744
|
+
return "delivered";
|
|
745
|
+
}
|
|
746
|
+
publishRecoveredTerminalRecords(records) {
|
|
747
|
+
const handoffs = [];
|
|
748
|
+
const directRecords = [];
|
|
749
|
+
for (const record of records) {
|
|
750
|
+
this.recovery.clearScheduledRetry(record.laneId);
|
|
751
|
+
const attempt = this.lifecycle.getActiveAttempt(record.laneId);
|
|
752
|
+
if (attempt && this.publishedTerminalAttemptIds.has(attempt.attemptId))
|
|
753
|
+
continue;
|
|
754
|
+
const childAgentId = attempt?.agentId ?? attempt?.dispatch.logicalLaneId ?? record.laneId;
|
|
755
|
+
const parentAgentId = this.lifecycle.getAgent(childAgentId)?.parentAgentId ?? attempt?.dispatch.parentAgentId;
|
|
756
|
+
if (attempt && parentAgentId) {
|
|
757
|
+
handoffs.push({
|
|
758
|
+
terminalAttemptId: attempt.attemptId,
|
|
759
|
+
parentAgentId,
|
|
760
|
+
childAgentId,
|
|
761
|
+
record,
|
|
762
|
+
});
|
|
763
|
+
}
|
|
764
|
+
else {
|
|
765
|
+
directRecords.push(record);
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
if (handoffs.length > 0) {
|
|
769
|
+
try {
|
|
770
|
+
this.terminalHandoffs.rehydrate(handoffs);
|
|
771
|
+
}
|
|
772
|
+
catch (error) {
|
|
773
|
+
this.safeWarn(`Worker terminal handoff recovery batch was retained durably but not adopted: ${error instanceof Error ? error.message : String(error)}`);
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
for (const record of directRecords)
|
|
777
|
+
this.publishTerminalRecord(record);
|
|
778
|
+
}
|
|
509
779
|
publishTerminalRecord(record) {
|
|
510
|
-
|
|
780
|
+
if (!isLaneTerminalStatus(record.status))
|
|
781
|
+
return;
|
|
782
|
+
this.recovery.clearScheduledRetry(record.laneId);
|
|
783
|
+
const attempt = this.lifecycle.getActiveAttempt(record.laneId);
|
|
511
784
|
const attemptId = attempt?.attemptId;
|
|
512
785
|
if (attemptId && this.publishedTerminalAttemptIds.has(attemptId))
|
|
513
786
|
return;
|
|
514
787
|
const childAgentId = attempt?.agentId ?? attempt?.dispatch.logicalLaneId ?? record.laneId;
|
|
515
788
|
const parentAgentId = this.lifecycle.getAgent(childAgentId)?.parentAgentId ?? attempt?.dispatch.parentAgentId;
|
|
516
|
-
let handedOffToParent = false;
|
|
517
789
|
if (attemptId && parentAgentId) {
|
|
518
790
|
try {
|
|
519
|
-
this.
|
|
791
|
+
this.terminalHandoffs.retain({
|
|
520
792
|
parentAgentId,
|
|
521
793
|
childAgentId,
|
|
522
794
|
terminalAttemptId: attemptId,
|
|
523
795
|
record,
|
|
524
796
|
});
|
|
525
|
-
|
|
797
|
+
this.terminalHandoffs.signal();
|
|
526
798
|
}
|
|
527
799
|
catch (error) {
|
|
528
|
-
this.safeWarn(`Worker terminal handoff for ${record.laneId} failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
529
|
-
}
|
|
530
|
-
}
|
|
531
|
-
if (handedOffToParent) {
|
|
532
|
-
const notification = this.lifecycle.getTerminalNotification(record.laneId);
|
|
533
|
-
if (notification?.status === "pending") {
|
|
534
|
-
this.lifecycle.markNotificationsDelivered([notification.notificationId]);
|
|
800
|
+
this.safeWarn(`Worker terminal handoff retention for ${record.laneId} failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
535
801
|
}
|
|
536
|
-
|
|
802
|
+
return;
|
|
537
803
|
}
|
|
538
|
-
|
|
539
|
-
|
|
804
|
+
this.recordTerminal(record);
|
|
805
|
+
if (attemptId) {
|
|
806
|
+
this.publishedTerminalAttemptIds.add(attemptId);
|
|
540
807
|
}
|
|
541
|
-
|
|
542
|
-
this.deps.emitAutonomyTelemetry({
|
|
543
|
-
type: AUTONOMY_TELEMETRY_EVENT_TYPES.workerTerminal,
|
|
544
|
-
timestamp: new Date().toISOString(),
|
|
545
|
-
payload: {
|
|
546
|
-
laneId: record.laneId,
|
|
547
|
-
laneType: record.type,
|
|
548
|
-
status: record.status,
|
|
549
|
-
reasonCode: record.reasonCode ?? null,
|
|
550
|
-
costUsd: record.costUsd ?? null,
|
|
551
|
-
},
|
|
552
|
-
});
|
|
808
|
+
this.publishTerminalObserversBestEffort(record);
|
|
553
809
|
this.agentControl.signalStateChanged();
|
|
554
|
-
if (attemptId)
|
|
555
|
-
this.publishedTerminalAttemptIds.add(attemptId);
|
|
556
810
|
}
|
|
557
811
|
/**
|
|
558
812
|
* A pre-execution denial still owns a prepared durable lane. Publish its terminal projection at
|
|
@@ -564,12 +818,14 @@ export class WorkerDelegationController {
|
|
|
564
818
|
const record = lifecycle.cancel(laneId, reasonCode);
|
|
565
819
|
if (record)
|
|
566
820
|
this.publishTerminalRecord(record);
|
|
821
|
+
if (record && !this.deps.isDisposed())
|
|
822
|
+
this.scheduler.drain();
|
|
567
823
|
return record;
|
|
568
824
|
}
|
|
569
825
|
hasWorkerCapacity(settings) {
|
|
570
826
|
const snapshot = this.getWorkerLifecycle().getTaskRuntimeSnapshot();
|
|
571
827
|
let yielded = 0;
|
|
572
|
-
for (const attemptId of
|
|
828
|
+
for (const attemptId of this.yieldedCapacityAttemptIds.keys()) {
|
|
573
829
|
const status = snapshot.attempts[attemptId]?.status;
|
|
574
830
|
if (status === "leased" || status === "running")
|
|
575
831
|
yielded += 1;
|
|
@@ -586,33 +842,205 @@ export class WorkerDelegationController {
|
|
|
586
842
|
const attempt = this.lifecycle.getLatestAgentAttempt(caller.agentId);
|
|
587
843
|
if (!attempt || (attempt.status !== "leased" && attempt.status !== "running"))
|
|
588
844
|
return () => undefined;
|
|
589
|
-
this.yieldedCapacityAttemptIds.
|
|
845
|
+
this.yieldedCapacityAttemptIds.set(attempt.attemptId, (this.yieldedCapacityAttemptIds.get(attempt.attemptId) ?? 0) + 1);
|
|
590
846
|
this.scheduler.drain();
|
|
591
847
|
let released = false;
|
|
592
848
|
return () => {
|
|
593
849
|
if (released)
|
|
594
850
|
return;
|
|
595
851
|
released = true;
|
|
852
|
+
const leases = this.yieldedCapacityAttemptIds.get(attempt.attemptId);
|
|
853
|
+
if (leases === undefined)
|
|
854
|
+
return;
|
|
855
|
+
if (leases > 1) {
|
|
856
|
+
this.yieldedCapacityAttemptIds.set(attempt.attemptId, leases - 1);
|
|
857
|
+
return;
|
|
858
|
+
}
|
|
596
859
|
this.yieldedCapacityAttemptIds.delete(attempt.attemptId);
|
|
597
860
|
if (!this.deps.isDisposed())
|
|
598
861
|
this.scheduler.drain();
|
|
599
862
|
};
|
|
600
863
|
}
|
|
601
864
|
workerDispatchAdmission(request, record) {
|
|
602
|
-
|
|
865
|
+
if (this.recovery.deferRetryIfNeeded(record, request))
|
|
866
|
+
return { action: "wait" };
|
|
867
|
+
const lifecycle = this.getWorkerLifecycle();
|
|
868
|
+
const attempt = lifecycle.getActiveAttempt(record.laneId);
|
|
869
|
+
if (!attempt)
|
|
870
|
+
return { action: "cancel", reasonCode: "orchestration_attempt_missing" };
|
|
871
|
+
if (attempt.status === "queued" || attempt.status === "suspended") {
|
|
872
|
+
const readiness = lifecycle.getAttemptDispatchReadiness(attempt.attemptId);
|
|
873
|
+
if (readiness.state === "waiting") {
|
|
874
|
+
return {
|
|
875
|
+
action: "wait",
|
|
876
|
+
reason: readiness.reasonCode === "objective_paused" ? "objective" : "dependencies",
|
|
877
|
+
};
|
|
878
|
+
}
|
|
879
|
+
if (readiness.state === "blocked")
|
|
880
|
+
return { action: "cancel", reasonCode: readiness.reasonCode };
|
|
881
|
+
}
|
|
882
|
+
const contract = attempt.dispatch.executionContract;
|
|
603
883
|
const admission = this.resolveWorkerAdmission(request, contract);
|
|
604
884
|
if (!admission.ok)
|
|
605
885
|
return { action: "cancel", reasonCode: admission.skipReason };
|
|
606
886
|
if (!this.hasWorkerCapacity(admission.settings))
|
|
607
887
|
return { action: "wait", reason: "capacity" };
|
|
608
|
-
const attempt = this.getWorkerLifecycle().getActiveAttempt(record.laneId);
|
|
609
|
-
if (!attempt)
|
|
610
|
-
return { action: "cancel", reasonCode: "orchestration_attempt_missing" };
|
|
611
888
|
const reservation = this.writeReservations.acquire(record.laneId, attempt, admission.executionPlan);
|
|
612
889
|
if (reservation.kind === "denied")
|
|
613
890
|
return { action: "cancel", reasonCode: reservation.reasonCode };
|
|
614
891
|
return reservation.kind === "granted" ? { action: "start" } : { action: "wait", reason: "write_reservation" };
|
|
615
892
|
}
|
|
893
|
+
workerProjectionHeadroomSkipReason(contract, verificationOfTaskId) {
|
|
894
|
+
const pendingVerifierSubjects = pendingVerifierSubjectTaskIds(this.lifecycle.getTaskRuntimeSnapshot());
|
|
895
|
+
if (verificationOfTaskId)
|
|
896
|
+
pendingVerifierSubjects.delete(verificationOfTaskId);
|
|
897
|
+
const slots = 1 + pendingVerifierSubjects.size + (contract.verifier ? 1 : 0);
|
|
898
|
+
try {
|
|
899
|
+
this.lifecycle.ledger.runtime.assertProjectionHeadroom({ tasks: slots, attempts: slots });
|
|
900
|
+
return undefined;
|
|
901
|
+
}
|
|
902
|
+
catch {
|
|
903
|
+
return "orchestration_projection_capacity_exhausted";
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
workerQueueReservationSkipReason(request, requestQueueSlots, unpersistedVerifierSlots) {
|
|
907
|
+
const pendingVerifierSubjects = pendingVerifierSubjectTaskIds(this.lifecycle.getTaskRuntimeSnapshot());
|
|
908
|
+
if (request.verificationOfTaskId)
|
|
909
|
+
pendingVerifierSubjects.delete(request.verificationOfTaskId);
|
|
910
|
+
const remainingQueueSlots = DEFAULT_WORKER_FLEET_LIMITS.maxQueuedDispatches - this.scheduler.queuedCount;
|
|
911
|
+
return requestQueueSlots + pendingVerifierSubjects.size + unpersistedVerifierSlots > remainingQueueSlots
|
|
912
|
+
? "worker_dispatch_queue_full"
|
|
913
|
+
: undefined;
|
|
914
|
+
}
|
|
915
|
+
workerContextParentModel(request) {
|
|
916
|
+
if (request.parentAgentId) {
|
|
917
|
+
const parent = this.lifecycle.getAgent(request.parentAgentId);
|
|
918
|
+
if (!parent)
|
|
919
|
+
throw new Error("Worker parent logical agent is missing.");
|
|
920
|
+
return workerContextModelIdentity(parent.resumeContext.modelRef);
|
|
921
|
+
}
|
|
922
|
+
const parent = this.deps.getModel();
|
|
923
|
+
if (!parent)
|
|
924
|
+
throw new Error("Foreground parent model is missing.");
|
|
925
|
+
return { provider: parent.provider, model: parent.id };
|
|
926
|
+
}
|
|
927
|
+
workerContextForkSource(request) {
|
|
928
|
+
const model = this.workerContextParentModel(request);
|
|
929
|
+
if (!request.parentAgentId) {
|
|
930
|
+
return { model, messages: this.deps.getSessionManager().buildSessionContext().messages };
|
|
931
|
+
}
|
|
932
|
+
const parent = this.lifecycle.getAgent(request.parentAgentId);
|
|
933
|
+
if (!parent)
|
|
934
|
+
throw new Error("Worker parent logical agent is missing.");
|
|
935
|
+
const conversation = this.conversations.open({
|
|
936
|
+
agentDir: this.deps.getAgentDir(),
|
|
937
|
+
resumeContext: parent.resumeContext,
|
|
938
|
+
expectedLogicalAgentId: parent.agentId,
|
|
939
|
+
});
|
|
940
|
+
return { model, messages: conversation.getProviderContext().messages };
|
|
941
|
+
}
|
|
942
|
+
workerContextForkMode(request, contract) {
|
|
943
|
+
if (request.verificationOfTaskId)
|
|
944
|
+
return { kind: "none" };
|
|
945
|
+
return resolveWorkerContextInheritanceMode({
|
|
946
|
+
parent: this.workerContextParentModel(request),
|
|
947
|
+
worker: {
|
|
948
|
+
provider: contract.worker.modelBinding.provider,
|
|
949
|
+
model: contract.worker.modelBinding.modelId,
|
|
950
|
+
},
|
|
951
|
+
...(request.forkTurns !== undefined ? { mode: request.forkTurns } : {}),
|
|
952
|
+
});
|
|
953
|
+
}
|
|
954
|
+
workerContextForkAdmissionSkipReason(request, contract) {
|
|
955
|
+
try {
|
|
956
|
+
this.workerContextForkMode(request, contract);
|
|
957
|
+
return undefined;
|
|
958
|
+
}
|
|
959
|
+
catch {
|
|
960
|
+
return "worker_context_inheritance_denied";
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
workerGoalDependencySkipReason(request) {
|
|
964
|
+
if (request.verificationOfTaskId)
|
|
965
|
+
return undefined;
|
|
966
|
+
const goal = this.deps.getGoalStateSnapshot();
|
|
967
|
+
if (!goal || !goal.requirements || goal.requirements.length === 0)
|
|
968
|
+
return undefined;
|
|
969
|
+
const reqIds = new Set();
|
|
970
|
+
for (const id of request.taskContext?.requirementIds ?? []) {
|
|
971
|
+
if (id.trim())
|
|
972
|
+
reqIds.add(id.trim());
|
|
973
|
+
}
|
|
974
|
+
if (reqIds.size === 0 && request.instructions) {
|
|
975
|
+
for (const req of goal.requirements) {
|
|
976
|
+
if (request.instructions.includes(req.id)) {
|
|
977
|
+
reqIds.add(req.id);
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
if (reqIds.size > 0) {
|
|
982
|
+
for (const reqId of reqIds) {
|
|
983
|
+
const req = goal.requirements.find((r) => r.id === reqId);
|
|
984
|
+
if (req?.dependencies && req.dependencies.length > 0) {
|
|
985
|
+
const unsatisfied = req.dependencies.some((depId) => {
|
|
986
|
+
const dep = goal.requirements.find((r) => r.id === depId);
|
|
987
|
+
return !dep || dep.status !== "satisfied";
|
|
988
|
+
});
|
|
989
|
+
if (unsatisfied)
|
|
990
|
+
return "goal_dependency_unsatisfied";
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
else {
|
|
995
|
+
const openReqs = goal.requirements.filter((r) => r.status === "open");
|
|
996
|
+
const hasOpenReqWithoutUnsatisfiedDeps = openReqs.some((req) => {
|
|
997
|
+
if (!req.dependencies || req.dependencies.length === 0)
|
|
998
|
+
return true;
|
|
999
|
+
return req.dependencies.every((depId) => {
|
|
1000
|
+
const dep = goal.requirements.find((r) => r.id === depId);
|
|
1001
|
+
return dep && dep.status === "satisfied";
|
|
1002
|
+
});
|
|
1003
|
+
});
|
|
1004
|
+
if (openReqs.length > 0 && !hasOpenReqWithoutUnsatisfiedDeps) {
|
|
1005
|
+
return "goal_dependency_unsatisfied";
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
return undefined;
|
|
1009
|
+
}
|
|
1010
|
+
/** One admission owner for every newly generated logical worker identity. */
|
|
1011
|
+
admitNewWorkerRequest(request, pinnedContract) {
|
|
1012
|
+
const goalDependencySkipReason = this.workerGoalDependencySkipReason(request);
|
|
1013
|
+
if (goalDependencySkipReason)
|
|
1014
|
+
return { ok: false, skipReason: goalDependencySkipReason };
|
|
1015
|
+
const fleetSkipReason = this.newWorkerFleetSkipReason(request);
|
|
1016
|
+
if (fleetSkipReason)
|
|
1017
|
+
return { ok: false, skipReason: fleetSkipReason };
|
|
1018
|
+
const admission = this.resolveWorkerAdmission(request, pinnedContract);
|
|
1019
|
+
if (!admission.ok)
|
|
1020
|
+
return admission;
|
|
1021
|
+
const contextForkSkipReason = this.workerContextForkAdmissionSkipReason(request, admission.executionContract);
|
|
1022
|
+
if (contextForkSkipReason)
|
|
1023
|
+
return { ok: false, skipReason: contextForkSkipReason };
|
|
1024
|
+
if (!request.verificationOfTaskId)
|
|
1025
|
+
this.recovery.recover();
|
|
1026
|
+
const headroomSkipReason = this.newWorkerFleetSkipReason(request, this.requiredAgentSlotsForAdmission(request, admission));
|
|
1027
|
+
if (headroomSkipReason)
|
|
1028
|
+
return { ok: false, skipReason: headroomSkipReason };
|
|
1029
|
+
const projectionHeadroomSkipReason = this.workerProjectionHeadroomSkipReason(admission.executionContract, request.verificationOfTaskId);
|
|
1030
|
+
if (projectionHeadroomSkipReason)
|
|
1031
|
+
return { ok: false, skipReason: projectionHeadroomSkipReason };
|
|
1032
|
+
const queueReservationSkipReason = this.workerQueueReservationSkipReason(request, 0, admission.verifierShipment ? 1 : 0);
|
|
1033
|
+
return queueReservationSkipReason ? { ok: false, skipReason: queueReservationSkipReason } : admission;
|
|
1034
|
+
}
|
|
1035
|
+
durableWorkerContextForkReferences() {
|
|
1036
|
+
const references = [];
|
|
1037
|
+
for (const attempt of Object.values(this.lifecycle.getTaskRuntimeSnapshot().attempts)) {
|
|
1038
|
+
if (attempt.dispatch.birthContextForkReference) {
|
|
1039
|
+
references.push(attempt.dispatch.birthContextForkReference);
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
return references;
|
|
1043
|
+
}
|
|
616
1044
|
buildWorkerExecutionPlan(profile, settings, requestedReadPaths, requestedWritePaths) {
|
|
617
1045
|
return buildWorkerExecutionPlan({
|
|
618
1046
|
profile,
|
|
@@ -637,21 +1065,102 @@ export class WorkerDelegationController {
|
|
|
637
1065
|
};
|
|
638
1066
|
}
|
|
639
1067
|
const goal = this.deps.getGoalStateSnapshot();
|
|
640
|
-
const
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
1068
|
+
const mode = this.workerContextForkMode(request, admission.executionContract);
|
|
1069
|
+
const messages = mode.kind === "none" ? [] : selectSanitizedContextFork(this.workerContextForkSource(request).messages, mode);
|
|
1070
|
+
for (let allocation = 0; allocation < DEFAULT_WORKER_FLEET_LIMITS.maxAgentsPerSession; allocation++) {
|
|
1071
|
+
const laneId = lifecycle.getNextAvailableLaneIdCandidate();
|
|
1072
|
+
let proposedReference;
|
|
1073
|
+
try {
|
|
1074
|
+
const captured = this.contextForks.captureAndPrepare({
|
|
1075
|
+
logicalAgentId: laneId,
|
|
1076
|
+
messages,
|
|
1077
|
+
readDurableReferences: () => this.durableWorkerContextForkReferences(),
|
|
1078
|
+
isLogicalIdentityClaimed: () => {
|
|
1079
|
+
const snapshot = lifecycle.getTaskRuntimeSnapshot();
|
|
1080
|
+
return (snapshot.tasks[laneId]?.attemptIds.some((attemptId) => snapshot.attempts[attemptId] !== undefined) ?? false);
|
|
1081
|
+
},
|
|
1082
|
+
prepare: (birthContextForkReference) => {
|
|
1083
|
+
proposedReference = birthContextForkReference;
|
|
1084
|
+
return lifecycle.prepare({
|
|
1085
|
+
instructions: admission.instructions,
|
|
1086
|
+
...(request.parentAgentId ? { parentAgentId: request.parentAgentId } : {}),
|
|
1087
|
+
birthContextForkReference,
|
|
1088
|
+
executionContract: admission.executionContract,
|
|
1089
|
+
requiredCapabilities: admission.executionPlan.requiredCapabilities,
|
|
1090
|
+
...(request.verificationOfTaskId ? { verificationOfTaskId: request.verificationOfTaskId } : {}),
|
|
1091
|
+
taskContext: {
|
|
1092
|
+
requirementIds: request.taskContext?.requirementIds ?? [],
|
|
1093
|
+
dependsOnTaskIds: request.taskContext?.dependsOnTaskIds ?? [],
|
|
1094
|
+
acceptanceCriterionIds: request.taskContext?.acceptanceCriterionIds ?? [],
|
|
1095
|
+
resourcePointerIds: admission.resourcePointerIds,
|
|
1096
|
+
},
|
|
1097
|
+
...(goal ? { goal } : {}),
|
|
1098
|
+
}, laneId);
|
|
1099
|
+
},
|
|
1100
|
+
});
|
|
1101
|
+
return { executionPlan: admission.executionPlan, lifecycle, ...captured.value };
|
|
1102
|
+
}
|
|
1103
|
+
catch (error) {
|
|
1104
|
+
if (error instanceof WorkerContextForkStoreError && error.code === "identity_claimed")
|
|
1105
|
+
continue;
|
|
1106
|
+
const snapshot = lifecycle.getTaskRuntimeSnapshot();
|
|
1107
|
+
const candidateTask = snapshot.tasks[laneId];
|
|
1108
|
+
const candidateAttempts = candidateTask?.attemptIds.map((attemptId) => snapshot.attempts[attemptId]) ?? [];
|
|
1109
|
+
const candidateOwnsDurableAttempt = candidateAttempts.some((attempt) => attempt !== undefined);
|
|
1110
|
+
const sameRequestReplay = candidateAttempts.some((attempt) => attempt !== undefined &&
|
|
1111
|
+
attempt.dispatch.instructions === admission.instructions &&
|
|
1112
|
+
attempt.dispatch.parentAgentId === request.parentAgentId &&
|
|
1113
|
+
isDeepStrictEqual(attempt.dispatch.executionContract, admission.executionContract) &&
|
|
1114
|
+
(proposedReference === undefined ||
|
|
1115
|
+
isDeepStrictEqual(attempt.dispatch.birthContextForkReference, proposedReference)));
|
|
1116
|
+
const corruptSnapshot = error instanceof WorkerContextForkStoreError &&
|
|
1117
|
+
(error.code === "snapshot_corrupt" || error.code === "snapshot_missing");
|
|
1118
|
+
const generatedIdWasConcurrentlyClaimed = candidateOwnsDurableAttempt && !sameRequestReplay && !corruptSnapshot;
|
|
1119
|
+
if (!generatedIdWasConcurrentlyClaimed)
|
|
1120
|
+
throw error;
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
throw new Error("Worker logical-agent allocation retries were exhausted.");
|
|
1124
|
+
}
|
|
1125
|
+
enqueuePreparedWorkerAttempt(record, request, profileId, priority) {
|
|
1126
|
+
try {
|
|
1127
|
+
this.scheduler.enqueue(record, { ...request, profileId }, false, priority);
|
|
1128
|
+
return undefined;
|
|
1129
|
+
}
|
|
1130
|
+
catch (error) {
|
|
1131
|
+
const reasonCode = error instanceof Error && error.message === "worker_dispatch_queue_full"
|
|
1132
|
+
? "worker_dispatch_queue_full"
|
|
1133
|
+
: "worker_dispatch_enqueue_error";
|
|
1134
|
+
this.cancelAndPublish(this.lifecycle, record.laneId, reasonCode);
|
|
1135
|
+
this.safeWarn(`Worker dispatch enqueue failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1136
|
+
return reasonCode;
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
queuePreparedWorkerAttempt(prepared, request, admission, options = {}) {
|
|
1140
|
+
if (this.workerQueueReservationSkipReason(request, 1, 0) ||
|
|
1141
|
+
!this.scheduler.hasQueueCapacity(request.verificationOfTaskId !== undefined)) {
|
|
1142
|
+
this.cancelAndPublish(prepared.lifecycle, prepared.record.laneId, "worker_dispatch_queue_full");
|
|
1143
|
+
return { started: false, skipReason: "worker_dispatch_queue_full" };
|
|
1144
|
+
}
|
|
1145
|
+
if (options.ensureAgent !== false) {
|
|
1146
|
+
try {
|
|
1147
|
+
this.ensurePreparedAgent(prepared, admission);
|
|
1148
|
+
}
|
|
1149
|
+
catch (error) {
|
|
1150
|
+
this.cancelAndPublish(prepared.lifecycle, prepared.record.laneId, "worker_conversation_unavailable");
|
|
1151
|
+
this.safeWarn(`Worker conversation setup failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1152
|
+
return { started: false, skipReason: "worker_conversation_unavailable" };
|
|
1153
|
+
}
|
|
1154
|
+
}
|
|
1155
|
+
const priority = request.verificationOfTaskId !== undefined;
|
|
1156
|
+
const enqueueSkipReason = this.enqueuePreparedWorkerAttempt(prepared.record, request, admission.shipment.profile.profileId, priority);
|
|
1157
|
+
if (enqueueSkipReason)
|
|
1158
|
+
return { started: false, skipReason: enqueueSkipReason };
|
|
1159
|
+
this.notifications.statusChanged();
|
|
1160
|
+
options.onStarted?.(prepared.record);
|
|
1161
|
+
if (options.drain)
|
|
1162
|
+
this.scheduler.drain();
|
|
1163
|
+
return { started: true, record: prepared.record };
|
|
655
1164
|
}
|
|
656
1165
|
ensurePreparedAgent(prepared, admission) {
|
|
657
1166
|
if (!prepared.attempt)
|
|
@@ -660,6 +1169,7 @@ export class WorkerDelegationController {
|
|
|
660
1169
|
if (!immutableWorker)
|
|
661
1170
|
throw new Error("Prepared worker execution contract is missing.");
|
|
662
1171
|
const agentId = prepared.attempt.agentId ?? prepared.attempt.dispatch.logicalLaneId ?? prepared.record.laneId;
|
|
1172
|
+
const birthContextForkReference = prepared.attempt.dispatch.birthContextForkReference;
|
|
663
1173
|
const existing = prepared.lifecycle.getAgent(agentId);
|
|
664
1174
|
if (existing) {
|
|
665
1175
|
const conversation = this.conversations.open({
|
|
@@ -667,6 +1177,9 @@ export class WorkerDelegationController {
|
|
|
667
1177
|
resumeContext: existing.resumeContext,
|
|
668
1178
|
expectedLogicalAgentId: existing.agentId,
|
|
669
1179
|
});
|
|
1180
|
+
if (!isDeepStrictEqual(conversation.getBirthContextForkReference(), birthContextForkReference)) {
|
|
1181
|
+
throw new Error("Worker conversation birth context conflicts with its durable attempt.");
|
|
1182
|
+
}
|
|
670
1183
|
const materialized = materializeWorkerResourceBundle(existing.resumeContext.contextPointers);
|
|
671
1184
|
if (!materialized.ok)
|
|
672
1185
|
throw new Error(`worker_resource_materialization_${materialized.code}`);
|
|
@@ -691,6 +1204,7 @@ export class WorkerDelegationController {
|
|
|
691
1204
|
modelRef: `${immutableWorker.modelBinding.provider}/${immutableWorker.modelBinding.modelId}`,
|
|
692
1205
|
resourceProfileNames: immutableWorker.profile.resourceProfileNames,
|
|
693
1206
|
contextPointers: materialized.pointers,
|
|
1207
|
+
...(birthContextForkReference ? { birthContextForkReference } : {}),
|
|
694
1208
|
});
|
|
695
1209
|
prepared.lifecycle.ensureAgent({
|
|
696
1210
|
agentId,
|
|
@@ -708,40 +1222,38 @@ export class WorkerDelegationController {
|
|
|
708
1222
|
return this.startInternal(request);
|
|
709
1223
|
}
|
|
710
1224
|
startInternal(request, pinnedContract) {
|
|
711
|
-
const admission = this.
|
|
1225
|
+
const admission = this.admitNewWorkerRequest(request, pinnedContract);
|
|
712
1226
|
if (!admission.ok)
|
|
713
1227
|
return { started: false, skipReason: admission.skipReason };
|
|
714
|
-
if (!request.verificationOfTaskId)
|
|
715
|
-
this.recovery.recover();
|
|
716
1228
|
const { settings, shipment } = admission;
|
|
717
1229
|
const foreground = this.deps.getModel();
|
|
718
1230
|
const contendsWithLocalForeground = foreground !== undefined && isLocalExecutionModel(foreground) && isLocalExecutionModel(shipment.model);
|
|
719
|
-
|
|
1231
|
+
const dependencyGated = (request.taskContext?.dependsOnTaskIds.length ?? 0) > 0;
|
|
1232
|
+
if (dependencyGated || contendsWithLocalForeground || !this.hasWorkerCapacity(settings)) {
|
|
720
1233
|
// A mandatory verifier is the continuation of an already admitted implementation, not a
|
|
721
1234
|
// new owner request. Reserve its queue admission so a burst of ordinary work cannot strand
|
|
722
1235
|
// the subject behind `independent_verification_required` with no terminal handoff.
|
|
723
|
-
|
|
1236
|
+
const priority = request.verificationOfTaskId !== undefined;
|
|
1237
|
+
if (this.workerQueueReservationSkipReason(request, 1, admission.verifierShipment ? 1 : 0) ||
|
|
1238
|
+
!this.scheduler.hasQueueCapacity(priority)) {
|
|
1239
|
+
return { started: false, skipReason: "worker_dispatch_queue_full" };
|
|
1240
|
+
}
|
|
1241
|
+
let prepared;
|
|
724
1242
|
try {
|
|
725
|
-
|
|
726
|
-
record = prepared.record;
|
|
727
|
-
this.ensurePreparedAgent(prepared, admission);
|
|
1243
|
+
prepared = this.prepareWorkerAttempt(request, admission);
|
|
728
1244
|
}
|
|
729
1245
|
catch (error) {
|
|
730
|
-
if (record)
|
|
731
|
-
this.cancelAndPublish(this.lifecycle, record.laneId, "worker_conversation_unavailable");
|
|
732
1246
|
this.safeWarn(`Worker dispatch was not persisted: ${error instanceof Error ? error.message : String(error)}`);
|
|
733
1247
|
return { started: false, skipReason: "orchestration_ledger_error" };
|
|
734
1248
|
}
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
this.notifications.statusChanged();
|
|
739
|
-
return { started: true, record };
|
|
1249
|
+
return this.queuePreparedWorkerAttempt(prepared, request, admission, {
|
|
1250
|
+
drain: dependencyGated,
|
|
1251
|
+
});
|
|
740
1252
|
}
|
|
741
1253
|
let startedRecord;
|
|
742
1254
|
const promise = this.runOnceWithAdmission(request, (record) => {
|
|
743
1255
|
startedRecord = record;
|
|
744
|
-
}, undefined, admission);
|
|
1256
|
+
}, undefined, admission, true);
|
|
745
1257
|
if (!startedRecord) {
|
|
746
1258
|
// Preparation is synchronous up to the first isolated completion await. A promise that
|
|
747
1259
|
// rejected before producing a lane is still observed below, so it cannot become unhandled.
|
|
@@ -754,14 +1266,18 @@ export class WorkerDelegationController {
|
|
|
754
1266
|
async runOnce(request, onStarted, existingRecord) {
|
|
755
1267
|
return this.runOnceWithAdmission(request, onStarted, existingRecord);
|
|
756
1268
|
}
|
|
757
|
-
async runOnceWithAdmission(request, onStarted, existingRecord, preparedAdmission) {
|
|
1269
|
+
async runOnceWithAdmission(request, onStarted, existingRecord, preparedAdmission, newWorkerAdmissionChecked = false) {
|
|
758
1270
|
const pinnedContract = existingRecord
|
|
759
1271
|
? this.getWorkerLifecycle().getActiveAttempt(existingRecord.laneId)?.dispatch.executionContract
|
|
760
1272
|
: undefined;
|
|
761
|
-
const admission =
|
|
1273
|
+
const admission = existingRecord
|
|
1274
|
+
? this.resolveWorkerAdmission(request, pinnedContract)
|
|
1275
|
+
: preparedAdmission && newWorkerAdmissionChecked
|
|
1276
|
+
? preparedAdmission
|
|
1277
|
+
: this.admitNewWorkerRequest(request, pinnedContract);
|
|
762
1278
|
if (!admission.ok)
|
|
763
1279
|
return { started: false, skipReason: admission.skipReason };
|
|
764
|
-
if (!request.verificationOfTaskId)
|
|
1280
|
+
if (existingRecord && !request.verificationOfTaskId)
|
|
765
1281
|
this.recovery.recover();
|
|
766
1282
|
const { instructions, settings, verifierShipment } = admission;
|
|
767
1283
|
const { model, modelBinding, profile: orchestrationProfile, soul } = admission.shipment;
|
|
@@ -774,6 +1290,16 @@ export class WorkerDelegationController {
|
|
|
774
1290
|
const { executionPlan, lifecycle } = prepared;
|
|
775
1291
|
if (!prepared.attempt)
|
|
776
1292
|
return { started: false, skipReason: "orchestration_attempt_missing" };
|
|
1293
|
+
if (prepared.attempt.status === "queued") {
|
|
1294
|
+
const readiness = lifecycle.getAttemptDispatchReadiness(prepared.attempt.attemptId);
|
|
1295
|
+
if (readiness.state === "blocked") {
|
|
1296
|
+
this.cancelAndPublish(lifecycle, prepared.record.laneId, readiness.reasonCode);
|
|
1297
|
+
return { started: false, skipReason: readiness.reasonCode };
|
|
1298
|
+
}
|
|
1299
|
+
if (readiness.state === "waiting") {
|
|
1300
|
+
return this.queuePreparedWorkerAttempt(prepared, request, admission, { onStarted });
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
777
1303
|
let preparedAgent;
|
|
778
1304
|
try {
|
|
779
1305
|
preparedAgent = this.ensurePreparedAgent(prepared, admission);
|
|
@@ -826,6 +1352,16 @@ export class WorkerDelegationController {
|
|
|
826
1352
|
return { started: false, skipReason: "orchestration_agent_missing" };
|
|
827
1353
|
}
|
|
828
1354
|
const conversation = preparedAgent.conversation;
|
|
1355
|
+
if (prepared.attempt.status === "queued" || conversation.usesAttemptUsageBoundaries()) {
|
|
1356
|
+
try {
|
|
1357
|
+
conversation.beginAttemptUsage(prepared.attempt.attemptId);
|
|
1358
|
+
}
|
|
1359
|
+
catch (error) {
|
|
1360
|
+
this.cancelAndPublish(lifecycle, prepared.record.laneId, "worker_conversation_unavailable");
|
|
1361
|
+
this.safeWarn(`Worker usage boundary setup failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1362
|
+
return { started: false, skipReason: "worker_conversation_unavailable" };
|
|
1363
|
+
}
|
|
1364
|
+
}
|
|
829
1365
|
if (prepared.attempt.status === "suspended")
|
|
830
1366
|
this.recovery.repairInterruptedToolResults(conversation);
|
|
831
1367
|
const reservation = this.writeReservations.acquire(prepared.record.laneId, prepared.attempt, executionPlan);
|
|
@@ -834,10 +1370,10 @@ export class WorkerDelegationController {
|
|
|
834
1370
|
return { started: false, skipReason: reservation.reasonCode };
|
|
835
1371
|
}
|
|
836
1372
|
if (reservation.kind === "blocked") {
|
|
837
|
-
this.
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
1373
|
+
return this.queuePreparedWorkerAttempt(prepared, request, admission, {
|
|
1374
|
+
ensureAgent: false,
|
|
1375
|
+
onStarted,
|
|
1376
|
+
});
|
|
841
1377
|
}
|
|
842
1378
|
let durableHandle;
|
|
843
1379
|
try {
|
|
@@ -864,9 +1400,11 @@ export class WorkerDelegationController {
|
|
|
864
1400
|
this.safeWarn("Worker write reservation fence did not match the durable attempt lease.");
|
|
865
1401
|
return { started: false, skipReason: record?.reasonCode ?? "write_reservation_fence_mismatch" };
|
|
866
1402
|
}
|
|
867
|
-
const recoveredTerminal = prepared.attempt.status === "suspended"
|
|
1403
|
+
const recoveredTerminal = prepared.attempt.status === "suspended"
|
|
1404
|
+
? this.recovery.recoveredTerminalCompletion(conversation, durableHandle.attemptId)
|
|
1405
|
+
: undefined;
|
|
868
1406
|
const checkpointUsage = lifecycle.getAttemptUsage(startedRecord.laneId);
|
|
869
|
-
const initialUsage = this.recovery.initialUsage(conversation, checkpointUsage);
|
|
1407
|
+
const initialUsage = this.recovery.initialUsage(conversation, checkpointUsage, durableHandle.attemptId);
|
|
870
1408
|
onStarted?.(startedRecord);
|
|
871
1409
|
const maxUsd = grant.budget.maxCostUsd;
|
|
872
1410
|
const executionPolicy = orchestrationProfile.executionPolicy;
|
|
@@ -875,7 +1413,11 @@ export class WorkerDelegationController {
|
|
|
875
1413
|
runWorkerDelegation: (childRequest) => this.runOnce({ ...childRequest, parentAgentId: agentId }),
|
|
876
1414
|
orchestrationProfiles: this.getProfileCatalog(),
|
|
877
1415
|
workerAgentControl: this.agentControl,
|
|
878
|
-
|
|
1416
|
+
caller: { kind: "worker", agentId },
|
|
1417
|
+
resolveMessageReplayScope: () => ({
|
|
1418
|
+
sessionId: this.deps.getSessionId(),
|
|
1419
|
+
branchId: durableHandle.attemptId,
|
|
1420
|
+
}),
|
|
879
1421
|
}));
|
|
880
1422
|
const agentBinding = lifecycle.getAgent(agentId);
|
|
881
1423
|
if (!agentBinding) {
|
|
@@ -893,6 +1435,9 @@ export class WorkerDelegationController {
|
|
|
893
1435
|
});
|
|
894
1436
|
const shellGranted = executionPlan.toolManifests.some((manifest) => manifest.toolName === STABLE_SHELL_TOOL_NAME);
|
|
895
1437
|
const shellSessionKey = shellGranted ? `worker:${this.deps.getSessionId()}:${agentId}` : undefined;
|
|
1438
|
+
const shellOutputDirectory = shellGranted
|
|
1439
|
+
? getProcessWorkRun(this.deps.getAgentDir(), "outputs", "tool-streams").path
|
|
1440
|
+
: undefined;
|
|
896
1441
|
if (shellSessionKey)
|
|
897
1442
|
this.shellSessionKeys.add(shellSessionKey);
|
|
898
1443
|
const toolSurface = createLaneToolSurface({
|
|
@@ -903,6 +1448,7 @@ export class WorkerDelegationController {
|
|
|
903
1448
|
...(executionPlan.processEnabled && executionPolicy ? { executionPolicy } : {}),
|
|
904
1449
|
processMaxWallClockMs: grant.budget.maxWallClockMs ?? 0,
|
|
905
1450
|
...(shellSessionKey ? { shellSessionKey } : {}),
|
|
1451
|
+
...(shellOutputDirectory ? { shellOutputDirectory } : {}),
|
|
906
1452
|
grant,
|
|
907
1453
|
toolManifests: executionPlan.toolManifests,
|
|
908
1454
|
additionalTools: [recursiveDelegateTool],
|
|
@@ -1007,6 +1553,23 @@ export class WorkerDelegationController {
|
|
|
1007
1553
|
});
|
|
1008
1554
|
const executionResult = await executor.run();
|
|
1009
1555
|
const rawOutcome = executionResult.rawOutcome;
|
|
1556
|
+
// Attempt ladder: a retryable bounded failure suspends and re-enqueues instead of
|
|
1557
|
+
// terminalizing, resuming from the persisted transcript under a fresh fence.
|
|
1558
|
+
if (rawOutcome.laneStatus === "failed" && !this.deps.isDisposed() && !workerSignal.aborted) {
|
|
1559
|
+
const retry = this.recovery.scheduleAttemptRetry({
|
|
1560
|
+
laneId: startedRecord.laneId,
|
|
1561
|
+
agentId,
|
|
1562
|
+
ownerId: this.agentControl.getProcessOwnerId(),
|
|
1563
|
+
request: { ...request, profileId: admission.shipment.profile.profileId },
|
|
1564
|
+
outcome: rawOutcome,
|
|
1565
|
+
provider: modelBinding.provider,
|
|
1566
|
+
...(grant.budget.maxAttempts !== undefined ? { maxAttempts: grant.budget.maxAttempts } : {}),
|
|
1567
|
+
});
|
|
1568
|
+
if (retry.scheduled) {
|
|
1569
|
+
this.notifications.statusChanged();
|
|
1570
|
+
return { started: true, record: retry.record };
|
|
1571
|
+
}
|
|
1572
|
+
}
|
|
1010
1573
|
const verificationRequired = orchestrationProfile.requireIndependentVerification &&
|
|
1011
1574
|
orchestrationProfile.role !== "verifier" &&
|
|
1012
1575
|
rawOutcome.claim.status === "completed";
|
|
@@ -1189,6 +1752,7 @@ export class WorkerDelegationController {
|
|
|
1189
1752
|
drain() {
|
|
1190
1753
|
this.recovery.recover();
|
|
1191
1754
|
this.scheduler.drain();
|
|
1755
|
+
this.terminalHandoffs.signal();
|
|
1192
1756
|
}
|
|
1193
1757
|
}
|
|
1194
1758
|
//# sourceMappingURL=worker-delegation-controller.js.map
|