@caupulican/pi-adaptative 0.86.3 → 0.86.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/core/agent-paths.d.ts +13 -0
- package/dist/core/agent-paths.d.ts.map +1 -1
- package/dist/core/agent-paths.js +23 -0
- package/dist/core/agent-paths.js.map +1 -1
- package/dist/core/autonomy/bounded-completion.d.ts +7 -0
- package/dist/core/autonomy/bounded-completion.d.ts.map +1 -1
- package/dist/core/autonomy/bounded-completion.js +12 -4
- package/dist/core/autonomy/bounded-completion.js.map +1 -1
- package/dist/core/autonomy/lane-tool-surface.d.ts +2 -0
- package/dist/core/autonomy/lane-tool-surface.d.ts.map +1 -1
- package/dist/core/autonomy/lane-tool-surface.js +6 -3
- package/dist/core/autonomy/lane-tool-surface.js.map +1 -1
- package/dist/core/autonomy/subagent-prompt.js +1 -1
- package/dist/core/autonomy/subagent-prompt.js.map +1 -1
- package/dist/core/background-lane-controller.d.ts +25 -2
- package/dist/core/background-lane-controller.d.ts.map +1 -1
- package/dist/core/background-lane-controller.js +67 -0
- package/dist/core/background-lane-controller.js.map +1 -1
- package/dist/core/delegation/replay-safe-mailbox-bounds.d.ts +4 -0
- package/dist/core/delegation/replay-safe-mailbox-bounds.d.ts.map +1 -0
- package/dist/core/delegation/replay-safe-mailbox-bounds.js +20 -0
- package/dist/core/delegation/replay-safe-mailbox-bounds.js.map +1 -0
- package/dist/core/delegation/sanitized-context-fork.d.ts +25 -0
- package/dist/core/delegation/sanitized-context-fork.d.ts.map +1 -0
- package/dist/core/delegation/sanitized-context-fork.js +299 -0
- package/dist/core/delegation/sanitized-context-fork.js.map +1 -0
- package/dist/core/delegation/session-root-mailbox.d.ts +75 -0
- package/dist/core/delegation/session-root-mailbox.d.ts.map +1 -0
- package/dist/core/delegation/session-root-mailbox.js +627 -0
- package/dist/core/delegation/session-root-mailbox.js.map +1 -0
- package/dist/core/delegation/worker-agent-control-coordinator.d.ts +102 -9
- package/dist/core/delegation/worker-agent-control-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-agent-control-coordinator.js +1307 -125
- package/dist/core/delegation/worker-agent-control-coordinator.js.map +1 -1
- package/dist/core/delegation/worker-agent-control.d.ts +183 -8
- package/dist/core/delegation/worker-agent-control.d.ts.map +1 -1
- package/dist/core/delegation/worker-agent-control.js +778 -48
- package/dist/core/delegation/worker-agent-control.js.map +1 -1
- package/dist/core/delegation/worker-attempt-executor.d.ts +9 -0
- package/dist/core/delegation/worker-attempt-executor.d.ts.map +1 -1
- package/dist/core/delegation/worker-attempt-executor.js +479 -214
- package/dist/core/delegation/worker-attempt-executor.js.map +1 -1
- package/dist/core/delegation/worker-authority-resolver.d.ts +9 -0
- package/dist/core/delegation/worker-authority-resolver.d.ts.map +1 -1
- package/dist/core/delegation/worker-authority-resolver.js +21 -2
- package/dist/core/delegation/worker-authority-resolver.js.map +1 -1
- package/dist/core/delegation/worker-context-fork-store.d.ts +71 -0
- package/dist/core/delegation/worker-context-fork-store.d.ts.map +1 -0
- package/dist/core/delegation/worker-context-fork-store.js +437 -0
- package/dist/core/delegation/worker-context-fork-store.js.map +1 -0
- package/dist/core/delegation/worker-context-inheritance-policy.d.ts +18 -0
- package/dist/core/delegation/worker-context-inheritance-policy.d.ts.map +1 -0
- package/dist/core/delegation/worker-context-inheritance-policy.js +28 -0
- package/dist/core/delegation/worker-context-inheritance-policy.js.map +1 -0
- package/dist/core/delegation/worker-conversation-revision.d.ts +31 -0
- package/dist/core/delegation/worker-conversation-revision.d.ts.map +1 -0
- package/dist/core/delegation/worker-conversation-revision.js +203 -0
- package/dist/core/delegation/worker-conversation-revision.js.map +1 -0
- package/dist/core/delegation/worker-conversation-store.d.ts +130 -18
- package/dist/core/delegation/worker-conversation-store.d.ts.map +1 -1
- package/dist/core/delegation/worker-conversation-store.js +1263 -79
- package/dist/core/delegation/worker-conversation-store.js.map +1 -1
- package/dist/core/delegation/worker-delegation-controller.d.ts +29 -2
- package/dist/core/delegation/worker-delegation-controller.d.ts.map +1 -1
- package/dist/core/delegation/worker-delegation-controller.js +625 -111
- package/dist/core/delegation/worker-delegation-controller.js.map +1 -1
- package/dist/core/delegation/worker-delegation-request.d.ts +2 -0
- package/dist/core/delegation/worker-delegation-request.d.ts.map +1 -1
- package/dist/core/delegation/worker-delegation-request.js.map +1 -1
- package/dist/core/delegation/worker-dispatch-scheduler.d.ts +20 -1
- package/dist/core/delegation/worker-dispatch-scheduler.d.ts.map +1 -1
- package/dist/core/delegation/worker-dispatch-scheduler.js +210 -37
- package/dist/core/delegation/worker-dispatch-scheduler.js.map +1 -1
- package/dist/core/delegation/worker-fleet-limits.d.ts +38 -0
- package/dist/core/delegation/worker-fleet-limits.d.ts.map +1 -0
- package/dist/core/delegation/worker-fleet-limits.js +113 -0
- package/dist/core/delegation/worker-fleet-limits.js.map +1 -0
- package/dist/core/delegation/worker-lane-projection.d.ts +1 -0
- package/dist/core/delegation/worker-lane-projection.d.ts.map +1 -1
- package/dist/core/delegation/worker-lane-projection.js +6 -2
- package/dist/core/delegation/worker-lane-projection.js.map +1 -1
- package/dist/core/delegation/worker-lifecycle.d.ts +20 -3
- package/dist/core/delegation/worker-lifecycle.d.ts.map +1 -1
- package/dist/core/delegation/worker-lifecycle.js +42 -18
- package/dist/core/delegation/worker-lifecycle.js.map +1 -1
- package/dist/core/delegation/worker-notification-coordinator.d.ts +4 -0
- package/dist/core/delegation/worker-notification-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-notification-coordinator.js +45 -4
- package/dist/core/delegation/worker-notification-coordinator.js.map +1 -1
- package/dist/core/delegation/worker-provider-turn-protocol.d.ts +50 -0
- package/dist/core/delegation/worker-provider-turn-protocol.d.ts.map +1 -0
- package/dist/core/delegation/worker-provider-turn-protocol.js +185 -0
- package/dist/core/delegation/worker-provider-turn-protocol.js.map +1 -0
- package/dist/core/delegation/worker-recovery-coordinator.d.ts +45 -4
- package/dist/core/delegation/worker-recovery-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-recovery-coordinator.js +223 -17
- package/dist/core/delegation/worker-recovery-coordinator.js.map +1 -1
- package/dist/core/delegation/worker-retry-policy.d.ts +36 -0
- package/dist/core/delegation/worker-retry-policy.d.ts.map +1 -0
- package/dist/core/delegation/worker-retry-policy.js +41 -0
- package/dist/core/delegation/worker-retry-policy.js.map +1 -0
- package/dist/core/delegation/worker-runner.d.ts +3 -1
- package/dist/core/delegation/worker-runner.d.ts.map +1 -1
- package/dist/core/delegation/worker-runner.js +4 -2
- package/dist/core/delegation/worker-runner.js.map +1 -1
- package/dist/core/delegation/worker-task-view.d.ts +43 -0
- package/dist/core/delegation/worker-task-view.d.ts.map +1 -0
- package/dist/core/delegation/worker-task-view.js +194 -0
- package/dist/core/delegation/worker-task-view.js.map +1 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.d.ts +55 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.d.ts.map +1 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.js +179 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.js.map +1 -0
- package/dist/core/delegation/worker-tree-budget-coordinator.d.ts +2 -2
- package/dist/core/delegation/worker-tree-budget-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-tree-budget-coordinator.js +12 -46
- package/dist/core/delegation/worker-tree-budget-coordinator.js.map +1 -1
- package/dist/core/keybindings.d.ts +10 -0
- package/dist/core/keybindings.d.ts.map +1 -1
- package/dist/core/keybindings.js +2 -0
- package/dist/core/keybindings.js.map +1 -1
- package/dist/core/orchestration/attempt-ordering.d.ts +9 -0
- package/dist/core/orchestration/attempt-ordering.d.ts.map +1 -0
- package/dist/core/orchestration/attempt-ordering.js +29 -0
- package/dist/core/orchestration/attempt-ordering.js.map +1 -0
- package/dist/core/orchestration/capability-gateway.d.ts +19 -0
- package/dist/core/orchestration/capability-gateway.d.ts.map +1 -1
- package/dist/core/orchestration/capability-gateway.js +19 -2
- package/dist/core/orchestration/capability-gateway.js.map +1 -1
- package/dist/core/orchestration/contracts.d.ts +43 -3
- package/dist/core/orchestration/contracts.d.ts.map +1 -1
- package/dist/core/orchestration/contracts.js +24 -0
- package/dist/core/orchestration/contracts.js.map +1 -1
- package/dist/core/orchestration/delegation-ledger.d.ts +8 -3
- package/dist/core/orchestration/delegation-ledger.d.ts.map +1 -1
- package/dist/core/orchestration/delegation-ledger.js +148 -55
- package/dist/core/orchestration/delegation-ledger.js.map +1 -1
- package/dist/core/orchestration/event-store.d.ts +6 -3
- package/dist/core/orchestration/event-store.d.ts.map +1 -1
- package/dist/core/orchestration/event-store.js +228 -30
- package/dist/core/orchestration/event-store.js.map +1 -1
- package/dist/core/orchestration/task-runtime-codecs.d.ts +31 -0
- package/dist/core/orchestration/task-runtime-codecs.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-codecs.js +1471 -0
- package/dist/core/orchestration/task-runtime-codecs.js.map +1 -0
- package/dist/core/orchestration/task-runtime-projection.d.ts +30 -0
- package/dist/core/orchestration/task-runtime-projection.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-projection.js +340 -0
- package/dist/core/orchestration/task-runtime-projection.js.map +1 -0
- package/dist/core/orchestration/task-runtime-reducer.d.ts +38 -0
- package/dist/core/orchestration/task-runtime-reducer.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-reducer.js +1270 -0
- package/dist/core/orchestration/task-runtime-reducer.js.map +1 -0
- package/dist/core/orchestration/task-runtime-state.d.ts +142 -0
- package/dist/core/orchestration/task-runtime-state.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-state.js +14 -0
- package/dist/core/orchestration/task-runtime-state.js.map +1 -0
- package/dist/core/orchestration/task-runtime.d.ts +44 -98
- package/dist/core/orchestration/task-runtime.d.ts.map +1 -1
- package/dist/core/orchestration/task-runtime.js +357 -1113
- package/dist/core/orchestration/task-runtime.js.map +1 -1
- package/dist/core/orchestration/worker-context-fork-reference.d.ts +20 -0
- package/dist/core/orchestration/worker-context-fork-reference.d.ts.map +1 -0
- package/dist/core/orchestration/worker-context-fork-reference.js +53 -0
- package/dist/core/orchestration/worker-context-fork-reference.js.map +1 -0
- package/dist/core/research/research-runner.d.ts +2 -0
- package/dist/core/research/research-runner.d.ts.map +1 -1
- package/dist/core/research/research-runner.js +1 -0
- package/dist/core/research/research-runner.js.map +1 -1
- package/dist/core/runtime-builder.d.ts.map +1 -1
- package/dist/core/runtime-builder.js +8 -0
- package/dist/core/runtime-builder.js.map +1 -1
- package/dist/core/secrets/credential-project.d.ts.map +1 -1
- package/dist/core/secrets/credential-project.js +24 -2
- package/dist/core/secrets/credential-project.js.map +1 -1
- package/dist/core/security/secret-text.d.ts +3 -0
- package/dist/core/security/secret-text.d.ts.map +1 -1
- package/dist/core/security/secret-text.js +15 -0
- package/dist/core/security/secret-text.js.map +1 -1
- package/dist/core/settings-manager.d.ts +3 -0
- package/dist/core/settings-manager.d.ts.map +1 -1
- package/dist/core/settings-manager.js +160 -8
- package/dist/core/settings-manager.js.map +1 -1
- package/dist/core/tools/delegate.d.ts +31 -4
- package/dist/core/tools/delegate.d.ts.map +1 -1
- package/dist/core/tools/delegate.js +749 -103
- package/dist/core/tools/delegate.js.map +1 -1
- package/dist/main.d.ts.map +1 -1
- package/dist/main.js +1 -0
- package/dist/main.js.map +1 -1
- package/dist/modes/interactive/components/activity-lane.d.ts +5 -0
- package/dist/modes/interactive/components/activity-lane.d.ts.map +1 -1
- package/dist/modes/interactive/components/activity-lane.js +133 -49
- package/dist/modes/interactive/components/activity-lane.js.map +1 -1
- package/dist/modes/interactive/components/agents-overlay.d.ts +40 -0
- package/dist/modes/interactive/components/agents-overlay.d.ts.map +1 -0
- package/dist/modes/interactive/components/agents-overlay.js +153 -0
- package/dist/modes/interactive/components/agents-overlay.js.map +1 -0
- package/dist/modes/interactive/components/settings-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/settings-selector.js +18 -13
- package/dist/modes/interactive/components/settings-selector.js.map +1 -1
- package/dist/modes/interactive/interactive-event-controller.d.ts +1 -1
- package/dist/modes/interactive/interactive-event-controller.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-event-controller.js +6 -3
- package/dist/modes/interactive/interactive-event-controller.js.map +1 -1
- package/dist/modes/interactive/interactive-mode.d.ts +5 -0
- package/dist/modes/interactive/interactive-mode.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-mode.js +51 -0
- package/dist/modes/interactive/interactive-mode.js.map +1 -1
- package/dist/modes/interactive/key-handlers.d.ts +1 -0
- package/dist/modes/interactive/key-handlers.d.ts.map +1 -1
- package/dist/modes/interactive/key-handlers.js +1 -0
- package/dist/modes/interactive/key-handlers.js.map +1 -1
- package/docs/harness-architecture.md +61 -17
- package/docs/settings.md +17 -9
- package/docs/worker-profiles.md +6 -2
- package/examples/extensions/custom-provider-anthropic/package-lock.json +2 -2
- package/examples/extensions/custom-provider-anthropic/package.json +1 -1
- package/examples/extensions/custom-provider-gitlab-duo/package.json +1 -1
- package/examples/extensions/sandbox/package-lock.json +2 -2
- package/examples/extensions/sandbox/package.json +1 -1
- package/examples/extensions/with-deps/package-lock.json +2 -2
- package/examples/extensions/with-deps/package.json +1 -1
- package/npm-shrinkwrap.json +12 -12
- package/package.json +4 -4
|
@@ -1,12 +1,16 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { isDeepStrictEqual } from "node:util";
|
|
3
|
+
import { getProcessWorkRun } from "../agent-paths.js";
|
|
2
4
|
import { getPrivateLaneDeniedPaths } from "../autonomy/lane-private-paths.js";
|
|
3
5
|
import { createLaneToolSurface } from "../autonomy/lane-tool-surface.js";
|
|
6
|
+
import { isLaneTerminalStatus } from "../autonomy/lane-tracker.js";
|
|
4
7
|
import { appendLaneRecordSnapshot } from "../autonomy/session-lane-record.js";
|
|
5
8
|
import { AUTONOMY_TELEMETRY_EVENT_TYPES } from "../autonomy/telemetry-events.js";
|
|
6
9
|
import { STABLE_SHELL_TOOL_NAME } from "../default-tool-surface.js";
|
|
7
10
|
import { deriveModelCapabilityProfile } from "../model-capability.js";
|
|
8
11
|
import { isLoopbackModelEndpoint } from "../models/model-endpoint.js";
|
|
9
12
|
import { providerUsageFromAttemptUsage } from "../orchestration/attempt-usage.js";
|
|
13
|
+
import { MAX_ORCHESTRATION_DISPATCH_INSTRUCTIONS_LENGTH, } from "../orchestration/contracts.js";
|
|
10
14
|
import { SessionTaskProfileStore } from "../orchestration/session-task-profile-store.js";
|
|
11
15
|
import { TaskProfileWriter, } from "../orchestration/task-profile-writer.js";
|
|
12
16
|
import { createWorkerExecutionContract, verifierWorkerExecutionContract, } from "../orchestration/worker-execution-contract.js";
|
|
@@ -14,20 +18,25 @@ import { registerInFlightWork } from "../reload-blockers.js";
|
|
|
14
18
|
import { createDelegateToolDefinition } from "../tools/delegate.js";
|
|
15
19
|
import { disposePersistentShellSession } from "../tools/shell-session.js";
|
|
16
20
|
import { wrapToolDefinition } from "../tools/tool-definition-wrapper.js";
|
|
21
|
+
import { selectSanitizedContextFork } from "./sanitized-context-fork.js";
|
|
17
22
|
import { applyWorkerActions } from "./worker-actions.js";
|
|
18
23
|
import { WorkerAgentControlCoordinator } from "./worker-agent-control-coordinator.js";
|
|
19
24
|
import { createWorkerAttemptExecutor } from "./worker-attempt-executor.js";
|
|
20
25
|
import { resolveWorkerAuthority } from "./worker-authority-resolver.js";
|
|
26
|
+
import { WorkerContextForkStore, WorkerContextForkStoreError } from "./worker-context-fork-store.js";
|
|
27
|
+
import { resolveWorkerContextInheritanceMode } from "./worker-context-inheritance-policy.js";
|
|
21
28
|
import { WorkerConversationStore, } from "./worker-conversation-store.js";
|
|
22
29
|
import { parseWorkerDelegationAuthorityRequest } from "./worker-delegation-request.js";
|
|
23
30
|
import { WorkerDispatchScheduler } from "./worker-dispatch-scheduler.js";
|
|
24
31
|
import { buildWorkerExecutionPlan, compileWorkerExecutionGrant, narrowWorkerExecutionPlan, workerExecutionAuthorityFromPlan, } from "./worker-execution-policy.js";
|
|
32
|
+
import { DEFAULT_WORKER_FLEET_LIMITS, evaluateNewWorkerAdmission, pendingVerifierSubjectTaskIds, } from "./worker-fleet-limits.js";
|
|
25
33
|
import { createLocalWorkerProcessOwnerId } from "./worker-process-owner.js";
|
|
26
34
|
import { WorkerProfileResolver } from "./worker-profile-resolver.js";
|
|
27
35
|
import { WorkerRecoveryCoordinator } from "./worker-recovery-coordinator.js";
|
|
28
36
|
import { selectWorkerResourcePointers } from "./worker-resource-catalog.js";
|
|
29
37
|
import { materializeWorkerResourceBundle } from "./worker-resource-materializer.js";
|
|
30
38
|
import { finalizeWorkerClaim } from "./worker-terminal-finalizer.js";
|
|
39
|
+
import { WorkerTerminalHandoffCoordinator, } from "./worker-terminal-handoff-coordinator.js";
|
|
31
40
|
import { collectWorkerTreeBudgetSeeds, WorkerTreeBudgetCoordinator } from "./worker-tree-budget-coordinator.js";
|
|
32
41
|
import { WorkerWriteReservationCoordinator } from "./worker-write-reservation-coordinator.js";
|
|
33
42
|
export function isLocalExecutionModel(model) {
|
|
@@ -57,36 +66,39 @@ function workerConversationRetentionPolicy(model, settingsManager) {
|
|
|
57
66
|
keepRecentTokens: Math.max(1, Math.min(settings.keepRecentTokens, Math.floor(contextWindow * 0.5), maxContextTokens - 1)),
|
|
58
67
|
};
|
|
59
68
|
}
|
|
69
|
+
function workerContextModelIdentity(modelRef) {
|
|
70
|
+
if (!modelRef)
|
|
71
|
+
throw new Error("Worker parent resume model is missing.");
|
|
72
|
+
const separator = modelRef.indexOf("/");
|
|
73
|
+
if (separator <= 0 || separator === modelRef.length - 1) {
|
|
74
|
+
throw new Error("Worker parent resume model is invalid.");
|
|
75
|
+
}
|
|
76
|
+
return { provider: modelRef.slice(0, separator), model: modelRef.slice(separator + 1) };
|
|
77
|
+
}
|
|
60
78
|
export class WorkerDelegationController {
|
|
61
79
|
constructor(deps, notifications, lifecycle) {
|
|
62
80
|
this.workerAbort = new AbortController();
|
|
63
81
|
this.laneAbortControllers = new Map();
|
|
64
82
|
this.shellSessionKeys = new Set();
|
|
65
83
|
this.publishedTerminalAttemptIds = new Set();
|
|
66
|
-
this.yieldedCapacityAttemptIds = new
|
|
84
|
+
this.yieldedCapacityAttemptIds = new Map();
|
|
67
85
|
this.conversations = new WorkerConversationStore();
|
|
68
86
|
this.treeBudgets = new WorkerTreeBudgetCoordinator();
|
|
69
87
|
this.inFlightLedgers = new Map();
|
|
70
88
|
this.deps = deps;
|
|
71
89
|
this.notifications = notifications;
|
|
72
90
|
this.lifecycle = lifecycle;
|
|
91
|
+
this.contextForks = new WorkerContextForkStore({
|
|
92
|
+
agentDir: this.deps.getAgentDir(),
|
|
93
|
+
parentSessionId: this.deps.getSessionId(),
|
|
94
|
+
});
|
|
73
95
|
this.scheduler = new WorkerDispatchScheduler({
|
|
74
96
|
agentDir: this.deps.getAgentDir?.() ?? "",
|
|
75
97
|
isDisposed: () => this.deps.isDisposed(),
|
|
76
98
|
admit: (request, record) => this.workerDispatchAdmission(request, record),
|
|
77
99
|
getRecord: (laneId) => this.getWorkerLifecycle().getRecord(laneId),
|
|
78
100
|
run: (request, record) => this.runOnce(request, undefined, record),
|
|
79
|
-
cancel: (laneId, reasonCode) =>
|
|
80
|
-
try {
|
|
81
|
-
this.writeReservations.release(laneId);
|
|
82
|
-
const terminal = this.getWorkerLifecycle().cancel(laneId, reasonCode);
|
|
83
|
-
if (terminal)
|
|
84
|
-
this.publishTerminalRecord(terminal);
|
|
85
|
-
}
|
|
86
|
-
catch (error) {
|
|
87
|
-
this.safeWarn(`Failed to cancel durable worker ${laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
88
|
-
}
|
|
89
|
-
},
|
|
101
|
+
cancel: (laneId, reasonCode) => this.cancelScheduledWorker(laneId, reasonCode),
|
|
90
102
|
warn: (message) => this.safeWarn(message),
|
|
91
103
|
});
|
|
92
104
|
this.recovery = new WorkerRecoveryCoordinator({
|
|
@@ -94,19 +106,33 @@ export class WorkerDelegationController {
|
|
|
94
106
|
scheduler: this.scheduler,
|
|
95
107
|
recoverWriteReservations: () => this.writeReservations.recoverProvenStale(),
|
|
96
108
|
publishTerminalRecord: (record) => this.publishTerminalRecord(record),
|
|
109
|
+
publishTerminalRecords: (records) => this.publishRecoveredTerminalRecords(records),
|
|
97
110
|
dispatchVerification: (recovery) => this.dispatchRecoveredVerification(recovery),
|
|
111
|
+
recoverTaskBearingMailboxTurns: () => this.agentControl.reconcileTaskBearingMailboxTurns(),
|
|
112
|
+
recoverSessionRootReplies: () => this.agentControl.reconcileSessionRootReplies(),
|
|
113
|
+
retryReady: () => {
|
|
114
|
+
if (this.deps.isDisposed())
|
|
115
|
+
return;
|
|
116
|
+
this.scheduler.drain();
|
|
117
|
+
this.terminalHandoffs.signal();
|
|
118
|
+
this.notifications.statusChanged();
|
|
119
|
+
},
|
|
98
120
|
warn: (message) => this.safeWarn(message),
|
|
99
121
|
});
|
|
100
122
|
this.agentControl = new WorkerAgentControlCoordinator({
|
|
101
123
|
agentDir: this.deps.getAgentDir(),
|
|
102
124
|
parentSessionId: this.deps.getSessionId(),
|
|
103
125
|
processOwnerId: createLocalWorkerProcessOwnerId(process.pid, randomUUID()),
|
|
126
|
+
conversationStore: this.conversations,
|
|
104
127
|
isControlAvailable: () => this.deps.isDelegateToolActive(),
|
|
105
128
|
getLifecycle: () => this.getWorkerLifecycle(),
|
|
106
129
|
recoveredRequest: (attempt) => this.recovery.recoveredRequest(attempt),
|
|
107
130
|
run: (request, record) => this.runOnce(request, undefined, record),
|
|
108
131
|
scheduler: this.scheduler,
|
|
109
|
-
statusChanged: () =>
|
|
132
|
+
statusChanged: () => {
|
|
133
|
+
this.terminalHandoffs.signal();
|
|
134
|
+
this.notifications.statusChanged();
|
|
135
|
+
},
|
|
110
136
|
abortLane: (laneId, reasonCode) => this.laneAbortControllers.get(laneId)?.abort(reasonCode),
|
|
111
137
|
cancelLane: (laneId, reasonCode) => {
|
|
112
138
|
this.scheduler.dropQueued(laneId);
|
|
@@ -114,10 +140,26 @@ export class WorkerDelegationController {
|
|
|
114
140
|
const terminal = this.getWorkerLifecycle().cancel(laneId, reasonCode);
|
|
115
141
|
if (terminal)
|
|
116
142
|
this.publishTerminalRecord(terminal);
|
|
143
|
+
if (terminal && !this.deps.isDisposed())
|
|
144
|
+
this.scheduler.drain();
|
|
117
145
|
return terminal;
|
|
118
146
|
},
|
|
147
|
+
taskStartHeadroomSkipReason: (agent) => {
|
|
148
|
+
const contract = this.lifecycle.getLatestAgentAttempt(agent.agentId)?.dispatch.executionContract;
|
|
149
|
+
return contract
|
|
150
|
+
? this.workerProjectionHeadroomSkipReason(contract)
|
|
151
|
+
: "orchestration_execution_contract_missing";
|
|
152
|
+
},
|
|
119
153
|
yieldCapacity: (callerAgentId, targetAgentId) => this.yieldWorkerCapacity(callerAgentId, targetAgentId),
|
|
154
|
+
warn: (message) => this.safeWarn(message),
|
|
155
|
+
});
|
|
156
|
+
this.terminalHandoffs = new WorkerTerminalHandoffCoordinator({
|
|
157
|
+
deliver: (handoff) => this.deliverTerminalHandoff(handoff),
|
|
158
|
+
onDeliveryError: (handoff, error) => {
|
|
159
|
+
this.safeWarn(`Worker terminal handoff for ${handoff.record.laneId} failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
160
|
+
},
|
|
120
161
|
});
|
|
162
|
+
this.agentControl.reconcileSessionRootReplies();
|
|
121
163
|
this.writeReservations = new WorkerWriteReservationCoordinator({
|
|
122
164
|
agentDir: this.deps.getAgentDir(),
|
|
123
165
|
getCwd: () => this.deps.getCwd(),
|
|
@@ -138,6 +180,33 @@ export class WorkerDelegationController {
|
|
|
138
180
|
// Disposal and recovery diagnostics must never throw.
|
|
139
181
|
}
|
|
140
182
|
}
|
|
183
|
+
cancelScheduledWorker(laneId, reasonCode) {
|
|
184
|
+
try {
|
|
185
|
+
this.writeReservations.release(laneId);
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
this.safeWarn(`Failed to release worker ${laneId} before cancellation: ${error instanceof Error ? error.message : String(error)}`);
|
|
189
|
+
throw error;
|
|
190
|
+
}
|
|
191
|
+
let terminal;
|
|
192
|
+
try {
|
|
193
|
+
terminal = this.getWorkerLifecycle().cancel(laneId, reasonCode);
|
|
194
|
+
}
|
|
195
|
+
catch (error) {
|
|
196
|
+
this.safeWarn(`Failed to cancel durable worker ${laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
197
|
+
throw error;
|
|
198
|
+
}
|
|
199
|
+
if (!terminal)
|
|
200
|
+
return;
|
|
201
|
+
try {
|
|
202
|
+
this.publishTerminalRecord(terminal);
|
|
203
|
+
}
|
|
204
|
+
catch (error) {
|
|
205
|
+
// The durable cancellation is authoritative. Publication remains recoverable from its
|
|
206
|
+
// pending notification and must not make the scheduler requeue a terminal attempt.
|
|
207
|
+
this.safeWarn(`Failed to publish canceled worker ${laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
141
210
|
/** Narrow model-facing control port. It owns control state; this controller only composes it. */
|
|
142
211
|
getAgentControl() {
|
|
143
212
|
return this.agentControl;
|
|
@@ -173,12 +242,27 @@ export class WorkerDelegationController {
|
|
|
173
242
|
this.lifecycle.markNotificationsDelivered(notificationIds);
|
|
174
243
|
}
|
|
175
244
|
abort() {
|
|
176
|
-
this.workerAbort.abort();
|
|
245
|
+
this.runTeardownStep("abort worker execution", () => this.workerAbort.abort());
|
|
177
246
|
// Bound attempts have an authoritative transcript and agent identity. A normal owner-session
|
|
178
247
|
// shutdown is an execution interruption, not an explicit worker cancellation: fence it into
|
|
179
248
|
// suspended state before the abort continuation can observe the signal.
|
|
180
|
-
const suspendedAttemptIds = new Set(
|
|
181
|
-
|
|
249
|
+
const suspendedAttemptIds = new Set();
|
|
250
|
+
try {
|
|
251
|
+
for (const attemptId of this.lifecycle.suspendBoundInProcessAttemptsForRestart(this.agentControl.getProcessOwnerId())) {
|
|
252
|
+
suspendedAttemptIds.add(attemptId);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
catch (error) {
|
|
256
|
+
this.safeWarn(`Failed to persist worker restart suspension during teardown: ${error instanceof Error ? error.message : String(error)}`);
|
|
257
|
+
}
|
|
258
|
+
let records = [];
|
|
259
|
+
try {
|
|
260
|
+
records = this.lifecycle.getRecords();
|
|
261
|
+
}
|
|
262
|
+
catch (error) {
|
|
263
|
+
this.safeWarn(`Failed to inspect durable worker records during teardown: ${error instanceof Error ? error.message : String(error)}`);
|
|
264
|
+
}
|
|
265
|
+
for (const record of records) {
|
|
182
266
|
if (record.status !== "queued" && record.status !== "running")
|
|
183
267
|
continue;
|
|
184
268
|
const ledger = this.inFlightLedgers.get(record.laneId);
|
|
@@ -235,11 +319,24 @@ export class WorkerDelegationController {
|
|
|
235
319
|
this.safeWarn(`Failed to persist canceled worker claim ${record.laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
236
320
|
}
|
|
237
321
|
}
|
|
238
|
-
this.
|
|
239
|
-
this.
|
|
240
|
-
|
|
241
|
-
|
|
322
|
+
this.inFlightLedgers.clear();
|
|
323
|
+
this.runTeardownStep("cancel queued worker dispatches", () => this.scheduler.cancelQueued());
|
|
324
|
+
this.runTeardownStep("dispose worker recovery", () => this.recovery.dispose());
|
|
325
|
+
this.runTeardownStep("dispose worker terminal handoffs", () => this.terminalHandoffs.dispose());
|
|
326
|
+
this.runTeardownStep("dispose worker write reservations", () => this.writeReservations.dispose());
|
|
327
|
+
for (const shellSessionKey of this.shellSessionKeys) {
|
|
328
|
+
this.runTeardownStep(`dispose worker shell ${shellSessionKey}`, () => disposePersistentShellSession(shellSessionKey));
|
|
329
|
+
}
|
|
242
330
|
this.shellSessionKeys.clear();
|
|
331
|
+
this.runTeardownStep("clear worker conversation cache", () => this.conversations.clearCache());
|
|
332
|
+
}
|
|
333
|
+
runTeardownStep(label, step) {
|
|
334
|
+
try {
|
|
335
|
+
step();
|
|
336
|
+
}
|
|
337
|
+
catch (error) {
|
|
338
|
+
this.safeWarn(`Failed to ${label}: ${error instanceof Error ? error.message : String(error)}`);
|
|
339
|
+
}
|
|
243
340
|
}
|
|
244
341
|
getWorkerLifecycle() {
|
|
245
342
|
return this.lifecycle;
|
|
@@ -285,6 +382,8 @@ export class WorkerDelegationController {
|
|
|
285
382
|
/** Reconcile the session goal into an already-loaded worker runtime without defeating lazy UAC. */
|
|
286
383
|
synchronizeGoalState(goal) {
|
|
287
384
|
this.publishGoalTerminalRecords(this.lifecycle.synchronizeGoalState(goal));
|
|
385
|
+
if (goal.status === "active" && !this.deps.isDisposed())
|
|
386
|
+
this.scheduler.drain();
|
|
288
387
|
}
|
|
289
388
|
publishGoalTerminalRecords(records) {
|
|
290
389
|
for (const record of records) {
|
|
@@ -292,6 +391,8 @@ export class WorkerDelegationController {
|
|
|
292
391
|
this.laneAbortControllers.get(record.laneId)?.abort("goal_terminal");
|
|
293
392
|
this.publishTerminalRecord(record);
|
|
294
393
|
}
|
|
394
|
+
if (records.length > 0 && !this.deps.isDisposed())
|
|
395
|
+
this.scheduler.drain();
|
|
295
396
|
}
|
|
296
397
|
dispatchRecoveredVerification(recovery) {
|
|
297
398
|
const legacyImplementation = recovery.verifierExecutionContract
|
|
@@ -334,9 +435,45 @@ export class WorkerDelegationController {
|
|
|
334
435
|
if (!profile.requireIndependentVerification)
|
|
335
436
|
return { ok: true };
|
|
336
437
|
const resolved = this.getWorkerProfileResolver().resolveVerifier(profile);
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
438
|
+
if (!resolved.ok)
|
|
439
|
+
return { ok: false, skipReason: `independent_verifier_unavailable:${resolved.reason}` };
|
|
440
|
+
// Fresh verifier profiles pass through the same shipment admission owner as fresh workers.
|
|
441
|
+
// Recovery-pinned verifier contracts bypass this path and preserve their immutable grant.
|
|
442
|
+
const admitted = resolveWorkerAuthority({
|
|
443
|
+
base: resolved.resolved,
|
|
444
|
+
modelRegistry: this.deps.getModelRegistry(),
|
|
445
|
+
isModelExhausted: (model) => this.deps.isModelExhausted(model),
|
|
446
|
+
});
|
|
447
|
+
return admitted.ok
|
|
448
|
+
? { ok: true, shipment: admitted.shipment }
|
|
449
|
+
: { ok: false, skipReason: `independent_verifier_unavailable:${admitted.reason}` };
|
|
450
|
+
}
|
|
451
|
+
/**
|
|
452
|
+
* A descendant may select a different routing profile, but that preset cannot introduce context
|
|
453
|
+
* its ancestor never admitted. Pointer ids alone are insufficient authority: retain only pointers
|
|
454
|
+
* whose complete immutable metadata matches the ancestral contract, and require exact soul text.
|
|
455
|
+
*/
|
|
456
|
+
narrowWorkerShipmentContext(shipment, boundary) {
|
|
457
|
+
const boundaryPointers = new Map(boundary.resourcePointers.map((pointer) => [pointer.id, pointer]));
|
|
458
|
+
const resourcePointers = [];
|
|
459
|
+
for (const pointer of shipment.resourcePointers) {
|
|
460
|
+
const inherited = boundaryPointers.get(pointer.id);
|
|
461
|
+
if (!inherited || !isDeepStrictEqual(inherited, pointer)) {
|
|
462
|
+
return { ok: false, skipReason: "orchestration_context_authority_exceeded" };
|
|
463
|
+
}
|
|
464
|
+
resourcePointers.push(structuredClone(inherited));
|
|
465
|
+
}
|
|
466
|
+
if (shipment.soul !== undefined && shipment.soul !== boundary.soul) {
|
|
467
|
+
return { ok: false, skipReason: "orchestration_context_authority_exceeded" };
|
|
468
|
+
}
|
|
469
|
+
return {
|
|
470
|
+
ok: true,
|
|
471
|
+
shipment: {
|
|
472
|
+
...shipment,
|
|
473
|
+
resourcePointers,
|
|
474
|
+
...(shipment.soul ? { soul: shipment.soul } : {}),
|
|
475
|
+
},
|
|
476
|
+
};
|
|
340
477
|
}
|
|
341
478
|
hasExactRecursiveCycle(parentAgentId, instructions, profileId) {
|
|
342
479
|
const normalizedInstructions = instructions.trim();
|
|
@@ -355,6 +492,17 @@ export class WorkerDelegationController {
|
|
|
355
492
|
}
|
|
356
493
|
return false;
|
|
357
494
|
}
|
|
495
|
+
/** Reject a new logical identity before creating its task, transcript, agent, or queue entry. */
|
|
496
|
+
newWorkerFleetSkipReason(request, requiredAgentSlots = 1) {
|
|
497
|
+
const decision = evaluateNewWorkerAdmission(this.lifecycle.getTaskRuntimeSnapshot().agents, request.parentAgentId, DEFAULT_WORKER_FLEET_LIMITS, requiredAgentSlots);
|
|
498
|
+
return decision.ok ? undefined : decision.reasonCode;
|
|
499
|
+
}
|
|
500
|
+
requiredAgentSlotsForAdmission(request, admission) {
|
|
501
|
+
const pendingVerifierSubjects = pendingVerifierSubjectTaskIds(this.lifecycle.getTaskRuntimeSnapshot());
|
|
502
|
+
if (request.verificationOfTaskId)
|
|
503
|
+
pendingVerifierSubjects.delete(request.verificationOfTaskId);
|
|
504
|
+
return 1 + pendingVerifierSubjects.size + (admission.verifierShipment ? 1 : 0);
|
|
505
|
+
}
|
|
358
506
|
/** Single admission contract shared by enqueue, scheduler revalidation, and execution. */
|
|
359
507
|
resolveWorkerAdmission(request, pinnedContract) {
|
|
360
508
|
if (this.deps.isDisposed())
|
|
@@ -421,7 +569,13 @@ export class WorkerDelegationController {
|
|
|
421
569
|
});
|
|
422
570
|
if (!adaptive.ok)
|
|
423
571
|
return { ok: false, skipReason: adaptive.reason };
|
|
424
|
-
|
|
572
|
+
let shipment = adaptive.shipment;
|
|
573
|
+
if (parentContract) {
|
|
574
|
+
const narrowed = this.narrowWorkerShipmentContext(shipment, parentContract.worker);
|
|
575
|
+
if (!narrowed.ok)
|
|
576
|
+
return narrowed;
|
|
577
|
+
shipment = narrowed.shipment;
|
|
578
|
+
}
|
|
425
579
|
if (parentAgent && this.hasExactRecursiveCycle(parentAgent.agentId, instructions, shipment.profile.profileId)) {
|
|
426
580
|
return { ok: false, skipReason: "recursive_delegation_cycle" };
|
|
427
581
|
}
|
|
@@ -445,6 +599,13 @@ export class WorkerDelegationController {
|
|
|
445
599
|
return verifier;
|
|
446
600
|
verifierShipment = verifier.shipment;
|
|
447
601
|
}
|
|
602
|
+
if (verifierShipment && parentContract) {
|
|
603
|
+
const verifierBoundary = parentContract.verifier ?? parentContract.worker;
|
|
604
|
+
const narrowed = this.narrowWorkerShipmentContext(verifierShipment, verifierBoundary);
|
|
605
|
+
if (!narrowed.ok)
|
|
606
|
+
return narrowed;
|
|
607
|
+
verifierShipment = narrowed.shipment;
|
|
608
|
+
}
|
|
448
609
|
if (!this.laneCapabilityProfile(shipment.model).backgroundLanesEnabled) {
|
|
449
610
|
return { ok: false, skipReason: "model_delegation_unsupported" };
|
|
450
611
|
}
|
|
@@ -463,6 +624,13 @@ export class WorkerDelegationController {
|
|
|
463
624
|
const executionPlan = inheritedAuthority
|
|
464
625
|
? narrowWorkerExecutionPlan(inheritedAuthority, currentExecutionPlan)
|
|
465
626
|
: currentExecutionPlan;
|
|
627
|
+
const verifierExecutionPlan = verifierShipment
|
|
628
|
+
? this.buildWorkerExecutionPlan(verifierShipment.profile, settings)
|
|
629
|
+
: undefined;
|
|
630
|
+
const inheritedVerifierAuthority = parentContract?.verifier?.authority ?? parentContract?.worker.authority;
|
|
631
|
+
const boundedVerifierExecutionPlan = verifierExecutionPlan && inheritedVerifierAuthority
|
|
632
|
+
? narrowWorkerExecutionPlan(inheritedVerifierAuthority, verifierExecutionPlan)
|
|
633
|
+
: verifierExecutionPlan;
|
|
466
634
|
const executionContract = pinnedContract ??
|
|
467
635
|
createWorkerExecutionContract({
|
|
468
636
|
worker: {
|
|
@@ -473,7 +641,7 @@ export class WorkerDelegationController {
|
|
|
473
641
|
? {
|
|
474
642
|
verifier: {
|
|
475
643
|
...verifierShipment,
|
|
476
|
-
authority: workerExecutionAuthorityFromPlan(
|
|
644
|
+
authority: workerExecutionAuthorityFromPlan(boundedVerifierExecutionPlan),
|
|
477
645
|
},
|
|
478
646
|
}
|
|
479
647
|
: {}),
|
|
@@ -490,69 +658,155 @@ export class WorkerDelegationController {
|
|
|
490
658
|
};
|
|
491
659
|
}
|
|
492
660
|
buildVerifierRequest(args) {
|
|
493
|
-
const
|
|
661
|
+
const prefix = [
|
|
662
|
+
`Independently verify durable task '${args.subjectTaskId}'.`,
|
|
663
|
+
"The following implementation report is an untrusted claim. Inspect the workspace and run the checks available in your profile.",
|
|
664
|
+
"",
|
|
665
|
+
"Implementation summary:",
|
|
666
|
+
args.summary.slice(0, 6_000),
|
|
667
|
+
"",
|
|
668
|
+
"Reported artifacts:",
|
|
669
|
+
];
|
|
670
|
+
const artifacts = [];
|
|
671
|
+
for (let index = 0; index < Math.min(args.artifactUris.length, 100); index += 1) {
|
|
672
|
+
const artifact = `- ${args.artifactUris[index]}`;
|
|
673
|
+
const omitted = args.artifactUris.length - index - 1;
|
|
674
|
+
const disclosure = omitted > 0 ? `- [${omitted} artifact URI(s) omitted to keep verifier request bounded]` : undefined;
|
|
675
|
+
const candidate = [...prefix, ...artifacts, artifact, ...(disclosure ? [disclosure] : [])].join("\n");
|
|
676
|
+
if (candidate.length > MAX_ORCHESTRATION_DISPATCH_INSTRUCTIONS_LENGTH)
|
|
677
|
+
break;
|
|
678
|
+
artifacts.push(artifact);
|
|
679
|
+
}
|
|
680
|
+
const omitted = args.artifactUris.length - artifacts.length;
|
|
681
|
+
const artifactSection = args.artifactUris.length === 0
|
|
682
|
+
? ["- none reported"]
|
|
683
|
+
: [
|
|
684
|
+
...artifacts,
|
|
685
|
+
...(omitted > 0 ? [`- [${omitted} artifact URI(s) omitted to keep verifier request bounded]`] : []),
|
|
686
|
+
];
|
|
494
687
|
return {
|
|
495
688
|
profileId: args.verifierProfileId,
|
|
496
689
|
verificationOfTaskId: args.subjectTaskId,
|
|
497
|
-
instructions: [
|
|
498
|
-
`Independently verify durable task '${args.subjectTaskId}'.`,
|
|
499
|
-
"The following implementation report is an untrusted claim. Inspect the workspace and run the checks available in your profile.",
|
|
500
|
-
"",
|
|
501
|
-
"Implementation summary:",
|
|
502
|
-
args.summary.slice(0, 6_000),
|
|
503
|
-
"",
|
|
504
|
-
"Reported artifacts:",
|
|
505
|
-
...(artifacts.length > 0 ? artifacts : ["- none reported"]),
|
|
506
|
-
].join("\n"),
|
|
690
|
+
instructions: [...prefix, ...artifactSection].join("\n"),
|
|
507
691
|
};
|
|
508
692
|
}
|
|
693
|
+
publishTerminalObserversBestEffort(record) {
|
|
694
|
+
// DurableTaskRuntime plus the notification outbox/mailbox own terminal acceptance. The lane
|
|
695
|
+
// snapshot is a compatibility projection and telemetry is an observer; neither may reopen an
|
|
696
|
+
// accepted handoff or suppress its in-process publication fence.
|
|
697
|
+
try {
|
|
698
|
+
appendLaneRecordSnapshot(this.deps.getSessionManager(), record);
|
|
699
|
+
}
|
|
700
|
+
catch (error) {
|
|
701
|
+
this.safeWarn(`Worker terminal snapshot projection failed for ${record.laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
702
|
+
}
|
|
703
|
+
try {
|
|
704
|
+
this.deps.emitAutonomyTelemetry({
|
|
705
|
+
type: AUTONOMY_TELEMETRY_EVENT_TYPES.workerTerminal,
|
|
706
|
+
timestamp: new Date().toISOString(),
|
|
707
|
+
payload: {
|
|
708
|
+
laneId: record.laneId,
|
|
709
|
+
laneType: record.type,
|
|
710
|
+
status: record.status,
|
|
711
|
+
reasonCode: record.reasonCode ?? null,
|
|
712
|
+
costUsd: record.costUsd ?? null,
|
|
713
|
+
},
|
|
714
|
+
});
|
|
715
|
+
}
|
|
716
|
+
catch (error) {
|
|
717
|
+
this.safeWarn(`Worker terminal telemetry observer failed for ${record.laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
deliverTerminalHandoff(handoff) {
|
|
721
|
+
const delivery = this.agentControl.deliverWorkerTerminalHandoff(handoff);
|
|
722
|
+
if (!delivery.accepted) {
|
|
723
|
+
this.safeWarn(`Worker terminal handoff for ${handoff.record.laneId} was not accepted${delivery.skipReason ? `: ${delivery.skipReason}` : "."}`);
|
|
724
|
+
return "retained";
|
|
725
|
+
}
|
|
726
|
+
const notification = this.lifecycle.getTerminalNotification(handoff.record.laneId);
|
|
727
|
+
if (notification?.status === "pending") {
|
|
728
|
+
this.lifecycle.markNotificationsDelivered([notification.notificationId]);
|
|
729
|
+
}
|
|
730
|
+
this.publishedTerminalAttemptIds.add(handoff.terminalAttemptId);
|
|
731
|
+
this.publishTerminalObserversBestEffort(handoff.record);
|
|
732
|
+
try {
|
|
733
|
+
this.notifications.statusChanged();
|
|
734
|
+
}
|
|
735
|
+
catch (error) {
|
|
736
|
+
this.safeWarn(`Worker terminal status observer failed for ${handoff.record.laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
737
|
+
}
|
|
738
|
+
try {
|
|
739
|
+
this.agentControl.signalStateChanged();
|
|
740
|
+
}
|
|
741
|
+
catch (error) {
|
|
742
|
+
this.safeWarn(`Worker terminal state observer failed for ${handoff.record.laneId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
743
|
+
}
|
|
744
|
+
return "delivered";
|
|
745
|
+
}
|
|
746
|
+
publishRecoveredTerminalRecords(records) {
|
|
747
|
+
const handoffs = [];
|
|
748
|
+
const directRecords = [];
|
|
749
|
+
for (const record of records) {
|
|
750
|
+
this.recovery.clearScheduledRetry(record.laneId);
|
|
751
|
+
const attempt = this.lifecycle.getActiveAttempt(record.laneId);
|
|
752
|
+
if (attempt && this.publishedTerminalAttemptIds.has(attempt.attemptId))
|
|
753
|
+
continue;
|
|
754
|
+
const childAgentId = attempt?.agentId ?? attempt?.dispatch.logicalLaneId ?? record.laneId;
|
|
755
|
+
const parentAgentId = this.lifecycle.getAgent(childAgentId)?.parentAgentId ?? attempt?.dispatch.parentAgentId;
|
|
756
|
+
if (attempt && parentAgentId) {
|
|
757
|
+
handoffs.push({
|
|
758
|
+
terminalAttemptId: attempt.attemptId,
|
|
759
|
+
parentAgentId,
|
|
760
|
+
childAgentId,
|
|
761
|
+
record,
|
|
762
|
+
});
|
|
763
|
+
}
|
|
764
|
+
else {
|
|
765
|
+
directRecords.push(record);
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
if (handoffs.length > 0) {
|
|
769
|
+
try {
|
|
770
|
+
this.terminalHandoffs.rehydrate(handoffs);
|
|
771
|
+
}
|
|
772
|
+
catch (error) {
|
|
773
|
+
this.safeWarn(`Worker terminal handoff recovery batch was retained durably but not adopted: ${error instanceof Error ? error.message : String(error)}`);
|
|
774
|
+
}
|
|
775
|
+
}
|
|
776
|
+
for (const record of directRecords)
|
|
777
|
+
this.publishTerminalRecord(record);
|
|
778
|
+
}
|
|
509
779
|
publishTerminalRecord(record) {
|
|
510
|
-
|
|
780
|
+
if (!isLaneTerminalStatus(record.status))
|
|
781
|
+
return;
|
|
782
|
+
this.recovery.clearScheduledRetry(record.laneId);
|
|
783
|
+
const attempt = this.lifecycle.getActiveAttempt(record.laneId);
|
|
511
784
|
const attemptId = attempt?.attemptId;
|
|
512
785
|
if (attemptId && this.publishedTerminalAttemptIds.has(attemptId))
|
|
513
786
|
return;
|
|
514
787
|
const childAgentId = attempt?.agentId ?? attempt?.dispatch.logicalLaneId ?? record.laneId;
|
|
515
788
|
const parentAgentId = this.lifecycle.getAgent(childAgentId)?.parentAgentId ?? attempt?.dispatch.parentAgentId;
|
|
516
|
-
let handedOffToParent = false;
|
|
517
789
|
if (attemptId && parentAgentId) {
|
|
518
790
|
try {
|
|
519
|
-
this.
|
|
791
|
+
this.terminalHandoffs.retain({
|
|
520
792
|
parentAgentId,
|
|
521
793
|
childAgentId,
|
|
522
794
|
terminalAttemptId: attemptId,
|
|
523
795
|
record,
|
|
524
796
|
});
|
|
525
|
-
|
|
797
|
+
this.terminalHandoffs.signal();
|
|
526
798
|
}
|
|
527
799
|
catch (error) {
|
|
528
|
-
this.safeWarn(`Worker terminal handoff for ${record.laneId} failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
800
|
+
this.safeWarn(`Worker terminal handoff retention for ${record.laneId} failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
529
801
|
}
|
|
802
|
+
return;
|
|
530
803
|
}
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
this.lifecycle.markNotificationsDelivered([notification.notificationId]);
|
|
535
|
-
}
|
|
536
|
-
this.notifications.statusChanged();
|
|
537
|
-
}
|
|
538
|
-
else {
|
|
539
|
-
this.recordTerminal(record);
|
|
804
|
+
this.recordTerminal(record);
|
|
805
|
+
if (attemptId) {
|
|
806
|
+
this.publishedTerminalAttemptIds.add(attemptId);
|
|
540
807
|
}
|
|
541
|
-
|
|
542
|
-
this.deps.emitAutonomyTelemetry({
|
|
543
|
-
type: AUTONOMY_TELEMETRY_EVENT_TYPES.workerTerminal,
|
|
544
|
-
timestamp: new Date().toISOString(),
|
|
545
|
-
payload: {
|
|
546
|
-
laneId: record.laneId,
|
|
547
|
-
laneType: record.type,
|
|
548
|
-
status: record.status,
|
|
549
|
-
reasonCode: record.reasonCode ?? null,
|
|
550
|
-
costUsd: record.costUsd ?? null,
|
|
551
|
-
},
|
|
552
|
-
});
|
|
808
|
+
this.publishTerminalObserversBestEffort(record);
|
|
553
809
|
this.agentControl.signalStateChanged();
|
|
554
|
-
if (attemptId)
|
|
555
|
-
this.publishedTerminalAttemptIds.add(attemptId);
|
|
556
810
|
}
|
|
557
811
|
/**
|
|
558
812
|
* A pre-execution denial still owns a prepared durable lane. Publish its terminal projection at
|
|
@@ -564,12 +818,14 @@ export class WorkerDelegationController {
|
|
|
564
818
|
const record = lifecycle.cancel(laneId, reasonCode);
|
|
565
819
|
if (record)
|
|
566
820
|
this.publishTerminalRecord(record);
|
|
821
|
+
if (record && !this.deps.isDisposed())
|
|
822
|
+
this.scheduler.drain();
|
|
567
823
|
return record;
|
|
568
824
|
}
|
|
569
825
|
hasWorkerCapacity(settings) {
|
|
570
826
|
const snapshot = this.getWorkerLifecycle().getTaskRuntimeSnapshot();
|
|
571
827
|
let yielded = 0;
|
|
572
|
-
for (const attemptId of
|
|
828
|
+
for (const attemptId of this.yieldedCapacityAttemptIds.keys()) {
|
|
573
829
|
const status = snapshot.attempts[attemptId]?.status;
|
|
574
830
|
if (status === "leased" || status === "running")
|
|
575
831
|
yielded += 1;
|
|
@@ -586,33 +842,155 @@ export class WorkerDelegationController {
|
|
|
586
842
|
const attempt = this.lifecycle.getLatestAgentAttempt(caller.agentId);
|
|
587
843
|
if (!attempt || (attempt.status !== "leased" && attempt.status !== "running"))
|
|
588
844
|
return () => undefined;
|
|
589
|
-
this.yieldedCapacityAttemptIds.
|
|
845
|
+
this.yieldedCapacityAttemptIds.set(attempt.attemptId, (this.yieldedCapacityAttemptIds.get(attempt.attemptId) ?? 0) + 1);
|
|
590
846
|
this.scheduler.drain();
|
|
591
847
|
let released = false;
|
|
592
848
|
return () => {
|
|
593
849
|
if (released)
|
|
594
850
|
return;
|
|
595
851
|
released = true;
|
|
852
|
+
const leases = this.yieldedCapacityAttemptIds.get(attempt.attemptId);
|
|
853
|
+
if (leases === undefined)
|
|
854
|
+
return;
|
|
855
|
+
if (leases > 1) {
|
|
856
|
+
this.yieldedCapacityAttemptIds.set(attempt.attemptId, leases - 1);
|
|
857
|
+
return;
|
|
858
|
+
}
|
|
596
859
|
this.yieldedCapacityAttemptIds.delete(attempt.attemptId);
|
|
597
860
|
if (!this.deps.isDisposed())
|
|
598
861
|
this.scheduler.drain();
|
|
599
862
|
};
|
|
600
863
|
}
|
|
601
864
|
workerDispatchAdmission(request, record) {
|
|
602
|
-
|
|
865
|
+
if (this.recovery.deferRetryIfNeeded(record, request))
|
|
866
|
+
return { action: "wait" };
|
|
867
|
+
const lifecycle = this.getWorkerLifecycle();
|
|
868
|
+
const attempt = lifecycle.getActiveAttempt(record.laneId);
|
|
869
|
+
if (!attempt)
|
|
870
|
+
return { action: "cancel", reasonCode: "orchestration_attempt_missing" };
|
|
871
|
+
if (attempt.status === "queued" || attempt.status === "suspended") {
|
|
872
|
+
const readiness = lifecycle.getAttemptDispatchReadiness(attempt.attemptId);
|
|
873
|
+
if (readiness.state === "waiting") {
|
|
874
|
+
return {
|
|
875
|
+
action: "wait",
|
|
876
|
+
reason: readiness.reasonCode === "objective_paused" ? "objective" : "dependencies",
|
|
877
|
+
};
|
|
878
|
+
}
|
|
879
|
+
if (readiness.state === "blocked")
|
|
880
|
+
return { action: "cancel", reasonCode: readiness.reasonCode };
|
|
881
|
+
}
|
|
882
|
+
const contract = attempt.dispatch.executionContract;
|
|
603
883
|
const admission = this.resolveWorkerAdmission(request, contract);
|
|
604
884
|
if (!admission.ok)
|
|
605
885
|
return { action: "cancel", reasonCode: admission.skipReason };
|
|
606
886
|
if (!this.hasWorkerCapacity(admission.settings))
|
|
607
887
|
return { action: "wait", reason: "capacity" };
|
|
608
|
-
const attempt = this.getWorkerLifecycle().getActiveAttempt(record.laneId);
|
|
609
|
-
if (!attempt)
|
|
610
|
-
return { action: "cancel", reasonCode: "orchestration_attempt_missing" };
|
|
611
888
|
const reservation = this.writeReservations.acquire(record.laneId, attempt, admission.executionPlan);
|
|
612
889
|
if (reservation.kind === "denied")
|
|
613
890
|
return { action: "cancel", reasonCode: reservation.reasonCode };
|
|
614
891
|
return reservation.kind === "granted" ? { action: "start" } : { action: "wait", reason: "write_reservation" };
|
|
615
892
|
}
|
|
893
|
+
workerProjectionHeadroomSkipReason(contract, verificationOfTaskId) {
|
|
894
|
+
const pendingVerifierSubjects = pendingVerifierSubjectTaskIds(this.lifecycle.getTaskRuntimeSnapshot());
|
|
895
|
+
if (verificationOfTaskId)
|
|
896
|
+
pendingVerifierSubjects.delete(verificationOfTaskId);
|
|
897
|
+
const slots = 1 + pendingVerifierSubjects.size + (contract.verifier ? 1 : 0);
|
|
898
|
+
try {
|
|
899
|
+
this.lifecycle.ledger.runtime.assertProjectionHeadroom({ tasks: slots, attempts: slots });
|
|
900
|
+
return undefined;
|
|
901
|
+
}
|
|
902
|
+
catch {
|
|
903
|
+
return "orchestration_projection_capacity_exhausted";
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
workerQueueReservationSkipReason(request, requestQueueSlots, unpersistedVerifierSlots) {
|
|
907
|
+
const pendingVerifierSubjects = pendingVerifierSubjectTaskIds(this.lifecycle.getTaskRuntimeSnapshot());
|
|
908
|
+
if (request.verificationOfTaskId)
|
|
909
|
+
pendingVerifierSubjects.delete(request.verificationOfTaskId);
|
|
910
|
+
const remainingQueueSlots = DEFAULT_WORKER_FLEET_LIMITS.maxQueuedDispatches - this.scheduler.queuedCount;
|
|
911
|
+
return requestQueueSlots + pendingVerifierSubjects.size + unpersistedVerifierSlots > remainingQueueSlots
|
|
912
|
+
? "worker_dispatch_queue_full"
|
|
913
|
+
: undefined;
|
|
914
|
+
}
|
|
915
|
+
workerContextParentModel(request) {
|
|
916
|
+
if (request.parentAgentId) {
|
|
917
|
+
const parent = this.lifecycle.getAgent(request.parentAgentId);
|
|
918
|
+
if (!parent)
|
|
919
|
+
throw new Error("Worker parent logical agent is missing.");
|
|
920
|
+
return workerContextModelIdentity(parent.resumeContext.modelRef);
|
|
921
|
+
}
|
|
922
|
+
const parent = this.deps.getModel();
|
|
923
|
+
if (!parent)
|
|
924
|
+
throw new Error("Foreground parent model is missing.");
|
|
925
|
+
return { provider: parent.provider, model: parent.id };
|
|
926
|
+
}
|
|
927
|
+
workerContextForkSource(request) {
|
|
928
|
+
const model = this.workerContextParentModel(request);
|
|
929
|
+
if (!request.parentAgentId) {
|
|
930
|
+
return { model, messages: this.deps.getSessionManager().buildSessionContext().messages };
|
|
931
|
+
}
|
|
932
|
+
const parent = this.lifecycle.getAgent(request.parentAgentId);
|
|
933
|
+
if (!parent)
|
|
934
|
+
throw new Error("Worker parent logical agent is missing.");
|
|
935
|
+
const conversation = this.conversations.open({
|
|
936
|
+
agentDir: this.deps.getAgentDir(),
|
|
937
|
+
resumeContext: parent.resumeContext,
|
|
938
|
+
expectedLogicalAgentId: parent.agentId,
|
|
939
|
+
});
|
|
940
|
+
return { model, messages: conversation.getProviderContext().messages };
|
|
941
|
+
}
|
|
942
|
+
workerContextForkMode(request, contract) {
|
|
943
|
+
if (request.verificationOfTaskId)
|
|
944
|
+
return { kind: "none" };
|
|
945
|
+
return resolveWorkerContextInheritanceMode({
|
|
946
|
+
parent: this.workerContextParentModel(request),
|
|
947
|
+
worker: {
|
|
948
|
+
provider: contract.worker.modelBinding.provider,
|
|
949
|
+
model: contract.worker.modelBinding.modelId,
|
|
950
|
+
},
|
|
951
|
+
...(request.forkTurns !== undefined ? { mode: request.forkTurns } : {}),
|
|
952
|
+
});
|
|
953
|
+
}
|
|
954
|
+
workerContextForkAdmissionSkipReason(request, contract) {
|
|
955
|
+
try {
|
|
956
|
+
this.workerContextForkMode(request, contract);
|
|
957
|
+
return undefined;
|
|
958
|
+
}
|
|
959
|
+
catch {
|
|
960
|
+
return "worker_context_inheritance_denied";
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
/** One admission owner for every newly generated logical worker identity. */
|
|
964
|
+
admitNewWorkerRequest(request, pinnedContract) {
|
|
965
|
+
const fleetSkipReason = this.newWorkerFleetSkipReason(request);
|
|
966
|
+
if (fleetSkipReason)
|
|
967
|
+
return { ok: false, skipReason: fleetSkipReason };
|
|
968
|
+
const admission = this.resolveWorkerAdmission(request, pinnedContract);
|
|
969
|
+
if (!admission.ok)
|
|
970
|
+
return admission;
|
|
971
|
+
const contextForkSkipReason = this.workerContextForkAdmissionSkipReason(request, admission.executionContract);
|
|
972
|
+
if (contextForkSkipReason)
|
|
973
|
+
return { ok: false, skipReason: contextForkSkipReason };
|
|
974
|
+
if (!request.verificationOfTaskId)
|
|
975
|
+
this.recovery.recover();
|
|
976
|
+
const headroomSkipReason = this.newWorkerFleetSkipReason(request, this.requiredAgentSlotsForAdmission(request, admission));
|
|
977
|
+
if (headroomSkipReason)
|
|
978
|
+
return { ok: false, skipReason: headroomSkipReason };
|
|
979
|
+
const projectionHeadroomSkipReason = this.workerProjectionHeadroomSkipReason(admission.executionContract, request.verificationOfTaskId);
|
|
980
|
+
if (projectionHeadroomSkipReason)
|
|
981
|
+
return { ok: false, skipReason: projectionHeadroomSkipReason };
|
|
982
|
+
const queueReservationSkipReason = this.workerQueueReservationSkipReason(request, 0, admission.verifierShipment ? 1 : 0);
|
|
983
|
+
return queueReservationSkipReason ? { ok: false, skipReason: queueReservationSkipReason } : admission;
|
|
984
|
+
}
|
|
985
|
+
durableWorkerContextForkReferences() {
|
|
986
|
+
const references = [];
|
|
987
|
+
for (const attempt of Object.values(this.lifecycle.getTaskRuntimeSnapshot().attempts)) {
|
|
988
|
+
if (attempt.dispatch.birthContextForkReference) {
|
|
989
|
+
references.push(attempt.dispatch.birthContextForkReference);
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
return references;
|
|
993
|
+
}
|
|
616
994
|
buildWorkerExecutionPlan(profile, settings, requestedReadPaths, requestedWritePaths) {
|
|
617
995
|
return buildWorkerExecutionPlan({
|
|
618
996
|
profile,
|
|
@@ -637,21 +1015,102 @@ export class WorkerDelegationController {
|
|
|
637
1015
|
};
|
|
638
1016
|
}
|
|
639
1017
|
const goal = this.deps.getGoalStateSnapshot();
|
|
640
|
-
const
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
1018
|
+
const mode = this.workerContextForkMode(request, admission.executionContract);
|
|
1019
|
+
const messages = mode.kind === "none" ? [] : selectSanitizedContextFork(this.workerContextForkSource(request).messages, mode);
|
|
1020
|
+
for (let allocation = 0; allocation < DEFAULT_WORKER_FLEET_LIMITS.maxAgentsPerSession; allocation++) {
|
|
1021
|
+
const laneId = lifecycle.getNextAvailableLaneIdCandidate();
|
|
1022
|
+
let proposedReference;
|
|
1023
|
+
try {
|
|
1024
|
+
const captured = this.contextForks.captureAndPrepare({
|
|
1025
|
+
logicalAgentId: laneId,
|
|
1026
|
+
messages,
|
|
1027
|
+
readDurableReferences: () => this.durableWorkerContextForkReferences(),
|
|
1028
|
+
isLogicalIdentityClaimed: () => {
|
|
1029
|
+
const snapshot = lifecycle.getTaskRuntimeSnapshot();
|
|
1030
|
+
return (snapshot.tasks[laneId]?.attemptIds.some((attemptId) => snapshot.attempts[attemptId] !== undefined) ?? false);
|
|
1031
|
+
},
|
|
1032
|
+
prepare: (birthContextForkReference) => {
|
|
1033
|
+
proposedReference = birthContextForkReference;
|
|
1034
|
+
return lifecycle.prepare({
|
|
1035
|
+
instructions: admission.instructions,
|
|
1036
|
+
...(request.parentAgentId ? { parentAgentId: request.parentAgentId } : {}),
|
|
1037
|
+
birthContextForkReference,
|
|
1038
|
+
executionContract: admission.executionContract,
|
|
1039
|
+
requiredCapabilities: admission.executionPlan.requiredCapabilities,
|
|
1040
|
+
...(request.verificationOfTaskId ? { verificationOfTaskId: request.verificationOfTaskId } : {}),
|
|
1041
|
+
taskContext: {
|
|
1042
|
+
requirementIds: request.taskContext?.requirementIds ?? [],
|
|
1043
|
+
dependsOnTaskIds: request.taskContext?.dependsOnTaskIds ?? [],
|
|
1044
|
+
acceptanceCriterionIds: request.taskContext?.acceptanceCriterionIds ?? [],
|
|
1045
|
+
resourcePointerIds: admission.resourcePointerIds,
|
|
1046
|
+
},
|
|
1047
|
+
...(goal ? { goal } : {}),
|
|
1048
|
+
}, laneId);
|
|
1049
|
+
},
|
|
1050
|
+
});
|
|
1051
|
+
return { executionPlan: admission.executionPlan, lifecycle, ...captured.value };
|
|
1052
|
+
}
|
|
1053
|
+
catch (error) {
|
|
1054
|
+
if (error instanceof WorkerContextForkStoreError && error.code === "identity_claimed")
|
|
1055
|
+
continue;
|
|
1056
|
+
const snapshot = lifecycle.getTaskRuntimeSnapshot();
|
|
1057
|
+
const candidateTask = snapshot.tasks[laneId];
|
|
1058
|
+
const candidateAttempts = candidateTask?.attemptIds.map((attemptId) => snapshot.attempts[attemptId]) ?? [];
|
|
1059
|
+
const candidateOwnsDurableAttempt = candidateAttempts.some((attempt) => attempt !== undefined);
|
|
1060
|
+
const sameRequestReplay = candidateAttempts.some((attempt) => attempt !== undefined &&
|
|
1061
|
+
attempt.dispatch.instructions === admission.instructions &&
|
|
1062
|
+
attempt.dispatch.parentAgentId === request.parentAgentId &&
|
|
1063
|
+
isDeepStrictEqual(attempt.dispatch.executionContract, admission.executionContract) &&
|
|
1064
|
+
(proposedReference === undefined ||
|
|
1065
|
+
isDeepStrictEqual(attempt.dispatch.birthContextForkReference, proposedReference)));
|
|
1066
|
+
const corruptSnapshot = error instanceof WorkerContextForkStoreError &&
|
|
1067
|
+
(error.code === "snapshot_corrupt" || error.code === "snapshot_missing");
|
|
1068
|
+
const generatedIdWasConcurrentlyClaimed = candidateOwnsDurableAttempt && !sameRequestReplay && !corruptSnapshot;
|
|
1069
|
+
if (!generatedIdWasConcurrentlyClaimed)
|
|
1070
|
+
throw error;
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
throw new Error("Worker logical-agent allocation retries were exhausted.");
|
|
1074
|
+
}
|
|
1075
|
+
enqueuePreparedWorkerAttempt(record, request, profileId, priority) {
|
|
1076
|
+
try {
|
|
1077
|
+
this.scheduler.enqueue(record, { ...request, profileId }, false, priority);
|
|
1078
|
+
return undefined;
|
|
1079
|
+
}
|
|
1080
|
+
catch (error) {
|
|
1081
|
+
const reasonCode = error instanceof Error && error.message === "worker_dispatch_queue_full"
|
|
1082
|
+
? "worker_dispatch_queue_full"
|
|
1083
|
+
: "worker_dispatch_enqueue_error";
|
|
1084
|
+
this.cancelAndPublish(this.lifecycle, record.laneId, reasonCode);
|
|
1085
|
+
this.safeWarn(`Worker dispatch enqueue failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1086
|
+
return reasonCode;
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
queuePreparedWorkerAttempt(prepared, request, admission, options = {}) {
|
|
1090
|
+
if (this.workerQueueReservationSkipReason(request, 1, 0) ||
|
|
1091
|
+
!this.scheduler.hasQueueCapacity(request.verificationOfTaskId !== undefined)) {
|
|
1092
|
+
this.cancelAndPublish(prepared.lifecycle, prepared.record.laneId, "worker_dispatch_queue_full");
|
|
1093
|
+
return { started: false, skipReason: "worker_dispatch_queue_full" };
|
|
1094
|
+
}
|
|
1095
|
+
if (options.ensureAgent !== false) {
|
|
1096
|
+
try {
|
|
1097
|
+
this.ensurePreparedAgent(prepared, admission);
|
|
1098
|
+
}
|
|
1099
|
+
catch (error) {
|
|
1100
|
+
this.cancelAndPublish(prepared.lifecycle, prepared.record.laneId, "worker_conversation_unavailable");
|
|
1101
|
+
this.safeWarn(`Worker conversation setup failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1102
|
+
return { started: false, skipReason: "worker_conversation_unavailable" };
|
|
1103
|
+
}
|
|
1104
|
+
}
|
|
1105
|
+
const priority = request.verificationOfTaskId !== undefined;
|
|
1106
|
+
const enqueueSkipReason = this.enqueuePreparedWorkerAttempt(prepared.record, request, admission.shipment.profile.profileId, priority);
|
|
1107
|
+
if (enqueueSkipReason)
|
|
1108
|
+
return { started: false, skipReason: enqueueSkipReason };
|
|
1109
|
+
this.notifications.statusChanged();
|
|
1110
|
+
options.onStarted?.(prepared.record);
|
|
1111
|
+
if (options.drain)
|
|
1112
|
+
this.scheduler.drain();
|
|
1113
|
+
return { started: true, record: prepared.record };
|
|
655
1114
|
}
|
|
656
1115
|
ensurePreparedAgent(prepared, admission) {
|
|
657
1116
|
if (!prepared.attempt)
|
|
@@ -660,6 +1119,7 @@ export class WorkerDelegationController {
|
|
|
660
1119
|
if (!immutableWorker)
|
|
661
1120
|
throw new Error("Prepared worker execution contract is missing.");
|
|
662
1121
|
const agentId = prepared.attempt.agentId ?? prepared.attempt.dispatch.logicalLaneId ?? prepared.record.laneId;
|
|
1122
|
+
const birthContextForkReference = prepared.attempt.dispatch.birthContextForkReference;
|
|
663
1123
|
const existing = prepared.lifecycle.getAgent(agentId);
|
|
664
1124
|
if (existing) {
|
|
665
1125
|
const conversation = this.conversations.open({
|
|
@@ -667,6 +1127,9 @@ export class WorkerDelegationController {
|
|
|
667
1127
|
resumeContext: existing.resumeContext,
|
|
668
1128
|
expectedLogicalAgentId: existing.agentId,
|
|
669
1129
|
});
|
|
1130
|
+
if (!isDeepStrictEqual(conversation.getBirthContextForkReference(), birthContextForkReference)) {
|
|
1131
|
+
throw new Error("Worker conversation birth context conflicts with its durable attempt.");
|
|
1132
|
+
}
|
|
670
1133
|
const materialized = materializeWorkerResourceBundle(existing.resumeContext.contextPointers);
|
|
671
1134
|
if (!materialized.ok)
|
|
672
1135
|
throw new Error(`worker_resource_materialization_${materialized.code}`);
|
|
@@ -691,6 +1154,7 @@ export class WorkerDelegationController {
|
|
|
691
1154
|
modelRef: `${immutableWorker.modelBinding.provider}/${immutableWorker.modelBinding.modelId}`,
|
|
692
1155
|
resourceProfileNames: immutableWorker.profile.resourceProfileNames,
|
|
693
1156
|
contextPointers: materialized.pointers,
|
|
1157
|
+
...(birthContextForkReference ? { birthContextForkReference } : {}),
|
|
694
1158
|
});
|
|
695
1159
|
prepared.lifecycle.ensureAgent({
|
|
696
1160
|
agentId,
|
|
@@ -708,40 +1172,38 @@ export class WorkerDelegationController {
|
|
|
708
1172
|
return this.startInternal(request);
|
|
709
1173
|
}
|
|
710
1174
|
startInternal(request, pinnedContract) {
|
|
711
|
-
const admission = this.
|
|
1175
|
+
const admission = this.admitNewWorkerRequest(request, pinnedContract);
|
|
712
1176
|
if (!admission.ok)
|
|
713
1177
|
return { started: false, skipReason: admission.skipReason };
|
|
714
|
-
if (!request.verificationOfTaskId)
|
|
715
|
-
this.recovery.recover();
|
|
716
1178
|
const { settings, shipment } = admission;
|
|
717
1179
|
const foreground = this.deps.getModel();
|
|
718
1180
|
const contendsWithLocalForeground = foreground !== undefined && isLocalExecutionModel(foreground) && isLocalExecutionModel(shipment.model);
|
|
719
|
-
|
|
1181
|
+
const dependencyGated = (request.taskContext?.dependsOnTaskIds.length ?? 0) > 0;
|
|
1182
|
+
if (dependencyGated || contendsWithLocalForeground || !this.hasWorkerCapacity(settings)) {
|
|
720
1183
|
// A mandatory verifier is the continuation of an already admitted implementation, not a
|
|
721
1184
|
// new owner request. Reserve its queue admission so a burst of ordinary work cannot strand
|
|
722
1185
|
// the subject behind `independent_verification_required` with no terminal handoff.
|
|
723
|
-
|
|
1186
|
+
const priority = request.verificationOfTaskId !== undefined;
|
|
1187
|
+
if (this.workerQueueReservationSkipReason(request, 1, admission.verifierShipment ? 1 : 0) ||
|
|
1188
|
+
!this.scheduler.hasQueueCapacity(priority)) {
|
|
1189
|
+
return { started: false, skipReason: "worker_dispatch_queue_full" };
|
|
1190
|
+
}
|
|
1191
|
+
let prepared;
|
|
724
1192
|
try {
|
|
725
|
-
|
|
726
|
-
record = prepared.record;
|
|
727
|
-
this.ensurePreparedAgent(prepared, admission);
|
|
1193
|
+
prepared = this.prepareWorkerAttempt(request, admission);
|
|
728
1194
|
}
|
|
729
1195
|
catch (error) {
|
|
730
|
-
if (record)
|
|
731
|
-
this.cancelAndPublish(this.lifecycle, record.laneId, "worker_conversation_unavailable");
|
|
732
1196
|
this.safeWarn(`Worker dispatch was not persisted: ${error instanceof Error ? error.message : String(error)}`);
|
|
733
1197
|
return { started: false, skipReason: "orchestration_ledger_error" };
|
|
734
1198
|
}
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
this.notifications.statusChanged();
|
|
739
|
-
return { started: true, record };
|
|
1199
|
+
return this.queuePreparedWorkerAttempt(prepared, request, admission, {
|
|
1200
|
+
drain: dependencyGated,
|
|
1201
|
+
});
|
|
740
1202
|
}
|
|
741
1203
|
let startedRecord;
|
|
742
1204
|
const promise = this.runOnceWithAdmission(request, (record) => {
|
|
743
1205
|
startedRecord = record;
|
|
744
|
-
}, undefined, admission);
|
|
1206
|
+
}, undefined, admission, true);
|
|
745
1207
|
if (!startedRecord) {
|
|
746
1208
|
// Preparation is synchronous up to the first isolated completion await. A promise that
|
|
747
1209
|
// rejected before producing a lane is still observed below, so it cannot become unhandled.
|
|
@@ -754,14 +1216,18 @@ export class WorkerDelegationController {
|
|
|
754
1216
|
async runOnce(request, onStarted, existingRecord) {
|
|
755
1217
|
return this.runOnceWithAdmission(request, onStarted, existingRecord);
|
|
756
1218
|
}
|
|
757
|
-
async runOnceWithAdmission(request, onStarted, existingRecord, preparedAdmission) {
|
|
1219
|
+
async runOnceWithAdmission(request, onStarted, existingRecord, preparedAdmission, newWorkerAdmissionChecked = false) {
|
|
758
1220
|
const pinnedContract = existingRecord
|
|
759
1221
|
? this.getWorkerLifecycle().getActiveAttempt(existingRecord.laneId)?.dispatch.executionContract
|
|
760
1222
|
: undefined;
|
|
761
|
-
const admission =
|
|
1223
|
+
const admission = existingRecord
|
|
1224
|
+
? this.resolveWorkerAdmission(request, pinnedContract)
|
|
1225
|
+
: preparedAdmission && newWorkerAdmissionChecked
|
|
1226
|
+
? preparedAdmission
|
|
1227
|
+
: this.admitNewWorkerRequest(request, pinnedContract);
|
|
762
1228
|
if (!admission.ok)
|
|
763
1229
|
return { started: false, skipReason: admission.skipReason };
|
|
764
|
-
if (!request.verificationOfTaskId)
|
|
1230
|
+
if (existingRecord && !request.verificationOfTaskId)
|
|
765
1231
|
this.recovery.recover();
|
|
766
1232
|
const { instructions, settings, verifierShipment } = admission;
|
|
767
1233
|
const { model, modelBinding, profile: orchestrationProfile, soul } = admission.shipment;
|
|
@@ -774,6 +1240,16 @@ export class WorkerDelegationController {
|
|
|
774
1240
|
const { executionPlan, lifecycle } = prepared;
|
|
775
1241
|
if (!prepared.attempt)
|
|
776
1242
|
return { started: false, skipReason: "orchestration_attempt_missing" };
|
|
1243
|
+
if (prepared.attempt.status === "queued") {
|
|
1244
|
+
const readiness = lifecycle.getAttemptDispatchReadiness(prepared.attempt.attemptId);
|
|
1245
|
+
if (readiness.state === "blocked") {
|
|
1246
|
+
this.cancelAndPublish(lifecycle, prepared.record.laneId, readiness.reasonCode);
|
|
1247
|
+
return { started: false, skipReason: readiness.reasonCode };
|
|
1248
|
+
}
|
|
1249
|
+
if (readiness.state === "waiting") {
|
|
1250
|
+
return this.queuePreparedWorkerAttempt(prepared, request, admission, { onStarted });
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
777
1253
|
let preparedAgent;
|
|
778
1254
|
try {
|
|
779
1255
|
preparedAgent = this.ensurePreparedAgent(prepared, admission);
|
|
@@ -826,6 +1302,16 @@ export class WorkerDelegationController {
|
|
|
826
1302
|
return { started: false, skipReason: "orchestration_agent_missing" };
|
|
827
1303
|
}
|
|
828
1304
|
const conversation = preparedAgent.conversation;
|
|
1305
|
+
if (prepared.attempt.status === "queued" || conversation.usesAttemptUsageBoundaries()) {
|
|
1306
|
+
try {
|
|
1307
|
+
conversation.beginAttemptUsage(prepared.attempt.attemptId);
|
|
1308
|
+
}
|
|
1309
|
+
catch (error) {
|
|
1310
|
+
this.cancelAndPublish(lifecycle, prepared.record.laneId, "worker_conversation_unavailable");
|
|
1311
|
+
this.safeWarn(`Worker usage boundary setup failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1312
|
+
return { started: false, skipReason: "worker_conversation_unavailable" };
|
|
1313
|
+
}
|
|
1314
|
+
}
|
|
829
1315
|
if (prepared.attempt.status === "suspended")
|
|
830
1316
|
this.recovery.repairInterruptedToolResults(conversation);
|
|
831
1317
|
const reservation = this.writeReservations.acquire(prepared.record.laneId, prepared.attempt, executionPlan);
|
|
@@ -834,10 +1320,10 @@ export class WorkerDelegationController {
|
|
|
834
1320
|
return { started: false, skipReason: reservation.reasonCode };
|
|
835
1321
|
}
|
|
836
1322
|
if (reservation.kind === "blocked") {
|
|
837
|
-
this.
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
1323
|
+
return this.queuePreparedWorkerAttempt(prepared, request, admission, {
|
|
1324
|
+
ensureAgent: false,
|
|
1325
|
+
onStarted,
|
|
1326
|
+
});
|
|
841
1327
|
}
|
|
842
1328
|
let durableHandle;
|
|
843
1329
|
try {
|
|
@@ -864,9 +1350,11 @@ export class WorkerDelegationController {
|
|
|
864
1350
|
this.safeWarn("Worker write reservation fence did not match the durable attempt lease.");
|
|
865
1351
|
return { started: false, skipReason: record?.reasonCode ?? "write_reservation_fence_mismatch" };
|
|
866
1352
|
}
|
|
867
|
-
const recoveredTerminal = prepared.attempt.status === "suspended"
|
|
1353
|
+
const recoveredTerminal = prepared.attempt.status === "suspended"
|
|
1354
|
+
? this.recovery.recoveredTerminalCompletion(conversation, durableHandle.attemptId)
|
|
1355
|
+
: undefined;
|
|
868
1356
|
const checkpointUsage = lifecycle.getAttemptUsage(startedRecord.laneId);
|
|
869
|
-
const initialUsage = this.recovery.initialUsage(conversation, checkpointUsage);
|
|
1357
|
+
const initialUsage = this.recovery.initialUsage(conversation, checkpointUsage, durableHandle.attemptId);
|
|
870
1358
|
onStarted?.(startedRecord);
|
|
871
1359
|
const maxUsd = grant.budget.maxCostUsd;
|
|
872
1360
|
const executionPolicy = orchestrationProfile.executionPolicy;
|
|
@@ -875,7 +1363,11 @@ export class WorkerDelegationController {
|
|
|
875
1363
|
runWorkerDelegation: (childRequest) => this.runOnce({ ...childRequest, parentAgentId: agentId }),
|
|
876
1364
|
orchestrationProfiles: this.getProfileCatalog(),
|
|
877
1365
|
workerAgentControl: this.agentControl,
|
|
878
|
-
|
|
1366
|
+
caller: { kind: "worker", agentId },
|
|
1367
|
+
resolveMessageReplayScope: () => ({
|
|
1368
|
+
sessionId: this.deps.getSessionId(),
|
|
1369
|
+
branchId: durableHandle.attemptId,
|
|
1370
|
+
}),
|
|
879
1371
|
}));
|
|
880
1372
|
const agentBinding = lifecycle.getAgent(agentId);
|
|
881
1373
|
if (!agentBinding) {
|
|
@@ -893,6 +1385,9 @@ export class WorkerDelegationController {
|
|
|
893
1385
|
});
|
|
894
1386
|
const shellGranted = executionPlan.toolManifests.some((manifest) => manifest.toolName === STABLE_SHELL_TOOL_NAME);
|
|
895
1387
|
const shellSessionKey = shellGranted ? `worker:${this.deps.getSessionId()}:${agentId}` : undefined;
|
|
1388
|
+
const shellOutputDirectory = shellGranted
|
|
1389
|
+
? getProcessWorkRun(this.deps.getAgentDir(), "outputs", "tool-streams").path
|
|
1390
|
+
: undefined;
|
|
896
1391
|
if (shellSessionKey)
|
|
897
1392
|
this.shellSessionKeys.add(shellSessionKey);
|
|
898
1393
|
const toolSurface = createLaneToolSurface({
|
|
@@ -903,6 +1398,7 @@ export class WorkerDelegationController {
|
|
|
903
1398
|
...(executionPlan.processEnabled && executionPolicy ? { executionPolicy } : {}),
|
|
904
1399
|
processMaxWallClockMs: grant.budget.maxWallClockMs ?? 0,
|
|
905
1400
|
...(shellSessionKey ? { shellSessionKey } : {}),
|
|
1401
|
+
...(shellOutputDirectory ? { shellOutputDirectory } : {}),
|
|
906
1402
|
grant,
|
|
907
1403
|
toolManifests: executionPlan.toolManifests,
|
|
908
1404
|
additionalTools: [recursiveDelegateTool],
|
|
@@ -1007,6 +1503,23 @@ export class WorkerDelegationController {
|
|
|
1007
1503
|
});
|
|
1008
1504
|
const executionResult = await executor.run();
|
|
1009
1505
|
const rawOutcome = executionResult.rawOutcome;
|
|
1506
|
+
// Attempt ladder: a retryable bounded failure suspends and re-enqueues instead of
|
|
1507
|
+
// terminalizing, resuming from the persisted transcript under a fresh fence.
|
|
1508
|
+
if (rawOutcome.laneStatus === "failed" && !this.deps.isDisposed() && !workerSignal.aborted) {
|
|
1509
|
+
const retry = this.recovery.scheduleAttemptRetry({
|
|
1510
|
+
laneId: startedRecord.laneId,
|
|
1511
|
+
agentId,
|
|
1512
|
+
ownerId: this.agentControl.getProcessOwnerId(),
|
|
1513
|
+
request: { ...request, profileId: admission.shipment.profile.profileId },
|
|
1514
|
+
outcome: rawOutcome,
|
|
1515
|
+
provider: modelBinding.provider,
|
|
1516
|
+
...(grant.budget.maxAttempts !== undefined ? { maxAttempts: grant.budget.maxAttempts } : {}),
|
|
1517
|
+
});
|
|
1518
|
+
if (retry.scheduled) {
|
|
1519
|
+
this.notifications.statusChanged();
|
|
1520
|
+
return { started: true, record: retry.record };
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1010
1523
|
const verificationRequired = orchestrationProfile.requireIndependentVerification &&
|
|
1011
1524
|
orchestrationProfile.role !== "verifier" &&
|
|
1012
1525
|
rawOutcome.claim.status === "completed";
|
|
@@ -1189,6 +1702,7 @@ export class WorkerDelegationController {
|
|
|
1189
1702
|
drain() {
|
|
1190
1703
|
this.recovery.recover();
|
|
1191
1704
|
this.scheduler.drain();
|
|
1705
|
+
this.terminalHandoffs.signal();
|
|
1192
1706
|
}
|
|
1193
1707
|
}
|
|
1194
1708
|
//# sourceMappingURL=worker-delegation-controller.js.map
|