@caupulican/pi-adaptative 0.86.3 → 0.86.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/core/agent-paths.d.ts +13 -0
- package/dist/core/agent-paths.d.ts.map +1 -1
- package/dist/core/agent-paths.js +23 -0
- package/dist/core/agent-paths.js.map +1 -1
- package/dist/core/autonomy/bounded-completion.d.ts +7 -0
- package/dist/core/autonomy/bounded-completion.d.ts.map +1 -1
- package/dist/core/autonomy/bounded-completion.js +12 -4
- package/dist/core/autonomy/bounded-completion.js.map +1 -1
- package/dist/core/autonomy/lane-tool-surface.d.ts +2 -0
- package/dist/core/autonomy/lane-tool-surface.d.ts.map +1 -1
- package/dist/core/autonomy/lane-tool-surface.js +6 -3
- package/dist/core/autonomy/lane-tool-surface.js.map +1 -1
- package/dist/core/autonomy/subagent-prompt.js +1 -1
- package/dist/core/autonomy/subagent-prompt.js.map +1 -1
- package/dist/core/background-lane-controller.d.ts +25 -2
- package/dist/core/background-lane-controller.d.ts.map +1 -1
- package/dist/core/background-lane-controller.js +67 -0
- package/dist/core/background-lane-controller.js.map +1 -1
- package/dist/core/delegation/replay-safe-mailbox-bounds.d.ts +4 -0
- package/dist/core/delegation/replay-safe-mailbox-bounds.d.ts.map +1 -0
- package/dist/core/delegation/replay-safe-mailbox-bounds.js +20 -0
- package/dist/core/delegation/replay-safe-mailbox-bounds.js.map +1 -0
- package/dist/core/delegation/sanitized-context-fork.d.ts +25 -0
- package/dist/core/delegation/sanitized-context-fork.d.ts.map +1 -0
- package/dist/core/delegation/sanitized-context-fork.js +299 -0
- package/dist/core/delegation/sanitized-context-fork.js.map +1 -0
- package/dist/core/delegation/session-root-mailbox.d.ts +75 -0
- package/dist/core/delegation/session-root-mailbox.d.ts.map +1 -0
- package/dist/core/delegation/session-root-mailbox.js +627 -0
- package/dist/core/delegation/session-root-mailbox.js.map +1 -0
- package/dist/core/delegation/worker-agent-control-coordinator.d.ts +102 -9
- package/dist/core/delegation/worker-agent-control-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-agent-control-coordinator.js +1307 -125
- package/dist/core/delegation/worker-agent-control-coordinator.js.map +1 -1
- package/dist/core/delegation/worker-agent-control.d.ts +183 -8
- package/dist/core/delegation/worker-agent-control.d.ts.map +1 -1
- package/dist/core/delegation/worker-agent-control.js +778 -48
- package/dist/core/delegation/worker-agent-control.js.map +1 -1
- package/dist/core/delegation/worker-attempt-executor.d.ts +9 -0
- package/dist/core/delegation/worker-attempt-executor.d.ts.map +1 -1
- package/dist/core/delegation/worker-attempt-executor.js +479 -214
- package/dist/core/delegation/worker-attempt-executor.js.map +1 -1
- package/dist/core/delegation/worker-authority-resolver.d.ts +9 -0
- package/dist/core/delegation/worker-authority-resolver.d.ts.map +1 -1
- package/dist/core/delegation/worker-authority-resolver.js +21 -2
- package/dist/core/delegation/worker-authority-resolver.js.map +1 -1
- package/dist/core/delegation/worker-context-fork-store.d.ts +71 -0
- package/dist/core/delegation/worker-context-fork-store.d.ts.map +1 -0
- package/dist/core/delegation/worker-context-fork-store.js +437 -0
- package/dist/core/delegation/worker-context-fork-store.js.map +1 -0
- package/dist/core/delegation/worker-context-inheritance-policy.d.ts +18 -0
- package/dist/core/delegation/worker-context-inheritance-policy.d.ts.map +1 -0
- package/dist/core/delegation/worker-context-inheritance-policy.js +28 -0
- package/dist/core/delegation/worker-context-inheritance-policy.js.map +1 -0
- package/dist/core/delegation/worker-conversation-revision.d.ts +31 -0
- package/dist/core/delegation/worker-conversation-revision.d.ts.map +1 -0
- package/dist/core/delegation/worker-conversation-revision.js +203 -0
- package/dist/core/delegation/worker-conversation-revision.js.map +1 -0
- package/dist/core/delegation/worker-conversation-store.d.ts +130 -18
- package/dist/core/delegation/worker-conversation-store.d.ts.map +1 -1
- package/dist/core/delegation/worker-conversation-store.js +1263 -79
- package/dist/core/delegation/worker-conversation-store.js.map +1 -1
- package/dist/core/delegation/worker-delegation-controller.d.ts +29 -2
- package/dist/core/delegation/worker-delegation-controller.d.ts.map +1 -1
- package/dist/core/delegation/worker-delegation-controller.js +625 -111
- package/dist/core/delegation/worker-delegation-controller.js.map +1 -1
- package/dist/core/delegation/worker-delegation-request.d.ts +2 -0
- package/dist/core/delegation/worker-delegation-request.d.ts.map +1 -1
- package/dist/core/delegation/worker-delegation-request.js.map +1 -1
- package/dist/core/delegation/worker-dispatch-scheduler.d.ts +20 -1
- package/dist/core/delegation/worker-dispatch-scheduler.d.ts.map +1 -1
- package/dist/core/delegation/worker-dispatch-scheduler.js +210 -37
- package/dist/core/delegation/worker-dispatch-scheduler.js.map +1 -1
- package/dist/core/delegation/worker-fleet-limits.d.ts +38 -0
- package/dist/core/delegation/worker-fleet-limits.d.ts.map +1 -0
- package/dist/core/delegation/worker-fleet-limits.js +113 -0
- package/dist/core/delegation/worker-fleet-limits.js.map +1 -0
- package/dist/core/delegation/worker-lane-projection.d.ts +1 -0
- package/dist/core/delegation/worker-lane-projection.d.ts.map +1 -1
- package/dist/core/delegation/worker-lane-projection.js +6 -2
- package/dist/core/delegation/worker-lane-projection.js.map +1 -1
- package/dist/core/delegation/worker-lifecycle.d.ts +20 -3
- package/dist/core/delegation/worker-lifecycle.d.ts.map +1 -1
- package/dist/core/delegation/worker-lifecycle.js +42 -18
- package/dist/core/delegation/worker-lifecycle.js.map +1 -1
- package/dist/core/delegation/worker-notification-coordinator.d.ts +4 -0
- package/dist/core/delegation/worker-notification-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-notification-coordinator.js +45 -4
- package/dist/core/delegation/worker-notification-coordinator.js.map +1 -1
- package/dist/core/delegation/worker-provider-turn-protocol.d.ts +50 -0
- package/dist/core/delegation/worker-provider-turn-protocol.d.ts.map +1 -0
- package/dist/core/delegation/worker-provider-turn-protocol.js +185 -0
- package/dist/core/delegation/worker-provider-turn-protocol.js.map +1 -0
- package/dist/core/delegation/worker-recovery-coordinator.d.ts +45 -4
- package/dist/core/delegation/worker-recovery-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-recovery-coordinator.js +223 -17
- package/dist/core/delegation/worker-recovery-coordinator.js.map +1 -1
- package/dist/core/delegation/worker-retry-policy.d.ts +36 -0
- package/dist/core/delegation/worker-retry-policy.d.ts.map +1 -0
- package/dist/core/delegation/worker-retry-policy.js +41 -0
- package/dist/core/delegation/worker-retry-policy.js.map +1 -0
- package/dist/core/delegation/worker-runner.d.ts +3 -1
- package/dist/core/delegation/worker-runner.d.ts.map +1 -1
- package/dist/core/delegation/worker-runner.js +4 -2
- package/dist/core/delegation/worker-runner.js.map +1 -1
- package/dist/core/delegation/worker-task-view.d.ts +43 -0
- package/dist/core/delegation/worker-task-view.d.ts.map +1 -0
- package/dist/core/delegation/worker-task-view.js +194 -0
- package/dist/core/delegation/worker-task-view.js.map +1 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.d.ts +55 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.d.ts.map +1 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.js +179 -0
- package/dist/core/delegation/worker-terminal-handoff-coordinator.js.map +1 -0
- package/dist/core/delegation/worker-tree-budget-coordinator.d.ts +2 -2
- package/dist/core/delegation/worker-tree-budget-coordinator.d.ts.map +1 -1
- package/dist/core/delegation/worker-tree-budget-coordinator.js +12 -46
- package/dist/core/delegation/worker-tree-budget-coordinator.js.map +1 -1
- package/dist/core/keybindings.d.ts +10 -0
- package/dist/core/keybindings.d.ts.map +1 -1
- package/dist/core/keybindings.js +2 -0
- package/dist/core/keybindings.js.map +1 -1
- package/dist/core/orchestration/attempt-ordering.d.ts +9 -0
- package/dist/core/orchestration/attempt-ordering.d.ts.map +1 -0
- package/dist/core/orchestration/attempt-ordering.js +29 -0
- package/dist/core/orchestration/attempt-ordering.js.map +1 -0
- package/dist/core/orchestration/capability-gateway.d.ts +19 -0
- package/dist/core/orchestration/capability-gateway.d.ts.map +1 -1
- package/dist/core/orchestration/capability-gateway.js +19 -2
- package/dist/core/orchestration/capability-gateway.js.map +1 -1
- package/dist/core/orchestration/contracts.d.ts +43 -3
- package/dist/core/orchestration/contracts.d.ts.map +1 -1
- package/dist/core/orchestration/contracts.js +24 -0
- package/dist/core/orchestration/contracts.js.map +1 -1
- package/dist/core/orchestration/delegation-ledger.d.ts +8 -3
- package/dist/core/orchestration/delegation-ledger.d.ts.map +1 -1
- package/dist/core/orchestration/delegation-ledger.js +148 -55
- package/dist/core/orchestration/delegation-ledger.js.map +1 -1
- package/dist/core/orchestration/event-store.d.ts +6 -3
- package/dist/core/orchestration/event-store.d.ts.map +1 -1
- package/dist/core/orchestration/event-store.js +228 -30
- package/dist/core/orchestration/event-store.js.map +1 -1
- package/dist/core/orchestration/task-runtime-codecs.d.ts +31 -0
- package/dist/core/orchestration/task-runtime-codecs.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-codecs.js +1471 -0
- package/dist/core/orchestration/task-runtime-codecs.js.map +1 -0
- package/dist/core/orchestration/task-runtime-projection.d.ts +30 -0
- package/dist/core/orchestration/task-runtime-projection.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-projection.js +340 -0
- package/dist/core/orchestration/task-runtime-projection.js.map +1 -0
- package/dist/core/orchestration/task-runtime-reducer.d.ts +38 -0
- package/dist/core/orchestration/task-runtime-reducer.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-reducer.js +1270 -0
- package/dist/core/orchestration/task-runtime-reducer.js.map +1 -0
- package/dist/core/orchestration/task-runtime-state.d.ts +142 -0
- package/dist/core/orchestration/task-runtime-state.d.ts.map +1 -0
- package/dist/core/orchestration/task-runtime-state.js +14 -0
- package/dist/core/orchestration/task-runtime-state.js.map +1 -0
- package/dist/core/orchestration/task-runtime.d.ts +44 -98
- package/dist/core/orchestration/task-runtime.d.ts.map +1 -1
- package/dist/core/orchestration/task-runtime.js +357 -1113
- package/dist/core/orchestration/task-runtime.js.map +1 -1
- package/dist/core/orchestration/worker-context-fork-reference.d.ts +20 -0
- package/dist/core/orchestration/worker-context-fork-reference.d.ts.map +1 -0
- package/dist/core/orchestration/worker-context-fork-reference.js +53 -0
- package/dist/core/orchestration/worker-context-fork-reference.js.map +1 -0
- package/dist/core/research/research-runner.d.ts +2 -0
- package/dist/core/research/research-runner.d.ts.map +1 -1
- package/dist/core/research/research-runner.js +1 -0
- package/dist/core/research/research-runner.js.map +1 -1
- package/dist/core/runtime-builder.d.ts.map +1 -1
- package/dist/core/runtime-builder.js +8 -0
- package/dist/core/runtime-builder.js.map +1 -1
- package/dist/core/secrets/credential-project.d.ts.map +1 -1
- package/dist/core/secrets/credential-project.js +24 -2
- package/dist/core/secrets/credential-project.js.map +1 -1
- package/dist/core/security/secret-text.d.ts +3 -0
- package/dist/core/security/secret-text.d.ts.map +1 -1
- package/dist/core/security/secret-text.js +15 -0
- package/dist/core/security/secret-text.js.map +1 -1
- package/dist/core/settings-manager.d.ts +3 -0
- package/dist/core/settings-manager.d.ts.map +1 -1
- package/dist/core/settings-manager.js +160 -8
- package/dist/core/settings-manager.js.map +1 -1
- package/dist/core/tools/delegate.d.ts +31 -4
- package/dist/core/tools/delegate.d.ts.map +1 -1
- package/dist/core/tools/delegate.js +749 -103
- package/dist/core/tools/delegate.js.map +1 -1
- package/dist/main.d.ts.map +1 -1
- package/dist/main.js +1 -0
- package/dist/main.js.map +1 -1
- package/dist/modes/interactive/components/activity-lane.d.ts +5 -0
- package/dist/modes/interactive/components/activity-lane.d.ts.map +1 -1
- package/dist/modes/interactive/components/activity-lane.js +133 -49
- package/dist/modes/interactive/components/activity-lane.js.map +1 -1
- package/dist/modes/interactive/components/agents-overlay.d.ts +40 -0
- package/dist/modes/interactive/components/agents-overlay.d.ts.map +1 -0
- package/dist/modes/interactive/components/agents-overlay.js +153 -0
- package/dist/modes/interactive/components/agents-overlay.js.map +1 -0
- package/dist/modes/interactive/components/settings-selector.d.ts.map +1 -1
- package/dist/modes/interactive/components/settings-selector.js +18 -13
- package/dist/modes/interactive/components/settings-selector.js.map +1 -1
- package/dist/modes/interactive/interactive-event-controller.d.ts +1 -1
- package/dist/modes/interactive/interactive-event-controller.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-event-controller.js +6 -3
- package/dist/modes/interactive/interactive-event-controller.js.map +1 -1
- package/dist/modes/interactive/interactive-mode.d.ts +5 -0
- package/dist/modes/interactive/interactive-mode.d.ts.map +1 -1
- package/dist/modes/interactive/interactive-mode.js +51 -0
- package/dist/modes/interactive/interactive-mode.js.map +1 -1
- package/dist/modes/interactive/key-handlers.d.ts +1 -0
- package/dist/modes/interactive/key-handlers.d.ts.map +1 -1
- package/dist/modes/interactive/key-handlers.js +1 -0
- package/dist/modes/interactive/key-handlers.js.map +1 -1
- package/docs/harness-architecture.md +61 -17
- package/docs/settings.md +17 -9
- package/docs/worker-profiles.md +6 -2
- package/examples/extensions/custom-provider-anthropic/package-lock.json +2 -2
- package/examples/extensions/custom-provider-anthropic/package.json +1 -1
- package/examples/extensions/custom-provider-gitlab-duo/package.json +1 -1
- package/examples/extensions/sandbox/package-lock.json +2 -2
- package/examples/extensions/sandbox/package.json +1 -1
- package/examples/extensions/with-deps/package-lock.json +2 -2
- package/examples/extensions/with-deps/package.json +1 -1
- package/npm-shrinkwrap.json +12 -12
- package/package.json +4 -4
|
@@ -1,5 +1,11 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { latestAgentAttemptsByDurableOrder } from "../orchestration/attempt-ordering.js";
|
|
2
|
+
import { MAX_ORCHESTRATION_COLLECTION_LENGTH, MAX_ORCHESTRATION_IDENTIFIER_LENGTH, } from "../orchestration/contracts.js";
|
|
3
|
+
import { SessionRootMailbox, sessionRootAddress, sessionRootReplyMessageId, } from "./session-root-mailbox.js";
|
|
4
|
+
import { normalizeWorkerAgentDependencyTaskIds, WorkerAgentMailbox, workerAgentBroadcastTargetIdempotencyKey, workerAgentMessageId, } from "./worker-agent-control.js";
|
|
5
|
+
import { MAX_WORKER_TRANSCRIPT_PAGE_MESSAGES, WorkerConversationStore, } from "./worker-conversation-store.js";
|
|
6
|
+
import { evaluateReusableWorkerTaskAdmission } from "./worker-fleet-limits.js";
|
|
7
|
+
import { projectWorkerTaskSessionView } from "./worker-task-view.js";
|
|
8
|
+
const MAX_BROADCAST_ERROR_CHARS = 512;
|
|
3
9
|
/**
|
|
4
10
|
* Sole owner of model-facing logical-agent controls and their durable inboxes.
|
|
5
11
|
*
|
|
@@ -10,88 +16,755 @@ export class WorkerAgentControlCoordinator {
|
|
|
10
16
|
constructor(options) {
|
|
11
17
|
this.mailboxes = new Map();
|
|
12
18
|
this.stateListeners = new Set();
|
|
13
|
-
this.
|
|
19
|
+
this.reconcilingTaskBearingAgentIds = new Set();
|
|
20
|
+
this.taskBearingContinuationAgentIds = new Set();
|
|
21
|
+
this.sessionRootReconciliationFailures = new Map();
|
|
22
|
+
this.workerReplyReconciliationFailures = new Map();
|
|
14
23
|
this.options = options;
|
|
24
|
+
this.conversations = options.conversationStore ?? new WorkerConversationStore();
|
|
25
|
+
this.sessionRootAddress = sessionRootAddress(options.parentSessionId);
|
|
26
|
+
this.sessionRootMailbox = new SessionRootMailbox({
|
|
27
|
+
agentDir: options.agentDir,
|
|
28
|
+
parentSessionId: options.parentSessionId,
|
|
29
|
+
});
|
|
15
30
|
}
|
|
16
31
|
getProcessOwnerId() {
|
|
17
32
|
return this.options.processOwnerId;
|
|
18
33
|
}
|
|
19
34
|
listWorkerAgents(scope = {}) {
|
|
20
35
|
this.requireControl();
|
|
21
|
-
const
|
|
22
|
-
|
|
36
|
+
const snapshot = this.options.getLifecycle().getTaskRuntimeSnapshot();
|
|
37
|
+
if (scope.callerAgentId) {
|
|
38
|
+
const callerAgentId = scope.callerAgentId.trim();
|
|
39
|
+
if (!callerAgentId || !snapshot.agents[callerAgentId]) {
|
|
40
|
+
throw new Error(`Unknown logical worker agent '${callerAgentId}'.`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const latestAttempts = this.latestAttemptsByAgent(snapshot);
|
|
44
|
+
const agents = Object.values(snapshot.agents);
|
|
23
45
|
return agents
|
|
24
|
-
.
|
|
25
|
-
.
|
|
46
|
+
.sort((left, right) => left.depth - right.depth || left.createdAt.localeCompare(right.createdAt))
|
|
47
|
+
.map((agent) => this.workerAgentView(agent, this.projectAgentActivity(agent, latestAttempts.get(agent.agentId))));
|
|
48
|
+
}
|
|
49
|
+
getWorkerTaskSessionView() {
|
|
50
|
+
this.requireControl();
|
|
51
|
+
return projectWorkerTaskSessionView(this.options.getLifecycle().getTaskRuntimeSnapshot());
|
|
52
|
+
}
|
|
53
|
+
/** Snapshot-only activity projection. Unlike waitForWorkerAgent, this never yields scheduler capacity. */
|
|
54
|
+
getWorkerAgentActivity(agentId, scope = {}) {
|
|
55
|
+
this.requireControl();
|
|
56
|
+
const canonicalAgentId = agentId.trim();
|
|
57
|
+
if (!canonicalAgentId)
|
|
58
|
+
throw new Error("Logical worker agent id is required.");
|
|
59
|
+
const agent = this.options.getLifecycle().getAgent(canonicalAgentId);
|
|
60
|
+
if (!agent)
|
|
61
|
+
return "unknown";
|
|
62
|
+
this.requireSessionPeer(canonicalAgentId, scope);
|
|
63
|
+
return this.activityForAgent(agent);
|
|
26
64
|
}
|
|
27
65
|
readWorkerAgentTranscript(agentId, options = {}) {
|
|
28
66
|
this.requireControl();
|
|
29
|
-
const agent = this.
|
|
67
|
+
const agent = this.requireControllableAgent(agentId, options);
|
|
30
68
|
const cursor = options.cursor ?? 0;
|
|
31
69
|
const maxMessages = options.maxMessages ?? 16;
|
|
32
70
|
if (!Number.isSafeInteger(cursor) || cursor < 0)
|
|
33
71
|
throw new TypeError("Worker transcript cursor is invalid.");
|
|
34
|
-
if (!Number.isSafeInteger(maxMessages) || maxMessages < 1 || maxMessages >
|
|
35
|
-
throw new TypeError(
|
|
72
|
+
if (!Number.isSafeInteger(maxMessages) || maxMessages < 1 || maxMessages > MAX_WORKER_TRANSCRIPT_PAGE_MESSAGES) {
|
|
73
|
+
throw new TypeError(`Worker transcript page size must be from 1 through ${MAX_WORKER_TRANSCRIPT_PAGE_MESSAGES} messages.`);
|
|
36
74
|
}
|
|
37
|
-
const
|
|
75
|
+
const page = this.conversations
|
|
38
76
|
.open({
|
|
39
77
|
agentDir: this.options.agentDir,
|
|
40
78
|
resumeContext: agent.resumeContext,
|
|
41
79
|
expectedLogicalAgentId: agent.agentId,
|
|
42
80
|
})
|
|
43
|
-
.
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
81
|
+
.getRawTranscriptPage({
|
|
82
|
+
cursor,
|
|
83
|
+
maxMessages,
|
|
84
|
+
...(options.maxBytes !== undefined ? { maxBytes: options.maxBytes } : {}),
|
|
85
|
+
});
|
|
48
86
|
return {
|
|
49
87
|
agentId: agent.agentId,
|
|
50
|
-
|
|
51
|
-
totalMessages: transcript.length,
|
|
52
|
-
messages,
|
|
53
|
-
...(nextCursor < transcript.length ? { nextCursor } : {}),
|
|
88
|
+
...page,
|
|
54
89
|
};
|
|
55
90
|
}
|
|
56
91
|
sendWorkerAgentMessage(agentId, message, options = {}) {
|
|
57
92
|
this.requireControl();
|
|
58
|
-
|
|
93
|
+
this.rejectGenericReplyOptions(options);
|
|
94
|
+
const agent = this.requireSessionPeer(agentId, { callerAgentId: options.senderAgentId });
|
|
95
|
+
this.assertAgentAcceptsNewMessages(agent, options.idempotencyKey);
|
|
59
96
|
const queued = this.enqueuePeerMessage(agent, "follow_up", message, options);
|
|
60
|
-
this.
|
|
97
|
+
this.notifyStateChangedBestEffort();
|
|
61
98
|
return { messageId: queued.messageId, queued: true };
|
|
62
99
|
}
|
|
100
|
+
broadcastWorkerAgentMessage(agentIds, message, options) {
|
|
101
|
+
this.requireControl();
|
|
102
|
+
const canonicalAgentIds = this.canonicalAgentIdSet(agentIds, "Worker broadcast");
|
|
103
|
+
const recipients = canonicalAgentIds.map((agentId) => ({
|
|
104
|
+
agentId,
|
|
105
|
+
idempotencyKey: workerAgentBroadcastTargetIdempotencyKey(options.idempotencyKey, agentId),
|
|
106
|
+
}));
|
|
107
|
+
const senderAgentId = options.senderAgentId ? this.requireKnownAgent(options.senderAgentId).agentId : undefined;
|
|
108
|
+
let created = false;
|
|
109
|
+
const results = recipients.map(({ agentId, idempotencyKey }) => {
|
|
110
|
+
try {
|
|
111
|
+
const target = this.requireSessionPeer(agentId, { callerAgentId: senderAgentId });
|
|
112
|
+
this.assertAgentAcceptsNewMessages(target, idempotencyKey);
|
|
113
|
+
const messageOptions = senderAgentId
|
|
114
|
+
? {
|
|
115
|
+
senderAgentId,
|
|
116
|
+
...(options.threadId !== undefined ? { threadId: options.threadId } : {}),
|
|
117
|
+
...(options.expectReply === true ? { expectReply: true } : {}),
|
|
118
|
+
idempotencyKey,
|
|
119
|
+
}
|
|
120
|
+
: this.sessionRootMessageOptions({
|
|
121
|
+
...(options.threadId !== undefined ? { threadId: options.threadId } : {}),
|
|
122
|
+
...(options.expectReply === true ? { expectReply: true } : {}),
|
|
123
|
+
idempotencyKey,
|
|
124
|
+
});
|
|
125
|
+
const queued = this.enqueuePeerMessage(target, "follow_up", message, messageOptions);
|
|
126
|
+
created ||= queued.status === "retained" && queued.created;
|
|
127
|
+
return {
|
|
128
|
+
agentId,
|
|
129
|
+
accepted: true,
|
|
130
|
+
queued: true,
|
|
131
|
+
replayed: queued.status === "completed_replay" || !queued.created,
|
|
132
|
+
messageId: queued.messageId,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
return {
|
|
137
|
+
agentId,
|
|
138
|
+
accepted: false,
|
|
139
|
+
error: (error instanceof Error ? error.message : String(error)).slice(0, MAX_BROADCAST_ERROR_CHARS),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
if (created)
|
|
144
|
+
this.notifyStateChangedBestEffort();
|
|
145
|
+
return { results };
|
|
146
|
+
}
|
|
63
147
|
followUpWorkerAgent(agentId, message, options = {}) {
|
|
64
148
|
this.requireControl();
|
|
65
|
-
const agent = this.
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
149
|
+
const agent = this.requireControllableAgent(agentId, { callerAgentId: options.senderAgentId });
|
|
150
|
+
return this.followUpAcceptedAgent(agent, message, options);
|
|
151
|
+
}
|
|
152
|
+
sendSessionRootWorkerAgentMessage(agentId, message, options = {}) {
|
|
153
|
+
this.requireControl();
|
|
154
|
+
const agent = this.requireKnownAgent(agentId);
|
|
155
|
+
this.assertAgentAcceptsNewMessages(agent, options.idempotencyKey);
|
|
156
|
+
const queued = this.enqueuePeerMessage(agent, "follow_up", message, this.sessionRootMessageOptions(options));
|
|
157
|
+
this.notifyStateChangedBestEffort();
|
|
158
|
+
return { messageId: queued.messageId, queued: true };
|
|
159
|
+
}
|
|
160
|
+
followUpSessionRootWorkerAgent(agentId, message, options = {}) {
|
|
161
|
+
this.requireControl();
|
|
162
|
+
return this.followUpAcceptedAgent(this.requireKnownAgent(agentId), message, this.sessionRootMessageOptions(options));
|
|
163
|
+
}
|
|
164
|
+
replyToWorkerAgentMessage(sourceAgentId, message, replyToMessageId) {
|
|
165
|
+
this.requireControl();
|
|
166
|
+
const source = this.requireKnownAgent(sourceAgentId);
|
|
167
|
+
const sourceMailbox = this.getMailbox(source.agentId);
|
|
168
|
+
const completedReply = sourceMailbox.resolveCompletedReply(replyToMessageId, message);
|
|
169
|
+
const activeAcknowledgementId = sourceMailbox.getReplyAcknowledgementId(replyToMessageId);
|
|
170
|
+
if (completedReply &&
|
|
171
|
+
activeAcknowledgementId !== undefined &&
|
|
172
|
+
activeAcknowledgementId !== completedReply.replyMessageId) {
|
|
173
|
+
throw new Error("Worker reply acknowledgement identity conflicts with its durable source receipt.");
|
|
174
|
+
}
|
|
175
|
+
if (completedReply && activeAcknowledgementId === undefined) {
|
|
176
|
+
if (completedReply.requestSenderId === this.sessionRootAddress) {
|
|
177
|
+
return { destination: "session_root", messageId: completedReply.replyMessageId };
|
|
178
|
+
}
|
|
179
|
+
const completedTarget = this.requireKnownAgent(completedReply.requestSenderId);
|
|
180
|
+
this.reconcileCompletedWorkerReplyAcknowledgement(sourceMailbox, replyToMessageId, completedTarget.agentId, completedReply.replyMessageId);
|
|
181
|
+
return {
|
|
182
|
+
destination: "worker",
|
|
183
|
+
messageId: completedReply.replyMessageId,
|
|
184
|
+
started: false,
|
|
185
|
+
steering: false,
|
|
186
|
+
skipReason: "worker_reply_already_accepted",
|
|
187
|
+
};
|
|
188
|
+
}
|
|
189
|
+
const request = sourceMailbox.getMessage(replyToMessageId);
|
|
190
|
+
if (!request || request.deliveredAt === undefined || request.expectReply !== true) {
|
|
191
|
+
throw new Error("Worker reply does not reference a delivered reply-expected message.");
|
|
192
|
+
}
|
|
193
|
+
if (!request.senderAgentId)
|
|
194
|
+
throw new Error("Worker reply request has no routable requester.");
|
|
195
|
+
if (request.senderAgentId === this.sessionRootAddress) {
|
|
196
|
+
return this.routeWorkerReplyToSessionRoot(source, sourceMailbox, request, message);
|
|
197
|
+
}
|
|
198
|
+
const target = this.requireKnownAgent(request.senderAgentId);
|
|
199
|
+
if (target.status === "retired" && activeAcknowledgementId === undefined) {
|
|
200
|
+
throw new Error(`Logical worker agent '${target.agentId}' is retired.`);
|
|
201
|
+
}
|
|
202
|
+
return this.routeWorkerReplyToAgent(target, source.agentId, request, message);
|
|
203
|
+
}
|
|
204
|
+
listSessionRootReplies(query = {}) {
|
|
205
|
+
this.requireControl();
|
|
206
|
+
this.reconcileSessionRootSurface();
|
|
207
|
+
return this.sessionRootMailbox.pendingReplies(query);
|
|
208
|
+
}
|
|
209
|
+
waitForSessionRootReplies(options = {}) {
|
|
210
|
+
this.requireControl();
|
|
211
|
+
this.reconcileSessionRootSurface();
|
|
212
|
+
return this.sessionRootMailbox.waitForReplies(options);
|
|
213
|
+
}
|
|
214
|
+
acknowledgeSessionRootReply(messageId, ackToken) {
|
|
215
|
+
this.requireControl();
|
|
216
|
+
this.reconcileSessionRootSurface();
|
|
217
|
+
const acknowledged = this.sessionRootMailbox.acknowledge(messageId, ackToken);
|
|
218
|
+
if (acknowledged) {
|
|
219
|
+
this.reconcileWorkerReplyOutboxesBestEffort();
|
|
220
|
+
this.notifyStateChangedBestEffort();
|
|
221
|
+
}
|
|
222
|
+
return acknowledged;
|
|
223
|
+
}
|
|
224
|
+
reconcileSessionRootSurface() {
|
|
225
|
+
this.reconcileSessionRootReplies();
|
|
226
|
+
this.reconcileWorkerReplyOutboxesBestEffort();
|
|
227
|
+
}
|
|
228
|
+
followUpAcceptedAgent(agent, message, options) {
|
|
229
|
+
this.rejectGenericReplyOptions(options);
|
|
230
|
+
if (this.activityForAgent(agent) === "active") {
|
|
231
|
+
const queued = this.enqueuePeerMessage(agent, "steer", message, options, { kind: "agent_turn" });
|
|
232
|
+
this.notifyStateChangedBestEffort();
|
|
233
|
+
if (queued.status === "completed_replay" ||
|
|
234
|
+
queued.message.deliveredAt !== undefined ||
|
|
235
|
+
queued.message.failedAt !== undefined) {
|
|
236
|
+
return {
|
|
237
|
+
started: false,
|
|
238
|
+
steering: false,
|
|
239
|
+
messageId: queued.messageId,
|
|
240
|
+
skipReason: "worker_message_already_finalized",
|
|
241
|
+
};
|
|
242
|
+
}
|
|
71
243
|
return { started: false, steering: true, messageId: queued.messageId };
|
|
72
244
|
}
|
|
73
245
|
if (agent.status !== "registered") {
|
|
74
246
|
return { started: false, steering: false, messageId: "", skipReason: `agent_${agent.status}` };
|
|
75
247
|
}
|
|
76
|
-
|
|
248
|
+
return this.startIdleAgentTask(agent, message, options);
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Atomically transition one idle persistent agent to a fresh task. There is no await between the
|
|
252
|
+
* activity check and durable prepare, so competing model calls cannot turn the loser into steering.
|
|
253
|
+
*/
|
|
254
|
+
startWorkerAgentTask(agentId, message, options = {}) {
|
|
255
|
+
this.requireControl();
|
|
256
|
+
const canonicalAgentId = agentId.trim();
|
|
257
|
+
if (!canonicalAgentId)
|
|
258
|
+
throw new Error("Logical worker agent id is required.");
|
|
259
|
+
const candidate = this.options.getLifecycle().getAgent(canonicalAgentId);
|
|
260
|
+
if (!candidate)
|
|
261
|
+
return { started: false, steering: false, messageId: "", skipReason: "unknown_agent" };
|
|
262
|
+
const agent = this.requireControllableAgent(canonicalAgentId, options);
|
|
263
|
+
const dependsOnTaskIds = normalizeWorkerAgentDependencyTaskIds(options.dependsOnTaskIds);
|
|
264
|
+
if (options.idempotencyKey !== undefined) {
|
|
265
|
+
const replay = this.replayWorkerAgentTask(agent, message, options.idempotencyKey, dependsOnTaskIds);
|
|
266
|
+
if (replay)
|
|
267
|
+
return replay;
|
|
268
|
+
}
|
|
269
|
+
const activity = this.activityForAgent(agent);
|
|
270
|
+
if (activity !== "idle") {
|
|
271
|
+
return { started: false, steering: false, messageId: "", skipReason: `worker_${activity}` };
|
|
272
|
+
}
|
|
273
|
+
if (agent.status !== "registered") {
|
|
274
|
+
return { started: false, steering: false, messageId: "", skipReason: `agent_${agent.status}` };
|
|
275
|
+
}
|
|
276
|
+
return this.startIdleAgentTask(agent, message, options.idempotencyKey === undefined ? {} : { idempotencyKey: options.idempotencyKey }, dependsOnTaskIds);
|
|
277
|
+
}
|
|
278
|
+
replayWorkerAgentTask(agent, message, idempotencyKey, dependsOnTaskIds) {
|
|
279
|
+
this.assertIdempotencyTarget(agent.agentId, idempotencyKey);
|
|
280
|
+
const messageId = workerAgentMessageId(this.options.parentSessionId, idempotencyKey);
|
|
281
|
+
const mailbox = this.getMailbox(agent.agentId);
|
|
282
|
+
const correlatedAttempt = this.controlMessageAttemptById(agent.agentId, messageId);
|
|
283
|
+
if (correlatedAttempt) {
|
|
284
|
+
if (correlatedAttempt.dispatch.instructions !== message.trim()) {
|
|
285
|
+
throw new Error("Worker control idempotency identity conflicts with its durable dispatch.");
|
|
286
|
+
}
|
|
287
|
+
this.assertAttemptDependencyIdentity(correlatedAttempt, dependsOnTaskIds);
|
|
288
|
+
const record = this.options.getLifecycle().getRecord(correlatedAttempt.taskId);
|
|
289
|
+
if (!record) {
|
|
290
|
+
return {
|
|
291
|
+
started: false,
|
|
292
|
+
steering: false,
|
|
293
|
+
messageId,
|
|
294
|
+
skipReason: "orchestration_projection_missing",
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
return { started: true, steering: false, messageId, record };
|
|
298
|
+
}
|
|
299
|
+
if (!mailbox.getMessage(messageId)) {
|
|
300
|
+
return mailbox.hasControlReplayReceipt(messageId)
|
|
301
|
+
? {
|
|
302
|
+
started: false,
|
|
303
|
+
steering: false,
|
|
304
|
+
messageId,
|
|
305
|
+
skipReason: "worker_task_receipt_without_attempt",
|
|
306
|
+
}
|
|
307
|
+
: undefined;
|
|
308
|
+
}
|
|
309
|
+
const acceptance = mailbox.enqueueWithReceipt({
|
|
310
|
+
kind: "follow_up",
|
|
311
|
+
content: message,
|
|
312
|
+
idempotencyKey,
|
|
313
|
+
task: {
|
|
314
|
+
kind: "agent_turn",
|
|
315
|
+
...(dependsOnTaskIds.length > 0 ? { dependsOnTaskIds } : {}),
|
|
316
|
+
},
|
|
317
|
+
});
|
|
318
|
+
if (acceptance.status === "completed_replay") {
|
|
319
|
+
return {
|
|
320
|
+
started: false,
|
|
321
|
+
steering: false,
|
|
322
|
+
messageId,
|
|
323
|
+
skipReason: "worker_task_receipt_without_attempt",
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
const accepted = acceptance.message;
|
|
327
|
+
const attempt = this.controlMessageAttempt(agent.agentId, accepted);
|
|
328
|
+
if (attempt) {
|
|
329
|
+
const record = this.options.getLifecycle().getRecord(attempt.taskId);
|
|
330
|
+
if (!record) {
|
|
331
|
+
return {
|
|
332
|
+
started: false,
|
|
333
|
+
steering: false,
|
|
334
|
+
messageId,
|
|
335
|
+
skipReason: "orchestration_projection_missing",
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
// Idempotent API replay returns the original accepted start even after that attempt became
|
|
339
|
+
// terminal. The record carries its current projection; no second task or scheduler entry is made.
|
|
340
|
+
return { started: true, steering: false, messageId, record };
|
|
341
|
+
}
|
|
342
|
+
if (accepted.deliveredAt !== undefined) {
|
|
343
|
+
return {
|
|
344
|
+
started: false,
|
|
345
|
+
steering: false,
|
|
346
|
+
messageId,
|
|
347
|
+
skipReason: "worker_task_delivered_without_attempt",
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
const reconciliation = this.reconcileTaskBearingMailbox(agent.agentId, messageId);
|
|
351
|
+
this.notifyStateChangedBestEffort();
|
|
352
|
+
return {
|
|
353
|
+
started: reconciliation.started,
|
|
354
|
+
steering: false,
|
|
355
|
+
messageId,
|
|
356
|
+
...(reconciliation.record ? { record: reconciliation.record } : {}),
|
|
357
|
+
...(reconciliation.skipReason ? { skipReason: reconciliation.skipReason } : {}),
|
|
358
|
+
};
|
|
359
|
+
}
|
|
360
|
+
startIdleAgentTask(agent, message, options, dependsOnTaskIds = []) {
|
|
361
|
+
if (!this.isAcceptedControlReplay(agent.agentId, options.idempotencyKey)) {
|
|
362
|
+
const headroomSkipReason = this.options.taskStartHeadroomSkipReason?.(agent);
|
|
363
|
+
if (headroomSkipReason)
|
|
364
|
+
return { started: false, steering: false, messageId: "", skipReason: headroomSkipReason };
|
|
365
|
+
const skipReason = this.reusableTaskAdmissionSkipReason(agent.agentId);
|
|
366
|
+
if (skipReason)
|
|
367
|
+
return { started: false, steering: false, messageId: "", skipReason };
|
|
368
|
+
}
|
|
369
|
+
const queued = this.enqueuePeerMessage(agent, "follow_up", message, options, {
|
|
370
|
+
kind: "agent_turn",
|
|
371
|
+
...(dependsOnTaskIds.length > 0 ? { dependsOnTaskIds } : {}),
|
|
372
|
+
});
|
|
373
|
+
if (queued.status === "completed_replay") {
|
|
374
|
+
return {
|
|
375
|
+
started: false,
|
|
376
|
+
steering: false,
|
|
377
|
+
messageId: queued.messageId,
|
|
378
|
+
skipReason: "worker_message_already_finalized",
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
const reconciliation = this.reconcileTaskBearingMailbox(agent.agentId, queued.messageId);
|
|
382
|
+
this.notifyStateChangedBestEffort();
|
|
383
|
+
return {
|
|
384
|
+
started: reconciliation.started,
|
|
385
|
+
steering: false,
|
|
386
|
+
messageId: queued.messageId,
|
|
387
|
+
...(reconciliation.record ? { record: reconciliation.record } : {}),
|
|
388
|
+
...(reconciliation.skipReason ? { skipReason: reconciliation.skipReason } : {}),
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
assertAttemptDependencyIdentity(attempt, dependsOnTaskIds) {
|
|
392
|
+
const snapshot = this.options.getLifecycle().getTaskRuntimeSnapshot();
|
|
393
|
+
const durableDependencies = snapshot.tasks?.[attempt.taskId]?.task.dependsOn;
|
|
394
|
+
if (!durableDependencies) {
|
|
395
|
+
if (dependsOnTaskIds.length > 0) {
|
|
396
|
+
throw new Error("Worker control idempotency dependency projection is missing.");
|
|
397
|
+
}
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
if (durableDependencies.length !== dependsOnTaskIds.length ||
|
|
401
|
+
durableDependencies.some((dependencyId, index) => dependencyId !== dependsOnTaskIds[index])) {
|
|
402
|
+
throw new Error("Worker control idempotency identity conflicts with its durable task dependencies.");
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
sessionRootMessageOptions(options) {
|
|
406
|
+
return {
|
|
407
|
+
senderAgentId: this.sessionRootAddress,
|
|
408
|
+
...(options.threadId !== undefined ? { threadId: options.threadId } : {}),
|
|
409
|
+
...(options.expectReply === true ? { expectReply: true } : {}),
|
|
410
|
+
...(options.idempotencyKey !== undefined ? { idempotencyKey: options.idempotencyKey } : {}),
|
|
411
|
+
};
|
|
412
|
+
}
|
|
413
|
+
rejectGenericReplyOptions(options) {
|
|
414
|
+
if ("replyToMessageId" in options) {
|
|
415
|
+
throw new Error("Worker replies must use the dedicated inferred-destination reply control.");
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
assertAgentAcceptsNewMessages(agent, idempotencyKey) {
|
|
419
|
+
if (agent.status !== "retired")
|
|
420
|
+
return;
|
|
421
|
+
if (idempotencyKey !== undefined) {
|
|
422
|
+
const messageId = workerAgentMessageId(this.options.parentSessionId, idempotencyKey);
|
|
423
|
+
const mailbox = this.getMailbox(agent.agentId);
|
|
424
|
+
if (mailbox.getMessage(messageId) || mailbox.hasControlReplayReceipt(messageId))
|
|
425
|
+
return;
|
|
426
|
+
}
|
|
427
|
+
throw new Error(`Logical worker agent '${agent.agentId}' is retired.`);
|
|
428
|
+
}
|
|
429
|
+
routeWorkerReplyToAgent(target, sourceAgentId, request, message) {
|
|
430
|
+
const activity = this.activityForAgent(target);
|
|
431
|
+
const idempotencyKey = `peer-reply:${sourceAgentId}:${request.messageId}`;
|
|
432
|
+
this.assertIdempotencyTarget(target.agentId, idempotencyKey);
|
|
433
|
+
const replyMessageId = workerAgentMessageId(this.options.parentSessionId, idempotencyKey);
|
|
434
|
+
const sourceMailbox = this.getMailbox(sourceAgentId);
|
|
435
|
+
const reservationAlreadyActive = sourceMailbox.getReplyAcknowledgementId(request.messageId) === replyMessageId;
|
|
436
|
+
if (!sourceMailbox.beginReplyAcknowledgement(request.messageId, replyMessageId, message)) {
|
|
437
|
+
throw new Error("Worker reply source acknowledgement could not be acquired.");
|
|
438
|
+
}
|
|
439
|
+
const transcriptInput = {
|
|
440
|
+
idempotencyKey,
|
|
441
|
+
content: message,
|
|
442
|
+
senderAgentId: sourceAgentId,
|
|
443
|
+
replyToMessageId: request.messageId,
|
|
444
|
+
...(request.threadId ? { threadId: request.threadId } : {}),
|
|
445
|
+
task: { kind: "agent_turn" },
|
|
446
|
+
};
|
|
447
|
+
const transcriptReplay = this.reconcileMandatoryControlTranscript(target, transcriptInput, false);
|
|
448
|
+
if (transcriptReplay.delivered) {
|
|
449
|
+
if (!sourceMailbox.commitReplyAcknowledgement(request.messageId, replyMessageId) &&
|
|
450
|
+
sourceMailbox.getReplyAcknowledgementId(request.messageId) !== undefined) {
|
|
451
|
+
throw new Error("Worker reply source acknowledgement did not commit after transcript replay.");
|
|
452
|
+
}
|
|
453
|
+
this.notifyStateChangedBestEffort();
|
|
454
|
+
return {
|
|
455
|
+
destination: "worker",
|
|
456
|
+
messageId: replyMessageId,
|
|
457
|
+
started: false,
|
|
458
|
+
steering: false,
|
|
459
|
+
skipReason: "worker_reply_already_accepted",
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
if (target.status === "retired") {
|
|
463
|
+
this.deliverMandatoryControlToRetiredTranscript(target, transcriptInput);
|
|
464
|
+
if (!sourceMailbox.commitReplyAcknowledgement(request.messageId, replyMessageId) &&
|
|
465
|
+
sourceMailbox.getReplyAcknowledgementId(request.messageId) !== undefined) {
|
|
466
|
+
throw new Error("Worker reply source acknowledgement did not commit after retired-target delivery.");
|
|
467
|
+
}
|
|
468
|
+
this.notifyStateChangedBestEffort();
|
|
469
|
+
return {
|
|
470
|
+
destination: "worker",
|
|
471
|
+
messageId: replyMessageId,
|
|
472
|
+
started: false,
|
|
473
|
+
steering: false,
|
|
474
|
+
skipReason: "worker_reply_retired_target_transcript_delivery",
|
|
475
|
+
};
|
|
476
|
+
}
|
|
477
|
+
const targetMailbox = this.getMailbox(target.agentId);
|
|
478
|
+
let queued;
|
|
77
479
|
try {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
480
|
+
queued = targetMailbox.enqueueWithReceipt({
|
|
481
|
+
kind: activity === "active" ? "steer" : "follow_up",
|
|
482
|
+
content: message,
|
|
483
|
+
senderAgentId: sourceAgentId,
|
|
484
|
+
replyToMessageId: request.messageId,
|
|
485
|
+
...(request.threadId ? { threadId: request.threadId } : {}),
|
|
486
|
+
idempotencyKey,
|
|
487
|
+
task: { kind: "agent_turn" },
|
|
488
|
+
});
|
|
86
489
|
}
|
|
87
490
|
catch (error) {
|
|
491
|
+
if (!reservationAlreadyActive) {
|
|
492
|
+
this.rollbackReplyReservationAfterTargetRejection(sourceMailbox, request.messageId, replyMessageId, error);
|
|
493
|
+
}
|
|
494
|
+
throw error;
|
|
495
|
+
}
|
|
496
|
+
if (queued.messageId !== replyMessageId) {
|
|
497
|
+
throw new Error("Worker reply target returned a divergent deterministic message identity.");
|
|
498
|
+
}
|
|
499
|
+
if (queued.status === "completed_replay") {
|
|
500
|
+
if (!targetMailbox.hasDeliveredControlReceipt(queued.messageId)) {
|
|
501
|
+
throw new Error("Worker reply target replay has no durable delivery evidence.");
|
|
502
|
+
}
|
|
503
|
+
if (!sourceMailbox.commitReplyAcknowledgement(request.messageId, queued.messageId) &&
|
|
504
|
+
sourceMailbox.getReplyAcknowledgementId(request.messageId) !== undefined) {
|
|
505
|
+
throw new Error("Worker reply source acknowledgement did not commit after target replay.");
|
|
506
|
+
}
|
|
507
|
+
this.notifyStateChangedBestEffort();
|
|
508
|
+
return {
|
|
509
|
+
destination: "worker",
|
|
510
|
+
messageId: queued.messageId,
|
|
511
|
+
started: false,
|
|
512
|
+
steering: false,
|
|
513
|
+
skipReason: "worker_reply_already_accepted",
|
|
514
|
+
};
|
|
515
|
+
}
|
|
516
|
+
this.beginWorkerReplyAcknowledgement(target, queued.message);
|
|
517
|
+
if (queued.message.deliveredAt !== undefined) {
|
|
518
|
+
if (!this.reconcileCompletedWorkerReplyAcknowledgement(sourceMailbox, request.messageId, target.agentId, queued.messageId) &&
|
|
519
|
+
sourceMailbox.getReplyAcknowledgementId(request.messageId) !== undefined) {
|
|
520
|
+
throw new Error("Worker reply source acknowledgement did not commit after target delivery.");
|
|
521
|
+
}
|
|
522
|
+
this.notifyStateChangedBestEffort();
|
|
88
523
|
return {
|
|
524
|
+
destination: "worker",
|
|
525
|
+
messageId: queued.messageId,
|
|
89
526
|
started: false,
|
|
90
527
|
steering: false,
|
|
528
|
+
skipReason: "worker_reply_already_accepted",
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
if (activity === "active") {
|
|
532
|
+
this.notifyStateChangedBestEffort();
|
|
533
|
+
if (!queued.created) {
|
|
534
|
+
return {
|
|
535
|
+
destination: "worker",
|
|
536
|
+
messageId: queued.messageId,
|
|
537
|
+
started: false,
|
|
538
|
+
steering: false,
|
|
539
|
+
skipReason: "worker_reply_already_accepted",
|
|
540
|
+
};
|
|
541
|
+
}
|
|
542
|
+
return {
|
|
543
|
+
destination: "worker",
|
|
91
544
|
messageId: queued.messageId,
|
|
92
|
-
|
|
545
|
+
started: false,
|
|
546
|
+
steering: true,
|
|
93
547
|
};
|
|
94
548
|
}
|
|
549
|
+
const reconciliation = this.reconcileTaskBearingMailbox(target.agentId, queued.messageId);
|
|
550
|
+
this.notifyStateChangedBestEffort();
|
|
551
|
+
return {
|
|
552
|
+
destination: "worker",
|
|
553
|
+
messageId: queued.messageId,
|
|
554
|
+
started: reconciliation.started,
|
|
555
|
+
steering: false,
|
|
556
|
+
...(reconciliation.record ? { record: reconciliation.record } : {}),
|
|
557
|
+
...(reconciliation.skipReason ? { skipReason: reconciliation.skipReason } : {}),
|
|
558
|
+
};
|
|
559
|
+
}
|
|
560
|
+
routeWorkerReplyToSessionRoot(source, sourceMailbox, request, message) {
|
|
561
|
+
const replyMessageId = sessionRootReplyMessageId(this.options.parentSessionId, source.agentId, request.messageId);
|
|
562
|
+
const reservationAlreadyActive = sourceMailbox.getReplyAcknowledgementId(request.messageId) === replyMessageId;
|
|
563
|
+
const replyInput = {
|
|
564
|
+
sourceAgentId: source.agentId,
|
|
565
|
+
requestMessageId: request.messageId,
|
|
566
|
+
...(request.threadId ? { threadId: request.threadId } : {}),
|
|
567
|
+
content: message,
|
|
568
|
+
};
|
|
569
|
+
try {
|
|
570
|
+
this.sessionRootMailbox.assertReplyInput(replyInput);
|
|
571
|
+
}
|
|
572
|
+
catch (error) {
|
|
573
|
+
if (reservationAlreadyActive) {
|
|
574
|
+
this.rollbackReplyReservationAfterTargetRejection(sourceMailbox, request.messageId, replyMessageId, error);
|
|
575
|
+
}
|
|
576
|
+
throw error;
|
|
577
|
+
}
|
|
578
|
+
if (!sourceMailbox.beginReplyAcknowledgement(request.messageId, replyMessageId, message)) {
|
|
579
|
+
throw new Error("Session root reply source acknowledgement could not be acquired.");
|
|
580
|
+
}
|
|
581
|
+
let accepted;
|
|
582
|
+
try {
|
|
583
|
+
accepted = this.sessionRootMailbox.enqueueSourceOwnedReply(replyInput);
|
|
584
|
+
}
|
|
585
|
+
catch (error) {
|
|
586
|
+
if (!reservationAlreadyActive) {
|
|
587
|
+
this.rollbackReplyReservationAfterTargetRejection(sourceMailbox, request.messageId, replyMessageId, error);
|
|
588
|
+
}
|
|
589
|
+
throw error;
|
|
590
|
+
}
|
|
591
|
+
if (accepted.messageId !== replyMessageId) {
|
|
592
|
+
throw new Error("Session root reply target returned a divergent deterministic message identity.");
|
|
593
|
+
}
|
|
594
|
+
if (accepted.status === "completed_replay") {
|
|
595
|
+
if (!sourceMailbox.commitReplyAcknowledgement(request.messageId, accepted.messageId) &&
|
|
596
|
+
sourceMailbox.getReplyAcknowledgementId(request.messageId) !== undefined) {
|
|
597
|
+
throw new Error("Session root reply source acknowledgement did not commit after target replay.");
|
|
598
|
+
}
|
|
599
|
+
this.releaseSessionRootSourceReceiptBestEffort(accepted.messageId);
|
|
600
|
+
}
|
|
601
|
+
else {
|
|
602
|
+
this.reconcileSessionRootReply(accepted.reply);
|
|
603
|
+
}
|
|
604
|
+
this.notifyStateChangedBestEffort();
|
|
605
|
+
return { destination: "session_root", messageId: accepted.messageId };
|
|
606
|
+
}
|
|
607
|
+
rollbackReplyReservationAfterTargetRejection(sourceMailbox, requestMessageId, replyMessageId, targetError) {
|
|
608
|
+
try {
|
|
609
|
+
if (!sourceMailbox.rollbackReplyAcknowledgement(requestMessageId, replyMessageId) &&
|
|
610
|
+
sourceMailbox.getReplyAcknowledgementId(requestMessageId) === replyMessageId) {
|
|
611
|
+
throw new Error("Worker reply source reservation did not roll back after target rejection.");
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
catch (rollbackError) {
|
|
615
|
+
throw new AggregateError([targetError, rollbackError], "Worker reply target rejected after its source reservation could not be rolled back.");
|
|
616
|
+
}
|
|
617
|
+
throw targetError;
|
|
618
|
+
}
|
|
619
|
+
deliverMandatoryControlToRetiredTranscript(target, input) {
|
|
620
|
+
if (target.status !== "retired" || this.activityForAgent(target) === "active") {
|
|
621
|
+
throw new Error("Mandatory transcript fallback requires a terminal logical worker target.");
|
|
622
|
+
}
|
|
623
|
+
const delivery = this.reconcileMandatoryControlTranscript(target, input, true);
|
|
624
|
+
return delivery.messageId;
|
|
625
|
+
}
|
|
626
|
+
reconcileMandatoryControlTranscript(target, input, appendIfMissing) {
|
|
627
|
+
const messageId = workerAgentMessageId(this.options.parentSessionId, input.idempotencyKey);
|
|
628
|
+
return this.reconcileControlTranscript(target, {
|
|
629
|
+
messageId,
|
|
630
|
+
kind: "follow_up",
|
|
631
|
+
content: input.content,
|
|
632
|
+
senderAgentId: input.senderAgentId,
|
|
633
|
+
...(input.threadId ? { threadId: input.threadId } : {}),
|
|
634
|
+
...(input.replyToMessageId ? { replyToMessageId: input.replyToMessageId } : {}),
|
|
635
|
+
task: input.task,
|
|
636
|
+
createdAt: new Date().toISOString(),
|
|
637
|
+
}, appendIfMissing);
|
|
638
|
+
}
|
|
639
|
+
reconcileControlTranscript(target, message, appendIfMissing) {
|
|
640
|
+
const messageId = message.messageId;
|
|
641
|
+
if (!target.resumeContext.sessionFile) {
|
|
642
|
+
if (appendIfMissing) {
|
|
643
|
+
throw new Error("Mandatory transcript fallback target has no durable session transcript.");
|
|
644
|
+
}
|
|
645
|
+
return { messageId, delivered: false };
|
|
646
|
+
}
|
|
647
|
+
const projected = this.mailboxMessage(message);
|
|
648
|
+
const conversation = this.conversations.open({
|
|
649
|
+
agentDir: this.options.agentDir,
|
|
650
|
+
resumeContext: target.resumeContext,
|
|
651
|
+
expectedLogicalAgentId: target.agentId,
|
|
652
|
+
});
|
|
653
|
+
if (typeof projected.content !== "string") {
|
|
654
|
+
throw new Error("Worker control transcript projection is not textual.");
|
|
655
|
+
}
|
|
656
|
+
const reconciliation = conversation.reconcileWorkerControlMessage({ messageId, content: projected.content }, projected, appendIfMissing);
|
|
657
|
+
return { messageId, delivered: reconciliation.delivered };
|
|
658
|
+
}
|
|
659
|
+
reconcileSessionRootReplies() {
|
|
660
|
+
let replies;
|
|
661
|
+
try {
|
|
662
|
+
replies = this.sessionRootMailbox.retainedReplies();
|
|
663
|
+
this.sessionRootReconciliationFailures.delete("session-root-mailbox");
|
|
664
|
+
}
|
|
665
|
+
catch (error) {
|
|
666
|
+
this.recordSessionRootReconciliationFailure("session-root-mailbox", error instanceof Error ? error.message : String(error));
|
|
667
|
+
return;
|
|
668
|
+
}
|
|
669
|
+
for (const reply of replies)
|
|
670
|
+
this.reconcileSessionRootReply(reply);
|
|
671
|
+
}
|
|
672
|
+
reconcileSessionRootReply(reply) {
|
|
673
|
+
try {
|
|
674
|
+
const source = this.options.getLifecycle().getAgent(reply.sourceAgentId);
|
|
675
|
+
if (!source) {
|
|
676
|
+
if (reply.sourceReconciledAt) {
|
|
677
|
+
this.sessionRootReconciliationFailures.delete(reply.messageId);
|
|
678
|
+
return;
|
|
679
|
+
}
|
|
680
|
+
throw new Error(`Unknown logical worker agent '${reply.sourceAgentId}'.`);
|
|
681
|
+
}
|
|
682
|
+
const sourceMailbox = this.getMailbox(source.agentId);
|
|
683
|
+
const request = sourceMailbox.getMessage(reply.requestMessageId);
|
|
684
|
+
if (!request) {
|
|
685
|
+
if (reply.sourceReconciledAt) {
|
|
686
|
+
this.releaseSessionRootSourceReceiptBestEffort(reply.messageId);
|
|
687
|
+
this.sessionRootReconciliationFailures.delete(reply.messageId);
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
throw new Error("Session root reply source request is missing before reconciliation.");
|
|
691
|
+
}
|
|
692
|
+
if (request.deliveredAt === undefined ||
|
|
693
|
+
request.expectReply !== true ||
|
|
694
|
+
request.senderAgentId !== this.sessionRootAddress ||
|
|
695
|
+
request.threadId !== reply.threadId) {
|
|
696
|
+
throw new Error("Session root reply source request conflicts with its durable target.");
|
|
697
|
+
}
|
|
698
|
+
let acknowledgementId = sourceMailbox.getReplyAcknowledgementId(request.messageId);
|
|
699
|
+
if (acknowledgementId && acknowledgementId !== reply.messageId) {
|
|
700
|
+
throw new Error("Session root reply source request has a divergent acknowledgement marker.");
|
|
701
|
+
}
|
|
702
|
+
if (request.repliedAt !== undefined && acknowledgementId === undefined) {
|
|
703
|
+
if (!reply.sourceReconciledAt) {
|
|
704
|
+
throw new Error("Session root reply source was marked replied without its exact acknowledgement marker.");
|
|
705
|
+
}
|
|
706
|
+
this.releaseSessionRootSourceReceiptBestEffort(reply.messageId);
|
|
707
|
+
this.sessionRootReconciliationFailures.delete(reply.messageId);
|
|
708
|
+
return;
|
|
709
|
+
}
|
|
710
|
+
if (request.repliedAt === undefined) {
|
|
711
|
+
if (!sourceMailbox.beginReplyAcknowledgement(request.messageId, reply.messageId, reply.content)) {
|
|
712
|
+
throw new Error("Session root reply source acknowledgement could not be acquired.");
|
|
713
|
+
}
|
|
714
|
+
acknowledgementId = reply.messageId;
|
|
715
|
+
}
|
|
716
|
+
if (!this.sessionRootMailbox.markSourceReconciled(reply.messageId)) {
|
|
717
|
+
throw new Error("Session root reply disappeared during source reconciliation.");
|
|
718
|
+
}
|
|
719
|
+
if (acknowledgementId === reply.messageId) {
|
|
720
|
+
if (!sourceMailbox.commitReplyAcknowledgement(request.messageId, reply.messageId)) {
|
|
721
|
+
const remaining = sourceMailbox.getReplyAcknowledgementId(request.messageId);
|
|
722
|
+
if (remaining !== undefined) {
|
|
723
|
+
throw new Error("Session root reply source acknowledgement did not commit.");
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
}
|
|
727
|
+
this.releaseSessionRootSourceReceiptBestEffort(reply.messageId);
|
|
728
|
+
this.sessionRootReconciliationFailures.delete(reply.messageId);
|
|
729
|
+
}
|
|
730
|
+
catch (error) {
|
|
731
|
+
this.recordSessionRootReconciliationFailure(reply.messageId, error instanceof Error ? error.message : String(error));
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
recordSessionRootReconciliationFailure(messageId, reason) {
|
|
735
|
+
if (this.sessionRootReconciliationFailures.get(messageId) === reason)
|
|
736
|
+
return;
|
|
737
|
+
if (!this.sessionRootReconciliationFailures.has(messageId) &&
|
|
738
|
+
this.sessionRootReconciliationFailures.size >= 128) {
|
|
739
|
+
const oldest = this.sessionRootReconciliationFailures.keys().next().value;
|
|
740
|
+
if (oldest)
|
|
741
|
+
this.sessionRootReconciliationFailures.delete(oldest);
|
|
742
|
+
}
|
|
743
|
+
this.sessionRootReconciliationFailures.set(messageId, reason);
|
|
744
|
+
try {
|
|
745
|
+
this.options.warn?.(`Session root reply reconciliation failed for ${messageId}: ${reason}`);
|
|
746
|
+
}
|
|
747
|
+
catch {
|
|
748
|
+
// Diagnostics are bounded observers; durable mailboxes remain authoritative.
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
releaseSessionRootSourceReceiptBestEffort(messageId) {
|
|
752
|
+
try {
|
|
753
|
+
this.sessionRootMailbox.releaseSourceReplayReceipt(messageId);
|
|
754
|
+
this.sessionRootReconciliationFailures.delete(`receipt:${messageId}`);
|
|
755
|
+
}
|
|
756
|
+
catch (error) {
|
|
757
|
+
this.recordSessionRootReconciliationFailure(`receipt:${messageId}`, error instanceof Error ? error.message : String(error));
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
notifyStateChangedBestEffort() {
|
|
761
|
+
try {
|
|
762
|
+
this.options.statusChanged();
|
|
763
|
+
}
|
|
764
|
+
catch {
|
|
765
|
+
// Notification observers cannot redefine durable task acceptance.
|
|
766
|
+
}
|
|
767
|
+
this.notifyStateListeners();
|
|
95
768
|
}
|
|
96
769
|
interruptWorkerAgent(agentId, scope = {}) {
|
|
97
770
|
const { agent, attempt } = this.controlledAgentAttempt(agentId, scope);
|
|
@@ -115,8 +788,27 @@ export class WorkerAgentControlCoordinator {
|
|
|
115
788
|
const record = this.options.getLifecycle().getRecord(attempt.taskId);
|
|
116
789
|
if (!record)
|
|
117
790
|
return { started: false, skipReason: "orchestration_projection_missing" };
|
|
118
|
-
|
|
119
|
-
|
|
791
|
+
try {
|
|
792
|
+
const request = this.options.recoveredRequest(attempt);
|
|
793
|
+
this.options.scheduler.enqueue(record, request, true, request.verificationOfTaskId !== undefined);
|
|
794
|
+
}
|
|
795
|
+
catch (error) {
|
|
796
|
+
return {
|
|
797
|
+
started: false,
|
|
798
|
+
record,
|
|
799
|
+
skipReason: error instanceof Error ? error.message : String(error),
|
|
800
|
+
};
|
|
801
|
+
}
|
|
802
|
+
try {
|
|
803
|
+
this.options.scheduler.drain();
|
|
804
|
+
}
|
|
805
|
+
catch (error) {
|
|
806
|
+
return {
|
|
807
|
+
started: true,
|
|
808
|
+
record,
|
|
809
|
+
skipReason: `worker_resume_recovery_pending:${error instanceof Error ? error.message : String(error)}`,
|
|
810
|
+
};
|
|
811
|
+
}
|
|
120
812
|
this.signalStateChanged();
|
|
121
813
|
return { started: true, record };
|
|
122
814
|
}
|
|
@@ -129,72 +821,141 @@ export class WorkerAgentControlCoordinator {
|
|
|
129
821
|
this.signalStateChanged();
|
|
130
822
|
return record;
|
|
131
823
|
}
|
|
132
|
-
|
|
824
|
+
retireWorkerAgent(agentId, scope = {}) {
|
|
825
|
+
this.requireControl();
|
|
826
|
+
let target = this.requireControllableAgent(agentId, scope);
|
|
827
|
+
if (target.status === "retired") {
|
|
828
|
+
return { agent: this.workerAgentView(target), retired: true, replayed: true };
|
|
829
|
+
}
|
|
830
|
+
const activity = this.activityForAgent(target);
|
|
831
|
+
if (activity !== "idle") {
|
|
832
|
+
throw new Error(`Logical worker agent '${target.agentId}' cannot retire while ${activity}.`);
|
|
833
|
+
}
|
|
834
|
+
const mailbox = this.getMailbox(target.agentId);
|
|
835
|
+
if (mailbox.pendingTaskBearing().length > 0) {
|
|
836
|
+
this.reconcileTaskBearingMailbox(target.agentId);
|
|
837
|
+
target = this.requireControllableAgent(target.agentId, scope);
|
|
838
|
+
}
|
|
839
|
+
this.reconcileWorkerReplyOutboxesBestEffort();
|
|
840
|
+
const pendingMessages = mailbox.pending();
|
|
841
|
+
if (pendingMessages.length > 0) {
|
|
842
|
+
throw new Error(`Logical worker agent '${target.agentId}' has ${pendingMessages.length} pending control message${pendingMessages.length === 1 ? "" : "s"}.`);
|
|
843
|
+
}
|
|
844
|
+
const unresolvedReplyCount = mailbox.awaitingReplies().length + mailbox.listReplyAcknowledgements().length;
|
|
845
|
+
if (unresolvedReplyCount > 0) {
|
|
846
|
+
throw new Error(`Logical worker agent '${target.agentId}' has ${unresolvedReplyCount} unresolved reply obligation${unresolvedReplyCount === 1 ? "" : "s"}.`);
|
|
847
|
+
}
|
|
848
|
+
const retired = this.options.getLifecycle().retireAgent(target.agentId);
|
|
849
|
+
this.notifyStateChangedBestEffort();
|
|
850
|
+
return { agent: this.workerAgentView(retired), retired: true, replayed: false };
|
|
851
|
+
}
|
|
852
|
+
/** Event-driven wait: durable projection plus one shared state notification, never output polling. */
|
|
133
853
|
waitForWorkerAgent(agentId, timeoutMs = 30_000, scope = {}) {
|
|
854
|
+
return this.waitForWorkerAgents([agentId], "all", timeoutMs, scope).then((result) => ({
|
|
855
|
+
status: result.statuses[0]?.status ?? "unknown",
|
|
856
|
+
}));
|
|
857
|
+
}
|
|
858
|
+
/** One event-driven wait set with one shared caller-capacity lease; never per-agent promise polling. */
|
|
859
|
+
waitForWorkerAgents(agentIds, mode, timeoutMs = 30_000, scope = {}) {
|
|
134
860
|
this.requireControl();
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
const
|
|
139
|
-
if (
|
|
140
|
-
|
|
861
|
+
if (mode !== "any" && mode !== "all")
|
|
862
|
+
throw new TypeError("Worker wait mode must be 'any' or 'all'.");
|
|
863
|
+
const canonicalAgentIds = this.canonicalAgentIdSet(agentIds, "Worker wait");
|
|
864
|
+
const baselineSnapshot = this.options.getLifecycle().getTaskRuntimeSnapshot();
|
|
865
|
+
if (scope.callerAgentId) {
|
|
866
|
+
const callerAgentId = scope.callerAgentId.trim();
|
|
867
|
+
if (!callerAgentId || !baselineSnapshot.agents[callerAgentId]) {
|
|
868
|
+
throw new Error(`Unknown logical worker agent '${callerAgentId}'.`);
|
|
869
|
+
}
|
|
870
|
+
}
|
|
141
871
|
const boundedTimeoutMs = Number.isFinite(timeoutMs)
|
|
142
872
|
? Math.max(1, Math.min(Math.floor(timeoutMs), 300_000))
|
|
143
873
|
: 30_000;
|
|
144
|
-
const
|
|
145
|
-
const
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
874
|
+
const statusesFromSnapshot = (snapshot) => {
|
|
875
|
+
const latestAttempts = this.latestAttemptsByAgent(snapshot);
|
|
876
|
+
return canonicalAgentIds.map((agentId) => {
|
|
877
|
+
const agent = snapshot.agents[agentId];
|
|
878
|
+
return {
|
|
879
|
+
agentId,
|
|
880
|
+
status: agent
|
|
881
|
+
? this.projectAgentActivity(agent, latestAttempts.get(agent.agentId))
|
|
882
|
+
: "unknown",
|
|
883
|
+
};
|
|
884
|
+
});
|
|
885
|
+
};
|
|
886
|
+
const currentStatuses = () => statusesFromSnapshot(this.options.getLifecycle().getTaskRuntimeSnapshot());
|
|
887
|
+
const baselineStatuses = statusesFromSnapshot(baselineSnapshot);
|
|
888
|
+
const baselineByAgentId = new Map(baselineStatuses.map(({ agentId, status }) => [agentId, status]));
|
|
889
|
+
const updatedAgentIds = new Set();
|
|
890
|
+
const recordUpdates = (statuses) => {
|
|
891
|
+
for (const { agentId, status } of statuses) {
|
|
892
|
+
if (baselineByAgentId.get(agentId) !== status)
|
|
893
|
+
updatedAgentIds.add(agentId);
|
|
894
|
+
}
|
|
895
|
+
};
|
|
896
|
+
const waitSatisfied = (statuses) => {
|
|
897
|
+
return mode === "any"
|
|
898
|
+
? statuses.some(({ status }) => status !== "active")
|
|
899
|
+
: statuses.every(({ status }) => status !== "active");
|
|
155
900
|
};
|
|
156
|
-
const
|
|
157
|
-
|
|
158
|
-
|
|
901
|
+
const result = (statuses, timedOut) => ({
|
|
902
|
+
statuses,
|
|
903
|
+
updatedAgentIds: canonicalAgentIds.filter((agentId) => updatedAgentIds.has(agentId)),
|
|
904
|
+
timedOut,
|
|
905
|
+
});
|
|
906
|
+
if (waitSatisfied(baselineStatuses))
|
|
907
|
+
return Promise.resolve(result(baselineStatuses, false));
|
|
159
908
|
return new Promise((resolve) => {
|
|
160
909
|
let settled = false;
|
|
161
|
-
let releaseYield
|
|
162
|
-
let unsubscribeMailbox = () => undefined;
|
|
910
|
+
let releaseYield;
|
|
163
911
|
let unsubscribeState = () => undefined;
|
|
164
912
|
let timeout;
|
|
913
|
+
const cleanup = () => {
|
|
914
|
+
unsubscribeState();
|
|
915
|
+
if (timeout)
|
|
916
|
+
clearTimeout(timeout);
|
|
917
|
+
const release = releaseYield;
|
|
918
|
+
releaseYield = undefined;
|
|
919
|
+
release?.();
|
|
920
|
+
};
|
|
165
921
|
const settle = () => {
|
|
166
922
|
if (settled)
|
|
167
923
|
return;
|
|
168
|
-
const
|
|
169
|
-
|
|
924
|
+
const statuses = currentStatuses();
|
|
925
|
+
recordUpdates(statuses);
|
|
926
|
+
if (!waitSatisfied(statuses))
|
|
170
927
|
return;
|
|
171
928
|
settled = true;
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
if (timeout)
|
|
175
|
-
clearTimeout(timeout);
|
|
176
|
-
releaseYield();
|
|
177
|
-
resolve({ status: next });
|
|
929
|
+
cleanup();
|
|
930
|
+
resolve(result(statuses, false));
|
|
178
931
|
};
|
|
179
|
-
unsubscribeMailbox = this.getMailbox(canonicalAgentId).subscribe(settle);
|
|
180
932
|
unsubscribeState = this.subscribeStateChanges(settle);
|
|
181
933
|
timeout = setTimeout(() => {
|
|
182
934
|
if (settled)
|
|
183
935
|
return;
|
|
184
936
|
settled = true;
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
resolve(
|
|
937
|
+
const statuses = currentStatuses();
|
|
938
|
+
recordUpdates(statuses);
|
|
939
|
+
cleanup();
|
|
940
|
+
resolve(result(statuses, true));
|
|
189
941
|
}, boundedTimeoutMs);
|
|
190
942
|
if (typeof timeout === "object" && "unref" in timeout)
|
|
191
943
|
timeout.unref();
|
|
192
944
|
if (scope.callerAgentId && this.options.yieldCapacity) {
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
945
|
+
// Capacity belongs to the waiting caller. Passing that identity through both slots keeps
|
|
946
|
+
// legacy controller adapters from treating an independent peer as the capacity owner.
|
|
947
|
+
try {
|
|
948
|
+
const yielded = this.options.yieldCapacity(scope.callerAgentId, scope.callerAgentId);
|
|
949
|
+
if (settled)
|
|
950
|
+
yielded();
|
|
951
|
+
else
|
|
952
|
+
releaseYield = yielded;
|
|
953
|
+
}
|
|
954
|
+
catch (error) {
|
|
955
|
+
settled = true;
|
|
956
|
+
cleanup();
|
|
957
|
+
throw error;
|
|
958
|
+
}
|
|
198
959
|
}
|
|
199
960
|
settle();
|
|
200
961
|
});
|
|
@@ -205,9 +966,15 @@ export class WorkerAgentControlCoordinator {
|
|
|
205
966
|
const pending = mailbox.pending();
|
|
206
967
|
if (pending.length === 0)
|
|
207
968
|
return [];
|
|
208
|
-
const delivered = conversation.findDeliveredWorkerControlMessageIds(pending.map((message) =>
|
|
969
|
+
const delivered = conversation.findDeliveredWorkerControlMessageIds(pending.map((message) => {
|
|
970
|
+
const projected = this.mailboxMessage(message);
|
|
971
|
+
if (typeof projected.content !== "string") {
|
|
972
|
+
throw new Error("Worker control transcript projection is not textual.");
|
|
973
|
+
}
|
|
974
|
+
return { messageId: message.messageId, content: projected.content };
|
|
975
|
+
}));
|
|
209
976
|
for (const messageId of delivered)
|
|
210
|
-
|
|
977
|
+
this.acknowledgeDeliveredMailboxMessage(agentId, messageId);
|
|
211
978
|
return mailbox
|
|
212
979
|
.pending()
|
|
213
980
|
.filter((message) => message.kind === "steer" || includeFollowUp)
|
|
@@ -219,12 +986,101 @@ export class WorkerAgentControlCoordinator {
|
|
|
219
986
|
return;
|
|
220
987
|
const messageId = /^\[Worker control (worker-message-[^\]\s]+)(?: [^\]]+)?\]\n/.exec(message.content)?.[1];
|
|
221
988
|
if (messageId)
|
|
222
|
-
this.
|
|
989
|
+
this.acknowledgeDeliveredMailboxMessage(agentId, messageId);
|
|
223
990
|
}
|
|
224
991
|
/** Called by controller-owned execution transitions after lifecycle state changed. */
|
|
225
992
|
signalStateChanged() {
|
|
226
|
-
|
|
227
|
-
|
|
993
|
+
this.reconcileTaskBearingMailboxTurns();
|
|
994
|
+
this.notifyStateListeners();
|
|
995
|
+
}
|
|
996
|
+
/** Restart/idle boundary: adopt or schedule every oldest pending executable mailbox intent. */
|
|
997
|
+
reconcileTaskBearingMailboxTurns() {
|
|
998
|
+
const agents = this.replyReconciliationAgents();
|
|
999
|
+
this.reconcileWorkerReplyOutboxes(agents);
|
|
1000
|
+
for (const agent of agents)
|
|
1001
|
+
this.reconcileTaskBearingMailbox(agent.agentId);
|
|
1002
|
+
}
|
|
1003
|
+
replyReconciliationAgents() {
|
|
1004
|
+
return Object.values(this.options.getLifecycle().getTaskRuntimeSnapshot().agents).sort((left, right) => left.createdAt.localeCompare(right.createdAt) || left.agentId.localeCompare(right.agentId));
|
|
1005
|
+
}
|
|
1006
|
+
reconcileWorkerReplyOutboxes(agents) {
|
|
1007
|
+
for (const agent of agents)
|
|
1008
|
+
this.reconcileWorkerReplyAcknowledgements(agent);
|
|
1009
|
+
}
|
|
1010
|
+
reconcileWorkerReplyOutboxesBestEffort() {
|
|
1011
|
+
try {
|
|
1012
|
+
this.reconcileWorkerReplyOutboxes(this.replyReconciliationAgents());
|
|
1013
|
+
this.workerReplyReconciliationFailures.delete("worker-reply-outbox-snapshot");
|
|
1014
|
+
}
|
|
1015
|
+
catch (error) {
|
|
1016
|
+
this.recordWorkerReplyReconciliationFailure("worker-reply-outbox-snapshot", error instanceof Error ? error.message : String(error));
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
reconcileWorkerReplyAcknowledgements(source) {
|
|
1020
|
+
const sourceMailbox = this.getMailbox(source.agentId);
|
|
1021
|
+
let acknowledgements;
|
|
1022
|
+
try {
|
|
1023
|
+
acknowledgements = sourceMailbox.listReplyAcknowledgements();
|
|
1024
|
+
this.workerReplyReconciliationFailures.delete(`source:${source.agentId}`);
|
|
1025
|
+
}
|
|
1026
|
+
catch (error) {
|
|
1027
|
+
this.recordWorkerReplyReconciliationFailure(`source:${source.agentId}`, error instanceof Error ? error.message : String(error));
|
|
1028
|
+
return;
|
|
1029
|
+
}
|
|
1030
|
+
for (const acknowledgement of acknowledgements) {
|
|
1031
|
+
const failureKey = `reply:${source.agentId}:${acknowledgement.messageId}`;
|
|
1032
|
+
try {
|
|
1033
|
+
const request = sourceMailbox.getMessage(acknowledgement.messageId);
|
|
1034
|
+
if (!request?.senderAgentId) {
|
|
1035
|
+
throw new Error("Worker reply acknowledgement source request is missing routing metadata.");
|
|
1036
|
+
}
|
|
1037
|
+
if (request.senderAgentId === this.sessionRootAddress) {
|
|
1038
|
+
this.routeWorkerReplyToSessionRoot(source, sourceMailbox, request, acknowledgement.replyContent);
|
|
1039
|
+
this.workerReplyReconciliationFailures.delete(failureKey);
|
|
1040
|
+
continue;
|
|
1041
|
+
}
|
|
1042
|
+
const target = this.requireKnownAgent(request.senderAgentId);
|
|
1043
|
+
this.routeWorkerReplyToAgent(target, source.agentId, request, acknowledgement.replyContent);
|
|
1044
|
+
this.workerReplyReconciliationFailures.delete(failureKey);
|
|
1045
|
+
}
|
|
1046
|
+
catch (error) {
|
|
1047
|
+
this.recordWorkerReplyReconciliationFailure(failureKey, error instanceof Error ? error.message : String(error));
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
recordWorkerReplyReconciliationFailure(failureKey, reason) {
|
|
1052
|
+
if (this.workerReplyReconciliationFailures.get(failureKey) === reason)
|
|
1053
|
+
return;
|
|
1054
|
+
if (!this.workerReplyReconciliationFailures.has(failureKey) &&
|
|
1055
|
+
this.workerReplyReconciliationFailures.size >= 128) {
|
|
1056
|
+
const oldest = this.workerReplyReconciliationFailures.keys().next().value;
|
|
1057
|
+
if (oldest)
|
|
1058
|
+
this.workerReplyReconciliationFailures.delete(oldest);
|
|
1059
|
+
}
|
|
1060
|
+
this.workerReplyReconciliationFailures.set(failureKey, reason);
|
|
1061
|
+
try {
|
|
1062
|
+
this.options.warn?.(`Worker reply acknowledgement recovery failed for ${failureKey}: ${reason}`);
|
|
1063
|
+
}
|
|
1064
|
+
catch {
|
|
1065
|
+
// Diagnostics are bounded observers; durable acknowledgement evidence remains authoritative.
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
reconcileCompletedWorkerReplyAcknowledgement(sourceMailbox, requestMessageId, targetAgentId, replyMessageId) {
|
|
1069
|
+
if (sourceMailbox.getReplyAcknowledgementId(requestMessageId) !== replyMessageId)
|
|
1070
|
+
return false;
|
|
1071
|
+
if (!this.getMailbox(targetAgentId).hasDeliveredControlReceipt(replyMessageId))
|
|
1072
|
+
return false;
|
|
1073
|
+
return sourceMailbox.commitReplyAcknowledgement(requestMessageId, replyMessageId);
|
|
1074
|
+
}
|
|
1075
|
+
notifyStateListeners() {
|
|
1076
|
+
for (const listener of this.stateListeners) {
|
|
1077
|
+
try {
|
|
1078
|
+
listener();
|
|
1079
|
+
}
|
|
1080
|
+
catch {
|
|
1081
|
+
// State listeners are advisory; one observer cannot redefine or hide a durable mutation.
|
|
1082
|
+
}
|
|
1083
|
+
}
|
|
228
1084
|
}
|
|
229
1085
|
/** Route a terminal child edge to its owning parent without injecting into an active model turn. */
|
|
230
1086
|
deliverWorkerTerminalHandoff(args) {
|
|
@@ -237,33 +1093,285 @@ export class WorkerAgentControlCoordinator {
|
|
|
237
1093
|
`laneId=${args.record.laneId}`,
|
|
238
1094
|
`status=${args.record.status}`,
|
|
239
1095
|
...(args.record.reasonCode ? [`reasonCode=${args.record.reasonCode}`] : []),
|
|
240
|
-
`Read
|
|
1096
|
+
`Read bounded raw transcript pages with delegate action="transcript" agentId="${args.childAgentId}". Messages present in each page are complete durable entries; omittedMessages discloses whole-message omissions, and an empty page may still continue via nextCursor.`,
|
|
241
1097
|
].join("\n");
|
|
242
|
-
const
|
|
1098
|
+
const idempotencyKey = `terminal-handoff:${args.terminalAttemptId}`;
|
|
1099
|
+
const transcriptInput = {
|
|
1100
|
+
idempotencyKey,
|
|
1101
|
+
content,
|
|
1102
|
+
senderAgentId: args.childAgentId,
|
|
1103
|
+
task: { kind: "terminal_handoff", sourceAttemptId: args.terminalAttemptId },
|
|
1104
|
+
};
|
|
1105
|
+
this.assertIdempotencyTarget(parent.agentId, idempotencyKey);
|
|
1106
|
+
const transcriptReplay = this.reconcileMandatoryControlTranscript(parent, transcriptInput, false);
|
|
1107
|
+
if (transcriptReplay.delivered) {
|
|
1108
|
+
this.notifyStateChangedBestEffort();
|
|
1109
|
+
return { messageId: transcriptReplay.messageId, started: false, accepted: true };
|
|
1110
|
+
}
|
|
1111
|
+
if (parent.status === "retired") {
|
|
1112
|
+
const messageId = this.deliverMandatoryControlToRetiredTranscript(parent, transcriptInput);
|
|
1113
|
+
this.notifyStateChangedBestEffort();
|
|
1114
|
+
return {
|
|
1115
|
+
messageId,
|
|
1116
|
+
started: false,
|
|
1117
|
+
accepted: true,
|
|
1118
|
+
skipReason: "terminal_handoff_retired_target_transcript_delivery",
|
|
1119
|
+
};
|
|
1120
|
+
}
|
|
1121
|
+
const queued = this.getMailbox(parent.agentId).enqueueWithReceipt({
|
|
243
1122
|
kind: active ? "steer" : "follow_up",
|
|
244
1123
|
content,
|
|
245
1124
|
senderAgentId: args.childAgentId,
|
|
246
|
-
idempotencyKey
|
|
1125
|
+
idempotencyKey,
|
|
1126
|
+
task: { kind: "terminal_handoff", sourceAttemptId: args.terminalAttemptId },
|
|
247
1127
|
});
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
1128
|
+
if (queued.status === "completed_replay") {
|
|
1129
|
+
this.notifyStateChangedBestEffort();
|
|
1130
|
+
return { messageId: queued.messageId, started: false, accepted: true };
|
|
1131
|
+
}
|
|
1132
|
+
if (queued.message.deliveredAt !== undefined) {
|
|
1133
|
+
this.notifyStateChangedBestEffort();
|
|
1134
|
+
return { messageId: queued.messageId, started: false, accepted: true };
|
|
1135
|
+
}
|
|
1136
|
+
const reconciliation = this.reconcileTaskBearingMailbox(parent.agentId, queued.messageId);
|
|
1137
|
+
this.notifyStateChangedBestEffort();
|
|
1138
|
+
return {
|
|
1139
|
+
messageId: queued.messageId,
|
|
1140
|
+
started: reconciliation.started,
|
|
1141
|
+
accepted: true,
|
|
1142
|
+
...(reconciliation.skipReason ? { skipReason: reconciliation.skipReason } : {}),
|
|
1143
|
+
};
|
|
1144
|
+
}
|
|
1145
|
+
reconcileTaskBearingMailbox(agentId, expectedMessageId) {
|
|
1146
|
+
if (this.reconcilingTaskBearingAgentIds.has(agentId)) {
|
|
1147
|
+
return { started: false, skipReason: "worker_task_reconciliation_in_progress" };
|
|
1148
|
+
}
|
|
1149
|
+
this.reconcilingTaskBearingAgentIds.add(agentId);
|
|
1150
|
+
try {
|
|
1151
|
+
const agent = this.options.getLifecycle().getAgent(agentId);
|
|
1152
|
+
if (!agent)
|
|
1153
|
+
return { started: false, skipReason: "unknown_agent" };
|
|
1154
|
+
const mailbox = this.getMailbox(agentId);
|
|
1155
|
+
for (let settlement = 0; settlement < 64; settlement++) {
|
|
1156
|
+
const message = mailbox.pendingTaskBearing()[0];
|
|
1157
|
+
if (!message)
|
|
1158
|
+
return { started: false };
|
|
1159
|
+
if (this.reconcileTaskBearingTranscriptDelivery(agent, message)) {
|
|
1160
|
+
if (expectedMessageId === message.messageId) {
|
|
1161
|
+
return { started: false, skipReason: "worker_control_transcript_delivery_reconciled" };
|
|
1162
|
+
}
|
|
1163
|
+
continue;
|
|
1164
|
+
}
|
|
1165
|
+
this.beginWorkerReplyAcknowledgement(agent, message);
|
|
1166
|
+
const correlated = this.controlMessageAttempt(agent.agentId, message);
|
|
1167
|
+
if (correlated) {
|
|
1168
|
+
const scheduled = this.scheduleCorrelatedTaskBearingAttempt(correlated);
|
|
1169
|
+
if (scheduled.skipReason?.startsWith("worker_task_terminal_")) {
|
|
1170
|
+
const settled = this.settleTerminalTaskBearingMessage(agent, mailbox, message, scheduled, expectedMessageId);
|
|
1171
|
+
if (settled)
|
|
1172
|
+
return settled;
|
|
1173
|
+
continue;
|
|
1174
|
+
}
|
|
1175
|
+
if (expectedMessageId && message.messageId !== expectedMessageId) {
|
|
1176
|
+
return { started: false, skipReason: "worker_task_waiting_for_older_message" };
|
|
1177
|
+
}
|
|
1178
|
+
return scheduled;
|
|
1179
|
+
}
|
|
1180
|
+
if (expectedMessageId && message.messageId !== expectedMessageId) {
|
|
1181
|
+
return { started: false, skipReason: "worker_task_waiting_for_older_message" };
|
|
1182
|
+
}
|
|
1183
|
+
const activity = this.activityForAgent(agent);
|
|
1184
|
+
if (activity !== "idle")
|
|
1185
|
+
return { started: false, skipReason: `worker_${activity}` };
|
|
1186
|
+
if (agent.status !== "registered") {
|
|
1187
|
+
const settled = this.settleTerminalTaskBearingMessage(agent, mailbox, message, { started: false, skipReason: `agent_${agent.status}` }, expectedMessageId);
|
|
1188
|
+
if (settled)
|
|
1189
|
+
return settled;
|
|
1190
|
+
continue;
|
|
1191
|
+
}
|
|
1192
|
+
const reuseSkipReason = this.reusableTaskAdmissionSkipReason(agent.agentId);
|
|
1193
|
+
if (reuseSkipReason) {
|
|
1194
|
+
this.reconcileControlTranscript(agent, message, true);
|
|
1195
|
+
this.acknowledgeDeliveredMailboxMessage(agent.agentId, message.messageId);
|
|
1196
|
+
const settlement = {
|
|
1197
|
+
started: false,
|
|
1198
|
+
skipReason: `${reuseSkipReason}:transcript_fallback_delivered`,
|
|
1199
|
+
};
|
|
1200
|
+
if (expectedMessageId === message.messageId)
|
|
1201
|
+
return settlement;
|
|
1202
|
+
continue;
|
|
1203
|
+
}
|
|
1204
|
+
try {
|
|
1205
|
+
this.enableAttemptAccountingForNextTask(agent);
|
|
1206
|
+
const prepared = this.options.getLifecycle().prepareAgentTurn({
|
|
1207
|
+
agentId: agent.agentId,
|
|
1208
|
+
instructions: message.content,
|
|
1209
|
+
controlMessageId: message.messageId,
|
|
1210
|
+
...(message.task?.kind === "agent_turn" && message.task.dependsOnTaskIds
|
|
1211
|
+
? { dependsOnTaskIds: message.task.dependsOnTaskIds }
|
|
1212
|
+
: {}),
|
|
1213
|
+
});
|
|
1214
|
+
return this.scheduleTaskBearingAttempt(prepared.record, prepared.attempt);
|
|
1215
|
+
}
|
|
1216
|
+
catch (error) {
|
|
1217
|
+
const recovered = this.controlMessageAttempt(agent.agentId, message);
|
|
1218
|
+
if (recovered) {
|
|
1219
|
+
const scheduled = this.scheduleCorrelatedTaskBearingAttempt(recovered);
|
|
1220
|
+
if (scheduled.skipReason?.startsWith("worker_task_terminal_")) {
|
|
1221
|
+
const settled = this.settleTerminalTaskBearingMessage(agent, mailbox, message, scheduled, expectedMessageId);
|
|
1222
|
+
if (settled)
|
|
1223
|
+
return settled;
|
|
1224
|
+
continue;
|
|
1225
|
+
}
|
|
1226
|
+
return {
|
|
1227
|
+
...scheduled,
|
|
1228
|
+
...(scheduled.skipReason
|
|
1229
|
+
? {}
|
|
1230
|
+
: { skipReason: "worker_task_recovered_after_prepare_interruption" }),
|
|
1231
|
+
};
|
|
1232
|
+
}
|
|
1233
|
+
return { started: false, skipReason: error instanceof Error ? error.message : String(error) };
|
|
1234
|
+
}
|
|
1235
|
+
}
|
|
1236
|
+
this.scheduleTaskBearingReconciliationContinuation(agentId);
|
|
1237
|
+
return { started: false, skipReason: "worker_task_reconciliation_bound_reached" };
|
|
1238
|
+
}
|
|
1239
|
+
catch (error) {
|
|
1240
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
1241
|
+
try {
|
|
1242
|
+
this.options.warn?.(`Worker task-bearing mailbox reconciliation failed for ${agentId}: ${reason}`);
|
|
1243
|
+
}
|
|
1244
|
+
catch {
|
|
1245
|
+
// Diagnostics are observers; the pending mailbox remains the recovery source of truth.
|
|
1246
|
+
}
|
|
1247
|
+
return { started: false, skipReason: reason };
|
|
1248
|
+
}
|
|
1249
|
+
finally {
|
|
1250
|
+
this.reconcilingTaskBearingAgentIds.delete(agentId);
|
|
1251
|
+
}
|
|
1252
|
+
}
|
|
1253
|
+
scheduleTaskBearingReconciliationContinuation(agentId) {
|
|
1254
|
+
if (this.taskBearingContinuationAgentIds.has(agentId))
|
|
1255
|
+
return;
|
|
1256
|
+
this.taskBearingContinuationAgentIds.add(agentId);
|
|
1257
|
+
queueMicrotask(() => {
|
|
1258
|
+
this.taskBearingContinuationAgentIds.delete(agentId);
|
|
1259
|
+
this.reconcileTaskBearingMailbox(agentId);
|
|
1260
|
+
});
|
|
1261
|
+
}
|
|
1262
|
+
reconcileTaskBearingTranscriptDelivery(target, message) {
|
|
1263
|
+
if (!this.reconcileControlTranscript(target, message, false).delivered)
|
|
1264
|
+
return false;
|
|
1265
|
+
this.acknowledgeDeliveredMailboxMessage(target.agentId, message.messageId);
|
|
1266
|
+
return true;
|
|
1267
|
+
}
|
|
1268
|
+
settleTerminalTaskBearingMessage(agent, mailbox, message, scheduled, expectedMessageId) {
|
|
1269
|
+
const returnSettlement = expectedMessageId === message.messageId;
|
|
1270
|
+
if (scheduled.skipReason && mailbox.deadLetterOrdinaryTask(message.messageId, scheduled.skipReason)) {
|
|
1271
|
+
return returnSettlement ? scheduled : undefined;
|
|
1272
|
+
}
|
|
1273
|
+
const retained = mailbox.getMessage(message.messageId);
|
|
1274
|
+
if (!retained || retained.deliveredAt !== undefined || retained.failedAt !== undefined) {
|
|
1275
|
+
return returnSettlement ? scheduled : undefined;
|
|
1276
|
+
}
|
|
1277
|
+
const activity = this.activityForAgent(agent);
|
|
1278
|
+
if (activity !== "idle") {
|
|
1279
|
+
return { ...scheduled, skipReason: `worker_${activity}:transcript_fallback_waiting` };
|
|
1280
|
+
}
|
|
1281
|
+
this.reconcileControlTranscript(agent, retained, true);
|
|
1282
|
+
this.acknowledgeDeliveredMailboxMessage(agent.agentId, retained.messageId);
|
|
1283
|
+
const settlement = {
|
|
1284
|
+
...scheduled,
|
|
1285
|
+
skipReason: `${scheduled.skipReason ?? "worker_task_terminal"}:transcript_fallback_delivered`,
|
|
1286
|
+
};
|
|
1287
|
+
return returnSettlement ? settlement : undefined;
|
|
1288
|
+
}
|
|
1289
|
+
controlMessageAttempt(agentId, message) {
|
|
1290
|
+
const attempt = this.controlMessageAttemptById(agentId, message.messageId);
|
|
1291
|
+
if (attempt && attempt.dispatch.instructions !== message.content) {
|
|
1292
|
+
throw new Error(`Worker control message '${message.messageId}' conflicts with its durable dispatch.`);
|
|
1293
|
+
}
|
|
1294
|
+
return attempt;
|
|
1295
|
+
}
|
|
1296
|
+
controlMessageAttemptById(agentId, messageId) {
|
|
1297
|
+
const matches = Object.values(this.options.getLifecycle().getTaskRuntimeSnapshot().attempts).filter((attempt) => attempt.dispatch.logicalLaneId === agentId && attempt.dispatch.controlMessageId === messageId);
|
|
1298
|
+
if (matches.length > 1) {
|
|
1299
|
+
throw new Error(`Worker control message '${messageId}' owns multiple durable attempts.`);
|
|
1300
|
+
}
|
|
1301
|
+
return matches[0];
|
|
1302
|
+
}
|
|
1303
|
+
scheduleCorrelatedTaskBearingAttempt(attempt) {
|
|
1304
|
+
const record = this.options.getLifecycle().getRecord(attempt.taskId);
|
|
1305
|
+
if (!record)
|
|
1306
|
+
return { started: false, skipReason: "orchestration_projection_missing" };
|
|
1307
|
+
if (attempt.status === "queued")
|
|
1308
|
+
return this.scheduleTaskBearingAttempt(record, attempt);
|
|
1309
|
+
if (attempt.status === "leased" || attempt.status === "running" || attempt.status === "suspended") {
|
|
1310
|
+
return { started: true, record };
|
|
1311
|
+
}
|
|
1312
|
+
return { started: false, record, skipReason: `worker_task_terminal_${attempt.status}` };
|
|
1313
|
+
}
|
|
1314
|
+
scheduleTaskBearingAttempt(record, attempt) {
|
|
1315
|
+
try {
|
|
1316
|
+
this.options.scheduler.enqueue(record, this.options.recoveredRequest(attempt));
|
|
1317
|
+
}
|
|
1318
|
+
catch (error) {
|
|
1319
|
+
return {
|
|
1320
|
+
started: false,
|
|
1321
|
+
record,
|
|
1322
|
+
skipReason: error instanceof Error ? error.message : String(error),
|
|
1323
|
+
};
|
|
1324
|
+
}
|
|
1325
|
+
try {
|
|
256
1326
|
this.options.scheduler.drain();
|
|
257
|
-
started
|
|
1327
|
+
return { started: true, record };
|
|
258
1328
|
}
|
|
259
|
-
|
|
260
|
-
|
|
1329
|
+
catch (error) {
|
|
1330
|
+
return {
|
|
1331
|
+
started: true,
|
|
1332
|
+
record,
|
|
1333
|
+
skipReason: `worker_task_recovery_pending:${error instanceof Error ? error.message : String(error)}`,
|
|
1334
|
+
};
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
enableAttemptAccountingForNextTask(agent) {
|
|
1338
|
+
if (!agent.resumeContext.sessionFile)
|
|
1339
|
+
return;
|
|
1340
|
+
this.conversations
|
|
1341
|
+
.open({
|
|
1342
|
+
agentDir: this.options.agentDir,
|
|
1343
|
+
resumeContext: agent.resumeContext,
|
|
1344
|
+
expectedLogicalAgentId: agent.agentId,
|
|
1345
|
+
})
|
|
1346
|
+
.enableAttemptUsageBoundaries();
|
|
261
1347
|
}
|
|
262
1348
|
requireControl() {
|
|
263
1349
|
if (!this.options.isControlAvailable()) {
|
|
264
1350
|
throw new Error("Worker delegation control is unavailable in this UAC surface.");
|
|
265
1351
|
}
|
|
266
1352
|
}
|
|
1353
|
+
canonicalAgentIdSet(agentIds, label) {
|
|
1354
|
+
if (!Array.isArray(agentIds) || agentIds.length < 1 || agentIds.length > MAX_ORCHESTRATION_COLLECTION_LENGTH) {
|
|
1355
|
+
throw new TypeError(`${label} agent ids must contain from 1 through ${MAX_ORCHESTRATION_COLLECTION_LENGTH} entries.`);
|
|
1356
|
+
}
|
|
1357
|
+
const canonicalAgentIds = [];
|
|
1358
|
+
const seenAgentIds = new Set();
|
|
1359
|
+
for (const agentId of agentIds) {
|
|
1360
|
+
if (typeof agentId !== "string")
|
|
1361
|
+
throw new TypeError("Logical worker agent id is required.");
|
|
1362
|
+
const canonicalAgentId = agentId.trim();
|
|
1363
|
+
if (!canonicalAgentId)
|
|
1364
|
+
throw new TypeError("Logical worker agent id is required.");
|
|
1365
|
+
if (canonicalAgentId.length > MAX_ORCHESTRATION_IDENTIFIER_LENGTH) {
|
|
1366
|
+
throw new TypeError(`Logical worker agent id exceeds ${MAX_ORCHESTRATION_IDENTIFIER_LENGTH} characters.`);
|
|
1367
|
+
}
|
|
1368
|
+
if (seenAgentIds.has(canonicalAgentId))
|
|
1369
|
+
continue;
|
|
1370
|
+
seenAgentIds.add(canonicalAgentId);
|
|
1371
|
+
canonicalAgentIds.push(canonicalAgentId);
|
|
1372
|
+
}
|
|
1373
|
+
return canonicalAgentIds;
|
|
1374
|
+
}
|
|
267
1375
|
requireKnownAgent(agentId) {
|
|
268
1376
|
const normalized = agentId.trim();
|
|
269
1377
|
if (!normalized)
|
|
@@ -273,18 +1381,14 @@ export class WorkerAgentControlCoordinator {
|
|
|
273
1381
|
throw new Error(`Unknown logical worker agent '${normalized}'.`);
|
|
274
1382
|
return agent;
|
|
275
1383
|
}
|
|
276
|
-
|
|
1384
|
+
requireSessionPeer(agentId, scope) {
|
|
277
1385
|
const target = this.requireKnownAgent(agentId);
|
|
278
|
-
if (
|
|
279
|
-
|
|
280
|
-
const caller = this.requireKnownAgent(scope.callerAgentId);
|
|
281
|
-
if (caller.rootAgentId !== target.rootAgentId) {
|
|
282
|
-
throw new Error(`Logical worker agent '${target.agentId}' is outside the caller's agent tree.`);
|
|
283
|
-
}
|
|
1386
|
+
if (scope.callerAgentId)
|
|
1387
|
+
this.requireKnownAgent(scope.callerAgentId);
|
|
284
1388
|
return target;
|
|
285
1389
|
}
|
|
286
1390
|
requireControllableAgent(agentId, scope) {
|
|
287
|
-
const target = this.
|
|
1391
|
+
const target = this.requireSessionPeer(agentId, scope);
|
|
288
1392
|
if (!scope.callerAgentId)
|
|
289
1393
|
return target;
|
|
290
1394
|
const caller = this.requireKnownAgent(scope.callerAgentId);
|
|
@@ -305,6 +1409,20 @@ export class WorkerAgentControlCoordinator {
|
|
|
305
1409
|
return latest;
|
|
306
1410
|
return agent.activeAttemptId ? lifecycle.getTaskRuntimeSnapshot().attempts[agent.activeAttemptId] : undefined;
|
|
307
1411
|
}
|
|
1412
|
+
latestAttemptsByAgent(snapshot) {
|
|
1413
|
+
return latestAgentAttemptsByDurableOrder(snapshot);
|
|
1414
|
+
}
|
|
1415
|
+
activityForAgent(agent) {
|
|
1416
|
+
return this.projectAgentActivity(agent, this.latestAgentAttempt(agent));
|
|
1417
|
+
}
|
|
1418
|
+
projectAgentActivity(agent, attempt) {
|
|
1419
|
+
if (attempt?.status === "suspended" || agent.status === "suspended")
|
|
1420
|
+
return "suspended";
|
|
1421
|
+
if (attempt?.status === "queued" || attempt?.status === "leased" || attempt?.status === "running") {
|
|
1422
|
+
return "active";
|
|
1423
|
+
}
|
|
1424
|
+
return "idle";
|
|
1425
|
+
}
|
|
308
1426
|
controlledAgentAttempt(agentId, scope) {
|
|
309
1427
|
this.requireControl();
|
|
310
1428
|
const agent = this.requireControllableAgent(agentId, scope);
|
|
@@ -323,39 +1441,103 @@ export class WorkerAgentControlCoordinator {
|
|
|
323
1441
|
}
|
|
324
1442
|
return mailbox;
|
|
325
1443
|
}
|
|
1444
|
+
isAcceptedControlReplay(agentId, idempotencyKey) {
|
|
1445
|
+
if (idempotencyKey === undefined)
|
|
1446
|
+
return false;
|
|
1447
|
+
const messageId = workerAgentMessageId(this.options.parentSessionId, idempotencyKey);
|
|
1448
|
+
const mailbox = this.getMailbox(agentId);
|
|
1449
|
+
return mailbox.getMessage(messageId) !== undefined || mailbox.hasControlReplayReceipt(messageId);
|
|
1450
|
+
}
|
|
1451
|
+
reusableTaskAdmissionSkipReason(agentId) {
|
|
1452
|
+
const admission = evaluateReusableWorkerTaskAdmission(this.options.getLifecycle().getTaskRuntimeSnapshot(), agentId);
|
|
1453
|
+
return admission.ok ? undefined : admission.reasonCode;
|
|
1454
|
+
}
|
|
326
1455
|
subscribeStateChanges(listener) {
|
|
327
1456
|
this.stateListeners.add(listener);
|
|
328
1457
|
return () => this.stateListeners.delete(listener);
|
|
329
1458
|
}
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
1459
|
+
acknowledgeDeliveredMailboxMessage(agentId, messageId) {
|
|
1460
|
+
const mailbox = this.getMailbox(agentId);
|
|
1461
|
+
const controlMessage = mailbox.getMessage(messageId);
|
|
1462
|
+
if (controlMessage) {
|
|
1463
|
+
const target = this.requireKnownAgent(agentId);
|
|
1464
|
+
const sourceMailbox = this.beginWorkerReplyAcknowledgement(target, controlMessage, true);
|
|
1465
|
+
mailbox.acknowledgeDelivered(messageId);
|
|
1466
|
+
if (sourceMailbox &&
|
|
1467
|
+
controlMessage.replyToMessageId &&
|
|
1468
|
+
!sourceMailbox.commitReplyAcknowledgement(controlMessage.replyToMessageId, controlMessage.messageId) &&
|
|
1469
|
+
sourceMailbox.getReplyAcknowledgementId(controlMessage.replyToMessageId) !== undefined) {
|
|
1470
|
+
throw new Error("Worker reply source acknowledgement did not commit.");
|
|
1471
|
+
}
|
|
1472
|
+
return;
|
|
1473
|
+
}
|
|
1474
|
+
mailbox.acknowledgeDelivered(messageId);
|
|
1475
|
+
}
|
|
1476
|
+
beginWorkerReplyAcknowledgement(target, reply, allowCommittedSource = false) {
|
|
1477
|
+
if (reply.replyToMessageId === undefined)
|
|
1478
|
+
return undefined;
|
|
1479
|
+
if (!reply.senderAgentId)
|
|
1480
|
+
throw new Error("Worker reply routing metadata is incomplete.");
|
|
1481
|
+
const source = this.requireKnownAgent(reply.senderAgentId);
|
|
1482
|
+
const sourceMailbox = this.getMailbox(source.agentId);
|
|
1483
|
+
const request = sourceMailbox.getMessage(reply.replyToMessageId);
|
|
341
1484
|
if (!request || request.expectReply !== true || request.deliveredAt === undefined) {
|
|
342
1485
|
throw new Error("Worker reply does not reference a delivered reply-expected message.");
|
|
343
1486
|
}
|
|
344
1487
|
if (request.senderAgentId !== target.agentId) {
|
|
345
1488
|
throw new Error("Worker reply target does not match the original requester.");
|
|
346
1489
|
}
|
|
347
|
-
if (request.threadId
|
|
1490
|
+
if (request.threadId !== reply.threadId) {
|
|
348
1491
|
throw new Error("Worker reply thread conflicts with the original request.");
|
|
349
1492
|
}
|
|
350
|
-
const
|
|
1493
|
+
const activeAcknowledgementId = sourceMailbox.getReplyAcknowledgementId(request.messageId);
|
|
1494
|
+
if (activeAcknowledgementId && activeAcknowledgementId !== reply.messageId) {
|
|
1495
|
+
throw new Error("Worker reply acknowledgement identity conflicts with an active transaction.");
|
|
1496
|
+
}
|
|
1497
|
+
if (request.repliedAt !== undefined && activeAcknowledgementId === undefined) {
|
|
1498
|
+
if (allowCommittedSource || reply.deliveredAt !== undefined)
|
|
1499
|
+
return sourceMailbox;
|
|
1500
|
+
throw new Error("Worker reply source was marked replied without its exact acknowledgement marker.");
|
|
1501
|
+
}
|
|
1502
|
+
if (!sourceMailbox.beginReplyAcknowledgement(request.messageId, reply.messageId, reply.content)) {
|
|
1503
|
+
throw new Error("Worker reply source acknowledgement could not be acquired.");
|
|
1504
|
+
}
|
|
1505
|
+
return sourceMailbox;
|
|
1506
|
+
}
|
|
1507
|
+
enqueuePeerMessage(target, kind, content, options, task) {
|
|
1508
|
+
if (options.idempotencyKey !== undefined) {
|
|
1509
|
+
this.assertIdempotencyTarget(target.agentId, options.idempotencyKey);
|
|
1510
|
+
}
|
|
1511
|
+
return this.getMailbox(target.agentId).enqueueWithReceipt({
|
|
351
1512
|
kind,
|
|
352
1513
|
content,
|
|
353
1514
|
...options,
|
|
354
|
-
...(
|
|
355
|
-
idempotencyKey: `peer-reply:${sender.agentId}:${request.messageId}`,
|
|
1515
|
+
...(task ? { task } : {}),
|
|
356
1516
|
});
|
|
357
|
-
|
|
358
|
-
|
|
1517
|
+
}
|
|
1518
|
+
workerAgentView(agent, activity = this.activityForAgent(agent)) {
|
|
1519
|
+
return {
|
|
1520
|
+
agentId: agent.agentId,
|
|
1521
|
+
...(agent.parentAgentId ? { parentAgentId: agent.parentAgentId } : {}),
|
|
1522
|
+
rootAgentId: agent.rootAgentId,
|
|
1523
|
+
depth: agent.depth,
|
|
1524
|
+
role: agent.role,
|
|
1525
|
+
status: agent.status,
|
|
1526
|
+
activity,
|
|
1527
|
+
createdAt: agent.createdAt,
|
|
1528
|
+
updatedAt: agent.updatedAt,
|
|
1529
|
+
};
|
|
1530
|
+
}
|
|
1531
|
+
assertIdempotencyTarget(targetAgentId, idempotencyKey) {
|
|
1532
|
+
const messageId = workerAgentMessageId(this.options.parentSessionId, idempotencyKey);
|
|
1533
|
+
for (const agent of Object.values(this.options.getLifecycle().getTaskRuntimeSnapshot().agents)) {
|
|
1534
|
+
if (agent.agentId === targetAgentId)
|
|
1535
|
+
continue;
|
|
1536
|
+
const mailbox = this.getMailbox(agent.agentId);
|
|
1537
|
+
if (mailbox.getMessage(messageId) || mailbox.hasControlReplayReceipt(messageId)) {
|
|
1538
|
+
throw new Error(`Worker control idempotency identity is already accepted by logical worker '${agent.agentId}'.`);
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
359
1541
|
}
|
|
360
1542
|
mailboxMessage(message) {
|
|
361
1543
|
const metadata = [
|