@producible/cereworker-core 26.520.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/abort.d.ts +5 -0
- package/dist/abort.d.ts.map +1 -0
- package/dist/abort.js +36 -0
- package/dist/abort.js.map +1 -0
- package/dist/context.d.ts +20 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +45 -0
- package/dist/context.js.map +1 -0
- package/dist/conversation.d.ts +48 -0
- package/dist/conversation.d.ts.map +1 -0
- package/dist/conversation.js +358 -0
- package/dist/conversation.js.map +1 -0
- package/dist/discovery.d.ts +32 -0
- package/dist/discovery.d.ts.map +1 -0
- package/dist/discovery.js +165 -0
- package/dist/discovery.js.map +1 -0
- package/dist/events.d.ts +222 -0
- package/dist/events.d.ts.map +1 -0
- package/dist/events.js +25 -0
- package/dist/events.js.map +1 -0
- package/dist/http-tools.d.ts +8 -0
- package/dist/http-tools.d.ts.map +1 -0
- package/dist/http-tools.js +137 -0
- package/dist/http-tools.js.map +1 -0
- package/dist/index.d.ts +34 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +20 -0
- package/dist/index.js.map +1 -0
- package/dist/instance.d.ts +56 -0
- package/dist/instance.d.ts.map +1 -0
- package/dist/instance.js +136 -0
- package/dist/instance.js.map +1 -0
- package/dist/legacy-sqlite.d.ts +6 -0
- package/dist/legacy-sqlite.d.ts.map +1 -0
- package/dist/legacy-sqlite.js +90 -0
- package/dist/legacy-sqlite.js.map +1 -0
- package/dist/logger.d.ts +15 -0
- package/dist/logger.d.ts.map +1 -0
- package/dist/logger.js +53 -0
- package/dist/logger.js.map +1 -0
- package/dist/orchestrator.d.ts +321 -0
- package/dist/orchestrator.d.ts.map +1 -0
- package/dist/orchestrator.js +2610 -0
- package/dist/orchestrator.js.map +1 -0
- package/dist/pairing.d.ts +41 -0
- package/dist/pairing.d.ts.map +1 -0
- package/dist/pairing.js +215 -0
- package/dist/pairing.js.map +1 -0
- package/dist/plan-store.d.ts +33 -0
- package/dist/plan-store.d.ts.map +1 -0
- package/dist/plan-store.js +113 -0
- package/dist/plan-store.js.map +1 -0
- package/dist/proactive.d.ts +64 -0
- package/dist/proactive.d.ts.map +1 -0
- package/dist/proactive.js +179 -0
- package/dist/proactive.js.map +1 -0
- package/dist/sub-agent-manager.d.ts +45 -0
- package/dist/sub-agent-manager.d.ts.map +1 -0
- package/dist/sub-agent-manager.js +509 -0
- package/dist/sub-agent-manager.js.map +1 -0
- package/dist/sub-agent-tools.d.ts +4 -0
- package/dist/sub-agent-tools.d.ts.map +1 -0
- package/dist/sub-agent-tools.js +94 -0
- package/dist/sub-agent-tools.js.map +1 -0
- package/dist/system-prompt.d.ts +34 -0
- package/dist/system-prompt.d.ts.map +1 -0
- package/dist/system-prompt.js +256 -0
- package/dist/system-prompt.js.map +1 -0
- package/dist/task-schedule.d.ts +13 -0
- package/dist/task-schedule.d.ts.map +1 -0
- package/dist/task-schedule.js +201 -0
- package/dist/task-schedule.js.map +1 -0
- package/dist/task-store.d.ts +22 -0
- package/dist/task-store.d.ts.map +1 -0
- package/dist/task-store.js +141 -0
- package/dist/task-store.js.map +1 -0
- package/dist/text-store.d.ts +18 -0
- package/dist/text-store.d.ts.map +1 -0
- package/dist/text-store.js +212 -0
- package/dist/text-store.js.map +1 -0
- package/dist/tool-runtime.d.ts +76 -0
- package/dist/tool-runtime.d.ts.map +1 -0
- package/dist/tool-runtime.js +443 -0
- package/dist/tool-runtime.js.map +1 -0
- package/dist/types.d.ts +392 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +31 -0
|
@@ -0,0 +1,2610 @@
|
|
|
1
|
+
import { nanoid } from 'nanoid';
|
|
2
|
+
import { TypedEventEmitter } from './events.js';
|
|
3
|
+
import { ConversationStore } from './conversation.js';
|
|
4
|
+
import { SubAgentManager } from './sub-agent-manager.js';
|
|
5
|
+
import { createSubAgentTools } from './sub-agent-tools.js';
|
|
6
|
+
import { createLogger } from './logger.js';
|
|
7
|
+
import { buildSystemPrompt } from './system-prompt.js';
|
|
8
|
+
import { estimateMessageTokens, shouldCompact, buildCompactionMessages } from './context.js';
|
|
9
|
+
import { createAbortError, throwIfAborted } from './abort.js';
|
|
10
|
+
import { ToolRuntime, } from './tool-runtime.js';
|
|
11
|
+
const log = createLogger('orchestrator');
|
|
12
|
+
const TASK_COMPLETE_TOOL = 'task_complete';
|
|
13
|
+
const TASK_BLOCKED_TOOL = 'task_blocked';
|
|
14
|
+
const TASK_CHECKPOINT_TOOL = 'task_checkpoint';
|
|
15
|
+
const TASK_UPSERT_TOOL = 'task_upsert';
|
|
16
|
+
const TASK_REMOVE_TOOL = 'task_remove';
|
|
17
|
+
const TASK_LIST_TOOL = 'task_list';
|
|
18
|
+
const TASK_GET_TOOL = 'task_get';
|
|
19
|
+
const INTERNAL_TASK_TOOL_NAMES = new Set([
|
|
20
|
+
TASK_COMPLETE_TOOL,
|
|
21
|
+
TASK_BLOCKED_TOOL,
|
|
22
|
+
TASK_CHECKPOINT_TOOL,
|
|
23
|
+
TASK_UPSERT_TOOL,
|
|
24
|
+
TASK_REMOVE_TOOL,
|
|
25
|
+
TASK_LIST_TOOL,
|
|
26
|
+
TASK_GET_TOOL,
|
|
27
|
+
]);
|
|
28
|
+
const SYSTEM_FALLBACK_COMPLETION_PROMPT = '[System fallback] The last turn ended without a final answer. Continue from the last verified state and end by calling task_complete or task_blocked before your final answer.';
|
|
29
|
+
const SYSTEM_FALLBACK_STALL_PROMPT = '[System fallback] The stalled turn is being retried from the last verified state.';
|
|
30
|
+
const DEBUG_TOOL_OUTPUT_MAX_CHARS = 8_000;
|
|
31
|
+
const DEBUG_TOOL_STRUCTURED_MAX_CHARS = 16_000;
|
|
32
|
+
const READ_ONLY_TOOL_NAMES = new Set([
|
|
33
|
+
'browserGetText',
|
|
34
|
+
'browserGetUrl',
|
|
35
|
+
'browserListTabs',
|
|
36
|
+
'browserWait',
|
|
37
|
+
'browserEval',
|
|
38
|
+
'readFile',
|
|
39
|
+
'listDirectory',
|
|
40
|
+
'searchFiles',
|
|
41
|
+
'glob',
|
|
42
|
+
'memory_read',
|
|
43
|
+
'webSearch',
|
|
44
|
+
'httpFetch',
|
|
45
|
+
]);
|
|
46
|
+
export class Orchestrator extends TypedEventEmitter {
|
|
47
|
+
conversations;
|
|
48
|
+
cerebrum = null;
|
|
49
|
+
cerebellum = null;
|
|
50
|
+
subAgentManager = null;
|
|
51
|
+
internalTools = new Map();
|
|
52
|
+
tools = new Map();
|
|
53
|
+
activeConversationId = null;
|
|
54
|
+
systemContext = null;
|
|
55
|
+
verificationEnabled = true;
|
|
56
|
+
verificationTimeoutMs = 5000;
|
|
57
|
+
monitorIntervalMs = 30_000;
|
|
58
|
+
monitorTimer = null;
|
|
59
|
+
abortController = null;
|
|
60
|
+
autoMode = false;
|
|
61
|
+
fineTunePoller = null;
|
|
62
|
+
fineTuneDataProvider = null;
|
|
63
|
+
fineTuneMethod = 'auto';
|
|
64
|
+
fineTuneSchedule = 'auto';
|
|
65
|
+
fineTuneStatus = {
|
|
66
|
+
status: 'idle',
|
|
67
|
+
jobId: '',
|
|
68
|
+
progress: 0,
|
|
69
|
+
currentStep: 0,
|
|
70
|
+
totalSteps: 0,
|
|
71
|
+
currentLoss: 0,
|
|
72
|
+
error: '',
|
|
73
|
+
checkpointPath: '',
|
|
74
|
+
startedAt: 0,
|
|
75
|
+
completedAt: 0,
|
|
76
|
+
};
|
|
77
|
+
_fineTuneHistory = [];
|
|
78
|
+
gatewayMode = 'standalone';
|
|
79
|
+
connectedNodes = 0;
|
|
80
|
+
gatewayUrl;
|
|
81
|
+
profile;
|
|
82
|
+
instanceStore = null;
|
|
83
|
+
proactiveEnabled = false;
|
|
84
|
+
discoveryMode = false;
|
|
85
|
+
onDiscoveryComplete = null;
|
|
86
|
+
lastStreamActivityAt = 0;
|
|
87
|
+
streamWatchdog = null;
|
|
88
|
+
streamNudgeCount = 0;
|
|
89
|
+
streamDeferredUntil = 0;
|
|
90
|
+
streamStallThreshold = 30_000;
|
|
91
|
+
maxNudgeRetries = 2;
|
|
92
|
+
maxCompletionRetries = 2;
|
|
93
|
+
turnJournalRetention = {
|
|
94
|
+
maxDays: 30,
|
|
95
|
+
maxFilesPerConversation: 100,
|
|
96
|
+
};
|
|
97
|
+
streamPhase = 'idle';
|
|
98
|
+
activeToolCall = null;
|
|
99
|
+
currentStreamTurn = null;
|
|
100
|
+
currentQuerySession = null;
|
|
101
|
+
currentAttemptCompletionState = null;
|
|
102
|
+
currentPartialContent = '';
|
|
103
|
+
currentLastContentKind = 'empty';
|
|
104
|
+
currentJournaledContentLength = 0;
|
|
105
|
+
pendingRecoveryDecision = null;
|
|
106
|
+
streamAbortGraceMs = 1_000;
|
|
107
|
+
taskConversations = new Map();
|
|
108
|
+
taskRunning = new Set();
|
|
109
|
+
recurringTasks = [];
|
|
110
|
+
toolRuntime;
|
|
111
|
+
compactionConfig = {
|
|
112
|
+
enabled: true,
|
|
113
|
+
threshold: 0.8,
|
|
114
|
+
keepRecentMessages: 10,
|
|
115
|
+
contextWindow: 128000,
|
|
116
|
+
};
|
|
117
|
+
constructor(options) {
|
|
118
|
+
super();
|
|
119
|
+
this.conversations = options?.conversationStore ?? new ConversationStore();
|
|
120
|
+
this.toolRuntime = new ToolRuntime(options?.toolRuntime);
|
|
121
|
+
this.registerInternalTools();
|
|
122
|
+
if (options?.compaction) {
|
|
123
|
+
this.compactionConfig = { ...this.compactionConfig, ...options.compaction };
|
|
124
|
+
}
|
|
125
|
+
if (options?.streamStallThreshold)
|
|
126
|
+
this.streamStallThreshold = options.streamStallThreshold * 1000;
|
|
127
|
+
if (options?.maxNudgeRetries) {
|
|
128
|
+
this.maxNudgeRetries = options.maxNudgeRetries;
|
|
129
|
+
this.maxCompletionRetries = options.maxNudgeRetries;
|
|
130
|
+
}
|
|
131
|
+
if (options?.turnJournalRetention) {
|
|
132
|
+
this.turnJournalRetention = {
|
|
133
|
+
...this.turnJournalRetention,
|
|
134
|
+
...options.turnJournalRetention,
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
setCerebrum(cerebrum) {
|
|
139
|
+
this.cerebrum = cerebrum;
|
|
140
|
+
}
|
|
141
|
+
setCerebellum(cerebellum, options) {
|
|
142
|
+
this.cerebellum = cerebellum;
|
|
143
|
+
if (options?.enabled !== undefined)
|
|
144
|
+
this.verificationEnabled = options.enabled;
|
|
145
|
+
if (options?.timeoutMs !== undefined)
|
|
146
|
+
this.verificationTimeoutMs = options.timeoutMs;
|
|
147
|
+
}
|
|
148
|
+
/** Set system context (e.g. skills prompt) prepended to every Cerebrum call */
|
|
149
|
+
setSystemContext(context) {
|
|
150
|
+
this.systemContext = context;
|
|
151
|
+
log.info('System context updated', { length: context.length });
|
|
152
|
+
}
|
|
153
|
+
getSystemContext() {
|
|
154
|
+
return this.systemContext;
|
|
155
|
+
}
|
|
156
|
+
setupSubAgents(options) {
|
|
157
|
+
if (!this.cerebrum)
|
|
158
|
+
return null;
|
|
159
|
+
if (options?.monitorIntervalMs)
|
|
160
|
+
this.monitorIntervalMs = options.monitorIntervalMs;
|
|
161
|
+
this.subAgentManager = new SubAgentManager({
|
|
162
|
+
cerebrum: this.cerebrum,
|
|
163
|
+
tools: this.tools,
|
|
164
|
+
maxConcurrent: options?.maxConcurrent,
|
|
165
|
+
baseDir: options?.baseDir,
|
|
166
|
+
toolRuntime: this.toolRuntime,
|
|
167
|
+
onProgress: (agentId, note, percent) => {
|
|
168
|
+
this.emit({ type: 'agent:progress', agentId, note, percent });
|
|
169
|
+
},
|
|
170
|
+
});
|
|
171
|
+
// Register sub-agent tools with the orchestrator
|
|
172
|
+
const agentTools = createSubAgentTools(this.subAgentManager);
|
|
173
|
+
for (const [name, tool] of Object.entries(agentTools)) {
|
|
174
|
+
this.tools.set(name, tool);
|
|
175
|
+
}
|
|
176
|
+
return this.subAgentManager;
|
|
177
|
+
}
|
|
178
|
+
getSubAgentManager() {
|
|
179
|
+
return this.subAgentManager;
|
|
180
|
+
}
|
|
181
|
+
getConversationStore() {
|
|
182
|
+
return this.conversations;
|
|
183
|
+
}
|
|
184
|
+
registerInternalTools() {
|
|
185
|
+
this.internalTools.set(TASK_COMPLETE_TOOL, {
|
|
186
|
+
description: 'Record that a tool-driven task is complete. Call this once right before your final answer with a concise summary and concrete evidence.',
|
|
187
|
+
parameters: {
|
|
188
|
+
type: 'object',
|
|
189
|
+
properties: {
|
|
190
|
+
summary: { type: 'string', description: 'Short summary of what was completed' },
|
|
191
|
+
evidence: { type: 'string', description: 'Concrete evidence proving completion' },
|
|
192
|
+
},
|
|
193
|
+
required: ['summary', 'evidence'],
|
|
194
|
+
additionalProperties: false,
|
|
195
|
+
},
|
|
196
|
+
execute: async (args) => this.recordCompletionSignal('complete', args),
|
|
197
|
+
});
|
|
198
|
+
this.internalTools.set(TASK_BLOCKED_TOOL, {
|
|
199
|
+
description: 'Record that you are blocked and cannot finish the task. Call this once right before your final answer with the blocker and evidence.',
|
|
200
|
+
parameters: {
|
|
201
|
+
type: 'object',
|
|
202
|
+
properties: {
|
|
203
|
+
blocker: { type: 'string', description: 'Specific blocker preventing completion' },
|
|
204
|
+
evidence: { type: 'string', description: 'Concrete evidence showing the blocker' },
|
|
205
|
+
},
|
|
206
|
+
required: ['blocker', 'evidence'],
|
|
207
|
+
additionalProperties: false,
|
|
208
|
+
},
|
|
209
|
+
execute: async (args) => this.recordCompletionSignal('blocked', args),
|
|
210
|
+
});
|
|
211
|
+
this.internalTools.set(TASK_CHECKPOINT_TOOL, {
|
|
212
|
+
description: 'Record a completed or in-progress milestone during a multi-step task. Use this after each major verified step so retries can resume from the right place.',
|
|
213
|
+
parameters: {
|
|
214
|
+
type: 'object',
|
|
215
|
+
properties: {
|
|
216
|
+
step: {
|
|
217
|
+
type: 'string',
|
|
218
|
+
description: 'Short milestone name, such as "profile continuity checked"',
|
|
219
|
+
},
|
|
220
|
+
status: {
|
|
221
|
+
type: 'string',
|
|
222
|
+
enum: ['done', 'in_progress'],
|
|
223
|
+
description: 'Whether the milestone is done or currently in progress',
|
|
224
|
+
},
|
|
225
|
+
evidence: {
|
|
226
|
+
type: 'string',
|
|
227
|
+
description: 'Concrete evidence showing what happened at this milestone',
|
|
228
|
+
},
|
|
229
|
+
},
|
|
230
|
+
required: ['step', 'status', 'evidence'],
|
|
231
|
+
additionalProperties: false,
|
|
232
|
+
},
|
|
233
|
+
execute: async (args) => this.recordTaskCheckpoint(args),
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
getAllTools() {
|
|
237
|
+
return new Map([...this.tools, ...this.internalTools]);
|
|
238
|
+
}
|
|
239
|
+
isInternalTaskSignalTool(name) {
|
|
240
|
+
return INTERNAL_TASK_TOOL_NAMES.has(name.trim() || name);
|
|
241
|
+
}
|
|
242
|
+
async recordCompletionSignal(signal, args) {
|
|
243
|
+
const state = this.currentAttemptCompletionState;
|
|
244
|
+
if (!state) {
|
|
245
|
+
return {
|
|
246
|
+
output: 'No active turn is available for task completion tracking.',
|
|
247
|
+
isError: true,
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
const evidence = String(args.evidence ?? '').trim();
|
|
251
|
+
if (!evidence) {
|
|
252
|
+
return {
|
|
253
|
+
output: 'A non-empty evidence field is required.',
|
|
254
|
+
isError: true,
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
if (signal === 'complete') {
|
|
258
|
+
const summary = String(args.summary ?? '').trim();
|
|
259
|
+
if (!summary) {
|
|
260
|
+
return {
|
|
261
|
+
output: 'A non-empty summary field is required.',
|
|
262
|
+
isError: true,
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
const hasVerifiedProgress = state.successfulExternalToolCount > 0 ||
|
|
266
|
+
state.continuity.progressLedger.some((entry) => entry.source === 'tool' && !entry.isError);
|
|
267
|
+
if (!hasVerifiedProgress) {
|
|
268
|
+
return {
|
|
269
|
+
output: 'task_complete requires at least one successful external tool result in this turn.',
|
|
270
|
+
isError: true,
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
state.signal = 'complete';
|
|
274
|
+
state.summary = summary;
|
|
275
|
+
state.blocker = undefined;
|
|
276
|
+
state.evidence = evidence;
|
|
277
|
+
}
|
|
278
|
+
else {
|
|
279
|
+
const blocker = String(args.blocker ?? '').trim();
|
|
280
|
+
if (!blocker) {
|
|
281
|
+
return {
|
|
282
|
+
output: 'A non-empty blocker field is required.',
|
|
283
|
+
isError: true,
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
state.signal = 'blocked';
|
|
287
|
+
state.blocker = blocker;
|
|
288
|
+
state.summary = undefined;
|
|
289
|
+
state.evidence = evidence;
|
|
290
|
+
}
|
|
291
|
+
const boundarySummary = signal === 'complete'
|
|
292
|
+
? `Task completion recorded: ${state.summary}`
|
|
293
|
+
: `Task blocker recorded: ${state.blocker}`;
|
|
294
|
+
this.recordBoundary(state.continuity, {
|
|
295
|
+
kind: 'completion',
|
|
296
|
+
action: signal === 'complete' ? TASK_COMPLETE_TOOL : TASK_BLOCKED_TOOL,
|
|
297
|
+
summary: boundarySummary,
|
|
298
|
+
stateChanging: true,
|
|
299
|
+
evidence,
|
|
300
|
+
});
|
|
301
|
+
this.appendTurnJournalEntry('completion_signal', boundarySummary, {
|
|
302
|
+
signal,
|
|
303
|
+
evidence,
|
|
304
|
+
summary: state.summary,
|
|
305
|
+
blocker: state.blocker,
|
|
306
|
+
});
|
|
307
|
+
this.emitCompletionTrace('signal_recorded', signal === 'complete'
|
|
308
|
+
? `Recorded task_complete signal with evidence: ${evidence}`
|
|
309
|
+
: `Recorded task_blocked signal with evidence: ${evidence}`, signal, 'info');
|
|
310
|
+
return {
|
|
311
|
+
output: signal === 'complete' ? 'Task completion recorded.' : 'Task blocker recorded.',
|
|
312
|
+
isError: false,
|
|
313
|
+
metadata: {
|
|
314
|
+
internal: true,
|
|
315
|
+
signal,
|
|
316
|
+
},
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
async recordTaskCheckpoint(args) {
|
|
320
|
+
const state = this.currentAttemptCompletionState;
|
|
321
|
+
if (!state) {
|
|
322
|
+
return {
|
|
323
|
+
output: 'No active turn is available for task checkpoint tracking.',
|
|
324
|
+
isError: true,
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
const step = String(args.step ?? '').trim();
|
|
328
|
+
const evidence = String(args.evidence ?? '').trim();
|
|
329
|
+
const statusValue = String(args.status ?? '').trim();
|
|
330
|
+
const status = statusValue === 'done' || statusValue === 'in_progress' ? statusValue : null;
|
|
331
|
+
if (!step) {
|
|
332
|
+
return {
|
|
333
|
+
output: 'A non-empty step field is required.',
|
|
334
|
+
isError: true,
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
if (!status) {
|
|
338
|
+
return {
|
|
339
|
+
output: 'status must be either "done" or "in_progress".',
|
|
340
|
+
isError: true,
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
if (!evidence) {
|
|
344
|
+
return {
|
|
345
|
+
output: 'A non-empty evidence field is required.',
|
|
346
|
+
isError: true,
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
const checkpoint = this.recordCheckpoint(state.continuity, step, status, evidence);
|
|
350
|
+
log.info('task_checkpoint_recorded', {
|
|
351
|
+
turnId: this.currentStreamTurn?.turnId,
|
|
352
|
+
attempt: this.currentStreamTurn?.attempt,
|
|
353
|
+
conversationId: this.currentStreamTurn?.conversationId,
|
|
354
|
+
step,
|
|
355
|
+
status,
|
|
356
|
+
evidence,
|
|
357
|
+
});
|
|
358
|
+
return {
|
|
359
|
+
output: `Checkpoint recorded: ${checkpoint.summary}`,
|
|
360
|
+
isError: false,
|
|
361
|
+
metadata: {
|
|
362
|
+
internal: true,
|
|
363
|
+
checkpoint,
|
|
364
|
+
},
|
|
365
|
+
};
|
|
366
|
+
}
|
|
367
|
+
registerTool(name, tool) {
|
|
368
|
+
if (this.internalTools.has(name)) {
|
|
369
|
+
throw new Error(`Tool name ${name} is reserved for internal task signaling`);
|
|
370
|
+
}
|
|
371
|
+
this.tools.set(name, tool);
|
|
372
|
+
}
|
|
373
|
+
registerTools(tools) {
|
|
374
|
+
for (const [name, tool] of Object.entries(tools)) {
|
|
375
|
+
this.registerTool(name, tool);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
async executeTool(name, args, options) {
|
|
379
|
+
return this.toolRuntime.execute({
|
|
380
|
+
toolCall: {
|
|
381
|
+
id: options?.callId ?? nanoid(10),
|
|
382
|
+
name,
|
|
383
|
+
args,
|
|
384
|
+
},
|
|
385
|
+
tools: this.getAllTools(),
|
|
386
|
+
conversationId: options?.conversationId,
|
|
387
|
+
sessionKey: options?.sessionKey,
|
|
388
|
+
scopeKey: options?.scopeKey,
|
|
389
|
+
turnId: options?.turnId,
|
|
390
|
+
attempt: options?.attempt,
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
unregisterTool(name) {
|
|
394
|
+
if (this.internalTools.has(name)) {
|
|
395
|
+
return false;
|
|
396
|
+
}
|
|
397
|
+
return this.tools.delete(name);
|
|
398
|
+
}
|
|
399
|
+
setProfile(profile) {
|
|
400
|
+
this.profile = profile;
|
|
401
|
+
}
|
|
402
|
+
setInstanceStore(store) {
|
|
403
|
+
this.instanceStore = store;
|
|
404
|
+
}
|
|
405
|
+
getInstanceStore() {
|
|
406
|
+
return this.instanceStore;
|
|
407
|
+
}
|
|
408
|
+
setProactiveEnabled(enabled) {
|
|
409
|
+
this.proactiveEnabled = enabled;
|
|
410
|
+
}
|
|
411
|
+
setDiscoveryMode(enabled) {
|
|
412
|
+
this.discoveryMode = enabled;
|
|
413
|
+
log.info('Discovery mode changed', { enabled });
|
|
414
|
+
}
|
|
415
|
+
isDiscoveryMode() {
|
|
416
|
+
return this.discoveryMode;
|
|
417
|
+
}
|
|
418
|
+
setDiscoveryCompleteHandler(handler) {
|
|
419
|
+
this.onDiscoveryComplete = handler;
|
|
420
|
+
}
|
|
421
|
+
sendProactiveMessage(content, source) {
|
|
422
|
+
this.emit({ type: 'message:proactive', content, source });
|
|
423
|
+
}
|
|
424
|
+
setAutoMode(enabled) {
|
|
425
|
+
this.autoMode = enabled;
|
|
426
|
+
log.info('Auto mode changed', { autoMode: enabled });
|
|
427
|
+
}
|
|
428
|
+
getAutoMode() {
|
|
429
|
+
return this.autoMode;
|
|
430
|
+
}
|
|
431
|
+
setGatewayMode(mode, extras) {
|
|
432
|
+
this.gatewayMode = mode;
|
|
433
|
+
if (extras?.connectedNodes !== undefined)
|
|
434
|
+
this.connectedNodes = extras.connectedNodes;
|
|
435
|
+
if (extras?.gatewayUrl !== undefined)
|
|
436
|
+
this.gatewayUrl = extras.gatewayUrl;
|
|
437
|
+
}
|
|
438
|
+
emergencyStop() {
|
|
439
|
+
// 1. Abort current stream
|
|
440
|
+
this.abortController?.abort();
|
|
441
|
+
this.abortController = null;
|
|
442
|
+
// 2. Cancel all sub-agents
|
|
443
|
+
if (this.subAgentManager) {
|
|
444
|
+
const agents = this.subAgentManager.listAgents();
|
|
445
|
+
agents
|
|
446
|
+
.filter((a) => a.status === 'running' || a.status === 'pending')
|
|
447
|
+
.forEach((a) => this.subAgentManager.cancel(a.id));
|
|
448
|
+
}
|
|
449
|
+
// 3. Emit event
|
|
450
|
+
this.emit({ type: 'emergency:stop' });
|
|
451
|
+
log.warn('Emergency stop triggered');
|
|
452
|
+
}
|
|
453
|
+
/** Set a callback that provides training pairs for fine-tuning (from HippocampusCurator). */
|
|
454
|
+
setFineTuneDataProvider(provider, method = 'auto') {
|
|
455
|
+
this.fineTuneDataProvider = provider;
|
|
456
|
+
this.fineTuneMethod = method;
|
|
457
|
+
}
|
|
458
|
+
getFineTuneMethod() {
|
|
459
|
+
return this.fineTuneMethod;
|
|
460
|
+
}
|
|
461
|
+
setFineTuneMethod(method) {
|
|
462
|
+
this.fineTuneMethod = method;
|
|
463
|
+
}
|
|
464
|
+
getFineTuneSchedule() {
|
|
465
|
+
return this.fineTuneSchedule;
|
|
466
|
+
}
|
|
467
|
+
setFineTuneSchedule(schedule) {
|
|
468
|
+
this.fineTuneSchedule = schedule;
|
|
469
|
+
}
|
|
470
|
+
/** Get current fine-tune status from Cerebellum (or cached local state). */
|
|
471
|
+
async getFineTuneStatus() {
|
|
472
|
+
if (this.cerebellum?.getFineTuneStatus) {
|
|
473
|
+
try {
|
|
474
|
+
const remote = await this.cerebellum.getFineTuneStatus();
|
|
475
|
+
if (remote) {
|
|
476
|
+
this.fineTuneStatus = remote;
|
|
477
|
+
return remote;
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
catch {
|
|
481
|
+
// Fall back to local cache
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
return this.fineTuneStatus;
|
|
485
|
+
}
|
|
486
|
+
getFineTuneHistory() {
|
|
487
|
+
return this._fineTuneHistory;
|
|
488
|
+
}
|
|
489
|
+
/** Minimum training pairs required before starting a fine-tune run. */
|
|
490
|
+
static MIN_TRAINING_PAIRS = 5;
|
|
491
|
+
/** Trigger a fine-tuning run: collect training data, send to cerebellum, start training, poll progress. */
|
|
492
|
+
async triggerFineTune() {
|
|
493
|
+
if (!this.cerebellum?.ingestTrainingData || !this.cerebellum?.startFineTune) {
|
|
494
|
+
throw new Error('Cerebellum fine-tuning not available');
|
|
495
|
+
}
|
|
496
|
+
// 1. Collect training data
|
|
497
|
+
let pairs = [];
|
|
498
|
+
if (this.fineTuneDataProvider) {
|
|
499
|
+
pairs = await this.fineTuneDataProvider();
|
|
500
|
+
}
|
|
501
|
+
// 2. Ingest training data
|
|
502
|
+
let totalPending = 0;
|
|
503
|
+
if (pairs.length > 0) {
|
|
504
|
+
totalPending = await this.cerebellum.ingestTrainingData(pairs);
|
|
505
|
+
log.info('Training data ingested', { newPairs: pairs.length, totalPending });
|
|
506
|
+
}
|
|
507
|
+
// 3. Check minimum threshold
|
|
508
|
+
if (totalPending < Orchestrator.MIN_TRAINING_PAIRS) {
|
|
509
|
+
log.info('Not enough training data, deferring fine-tune', {
|
|
510
|
+
totalPending,
|
|
511
|
+
threshold: Orchestrator.MIN_TRAINING_PAIRS,
|
|
512
|
+
});
|
|
513
|
+
throw new Error(`Not enough training data (${totalPending}/${Orchestrator.MIN_TRAINING_PAIRS} pairs). ` +
|
|
514
|
+
'Data has been saved — training will start automatically when enough accumulates.');
|
|
515
|
+
}
|
|
516
|
+
// 4. Start fine-tuning
|
|
517
|
+
const result = await this.cerebellum.startFineTune({ method: this.fineTuneMethod });
|
|
518
|
+
if (!result.started) {
|
|
519
|
+
throw new Error(result.error || 'Failed to start fine-tuning');
|
|
520
|
+
}
|
|
521
|
+
this.fineTuneStatus = {
|
|
522
|
+
...this.fineTuneStatus,
|
|
523
|
+
status: 'running',
|
|
524
|
+
jobId: result.jobId,
|
|
525
|
+
startedAt: Date.now(),
|
|
526
|
+
};
|
|
527
|
+
this.emit({ type: 'finetune:start', jobId: result.jobId });
|
|
528
|
+
log.info('Fine-tuning started', { jobId: result.jobId });
|
|
529
|
+
// 5. Poll for progress
|
|
530
|
+
this.stopFineTunePoller();
|
|
531
|
+
this.fineTunePoller = setInterval(async () => {
|
|
532
|
+
try {
|
|
533
|
+
const status = await this.cerebellum.getFineTuneStatus();
|
|
534
|
+
if (!status)
|
|
535
|
+
return;
|
|
536
|
+
this.fineTuneStatus = status;
|
|
537
|
+
if (status.status === 'running') {
|
|
538
|
+
this.emit({
|
|
539
|
+
type: 'finetune:progress',
|
|
540
|
+
jobId: status.jobId,
|
|
541
|
+
progress: status.progress,
|
|
542
|
+
loss: status.currentLoss,
|
|
543
|
+
});
|
|
544
|
+
}
|
|
545
|
+
else if (status.status === 'completed') {
|
|
546
|
+
this._fineTuneHistory.push({
|
|
547
|
+
jobId: status.jobId,
|
|
548
|
+
status: 'completed',
|
|
549
|
+
completedAt: status.completedAt || Date.now(),
|
|
550
|
+
loss: status.currentLoss,
|
|
551
|
+
});
|
|
552
|
+
this.instanceStore?.recordFineTune({
|
|
553
|
+
jobId: status.jobId,
|
|
554
|
+
method: this.fineTuneMethod,
|
|
555
|
+
completedAt: new Date(status.completedAt || Date.now()).toISOString(),
|
|
556
|
+
checkpointPath: status.checkpointPath,
|
|
557
|
+
loss: status.currentLoss,
|
|
558
|
+
trainingPairs: status.totalSteps,
|
|
559
|
+
});
|
|
560
|
+
this.emit({
|
|
561
|
+
type: 'finetune:complete',
|
|
562
|
+
jobId: status.jobId,
|
|
563
|
+
checkpointPath: status.checkpointPath,
|
|
564
|
+
});
|
|
565
|
+
log.info('Fine-tuning completed', {
|
|
566
|
+
jobId: status.jobId,
|
|
567
|
+
checkpoint: status.checkpointPath,
|
|
568
|
+
});
|
|
569
|
+
this.stopFineTunePoller();
|
|
570
|
+
}
|
|
571
|
+
else if (status.status === 'failed') {
|
|
572
|
+
this._fineTuneHistory.push({
|
|
573
|
+
jobId: status.jobId,
|
|
574
|
+
status: 'failed',
|
|
575
|
+
completedAt: Date.now(),
|
|
576
|
+
loss: status.currentLoss,
|
|
577
|
+
});
|
|
578
|
+
this.emit({
|
|
579
|
+
type: 'finetune:error',
|
|
580
|
+
jobId: status.jobId,
|
|
581
|
+
error: status.error,
|
|
582
|
+
});
|
|
583
|
+
log.error('Fine-tuning failed', { jobId: status.jobId, error: status.error });
|
|
584
|
+
this.stopFineTunePoller();
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
catch {
|
|
588
|
+
// Polling failure is non-blocking
|
|
589
|
+
}
|
|
590
|
+
}, 10_000);
|
|
591
|
+
}
|
|
592
|
+
stopFineTunePoller() {
|
|
593
|
+
if (this.fineTunePoller) {
|
|
594
|
+
clearInterval(this.fineTunePoller);
|
|
595
|
+
this.fineTunePoller = null;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
parseDiscoveryCompletion(text) {
|
|
599
|
+
const match = text.match(/<discovery_complete>\s*([\s\S]*?)\s*<\/discovery_complete>/);
|
|
600
|
+
if (!match)
|
|
601
|
+
return null;
|
|
602
|
+
const block = match[1];
|
|
603
|
+
const nameMatch = block.match(/name:\s*(.+)/i);
|
|
604
|
+
const roleMatch = block.match(/role:\s*(.+)/i);
|
|
605
|
+
const traitsMatch = block.match(/traits:\s*(.+)/i);
|
|
606
|
+
return {
|
|
607
|
+
name: nameMatch?.[1]?.trim() || 'Cere',
|
|
608
|
+
role: roleMatch?.[1]?.trim() || 'general-purpose assistant',
|
|
609
|
+
traits: traitsMatch?.[1]
|
|
610
|
+
?.split(',')
|
|
611
|
+
.map((t) => t.trim().toLowerCase())
|
|
612
|
+
.filter(Boolean) ?? [],
|
|
613
|
+
};
|
|
614
|
+
}
|
|
615
|
+
getActiveConversationId() {
|
|
616
|
+
return this.activeConversationId;
|
|
617
|
+
}
|
|
618
|
+
getConversation(id) {
|
|
619
|
+
return this.conversations.get(id);
|
|
620
|
+
}
|
|
621
|
+
getMessages(conversationId) {
|
|
622
|
+
const id = conversationId ?? this.activeConversationId;
|
|
623
|
+
if (!id)
|
|
624
|
+
return [];
|
|
625
|
+
return this.conversations.getMessages(id);
|
|
626
|
+
}
|
|
627
|
+
getQuerySession(conversationId, sessionId) {
|
|
628
|
+
return this.conversations.getQuerySession(conversationId, sessionId);
|
|
629
|
+
}
|
|
630
|
+
getSessionEvents(conversationId, sessionId) {
|
|
631
|
+
return this.conversations.getSessionEvents(conversationId, sessionId);
|
|
632
|
+
}
|
|
633
|
+
recordSessionEvent(conversationId, sessionId, type, summary, data) {
|
|
634
|
+
const session = this.conversations.getQuerySession(conversationId, sessionId);
|
|
635
|
+
if (!session)
|
|
636
|
+
return;
|
|
637
|
+
const updatedAt = Date.now();
|
|
638
|
+
const updatedSession = {
|
|
639
|
+
...session,
|
|
640
|
+
updatedAt,
|
|
641
|
+
summary: this.truncateResumeText(summary, 500),
|
|
642
|
+
};
|
|
643
|
+
this.conversations.saveQuerySession(conversationId, updatedSession);
|
|
644
|
+
const event = {
|
|
645
|
+
sessionId,
|
|
646
|
+
conversationId,
|
|
647
|
+
turnId: session.turnId,
|
|
648
|
+
attempt: session.attempt,
|
|
649
|
+
timestamp: updatedAt,
|
|
650
|
+
type,
|
|
651
|
+
state: updatedSession.state,
|
|
652
|
+
summary: this.truncateResumeText(summary, 500),
|
|
653
|
+
instanceId: updatedSession.instanceId,
|
|
654
|
+
checkpointPath: updatedSession.checkpointPath ?? null,
|
|
655
|
+
data,
|
|
656
|
+
};
|
|
657
|
+
this.conversations.appendSessionEvent(conversationId, sessionId, event);
|
|
658
|
+
}
|
|
659
|
+
recordSessionMemoryUpdate(conversationId, sessionId, snapshot) {
|
|
660
|
+
const session = this.conversations.getQuerySession(conversationId, sessionId);
|
|
661
|
+
if (!session)
|
|
662
|
+
return;
|
|
663
|
+
const updated = {
|
|
664
|
+
...session,
|
|
665
|
+
memory: snapshot,
|
|
666
|
+
updatedAt: Date.now(),
|
|
667
|
+
summary: snapshot.summary || session.summary,
|
|
668
|
+
};
|
|
669
|
+
this.conversations.saveQuerySession(conversationId, updated);
|
|
670
|
+
const event = {
|
|
671
|
+
sessionId,
|
|
672
|
+
conversationId,
|
|
673
|
+
turnId: session.turnId,
|
|
674
|
+
attempt: session.attempt,
|
|
675
|
+
timestamp: snapshot.updatedAt,
|
|
676
|
+
type: 'memory_updated',
|
|
677
|
+
state: updated.state,
|
|
678
|
+
summary: this.truncateResumeText(`Session memory updated: ${snapshot.summary}`, 500),
|
|
679
|
+
instanceId: updated.instanceId,
|
|
680
|
+
checkpointPath: updated.checkpointPath ?? null,
|
|
681
|
+
data: {
|
|
682
|
+
excerpt: snapshot.excerpt,
|
|
683
|
+
},
|
|
684
|
+
};
|
|
685
|
+
this.conversations.appendSessionEvent(conversationId, sessionId, event);
|
|
686
|
+
this.emit({ type: 'session:memory-updated', conversationId, sessionId, snapshot });
|
|
687
|
+
}
|
|
688
|
+
startConversation() {
|
|
689
|
+
const conversation = this.conversations.create();
|
|
690
|
+
this.activeConversationId = conversation.id;
|
|
691
|
+
this.instanceStore?.incrementConversation();
|
|
692
|
+
log.info('Started conversation', { id: conversation.id });
|
|
693
|
+
return conversation.id;
|
|
694
|
+
}
|
|
695
|
+
/** Resume an existing conversation by ID */
|
|
696
|
+
resumeConversation(id) {
|
|
697
|
+
const conversation = this.conversations.get(id);
|
|
698
|
+
if (!conversation)
|
|
699
|
+
return false;
|
|
700
|
+
this.activeConversationId = id;
|
|
701
|
+
const messages = this.conversations.getMessages(id);
|
|
702
|
+
this.emit({ type: 'conversation:resumed', conversationId: id, messages });
|
|
703
|
+
log.info('Resumed conversation', { id, messageCount: messages.length });
|
|
704
|
+
return true;
|
|
705
|
+
}
|
|
706
|
+
// --- Recurring Task Execution ---
|
|
707
|
+
setRecurringTasks(tasks) {
|
|
708
|
+
this.recurringTasks = tasks;
|
|
709
|
+
}
|
|
710
|
+
setTaskConversation(taskId, conversationId) {
|
|
711
|
+
this.taskConversations.set(taskId, conversationId);
|
|
712
|
+
}
|
|
713
|
+
getTaskConversation(taskId) {
|
|
714
|
+
return this.taskConversations.get(taskId);
|
|
715
|
+
}
|
|
716
|
+
isTaskRunning(taskId) {
|
|
717
|
+
return this.taskRunning.has(taskId);
|
|
718
|
+
}
|
|
719
|
+
async runTask(taskId, goal, options) {
|
|
720
|
+
if (this.taskRunning.has(taskId)) {
|
|
721
|
+
return { success: false, error: 'Task already running' };
|
|
722
|
+
}
|
|
723
|
+
// Get or create a dedicated conversation for this task
|
|
724
|
+
let convId = this.taskConversations.get(taskId);
|
|
725
|
+
if (!convId || !this.conversations.get(convId)) {
|
|
726
|
+
const conv = this.conversations.create();
|
|
727
|
+
convId = conv.id;
|
|
728
|
+
this.taskConversations.set(taskId, convId);
|
|
729
|
+
}
|
|
730
|
+
const prevAutoMode = this.autoMode;
|
|
731
|
+
if (options?.autoMode !== undefined) {
|
|
732
|
+
this.autoMode = options.autoMode;
|
|
733
|
+
}
|
|
734
|
+
this.taskRunning.add(taskId);
|
|
735
|
+
this.emit({ type: 'task:start', taskId, goal });
|
|
736
|
+
log.info('Running recurring task', { taskId, conversationId: convId });
|
|
737
|
+
const timeoutMs = options?.timeoutMs ?? 600_000;
|
|
738
|
+
try {
|
|
739
|
+
const now = new Date().toISOString();
|
|
740
|
+
const taskDef = this.recurringTasks.find((t) => t.id === taskId);
|
|
741
|
+
const schedule = taskDef?.schedule ?? 'unknown';
|
|
742
|
+
const prompt = [
|
|
743
|
+
`[Recurring Task: ${taskId}]`,
|
|
744
|
+
`Schedule: ${schedule}`,
|
|
745
|
+
`Current time: ${now}`,
|
|
746
|
+
`Goal: ${goal}`,
|
|
747
|
+
'',
|
|
748
|
+
'Execute this goal using your available tools and skills.',
|
|
749
|
+
'Review your conversation history for learnings from previous runs.',
|
|
750
|
+
'If a step fails, try an alternative approach before giving up.',
|
|
751
|
+
'After completing, use memory_log to record what you did, outcomes, and any issues.',
|
|
752
|
+
].join('\n');
|
|
753
|
+
await Promise.race([
|
|
754
|
+
this.sendMessage(prompt, convId),
|
|
755
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error(`Task timed out after ${timeoutMs / 1000}s`)), timeoutMs)),
|
|
756
|
+
]);
|
|
757
|
+
this.emit({ type: 'task:complete', taskId });
|
|
758
|
+
log.info('Recurring task completed', { taskId });
|
|
759
|
+
return { success: true };
|
|
760
|
+
}
|
|
761
|
+
catch (err) {
|
|
762
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
763
|
+
this.emit({ type: 'task:error', taskId, error });
|
|
764
|
+
log.warn('Recurring task failed', { taskId, error });
|
|
765
|
+
return { success: false, error };
|
|
766
|
+
}
|
|
767
|
+
finally {
|
|
768
|
+
this.taskRunning.delete(taskId);
|
|
769
|
+
this.autoMode = prevAutoMode;
|
|
770
|
+
}
|
|
771
|
+
}
|
|
772
|
+
getStreamState() {
|
|
773
|
+
const stallThresholdMs = this.getCurrentStallThresholdMs();
|
|
774
|
+
return {
|
|
775
|
+
streaming: this.streamWatchdog !== null,
|
|
776
|
+
lastActivityAt: this.lastStreamActivityAt,
|
|
777
|
+
stallDetected: this.streamWatchdog !== null && Date.now() - this.lastStreamActivityAt > stallThresholdMs,
|
|
778
|
+
nudgeCount: this.streamNudgeCount,
|
|
779
|
+
phase: this.streamPhase,
|
|
780
|
+
activeToolName: this.activeToolCall?.name,
|
|
781
|
+
activeToolCallId: this.activeToolCall?.id,
|
|
782
|
+
activeToolStartedAt: this.activeToolCall?.startedAt,
|
|
783
|
+
};
|
|
784
|
+
}
|
|
785
|
+
markStreamWaitingModel(activityAt = Date.now()) {
|
|
786
|
+
const phaseChanged = this.streamPhase !== 'waiting_model' || this.activeToolCall !== null;
|
|
787
|
+
this.lastStreamActivityAt = activityAt;
|
|
788
|
+
this.streamPhase = 'waiting_model';
|
|
789
|
+
this.activeToolCall = null;
|
|
790
|
+
if (phaseChanged) {
|
|
791
|
+
this.logStreamDebug('stream_phase_changed', {
|
|
792
|
+
phase: this.streamPhase,
|
|
793
|
+
});
|
|
794
|
+
}
|
|
795
|
+
}
|
|
796
|
+
markStreamWaitingTool(toolCall, activityAt = Date.now()) {
|
|
797
|
+
const normalizedToolName = toolCall.name.trim() || toolCall.name;
|
|
798
|
+
const phaseChanged = this.streamPhase !== 'waiting_tool' ||
|
|
799
|
+
this.activeToolCall?.id !== toolCall.id ||
|
|
800
|
+
this.activeToolCall?.name !== normalizedToolName;
|
|
801
|
+
this.lastStreamActivityAt = activityAt;
|
|
802
|
+
this.streamPhase = 'waiting_tool';
|
|
803
|
+
this.activeToolCall = {
|
|
804
|
+
id: toolCall.id,
|
|
805
|
+
name: normalizedToolName,
|
|
806
|
+
startedAt: activityAt,
|
|
807
|
+
};
|
|
808
|
+
if (phaseChanged) {
|
|
809
|
+
this.logStreamDebug('stream_phase_changed', {
|
|
810
|
+
phase: this.streamPhase,
|
|
811
|
+
activeToolName: normalizedToolName,
|
|
812
|
+
activeToolCallId: toolCall.id,
|
|
813
|
+
});
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
resetStreamState() {
|
|
817
|
+
this.streamPhase = 'idle';
|
|
818
|
+
this.activeToolCall = null;
|
|
819
|
+
this.streamDeferredUntil = 0;
|
|
820
|
+
this.currentPartialContent = '';
|
|
821
|
+
this.currentLastContentKind = 'empty';
|
|
822
|
+
this.currentJournaledContentLength = 0;
|
|
823
|
+
}
|
|
824
|
+
createQuerySession(conversationId, turnId, attempt, source, latestUserMessage, stallRetryCount, completionRetryCount, priorSession) {
|
|
825
|
+
const timestamp = Date.now();
|
|
826
|
+
const instance = this.instanceStore?.get();
|
|
827
|
+
return {
|
|
828
|
+
id: turnId,
|
|
829
|
+
conversationId,
|
|
830
|
+
turnId,
|
|
831
|
+
attempt,
|
|
832
|
+
source,
|
|
833
|
+
state: 'ready',
|
|
834
|
+
startedAt: timestamp,
|
|
835
|
+
updatedAt: timestamp,
|
|
836
|
+
summary: `Turn attempt ${attempt} started.`,
|
|
837
|
+
latestUserMessage: this.truncateResumeText(latestUserMessage, 1200),
|
|
838
|
+
stallRetryCount,
|
|
839
|
+
completionRetryCount,
|
|
840
|
+
instanceId: instance?.id,
|
|
841
|
+
checkpointPath: instance?.activeCheckpoint ?? null,
|
|
842
|
+
// Carry forward recovery state from prior attempt for crash recovery
|
|
843
|
+
...(priorSession?.latestBoundary ? { latestBoundary: priorSession.latestBoundary } : {}),
|
|
844
|
+
...(priorSession?.lastOutcome ? { lastOutcome: priorSession.lastOutcome } : {}),
|
|
845
|
+
...(priorSession?.lastError ? { lastError: priorSession.lastError } : {}),
|
|
846
|
+
};
|
|
847
|
+
}
|
|
848
|
+
saveCurrentQuerySession() {
|
|
849
|
+
if (!this.currentStreamTurn || !this.currentQuerySession)
|
|
850
|
+
return;
|
|
851
|
+
this.conversations.saveQuerySession(this.currentStreamTurn.conversationId, this.currentQuerySession);
|
|
852
|
+
}
|
|
853
|
+
updateCurrentQuerySession(type, summary, data) {
|
|
854
|
+
if (!this.currentStreamTurn || !this.currentQuerySession)
|
|
855
|
+
return;
|
|
856
|
+
const eventState = this.getQuerySessionState(type, data);
|
|
857
|
+
const updatedAt = Date.now();
|
|
858
|
+
const next = {
|
|
859
|
+
...this.currentQuerySession,
|
|
860
|
+
attempt: this.currentStreamTurn.attempt,
|
|
861
|
+
updatedAt,
|
|
862
|
+
summary: this.truncateResumeText(summary, 500),
|
|
863
|
+
state: this.resolveQuerySessionState(type, eventState, data),
|
|
864
|
+
checkpointPath: this.instanceStore?.get()?.activeCheckpoint ?? this.currentQuerySession.checkpointPath ?? null,
|
|
865
|
+
};
|
|
866
|
+
if (type === 'partial_text') {
|
|
867
|
+
const excerpt = typeof data?.excerpt === 'string' ? data.excerpt : summary;
|
|
868
|
+
next.latestAssistantMessage = this.truncateResumeText(excerpt, 1200);
|
|
869
|
+
}
|
|
870
|
+
if (type === 'tool_start') {
|
|
871
|
+
next.activeToolName = typeof data?.toolName === 'string' ? data.toolName : undefined;
|
|
872
|
+
next.activeToolCallId = typeof data?.callId === 'string' ? data.callId : undefined;
|
|
873
|
+
}
|
|
874
|
+
else if (type !== 'tool_end') {
|
|
875
|
+
next.activeToolName = undefined;
|
|
876
|
+
next.activeToolCallId = undefined;
|
|
877
|
+
}
|
|
878
|
+
if (type === 'tool_end') {
|
|
879
|
+
next.activeToolName = undefined;
|
|
880
|
+
next.activeToolCallId = undefined;
|
|
881
|
+
}
|
|
882
|
+
if (type === 'turn_finished') {
|
|
883
|
+
const finalContent = typeof data?.finalContent === 'string' ? data.finalContent.trim() : '';
|
|
884
|
+
if (finalContent) {
|
|
885
|
+
next.latestAssistantMessage = this.truncateResumeText(finalContent, 1200);
|
|
886
|
+
}
|
|
887
|
+
if (typeof data?.turnOutcome === 'string') {
|
|
888
|
+
next.lastOutcome = data.turnOutcome;
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
if (type === 'turn_error') {
|
|
892
|
+
next.lastError = typeof data?.error === 'string' ? data.error : summary;
|
|
893
|
+
if (data?.aborted === true) {
|
|
894
|
+
next.lastOutcome = 'aborted';
|
|
895
|
+
}
|
|
896
|
+
else {
|
|
897
|
+
next.lastOutcome = 'protocol_error';
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
if (type === 'completion_signal' && typeof data?.signal === 'string') {
|
|
901
|
+
next.summary = this.truncateResumeText(summary, 500);
|
|
902
|
+
}
|
|
903
|
+
this.currentQuerySession = next;
|
|
904
|
+
this.saveCurrentQuerySession();
|
|
905
|
+
}
|
|
906
|
+
resolveQuerySessionState(type, defaultState, data) {
|
|
907
|
+
if (type === 'turn_finished') {
|
|
908
|
+
const turnOutcome = data?.turnOutcome;
|
|
909
|
+
if (turnOutcome === 'completed' || turnOutcome === 'completed_no_text')
|
|
910
|
+
return 'completed';
|
|
911
|
+
if (turnOutcome === 'stalled')
|
|
912
|
+
return 'stalled';
|
|
913
|
+
if (turnOutcome === 'aborted')
|
|
914
|
+
return 'aborted';
|
|
915
|
+
if (turnOutcome === 'ended_on_tool_calls' || turnOutcome === 'completion_signal_missing') {
|
|
916
|
+
return 'waiting_followup';
|
|
917
|
+
}
|
|
918
|
+
if (turnOutcome === 'protocol_error')
|
|
919
|
+
return 'failed';
|
|
920
|
+
}
|
|
921
|
+
return defaultState;
|
|
922
|
+
}
|
|
923
|
+
appendTurnJournalEntry(type, summary, data) {
|
|
924
|
+
if (!this.currentStreamTurn)
|
|
925
|
+
return;
|
|
926
|
+
const entry = {
|
|
927
|
+
turnId: this.currentStreamTurn.turnId,
|
|
928
|
+
attempt: this.currentStreamTurn.attempt,
|
|
929
|
+
timestamp: Date.now(),
|
|
930
|
+
type,
|
|
931
|
+
summary: this.truncateResumeText(summary, 500),
|
|
932
|
+
...(data ? { data } : {}),
|
|
933
|
+
};
|
|
934
|
+
this.conversations.appendTurnJournalEntry(this.currentStreamTurn.conversationId, this.currentStreamTurn.turnId, entry);
|
|
935
|
+
this.appendSessionEvent(type, entry.summary, data);
|
|
936
|
+
}
|
|
937
|
+
appendSessionEvent(type, summary, data) {
|
|
938
|
+
if (!this.currentStreamTurn)
|
|
939
|
+
return;
|
|
940
|
+
const instance = this.instanceStore?.get();
|
|
941
|
+
const event = {
|
|
942
|
+
sessionId: this.currentStreamTurn.sessionId,
|
|
943
|
+
conversationId: this.currentStreamTurn.conversationId,
|
|
944
|
+
turnId: this.currentStreamTurn.turnId,
|
|
945
|
+
attempt: this.currentStreamTurn.attempt,
|
|
946
|
+
timestamp: Date.now(),
|
|
947
|
+
type: this.mapJournalEntryToSessionEvent(type),
|
|
948
|
+
state: this.getQuerySessionState(type, data),
|
|
949
|
+
summary: this.truncateResumeText(summary, 500),
|
|
950
|
+
instanceId: instance?.id,
|
|
951
|
+
checkpointPath: instance?.activeCheckpoint ?? null,
|
|
952
|
+
...(data ? { data } : {}),
|
|
953
|
+
};
|
|
954
|
+
this.conversations.appendSessionEvent(this.currentStreamTurn.conversationId, this.currentStreamTurn.sessionId, event);
|
|
955
|
+
this.updateCurrentQuerySession(type, summary, data);
|
|
956
|
+
}
|
|
957
|
+
mapJournalEntryToSessionEvent(type) {
|
|
958
|
+
switch (type) {
|
|
959
|
+
case 'tool_start':
|
|
960
|
+
return 'tool_started';
|
|
961
|
+
case 'tool_end':
|
|
962
|
+
return 'tool_finished';
|
|
963
|
+
case 'checkpoint':
|
|
964
|
+
return 'checkpoint_recorded';
|
|
965
|
+
case 'boundary':
|
|
966
|
+
return 'boundary_committed';
|
|
967
|
+
case 'completion_signal':
|
|
968
|
+
return 'completion_signal_recorded';
|
|
969
|
+
case 'recovery':
|
|
970
|
+
return 'recovery_assessed';
|
|
971
|
+
case 'turn_error':
|
|
972
|
+
return 'turn_failed';
|
|
973
|
+
default:
|
|
974
|
+
return type;
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
getQuerySessionState(type, data) {
|
|
978
|
+
switch (type) {
|
|
979
|
+
case 'turn_started':
|
|
980
|
+
return 'ready';
|
|
981
|
+
case 'partial_text':
|
|
982
|
+
return 'sampling';
|
|
983
|
+
case 'tool_start':
|
|
984
|
+
return 'tool_execution';
|
|
985
|
+
case 'tool_end':
|
|
986
|
+
case 'checkpoint':
|
|
987
|
+
case 'boundary':
|
|
988
|
+
case 'completion_signal':
|
|
989
|
+
return 'waiting_followup';
|
|
990
|
+
case 'recovery':
|
|
991
|
+
return data?.cause === 'stall' ? 'stalled' : 'waiting_followup';
|
|
992
|
+
case 'turn_finished':
|
|
993
|
+
return 'completed';
|
|
994
|
+
case 'turn_error':
|
|
995
|
+
return data?.aborted ? 'aborted' : 'failed';
|
|
996
|
+
default:
|
|
997
|
+
return 'waiting_followup';
|
|
998
|
+
}
|
|
999
|
+
}
|
|
1000
|
+
pruneTurnJournals(conversationId) {
|
|
1001
|
+
try {
|
|
1002
|
+
const result = this.conversations.pruneTurnJournals(conversationId, this.turnJournalRetention);
|
|
1003
|
+
if (result.prunedByAge > 0 || result.prunedByCount > 0) {
|
|
1004
|
+
log.debug('Pruned turn journals', {
|
|
1005
|
+
conversationId,
|
|
1006
|
+
prunedByAge: result.prunedByAge,
|
|
1007
|
+
prunedByCount: result.prunedByCount,
|
|
1008
|
+
remaining: result.remaining,
|
|
1009
|
+
});
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
catch (error) {
|
|
1013
|
+
log.warn('Failed to prune turn journals', {
|
|
1014
|
+
conversationId,
|
|
1015
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1016
|
+
});
|
|
1017
|
+
}
|
|
1018
|
+
}
|
|
1019
|
+
persistPartialContentSnapshot(force = false) {
|
|
1020
|
+
const normalized = this.currentPartialContent.trim();
|
|
1021
|
+
if (!normalized)
|
|
1022
|
+
return;
|
|
1023
|
+
if (!force && this.currentPartialContent.length - this.currentJournaledContentLength < 200) {
|
|
1024
|
+
return;
|
|
1025
|
+
}
|
|
1026
|
+
this.currentJournaledContentLength = this.currentPartialContent.length;
|
|
1027
|
+
this.appendTurnJournalEntry('partial_text', `Assistant partial text: ${this.truncateResumeText(normalized, 220)}`, {
|
|
1028
|
+
chars: this.currentPartialContent.length,
|
|
1029
|
+
excerpt: this.truncateResumeText(normalized, 1200),
|
|
1030
|
+
});
|
|
1031
|
+
}
|
|
1032
|
+
recordBoundary(continuity, boundary) {
|
|
1033
|
+
const createdAt = Date.now();
|
|
1034
|
+
const summary = {
|
|
1035
|
+
id: nanoid(10),
|
|
1036
|
+
createdAt,
|
|
1037
|
+
browserState: this.cloneBrowserState(boundary.browserState ?? continuity.browserState),
|
|
1038
|
+
...boundary,
|
|
1039
|
+
};
|
|
1040
|
+
continuity.boundaries.push(summary);
|
|
1041
|
+
while (continuity.boundaries.length > 20) {
|
|
1042
|
+
continuity.boundaries.shift();
|
|
1043
|
+
}
|
|
1044
|
+
this.appendTurnJournalEntry('boundary', summary.summary, {
|
|
1045
|
+
boundaryId: summary.id,
|
|
1046
|
+
kind: summary.kind,
|
|
1047
|
+
action: summary.action,
|
|
1048
|
+
stateChanging: summary.stateChanging,
|
|
1049
|
+
url: summary.url,
|
|
1050
|
+
tabId: summary.tabId,
|
|
1051
|
+
checkpointStatus: summary.checkpointStatus,
|
|
1052
|
+
evidence: summary.evidence,
|
|
1053
|
+
browserState: summary.browserState,
|
|
1054
|
+
});
|
|
1055
|
+
if (this.currentQuerySession) {
|
|
1056
|
+
this.currentQuerySession = {
|
|
1057
|
+
...this.currentQuerySession,
|
|
1058
|
+
latestBoundary: summary,
|
|
1059
|
+
updatedAt: Date.now(),
|
|
1060
|
+
};
|
|
1061
|
+
this.saveCurrentQuerySession();
|
|
1062
|
+
}
|
|
1063
|
+
return summary;
|
|
1064
|
+
}
|
|
1065
|
+
deriveRepetitionSignals(continuity) {
|
|
1066
|
+
const signals = [];
|
|
1067
|
+
const recentToolEntries = continuity.progressLedger
|
|
1068
|
+
.filter((entry) => entry.source === 'tool')
|
|
1069
|
+
.slice(-10);
|
|
1070
|
+
const recentActions = recentToolEntries.map((entry) => entry.action);
|
|
1071
|
+
if (recentActions.length >= 4 && new Set(recentActions.slice(-4)).size <= 2) {
|
|
1072
|
+
signals.push(`Recent tool actions are cycling between ${Array.from(new Set(recentActions.slice(-4))).join(', ')}`);
|
|
1073
|
+
}
|
|
1074
|
+
const summaryCounts = new Map();
|
|
1075
|
+
for (const boundary of continuity.boundaries.slice(-12)) {
|
|
1076
|
+
if (boundary.kind !== 'tool' || !boundary.stateChanging)
|
|
1077
|
+
continue;
|
|
1078
|
+
const key = `${boundary.action}|${boundary.summary}`;
|
|
1079
|
+
summaryCounts.set(key, (summaryCounts.get(key) ?? 0) + 1);
|
|
1080
|
+
}
|
|
1081
|
+
for (const [key, count] of summaryCounts.entries()) {
|
|
1082
|
+
if (count < 2)
|
|
1083
|
+
continue;
|
|
1084
|
+
const [, summary] = key.split('|', 2);
|
|
1085
|
+
signals.push(`Repeated verified action x${count}: ${summary}`);
|
|
1086
|
+
}
|
|
1087
|
+
return signals.slice(0, 5);
|
|
1088
|
+
}
|
|
1089
|
+
classifyTurnOutcome(displayContent, finishMeta, completionState) {
|
|
1090
|
+
const trimmed = displayContent.trim();
|
|
1091
|
+
const endedOnToolCalls = finishMeta?.finishReason === 'tool-calls' ||
|
|
1092
|
+
finishMeta?.stepFinishReasons.at(-1) === 'tool-calls' ||
|
|
1093
|
+
finishMeta?.endedWithToolCall === true;
|
|
1094
|
+
if (endedOnToolCalls) {
|
|
1095
|
+
return 'ended_on_tool_calls';
|
|
1096
|
+
}
|
|
1097
|
+
if (completionState.externalToolCallCount > 0 && completionState.signal === 'none') {
|
|
1098
|
+
return 'completion_signal_missing';
|
|
1099
|
+
}
|
|
1100
|
+
if (finishMeta?.finishReason === 'error' || finishMeta?.lastContentKind === 'error') {
|
|
1101
|
+
return 'protocol_error';
|
|
1102
|
+
}
|
|
1103
|
+
if (trimmed.length > 0) {
|
|
1104
|
+
return 'completed';
|
|
1105
|
+
}
|
|
1106
|
+
return 'completed_no_text';
|
|
1107
|
+
}
|
|
1108
|
+
getStreamDiagnostics(elapsedSeconds) {
|
|
1109
|
+
return {
|
|
1110
|
+
elapsedSeconds,
|
|
1111
|
+
phase: this.streamPhase,
|
|
1112
|
+
activeToolName: this.activeToolCall?.name,
|
|
1113
|
+
activeToolCallId: this.activeToolCall?.id,
|
|
1114
|
+
activeToolStartedAt: this.activeToolCall?.startedAt,
|
|
1115
|
+
};
|
|
1116
|
+
}
|
|
1117
|
+
describeStreamLocation(phase = this.streamPhase, activeToolName = this.activeToolCall?.name) {
|
|
1118
|
+
if (phase === 'waiting_tool') {
|
|
1119
|
+
return activeToolName ? `waiting_tool/${activeToolName}` : 'waiting_tool';
|
|
1120
|
+
}
|
|
1121
|
+
return phase;
|
|
1122
|
+
}
|
|
1123
|
+
getCurrentStallThresholdMs(phase = this.streamPhase, nudgeCount = this.streamNudgeCount) {
|
|
1124
|
+
const phaseBase = phase === 'waiting_model' ? this.streamStallThreshold * 3 : this.streamStallThreshold;
|
|
1125
|
+
return phaseBase + nudgeCount * this.streamStallThreshold;
|
|
1126
|
+
}
|
|
1127
|
+
logStreamDebug(msg, data) {
|
|
1128
|
+
if (!this.currentStreamTurn)
|
|
1129
|
+
return;
|
|
1130
|
+
log.debug(msg, {
|
|
1131
|
+
turnId: this.currentStreamTurn.turnId,
|
|
1132
|
+
attempt: this.currentStreamTurn.attempt,
|
|
1133
|
+
conversationId: this.currentStreamTurn.conversationId,
|
|
1134
|
+
...data,
|
|
1135
|
+
});
|
|
1136
|
+
}
|
|
1137
|
+
emitWatchdog(stage, message, options) {
|
|
1138
|
+
if (!this.currentStreamTurn)
|
|
1139
|
+
return;
|
|
1140
|
+
const payload = {
|
|
1141
|
+
stage,
|
|
1142
|
+
turnId: this.currentStreamTurn.turnId,
|
|
1143
|
+
attempt: this.currentStreamTurn.attempt,
|
|
1144
|
+
conversationId: this.currentStreamTurn.conversationId,
|
|
1145
|
+
message,
|
|
1146
|
+
...this.getStreamDiagnostics(options?.elapsedSeconds),
|
|
1147
|
+
};
|
|
1148
|
+
const level = options?.level ?? 'info';
|
|
1149
|
+
switch (level) {
|
|
1150
|
+
case 'debug':
|
|
1151
|
+
log.debug(`watchdog_${stage}`, payload);
|
|
1152
|
+
break;
|
|
1153
|
+
case 'warn':
|
|
1154
|
+
log.warn(`watchdog_${stage}`, payload);
|
|
1155
|
+
break;
|
|
1156
|
+
case 'error':
|
|
1157
|
+
log.error(`watchdog_${stage}`, payload);
|
|
1158
|
+
break;
|
|
1159
|
+
default:
|
|
1160
|
+
log.info(`watchdog_${stage}`, payload);
|
|
1161
|
+
break;
|
|
1162
|
+
}
|
|
1163
|
+
this.emit({ type: 'cerebrum:watchdog', ...payload });
|
|
1164
|
+
}
|
|
1165
|
+
emitCompletionTrace(stage, message, signal, level = 'info') {
|
|
1166
|
+
if (!this.currentStreamTurn)
|
|
1167
|
+
return;
|
|
1168
|
+
const payload = {
|
|
1169
|
+
stage,
|
|
1170
|
+
turnId: this.currentStreamTurn.turnId,
|
|
1171
|
+
attempt: this.currentStreamTurn.attempt,
|
|
1172
|
+
conversationId: this.currentStreamTurn.conversationId,
|
|
1173
|
+
signal,
|
|
1174
|
+
message,
|
|
1175
|
+
...this.getStreamDiagnostics(),
|
|
1176
|
+
};
|
|
1177
|
+
switch (level) {
|
|
1178
|
+
case 'debug':
|
|
1179
|
+
log.debug(`completion_${stage}`, payload);
|
|
1180
|
+
break;
|
|
1181
|
+
case 'warn':
|
|
1182
|
+
log.warn(`completion_${stage}`, payload);
|
|
1183
|
+
break;
|
|
1184
|
+
case 'error':
|
|
1185
|
+
log.error(`completion_${stage}`, payload);
|
|
1186
|
+
break;
|
|
1187
|
+
default:
|
|
1188
|
+
log.info(`completion_${stage}`, payload);
|
|
1189
|
+
break;
|
|
1190
|
+
}
|
|
1191
|
+
this.emit({ type: 'cerebrum:completion', ...payload });
|
|
1192
|
+
}
|
|
1193
|
+
createAttemptCompletionState(continuity) {
|
|
1194
|
+
return {
|
|
1195
|
+
signal: 'none',
|
|
1196
|
+
evidence: '',
|
|
1197
|
+
successfulExternalToolCount: 0,
|
|
1198
|
+
externalToolCallCount: 0,
|
|
1199
|
+
internalToolCallCount: 0,
|
|
1200
|
+
continuity,
|
|
1201
|
+
};
|
|
1202
|
+
}
|
|
1203
|
+
createTurnContinuityState() {
|
|
1204
|
+
return {
|
|
1205
|
+
progressLedger: [],
|
|
1206
|
+
taskCheckpoints: [],
|
|
1207
|
+
browserState: {},
|
|
1208
|
+
boundaries: [],
|
|
1209
|
+
};
|
|
1210
|
+
}
|
|
1211
|
+
buildRecoveryRequest(params) {
|
|
1212
|
+
const partialContent = this.truncateResumeText(params.partialContent, 600);
|
|
1213
|
+
const continuity = params.completionState.continuity;
|
|
1214
|
+
const latestBoundary = continuity.boundaries.at(-1);
|
|
1215
|
+
return {
|
|
1216
|
+
conversationId: this.currentStreamTurn?.conversationId ?? '',
|
|
1217
|
+
turnId: this.currentStreamTurn?.turnId ?? '',
|
|
1218
|
+
attempt: params.attempt,
|
|
1219
|
+
cause: params.cause,
|
|
1220
|
+
phase: this.streamPhase,
|
|
1221
|
+
activeToolName: this.activeToolCall?.name,
|
|
1222
|
+
activeToolCallId: this.activeToolCall?.id,
|
|
1223
|
+
stallRetryCount: this.streamNudgeCount,
|
|
1224
|
+
completionRetryCount: params.completionRetryCount ?? 0,
|
|
1225
|
+
finishReason: params.finishMeta?.finishReason ?? params.finishMeta?.stepFinishReasons.at(-1),
|
|
1226
|
+
turnOutcome: params.turnOutcome,
|
|
1227
|
+
lastContentKind: params.finishMeta?.lastContentKind ?? this.currentLastContentKind,
|
|
1228
|
+
elapsedSeconds: params.elapsedSeconds,
|
|
1229
|
+
partialContent: partialContent || undefined,
|
|
1230
|
+
latestUserMessage: params.latestUserMessage
|
|
1231
|
+
? this.truncateResumeText(params.latestUserMessage, 600)
|
|
1232
|
+
: undefined,
|
|
1233
|
+
progressEntries: continuity.progressLedger.slice(-50).map((entry) => ({ ...entry })),
|
|
1234
|
+
taskCheckpoints: continuity.taskCheckpoints.map((checkpoint) => ({ ...checkpoint })),
|
|
1235
|
+
browserState: this.cloneBrowserState(continuity.browserState),
|
|
1236
|
+
latestBoundary: latestBoundary
|
|
1237
|
+
? {
|
|
1238
|
+
...latestBoundary,
|
|
1239
|
+
...(latestBoundary.browserState
|
|
1240
|
+
? { browserState: this.cloneBrowserState(latestBoundary.browserState) }
|
|
1241
|
+
: {}),
|
|
1242
|
+
}
|
|
1243
|
+
: undefined,
|
|
1244
|
+
recentBoundaries: continuity.boundaries.slice(-20).map((boundary) => ({
|
|
1245
|
+
...boundary,
|
|
1246
|
+
...(boundary.browserState
|
|
1247
|
+
? { browserState: this.cloneBrowserState(boundary.browserState) }
|
|
1248
|
+
: {}),
|
|
1249
|
+
})),
|
|
1250
|
+
repetitionSignals: this.deriveRepetitionSignals(continuity),
|
|
1251
|
+
};
|
|
1252
|
+
}
|
|
1253
|
+
emitRecoveryTrace(cause, source, assessment, level = 'info') {
|
|
1254
|
+
if (!this.currentStreamTurn)
|
|
1255
|
+
return;
|
|
1256
|
+
const payload = {
|
|
1257
|
+
type: 'cerebellum:recovery',
|
|
1258
|
+
cause,
|
|
1259
|
+
action: assessment.action,
|
|
1260
|
+
turnId: this.currentStreamTurn.turnId,
|
|
1261
|
+
attempt: this.currentStreamTurn.attempt,
|
|
1262
|
+
conversationId: this.currentStreamTurn.conversationId,
|
|
1263
|
+
message: assessment.operatorMessage,
|
|
1264
|
+
operatorMessage: assessment.operatorMessage,
|
|
1265
|
+
diagnosis: assessment.diagnosis,
|
|
1266
|
+
nextStep: assessment.nextStep,
|
|
1267
|
+
completedSteps: assessment.completedSteps,
|
|
1268
|
+
waitSeconds: assessment.waitSeconds,
|
|
1269
|
+
source,
|
|
1270
|
+
...this.getStreamDiagnostics(),
|
|
1271
|
+
};
|
|
1272
|
+
switch (level) {
|
|
1273
|
+
case 'debug':
|
|
1274
|
+
log.debug('cerebellum_recovery', payload);
|
|
1275
|
+
break;
|
|
1276
|
+
case 'warn':
|
|
1277
|
+
log.warn('cerebellum_recovery', payload);
|
|
1278
|
+
break;
|
|
1279
|
+
case 'error':
|
|
1280
|
+
log.error('cerebellum_recovery', payload);
|
|
1281
|
+
break;
|
|
1282
|
+
default:
|
|
1283
|
+
log.info('cerebellum_recovery', payload);
|
|
1284
|
+
break;
|
|
1285
|
+
}
|
|
1286
|
+
this.emit(payload);
|
|
1287
|
+
}
|
|
1288
|
+
async assessTurnRecovery(request) {
|
|
1289
|
+
log.debug('turn_recovery_request', {
|
|
1290
|
+
turnId: request.turnId,
|
|
1291
|
+
attempt: request.attempt,
|
|
1292
|
+
conversationId: request.conversationId,
|
|
1293
|
+
cause: request.cause,
|
|
1294
|
+
phase: request.phase,
|
|
1295
|
+
activeToolName: request.activeToolName,
|
|
1296
|
+
activeToolCallId: request.activeToolCallId,
|
|
1297
|
+
stallRetryCount: request.stallRetryCount,
|
|
1298
|
+
completionRetryCount: request.completionRetryCount,
|
|
1299
|
+
finishReason: request.finishReason,
|
|
1300
|
+
turnOutcome: request.turnOutcome,
|
|
1301
|
+
lastContentKind: request.lastContentKind,
|
|
1302
|
+
elapsedSeconds: request.elapsedSeconds,
|
|
1303
|
+
hasPartialContent: Boolean(request.partialContent),
|
|
1304
|
+
latestUserMessage: request.latestUserMessage
|
|
1305
|
+
? this.truncateResumeText(request.latestUserMessage, 300)
|
|
1306
|
+
: '',
|
|
1307
|
+
browserState: request.browserState,
|
|
1308
|
+
progressEntries: request.progressEntries,
|
|
1309
|
+
taskCheckpoints: request.taskCheckpoints,
|
|
1310
|
+
latestBoundary: request.latestBoundary,
|
|
1311
|
+
recentBoundaries: request.recentBoundaries,
|
|
1312
|
+
repetitionSignals: request.repetitionSignals,
|
|
1313
|
+
});
|
|
1314
|
+
if (this.cerebellum?.isConnected() && this.cerebellum.assessTurnRecovery) {
|
|
1315
|
+
try {
|
|
1316
|
+
const assessment = await this.cerebellum.assessTurnRecovery(request);
|
|
1317
|
+
if (assessment) {
|
|
1318
|
+
if (request.cause === 'completion' && assessment.action === 'wait') {
|
|
1319
|
+
return {
|
|
1320
|
+
source: 'cerebellum',
|
|
1321
|
+
assessment: {
|
|
1322
|
+
...assessment,
|
|
1323
|
+
action: 'retry',
|
|
1324
|
+
waitSeconds: undefined,
|
|
1325
|
+
},
|
|
1326
|
+
};
|
|
1327
|
+
}
|
|
1328
|
+
return { source: 'cerebellum', assessment };
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
catch (error) {
|
|
1332
|
+
log.warn('Turn recovery assessment failed', {
|
|
1333
|
+
turnId: request.turnId,
|
|
1334
|
+
attempt: request.attempt,
|
|
1335
|
+
conversationId: request.conversationId,
|
|
1336
|
+
cause: request.cause,
|
|
1337
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1338
|
+
});
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
return {
|
|
1342
|
+
source: 'fallback',
|
|
1343
|
+
assessment: this.buildFallbackRecoveryAssessment(request),
|
|
1344
|
+
};
|
|
1345
|
+
}
|
|
1346
|
+
deriveCompletedSteps(request) {
|
|
1347
|
+
const completed = new Set();
|
|
1348
|
+
for (const boundary of request.recentBoundaries) {
|
|
1349
|
+
if ((boundary.kind === 'tool' ||
|
|
1350
|
+
boundary.kind === 'checkpoint' ||
|
|
1351
|
+
boundary.kind === 'completion') &&
|
|
1352
|
+
boundary.summary) {
|
|
1353
|
+
completed.add(boundary.summary);
|
|
1354
|
+
}
|
|
1355
|
+
}
|
|
1356
|
+
for (const checkpoint of request.taskCheckpoints) {
|
|
1357
|
+
if (checkpoint.status === 'done') {
|
|
1358
|
+
completed.add(checkpoint.summary);
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
for (const entry of request.progressEntries) {
|
|
1362
|
+
if (entry.source === 'tool' && entry.stateChanging && !entry.isError) {
|
|
1363
|
+
completed.add(entry.summary);
|
|
1364
|
+
}
|
|
1365
|
+
}
|
|
1366
|
+
return Array.from(completed).slice(-10);
|
|
1367
|
+
}
|
|
1368
|
+
buildFallbackRecoveryAssessment(request, options) {
|
|
1369
|
+
const completedSteps = this.deriveCompletedSteps(request);
|
|
1370
|
+
const browserHints = [];
|
|
1371
|
+
if (request.browserState.currentUrl)
|
|
1372
|
+
browserHints.push(`Current URL: ${request.browserState.currentUrl}`);
|
|
1373
|
+
if (request.browserState.activeTabId)
|
|
1374
|
+
browserHints.push(`Active tab: ${request.browserState.activeTabId}`);
|
|
1375
|
+
const diagnosis = options?.reason ??
|
|
1376
|
+
(request.cause === 'stall'
|
|
1377
|
+
? `Recovery guidance is unavailable while the stream is stalled in ${this.describeStreamLocation(request.phase, request.activeToolName)}.`
|
|
1378
|
+
: `Recovery guidance is unavailable after the turn ended with ${request.turnOutcome} (${request.finishReason ?? 'no final answer'}).`);
|
|
1379
|
+
const nextStep = request.cause === 'stall'
|
|
1380
|
+
? 'Resume from the last verified browser state and continue with the next unfinished step.'
|
|
1381
|
+
: 'Use the verified progress below to continue from the next unfinished step and avoid repeating confirmed work.';
|
|
1382
|
+
const lines = [
|
|
1383
|
+
'[System fallback recovery]',
|
|
1384
|
+
diagnosis,
|
|
1385
|
+
'The failed attempt tool history has been removed; rely on this verified summary instead.',
|
|
1386
|
+
];
|
|
1387
|
+
if (completedSteps.length > 0) {
|
|
1388
|
+
lines.push('', 'Completed steps:');
|
|
1389
|
+
for (const step of completedSteps)
|
|
1390
|
+
lines.push(`- ${step}`);
|
|
1391
|
+
}
|
|
1392
|
+
if (browserHints.length > 0) {
|
|
1393
|
+
lines.push('', 'Last known browser state:');
|
|
1394
|
+
for (const hint of browserHints)
|
|
1395
|
+
lines.push(`- ${hint}`);
|
|
1396
|
+
}
|
|
1397
|
+
if (request.latestBoundary) {
|
|
1398
|
+
lines.push('', `Latest verified boundary: ${request.latestBoundary.summary}`);
|
|
1399
|
+
}
|
|
1400
|
+
if (request.repetitionSignals.length > 0) {
|
|
1401
|
+
lines.push('', 'Repetition warnings:');
|
|
1402
|
+
for (const signal of request.repetitionSignals)
|
|
1403
|
+
lines.push(`- ${signal}`);
|
|
1404
|
+
}
|
|
1405
|
+
if (request.partialContent) {
|
|
1406
|
+
lines.push('', 'Partial assistant text from the failed attempt:', request.partialContent);
|
|
1407
|
+
}
|
|
1408
|
+
lines.push('', `Next step: ${nextStep}`);
|
|
1409
|
+
lines.push('Only repeat a completed action if the current page state clearly contradicts this summary.');
|
|
1410
|
+
lines.push('End your final answer by calling task_complete or task_blocked.');
|
|
1411
|
+
return {
|
|
1412
|
+
action: options?.action ?? 'retry',
|
|
1413
|
+
operatorMessage: request.cause === 'stall'
|
|
1414
|
+
? SYSTEM_FALLBACK_STALL_PROMPT
|
|
1415
|
+
: SYSTEM_FALLBACK_COMPLETION_PROMPT,
|
|
1416
|
+
modelMessage: lines.join('\n'),
|
|
1417
|
+
diagnosis,
|
|
1418
|
+
nextStep,
|
|
1419
|
+
completedSteps,
|
|
1420
|
+
};
|
|
1421
|
+
}
|
|
1422
|
+
buildRetryContextMessage(cause, attempt, modelMessage, source) {
|
|
1423
|
+
return {
|
|
1424
|
+
id: `system:${cause}-retry:${attempt}`,
|
|
1425
|
+
role: 'system',
|
|
1426
|
+
content: modelMessage,
|
|
1427
|
+
timestamp: 0,
|
|
1428
|
+
metadata: {
|
|
1429
|
+
transient: true,
|
|
1430
|
+
source: cause === 'stall' ? 'watchdog-resume' : 'completion-resume',
|
|
1431
|
+
recoverySource: source,
|
|
1432
|
+
},
|
|
1433
|
+
};
|
|
1434
|
+
}
|
|
1435
|
+
formatToolOutputPreview(output) {
|
|
1436
|
+
return this.truncateResumeText(output, 180).replace(/\s+/g, ' ').trim();
|
|
1437
|
+
}
|
|
1438
|
+
truncateResumeText(text, maxChars) {
|
|
1439
|
+
const normalized = text.replace(/\r\n/g, '\n').trim();
|
|
1440
|
+
if (normalized.length <= maxChars)
|
|
1441
|
+
return normalized;
|
|
1442
|
+
return `${normalized.slice(0, Math.max(0, maxChars - 1)).trimEnd()}…`;
|
|
1443
|
+
}
|
|
1444
|
+
serializeDebugValue(value, maxChars) {
|
|
1445
|
+
const raw = typeof value === 'string' ? value : (JSON.stringify(value, null, 2) ?? String(value));
|
|
1446
|
+
if (raw.length <= maxChars) {
|
|
1447
|
+
return { value: raw, truncated: false };
|
|
1448
|
+
}
|
|
1449
|
+
return {
|
|
1450
|
+
value: `${raw.slice(0, Math.max(0, maxChars - 1)).trimEnd()}…`,
|
|
1451
|
+
truncated: true,
|
|
1452
|
+
};
|
|
1453
|
+
}
|
|
1454
|
+
buildToolDebugPayload(toolCall, result, toolName) {
|
|
1455
|
+
const argsPreview = this.serializeDebugValue(toolCall.args, DEBUG_TOOL_STRUCTURED_MAX_CHARS);
|
|
1456
|
+
if (!result) {
|
|
1457
|
+
return {
|
|
1458
|
+
requestedToolName: toolCall.name,
|
|
1459
|
+
toolName: toolName ?? (toolCall.name.trim() || toolCall.name),
|
|
1460
|
+
toolCallId: toolCall.id,
|
|
1461
|
+
toolArgs: argsPreview.value,
|
|
1462
|
+
debugPayloadTruncated: argsPreview.truncated,
|
|
1463
|
+
};
|
|
1464
|
+
}
|
|
1465
|
+
const outputPreview = this.serializeDebugValue(result.output, DEBUG_TOOL_OUTPUT_MAX_CHARS);
|
|
1466
|
+
const detailsPreview = result.details
|
|
1467
|
+
? this.serializeDebugValue(result.details, DEBUG_TOOL_STRUCTURED_MAX_CHARS)
|
|
1468
|
+
: null;
|
|
1469
|
+
const resumeMetadata = result.metadata && typeof result.metadata === 'object'
|
|
1470
|
+
? (result.metadata.resume ?? null)
|
|
1471
|
+
: null;
|
|
1472
|
+
const resumePreview = resumeMetadata
|
|
1473
|
+
? this.serializeDebugValue(resumeMetadata, DEBUG_TOOL_STRUCTURED_MAX_CHARS)
|
|
1474
|
+
: null;
|
|
1475
|
+
return {
|
|
1476
|
+
requestedToolName: toolCall.name,
|
|
1477
|
+
toolName: toolName ?? (toolCall.name.trim() || toolCall.name),
|
|
1478
|
+
toolCallId: toolCall.id,
|
|
1479
|
+
toolArgs: argsPreview.value,
|
|
1480
|
+
toolOutput: outputPreview.value,
|
|
1481
|
+
toolDetails: detailsPreview?.value ?? null,
|
|
1482
|
+
toolResume: resumePreview?.value ?? null,
|
|
1483
|
+
isError: result.isError,
|
|
1484
|
+
warnings: result.warnings ?? [],
|
|
1485
|
+
truncated: result.truncated ?? false,
|
|
1486
|
+
debugPayloadTruncated: argsPreview.truncated ||
|
|
1487
|
+
outputPreview.truncated ||
|
|
1488
|
+
Boolean(detailsPreview?.truncated) ||
|
|
1489
|
+
Boolean(resumePreview?.truncated),
|
|
1490
|
+
};
|
|
1491
|
+
}
|
|
1492
|
+
recordCheckpoint(continuity, step, status, evidence) {
|
|
1493
|
+
const checkpoint = {
|
|
1494
|
+
step,
|
|
1495
|
+
status,
|
|
1496
|
+
evidence,
|
|
1497
|
+
summary: `${step} (${status}): ${this.truncateResumeText(evidence, 220)}`,
|
|
1498
|
+
};
|
|
1499
|
+
const existingIndex = continuity.taskCheckpoints.findIndex((entry) => entry.step === step);
|
|
1500
|
+
if (existingIndex >= 0) {
|
|
1501
|
+
continuity.taskCheckpoints[existingIndex] = checkpoint;
|
|
1502
|
+
}
|
|
1503
|
+
else {
|
|
1504
|
+
continuity.taskCheckpoints.push(checkpoint);
|
|
1505
|
+
}
|
|
1506
|
+
this.recordProgressEntry(continuity, {
|
|
1507
|
+
source: 'checkpoint',
|
|
1508
|
+
action: 'task_checkpoint',
|
|
1509
|
+
summary: checkpoint.summary,
|
|
1510
|
+
stateChanging: status === 'done',
|
|
1511
|
+
isError: false,
|
|
1512
|
+
checkpointStatus: status,
|
|
1513
|
+
});
|
|
1514
|
+
this.recordBoundary(continuity, {
|
|
1515
|
+
kind: 'checkpoint',
|
|
1516
|
+
action: 'task_checkpoint',
|
|
1517
|
+
summary: checkpoint.summary,
|
|
1518
|
+
stateChanging: status === 'done',
|
|
1519
|
+
evidence,
|
|
1520
|
+
checkpointStatus: status,
|
|
1521
|
+
});
|
|
1522
|
+
this.appendTurnJournalEntry('checkpoint', checkpoint.summary, {
|
|
1523
|
+
step,
|
|
1524
|
+
status,
|
|
1525
|
+
evidence,
|
|
1526
|
+
});
|
|
1527
|
+
return checkpoint;
|
|
1528
|
+
}
|
|
1529
|
+
recordAttemptToolProgress(completionState, toolName, result) {
|
|
1530
|
+
const continuity = completionState.continuity;
|
|
1531
|
+
const entry = this.createProgressEntry(toolName, result);
|
|
1532
|
+
if (!entry)
|
|
1533
|
+
return;
|
|
1534
|
+
this.recordProgressEntry(continuity, entry);
|
|
1535
|
+
this.updateBrowserState(continuity.browserState, result);
|
|
1536
|
+
if (entry.stateChanging && !entry.isError) {
|
|
1537
|
+
this.recordBoundary(continuity, {
|
|
1538
|
+
kind: 'tool',
|
|
1539
|
+
action: entry.action,
|
|
1540
|
+
summary: entry.summary,
|
|
1541
|
+
stateChanging: true,
|
|
1542
|
+
url: entry.url,
|
|
1543
|
+
tabId: entry.tabId,
|
|
1544
|
+
});
|
|
1545
|
+
}
|
|
1546
|
+
}
|
|
1547
|
+
createProgressEntry(toolName, result) {
|
|
1548
|
+
const resume = this.getBrowserResumeMetadata(result);
|
|
1549
|
+
if (resume?.summary) {
|
|
1550
|
+
return {
|
|
1551
|
+
source: 'tool',
|
|
1552
|
+
toolName,
|
|
1553
|
+
action: resume.action ?? toolName,
|
|
1554
|
+
summary: this.truncateResumeText(resume.summary, 220),
|
|
1555
|
+
url: resume.url,
|
|
1556
|
+
tabId: resume.tabId ?? resume.activeTabId,
|
|
1557
|
+
stateChanging: resume.stateChanging ?? this.isLikelyStateChangingTool(toolName),
|
|
1558
|
+
isError: result.isError,
|
|
1559
|
+
};
|
|
1560
|
+
}
|
|
1561
|
+
const outputPreview = this.formatToolOutputPreview(result.output);
|
|
1562
|
+
if (!outputPreview)
|
|
1563
|
+
return null;
|
|
1564
|
+
return {
|
|
1565
|
+
source: 'tool',
|
|
1566
|
+
toolName,
|
|
1567
|
+
action: toolName,
|
|
1568
|
+
summary: `${toolName}: ${outputPreview}`,
|
|
1569
|
+
stateChanging: this.isLikelyStateChangingTool(toolName),
|
|
1570
|
+
isError: result.isError,
|
|
1571
|
+
};
|
|
1572
|
+
}
|
|
1573
|
+
recordProgressEntry(continuity, entry) {
|
|
1574
|
+
const last = continuity.progressLedger.at(-1);
|
|
1575
|
+
if (last &&
|
|
1576
|
+
entry.source === 'tool' &&
|
|
1577
|
+
last.source === 'tool' &&
|
|
1578
|
+
!entry.stateChanging &&
|
|
1579
|
+
!last.stateChanging &&
|
|
1580
|
+
last.action === entry.action &&
|
|
1581
|
+
last.summary === entry.summary &&
|
|
1582
|
+
last.url === entry.url &&
|
|
1583
|
+
last.tabId === entry.tabId) {
|
|
1584
|
+
return;
|
|
1585
|
+
}
|
|
1586
|
+
continuity.progressLedger.push(entry);
|
|
1587
|
+
while (continuity.progressLedger.length > 50) {
|
|
1588
|
+
const removableIndex = continuity.progressLedger.findIndex((candidate) => candidate.source === 'tool' && !candidate.stateChanging);
|
|
1589
|
+
continuity.progressLedger.splice(removableIndex >= 0 ? removableIndex : 0, 1);
|
|
1590
|
+
}
|
|
1591
|
+
}
|
|
1592
|
+
getBrowserResumeMetadata(result) {
|
|
1593
|
+
const metadata = result.metadata;
|
|
1594
|
+
if (!metadata || typeof metadata !== 'object')
|
|
1595
|
+
return null;
|
|
1596
|
+
const resume = metadata.resume;
|
|
1597
|
+
if (!resume || typeof resume !== 'object')
|
|
1598
|
+
return null;
|
|
1599
|
+
return resume;
|
|
1600
|
+
}
|
|
1601
|
+
updateBrowserState(browserState, result) {
|
|
1602
|
+
const resume = this.getBrowserResumeMetadata(result);
|
|
1603
|
+
if (!resume)
|
|
1604
|
+
return;
|
|
1605
|
+
const stateDelta = resume.stateDelta && typeof resume.stateDelta === 'object'
|
|
1606
|
+
? resume.stateDelta
|
|
1607
|
+
: null;
|
|
1608
|
+
const deltaUrl = typeof stateDelta?.currentUrl === 'string' ? stateDelta.currentUrl : undefined;
|
|
1609
|
+
const deltaActiveTabId = typeof stateDelta?.activeTabId === 'string' ? stateDelta.activeTabId : undefined;
|
|
1610
|
+
const deltaTabs = Array.isArray(stateDelta?.tabs)
|
|
1611
|
+
? stateDelta.tabs
|
|
1612
|
+
: undefined;
|
|
1613
|
+
if (resume.url ?? deltaUrl) {
|
|
1614
|
+
browserState.currentUrl = resume.url ?? deltaUrl;
|
|
1615
|
+
}
|
|
1616
|
+
if (resume.activeTabId ?? deltaActiveTabId) {
|
|
1617
|
+
browserState.activeTabId = resume.activeTabId ?? deltaActiveTabId;
|
|
1618
|
+
}
|
|
1619
|
+
else if (resume.tabId && resume.stateChanging) {
|
|
1620
|
+
browserState.activeTabId = resume.tabId;
|
|
1621
|
+
}
|
|
1622
|
+
const nextTabs = resume.tabs?.length ? resume.tabs : deltaTabs;
|
|
1623
|
+
if (nextTabs?.length) {
|
|
1624
|
+
browserState.tabs = nextTabs.map((tab) => ({ ...tab }));
|
|
1625
|
+
const active = nextTabs.find((tab) => tab.active);
|
|
1626
|
+
if (active) {
|
|
1627
|
+
browserState.activeTabId = active.id;
|
|
1628
|
+
browserState.currentUrl = active.url;
|
|
1629
|
+
}
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
cloneBrowserState(browserState) {
|
|
1633
|
+
return {
|
|
1634
|
+
currentUrl: browserState.currentUrl,
|
|
1635
|
+
activeTabId: browserState.activeTabId,
|
|
1636
|
+
tabs: browserState.tabs?.map((tab) => ({ ...tab })),
|
|
1637
|
+
};
|
|
1638
|
+
}
|
|
1639
|
+
isLikelyStateChangingTool(toolName) {
|
|
1640
|
+
return !READ_ONLY_TOOL_NAMES.has(toolName);
|
|
1641
|
+
}
|
|
1642
|
+
evaluateCompletionGuard(displayContent, finishMeta, completionState) {
|
|
1643
|
+
const trimmedContent = displayContent.trim();
|
|
1644
|
+
const hadExternalToolActivity = completionState.externalToolCallCount > 0;
|
|
1645
|
+
const endedOnToolCalls = finishMeta?.finishReason === 'tool-calls' ||
|
|
1646
|
+
finishMeta?.stepFinishReasons.at(-1) === 'tool-calls';
|
|
1647
|
+
if (trimmedContent.length === 0 && hadExternalToolActivity) {
|
|
1648
|
+
return {
|
|
1649
|
+
message: 'Turn ended after tool activity without a final answer.',
|
|
1650
|
+
signal: completionState.signal,
|
|
1651
|
+
};
|
|
1652
|
+
}
|
|
1653
|
+
if (endedOnToolCalls) {
|
|
1654
|
+
return {
|
|
1655
|
+
message: 'Turn ended on tool-calls without a final answer.',
|
|
1656
|
+
signal: completionState.signal,
|
|
1657
|
+
};
|
|
1658
|
+
}
|
|
1659
|
+
if (hadExternalToolActivity && completionState.signal === 'none') {
|
|
1660
|
+
return {
|
|
1661
|
+
message: 'Tool-driven turn ended without task_complete or task_blocked.',
|
|
1662
|
+
signal: completionState.signal,
|
|
1663
|
+
};
|
|
1664
|
+
}
|
|
1665
|
+
return null;
|
|
1666
|
+
}
|
|
1667
|
+
async awaitStreamAttempt(streamPromise, abortController) {
|
|
1668
|
+
return new Promise((resolve, reject) => {
|
|
1669
|
+
let settled = false;
|
|
1670
|
+
let abortTimer = null;
|
|
1671
|
+
const cleanup = () => {
|
|
1672
|
+
abortController.signal.removeEventListener('abort', onAbort);
|
|
1673
|
+
if (abortTimer) {
|
|
1674
|
+
clearTimeout(abortTimer);
|
|
1675
|
+
abortTimer = null;
|
|
1676
|
+
}
|
|
1677
|
+
};
|
|
1678
|
+
const settleResolve = () => {
|
|
1679
|
+
if (settled)
|
|
1680
|
+
return;
|
|
1681
|
+
settled = true;
|
|
1682
|
+
cleanup();
|
|
1683
|
+
resolve();
|
|
1684
|
+
};
|
|
1685
|
+
const settleReject = (error) => {
|
|
1686
|
+
if (settled)
|
|
1687
|
+
return;
|
|
1688
|
+
settled = true;
|
|
1689
|
+
cleanup();
|
|
1690
|
+
reject(error);
|
|
1691
|
+
};
|
|
1692
|
+
const onAbort = () => {
|
|
1693
|
+
this.logStreamDebug('provider_abort_observed', {
|
|
1694
|
+
phase: this.streamPhase,
|
|
1695
|
+
activeToolName: this.activeToolCall?.name,
|
|
1696
|
+
activeToolCallId: this.activeToolCall?.id,
|
|
1697
|
+
});
|
|
1698
|
+
if (abortTimer)
|
|
1699
|
+
return;
|
|
1700
|
+
abortTimer = setTimeout(() => {
|
|
1701
|
+
if (settled)
|
|
1702
|
+
return;
|
|
1703
|
+
const elapsedSeconds = Math.max(1, Math.round((Date.now() - this.lastStreamActivityAt) / 1000));
|
|
1704
|
+
this.emitWatchdog('teardown_timeout', `Provider did not settle within ${this.streamAbortGraceMs}ms after abort; continuing retry.`, { level: 'warn', elapsedSeconds });
|
|
1705
|
+
settleReject(createAbortError('Stream aborted'));
|
|
1706
|
+
}, this.streamAbortGraceMs);
|
|
1707
|
+
};
|
|
1708
|
+
abortController.signal.addEventListener('abort', onAbort, { once: true });
|
|
1709
|
+
if (abortController.signal.aborted) {
|
|
1710
|
+
onAbort();
|
|
1711
|
+
}
|
|
1712
|
+
streamPromise.then(settleResolve, settleReject);
|
|
1713
|
+
});
|
|
1714
|
+
}
|
|
1715
|
+
startStreamWatchdog(latestUserMessage) {
|
|
1716
|
+
this.stopStreamWatchdog();
|
|
1717
|
+
this.markStreamWaitingModel();
|
|
1718
|
+
this.streamDeferredUntil = 0;
|
|
1719
|
+
this.streamWatchdog = setInterval(() => {
|
|
1720
|
+
if (!this.currentAttemptCompletionState || !this.currentStreamTurn)
|
|
1721
|
+
return;
|
|
1722
|
+
if (this.streamDeferredUntil > Date.now())
|
|
1723
|
+
return;
|
|
1724
|
+
const elapsed = Date.now() - this.lastStreamActivityAt;
|
|
1725
|
+
const stallThresholdMs = this.getCurrentStallThresholdMs();
|
|
1726
|
+
if (elapsed < stallThresholdMs)
|
|
1727
|
+
return;
|
|
1728
|
+
if (this.streamNudgeCount >= this.maxNudgeRetries)
|
|
1729
|
+
return;
|
|
1730
|
+
if (this._nudgeInFlight)
|
|
1731
|
+
return;
|
|
1732
|
+
const elapsedSeconds = Math.round(elapsed / 1000);
|
|
1733
|
+
const diagnostics = this.getStreamDiagnostics(elapsedSeconds);
|
|
1734
|
+
this.emitWatchdog('stalled', `Stalled after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'warn', elapsedSeconds });
|
|
1735
|
+
this.emit({ type: 'cerebrum:stall', ...diagnostics });
|
|
1736
|
+
if (!this.cerebellum?.isConnected()) {
|
|
1737
|
+
this.emitWatchdog('abort_issued', 'Cerebellum disconnected during an active stream; aborting the turn.', { level: 'warn', elapsedSeconds });
|
|
1738
|
+
this.abortController?.abort();
|
|
1739
|
+
return;
|
|
1740
|
+
}
|
|
1741
|
+
this._nudgeInFlight = true;
|
|
1742
|
+
void (async () => {
|
|
1743
|
+
try {
|
|
1744
|
+
const request = this.buildRecoveryRequest({
|
|
1745
|
+
cause: 'stall',
|
|
1746
|
+
attempt: this.currentStreamTurn.attempt,
|
|
1747
|
+
partialContent: this.currentPartialContent,
|
|
1748
|
+
completionState: this.currentAttemptCompletionState,
|
|
1749
|
+
turnOutcome: 'stalled',
|
|
1750
|
+
latestUserMessage,
|
|
1751
|
+
elapsedSeconds,
|
|
1752
|
+
});
|
|
1753
|
+
const { source, assessment } = await this.assessTurnRecovery(request);
|
|
1754
|
+
this.emitRecoveryTrace('stall', source, assessment, assessment.action === 'stop' ? 'warn' : 'info');
|
|
1755
|
+
this.appendTurnJournalEntry('recovery', assessment.operatorMessage, {
|
|
1756
|
+
cause: 'stall',
|
|
1757
|
+
source,
|
|
1758
|
+
action: assessment.action,
|
|
1759
|
+
diagnosis: assessment.diagnosis,
|
|
1760
|
+
nextStep: assessment.nextStep,
|
|
1761
|
+
waitSeconds: assessment.waitSeconds,
|
|
1762
|
+
completedSteps: assessment.completedSteps,
|
|
1763
|
+
});
|
|
1764
|
+
if (assessment.action === 'wait') {
|
|
1765
|
+
const waitSeconds = Math.max(15, assessment.waitSeconds ?? this.streamStallThreshold / 1000);
|
|
1766
|
+
this.streamDeferredUntil = Date.now() + waitSeconds * 1000;
|
|
1767
|
+
return;
|
|
1768
|
+
}
|
|
1769
|
+
if (assessment.action === 'retry') {
|
|
1770
|
+
this.streamNudgeCount++;
|
|
1771
|
+
this.pendingRecoveryDecision = { cause: 'stall', source, assessment };
|
|
1772
|
+
this.recordBoundary(this.currentAttemptCompletionState.continuity, {
|
|
1773
|
+
kind: 'recovery',
|
|
1774
|
+
action: 'stall_retry',
|
|
1775
|
+
summary: assessment.diagnosis,
|
|
1776
|
+
stateChanging: false,
|
|
1777
|
+
});
|
|
1778
|
+
this.emitWatchdog('nudge_requested', `Cerebellum requested nudge ${this.streamNudgeCount}/${this.maxNudgeRetries} after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'info', elapsedSeconds });
|
|
1779
|
+
this.emit({
|
|
1780
|
+
type: 'cerebrum:stall:nudge',
|
|
1781
|
+
attempt: this.streamNudgeCount,
|
|
1782
|
+
...diagnostics,
|
|
1783
|
+
});
|
|
1784
|
+
this.emitWatchdog('abort_issued', `Aborting stalled stream attempt ${this.currentStreamTurn?.attempt ?? 0}.`, { level: 'warn', elapsedSeconds });
|
|
1785
|
+
this.abortController?.abort();
|
|
1786
|
+
return;
|
|
1787
|
+
}
|
|
1788
|
+
this.pendingRecoveryDecision = { cause: 'stall', source, assessment };
|
|
1789
|
+
this.recordBoundary(this.currentAttemptCompletionState.continuity, {
|
|
1790
|
+
kind: 'recovery',
|
|
1791
|
+
action: 'stall_stop',
|
|
1792
|
+
summary: assessment.diagnosis,
|
|
1793
|
+
stateChanging: false,
|
|
1794
|
+
});
|
|
1795
|
+
this.emitWatchdog('abort_issued', 'Aborting stalled stream because recovery guidance requested stop.', { level: 'warn', elapsedSeconds });
|
|
1796
|
+
this.abortController?.abort();
|
|
1797
|
+
}
|
|
1798
|
+
catch {
|
|
1799
|
+
const request = this.buildRecoveryRequest({
|
|
1800
|
+
cause: 'stall',
|
|
1801
|
+
attempt: this.currentStreamTurn.attempt,
|
|
1802
|
+
partialContent: this.currentPartialContent,
|
|
1803
|
+
completionState: this.currentAttemptCompletionState,
|
|
1804
|
+
turnOutcome: 'stalled',
|
|
1805
|
+
latestUserMessage,
|
|
1806
|
+
elapsedSeconds,
|
|
1807
|
+
});
|
|
1808
|
+
const assessment = this.buildFallbackRecoveryAssessment(request, {
|
|
1809
|
+
reason: `Recovery assessment failed after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`,
|
|
1810
|
+
});
|
|
1811
|
+
this.pendingRecoveryDecision = { cause: 'stall', source: 'fallback', assessment };
|
|
1812
|
+
this.emitRecoveryTrace('stall', 'fallback', assessment, 'warn');
|
|
1813
|
+
this.appendTurnJournalEntry('recovery', assessment.operatorMessage, {
|
|
1814
|
+
cause: 'stall',
|
|
1815
|
+
source: 'fallback',
|
|
1816
|
+
action: assessment.action,
|
|
1817
|
+
diagnosis: assessment.diagnosis,
|
|
1818
|
+
nextStep: assessment.nextStep,
|
|
1819
|
+
waitSeconds: assessment.waitSeconds,
|
|
1820
|
+
completedSteps: assessment.completedSteps,
|
|
1821
|
+
});
|
|
1822
|
+
this.streamNudgeCount++;
|
|
1823
|
+
this.recordBoundary(this.currentAttemptCompletionState.continuity, {
|
|
1824
|
+
kind: 'recovery',
|
|
1825
|
+
action: 'stall_retry',
|
|
1826
|
+
summary: assessment.diagnosis,
|
|
1827
|
+
stateChanging: false,
|
|
1828
|
+
});
|
|
1829
|
+
this.emitWatchdog('nudge_requested', `Fallback retry ${this.streamNudgeCount}/${this.maxNudgeRetries} after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'info', elapsedSeconds });
|
|
1830
|
+
this.emit({
|
|
1831
|
+
type: 'cerebrum:stall:nudge',
|
|
1832
|
+
attempt: this.streamNudgeCount,
|
|
1833
|
+
...diagnostics,
|
|
1834
|
+
});
|
|
1835
|
+
this.emitWatchdog('abort_issued', `Aborting stalled stream attempt ${this.currentStreamTurn?.attempt ?? 0}.`, { level: 'warn', elapsedSeconds });
|
|
1836
|
+
this.abortController?.abort();
|
|
1837
|
+
}
|
|
1838
|
+
finally {
|
|
1839
|
+
this._nudgeInFlight = false;
|
|
1840
|
+
}
|
|
1841
|
+
})();
|
|
1842
|
+
}, 15_000);
|
|
1843
|
+
}
|
|
1844
|
+
_nudgeInFlight = false;
|
|
1845
|
+
stopStreamWatchdog() {
|
|
1846
|
+
if (this.streamWatchdog) {
|
|
1847
|
+
clearInterval(this.streamWatchdog);
|
|
1848
|
+
this.streamWatchdog = null;
|
|
1849
|
+
}
|
|
1850
|
+
this.resetStreamState();
|
|
1851
|
+
}
|
|
1852
|
+
async sendMessage(content, conversationId, options) {
|
|
1853
|
+
if (!this.cerebrum)
|
|
1854
|
+
throw new Error('Cerebrum not connected');
|
|
1855
|
+
if (this.cerebellum && !this.cerebellum.isConnected()) {
|
|
1856
|
+
throw new Error('Cerebellum is offline. Fix the Cerebellum connection before continuing. Run: docker compose up -d cerebellum');
|
|
1857
|
+
}
|
|
1858
|
+
const convId = conversationId ?? this.activeConversationId;
|
|
1859
|
+
if (!convId)
|
|
1860
|
+
throw new Error('No active conversation');
|
|
1861
|
+
if (content) {
|
|
1862
|
+
const userMessage = this.conversations.appendMessage(convId, 'user', content);
|
|
1863
|
+
this.emit({ type: 'message:user', message: userMessage });
|
|
1864
|
+
}
|
|
1865
|
+
const latestUserMessage = content ||
|
|
1866
|
+
[...this.conversations.getMessages(convId)]
|
|
1867
|
+
.reverse()
|
|
1868
|
+
.find((message) => message.role === 'user')?.content ||
|
|
1869
|
+
'';
|
|
1870
|
+
this.streamNudgeCount = 0;
|
|
1871
|
+
let completionRetryCount = 0;
|
|
1872
|
+
let nextRetryContext = null;
|
|
1873
|
+
// Track message IDs from failed attempts so they can be excluded from retries and cleaned up.
|
|
1874
|
+
const failedAttemptMessageIds = [];
|
|
1875
|
+
const turnId = options?.turnId ?? nanoid(10);
|
|
1876
|
+
const maxTotalAttempts = 1 + this.maxNudgeRetries + this.maxCompletionRetries;
|
|
1877
|
+
let loopTerminated = false;
|
|
1878
|
+
let nextRetryCause = null;
|
|
1879
|
+
const turnContinuity = this.createTurnContinuityState();
|
|
1880
|
+
try {
|
|
1881
|
+
for (let attempt = 0; attempt < maxTotalAttempts; attempt++) {
|
|
1882
|
+
const abortController = new AbortController();
|
|
1883
|
+
const attemptNumber = attempt + 1;
|
|
1884
|
+
const retryCause = nextRetryCause;
|
|
1885
|
+
nextRetryCause = null;
|
|
1886
|
+
const completionState = this.createAttemptCompletionState(turnContinuity);
|
|
1887
|
+
let completionGuardFailure = null;
|
|
1888
|
+
const stallRetryCountAtStart = this.streamNudgeCount;
|
|
1889
|
+
const attemptMessageIds = [];
|
|
1890
|
+
const isCurrentAttempt = () => this.abortController === abortController;
|
|
1891
|
+
this.abortController = abortController;
|
|
1892
|
+
this.currentAttemptCompletionState = completionState;
|
|
1893
|
+
const sessionSource = options?.source ?? 'local';
|
|
1894
|
+
this.currentStreamTurn = {
|
|
1895
|
+
turnId,
|
|
1896
|
+
attempt: attemptNumber,
|
|
1897
|
+
conversationId: convId,
|
|
1898
|
+
sessionId: turnId,
|
|
1899
|
+
source: sessionSource,
|
|
1900
|
+
};
|
|
1901
|
+
const priorSession = this.currentQuerySession;
|
|
1902
|
+
this.currentQuerySession = this.createQuerySession(convId, turnId, attemptNumber, sessionSource, latestUserMessage, this.streamNudgeCount, completionRetryCount, priorSession);
|
|
1903
|
+
this.saveCurrentQuerySession();
|
|
1904
|
+
if (attemptNumber === 1 && sessionSource === 'channel' && options?.ingress) {
|
|
1905
|
+
this.recordSessionEvent(convId, turnId, 'channel_ingress', `Received channel message from ${options.ingress.senderName || options.ingress.senderId || 'unknown sender'}.`, {
|
|
1906
|
+
channelId: options.ingress.channelId,
|
|
1907
|
+
routeTo: options.ingress.routeTo,
|
|
1908
|
+
senderId: options.ingress.senderId,
|
|
1909
|
+
senderName: options.ingress.senderName,
|
|
1910
|
+
sessionId: options.ingress.sessionId,
|
|
1911
|
+
threadId: options.ingress.threadId,
|
|
1912
|
+
replyToId: options.ingress.replyToId,
|
|
1913
|
+
timestamp: options.ingress.timestamp,
|
|
1914
|
+
});
|
|
1915
|
+
}
|
|
1916
|
+
this.currentPartialContent = '';
|
|
1917
|
+
this.currentLastContentKind = 'empty';
|
|
1918
|
+
this.currentJournaledContentLength = 0;
|
|
1919
|
+
this.pendingRecoveryDecision = null;
|
|
1920
|
+
log.info('stream_started', {
|
|
1921
|
+
turnId,
|
|
1922
|
+
attempt: attemptNumber,
|
|
1923
|
+
conversationId: convId,
|
|
1924
|
+
stallRetryCount: this.streamNudgeCount,
|
|
1925
|
+
completionRetryCount,
|
|
1926
|
+
retryCause,
|
|
1927
|
+
});
|
|
1928
|
+
this.appendTurnJournalEntry('turn_started', `Turn attempt ${attemptNumber} started.`, {
|
|
1929
|
+
retryCause: retryCause ?? null,
|
|
1930
|
+
latestUserMessage: this.truncateResumeText(latestUserMessage, 600),
|
|
1931
|
+
stallRetryCount: this.streamNudgeCount,
|
|
1932
|
+
completionRetryCount,
|
|
1933
|
+
});
|
|
1934
|
+
this.emit({
|
|
1935
|
+
type: 'message:cerebrum:start',
|
|
1936
|
+
conversationId: convId,
|
|
1937
|
+
turnId,
|
|
1938
|
+
sessionId: turnId,
|
|
1939
|
+
source: sessionSource,
|
|
1940
|
+
});
|
|
1941
|
+
this.startStreamWatchdog(latestUserMessage);
|
|
1942
|
+
let messages = this.conversations.getMessages(convId);
|
|
1943
|
+
// On retry: exclude failed attempts' messages from history.
|
|
1944
|
+
// The resume context already summarizes what happened — sending the raw tool calls
|
|
1945
|
+
// causes the model to repeat the exact same steps instead of continuing.
|
|
1946
|
+
if (failedAttemptMessageIds.length > 0) {
|
|
1947
|
+
const excludeSet = new Set(failedAttemptMessageIds);
|
|
1948
|
+
messages = messages.filter((m) => !excludeSet.has(m.id));
|
|
1949
|
+
}
|
|
1950
|
+
// Context window compaction
|
|
1951
|
+
if (this.compactionConfig.enabled &&
|
|
1952
|
+
this.cerebrum?.summarize &&
|
|
1953
|
+
shouldCompact(messages, this.compactionConfig.contextWindow, this.compactionConfig.threshold)) {
|
|
1954
|
+
try {
|
|
1955
|
+
const keepRecent = this.compactionConfig.keepRecentMessages;
|
|
1956
|
+
const olderMessages = messages.slice(0, Math.max(0, messages.length - keepRecent));
|
|
1957
|
+
if (olderMessages.length > 0) {
|
|
1958
|
+
log.info('Compacting conversation', {
|
|
1959
|
+
totalMessages: messages.length,
|
|
1960
|
+
compactingMessages: olderMessages.length,
|
|
1961
|
+
estimatedTokens: estimateMessageTokens(messages),
|
|
1962
|
+
});
|
|
1963
|
+
const summary = await this.cerebrum.summarize(olderMessages);
|
|
1964
|
+
messages = buildCompactionMessages(messages, summary, keepRecent);
|
|
1965
|
+
}
|
|
1966
|
+
}
|
|
1967
|
+
catch (error) {
|
|
1968
|
+
log.warn('Compaction failed, continuing with full context', {
|
|
1969
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1970
|
+
});
|
|
1971
|
+
}
|
|
1972
|
+
}
|
|
1973
|
+
// Build system prompt with runtime state + skills context
|
|
1974
|
+
const instance = this.instanceStore?.get();
|
|
1975
|
+
const allTools = this.getAllTools();
|
|
1976
|
+
const basePrompt = buildSystemPrompt({
|
|
1977
|
+
cerebellumConnected: this.cerebellum?.isConnected() ?? false,
|
|
1978
|
+
tools: allTools,
|
|
1979
|
+
autoMode: this.autoMode,
|
|
1980
|
+
gatewayMode: this.gatewayMode,
|
|
1981
|
+
connectedNodes: this.connectedNodes,
|
|
1982
|
+
gatewayUrl: this.gatewayUrl,
|
|
1983
|
+
profile: this.profile,
|
|
1984
|
+
finetuneStatus: {
|
|
1985
|
+
enabled: !!this.fineTuneDataProvider,
|
|
1986
|
+
status: this.fineTuneStatus.status,
|
|
1987
|
+
progress: this.fineTuneStatus.progress,
|
|
1988
|
+
lastJobId: this.fineTuneStatus.jobId || undefined,
|
|
1989
|
+
},
|
|
1990
|
+
recurringTasks: this.recurringTasks,
|
|
1991
|
+
instanceId: instance?.id,
|
|
1992
|
+
instanceCreatedAt: instance?.createdAt,
|
|
1993
|
+
finetuneCount: instance?.finetuneLineage.length,
|
|
1994
|
+
proactiveEnabled: this.proactiveEnabled,
|
|
1995
|
+
discoveryMode: this.discoveryMode,
|
|
1996
|
+
});
|
|
1997
|
+
const systemParts = [basePrompt];
|
|
1998
|
+
if (this.systemContext)
|
|
1999
|
+
systemParts.push(this.systemContext);
|
|
2000
|
+
const fullSystemPrompt = systemParts.join('\n\n---\n\n');
|
|
2001
|
+
const transientRetryMessages = nextRetryContext ? [nextRetryContext] : [];
|
|
2002
|
+
nextRetryContext = null;
|
|
2003
|
+
const allMessages = [
|
|
2004
|
+
{ id: 'system', role: 'system', content: fullSystemPrompt, timestamp: 0 },
|
|
2005
|
+
...transientRetryMessages,
|
|
2006
|
+
...messages,
|
|
2007
|
+
];
|
|
2008
|
+
const toolDefs = Object.fromEntries(allTools);
|
|
2009
|
+
let fullContent = '';
|
|
2010
|
+
let finalDisplayContent = '';
|
|
2011
|
+
let attemptFinishMeta;
|
|
2012
|
+
const throwIfToolAttemptAborted = () => {
|
|
2013
|
+
if (!isCurrentAttempt()) {
|
|
2014
|
+
throw createAbortError('Tool execution aborted');
|
|
2015
|
+
}
|
|
2016
|
+
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
2017
|
+
};
|
|
2018
|
+
try {
|
|
2019
|
+
const streamPromise = this.cerebrum.stream(allMessages, toolDefs, {
|
|
2020
|
+
onChunk: (chunk) => {
|
|
2021
|
+
if (!isCurrentAttempt() || abortController.signal.aborted)
|
|
2022
|
+
return;
|
|
2023
|
+
fullContent += chunk;
|
|
2024
|
+
this.currentPartialContent = fullContent;
|
|
2025
|
+
this.currentLastContentKind = 'text';
|
|
2026
|
+
this.markStreamWaitingModel();
|
|
2027
|
+
this.persistPartialContentSnapshot();
|
|
2028
|
+
this.emit({ type: 'message:cerebrum:chunk', chunk });
|
|
2029
|
+
},
|
|
2030
|
+
onToolCall: async (toolCall) => {
|
|
2031
|
+
throwIfToolAttemptAborted();
|
|
2032
|
+
this.logStreamDebug('tool_callback_started', this.buildToolDebugPayload(toolCall));
|
|
2033
|
+
this.markStreamWaitingTool(toolCall);
|
|
2034
|
+
this.currentLastContentKind = 'tool-call';
|
|
2035
|
+
const requestedToolName = toolCall.name;
|
|
2036
|
+
const normalizedToolName = requestedToolName.trim() || requestedToolName;
|
|
2037
|
+
this.appendTurnJournalEntry('tool_start', `Calling ${normalizedToolName}`, {
|
|
2038
|
+
requestedToolName,
|
|
2039
|
+
toolName: normalizedToolName,
|
|
2040
|
+
callId: toolCall.id,
|
|
2041
|
+
args: toolCall.args,
|
|
2042
|
+
});
|
|
2043
|
+
const isInternalTaskSignal = this.isInternalTaskSignalTool(normalizedToolName);
|
|
2044
|
+
const toolIngress = isInternalTaskSignal ? options?.ingress : undefined;
|
|
2045
|
+
if (isInternalTaskSignal) {
|
|
2046
|
+
completionState.internalToolCallCount++;
|
|
2047
|
+
}
|
|
2048
|
+
else {
|
|
2049
|
+
completionState.externalToolCallCount++;
|
|
2050
|
+
this.emit({
|
|
2051
|
+
type: 'message:cerebrum:toolcall',
|
|
2052
|
+
toolCall: { ...toolCall, name: normalizedToolName },
|
|
2053
|
+
});
|
|
2054
|
+
this.emit({
|
|
2055
|
+
type: 'tool:start',
|
|
2056
|
+
callId: toolCall.id,
|
|
2057
|
+
name: normalizedToolName,
|
|
2058
|
+
requestedName: requestedToolName !== normalizedToolName ? requestedToolName : undefined,
|
|
2059
|
+
args: toolCall.args,
|
|
2060
|
+
});
|
|
2061
|
+
}
|
|
2062
|
+
const { toolName, result } = await this.toolRuntime.execute({
|
|
2063
|
+
toolCall,
|
|
2064
|
+
tools: allTools,
|
|
2065
|
+
conversationId: convId,
|
|
2066
|
+
sessionKey: turnId,
|
|
2067
|
+
scopeKey: convId,
|
|
2068
|
+
turnId,
|
|
2069
|
+
attempt: attemptNumber,
|
|
2070
|
+
ingress: toolIngress,
|
|
2071
|
+
abortSignal: abortController.signal,
|
|
2072
|
+
});
|
|
2073
|
+
this.logStreamDebug('tool_callback_finished', this.buildToolDebugPayload(toolCall, result, toolName));
|
|
2074
|
+
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
2075
|
+
this.markStreamWaitingModel();
|
|
2076
|
+
this.currentLastContentKind = result.isError ? 'error' : 'tool-call';
|
|
2077
|
+
if (!isInternalTaskSignal) {
|
|
2078
|
+
this.emit({
|
|
2079
|
+
type: 'tool:end',
|
|
2080
|
+
callId: toolCall.id,
|
|
2081
|
+
name: toolName,
|
|
2082
|
+
requestedName: requestedToolName !== toolName ? requestedToolName : undefined,
|
|
2083
|
+
args: toolCall.args,
|
|
2084
|
+
result,
|
|
2085
|
+
});
|
|
2086
|
+
}
|
|
2087
|
+
if (!isInternalTaskSignal && !result.isError) {
|
|
2088
|
+
completionState.successfulExternalToolCount++;
|
|
2089
|
+
}
|
|
2090
|
+
// Cerebellum verification (non-blocking)
|
|
2091
|
+
if (!isInternalTaskSignal &&
|
|
2092
|
+
this.cerebellum?.isConnected() &&
|
|
2093
|
+
this.verificationEnabled) {
|
|
2094
|
+
try {
|
|
2095
|
+
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
2096
|
+
this.emit({ type: 'verification:start', callId: toolCall.id, toolName });
|
|
2097
|
+
const toolArgs = {};
|
|
2098
|
+
for (const [k, v] of Object.entries(toolCall.args)) {
|
|
2099
|
+
toolArgs[k] = String(v);
|
|
2100
|
+
}
|
|
2101
|
+
const verifyPromise = this.cerebellum.verifyToolResult(toolName, toolArgs, result.output, !result.isError);
|
|
2102
|
+
const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve(null), this.verificationTimeoutMs));
|
|
2103
|
+
const verification = await Promise.race([verifyPromise, timeoutPromise]);
|
|
2104
|
+
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
2105
|
+
if (verification && !verification.passed) {
|
|
2106
|
+
const failedChecks = verification.checks
|
|
2107
|
+
.filter((c) => !c.passed)
|
|
2108
|
+
.map((c) => c.description)
|
|
2109
|
+
.join(', ');
|
|
2110
|
+
result.output += `\n[Cerebellum warning: ${failedChecks}]`;
|
|
2111
|
+
}
|
|
2112
|
+
if (verification) {
|
|
2113
|
+
result.metadata = {
|
|
2114
|
+
...(result.metadata ?? {}),
|
|
2115
|
+
verification: {
|
|
2116
|
+
passed: verification.passed,
|
|
2117
|
+
modelVerdict: verification.modelVerdict,
|
|
2118
|
+
failedChecks: verification.checks
|
|
2119
|
+
.filter((check) => !check.passed)
|
|
2120
|
+
.map((check) => check.description),
|
|
2121
|
+
},
|
|
2122
|
+
};
|
|
2123
|
+
const vResult = {
|
|
2124
|
+
passed: verification.passed,
|
|
2125
|
+
checks: verification.checks,
|
|
2126
|
+
modelVerdict: verification.modelVerdict,
|
|
2127
|
+
toolCallId: toolCall.id,
|
|
2128
|
+
toolName,
|
|
2129
|
+
};
|
|
2130
|
+
this.emit({
|
|
2131
|
+
type: 'verification:end',
|
|
2132
|
+
result: vResult,
|
|
2133
|
+
conversationId: convId,
|
|
2134
|
+
sessionId: turnId,
|
|
2135
|
+
});
|
|
2136
|
+
}
|
|
2137
|
+
}
|
|
2138
|
+
catch {
|
|
2139
|
+
// Verification failure should never block tool execution
|
|
2140
|
+
}
|
|
2141
|
+
}
|
|
2142
|
+
throwIfToolAttemptAborted();
|
|
2143
|
+
const toolJournalEntry = this.createProgressEntry(toolName, result);
|
|
2144
|
+
this.appendTurnJournalEntry('tool_end', toolJournalEntry?.summary ??
|
|
2145
|
+
`${toolName}: ${this.formatToolOutputPreview(result.output)}`, {
|
|
2146
|
+
requestedToolName,
|
|
2147
|
+
toolName,
|
|
2148
|
+
callId: toolCall.id,
|
|
2149
|
+
isError: result.isError,
|
|
2150
|
+
args: toolCall.args,
|
|
2151
|
+
output: this.truncateResumeText(result.output, 2_000),
|
|
2152
|
+
details: result.details,
|
|
2153
|
+
metadata: result.metadata,
|
|
2154
|
+
});
|
|
2155
|
+
if (!isInternalTaskSignal) {
|
|
2156
|
+
this.recordAttemptToolProgress(completionState, toolName, result);
|
|
2157
|
+
const toolMsg = this.conversations.appendMessage(convId, 'tool', result.output, {
|
|
2158
|
+
toolResult: result,
|
|
2159
|
+
metadata: {
|
|
2160
|
+
toolName,
|
|
2161
|
+
...(requestedToolName !== toolName ? { requestedToolName } : {}),
|
|
2162
|
+
},
|
|
2163
|
+
});
|
|
2164
|
+
attemptMessageIds.push(toolMsg.id);
|
|
2165
|
+
}
|
|
2166
|
+
return result;
|
|
2167
|
+
},
|
|
2168
|
+
onFinish: (content, toolCalls, finishMeta) => {
|
|
2169
|
+
if (!isCurrentAttempt() || abortController.signal.aborted)
|
|
2170
|
+
return;
|
|
2171
|
+
this.stopStreamWatchdog();
|
|
2172
|
+
let displayContent = content;
|
|
2173
|
+
finalDisplayContent = content;
|
|
2174
|
+
attemptFinishMeta = finishMeta;
|
|
2175
|
+
this.currentLastContentKind =
|
|
2176
|
+
finishMeta?.lastContentKind ??
|
|
2177
|
+
(content.trim() ? 'text' : this.currentLastContentKind);
|
|
2178
|
+
this.persistPartialContentSnapshot(true);
|
|
2179
|
+
const visibleToolCalls = toolCalls?.filter((toolCall) => !this.isInternalTaskSignalTool(toolCall.name));
|
|
2180
|
+
const turnOutcome = this.classifyTurnOutcome(displayContent, finishMeta, completionState);
|
|
2181
|
+
log.info('stream_finish_observed', {
|
|
2182
|
+
turnId,
|
|
2183
|
+
attempt: attemptNumber,
|
|
2184
|
+
conversationId: convId,
|
|
2185
|
+
finishReason: finishMeta?.finishReason,
|
|
2186
|
+
rawFinishReason: finishMeta?.rawFinishReason,
|
|
2187
|
+
lastContentKind: finishMeta?.lastContentKind,
|
|
2188
|
+
stepCount: finishMeta?.stepCount ?? 0,
|
|
2189
|
+
chunkCount: finishMeta?.chunkCount ?? 0,
|
|
2190
|
+
toolCallCount: finishMeta?.toolCallCount ?? 0,
|
|
2191
|
+
textChars: finishMeta?.textChars ?? content.length,
|
|
2192
|
+
completionSignal: completionState.signal,
|
|
2193
|
+
turnOutcome,
|
|
2194
|
+
});
|
|
2195
|
+
// Check for discovery completion — parse and strip the tag before storing
|
|
2196
|
+
if (this.discoveryMode && content.includes('<discovery_complete>')) {
|
|
2197
|
+
const parsed = this.parseDiscoveryCompletion(content);
|
|
2198
|
+
// Strip the tag block from the displayed/stored content
|
|
2199
|
+
displayContent = content
|
|
2200
|
+
.replace(/<discovery_complete>[\s\S]*?<\/discovery_complete>/g, '')
|
|
2201
|
+
.trim();
|
|
2202
|
+
finalDisplayContent = displayContent;
|
|
2203
|
+
if (parsed && this.onDiscoveryComplete) {
|
|
2204
|
+
this.discoveryMode = false;
|
|
2205
|
+
this.onDiscoveryComplete(parsed);
|
|
2206
|
+
log.info('Discovery completed', { name: parsed.name });
|
|
2207
|
+
}
|
|
2208
|
+
}
|
|
2209
|
+
this.appendTurnJournalEntry('turn_finished', `Turn ended with ${turnOutcome}.`, {
|
|
2210
|
+
finishReason: finishMeta?.finishReason,
|
|
2211
|
+
rawFinishReason: finishMeta?.rawFinishReason,
|
|
2212
|
+
lastContentKind: finishMeta?.lastContentKind ?? this.currentLastContentKind,
|
|
2213
|
+
turnOutcome,
|
|
2214
|
+
textChars: finishMeta?.textChars ?? displayContent.length,
|
|
2215
|
+
toolCallCount: finishMeta?.toolCallCount ?? 0,
|
|
2216
|
+
completionSignal: completionState.signal,
|
|
2217
|
+
finalContent: this.truncateResumeText(displayContent, 2_000),
|
|
2218
|
+
});
|
|
2219
|
+
this.pruneTurnJournals(convId);
|
|
2220
|
+
const guardFailure = this.evaluateCompletionGuard(displayContent, finishMeta, completionState);
|
|
2221
|
+
if (guardFailure) {
|
|
2222
|
+
completionGuardFailure = guardFailure;
|
|
2223
|
+
finalDisplayContent = displayContent;
|
|
2224
|
+
this.emitCompletionTrace('guard_triggered', guardFailure.message, guardFailure.signal, 'warn');
|
|
2225
|
+
log.warn('completion_guard_triggered', {
|
|
2226
|
+
turnId,
|
|
2227
|
+
attempt: attemptNumber,
|
|
2228
|
+
conversationId: convId,
|
|
2229
|
+
finishReason: finishMeta?.finishReason,
|
|
2230
|
+
rawFinishReason: finishMeta?.rawFinishReason,
|
|
2231
|
+
lastContentKind: finishMeta?.lastContentKind,
|
|
2232
|
+
stepCount: finishMeta?.stepCount ?? 0,
|
|
2233
|
+
chunkCount: finishMeta?.chunkCount ?? 0,
|
|
2234
|
+
toolCallCount: finishMeta?.toolCallCount ?? 0,
|
|
2235
|
+
textChars: finishMeta?.textChars ?? displayContent.length,
|
|
2236
|
+
completionSignal: completionState.signal,
|
|
2237
|
+
turnOutcome,
|
|
2238
|
+
});
|
|
2239
|
+
return;
|
|
2240
|
+
}
|
|
2241
|
+
// Clean up failed attempt messages from conversation store
|
|
2242
|
+
// so they don't leak into future turns (preserves successful attempt's tool results)
|
|
2243
|
+
if (failedAttemptMessageIds.length > 0) {
|
|
2244
|
+
const deleted = this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
2245
|
+
if (deleted > 0) {
|
|
2246
|
+
log.info('Cleaned up failed attempt messages', {
|
|
2247
|
+
deleted,
|
|
2248
|
+
convId,
|
|
2249
|
+
attempt: attemptNumber,
|
|
2250
|
+
});
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
2253
|
+
const cerebrumMessage = this.conversations.appendMessage(convId, 'cerebrum', displayContent, visibleToolCalls?.length ? { toolCalls: visibleToolCalls } : undefined);
|
|
2254
|
+
this.emit({
|
|
2255
|
+
type: 'message:cerebrum:end',
|
|
2256
|
+
conversationId: convId,
|
|
2257
|
+
turnId,
|
|
2258
|
+
sessionId: turnId,
|
|
2259
|
+
source: sessionSource,
|
|
2260
|
+
message: cerebrumMessage,
|
|
2261
|
+
});
|
|
2262
|
+
log.info('stream_finished', {
|
|
2263
|
+
turnId,
|
|
2264
|
+
attempt: attemptNumber,
|
|
2265
|
+
conversationId: convId,
|
|
2266
|
+
stallRetryCount: this.streamNudgeCount,
|
|
2267
|
+
completionRetryCount,
|
|
2268
|
+
retryCause,
|
|
2269
|
+
});
|
|
2270
|
+
if (retryCause === 'completion') {
|
|
2271
|
+
this.emitCompletionTrace('retry_recovered', `Completion retry ${completionRetryCount}/${this.maxCompletionRetries} recovered on attempt ${attemptNumber}.`, completionState.signal, 'info');
|
|
2272
|
+
}
|
|
2273
|
+
else if (retryCause === 'stall') {
|
|
2274
|
+
this.emitWatchdog('retry_recovered', `Stall retry ${this.streamNudgeCount}/${this.maxNudgeRetries} recovered on attempt ${attemptNumber}.`, { level: 'info' });
|
|
2275
|
+
}
|
|
2276
|
+
},
|
|
2277
|
+
onError: (error) => {
|
|
2278
|
+
if (!isCurrentAttempt())
|
|
2279
|
+
return;
|
|
2280
|
+
this.stopStreamWatchdog();
|
|
2281
|
+
// Don't log/emit if the abort was intentional (nudge or Cerebellum disconnect) — catch block handles it
|
|
2282
|
+
if (abortController.signal.aborted)
|
|
2283
|
+
return;
|
|
2284
|
+
log.error('Cerebrum stream error', { error: error.message });
|
|
2285
|
+
this.emit({ type: 'error', error });
|
|
2286
|
+
},
|
|
2287
|
+
}, { abortSignal: abortController.signal });
|
|
2288
|
+
await this.awaitStreamAttempt(streamPromise, abortController);
|
|
2289
|
+
const completionFailure = completionGuardFailure;
|
|
2290
|
+
if (completionFailure !== null) {
|
|
2291
|
+
const turnOutcome = this.classifyTurnOutcome(finalDisplayContent, attemptFinishMeta, completionState);
|
|
2292
|
+
const completionSignal = completionFailure.signal;
|
|
2293
|
+
const recoveryRequest = this.buildRecoveryRequest({
|
|
2294
|
+
cause: 'completion',
|
|
2295
|
+
attempt: attemptNumber,
|
|
2296
|
+
partialContent: fullContent || finalDisplayContent,
|
|
2297
|
+
completionState,
|
|
2298
|
+
turnOutcome,
|
|
2299
|
+
latestUserMessage,
|
|
2300
|
+
completionRetryCount,
|
|
2301
|
+
finishMeta: attemptFinishMeta,
|
|
2302
|
+
});
|
|
2303
|
+
const { source, assessment } = await this.assessTurnRecovery(recoveryRequest);
|
|
2304
|
+
this.emitRecoveryTrace('completion', source, assessment, assessment.action === 'stop' ? 'warn' : 'info');
|
|
2305
|
+
nextRetryContext = this.buildRetryContextMessage('completion', attemptNumber, assessment.modelMessage, source);
|
|
2306
|
+
this.appendTurnJournalEntry('recovery', assessment.operatorMessage, {
|
|
2307
|
+
cause: 'completion',
|
|
2308
|
+
source,
|
|
2309
|
+
action: assessment.action,
|
|
2310
|
+
diagnosis: assessment.diagnosis,
|
|
2311
|
+
nextStep: assessment.nextStep,
|
|
2312
|
+
completedSteps: assessment.completedSteps,
|
|
2313
|
+
turnOutcome,
|
|
2314
|
+
finishReason: attemptFinishMeta?.finishReason,
|
|
2315
|
+
});
|
|
2316
|
+
this.recordBoundary(completionState.continuity, {
|
|
2317
|
+
kind: 'recovery',
|
|
2318
|
+
action: assessment.action === 'stop' ? 'completion_stop' : 'completion_retry',
|
|
2319
|
+
summary: assessment.diagnosis,
|
|
2320
|
+
stateChanging: false,
|
|
2321
|
+
});
|
|
2322
|
+
log.info('completion_retry_context_prepared', {
|
|
2323
|
+
turnId,
|
|
2324
|
+
attempt: attemptNumber,
|
|
2325
|
+
conversationId: convId,
|
|
2326
|
+
source,
|
|
2327
|
+
action: assessment.action,
|
|
2328
|
+
finishReason: attemptFinishMeta?.finishReason,
|
|
2329
|
+
rawFinishReason: attemptFinishMeta?.rawFinishReason,
|
|
2330
|
+
hasPartialContent: (fullContent || finalDisplayContent).trim().length > 0,
|
|
2331
|
+
progressEntries: completionState.continuity.progressLedger.length,
|
|
2332
|
+
taskCheckpoints: completionState.continuity.taskCheckpoints.length,
|
|
2333
|
+
completedSteps: assessment.completedSteps,
|
|
2334
|
+
nextStep: assessment.nextStep,
|
|
2335
|
+
turnOutcome,
|
|
2336
|
+
lastContentKind: attemptFinishMeta?.lastContentKind,
|
|
2337
|
+
latestBoundary: completionState.continuity.boundaries.at(-1),
|
|
2338
|
+
repetitionSignals: recoveryRequest.repetitionSignals,
|
|
2339
|
+
});
|
|
2340
|
+
if (assessment.action === 'stop') {
|
|
2341
|
+
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
2342
|
+
const diagnosticMessage = this.conversations.appendMessage(convId, 'system', assessment.operatorMessage);
|
|
2343
|
+
this.emit({ type: 'message:system', message: diagnosticMessage });
|
|
2344
|
+
this.emitCompletionTrace('retry_failed', assessment.diagnosis, completionSignal, 'error');
|
|
2345
|
+
this.appendTurnJournalEntry('turn_error', assessment.diagnosis || 'Recovery returned stop.', {
|
|
2346
|
+
retryCause: 'completion',
|
|
2347
|
+
completionRetryCount,
|
|
2348
|
+
stallRetryCount: this.streamNudgeCount,
|
|
2349
|
+
error: assessment.diagnosis || 'Recovery returned stop.',
|
|
2350
|
+
});
|
|
2351
|
+
this.emit({
|
|
2352
|
+
type: 'error',
|
|
2353
|
+
error: new Error(assessment.diagnosis ||
|
|
2354
|
+
'Turn ended without a valid completion signal or final answer.'),
|
|
2355
|
+
});
|
|
2356
|
+
if (failedAttemptMessageIds.length > 0) {
|
|
2357
|
+
this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
2358
|
+
}
|
|
2359
|
+
loopTerminated = true;
|
|
2360
|
+
break;
|
|
2361
|
+
}
|
|
2362
|
+
if (completionRetryCount < this.maxCompletionRetries) {
|
|
2363
|
+
completionRetryCount++;
|
|
2364
|
+
const systemMessage = this.conversations.appendMessage(convId, 'system', assessment.operatorMessage);
|
|
2365
|
+
attemptMessageIds.push(systemMessage.id);
|
|
2366
|
+
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
2367
|
+
this.emit({ type: 'message:system', message: systemMessage });
|
|
2368
|
+
this.emitCompletionTrace('retry_started', `Retrying attempt ${attemptNumber + 1} after incomplete completion (${completionRetryCount}/${this.maxCompletionRetries}).`, completionSignal, 'info');
|
|
2369
|
+
nextRetryCause = 'completion';
|
|
2370
|
+
continue;
|
|
2371
|
+
}
|
|
2372
|
+
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
2373
|
+
const diagnosticMessage = this.conversations.appendMessage(convId, 'system', source === 'cerebellum'
|
|
2374
|
+
? '[Cerebellum] The turn ended repeatedly without a valid completion signal or final answer.'
|
|
2375
|
+
: '[System fallback] The turn ended repeatedly without a valid completion signal or final answer.');
|
|
2376
|
+
this.emit({ type: 'message:system', message: diagnosticMessage });
|
|
2377
|
+
this.emitCompletionTrace('retry_failed', `Completion retries exhausted after ${completionRetryCount}/${this.maxCompletionRetries}: ${assessment.diagnosis || completionFailure.message}`, completionSignal, 'error');
|
|
2378
|
+
this.appendTurnJournalEntry('turn_error', `Completion retries exhausted (${completionRetryCount}/${this.maxCompletionRetries}).`, {
|
|
2379
|
+
retryCause: 'completion',
|
|
2380
|
+
completionRetryCount,
|
|
2381
|
+
stallRetryCount: this.streamNudgeCount,
|
|
2382
|
+
error: assessment.diagnosis || 'Completion retries exhausted.',
|
|
2383
|
+
});
|
|
2384
|
+
this.emit({
|
|
2385
|
+
type: 'error',
|
|
2386
|
+
error: new Error(assessment.diagnosis ||
|
|
2387
|
+
'Turn ended without a valid completion signal or final answer.'),
|
|
2388
|
+
});
|
|
2389
|
+
// Clean up all failed attempt messages on exhaustion
|
|
2390
|
+
if (failedAttemptMessageIds.length > 0) {
|
|
2391
|
+
this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
2392
|
+
}
|
|
2393
|
+
loopTerminated = true;
|
|
2394
|
+
break;
|
|
2395
|
+
}
|
|
2396
|
+
loopTerminated = true;
|
|
2397
|
+
break; // success — exit retry loop
|
|
2398
|
+
}
|
|
2399
|
+
catch (error) {
|
|
2400
|
+
const failureState = this.getStreamState();
|
|
2401
|
+
this.stopStreamWatchdog();
|
|
2402
|
+
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
2403
|
+
const recoveryDecision = this.pendingRecoveryDecision;
|
|
2404
|
+
this.pendingRecoveryDecision = null;
|
|
2405
|
+
const stallRecovery = recoveryDecision;
|
|
2406
|
+
const isRecoveryRetryAbort = abortController.signal.aborted &&
|
|
2407
|
+
stallRecovery !== null &&
|
|
2408
|
+
stallRecovery.assessment.action === 'retry' &&
|
|
2409
|
+
this.streamNudgeCount > stallRetryCountAtStart &&
|
|
2410
|
+
this.streamNudgeCount <= this.maxNudgeRetries;
|
|
2411
|
+
if (isRecoveryRetryAbort && stallRecovery) {
|
|
2412
|
+
nextRetryContext = this.buildRetryContextMessage('stall', attemptNumber, stallRecovery.assessment.modelMessage, stallRecovery.source);
|
|
2413
|
+
const systemMessage = this.conversations.appendMessage(convId, 'system', stallRecovery.assessment.operatorMessage);
|
|
2414
|
+
attemptMessageIds.push(systemMessage.id);
|
|
2415
|
+
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
2416
|
+
this.emit({ type: 'message:system', message: systemMessage });
|
|
2417
|
+
this.emitWatchdog('retry_started', `Retrying stalled turn with attempt ${attemptNumber + 1} (stall retry ${this.streamNudgeCount}/${this.maxNudgeRetries}).`, { level: 'info' });
|
|
2418
|
+
nextRetryCause = 'stall';
|
|
2419
|
+
continue; // retry loop
|
|
2420
|
+
}
|
|
2421
|
+
if (abortController.signal.aborted &&
|
|
2422
|
+
stallRecovery !== null &&
|
|
2423
|
+
stallRecovery.assessment.action === 'stop') {
|
|
2424
|
+
const systemMessage = this.conversations.appendMessage(convId, 'system', stallRecovery.assessment.operatorMessage);
|
|
2425
|
+
this.emit({ type: 'message:system', message: systemMessage });
|
|
2426
|
+
this.appendTurnJournalEntry('turn_error', stallRecovery.assessment.diagnosis || 'Stall recovery returned stop.', {
|
|
2427
|
+
retryCause: 'stall',
|
|
2428
|
+
stallRetryCount: this.streamNudgeCount,
|
|
2429
|
+
completionRetryCount,
|
|
2430
|
+
error: stallRecovery.assessment.diagnosis || 'Stall recovery returned stop.',
|
|
2431
|
+
});
|
|
2432
|
+
this.emit({ type: 'error', error: new Error(stallRecovery.assessment.diagnosis) });
|
|
2433
|
+
if (failedAttemptMessageIds.length > 0) {
|
|
2434
|
+
this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
2435
|
+
}
|
|
2436
|
+
loopTerminated = true;
|
|
2437
|
+
break;
|
|
2438
|
+
}
|
|
2439
|
+
// Check if Cerebellum dropped mid-stream
|
|
2440
|
+
if (this.cerebellum && !this.cerebellum.isConnected() && abortController.signal.aborted) {
|
|
2441
|
+
const err = new Error('Cerebellum disconnected during active response. Restart it with: docker compose up -d cerebellum');
|
|
2442
|
+
log.error('Cerebellum disconnected mid-stream', { error: err.message });
|
|
2443
|
+
this.appendTurnJournalEntry('turn_error', err.message, {
|
|
2444
|
+
retryCause,
|
|
2445
|
+
stallRetryCount: this.streamNudgeCount,
|
|
2446
|
+
completionRetryCount,
|
|
2447
|
+
error: err.message,
|
|
2448
|
+
aborted: true,
|
|
2449
|
+
});
|
|
2450
|
+
this.emit({ type: 'error', error: err });
|
|
2451
|
+
if (failedAttemptMessageIds.length > 0) {
|
|
2452
|
+
this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
2453
|
+
}
|
|
2454
|
+
loopTerminated = true;
|
|
2455
|
+
break;
|
|
2456
|
+
}
|
|
2457
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
2458
|
+
this.appendTurnJournalEntry('turn_error', `Turn attempt ${attemptNumber} failed: ${err.message}`, {
|
|
2459
|
+
retryCause,
|
|
2460
|
+
stallRetryCount: this.streamNudgeCount,
|
|
2461
|
+
completionRetryCount,
|
|
2462
|
+
phase: failureState.phase,
|
|
2463
|
+
activeToolName: failureState.activeToolName,
|
|
2464
|
+
activeToolCallId: failureState.activeToolCallId,
|
|
2465
|
+
error: err.message,
|
|
2466
|
+
});
|
|
2467
|
+
this.pruneTurnJournals(convId);
|
|
2468
|
+
if (retryCause === 'completion') {
|
|
2469
|
+
this.emitCompletionTrace('retry_failed', `Completion retry attempt ${attemptNumber} failed: ${err.message}`, completionState.signal, 'error');
|
|
2470
|
+
}
|
|
2471
|
+
else if (retryCause === 'stall') {
|
|
2472
|
+
this.emitWatchdog('retry_failed', `Stall retry attempt ${attemptNumber} failed: ${err.message}`, { level: 'error' });
|
|
2473
|
+
}
|
|
2474
|
+
log.error('Send message failed', {
|
|
2475
|
+
error: err.message,
|
|
2476
|
+
turnId,
|
|
2477
|
+
attempt: attemptNumber,
|
|
2478
|
+
conversationId: convId,
|
|
2479
|
+
phase: failureState.phase,
|
|
2480
|
+
activeToolName: failureState.activeToolName,
|
|
2481
|
+
activeToolCallId: failureState.activeToolCallId,
|
|
2482
|
+
activeToolStartedAt: failureState.activeToolStartedAt,
|
|
2483
|
+
stallRetryCount: this.streamNudgeCount,
|
|
2484
|
+
completionRetryCount,
|
|
2485
|
+
retryCause,
|
|
2486
|
+
});
|
|
2487
|
+
this.emit({ type: 'error', error: err });
|
|
2488
|
+
// Clean up all failed attempt messages on error
|
|
2489
|
+
if (failedAttemptMessageIds.length > 0) {
|
|
2490
|
+
this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
2491
|
+
}
|
|
2492
|
+
loopTerminated = true;
|
|
2493
|
+
break;
|
|
2494
|
+
}
|
|
2495
|
+
finally {
|
|
2496
|
+
this.currentAttemptCompletionState = null;
|
|
2497
|
+
}
|
|
2498
|
+
}
|
|
2499
|
+
if (!loopTerminated) {
|
|
2500
|
+
const err = new Error(`Retry safety limit reached after ${maxTotalAttempts} attempts.`);
|
|
2501
|
+
log.error('Send message failed', {
|
|
2502
|
+
error: err.message,
|
|
2503
|
+
turnId,
|
|
2504
|
+
conversationId: convId,
|
|
2505
|
+
stallRetryCount: this.streamNudgeCount,
|
|
2506
|
+
completionRetryCount,
|
|
2507
|
+
});
|
|
2508
|
+
this.emit({ type: 'error', error: err });
|
|
2509
|
+
}
|
|
2510
|
+
}
|
|
2511
|
+
finally {
|
|
2512
|
+
this.currentQuerySession = null;
|
|
2513
|
+
this.currentStreamTurn = null;
|
|
2514
|
+
}
|
|
2515
|
+
}
|
|
2516
|
+
async start() {
|
|
2517
|
+
if (!this.activeConversationId) {
|
|
2518
|
+
// Resume the most recent conversation, or create a new one
|
|
2519
|
+
const convs = this.conversations.list();
|
|
2520
|
+
if (convs.length > 0) {
|
|
2521
|
+
this.resumeConversation(convs[0].id);
|
|
2522
|
+
log.info('Auto-resumed last conversation', { id: convs[0].id });
|
|
2523
|
+
}
|
|
2524
|
+
else {
|
|
2525
|
+
this.startConversation();
|
|
2526
|
+
}
|
|
2527
|
+
}
|
|
2528
|
+
// Start sub-agent monitoring loop if cerebellum + sub-agents are enabled
|
|
2529
|
+
if (this.subAgentManager && this.cerebellum?.reportAgentStates) {
|
|
2530
|
+
this.startMonitorLoop();
|
|
2531
|
+
}
|
|
2532
|
+
// Recover interrupted sub-agents from disk
|
|
2533
|
+
if (this.subAgentManager) {
|
|
2534
|
+
const recovered = await this.subAgentManager.recoverFromDisk();
|
|
2535
|
+
for (const agentId of recovered) {
|
|
2536
|
+
const agent = this.subAgentManager.getAgent(agentId);
|
|
2537
|
+
if (agent) {
|
|
2538
|
+
this.emit({ type: 'agent:recovered', agentId, task: agent.task });
|
|
2539
|
+
}
|
|
2540
|
+
}
|
|
2541
|
+
if (recovered.length > 0) {
|
|
2542
|
+
log.info(`Recovered ${recovered.length} interrupted sub-agent(s)`);
|
|
2543
|
+
}
|
|
2544
|
+
}
|
|
2545
|
+
}
|
|
2546
|
+
async stop() {
|
|
2547
|
+
if (this.monitorTimer) {
|
|
2548
|
+
clearInterval(this.monitorTimer);
|
|
2549
|
+
this.monitorTimer = null;
|
|
2550
|
+
}
|
|
2551
|
+
this.stopFineTunePoller();
|
|
2552
|
+
this.removeAllListeners();
|
|
2553
|
+
log.info('Orchestrator stopped');
|
|
2554
|
+
}
|
|
2555
|
+
startMonitorLoop() {
|
|
2556
|
+
if (this.monitorTimer)
|
|
2557
|
+
return;
|
|
2558
|
+
this.monitorTimer = setInterval(async () => {
|
|
2559
|
+
if (!this.subAgentManager || !this.cerebellum?.reportAgentStates)
|
|
2560
|
+
return;
|
|
2561
|
+
const agents = this.subAgentManager.listAgents();
|
|
2562
|
+
const activeAgents = agents.filter((a) => a.status === 'running' || a.status === 'pending');
|
|
2563
|
+
if (activeAgents.length === 0)
|
|
2564
|
+
return;
|
|
2565
|
+
try {
|
|
2566
|
+
const actions = await this.cerebellum.reportAgentStates(activeAgents.map((a) => ({
|
|
2567
|
+
id: a.id,
|
|
2568
|
+
task: a.task,
|
|
2569
|
+
status: a.status,
|
|
2570
|
+
spawnedAt: a.spawnedAt,
|
|
2571
|
+
lastActivityAt: a.lastActivityAt,
|
|
2572
|
+
timeoutMs: a.timeoutMs,
|
|
2573
|
+
messagesCount: a.messagesCount,
|
|
2574
|
+
toolCallsCount: a.toolCallsCount,
|
|
2575
|
+
retryCount: a.retryCount,
|
|
2576
|
+
progressNote: a.progressNote ?? '',
|
|
2577
|
+
progressPercent: a.progressPercent ?? -1,
|
|
2578
|
+
lastProgressAt: a.lastProgressAt ?? 0,
|
|
2579
|
+
deadlineAt: a.deadlineAt,
|
|
2580
|
+
})));
|
|
2581
|
+
const actionable = actions.filter((a) => a.action !== 'ok');
|
|
2582
|
+
if (actionable.length > 0) {
|
|
2583
|
+
this.emit({ type: 'agent:health', actions: actionable });
|
|
2584
|
+
}
|
|
2585
|
+
for (const action of actions) {
|
|
2586
|
+
switch (action.action) {
|
|
2587
|
+
case 'ping':
|
|
2588
|
+
this.subAgentManager.ping(action.agentId);
|
|
2589
|
+
break;
|
|
2590
|
+
case 'retry':
|
|
2591
|
+
await this.subAgentManager.retry(action.agentId);
|
|
2592
|
+
break;
|
|
2593
|
+
case 'cancel':
|
|
2594
|
+
this.subAgentManager.cancel(action.agentId);
|
|
2595
|
+
break;
|
|
2596
|
+
case 'timeout':
|
|
2597
|
+
this.subAgentManager.timeout(action.agentId);
|
|
2598
|
+
break;
|
|
2599
|
+
}
|
|
2600
|
+
}
|
|
2601
|
+
}
|
|
2602
|
+
catch {
|
|
2603
|
+
// Monitor failure is non-blocking
|
|
2604
|
+
}
|
|
2605
|
+
// Prune old completed agents
|
|
2606
|
+
this.subAgentManager.prune();
|
|
2607
|
+
}, this.monitorIntervalMs);
|
|
2608
|
+
}
|
|
2609
|
+
}
|
|
2610
|
+
//# sourceMappingURL=orchestrator.js.map
|