@cereworker/core 26.330.2 → 26.403.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/conversation.d.ts +27 -5
- package/dist/conversation.d.ts.map +1 -1
- package/dist/conversation.js +254 -108
- package/dist/conversation.js.map +1 -1
- package/dist/events.d.ts +21 -1
- package/dist/events.d.ts.map +1 -1
- package/dist/events.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/legacy-sqlite.d.ts +6 -0
- package/dist/legacy-sqlite.d.ts.map +1 -0
- package/dist/legacy-sqlite.js +90 -0
- package/dist/legacy-sqlite.js.map +1 -0
- package/dist/orchestrator.d.ts +23 -4
- package/dist/orchestrator.d.ts.map +1 -1
- package/dist/orchestrator.js +795 -201
- package/dist/orchestrator.js.map +1 -1
- package/dist/pairing.d.ts +11 -2
- package/dist/pairing.d.ts.map +1 -1
- package/dist/pairing.js +166 -95
- package/dist/pairing.js.map +1 -1
- package/dist/plan-store.d.ts +4 -3
- package/dist/plan-store.d.ts.map +1 -1
- package/dist/plan-store.js +69 -72
- package/dist/plan-store.js.map +1 -1
- package/dist/system-prompt.d.ts.map +1 -1
- package/dist/system-prompt.js +31 -11
- package/dist/system-prompt.js.map +1 -1
- package/dist/text-store.d.ts +18 -0
- package/dist/text-store.d.ts.map +1 -0
- package/dist/text-store.js +212 -0
- package/dist/text-store.js.map +1 -0
- package/dist/types.d.ts +64 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -2
package/dist/orchestrator.js
CHANGED
|
@@ -12,8 +12,15 @@ const log = createLogger('orchestrator');
|
|
|
12
12
|
const TASK_COMPLETE_TOOL = 'task_complete';
|
|
13
13
|
const TASK_BLOCKED_TOOL = 'task_blocked';
|
|
14
14
|
const TASK_CHECKPOINT_TOOL = 'task_checkpoint';
|
|
15
|
-
const INTERNAL_TASK_TOOL_NAMES = new Set([
|
|
16
|
-
|
|
15
|
+
const INTERNAL_TASK_TOOL_NAMES = new Set([
|
|
16
|
+
TASK_COMPLETE_TOOL,
|
|
17
|
+
TASK_BLOCKED_TOOL,
|
|
18
|
+
TASK_CHECKPOINT_TOOL,
|
|
19
|
+
]);
|
|
20
|
+
const SYSTEM_FALLBACK_COMPLETION_PROMPT = '[System fallback] The last turn ended without a final answer. Continue from the last verified state and end by calling task_complete or task_blocked before your final answer.';
|
|
21
|
+
const SYSTEM_FALLBACK_STALL_PROMPT = '[System fallback] The stalled turn is being retried from the last verified state.';
|
|
22
|
+
const DEBUG_TOOL_OUTPUT_MAX_CHARS = 8_000;
|
|
23
|
+
const DEBUG_TOOL_STRUCTURED_MAX_CHARS = 16_000;
|
|
17
24
|
const READ_ONLY_TOOL_NAMES = new Set([
|
|
18
25
|
'browserGetText',
|
|
19
26
|
'browserGetUrl',
|
|
@@ -48,9 +55,16 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
48
55
|
fineTuneMethod = 'auto';
|
|
49
56
|
fineTuneSchedule = 'auto';
|
|
50
57
|
fineTuneStatus = {
|
|
51
|
-
status: 'idle',
|
|
52
|
-
|
|
53
|
-
|
|
58
|
+
status: 'idle',
|
|
59
|
+
jobId: '',
|
|
60
|
+
progress: 0,
|
|
61
|
+
currentStep: 0,
|
|
62
|
+
totalSteps: 0,
|
|
63
|
+
currentLoss: 0,
|
|
64
|
+
error: '',
|
|
65
|
+
checkpointPath: '',
|
|
66
|
+
startedAt: 0,
|
|
67
|
+
completedAt: 0,
|
|
54
68
|
};
|
|
55
69
|
_fineTuneHistory = [];
|
|
56
70
|
gatewayMode = 'standalone';
|
|
@@ -64,13 +78,22 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
64
78
|
lastStreamActivityAt = 0;
|
|
65
79
|
streamWatchdog = null;
|
|
66
80
|
streamNudgeCount = 0;
|
|
81
|
+
streamDeferredUntil = 0;
|
|
67
82
|
streamStallThreshold = 30_000;
|
|
68
83
|
maxNudgeRetries = 2;
|
|
69
84
|
maxCompletionRetries = 2;
|
|
85
|
+
turnJournalRetention = {
|
|
86
|
+
maxDays: 30,
|
|
87
|
+
maxFilesPerConversation: 100,
|
|
88
|
+
};
|
|
70
89
|
streamPhase = 'idle';
|
|
71
90
|
activeToolCall = null;
|
|
72
91
|
currentStreamTurn = null;
|
|
73
92
|
currentAttemptCompletionState = null;
|
|
93
|
+
currentPartialContent = '';
|
|
94
|
+
currentLastContentKind = 'empty';
|
|
95
|
+
currentJournaledContentLength = 0;
|
|
96
|
+
pendingRecoveryDecision = null;
|
|
74
97
|
streamAbortGraceMs = 1_000;
|
|
75
98
|
taskConversations = new Map();
|
|
76
99
|
taskRunning = new Set();
|
|
@@ -96,6 +119,12 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
96
119
|
this.maxNudgeRetries = options.maxNudgeRetries;
|
|
97
120
|
this.maxCompletionRetries = options.maxNudgeRetries;
|
|
98
121
|
}
|
|
122
|
+
if (options?.turnJournalRetention) {
|
|
123
|
+
this.turnJournalRetention = {
|
|
124
|
+
...this.turnJournalRetention,
|
|
125
|
+
...options.turnJournalRetention,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
99
128
|
}
|
|
100
129
|
setCerebrum(cerebrum) {
|
|
101
130
|
this.cerebrum = cerebrum;
|
|
@@ -175,9 +204,19 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
175
204
|
parameters: {
|
|
176
205
|
type: 'object',
|
|
177
206
|
properties: {
|
|
178
|
-
step: {
|
|
179
|
-
|
|
180
|
-
|
|
207
|
+
step: {
|
|
208
|
+
type: 'string',
|
|
209
|
+
description: 'Short milestone name, such as "profile continuity checked"',
|
|
210
|
+
},
|
|
211
|
+
status: {
|
|
212
|
+
type: 'string',
|
|
213
|
+
enum: ['done', 'in_progress'],
|
|
214
|
+
description: 'Whether the milestone is done or currently in progress',
|
|
215
|
+
},
|
|
216
|
+
evidence: {
|
|
217
|
+
type: 'string',
|
|
218
|
+
description: 'Concrete evidence showing what happened at this milestone',
|
|
219
|
+
},
|
|
181
220
|
},
|
|
182
221
|
required: ['step', 'status', 'evidence'],
|
|
183
222
|
additionalProperties: false,
|
|
@@ -214,8 +253,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
214
253
|
isError: true,
|
|
215
254
|
};
|
|
216
255
|
}
|
|
217
|
-
const hasVerifiedProgress = state.successfulExternalToolCount > 0
|
|
218
|
-
|
|
256
|
+
const hasVerifiedProgress = state.successfulExternalToolCount > 0 ||
|
|
257
|
+
state.continuity.progressLedger.some((entry) => entry.source === 'tool' && !entry.isError);
|
|
219
258
|
if (!hasVerifiedProgress) {
|
|
220
259
|
return {
|
|
221
260
|
output: 'task_complete requires at least one successful external tool result in this turn.',
|
|
@@ -240,6 +279,22 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
240
279
|
state.summary = undefined;
|
|
241
280
|
state.evidence = evidence;
|
|
242
281
|
}
|
|
282
|
+
const boundarySummary = signal === 'complete'
|
|
283
|
+
? `Task completion recorded: ${state.summary}`
|
|
284
|
+
: `Task blocker recorded: ${state.blocker}`;
|
|
285
|
+
this.recordBoundary(state.continuity, {
|
|
286
|
+
kind: 'completion',
|
|
287
|
+
action: signal === 'complete' ? TASK_COMPLETE_TOOL : TASK_BLOCKED_TOOL,
|
|
288
|
+
summary: boundarySummary,
|
|
289
|
+
stateChanging: true,
|
|
290
|
+
evidence,
|
|
291
|
+
});
|
|
292
|
+
this.appendTurnJournalEntry('completion_signal', boundarySummary, {
|
|
293
|
+
signal,
|
|
294
|
+
evidence,
|
|
295
|
+
summary: state.summary,
|
|
296
|
+
blocker: state.blocker,
|
|
297
|
+
});
|
|
243
298
|
this.emitCompletionTrace('signal_recorded', signal === 'complete'
|
|
244
299
|
? `Recorded task_complete signal with evidence: ${evidence}`
|
|
245
300
|
: `Recorded task_blocked signal with evidence: ${evidence}`, signal, 'info');
|
|
@@ -263,9 +318,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
263
318
|
const step = String(args.step ?? '').trim();
|
|
264
319
|
const evidence = String(args.evidence ?? '').trim();
|
|
265
320
|
const statusValue = String(args.status ?? '').trim();
|
|
266
|
-
const status = statusValue === 'done' || statusValue === 'in_progress'
|
|
267
|
-
? statusValue
|
|
268
|
-
: null;
|
|
321
|
+
const status = statusValue === 'done' || statusValue === 'in_progress' ? statusValue : null;
|
|
269
322
|
if (!step) {
|
|
270
323
|
return {
|
|
271
324
|
output: 'A non-empty step field is required.',
|
|
@@ -454,7 +507,12 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
454
507
|
if (!result.started) {
|
|
455
508
|
throw new Error(result.error || 'Failed to start fine-tuning');
|
|
456
509
|
}
|
|
457
|
-
this.fineTuneStatus = {
|
|
510
|
+
this.fineTuneStatus = {
|
|
511
|
+
...this.fineTuneStatus,
|
|
512
|
+
status: 'running',
|
|
513
|
+
jobId: result.jobId,
|
|
514
|
+
startedAt: Date.now(),
|
|
515
|
+
};
|
|
458
516
|
this.emit({ type: 'finetune:start', jobId: result.jobId });
|
|
459
517
|
log.info('Fine-tuning started', { jobId: result.jobId });
|
|
460
518
|
// 5. Poll for progress
|
|
@@ -493,7 +551,10 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
493
551
|
jobId: status.jobId,
|
|
494
552
|
checkpointPath: status.checkpointPath,
|
|
495
553
|
});
|
|
496
|
-
log.info('Fine-tuning completed', {
|
|
554
|
+
log.info('Fine-tuning completed', {
|
|
555
|
+
jobId: status.jobId,
|
|
556
|
+
checkpoint: status.checkpointPath,
|
|
557
|
+
});
|
|
497
558
|
this.stopFineTunePoller();
|
|
498
559
|
}
|
|
499
560
|
else if (status.status === 'failed') {
|
|
@@ -534,7 +595,10 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
534
595
|
return {
|
|
535
596
|
name: nameMatch?.[1]?.trim() || 'Cere',
|
|
536
597
|
role: roleMatch?.[1]?.trim() || 'general-purpose assistant',
|
|
537
|
-
traits: traitsMatch?.[1]
|
|
598
|
+
traits: traitsMatch?.[1]
|
|
599
|
+
?.split(',')
|
|
600
|
+
.map((t) => t.trim().toLowerCase())
|
|
601
|
+
.filter(Boolean) ?? [],
|
|
538
602
|
};
|
|
539
603
|
}
|
|
540
604
|
getActiveConversationId() {
|
|
@@ -638,7 +702,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
638
702
|
return {
|
|
639
703
|
streaming: this.streamWatchdog !== null,
|
|
640
704
|
lastActivityAt: this.lastStreamActivityAt,
|
|
641
|
-
stallDetected: this.streamWatchdog !== null &&
|
|
705
|
+
stallDetected: this.streamWatchdog !== null && Date.now() - this.lastStreamActivityAt > stallThresholdMs,
|
|
642
706
|
nudgeCount: this.streamNudgeCount,
|
|
643
707
|
phase: this.streamPhase,
|
|
644
708
|
activeToolName: this.activeToolCall?.name,
|
|
@@ -659,9 +723,9 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
659
723
|
}
|
|
660
724
|
markStreamWaitingTool(toolCall, activityAt = Date.now()) {
|
|
661
725
|
const normalizedToolName = toolCall.name.trim() || toolCall.name;
|
|
662
|
-
const phaseChanged = this.streamPhase !== 'waiting_tool'
|
|
663
|
-
|
|
664
|
-
|
|
726
|
+
const phaseChanged = this.streamPhase !== 'waiting_tool' ||
|
|
727
|
+
this.activeToolCall?.id !== toolCall.id ||
|
|
728
|
+
this.activeToolCall?.name !== normalizedToolName;
|
|
665
729
|
this.lastStreamActivityAt = activityAt;
|
|
666
730
|
this.streamPhase = 'waiting_tool';
|
|
667
731
|
this.activeToolCall = {
|
|
@@ -680,6 +744,123 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
680
744
|
resetStreamState() {
|
|
681
745
|
this.streamPhase = 'idle';
|
|
682
746
|
this.activeToolCall = null;
|
|
747
|
+
this.streamDeferredUntil = 0;
|
|
748
|
+
this.currentPartialContent = '';
|
|
749
|
+
this.currentLastContentKind = 'empty';
|
|
750
|
+
this.currentJournaledContentLength = 0;
|
|
751
|
+
}
|
|
752
|
+
appendTurnJournalEntry(type, summary, data) {
|
|
753
|
+
if (!this.currentStreamTurn)
|
|
754
|
+
return;
|
|
755
|
+
const entry = {
|
|
756
|
+
turnId: this.currentStreamTurn.turnId,
|
|
757
|
+
attempt: this.currentStreamTurn.attempt,
|
|
758
|
+
timestamp: Date.now(),
|
|
759
|
+
type,
|
|
760
|
+
summary: this.truncateResumeText(summary, 500),
|
|
761
|
+
...(data ? { data } : {}),
|
|
762
|
+
};
|
|
763
|
+
this.conversations.appendTurnJournalEntry(this.currentStreamTurn.conversationId, this.currentStreamTurn.turnId, entry);
|
|
764
|
+
}
|
|
765
|
+
pruneTurnJournals(conversationId) {
|
|
766
|
+
try {
|
|
767
|
+
const result = this.conversations.pruneTurnJournals(conversationId, this.turnJournalRetention);
|
|
768
|
+
if (result.prunedByAge > 0 || result.prunedByCount > 0) {
|
|
769
|
+
log.debug('Pruned turn journals', {
|
|
770
|
+
conversationId,
|
|
771
|
+
prunedByAge: result.prunedByAge,
|
|
772
|
+
prunedByCount: result.prunedByCount,
|
|
773
|
+
remaining: result.remaining,
|
|
774
|
+
});
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
catch (error) {
|
|
778
|
+
log.warn('Failed to prune turn journals', {
|
|
779
|
+
conversationId,
|
|
780
|
+
error: error instanceof Error ? error.message : String(error),
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
persistPartialContentSnapshot(force = false) {
|
|
785
|
+
const normalized = this.currentPartialContent.trim();
|
|
786
|
+
if (!normalized)
|
|
787
|
+
return;
|
|
788
|
+
if (!force && this.currentPartialContent.length - this.currentJournaledContentLength < 200) {
|
|
789
|
+
return;
|
|
790
|
+
}
|
|
791
|
+
this.currentJournaledContentLength = this.currentPartialContent.length;
|
|
792
|
+
this.appendTurnJournalEntry('partial_text', `Assistant partial text: ${this.truncateResumeText(normalized, 220)}`, {
|
|
793
|
+
chars: this.currentPartialContent.length,
|
|
794
|
+
excerpt: this.truncateResumeText(normalized, 1200),
|
|
795
|
+
});
|
|
796
|
+
}
|
|
797
|
+
recordBoundary(continuity, boundary) {
|
|
798
|
+
const createdAt = Date.now();
|
|
799
|
+
const summary = {
|
|
800
|
+
id: nanoid(10),
|
|
801
|
+
createdAt,
|
|
802
|
+
browserState: this.cloneBrowserState(boundary.browserState ?? continuity.browserState),
|
|
803
|
+
...boundary,
|
|
804
|
+
};
|
|
805
|
+
continuity.boundaries.push(summary);
|
|
806
|
+
while (continuity.boundaries.length > 20) {
|
|
807
|
+
continuity.boundaries.shift();
|
|
808
|
+
}
|
|
809
|
+
this.appendTurnJournalEntry('boundary', summary.summary, {
|
|
810
|
+
boundaryId: summary.id,
|
|
811
|
+
kind: summary.kind,
|
|
812
|
+
action: summary.action,
|
|
813
|
+
stateChanging: summary.stateChanging,
|
|
814
|
+
url: summary.url,
|
|
815
|
+
tabId: summary.tabId,
|
|
816
|
+
checkpointStatus: summary.checkpointStatus,
|
|
817
|
+
evidence: summary.evidence,
|
|
818
|
+
browserState: summary.browserState,
|
|
819
|
+
});
|
|
820
|
+
return summary;
|
|
821
|
+
}
|
|
822
|
+
deriveRepetitionSignals(continuity) {
|
|
823
|
+
const signals = [];
|
|
824
|
+
const recentToolEntries = continuity.progressLedger
|
|
825
|
+
.filter((entry) => entry.source === 'tool')
|
|
826
|
+
.slice(-10);
|
|
827
|
+
const recentActions = recentToolEntries.map((entry) => entry.action);
|
|
828
|
+
if (recentActions.length >= 4 && new Set(recentActions.slice(-4)).size <= 2) {
|
|
829
|
+
signals.push(`Recent tool actions are cycling between ${Array.from(new Set(recentActions.slice(-4))).join(', ')}`);
|
|
830
|
+
}
|
|
831
|
+
const summaryCounts = new Map();
|
|
832
|
+
for (const boundary of continuity.boundaries.slice(-12)) {
|
|
833
|
+
if (boundary.kind !== 'tool' || !boundary.stateChanging)
|
|
834
|
+
continue;
|
|
835
|
+
const key = `${boundary.action}|${boundary.summary}`;
|
|
836
|
+
summaryCounts.set(key, (summaryCounts.get(key) ?? 0) + 1);
|
|
837
|
+
}
|
|
838
|
+
for (const [key, count] of summaryCounts.entries()) {
|
|
839
|
+
if (count < 2)
|
|
840
|
+
continue;
|
|
841
|
+
const [, summary] = key.split('|', 2);
|
|
842
|
+
signals.push(`Repeated verified action x${count}: ${summary}`);
|
|
843
|
+
}
|
|
844
|
+
return signals.slice(0, 5);
|
|
845
|
+
}
|
|
846
|
+
classifyTurnOutcome(displayContent, finishMeta, completionState) {
|
|
847
|
+
const trimmed = displayContent.trim();
|
|
848
|
+
const endedOnToolCalls = finishMeta?.finishReason === 'tool-calls' ||
|
|
849
|
+
finishMeta?.stepFinishReasons.at(-1) === 'tool-calls' ||
|
|
850
|
+
finishMeta?.endedWithToolCall === true;
|
|
851
|
+
if (endedOnToolCalls) {
|
|
852
|
+
return 'ended_on_tool_calls';
|
|
853
|
+
}
|
|
854
|
+
if (completionState.externalToolCallCount > 0 && completionState.signal === 'none') {
|
|
855
|
+
return 'completion_signal_missing';
|
|
856
|
+
}
|
|
857
|
+
if (finishMeta?.finishReason === 'error' || finishMeta?.lastContentKind === 'error') {
|
|
858
|
+
return 'protocol_error';
|
|
859
|
+
}
|
|
860
|
+
if (trimmed.length > 0) {
|
|
861
|
+
return 'completed';
|
|
862
|
+
}
|
|
863
|
+
return 'completed_no_text';
|
|
683
864
|
}
|
|
684
865
|
getStreamDiagnostics(elapsedSeconds) {
|
|
685
866
|
return {
|
|
@@ -692,17 +873,13 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
692
873
|
}
|
|
693
874
|
describeStreamLocation(phase = this.streamPhase, activeToolName = this.activeToolCall?.name) {
|
|
694
875
|
if (phase === 'waiting_tool') {
|
|
695
|
-
return activeToolName
|
|
696
|
-
? `waiting_tool/${activeToolName}`
|
|
697
|
-
: 'waiting_tool';
|
|
876
|
+
return activeToolName ? `waiting_tool/${activeToolName}` : 'waiting_tool';
|
|
698
877
|
}
|
|
699
878
|
return phase;
|
|
700
879
|
}
|
|
701
880
|
getCurrentStallThresholdMs(phase = this.streamPhase, nudgeCount = this.streamNudgeCount) {
|
|
702
|
-
const phaseBase = phase === 'waiting_model'
|
|
703
|
-
|
|
704
|
-
: this.streamStallThreshold;
|
|
705
|
-
return phaseBase + (nudgeCount * this.streamStallThreshold);
|
|
881
|
+
const phaseBase = phase === 'waiting_model' ? this.streamStallThreshold * 3 : this.streamStallThreshold;
|
|
882
|
+
return phaseBase + nudgeCount * this.streamStallThreshold;
|
|
706
883
|
}
|
|
707
884
|
logStreamDebug(msg, data) {
|
|
708
885
|
if (!this.currentStreamTurn)
|
|
@@ -785,121 +962,235 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
785
962
|
progressLedger: [],
|
|
786
963
|
taskCheckpoints: [],
|
|
787
964
|
browserState: {},
|
|
965
|
+
boundaries: [],
|
|
788
966
|
};
|
|
789
967
|
}
|
|
790
|
-
|
|
968
|
+
buildRecoveryRequest(params) {
|
|
791
969
|
const partialContent = this.truncateResumeText(params.partialContent, 600);
|
|
792
970
|
const continuity = params.completionState.continuity;
|
|
793
|
-
|
|
794
|
-
&& continuity.progressLedger.length === 0
|
|
795
|
-
&& continuity.taskCheckpoints.length === 0
|
|
796
|
-
&& !params.activeToolName
|
|
797
|
-
&& !continuity.browserState.currentUrl
|
|
798
|
-
&& !continuity.browserState.activeTabId) {
|
|
799
|
-
return null;
|
|
800
|
-
}
|
|
971
|
+
const latestBoundary = continuity.boundaries.at(-1);
|
|
801
972
|
return {
|
|
802
|
-
|
|
973
|
+
conversationId: this.currentStreamTurn?.conversationId ?? '',
|
|
974
|
+
turnId: this.currentStreamTurn?.turnId ?? '',
|
|
803
975
|
attempt: params.attempt,
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
976
|
+
cause: params.cause,
|
|
977
|
+
phase: this.streamPhase,
|
|
978
|
+
activeToolName: this.activeToolCall?.name,
|
|
979
|
+
activeToolCallId: this.activeToolCall?.id,
|
|
980
|
+
stallRetryCount: this.streamNudgeCount,
|
|
981
|
+
completionRetryCount: params.completionRetryCount ?? 0,
|
|
982
|
+
finishReason: params.finishMeta?.finishReason ?? params.finishMeta?.stepFinishReasons.at(-1),
|
|
983
|
+
turnOutcome: params.turnOutcome,
|
|
984
|
+
lastContentKind: params.finishMeta?.lastContentKind ?? this.currentLastContentKind,
|
|
985
|
+
elapsedSeconds: params.elapsedSeconds,
|
|
807
986
|
partialContent: partialContent || undefined,
|
|
808
|
-
|
|
809
|
-
|
|
987
|
+
latestUserMessage: params.latestUserMessage
|
|
988
|
+
? this.truncateResumeText(params.latestUserMessage, 600)
|
|
989
|
+
: undefined,
|
|
990
|
+
progressEntries: continuity.progressLedger.slice(-50).map((entry) => ({ ...entry })),
|
|
991
|
+
taskCheckpoints: continuity.taskCheckpoints.map((checkpoint) => ({ ...checkpoint })),
|
|
810
992
|
browserState: this.cloneBrowserState(continuity.browserState),
|
|
993
|
+
latestBoundary: latestBoundary
|
|
994
|
+
? {
|
|
995
|
+
...latestBoundary,
|
|
996
|
+
...(latestBoundary.browserState
|
|
997
|
+
? { browserState: this.cloneBrowserState(latestBoundary.browserState) }
|
|
998
|
+
: {}),
|
|
999
|
+
}
|
|
1000
|
+
: undefined,
|
|
1001
|
+
recentBoundaries: continuity.boundaries.slice(-20).map((boundary) => ({
|
|
1002
|
+
...boundary,
|
|
1003
|
+
...(boundary.browserState
|
|
1004
|
+
? { browserState: this.cloneBrowserState(boundary.browserState) }
|
|
1005
|
+
: {}),
|
|
1006
|
+
})),
|
|
1007
|
+
repetitionSignals: this.deriveRepetitionSignals(continuity),
|
|
811
1008
|
};
|
|
812
1009
|
}
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
const
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
1010
|
+
emitRecoveryTrace(cause, source, assessment, level = 'info') {
|
|
1011
|
+
if (!this.currentStreamTurn)
|
|
1012
|
+
return;
|
|
1013
|
+
const payload = {
|
|
1014
|
+
type: 'cerebellum:recovery',
|
|
1015
|
+
cause,
|
|
1016
|
+
action: assessment.action,
|
|
1017
|
+
turnId: this.currentStreamTurn.turnId,
|
|
1018
|
+
attempt: this.currentStreamTurn.attempt,
|
|
1019
|
+
conversationId: this.currentStreamTurn.conversationId,
|
|
1020
|
+
message: assessment.operatorMessage,
|
|
1021
|
+
operatorMessage: assessment.operatorMessage,
|
|
1022
|
+
diagnosis: assessment.diagnosis,
|
|
1023
|
+
nextStep: assessment.nextStep,
|
|
1024
|
+
completedSteps: assessment.completedSteps,
|
|
1025
|
+
waitSeconds: assessment.waitSeconds,
|
|
1026
|
+
source,
|
|
1027
|
+
...this.getStreamDiagnostics(),
|
|
1028
|
+
};
|
|
1029
|
+
switch (level) {
|
|
1030
|
+
case 'debug':
|
|
1031
|
+
log.debug('cerebellum_recovery', payload);
|
|
1032
|
+
break;
|
|
1033
|
+
case 'warn':
|
|
1034
|
+
log.warn('cerebellum_recovery', payload);
|
|
1035
|
+
break;
|
|
1036
|
+
case 'error':
|
|
1037
|
+
log.error('cerebellum_recovery', payload);
|
|
1038
|
+
break;
|
|
1039
|
+
default:
|
|
1040
|
+
log.info('cerebellum_recovery', payload);
|
|
1041
|
+
break;
|
|
1042
|
+
}
|
|
1043
|
+
this.emit(payload);
|
|
1044
|
+
}
|
|
1045
|
+
async assessTurnRecovery(request) {
|
|
1046
|
+
log.debug('turn_recovery_request', {
|
|
1047
|
+
turnId: request.turnId,
|
|
1048
|
+
attempt: request.attempt,
|
|
1049
|
+
conversationId: request.conversationId,
|
|
1050
|
+
cause: request.cause,
|
|
1051
|
+
phase: request.phase,
|
|
1052
|
+
activeToolName: request.activeToolName,
|
|
1053
|
+
activeToolCallId: request.activeToolCallId,
|
|
1054
|
+
stallRetryCount: request.stallRetryCount,
|
|
1055
|
+
completionRetryCount: request.completionRetryCount,
|
|
1056
|
+
finishReason: request.finishReason,
|
|
1057
|
+
turnOutcome: request.turnOutcome,
|
|
1058
|
+
lastContentKind: request.lastContentKind,
|
|
1059
|
+
elapsedSeconds: request.elapsedSeconds,
|
|
1060
|
+
hasPartialContent: Boolean(request.partialContent),
|
|
1061
|
+
latestUserMessage: request.latestUserMessage
|
|
1062
|
+
? this.truncateResumeText(request.latestUserMessage, 300)
|
|
1063
|
+
: '',
|
|
1064
|
+
browserState: request.browserState,
|
|
1065
|
+
progressEntries: request.progressEntries,
|
|
1066
|
+
taskCheckpoints: request.taskCheckpoints,
|
|
1067
|
+
latestBoundary: request.latestBoundary,
|
|
1068
|
+
recentBoundaries: request.recentBoundaries,
|
|
1069
|
+
repetitionSignals: request.repetitionSignals,
|
|
1070
|
+
});
|
|
1071
|
+
if (this.cerebellum?.isConnected() && this.cerebellum.assessTurnRecovery) {
|
|
1072
|
+
try {
|
|
1073
|
+
const assessment = await this.cerebellum.assessTurnRecovery(request);
|
|
1074
|
+
if (assessment) {
|
|
1075
|
+
if (request.cause === 'completion' && assessment.action === 'wait') {
|
|
1076
|
+
return {
|
|
1077
|
+
source: 'cerebellum',
|
|
1078
|
+
assessment: {
|
|
1079
|
+
...assessment,
|
|
1080
|
+
action: 'retry',
|
|
1081
|
+
waitSeconds: undefined,
|
|
1082
|
+
},
|
|
1083
|
+
};
|
|
1084
|
+
}
|
|
1085
|
+
return { source: 'cerebellum', assessment };
|
|
1086
|
+
}
|
|
1087
|
+
}
|
|
1088
|
+
catch (error) {
|
|
1089
|
+
log.warn('Turn recovery assessment failed', {
|
|
1090
|
+
turnId: request.turnId,
|
|
1091
|
+
attempt: request.attempt,
|
|
1092
|
+
conversationId: request.conversationId,
|
|
1093
|
+
cause: request.cause,
|
|
1094
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1095
|
+
});
|
|
1096
|
+
}
|
|
824
1097
|
}
|
|
825
1098
|
return {
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
finishReason,
|
|
829
|
-
partialContent: partialContent || undefined,
|
|
830
|
-
progressEntries: continuity.progressLedger.slice(-20),
|
|
831
|
-
taskCheckpoints: continuity.taskCheckpoints.slice(-8),
|
|
832
|
-
browserState: this.cloneBrowserState(continuity.browserState),
|
|
1099
|
+
source: 'fallback',
|
|
1100
|
+
assessment: this.buildFallbackRecoveryAssessment(request),
|
|
833
1101
|
};
|
|
834
1102
|
}
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
? `The previous attempt (${snapshot.attempt}) was interrupted after stalling while ${this.describeStreamLocation(snapshot.phase, snapshot.activeToolName)}.`
|
|
844
|
-
: `The previous attempt (${snapshot.attempt}) ended without a valid completion${snapshot.finishReason ? ` (finish reason: ${snapshot.finishReason})` : ''}.`,
|
|
845
|
-
'IMPORTANT: The tool call history from the failed attempt has been removed from this conversation. The ledger below is the authoritative record of what was already verified.',
|
|
846
|
-
'Do NOT repeat completed steps unless the current page state clearly contradicts this ledger.',
|
|
847
|
-
'Continue from the NEXT incomplete step, then either finish the task or report a concrete blocker.',
|
|
848
|
-
];
|
|
849
|
-
if (snapshot.browserState.currentUrl || snapshot.browserState.activeTabId || snapshot.browserState.tabs?.length) {
|
|
850
|
-
lines.push('', 'Last known browser state:');
|
|
851
|
-
if (snapshot.browserState.currentUrl) {
|
|
852
|
-
lines.push(`- Current URL: ${snapshot.browserState.currentUrl}`);
|
|
853
|
-
}
|
|
854
|
-
if (snapshot.browserState.activeTabId) {
|
|
855
|
-
lines.push(`- Active tab: ${snapshot.browserState.activeTabId}`);
|
|
856
|
-
}
|
|
857
|
-
if (snapshot.browserState.tabs?.length) {
|
|
858
|
-
const visibleTabs = snapshot.browserState.tabs.slice(0, 6);
|
|
859
|
-
for (const tab of visibleTabs) {
|
|
860
|
-
lines.push(`- Tab ${tab.id}${tab.active ? ' [active]' : ''}: ${tab.url}${tab.title ? ` (${tab.title})` : ''}`);
|
|
861
|
-
}
|
|
862
|
-
if (snapshot.browserState.tabs.length > visibleTabs.length) {
|
|
863
|
-
lines.push(`- ... ${snapshot.browserState.tabs.length - visibleTabs.length} more tab(s)`);
|
|
864
|
-
}
|
|
1103
|
+
deriveCompletedSteps(request) {
|
|
1104
|
+
const completed = new Set();
|
|
1105
|
+
for (const boundary of request.recentBoundaries) {
|
|
1106
|
+
if ((boundary.kind === 'tool' ||
|
|
1107
|
+
boundary.kind === 'checkpoint' ||
|
|
1108
|
+
boundary.kind === 'completion') &&
|
|
1109
|
+
boundary.summary) {
|
|
1110
|
+
completed.add(boundary.summary);
|
|
865
1111
|
}
|
|
866
1112
|
}
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
lines.push(`- ${checkpoint.summary}`);
|
|
1113
|
+
for (const checkpoint of request.taskCheckpoints) {
|
|
1114
|
+
if (checkpoint.status === 'done') {
|
|
1115
|
+
completed.add(checkpoint.summary);
|
|
871
1116
|
}
|
|
872
1117
|
}
|
|
873
|
-
const
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
for (const entry of toolEntries) {
|
|
877
|
-
const prefix = entry.isError ? '[error]' : entry.stateChanging ? '[done]' : '[seen]';
|
|
878
|
-
lines.push(`- ${prefix} ${entry.summary}`);
|
|
1118
|
+
for (const entry of request.progressEntries) {
|
|
1119
|
+
if (entry.source === 'tool' && entry.stateChanging && !entry.isError) {
|
|
1120
|
+
completed.add(entry.summary);
|
|
879
1121
|
}
|
|
880
1122
|
}
|
|
881
|
-
|
|
882
|
-
|
|
1123
|
+
return Array.from(completed).slice(-10);
|
|
1124
|
+
}
|
|
1125
|
+
buildFallbackRecoveryAssessment(request, options) {
|
|
1126
|
+
const completedSteps = this.deriveCompletedSteps(request);
|
|
1127
|
+
const browserHints = [];
|
|
1128
|
+
if (request.browserState.currentUrl)
|
|
1129
|
+
browserHints.push(`Current URL: ${request.browserState.currentUrl}`);
|
|
1130
|
+
if (request.browserState.activeTabId)
|
|
1131
|
+
browserHints.push(`Active tab: ${request.browserState.activeTabId}`);
|
|
1132
|
+
const diagnosis = options?.reason ??
|
|
1133
|
+
(request.cause === 'stall'
|
|
1134
|
+
? `Recovery guidance is unavailable while the stream is stalled in ${this.describeStreamLocation(request.phase, request.activeToolName)}.`
|
|
1135
|
+
: `Recovery guidance is unavailable after the turn ended with ${request.turnOutcome} (${request.finishReason ?? 'no final answer'}).`);
|
|
1136
|
+
const nextStep = request.cause === 'stall'
|
|
1137
|
+
? 'Resume from the last verified browser state and continue with the next unfinished step.'
|
|
1138
|
+
: 'Use the verified progress below to continue from the next unfinished step and avoid repeating confirmed work.';
|
|
1139
|
+
const lines = [
|
|
1140
|
+
'[System fallback recovery]',
|
|
1141
|
+
diagnosis,
|
|
1142
|
+
'The failed attempt tool history has been removed; rely on this verified summary instead.',
|
|
1143
|
+
];
|
|
1144
|
+
if (completedSteps.length > 0) {
|
|
1145
|
+
lines.push('', 'Completed steps:');
|
|
1146
|
+
for (const step of completedSteps)
|
|
1147
|
+
lines.push(`- ${step}`);
|
|
1148
|
+
}
|
|
1149
|
+
if (browserHints.length > 0) {
|
|
1150
|
+
lines.push('', 'Last known browser state:');
|
|
1151
|
+
for (const hint of browserHints)
|
|
1152
|
+
lines.push(`- ${hint}`);
|
|
1153
|
+
}
|
|
1154
|
+
if (request.latestBoundary) {
|
|
1155
|
+
lines.push('', `Latest verified boundary: ${request.latestBoundary.summary}`);
|
|
1156
|
+
}
|
|
1157
|
+
if (request.repetitionSignals.length > 0) {
|
|
1158
|
+
lines.push('', 'Repetition warnings:');
|
|
1159
|
+
for (const signal of request.repetitionSignals)
|
|
1160
|
+
lines.push(`- ${signal}`);
|
|
883
1161
|
}
|
|
884
|
-
if (
|
|
885
|
-
lines.push('', 'Partial assistant text
|
|
1162
|
+
if (request.partialContent) {
|
|
1163
|
+
lines.push('', 'Partial assistant text from the failed attempt:', request.partialContent);
|
|
886
1164
|
}
|
|
887
|
-
lines.push('',
|
|
1165
|
+
lines.push('', `Next step: ${nextStep}`);
|
|
1166
|
+
lines.push('Only repeat a completed action if the current page state clearly contradicts this summary.');
|
|
1167
|
+
lines.push('End your final answer by calling task_complete or task_blocked.');
|
|
1168
|
+
return {
|
|
1169
|
+
action: options?.action ?? 'retry',
|
|
1170
|
+
operatorMessage: request.cause === 'stall'
|
|
1171
|
+
? SYSTEM_FALLBACK_STALL_PROMPT
|
|
1172
|
+
: SYSTEM_FALLBACK_COMPLETION_PROMPT,
|
|
1173
|
+
modelMessage: lines.join('\n'),
|
|
1174
|
+
diagnosis,
|
|
1175
|
+
nextStep,
|
|
1176
|
+
completedSteps,
|
|
1177
|
+
};
|
|
1178
|
+
}
|
|
1179
|
+
buildRetryContextMessage(cause, attempt, modelMessage, source) {
|
|
888
1180
|
return {
|
|
889
|
-
id: `system:${
|
|
1181
|
+
id: `system:${cause}-retry:${attempt}`,
|
|
890
1182
|
role: 'system',
|
|
891
|
-
content:
|
|
1183
|
+
content: modelMessage,
|
|
892
1184
|
timestamp: 0,
|
|
893
1185
|
metadata: {
|
|
894
1186
|
transient: true,
|
|
895
|
-
source:
|
|
1187
|
+
source: cause === 'stall' ? 'watchdog-resume' : 'completion-resume',
|
|
1188
|
+
recoverySource: source,
|
|
896
1189
|
},
|
|
897
1190
|
};
|
|
898
1191
|
}
|
|
899
1192
|
formatToolOutputPreview(output) {
|
|
900
|
-
return this.truncateResumeText(output, 180)
|
|
901
|
-
.replace(/\s+/g, ' ')
|
|
902
|
-
.trim();
|
|
1193
|
+
return this.truncateResumeText(output, 180).replace(/\s+/g, ' ').trim();
|
|
903
1194
|
}
|
|
904
1195
|
truncateResumeText(text, maxChars) {
|
|
905
1196
|
const normalized = text.replace(/\r\n/g, '\n').trim();
|
|
@@ -907,6 +1198,54 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
907
1198
|
return normalized;
|
|
908
1199
|
return `${normalized.slice(0, Math.max(0, maxChars - 1)).trimEnd()}…`;
|
|
909
1200
|
}
|
|
1201
|
+
serializeDebugValue(value, maxChars) {
|
|
1202
|
+
const raw = typeof value === 'string' ? value : (JSON.stringify(value, null, 2) ?? String(value));
|
|
1203
|
+
if (raw.length <= maxChars) {
|
|
1204
|
+
return { value: raw, truncated: false };
|
|
1205
|
+
}
|
|
1206
|
+
return {
|
|
1207
|
+
value: `${raw.slice(0, Math.max(0, maxChars - 1)).trimEnd()}…`,
|
|
1208
|
+
truncated: true,
|
|
1209
|
+
};
|
|
1210
|
+
}
|
|
1211
|
+
buildToolDebugPayload(toolCall, result, toolName) {
|
|
1212
|
+
const argsPreview = this.serializeDebugValue(toolCall.args, DEBUG_TOOL_STRUCTURED_MAX_CHARS);
|
|
1213
|
+
if (!result) {
|
|
1214
|
+
return {
|
|
1215
|
+
requestedToolName: toolCall.name,
|
|
1216
|
+
toolName: toolName ?? (toolCall.name.trim() || toolCall.name),
|
|
1217
|
+
toolCallId: toolCall.id,
|
|
1218
|
+
toolArgs: argsPreview.value,
|
|
1219
|
+
debugPayloadTruncated: argsPreview.truncated,
|
|
1220
|
+
};
|
|
1221
|
+
}
|
|
1222
|
+
const outputPreview = this.serializeDebugValue(result.output, DEBUG_TOOL_OUTPUT_MAX_CHARS);
|
|
1223
|
+
const detailsPreview = result.details
|
|
1224
|
+
? this.serializeDebugValue(result.details, DEBUG_TOOL_STRUCTURED_MAX_CHARS)
|
|
1225
|
+
: null;
|
|
1226
|
+
const resumeMetadata = result.metadata && typeof result.metadata === 'object'
|
|
1227
|
+
? (result.metadata.resume ?? null)
|
|
1228
|
+
: null;
|
|
1229
|
+
const resumePreview = resumeMetadata
|
|
1230
|
+
? this.serializeDebugValue(resumeMetadata, DEBUG_TOOL_STRUCTURED_MAX_CHARS)
|
|
1231
|
+
: null;
|
|
1232
|
+
return {
|
|
1233
|
+
requestedToolName: toolCall.name,
|
|
1234
|
+
toolName: toolName ?? (toolCall.name.trim() || toolCall.name),
|
|
1235
|
+
toolCallId: toolCall.id,
|
|
1236
|
+
toolArgs: argsPreview.value,
|
|
1237
|
+
toolOutput: outputPreview.value,
|
|
1238
|
+
toolDetails: detailsPreview?.value ?? null,
|
|
1239
|
+
toolResume: resumePreview?.value ?? null,
|
|
1240
|
+
isError: result.isError,
|
|
1241
|
+
warnings: result.warnings ?? [],
|
|
1242
|
+
truncated: result.truncated ?? false,
|
|
1243
|
+
debugPayloadTruncated: argsPreview.truncated ||
|
|
1244
|
+
outputPreview.truncated ||
|
|
1245
|
+
Boolean(detailsPreview?.truncated) ||
|
|
1246
|
+
Boolean(resumePreview?.truncated),
|
|
1247
|
+
};
|
|
1248
|
+
}
|
|
910
1249
|
recordCheckpoint(continuity, step, status, evidence) {
|
|
911
1250
|
const checkpoint = {
|
|
912
1251
|
step,
|
|
@@ -929,6 +1268,19 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
929
1268
|
isError: false,
|
|
930
1269
|
checkpointStatus: status,
|
|
931
1270
|
});
|
|
1271
|
+
this.recordBoundary(continuity, {
|
|
1272
|
+
kind: 'checkpoint',
|
|
1273
|
+
action: 'task_checkpoint',
|
|
1274
|
+
summary: checkpoint.summary,
|
|
1275
|
+
stateChanging: status === 'done',
|
|
1276
|
+
evidence,
|
|
1277
|
+
checkpointStatus: status,
|
|
1278
|
+
});
|
|
1279
|
+
this.appendTurnJournalEntry('checkpoint', checkpoint.summary, {
|
|
1280
|
+
step,
|
|
1281
|
+
status,
|
|
1282
|
+
evidence,
|
|
1283
|
+
});
|
|
932
1284
|
return checkpoint;
|
|
933
1285
|
}
|
|
934
1286
|
recordAttemptToolProgress(completionState, toolName, result) {
|
|
@@ -938,6 +1290,16 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
938
1290
|
return;
|
|
939
1291
|
this.recordProgressEntry(continuity, entry);
|
|
940
1292
|
this.updateBrowserState(continuity.browserState, result);
|
|
1293
|
+
if (entry.stateChanging && !entry.isError) {
|
|
1294
|
+
this.recordBoundary(continuity, {
|
|
1295
|
+
kind: 'tool',
|
|
1296
|
+
action: entry.action,
|
|
1297
|
+
summary: entry.summary,
|
|
1298
|
+
stateChanging: true,
|
|
1299
|
+
url: entry.url,
|
|
1300
|
+
tabId: entry.tabId,
|
|
1301
|
+
});
|
|
1302
|
+
}
|
|
941
1303
|
}
|
|
942
1304
|
createProgressEntry(toolName, result) {
|
|
943
1305
|
const resume = this.getBrowserResumeMetadata(result);
|
|
@@ -967,19 +1329,19 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
967
1329
|
}
|
|
968
1330
|
recordProgressEntry(continuity, entry) {
|
|
969
1331
|
const last = continuity.progressLedger.at(-1);
|
|
970
|
-
if (last
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
1332
|
+
if (last &&
|
|
1333
|
+
entry.source === 'tool' &&
|
|
1334
|
+
last.source === 'tool' &&
|
|
1335
|
+
!entry.stateChanging &&
|
|
1336
|
+
!last.stateChanging &&
|
|
1337
|
+
last.action === entry.action &&
|
|
1338
|
+
last.summary === entry.summary &&
|
|
1339
|
+
last.url === entry.url &&
|
|
1340
|
+
last.tabId === entry.tabId) {
|
|
979
1341
|
return;
|
|
980
1342
|
}
|
|
981
1343
|
continuity.progressLedger.push(entry);
|
|
982
|
-
while (continuity.progressLedger.length >
|
|
1344
|
+
while (continuity.progressLedger.length > 50) {
|
|
983
1345
|
const removableIndex = continuity.progressLedger.findIndex((candidate) => candidate.source === 'tool' && !candidate.stateChanging);
|
|
984
1346
|
continuity.progressLedger.splice(removableIndex >= 0 ? removableIndex : 0, 1);
|
|
985
1347
|
}
|
|
@@ -997,18 +1359,27 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
997
1359
|
const resume = this.getBrowserResumeMetadata(result);
|
|
998
1360
|
if (!resume)
|
|
999
1361
|
return;
|
|
1000
|
-
|
|
1001
|
-
|
|
1362
|
+
const stateDelta = resume.stateDelta && typeof resume.stateDelta === 'object'
|
|
1363
|
+
? resume.stateDelta
|
|
1364
|
+
: null;
|
|
1365
|
+
const deltaUrl = typeof stateDelta?.currentUrl === 'string' ? stateDelta.currentUrl : undefined;
|
|
1366
|
+
const deltaActiveTabId = typeof stateDelta?.activeTabId === 'string' ? stateDelta.activeTabId : undefined;
|
|
1367
|
+
const deltaTabs = Array.isArray(stateDelta?.tabs)
|
|
1368
|
+
? stateDelta.tabs
|
|
1369
|
+
: undefined;
|
|
1370
|
+
if (resume.url ?? deltaUrl) {
|
|
1371
|
+
browserState.currentUrl = resume.url ?? deltaUrl;
|
|
1002
1372
|
}
|
|
1003
|
-
if (resume.activeTabId) {
|
|
1004
|
-
browserState.activeTabId = resume.activeTabId;
|
|
1373
|
+
if (resume.activeTabId ?? deltaActiveTabId) {
|
|
1374
|
+
browserState.activeTabId = resume.activeTabId ?? deltaActiveTabId;
|
|
1005
1375
|
}
|
|
1006
1376
|
else if (resume.tabId && resume.stateChanging) {
|
|
1007
1377
|
browserState.activeTabId = resume.tabId;
|
|
1008
1378
|
}
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1379
|
+
const nextTabs = resume.tabs?.length ? resume.tabs : deltaTabs;
|
|
1380
|
+
if (nextTabs?.length) {
|
|
1381
|
+
browserState.tabs = nextTabs.map((tab) => ({ ...tab }));
|
|
1382
|
+
const active = nextTabs.find((tab) => tab.active);
|
|
1012
1383
|
if (active) {
|
|
1013
1384
|
browserState.activeTabId = active.id;
|
|
1014
1385
|
browserState.currentUrl = active.url;
|
|
@@ -1028,8 +1399,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1028
1399
|
evaluateCompletionGuard(displayContent, finishMeta, completionState) {
|
|
1029
1400
|
const trimmedContent = displayContent.trim();
|
|
1030
1401
|
const hadExternalToolActivity = completionState.externalToolCallCount > 0;
|
|
1031
|
-
const endedOnToolCalls = finishMeta?.finishReason === 'tool-calls'
|
|
1032
|
-
|
|
1402
|
+
const endedOnToolCalls = finishMeta?.finishReason === 'tool-calls' ||
|
|
1403
|
+
finishMeta?.stepFinishReasons.at(-1) === 'tool-calls';
|
|
1033
1404
|
if (trimmedContent.length === 0 && hadExternalToolActivity) {
|
|
1034
1405
|
return {
|
|
1035
1406
|
message: 'Turn ended after tool activity without a final answer.',
|
|
@@ -1098,10 +1469,15 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1098
1469
|
streamPromise.then(settleResolve, settleReject);
|
|
1099
1470
|
});
|
|
1100
1471
|
}
|
|
1101
|
-
startStreamWatchdog() {
|
|
1472
|
+
startStreamWatchdog(latestUserMessage) {
|
|
1102
1473
|
this.stopStreamWatchdog();
|
|
1103
1474
|
this.markStreamWaitingModel();
|
|
1475
|
+
this.streamDeferredUntil = 0;
|
|
1104
1476
|
this.streamWatchdog = setInterval(() => {
|
|
1477
|
+
if (!this.currentAttemptCompletionState || !this.currentStreamTurn)
|
|
1478
|
+
return;
|
|
1479
|
+
if (this.streamDeferredUntil > Date.now())
|
|
1480
|
+
return;
|
|
1105
1481
|
const elapsed = Date.now() - this.lastStreamActivityAt;
|
|
1106
1482
|
const stallThresholdMs = this.getCurrentStallThresholdMs();
|
|
1107
1483
|
if (elapsed < stallThresholdMs)
|
|
@@ -1115,31 +1491,106 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1115
1491
|
this.emitWatchdog('stalled', `Stalled after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'warn', elapsedSeconds });
|
|
1116
1492
|
this.emit({ type: 'cerebrum:stall', ...diagnostics });
|
|
1117
1493
|
if (!this.cerebellum?.isConnected()) {
|
|
1118
|
-
// Cerebellum dropped mid-stream — abort the current turn
|
|
1119
1494
|
this.emitWatchdog('abort_issued', 'Cerebellum disconnected during an active stream; aborting the turn.', { level: 'warn', elapsedSeconds });
|
|
1120
1495
|
this.abortController?.abort();
|
|
1121
1496
|
return;
|
|
1122
1497
|
}
|
|
1123
1498
|
this._nudgeInFlight = true;
|
|
1124
|
-
const doNudge = () => {
|
|
1125
|
-
this.streamNudgeCount++;
|
|
1126
|
-
this.emitWatchdog('nudge_requested', `Cerebellum requested nudge ${this.streamNudgeCount}/${this.maxNudgeRetries} after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'info', elapsedSeconds });
|
|
1127
|
-
this.emit({ type: 'cerebrum:stall:nudge', attempt: this.streamNudgeCount, ...diagnostics });
|
|
1128
|
-
this.emitWatchdog('abort_issued', `Aborting stalled stream attempt ${this.currentStreamTurn?.attempt ?? 0}.`, { level: 'warn', elapsedSeconds });
|
|
1129
|
-
this.abortController?.abort();
|
|
1130
|
-
};
|
|
1131
1499
|
void (async () => {
|
|
1132
1500
|
try {
|
|
1133
|
-
const
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1501
|
+
const request = this.buildRecoveryRequest({
|
|
1502
|
+
cause: 'stall',
|
|
1503
|
+
attempt: this.currentStreamTurn.attempt,
|
|
1504
|
+
partialContent: this.currentPartialContent,
|
|
1505
|
+
completionState: this.currentAttemptCompletionState,
|
|
1506
|
+
turnOutcome: 'stalled',
|
|
1507
|
+
latestUserMessage,
|
|
1508
|
+
elapsedSeconds,
|
|
1509
|
+
});
|
|
1510
|
+
const { source, assessment } = await this.assessTurnRecovery(request);
|
|
1511
|
+
this.emitRecoveryTrace('stall', source, assessment, assessment.action === 'stop' ? 'warn' : 'info');
|
|
1512
|
+
this.appendTurnJournalEntry('recovery', assessment.operatorMessage, {
|
|
1513
|
+
cause: 'stall',
|
|
1514
|
+
source,
|
|
1515
|
+
action: assessment.action,
|
|
1516
|
+
diagnosis: assessment.diagnosis,
|
|
1517
|
+
nextStep: assessment.nextStep,
|
|
1518
|
+
waitSeconds: assessment.waitSeconds,
|
|
1519
|
+
completedSteps: assessment.completedSteps,
|
|
1520
|
+
});
|
|
1521
|
+
if (assessment.action === 'wait') {
|
|
1522
|
+
const waitSeconds = Math.max(15, assessment.waitSeconds ?? this.streamStallThreshold / 1000);
|
|
1523
|
+
this.streamDeferredUntil = Date.now() + waitSeconds * 1000;
|
|
1524
|
+
return;
|
|
1525
|
+
}
|
|
1526
|
+
if (assessment.action === 'retry') {
|
|
1527
|
+
this.streamNudgeCount++;
|
|
1528
|
+
this.pendingRecoveryDecision = { cause: 'stall', source, assessment };
|
|
1529
|
+
this.recordBoundary(this.currentAttemptCompletionState.continuity, {
|
|
1530
|
+
kind: 'recovery',
|
|
1531
|
+
action: 'stall_retry',
|
|
1532
|
+
summary: assessment.diagnosis,
|
|
1533
|
+
stateChanging: false,
|
|
1534
|
+
});
|
|
1535
|
+
this.emitWatchdog('nudge_requested', `Cerebellum requested nudge ${this.streamNudgeCount}/${this.maxNudgeRetries} after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'info', elapsedSeconds });
|
|
1536
|
+
this.emit({
|
|
1537
|
+
type: 'cerebrum:stall:nudge',
|
|
1538
|
+
attempt: this.streamNudgeCount,
|
|
1539
|
+
...diagnostics,
|
|
1540
|
+
});
|
|
1541
|
+
this.emitWatchdog('abort_issued', `Aborting stalled stream attempt ${this.currentStreamTurn?.attempt ?? 0}.`, { level: 'warn', elapsedSeconds });
|
|
1542
|
+
this.abortController?.abort();
|
|
1543
|
+
return;
|
|
1138
1544
|
}
|
|
1545
|
+
this.pendingRecoveryDecision = { cause: 'stall', source, assessment };
|
|
1546
|
+
this.recordBoundary(this.currentAttemptCompletionState.continuity, {
|
|
1547
|
+
kind: 'recovery',
|
|
1548
|
+
action: 'stall_stop',
|
|
1549
|
+
summary: assessment.diagnosis,
|
|
1550
|
+
stateChanging: false,
|
|
1551
|
+
});
|
|
1552
|
+
this.emitWatchdog('abort_issued', 'Aborting stalled stream because recovery guidance requested stop.', { level: 'warn', elapsedSeconds });
|
|
1553
|
+
this.abortController?.abort();
|
|
1139
1554
|
}
|
|
1140
1555
|
catch {
|
|
1141
|
-
|
|
1142
|
-
|
|
1556
|
+
const request = this.buildRecoveryRequest({
|
|
1557
|
+
cause: 'stall',
|
|
1558
|
+
attempt: this.currentStreamTurn.attempt,
|
|
1559
|
+
partialContent: this.currentPartialContent,
|
|
1560
|
+
completionState: this.currentAttemptCompletionState,
|
|
1561
|
+
turnOutcome: 'stalled',
|
|
1562
|
+
latestUserMessage,
|
|
1563
|
+
elapsedSeconds,
|
|
1564
|
+
});
|
|
1565
|
+
const assessment = this.buildFallbackRecoveryAssessment(request, {
|
|
1566
|
+
reason: `Recovery assessment failed after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`,
|
|
1567
|
+
});
|
|
1568
|
+
this.pendingRecoveryDecision = { cause: 'stall', source: 'fallback', assessment };
|
|
1569
|
+
this.emitRecoveryTrace('stall', 'fallback', assessment, 'warn');
|
|
1570
|
+
this.appendTurnJournalEntry('recovery', assessment.operatorMessage, {
|
|
1571
|
+
cause: 'stall',
|
|
1572
|
+
source: 'fallback',
|
|
1573
|
+
action: assessment.action,
|
|
1574
|
+
diagnosis: assessment.diagnosis,
|
|
1575
|
+
nextStep: assessment.nextStep,
|
|
1576
|
+
waitSeconds: assessment.waitSeconds,
|
|
1577
|
+
completedSteps: assessment.completedSteps,
|
|
1578
|
+
});
|
|
1579
|
+
this.streamNudgeCount++;
|
|
1580
|
+
this.recordBoundary(this.currentAttemptCompletionState.continuity, {
|
|
1581
|
+
kind: 'recovery',
|
|
1582
|
+
action: 'stall_retry',
|
|
1583
|
+
summary: assessment.diagnosis,
|
|
1584
|
+
stateChanging: false,
|
|
1585
|
+
});
|
|
1586
|
+
this.emitWatchdog('nudge_requested', `Fallback retry ${this.streamNudgeCount}/${this.maxNudgeRetries} after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'info', elapsedSeconds });
|
|
1587
|
+
this.emit({
|
|
1588
|
+
type: 'cerebrum:stall:nudge',
|
|
1589
|
+
attempt: this.streamNudgeCount,
|
|
1590
|
+
...diagnostics,
|
|
1591
|
+
});
|
|
1592
|
+
this.emitWatchdog('abort_issued', `Aborting stalled stream attempt ${this.currentStreamTurn?.attempt ?? 0}.`, { level: 'warn', elapsedSeconds });
|
|
1593
|
+
this.abortController?.abort();
|
|
1143
1594
|
}
|
|
1144
1595
|
finally {
|
|
1145
1596
|
this._nudgeInFlight = false;
|
|
@@ -1168,6 +1619,11 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1168
1619
|
const userMessage = this.conversations.appendMessage(convId, 'user', content);
|
|
1169
1620
|
this.emit({ type: 'message:user', message: userMessage });
|
|
1170
1621
|
}
|
|
1622
|
+
const latestUserMessage = content ||
|
|
1623
|
+
[...this.conversations.getMessages(convId)]
|
|
1624
|
+
.reverse()
|
|
1625
|
+
.find((message) => message.role === 'user')?.content ||
|
|
1626
|
+
'';
|
|
1171
1627
|
this.streamNudgeCount = 0;
|
|
1172
1628
|
let completionRetryCount = 0;
|
|
1173
1629
|
let nextRetryContext = null;
|
|
@@ -1196,6 +1652,10 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1196
1652
|
attempt: attemptNumber,
|
|
1197
1653
|
conversationId: convId,
|
|
1198
1654
|
};
|
|
1655
|
+
this.currentPartialContent = '';
|
|
1656
|
+
this.currentLastContentKind = 'empty';
|
|
1657
|
+
this.currentJournaledContentLength = 0;
|
|
1658
|
+
this.pendingRecoveryDecision = null;
|
|
1199
1659
|
log.info('stream_started', {
|
|
1200
1660
|
turnId,
|
|
1201
1661
|
attempt: attemptNumber,
|
|
@@ -1204,8 +1664,14 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1204
1664
|
completionRetryCount,
|
|
1205
1665
|
retryCause,
|
|
1206
1666
|
});
|
|
1667
|
+
this.appendTurnJournalEntry('turn_started', `Turn attempt ${attemptNumber} started.`, {
|
|
1668
|
+
retryCause: retryCause ?? null,
|
|
1669
|
+
latestUserMessage: this.truncateResumeText(latestUserMessage, 600),
|
|
1670
|
+
stallRetryCount: this.streamNudgeCount,
|
|
1671
|
+
completionRetryCount,
|
|
1672
|
+
});
|
|
1207
1673
|
this.emit({ type: 'message:cerebrum:start', conversationId: convId });
|
|
1208
|
-
this.startStreamWatchdog();
|
|
1674
|
+
this.startStreamWatchdog(latestUserMessage);
|
|
1209
1675
|
let messages = this.conversations.getMessages(convId);
|
|
1210
1676
|
// On retry: exclude failed attempts' messages from history.
|
|
1211
1677
|
// The resume context already summarizes what happened — sending the raw tool calls
|
|
@@ -1274,6 +1740,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1274
1740
|
];
|
|
1275
1741
|
const toolDefs = Object.fromEntries(allTools);
|
|
1276
1742
|
let fullContent = '';
|
|
1743
|
+
let finalDisplayContent = '';
|
|
1744
|
+
let attemptFinishMeta;
|
|
1277
1745
|
const throwIfToolAttemptAborted = () => {
|
|
1278
1746
|
if (!isCurrentAttempt()) {
|
|
1279
1747
|
throw createAbortError('Tool execution aborted');
|
|
@@ -1286,26 +1754,42 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1286
1754
|
if (!isCurrentAttempt() || abortController.signal.aborted)
|
|
1287
1755
|
return;
|
|
1288
1756
|
fullContent += chunk;
|
|
1757
|
+
this.currentPartialContent = fullContent;
|
|
1758
|
+
this.currentLastContentKind = 'text';
|
|
1289
1759
|
this.markStreamWaitingModel();
|
|
1760
|
+
this.persistPartialContentSnapshot();
|
|
1290
1761
|
this.emit({ type: 'message:cerebrum:chunk', chunk });
|
|
1291
1762
|
},
|
|
1292
1763
|
onToolCall: async (toolCall) => {
|
|
1293
1764
|
throwIfToolAttemptAborted();
|
|
1294
|
-
this.logStreamDebug('tool_callback_started',
|
|
1295
|
-
toolName: toolCall.name.trim() || toolCall.name,
|
|
1296
|
-
toolCallId: toolCall.id,
|
|
1297
|
-
});
|
|
1765
|
+
this.logStreamDebug('tool_callback_started', this.buildToolDebugPayload(toolCall));
|
|
1298
1766
|
this.markStreamWaitingTool(toolCall);
|
|
1767
|
+
this.currentLastContentKind = 'tool-call';
|
|
1299
1768
|
const requestedToolName = toolCall.name;
|
|
1300
1769
|
const normalizedToolName = requestedToolName.trim() || requestedToolName;
|
|
1770
|
+
this.appendTurnJournalEntry('tool_start', `Calling ${normalizedToolName}`, {
|
|
1771
|
+
requestedToolName,
|
|
1772
|
+
toolName: normalizedToolName,
|
|
1773
|
+
callId: toolCall.id,
|
|
1774
|
+
args: toolCall.args,
|
|
1775
|
+
});
|
|
1301
1776
|
const isInternalTaskSignal = this.isInternalTaskSignalTool(normalizedToolName);
|
|
1302
1777
|
if (isInternalTaskSignal) {
|
|
1303
1778
|
completionState.internalToolCallCount++;
|
|
1304
1779
|
}
|
|
1305
1780
|
else {
|
|
1306
1781
|
completionState.externalToolCallCount++;
|
|
1307
|
-
this.emit({
|
|
1308
|
-
|
|
1782
|
+
this.emit({
|
|
1783
|
+
type: 'message:cerebrum:toolcall',
|
|
1784
|
+
toolCall: { ...toolCall, name: normalizedToolName },
|
|
1785
|
+
});
|
|
1786
|
+
this.emit({
|
|
1787
|
+
type: 'tool:start',
|
|
1788
|
+
callId: toolCall.id,
|
|
1789
|
+
name: normalizedToolName,
|
|
1790
|
+
requestedName: requestedToolName !== normalizedToolName ? requestedToolName : undefined,
|
|
1791
|
+
args: toolCall.args,
|
|
1792
|
+
});
|
|
1309
1793
|
}
|
|
1310
1794
|
const { toolName, result } = await this.toolRuntime.execute({
|
|
1311
1795
|
toolCall,
|
|
@@ -1315,21 +1799,27 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1315
1799
|
scopeKey: convId,
|
|
1316
1800
|
abortSignal: abortController.signal,
|
|
1317
1801
|
});
|
|
1318
|
-
this.logStreamDebug('tool_callback_finished',
|
|
1319
|
-
toolName,
|
|
1320
|
-
toolCallId: toolCall.id,
|
|
1321
|
-
isError: result.isError,
|
|
1322
|
-
});
|
|
1802
|
+
this.logStreamDebug('tool_callback_finished', this.buildToolDebugPayload(toolCall, result, toolName));
|
|
1323
1803
|
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
1324
1804
|
this.markStreamWaitingModel();
|
|
1805
|
+
this.currentLastContentKind = result.isError ? 'error' : 'tool-call';
|
|
1325
1806
|
if (!isInternalTaskSignal) {
|
|
1326
|
-
this.emit({
|
|
1807
|
+
this.emit({
|
|
1808
|
+
type: 'tool:end',
|
|
1809
|
+
callId: toolCall.id,
|
|
1810
|
+
name: toolName,
|
|
1811
|
+
requestedName: requestedToolName !== toolName ? requestedToolName : undefined,
|
|
1812
|
+
args: toolCall.args,
|
|
1813
|
+
result,
|
|
1814
|
+
});
|
|
1327
1815
|
}
|
|
1328
1816
|
if (!isInternalTaskSignal && !result.isError) {
|
|
1329
1817
|
completionState.successfulExternalToolCount++;
|
|
1330
1818
|
}
|
|
1331
1819
|
// Cerebellum verification (non-blocking)
|
|
1332
|
-
if (!isInternalTaskSignal &&
|
|
1820
|
+
if (!isInternalTaskSignal &&
|
|
1821
|
+
this.cerebellum?.isConnected() &&
|
|
1822
|
+
this.verificationEnabled) {
|
|
1333
1823
|
try {
|
|
1334
1824
|
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
1335
1825
|
this.emit({ type: 'verification:start', callId: toolCall.id, toolName });
|
|
@@ -1364,6 +1854,18 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1364
1854
|
}
|
|
1365
1855
|
}
|
|
1366
1856
|
throwIfToolAttemptAborted();
|
|
1857
|
+
const toolJournalEntry = this.createProgressEntry(toolName, result);
|
|
1858
|
+
this.appendTurnJournalEntry('tool_end', toolJournalEntry?.summary ??
|
|
1859
|
+
`${toolName}: ${this.formatToolOutputPreview(result.output)}`, {
|
|
1860
|
+
requestedToolName,
|
|
1861
|
+
toolName,
|
|
1862
|
+
callId: toolCall.id,
|
|
1863
|
+
isError: result.isError,
|
|
1864
|
+
args: toolCall.args,
|
|
1865
|
+
output: this.truncateResumeText(result.output, 2_000),
|
|
1866
|
+
details: result.details,
|
|
1867
|
+
metadata: result.metadata,
|
|
1868
|
+
});
|
|
1367
1869
|
if (!isInternalTaskSignal) {
|
|
1368
1870
|
this.recordAttemptToolProgress(completionState, toolName, result);
|
|
1369
1871
|
const toolMsg = this.conversations.appendMessage(convId, 'tool', result.output, {
|
|
@@ -1382,18 +1884,27 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1382
1884
|
return;
|
|
1383
1885
|
this.stopStreamWatchdog();
|
|
1384
1886
|
let displayContent = content;
|
|
1887
|
+
finalDisplayContent = content;
|
|
1888
|
+
attemptFinishMeta = finishMeta;
|
|
1889
|
+
this.currentLastContentKind =
|
|
1890
|
+
finishMeta?.lastContentKind ??
|
|
1891
|
+
(content.trim() ? 'text' : this.currentLastContentKind);
|
|
1892
|
+
this.persistPartialContentSnapshot(true);
|
|
1385
1893
|
const visibleToolCalls = toolCalls?.filter((toolCall) => !this.isInternalTaskSignalTool(toolCall.name));
|
|
1894
|
+
const turnOutcome = this.classifyTurnOutcome(displayContent, finishMeta, completionState);
|
|
1386
1895
|
log.info('stream_finish_observed', {
|
|
1387
1896
|
turnId,
|
|
1388
1897
|
attempt: attemptNumber,
|
|
1389
1898
|
conversationId: convId,
|
|
1390
1899
|
finishReason: finishMeta?.finishReason,
|
|
1391
1900
|
rawFinishReason: finishMeta?.rawFinishReason,
|
|
1901
|
+
lastContentKind: finishMeta?.lastContentKind,
|
|
1392
1902
|
stepCount: finishMeta?.stepCount ?? 0,
|
|
1393
1903
|
chunkCount: finishMeta?.chunkCount ?? 0,
|
|
1394
1904
|
toolCallCount: finishMeta?.toolCallCount ?? 0,
|
|
1395
1905
|
textChars: finishMeta?.textChars ?? content.length,
|
|
1396
1906
|
completionSignal: completionState.signal,
|
|
1907
|
+
turnOutcome,
|
|
1397
1908
|
});
|
|
1398
1909
|
// Check for discovery completion — parse and strip the tag before storing
|
|
1399
1910
|
if (this.discoveryMode && content.includes('<discovery_complete>')) {
|
|
@@ -1402,36 +1913,28 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1402
1913
|
displayContent = content
|
|
1403
1914
|
.replace(/<discovery_complete>[\s\S]*?<\/discovery_complete>/g, '')
|
|
1404
1915
|
.trim();
|
|
1916
|
+
finalDisplayContent = displayContent;
|
|
1405
1917
|
if (parsed && this.onDiscoveryComplete) {
|
|
1406
1918
|
this.discoveryMode = false;
|
|
1407
1919
|
this.onDiscoveryComplete(parsed);
|
|
1408
1920
|
log.info('Discovery completed', { name: parsed.name });
|
|
1409
1921
|
}
|
|
1410
1922
|
}
|
|
1923
|
+
this.appendTurnJournalEntry('turn_finished', `Turn ended with ${turnOutcome}.`, {
|
|
1924
|
+
finishReason: finishMeta?.finishReason,
|
|
1925
|
+
rawFinishReason: finishMeta?.rawFinishReason,
|
|
1926
|
+
lastContentKind: finishMeta?.lastContentKind ?? this.currentLastContentKind,
|
|
1927
|
+
turnOutcome,
|
|
1928
|
+
textChars: finishMeta?.textChars ?? displayContent.length,
|
|
1929
|
+
toolCallCount: finishMeta?.toolCallCount ?? 0,
|
|
1930
|
+
completionSignal: completionState.signal,
|
|
1931
|
+
finalContent: this.truncateResumeText(displayContent, 2_000),
|
|
1932
|
+
});
|
|
1933
|
+
this.pruneTurnJournals(convId);
|
|
1411
1934
|
const guardFailure = this.evaluateCompletionGuard(displayContent, finishMeta, completionState);
|
|
1412
1935
|
if (guardFailure) {
|
|
1413
1936
|
completionGuardFailure = guardFailure;
|
|
1414
|
-
|
|
1415
|
-
attempt: attemptNumber,
|
|
1416
|
-
partialContent: fullContent || displayContent,
|
|
1417
|
-
completionState,
|
|
1418
|
-
finishMeta,
|
|
1419
|
-
}));
|
|
1420
|
-
if (nextRetryContext) {
|
|
1421
|
-
log.info('completion_retry_context_prepared', {
|
|
1422
|
-
turnId,
|
|
1423
|
-
attempt: attemptNumber,
|
|
1424
|
-
conversationId: convId,
|
|
1425
|
-
finishReason: finishMeta?.finishReason,
|
|
1426
|
-
rawFinishReason: finishMeta?.rawFinishReason,
|
|
1427
|
-
hasPartialContent: (fullContent || displayContent).trim().length > 0,
|
|
1428
|
-
progressEntries: completionState.continuity.progressLedger.length,
|
|
1429
|
-
taskCheckpoints: completionState.continuity.taskCheckpoints.length,
|
|
1430
|
-
hasBrowserState: Boolean(completionState.continuity.browserState.currentUrl
|
|
1431
|
-
|| completionState.continuity.browserState.activeTabId
|
|
1432
|
-
|| completionState.continuity.browserState.tabs?.length),
|
|
1433
|
-
});
|
|
1434
|
-
}
|
|
1937
|
+
finalDisplayContent = displayContent;
|
|
1435
1938
|
this.emitCompletionTrace('guard_triggered', guardFailure.message, guardFailure.signal, 'warn');
|
|
1436
1939
|
log.warn('completion_guard_triggered', {
|
|
1437
1940
|
turnId,
|
|
@@ -1439,11 +1942,13 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1439
1942
|
conversationId: convId,
|
|
1440
1943
|
finishReason: finishMeta?.finishReason,
|
|
1441
1944
|
rawFinishReason: finishMeta?.rawFinishReason,
|
|
1945
|
+
lastContentKind: finishMeta?.lastContentKind,
|
|
1442
1946
|
stepCount: finishMeta?.stepCount ?? 0,
|
|
1443
1947
|
chunkCount: finishMeta?.chunkCount ?? 0,
|
|
1444
1948
|
toolCallCount: finishMeta?.toolCallCount ?? 0,
|
|
1445
1949
|
textChars: finishMeta?.textChars ?? displayContent.length,
|
|
1446
1950
|
completionSignal: completionState.signal,
|
|
1951
|
+
turnOutcome,
|
|
1447
1952
|
});
|
|
1448
1953
|
return;
|
|
1449
1954
|
}
|
|
@@ -1452,7 +1957,11 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1452
1957
|
if (failedAttemptMessageIds.length > 0) {
|
|
1453
1958
|
const deleted = this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
1454
1959
|
if (deleted > 0) {
|
|
1455
|
-
log.info('Cleaned up failed attempt messages', {
|
|
1960
|
+
log.info('Cleaned up failed attempt messages', {
|
|
1961
|
+
deleted,
|
|
1962
|
+
convId,
|
|
1963
|
+
attempt: attemptNumber,
|
|
1964
|
+
});
|
|
1456
1965
|
}
|
|
1457
1966
|
}
|
|
1458
1967
|
const cerebrumMessage = this.conversations.appendMessage(convId, 'cerebrum', displayContent, visibleToolCalls?.length ? { toolCalls: visibleToolCalls } : undefined);
|
|
@@ -1486,10 +1995,74 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1486
1995
|
await this.awaitStreamAttempt(streamPromise, abortController);
|
|
1487
1996
|
const completionFailure = completionGuardFailure;
|
|
1488
1997
|
if (completionFailure !== null) {
|
|
1998
|
+
const turnOutcome = this.classifyTurnOutcome(finalDisplayContent, attemptFinishMeta, completionState);
|
|
1489
1999
|
const completionSignal = completionFailure.signal;
|
|
2000
|
+
const recoveryRequest = this.buildRecoveryRequest({
|
|
2001
|
+
cause: 'completion',
|
|
2002
|
+
attempt: attemptNumber,
|
|
2003
|
+
partialContent: fullContent || finalDisplayContent,
|
|
2004
|
+
completionState,
|
|
2005
|
+
turnOutcome,
|
|
2006
|
+
latestUserMessage,
|
|
2007
|
+
completionRetryCount,
|
|
2008
|
+
finishMeta: attemptFinishMeta,
|
|
2009
|
+
});
|
|
2010
|
+
const { source, assessment } = await this.assessTurnRecovery(recoveryRequest);
|
|
2011
|
+
this.emitRecoveryTrace('completion', source, assessment, assessment.action === 'stop' ? 'warn' : 'info');
|
|
2012
|
+
nextRetryContext = this.buildRetryContextMessage('completion', attemptNumber, assessment.modelMessage, source);
|
|
2013
|
+
this.appendTurnJournalEntry('recovery', assessment.operatorMessage, {
|
|
2014
|
+
cause: 'completion',
|
|
2015
|
+
source,
|
|
2016
|
+
action: assessment.action,
|
|
2017
|
+
diagnosis: assessment.diagnosis,
|
|
2018
|
+
nextStep: assessment.nextStep,
|
|
2019
|
+
completedSteps: assessment.completedSteps,
|
|
2020
|
+
turnOutcome,
|
|
2021
|
+
finishReason: attemptFinishMeta?.finishReason,
|
|
2022
|
+
});
|
|
2023
|
+
this.recordBoundary(completionState.continuity, {
|
|
2024
|
+
kind: 'recovery',
|
|
2025
|
+
action: assessment.action === 'stop' ? 'completion_stop' : 'completion_retry',
|
|
2026
|
+
summary: assessment.diagnosis,
|
|
2027
|
+
stateChanging: false,
|
|
2028
|
+
});
|
|
2029
|
+
log.info('completion_retry_context_prepared', {
|
|
2030
|
+
turnId,
|
|
2031
|
+
attempt: attemptNumber,
|
|
2032
|
+
conversationId: convId,
|
|
2033
|
+
source,
|
|
2034
|
+
action: assessment.action,
|
|
2035
|
+
finishReason: attemptFinishMeta?.finishReason,
|
|
2036
|
+
rawFinishReason: attemptFinishMeta?.rawFinishReason,
|
|
2037
|
+
hasPartialContent: (fullContent || finalDisplayContent).trim().length > 0,
|
|
2038
|
+
progressEntries: completionState.continuity.progressLedger.length,
|
|
2039
|
+
taskCheckpoints: completionState.continuity.taskCheckpoints.length,
|
|
2040
|
+
completedSteps: assessment.completedSteps,
|
|
2041
|
+
nextStep: assessment.nextStep,
|
|
2042
|
+
turnOutcome,
|
|
2043
|
+
lastContentKind: attemptFinishMeta?.lastContentKind,
|
|
2044
|
+
latestBoundary: completionState.continuity.boundaries.at(-1),
|
|
2045
|
+
repetitionSignals: recoveryRequest.repetitionSignals,
|
|
2046
|
+
});
|
|
2047
|
+
if (assessment.action === 'stop') {
|
|
2048
|
+
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
2049
|
+
const diagnosticMessage = this.conversations.appendMessage(convId, 'system', assessment.operatorMessage);
|
|
2050
|
+
this.emit({ type: 'message:system', message: diagnosticMessage });
|
|
2051
|
+
this.emitCompletionTrace('retry_failed', assessment.diagnosis, completionSignal, 'error');
|
|
2052
|
+
this.emit({
|
|
2053
|
+
type: 'error',
|
|
2054
|
+
error: new Error(assessment.diagnosis ||
|
|
2055
|
+
'Turn ended without a valid completion signal or final answer.'),
|
|
2056
|
+
});
|
|
2057
|
+
if (failedAttemptMessageIds.length > 0) {
|
|
2058
|
+
this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
2059
|
+
}
|
|
2060
|
+
loopTerminated = true;
|
|
2061
|
+
break;
|
|
2062
|
+
}
|
|
1490
2063
|
if (completionRetryCount < this.maxCompletionRetries) {
|
|
1491
2064
|
completionRetryCount++;
|
|
1492
|
-
const systemMessage = this.conversations.appendMessage(convId, 'system',
|
|
2065
|
+
const systemMessage = this.conversations.appendMessage(convId, 'system', assessment.operatorMessage);
|
|
1493
2066
|
attemptMessageIds.push(systemMessage.id);
|
|
1494
2067
|
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
1495
2068
|
this.emit({ type: 'message:system', message: systemMessage });
|
|
@@ -1498,12 +2071,15 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1498
2071
|
continue;
|
|
1499
2072
|
}
|
|
1500
2073
|
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
1501
|
-
const diagnosticMessage = this.conversations.appendMessage(convId, 'system',
|
|
2074
|
+
const diagnosticMessage = this.conversations.appendMessage(convId, 'system', source === 'cerebellum'
|
|
2075
|
+
? '[Cerebellum] The turn ended repeatedly without a valid completion signal or final answer.'
|
|
2076
|
+
: '[System fallback] The turn ended repeatedly without a valid completion signal or final answer.');
|
|
1502
2077
|
this.emit({ type: 'message:system', message: diagnosticMessage });
|
|
1503
|
-
this.emitCompletionTrace('retry_failed', `Completion retries exhausted after ${completionRetryCount}/${this.maxCompletionRetries}: ${completionFailure.message}`, completionSignal, 'error');
|
|
2078
|
+
this.emitCompletionTrace('retry_failed', `Completion retries exhausted after ${completionRetryCount}/${this.maxCompletionRetries}: ${assessment.diagnosis || completionFailure.message}`, completionSignal, 'error');
|
|
1504
2079
|
this.emit({
|
|
1505
2080
|
type: 'error',
|
|
1506
|
-
error: new Error(
|
|
2081
|
+
error: new Error(assessment.diagnosis ||
|
|
2082
|
+
'Turn ended without a valid completion signal or final answer.'),
|
|
1507
2083
|
});
|
|
1508
2084
|
// Clean up all failed attempt messages on exhaustion
|
|
1509
2085
|
if (failedAttemptMessageIds.length > 0) {
|
|
@@ -1519,21 +2095,17 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1519
2095
|
const failureState = this.getStreamState();
|
|
1520
2096
|
this.stopStreamWatchdog();
|
|
1521
2097
|
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
completionState,
|
|
1534
|
-
}));
|
|
1535
|
-
// Inject nudge message and retry via the loop
|
|
1536
|
-
const systemMessage = this.conversations.appendMessage(convId, 'system', '[Cerebellum] You stopped mid-response. Continue from where you left off.');
|
|
2098
|
+
const recoveryDecision = this.pendingRecoveryDecision;
|
|
2099
|
+
this.pendingRecoveryDecision = null;
|
|
2100
|
+
const stallRecovery = recoveryDecision;
|
|
2101
|
+
const isRecoveryRetryAbort = abortController.signal.aborted &&
|
|
2102
|
+
stallRecovery !== null &&
|
|
2103
|
+
stallRecovery.assessment.action === 'retry' &&
|
|
2104
|
+
this.streamNudgeCount > stallRetryCountAtStart &&
|
|
2105
|
+
this.streamNudgeCount <= this.maxNudgeRetries;
|
|
2106
|
+
if (isRecoveryRetryAbort && stallRecovery) {
|
|
2107
|
+
nextRetryContext = this.buildRetryContextMessage('stall', attemptNumber, stallRecovery.assessment.modelMessage, stallRecovery.source);
|
|
2108
|
+
const systemMessage = this.conversations.appendMessage(convId, 'system', stallRecovery.assessment.operatorMessage);
|
|
1537
2109
|
attemptMessageIds.push(systemMessage.id);
|
|
1538
2110
|
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
1539
2111
|
this.emit({ type: 'message:system', message: systemMessage });
|
|
@@ -1541,6 +2113,18 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1541
2113
|
nextRetryCause = 'stall';
|
|
1542
2114
|
continue; // retry loop
|
|
1543
2115
|
}
|
|
2116
|
+
if (abortController.signal.aborted &&
|
|
2117
|
+
stallRecovery !== null &&
|
|
2118
|
+
stallRecovery.assessment.action === 'stop') {
|
|
2119
|
+
const systemMessage = this.conversations.appendMessage(convId, 'system', stallRecovery.assessment.operatorMessage);
|
|
2120
|
+
this.emit({ type: 'message:system', message: systemMessage });
|
|
2121
|
+
this.emit({ type: 'error', error: new Error(stallRecovery.assessment.diagnosis) });
|
|
2122
|
+
if (failedAttemptMessageIds.length > 0) {
|
|
2123
|
+
this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
2124
|
+
}
|
|
2125
|
+
loopTerminated = true;
|
|
2126
|
+
break;
|
|
2127
|
+
}
|
|
1544
2128
|
// Check if Cerebellum dropped mid-stream
|
|
1545
2129
|
if (this.cerebellum && !this.cerebellum.isConnected() && abortController.signal.aborted) {
|
|
1546
2130
|
const err = new Error('Cerebellum disconnected during active response. Restart it with: docker compose up -d cerebellum');
|
|
@@ -1553,6 +2137,16 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1553
2137
|
break;
|
|
1554
2138
|
}
|
|
1555
2139
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
2140
|
+
this.appendTurnJournalEntry('turn_error', `Turn attempt ${attemptNumber} failed: ${err.message}`, {
|
|
2141
|
+
retryCause,
|
|
2142
|
+
stallRetryCount: this.streamNudgeCount,
|
|
2143
|
+
completionRetryCount,
|
|
2144
|
+
phase: failureState.phase,
|
|
2145
|
+
activeToolName: failureState.activeToolName,
|
|
2146
|
+
activeToolCallId: failureState.activeToolCallId,
|
|
2147
|
+
error: err.message,
|
|
2148
|
+
});
|
|
2149
|
+
this.pruneTurnJournals(convId);
|
|
1556
2150
|
if (retryCause === 'completion') {
|
|
1557
2151
|
this.emitCompletionTrace('retry_failed', `Completion retry attempt ${attemptNumber} failed: ${err.message}`, completionState.signal, 'error');
|
|
1558
2152
|
}
|