@cereworker/core 26.330.1 → 26.330.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/events.d.ts +21 -1
- package/dist/events.d.ts.map +1 -1
- package/dist/events.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/orchestrator.d.ts +23 -6
- package/dist/orchestrator.d.ts.map +1 -1
- package/dist/orchestrator.js +572 -146
- package/dist/orchestrator.js.map +1 -1
- package/dist/system-prompt.d.ts.map +1 -1
- package/dist/system-prompt.js +4 -3
- package/dist/system-prompt.js.map +1 -1
- package/dist/types.d.ts +58 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -2
package/dist/orchestrator.js
CHANGED
|
@@ -11,8 +11,26 @@ import { ToolRuntime, } from './tool-runtime.js';
|
|
|
11
11
|
const log = createLogger('orchestrator');
|
|
12
12
|
const TASK_COMPLETE_TOOL = 'task_complete';
|
|
13
13
|
const TASK_BLOCKED_TOOL = 'task_blocked';
|
|
14
|
-
const
|
|
15
|
-
const
|
|
14
|
+
const TASK_CHECKPOINT_TOOL = 'task_checkpoint';
|
|
15
|
+
const INTERNAL_TASK_TOOL_NAMES = new Set([TASK_COMPLETE_TOOL, TASK_BLOCKED_TOOL, TASK_CHECKPOINT_TOOL]);
|
|
16
|
+
const SYSTEM_FALLBACK_COMPLETION_PROMPT = '[System fallback] The last turn ended without a final answer. Continue from the last verified state and end by calling task_complete or task_blocked before your final answer.';
|
|
17
|
+
const SYSTEM_FALLBACK_STALL_PROMPT = '[System fallback] The stalled turn is being retried from the last verified state.';
|
|
18
|
+
const DEBUG_TOOL_OUTPUT_MAX_CHARS = 8_000;
|
|
19
|
+
const DEBUG_TOOL_STRUCTURED_MAX_CHARS = 16_000;
|
|
20
|
+
const READ_ONLY_TOOL_NAMES = new Set([
|
|
21
|
+
'browserGetText',
|
|
22
|
+
'browserGetUrl',
|
|
23
|
+
'browserListTabs',
|
|
24
|
+
'browserWait',
|
|
25
|
+
'browserEval',
|
|
26
|
+
'readFile',
|
|
27
|
+
'listDirectory',
|
|
28
|
+
'searchFiles',
|
|
29
|
+
'glob',
|
|
30
|
+
'memory_read',
|
|
31
|
+
'webSearch',
|
|
32
|
+
'httpFetch',
|
|
33
|
+
]);
|
|
16
34
|
export class Orchestrator extends TypedEventEmitter {
|
|
17
35
|
conversations;
|
|
18
36
|
cerebrum = null;
|
|
@@ -49,6 +67,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
49
67
|
lastStreamActivityAt = 0;
|
|
50
68
|
streamWatchdog = null;
|
|
51
69
|
streamNudgeCount = 0;
|
|
70
|
+
streamDeferredUntil = 0;
|
|
52
71
|
streamStallThreshold = 30_000;
|
|
53
72
|
maxNudgeRetries = 2;
|
|
54
73
|
maxCompletionRetries = 2;
|
|
@@ -56,6 +75,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
56
75
|
activeToolCall = null;
|
|
57
76
|
currentStreamTurn = null;
|
|
58
77
|
currentAttemptCompletionState = null;
|
|
78
|
+
currentPartialContent = '';
|
|
79
|
+
pendingRecoveryDecision = null;
|
|
59
80
|
streamAbortGraceMs = 1_000;
|
|
60
81
|
taskConversations = new Map();
|
|
61
82
|
taskRunning = new Set();
|
|
@@ -155,12 +176,26 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
155
176
|
},
|
|
156
177
|
execute: async (args) => this.recordCompletionSignal('blocked', args),
|
|
157
178
|
});
|
|
179
|
+
this.internalTools.set(TASK_CHECKPOINT_TOOL, {
|
|
180
|
+
description: 'Record a completed or in-progress milestone during a multi-step task. Use this after each major verified step so retries can resume from the right place.',
|
|
181
|
+
parameters: {
|
|
182
|
+
type: 'object',
|
|
183
|
+
properties: {
|
|
184
|
+
step: { type: 'string', description: 'Short milestone name, such as "profile continuity checked"' },
|
|
185
|
+
status: { type: 'string', enum: ['done', 'in_progress'], description: 'Whether the milestone is done or currently in progress' },
|
|
186
|
+
evidence: { type: 'string', description: 'Concrete evidence showing what happened at this milestone' },
|
|
187
|
+
},
|
|
188
|
+
required: ['step', 'status', 'evidence'],
|
|
189
|
+
additionalProperties: false,
|
|
190
|
+
},
|
|
191
|
+
execute: async (args) => this.recordTaskCheckpoint(args),
|
|
192
|
+
});
|
|
158
193
|
}
|
|
159
194
|
getAllTools() {
|
|
160
195
|
return new Map([...this.tools, ...this.internalTools]);
|
|
161
196
|
}
|
|
162
197
|
isInternalTaskSignalTool(name) {
|
|
163
|
-
return
|
|
198
|
+
return INTERNAL_TASK_TOOL_NAMES.has(name.trim() || name);
|
|
164
199
|
}
|
|
165
200
|
async recordCompletionSignal(signal, args) {
|
|
166
201
|
const state = this.currentAttemptCompletionState;
|
|
@@ -185,9 +220,11 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
185
220
|
isError: true,
|
|
186
221
|
};
|
|
187
222
|
}
|
|
188
|
-
|
|
223
|
+
const hasVerifiedProgress = state.successfulExternalToolCount > 0
|
|
224
|
+
|| state.continuity.progressLedger.some((entry) => entry.source === 'tool' && !entry.isError);
|
|
225
|
+
if (!hasVerifiedProgress) {
|
|
189
226
|
return {
|
|
190
|
-
output: 'task_complete requires at least one successful external tool result in this
|
|
227
|
+
output: 'task_complete requires at least one successful external tool result in this turn.',
|
|
191
228
|
isError: true,
|
|
192
229
|
};
|
|
193
230
|
}
|
|
@@ -221,6 +258,56 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
221
258
|
},
|
|
222
259
|
};
|
|
223
260
|
}
|
|
261
|
+
async recordTaskCheckpoint(args) {
|
|
262
|
+
const state = this.currentAttemptCompletionState;
|
|
263
|
+
if (!state) {
|
|
264
|
+
return {
|
|
265
|
+
output: 'No active turn is available for task checkpoint tracking.',
|
|
266
|
+
isError: true,
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
const step = String(args.step ?? '').trim();
|
|
270
|
+
const evidence = String(args.evidence ?? '').trim();
|
|
271
|
+
const statusValue = String(args.status ?? '').trim();
|
|
272
|
+
const status = statusValue === 'done' || statusValue === 'in_progress'
|
|
273
|
+
? statusValue
|
|
274
|
+
: null;
|
|
275
|
+
if (!step) {
|
|
276
|
+
return {
|
|
277
|
+
output: 'A non-empty step field is required.',
|
|
278
|
+
isError: true,
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
if (!status) {
|
|
282
|
+
return {
|
|
283
|
+
output: 'status must be either "done" or "in_progress".',
|
|
284
|
+
isError: true,
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
if (!evidence) {
|
|
288
|
+
return {
|
|
289
|
+
output: 'A non-empty evidence field is required.',
|
|
290
|
+
isError: true,
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
const checkpoint = this.recordCheckpoint(state.continuity, step, status, evidence);
|
|
294
|
+
log.info('task_checkpoint_recorded', {
|
|
295
|
+
turnId: this.currentStreamTurn?.turnId,
|
|
296
|
+
attempt: this.currentStreamTurn?.attempt,
|
|
297
|
+
conversationId: this.currentStreamTurn?.conversationId,
|
|
298
|
+
step,
|
|
299
|
+
status,
|
|
300
|
+
evidence,
|
|
301
|
+
});
|
|
302
|
+
return {
|
|
303
|
+
output: `Checkpoint recorded: ${checkpoint.summary}`,
|
|
304
|
+
isError: false,
|
|
305
|
+
metadata: {
|
|
306
|
+
internal: true,
|
|
307
|
+
checkpoint,
|
|
308
|
+
},
|
|
309
|
+
};
|
|
310
|
+
}
|
|
224
311
|
registerTool(name, tool) {
|
|
225
312
|
if (this.internalTools.has(name)) {
|
|
226
313
|
throw new Error(`Tool name ${name} is reserved for internal task signaling`);
|
|
@@ -599,6 +686,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
599
686
|
resetStreamState() {
|
|
600
687
|
this.streamPhase = 'idle';
|
|
601
688
|
this.activeToolCall = null;
|
|
689
|
+
this.streamDeferredUntil = 0;
|
|
690
|
+
this.currentPartialContent = '';
|
|
602
691
|
}
|
|
603
692
|
getStreamDiagnostics(elapsedSeconds) {
|
|
604
693
|
return {
|
|
@@ -689,106 +778,201 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
689
778
|
}
|
|
690
779
|
this.emit({ type: 'cerebrum:completion', ...payload });
|
|
691
780
|
}
|
|
692
|
-
createAttemptCompletionState() {
|
|
781
|
+
createAttemptCompletionState(continuity) {
|
|
693
782
|
return {
|
|
694
783
|
signal: 'none',
|
|
695
784
|
evidence: '',
|
|
696
785
|
successfulExternalToolCount: 0,
|
|
697
786
|
externalToolCallCount: 0,
|
|
698
787
|
internalToolCallCount: 0,
|
|
699
|
-
|
|
788
|
+
continuity,
|
|
789
|
+
};
|
|
790
|
+
}
|
|
791
|
+
createTurnContinuityState() {
|
|
792
|
+
return {
|
|
793
|
+
progressLedger: [],
|
|
794
|
+
taskCheckpoints: [],
|
|
795
|
+
browserState: {},
|
|
700
796
|
};
|
|
701
797
|
}
|
|
702
|
-
|
|
798
|
+
buildRecoveryRequest(params) {
|
|
703
799
|
const partialContent = this.truncateResumeText(params.partialContent, 600);
|
|
704
|
-
const
|
|
705
|
-
if (!partialContent && recentExternalToolSummaries.length === 0 && !params.activeToolName) {
|
|
706
|
-
return null;
|
|
707
|
-
}
|
|
800
|
+
const continuity = params.completionState.continuity;
|
|
708
801
|
return {
|
|
802
|
+
conversationId: this.currentStreamTurn?.conversationId ?? '',
|
|
803
|
+
turnId: this.currentStreamTurn?.turnId ?? '',
|
|
709
804
|
attempt: params.attempt,
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
805
|
+
cause: params.cause,
|
|
806
|
+
phase: this.streamPhase,
|
|
807
|
+
activeToolName: this.activeToolCall?.name,
|
|
808
|
+
activeToolCallId: this.activeToolCall?.id,
|
|
809
|
+
stallRetryCount: this.streamNudgeCount,
|
|
810
|
+
completionRetryCount: params.completionRetryCount ?? 0,
|
|
811
|
+
finishReason: params.finishMeta?.finishReason ?? params.finishMeta?.stepFinishReasons.at(-1),
|
|
812
|
+
elapsedSeconds: params.elapsedSeconds,
|
|
713
813
|
partialContent: partialContent || undefined,
|
|
714
|
-
|
|
814
|
+
latestUserMessage: params.latestUserMessage ? this.truncateResumeText(params.latestUserMessage, 600) : undefined,
|
|
815
|
+
progressEntries: continuity.progressLedger.slice(-50).map((entry) => ({ ...entry })),
|
|
816
|
+
taskCheckpoints: continuity.taskCheckpoints.map((checkpoint) => ({ ...checkpoint })),
|
|
817
|
+
browserState: this.cloneBrowserState(continuity.browserState),
|
|
715
818
|
};
|
|
716
819
|
}
|
|
717
|
-
|
|
718
|
-
if (!
|
|
719
|
-
return
|
|
720
|
-
const
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
820
|
+
emitRecoveryTrace(cause, source, assessment, level = 'info') {
|
|
821
|
+
if (!this.currentStreamTurn)
|
|
822
|
+
return;
|
|
823
|
+
const payload = {
|
|
824
|
+
type: 'cerebellum:recovery',
|
|
825
|
+
cause,
|
|
826
|
+
action: assessment.action,
|
|
827
|
+
turnId: this.currentStreamTurn.turnId,
|
|
828
|
+
attempt: this.currentStreamTurn.attempt,
|
|
829
|
+
conversationId: this.currentStreamTurn.conversationId,
|
|
830
|
+
message: assessment.operatorMessage,
|
|
831
|
+
operatorMessage: assessment.operatorMessage,
|
|
832
|
+
diagnosis: assessment.diagnosis,
|
|
833
|
+
nextStep: assessment.nextStep,
|
|
834
|
+
completedSteps: assessment.completedSteps,
|
|
835
|
+
waitSeconds: assessment.waitSeconds,
|
|
836
|
+
source,
|
|
837
|
+
...this.getStreamDiagnostics(),
|
|
838
|
+
};
|
|
839
|
+
switch (level) {
|
|
840
|
+
case 'debug':
|
|
841
|
+
log.debug('cerebellum_recovery', payload);
|
|
842
|
+
break;
|
|
843
|
+
case 'warn':
|
|
844
|
+
log.warn('cerebellum_recovery', payload);
|
|
845
|
+
break;
|
|
846
|
+
case 'error':
|
|
847
|
+
log.error('cerebellum_recovery', payload);
|
|
848
|
+
break;
|
|
849
|
+
default:
|
|
850
|
+
log.info('cerebellum_recovery', payload);
|
|
851
|
+
break;
|
|
735
852
|
}
|
|
736
|
-
|
|
737
|
-
|
|
853
|
+
this.emit(payload);
|
|
854
|
+
}
|
|
855
|
+
async assessTurnRecovery(request) {
|
|
856
|
+
log.debug('turn_recovery_request', {
|
|
857
|
+
turnId: request.turnId,
|
|
858
|
+
attempt: request.attempt,
|
|
859
|
+
conversationId: request.conversationId,
|
|
860
|
+
cause: request.cause,
|
|
861
|
+
phase: request.phase,
|
|
862
|
+
activeToolName: request.activeToolName,
|
|
863
|
+
activeToolCallId: request.activeToolCallId,
|
|
864
|
+
stallRetryCount: request.stallRetryCount,
|
|
865
|
+
completionRetryCount: request.completionRetryCount,
|
|
866
|
+
finishReason: request.finishReason,
|
|
867
|
+
elapsedSeconds: request.elapsedSeconds,
|
|
868
|
+
hasPartialContent: Boolean(request.partialContent),
|
|
869
|
+
latestUserMessage: request.latestUserMessage ? this.truncateResumeText(request.latestUserMessage, 300) : '',
|
|
870
|
+
browserState: request.browserState,
|
|
871
|
+
progressEntries: request.progressEntries,
|
|
872
|
+
taskCheckpoints: request.taskCheckpoints,
|
|
873
|
+
});
|
|
874
|
+
if (this.cerebellum?.isConnected() && this.cerebellum.assessTurnRecovery) {
|
|
875
|
+
try {
|
|
876
|
+
const assessment = await this.cerebellum.assessTurnRecovery(request);
|
|
877
|
+
if (assessment) {
|
|
878
|
+
if (request.cause === 'completion' && assessment.action === 'wait') {
|
|
879
|
+
return {
|
|
880
|
+
source: 'cerebellum',
|
|
881
|
+
assessment: {
|
|
882
|
+
...assessment,
|
|
883
|
+
action: 'retry',
|
|
884
|
+
waitSeconds: undefined,
|
|
885
|
+
},
|
|
886
|
+
};
|
|
887
|
+
}
|
|
888
|
+
return { source: 'cerebellum', assessment };
|
|
889
|
+
}
|
|
890
|
+
}
|
|
891
|
+
catch (error) {
|
|
892
|
+
log.warn('Turn recovery assessment failed', {
|
|
893
|
+
turnId: request.turnId,
|
|
894
|
+
attempt: request.attempt,
|
|
895
|
+
conversationId: request.conversationId,
|
|
896
|
+
cause: request.cause,
|
|
897
|
+
error: error instanceof Error ? error.message : String(error),
|
|
898
|
+
});
|
|
899
|
+
}
|
|
738
900
|
}
|
|
739
901
|
return {
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
content: lines.join('\n'),
|
|
743
|
-
timestamp: 0,
|
|
744
|
-
metadata: {
|
|
745
|
-
transient: true,
|
|
746
|
-
source: 'watchdog-resume',
|
|
747
|
-
},
|
|
902
|
+
source: 'fallback',
|
|
903
|
+
assessment: this.buildFallbackRecoveryAssessment(request),
|
|
748
904
|
};
|
|
749
905
|
}
|
|
750
|
-
|
|
751
|
-
const
|
|
752
|
-
const
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
906
|
+
deriveCompletedSteps(request) {
|
|
907
|
+
const completed = new Set();
|
|
908
|
+
for (const checkpoint of request.taskCheckpoints) {
|
|
909
|
+
if (checkpoint.status === 'done') {
|
|
910
|
+
completed.add(checkpoint.summary);
|
|
911
|
+
}
|
|
756
912
|
}
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
}
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
913
|
+
for (const entry of request.progressEntries) {
|
|
914
|
+
if (entry.source === 'tool' && entry.stateChanging && !entry.isError) {
|
|
915
|
+
completed.add(entry.summary);
|
|
916
|
+
}
|
|
917
|
+
}
|
|
918
|
+
return Array.from(completed).slice(-10);
|
|
919
|
+
}
|
|
920
|
+
buildFallbackRecoveryAssessment(request, options) {
|
|
921
|
+
const completedSteps = this.deriveCompletedSteps(request);
|
|
922
|
+
const browserHints = [];
|
|
923
|
+
if (request.browserState.currentUrl)
|
|
924
|
+
browserHints.push(`Current URL: ${request.browserState.currentUrl}`);
|
|
925
|
+
if (request.browserState.activeTabId)
|
|
926
|
+
browserHints.push(`Active tab: ${request.browserState.activeTabId}`);
|
|
927
|
+
const diagnosis = options?.reason
|
|
928
|
+
?? (request.cause === 'stall'
|
|
929
|
+
? `Recovery guidance is unavailable while the stream is stalled in ${this.describeStreamLocation(request.phase, request.activeToolName)}.`
|
|
930
|
+
: `Recovery guidance is unavailable after the turn ended with ${request.finishReason ?? 'no final answer'}.`);
|
|
931
|
+
const nextStep = request.cause === 'stall'
|
|
932
|
+
? 'Resume from the last verified browser state and continue with the next unfinished step.'
|
|
933
|
+
: 'Use the verified progress below to continue from the next unfinished step and avoid repeating confirmed work.';
|
|
767
934
|
const lines = [
|
|
768
|
-
'[
|
|
769
|
-
|
|
770
|
-
'
|
|
771
|
-
'Do NOT repeat these steps. Start from the NEXT action after the last confirmed result.',
|
|
772
|
-
'Continue from that state, then either finish the task or report a concrete blocker. End by calling task_complete or task_blocked before your final answer.',
|
|
935
|
+
'[System fallback recovery]',
|
|
936
|
+
diagnosis,
|
|
937
|
+
'The failed attempt tool history has been removed; rely on this verified summary instead.',
|
|
773
938
|
];
|
|
774
|
-
if (
|
|
775
|
-
lines.push('', '
|
|
776
|
-
for (const
|
|
777
|
-
|
|
778
|
-
lines.push(`- ${summary.toolName}: ${prefix} ${summary.outputPreview}`);
|
|
779
|
-
}
|
|
939
|
+
if (completedSteps.length > 0) {
|
|
940
|
+
lines.push('', 'Completed steps:');
|
|
941
|
+
for (const step of completedSteps)
|
|
942
|
+
lines.push(`- ${step}`);
|
|
780
943
|
}
|
|
781
|
-
if (
|
|
782
|
-
lines.push('', '
|
|
944
|
+
if (browserHints.length > 0) {
|
|
945
|
+
lines.push('', 'Last known browser state:');
|
|
946
|
+
for (const hint of browserHints)
|
|
947
|
+
lines.push(`- ${hint}`);
|
|
783
948
|
}
|
|
949
|
+
if (request.partialContent) {
|
|
950
|
+
lines.push('', 'Partial assistant text from the failed attempt:', request.partialContent);
|
|
951
|
+
}
|
|
952
|
+
lines.push('', `Next step: ${nextStep}`);
|
|
953
|
+
lines.push('Only repeat a completed action if the current page state clearly contradicts this summary.');
|
|
954
|
+
lines.push('End your final answer by calling task_complete or task_blocked.');
|
|
784
955
|
return {
|
|
785
|
-
|
|
956
|
+
action: options?.action ?? 'retry',
|
|
957
|
+
operatorMessage: request.cause === 'stall'
|
|
958
|
+
? SYSTEM_FALLBACK_STALL_PROMPT
|
|
959
|
+
: SYSTEM_FALLBACK_COMPLETION_PROMPT,
|
|
960
|
+
modelMessage: lines.join('\n'),
|
|
961
|
+
diagnosis,
|
|
962
|
+
nextStep,
|
|
963
|
+
completedSteps,
|
|
964
|
+
};
|
|
965
|
+
}
|
|
966
|
+
buildRetryContextMessage(cause, attempt, modelMessage, source) {
|
|
967
|
+
return {
|
|
968
|
+
id: `system:${cause}-retry:${attempt}`,
|
|
786
969
|
role: 'system',
|
|
787
|
-
content:
|
|
970
|
+
content: modelMessage,
|
|
788
971
|
timestamp: 0,
|
|
789
972
|
metadata: {
|
|
790
973
|
transient: true,
|
|
791
|
-
source: 'completion-resume',
|
|
974
|
+
source: cause === 'stall' ? 'watchdog-resume' : 'completion-resume',
|
|
975
|
+
recoverySource: source,
|
|
792
976
|
},
|
|
793
977
|
};
|
|
794
978
|
}
|
|
@@ -803,16 +987,171 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
803
987
|
return normalized;
|
|
804
988
|
return `${normalized.slice(0, Math.max(0, maxChars - 1)).trimEnd()}…`;
|
|
805
989
|
}
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
990
|
+
serializeDebugValue(value, maxChars) {
|
|
991
|
+
const raw = typeof value === 'string'
|
|
992
|
+
? value
|
|
993
|
+
: JSON.stringify(value, null, 2) ?? String(value);
|
|
994
|
+
if (raw.length <= maxChars) {
|
|
995
|
+
return { value: raw, truncated: false };
|
|
996
|
+
}
|
|
997
|
+
return {
|
|
998
|
+
value: `${raw.slice(0, Math.max(0, maxChars - 1)).trimEnd()}…`,
|
|
999
|
+
truncated: true,
|
|
1000
|
+
};
|
|
1001
|
+
}
|
|
1002
|
+
buildToolDebugPayload(toolCall, result, toolName) {
|
|
1003
|
+
const argsPreview = this.serializeDebugValue(toolCall.args, DEBUG_TOOL_STRUCTURED_MAX_CHARS);
|
|
1004
|
+
if (!result) {
|
|
1005
|
+
return {
|
|
1006
|
+
requestedToolName: toolCall.name,
|
|
1007
|
+
toolName: toolName ?? (toolCall.name.trim() || toolCall.name),
|
|
1008
|
+
toolCallId: toolCall.id,
|
|
1009
|
+
toolArgs: argsPreview.value,
|
|
1010
|
+
debugPayloadTruncated: argsPreview.truncated,
|
|
1011
|
+
};
|
|
1012
|
+
}
|
|
1013
|
+
const outputPreview = this.serializeDebugValue(result.output, DEBUG_TOOL_OUTPUT_MAX_CHARS);
|
|
1014
|
+
const detailsPreview = result.details
|
|
1015
|
+
? this.serializeDebugValue(result.details, DEBUG_TOOL_STRUCTURED_MAX_CHARS)
|
|
1016
|
+
: null;
|
|
1017
|
+
const resumeMetadata = result.metadata && typeof result.metadata === 'object'
|
|
1018
|
+
? (result.metadata.resume ?? null)
|
|
1019
|
+
: null;
|
|
1020
|
+
const resumePreview = resumeMetadata
|
|
1021
|
+
? this.serializeDebugValue(resumeMetadata, DEBUG_TOOL_STRUCTURED_MAX_CHARS)
|
|
1022
|
+
: null;
|
|
1023
|
+
return {
|
|
1024
|
+
requestedToolName: toolCall.name,
|
|
1025
|
+
toolName: toolName ?? (toolCall.name.trim() || toolCall.name),
|
|
1026
|
+
toolCallId: toolCall.id,
|
|
1027
|
+
toolArgs: argsPreview.value,
|
|
1028
|
+
toolOutput: outputPreview.value,
|
|
1029
|
+
toolDetails: detailsPreview?.value ?? null,
|
|
1030
|
+
toolResume: resumePreview?.value ?? null,
|
|
810
1031
|
isError: result.isError,
|
|
1032
|
+
warnings: result.warnings ?? [],
|
|
1033
|
+
truncated: result.truncated ?? false,
|
|
1034
|
+
debugPayloadTruncated: argsPreview.truncated || outputPreview.truncated || Boolean(detailsPreview?.truncated) || Boolean(resumePreview?.truncated),
|
|
1035
|
+
};
|
|
1036
|
+
}
|
|
1037
|
+
recordCheckpoint(continuity, step, status, evidence) {
|
|
1038
|
+
const checkpoint = {
|
|
1039
|
+
step,
|
|
1040
|
+
status,
|
|
1041
|
+
evidence,
|
|
1042
|
+
summary: `${step} (${status}): ${this.truncateResumeText(evidence, 220)}`,
|
|
1043
|
+
};
|
|
1044
|
+
const existingIndex = continuity.taskCheckpoints.findIndex((entry) => entry.step === step);
|
|
1045
|
+
if (existingIndex >= 0) {
|
|
1046
|
+
continuity.taskCheckpoints[existingIndex] = checkpoint;
|
|
1047
|
+
}
|
|
1048
|
+
else {
|
|
1049
|
+
continuity.taskCheckpoints.push(checkpoint);
|
|
1050
|
+
}
|
|
1051
|
+
this.recordProgressEntry(continuity, {
|
|
1052
|
+
source: 'checkpoint',
|
|
1053
|
+
action: 'task_checkpoint',
|
|
1054
|
+
summary: checkpoint.summary,
|
|
1055
|
+
stateChanging: status === 'done',
|
|
1056
|
+
isError: false,
|
|
1057
|
+
checkpointStatus: status,
|
|
811
1058
|
});
|
|
812
|
-
|
|
813
|
-
|
|
1059
|
+
return checkpoint;
|
|
1060
|
+
}
|
|
1061
|
+
recordAttemptToolProgress(completionState, toolName, result) {
|
|
1062
|
+
const continuity = completionState.continuity;
|
|
1063
|
+
const entry = this.createProgressEntry(toolName, result);
|
|
1064
|
+
if (!entry)
|
|
1065
|
+
return;
|
|
1066
|
+
this.recordProgressEntry(continuity, entry);
|
|
1067
|
+
this.updateBrowserState(continuity.browserState, result);
|
|
1068
|
+
}
|
|
1069
|
+
createProgressEntry(toolName, result) {
|
|
1070
|
+
const resume = this.getBrowserResumeMetadata(result);
|
|
1071
|
+
if (resume?.summary) {
|
|
1072
|
+
return {
|
|
1073
|
+
source: 'tool',
|
|
1074
|
+
toolName,
|
|
1075
|
+
action: resume.action ?? toolName,
|
|
1076
|
+
summary: this.truncateResumeText(resume.summary, 220),
|
|
1077
|
+
url: resume.url,
|
|
1078
|
+
tabId: resume.tabId ?? resume.activeTabId,
|
|
1079
|
+
stateChanging: resume.stateChanging ?? this.isLikelyStateChangingTool(toolName),
|
|
1080
|
+
isError: result.isError,
|
|
1081
|
+
};
|
|
1082
|
+
}
|
|
1083
|
+
const outputPreview = this.formatToolOutputPreview(result.output);
|
|
1084
|
+
if (!outputPreview)
|
|
1085
|
+
return null;
|
|
1086
|
+
return {
|
|
1087
|
+
source: 'tool',
|
|
1088
|
+
toolName,
|
|
1089
|
+
action: toolName,
|
|
1090
|
+
summary: `${toolName}: ${outputPreview}`,
|
|
1091
|
+
stateChanging: this.isLikelyStateChangingTool(toolName),
|
|
1092
|
+
isError: result.isError,
|
|
1093
|
+
};
|
|
1094
|
+
}
|
|
1095
|
+
recordProgressEntry(continuity, entry) {
|
|
1096
|
+
const last = continuity.progressLedger.at(-1);
|
|
1097
|
+
if (last
|
|
1098
|
+
&& entry.source === 'tool'
|
|
1099
|
+
&& last.source === 'tool'
|
|
1100
|
+
&& !entry.stateChanging
|
|
1101
|
+
&& !last.stateChanging
|
|
1102
|
+
&& last.action === entry.action
|
|
1103
|
+
&& last.summary === entry.summary
|
|
1104
|
+
&& last.url === entry.url
|
|
1105
|
+
&& last.tabId === entry.tabId) {
|
|
1106
|
+
return;
|
|
1107
|
+
}
|
|
1108
|
+
continuity.progressLedger.push(entry);
|
|
1109
|
+
while (continuity.progressLedger.length > 50) {
|
|
1110
|
+
const removableIndex = continuity.progressLedger.findIndex((candidate) => candidate.source === 'tool' && !candidate.stateChanging);
|
|
1111
|
+
continuity.progressLedger.splice(removableIndex >= 0 ? removableIndex : 0, 1);
|
|
814
1112
|
}
|
|
815
1113
|
}
|
|
1114
|
+
getBrowserResumeMetadata(result) {
|
|
1115
|
+
const metadata = result.metadata;
|
|
1116
|
+
if (!metadata || typeof metadata !== 'object')
|
|
1117
|
+
return null;
|
|
1118
|
+
const resume = metadata.resume;
|
|
1119
|
+
if (!resume || typeof resume !== 'object')
|
|
1120
|
+
return null;
|
|
1121
|
+
return resume;
|
|
1122
|
+
}
|
|
1123
|
+
updateBrowserState(browserState, result) {
|
|
1124
|
+
const resume = this.getBrowserResumeMetadata(result);
|
|
1125
|
+
if (!resume)
|
|
1126
|
+
return;
|
|
1127
|
+
if (resume.url) {
|
|
1128
|
+
browserState.currentUrl = resume.url;
|
|
1129
|
+
}
|
|
1130
|
+
if (resume.activeTabId) {
|
|
1131
|
+
browserState.activeTabId = resume.activeTabId;
|
|
1132
|
+
}
|
|
1133
|
+
else if (resume.tabId && resume.stateChanging) {
|
|
1134
|
+
browserState.activeTabId = resume.tabId;
|
|
1135
|
+
}
|
|
1136
|
+
if (resume.tabs?.length) {
|
|
1137
|
+
browserState.tabs = resume.tabs.map((tab) => ({ ...tab }));
|
|
1138
|
+
const active = resume.tabs.find((tab) => tab.active);
|
|
1139
|
+
if (active) {
|
|
1140
|
+
browserState.activeTabId = active.id;
|
|
1141
|
+
browserState.currentUrl = active.url;
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
}
|
|
1145
|
+
cloneBrowserState(browserState) {
|
|
1146
|
+
return {
|
|
1147
|
+
currentUrl: browserState.currentUrl,
|
|
1148
|
+
activeTabId: browserState.activeTabId,
|
|
1149
|
+
tabs: browserState.tabs?.map((tab) => ({ ...tab })),
|
|
1150
|
+
};
|
|
1151
|
+
}
|
|
1152
|
+
isLikelyStateChangingTool(toolName) {
|
|
1153
|
+
return !READ_ONLY_TOOL_NAMES.has(toolName);
|
|
1154
|
+
}
|
|
816
1155
|
evaluateCompletionGuard(displayContent, finishMeta, completionState) {
|
|
817
1156
|
const trimmedContent = displayContent.trim();
|
|
818
1157
|
const hadExternalToolActivity = completionState.externalToolCallCount > 0;
|
|
@@ -886,10 +1225,15 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
886
1225
|
streamPromise.then(settleResolve, settleReject);
|
|
887
1226
|
});
|
|
888
1227
|
}
|
|
889
|
-
startStreamWatchdog() {
|
|
1228
|
+
startStreamWatchdog(latestUserMessage) {
|
|
890
1229
|
this.stopStreamWatchdog();
|
|
891
1230
|
this.markStreamWaitingModel();
|
|
1231
|
+
this.streamDeferredUntil = 0;
|
|
892
1232
|
this.streamWatchdog = setInterval(() => {
|
|
1233
|
+
if (!this.currentAttemptCompletionState || !this.currentStreamTurn)
|
|
1234
|
+
return;
|
|
1235
|
+
if (this.streamDeferredUntil > Date.now())
|
|
1236
|
+
return;
|
|
893
1237
|
const elapsed = Date.now() - this.lastStreamActivityAt;
|
|
894
1238
|
const stallThresholdMs = this.getCurrentStallThresholdMs();
|
|
895
1239
|
if (elapsed < stallThresholdMs)
|
|
@@ -903,31 +1247,60 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
903
1247
|
this.emitWatchdog('stalled', `Stalled after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'warn', elapsedSeconds });
|
|
904
1248
|
this.emit({ type: 'cerebrum:stall', ...diagnostics });
|
|
905
1249
|
if (!this.cerebellum?.isConnected()) {
|
|
906
|
-
// Cerebellum dropped mid-stream — abort the current turn
|
|
907
1250
|
this.emitWatchdog('abort_issued', 'Cerebellum disconnected during an active stream; aborting the turn.', { level: 'warn', elapsedSeconds });
|
|
908
1251
|
this.abortController?.abort();
|
|
909
1252
|
return;
|
|
910
1253
|
}
|
|
911
1254
|
this._nudgeInFlight = true;
|
|
912
|
-
const doNudge = () => {
|
|
913
|
-
this.streamNudgeCount++;
|
|
914
|
-
this.emitWatchdog('nudge_requested', `Cerebellum requested nudge ${this.streamNudgeCount}/${this.maxNudgeRetries} after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'info', elapsedSeconds });
|
|
915
|
-
this.emit({ type: 'cerebrum:stall:nudge', attempt: this.streamNudgeCount, ...diagnostics });
|
|
916
|
-
this.emitWatchdog('abort_issued', `Aborting stalled stream attempt ${this.currentStreamTurn?.attempt ?? 0}.`, { level: 'warn', elapsedSeconds });
|
|
917
|
-
this.abortController?.abort();
|
|
918
|
-
};
|
|
919
1255
|
void (async () => {
|
|
920
1256
|
try {
|
|
921
|
-
const
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
1257
|
+
const request = this.buildRecoveryRequest({
|
|
1258
|
+
cause: 'stall',
|
|
1259
|
+
attempt: this.currentStreamTurn.attempt,
|
|
1260
|
+
partialContent: this.currentPartialContent,
|
|
1261
|
+
completionState: this.currentAttemptCompletionState,
|
|
1262
|
+
latestUserMessage,
|
|
1263
|
+
elapsedSeconds,
|
|
1264
|
+
});
|
|
1265
|
+
const { source, assessment } = await this.assessTurnRecovery(request);
|
|
1266
|
+
this.emitRecoveryTrace('stall', source, assessment, assessment.action === 'stop' ? 'warn' : 'info');
|
|
1267
|
+
if (assessment.action === 'wait') {
|
|
1268
|
+
const waitSeconds = Math.max(15, assessment.waitSeconds ?? this.streamStallThreshold / 1000);
|
|
1269
|
+
this.streamDeferredUntil = Date.now() + (waitSeconds * 1000);
|
|
1270
|
+
return;
|
|
1271
|
+
}
|
|
1272
|
+
if (assessment.action === 'retry') {
|
|
1273
|
+
this.streamNudgeCount++;
|
|
1274
|
+
this.pendingRecoveryDecision = { cause: 'stall', source, assessment };
|
|
1275
|
+
this.emitWatchdog('nudge_requested', `Cerebellum requested nudge ${this.streamNudgeCount}/${this.maxNudgeRetries} after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'info', elapsedSeconds });
|
|
1276
|
+
this.emit({ type: 'cerebrum:stall:nudge', attempt: this.streamNudgeCount, ...diagnostics });
|
|
1277
|
+
this.emitWatchdog('abort_issued', `Aborting stalled stream attempt ${this.currentStreamTurn?.attempt ?? 0}.`, { level: 'warn', elapsedSeconds });
|
|
1278
|
+
this.abortController?.abort();
|
|
1279
|
+
return;
|
|
926
1280
|
}
|
|
1281
|
+
this.pendingRecoveryDecision = { cause: 'stall', source, assessment };
|
|
1282
|
+
this.emitWatchdog('abort_issued', 'Aborting stalled stream because recovery guidance requested stop.', { level: 'warn', elapsedSeconds });
|
|
1283
|
+
this.abortController?.abort();
|
|
927
1284
|
}
|
|
928
1285
|
catch {
|
|
929
|
-
|
|
930
|
-
|
|
1286
|
+
const request = this.buildRecoveryRequest({
|
|
1287
|
+
cause: 'stall',
|
|
1288
|
+
attempt: this.currentStreamTurn.attempt,
|
|
1289
|
+
partialContent: this.currentPartialContent,
|
|
1290
|
+
completionState: this.currentAttemptCompletionState,
|
|
1291
|
+
latestUserMessage,
|
|
1292
|
+
elapsedSeconds,
|
|
1293
|
+
});
|
|
1294
|
+
const assessment = this.buildFallbackRecoveryAssessment(request, {
|
|
1295
|
+
reason: `Recovery assessment failed after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`,
|
|
1296
|
+
});
|
|
1297
|
+
this.pendingRecoveryDecision = { cause: 'stall', source: 'fallback', assessment };
|
|
1298
|
+
this.emitRecoveryTrace('stall', 'fallback', assessment, 'warn');
|
|
1299
|
+
this.streamNudgeCount++;
|
|
1300
|
+
this.emitWatchdog('nudge_requested', `Fallback retry ${this.streamNudgeCount}/${this.maxNudgeRetries} after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'info', elapsedSeconds });
|
|
1301
|
+
this.emit({ type: 'cerebrum:stall:nudge', attempt: this.streamNudgeCount, ...diagnostics });
|
|
1302
|
+
this.emitWatchdog('abort_issued', `Aborting stalled stream attempt ${this.currentStreamTurn?.attempt ?? 0}.`, { level: 'warn', elapsedSeconds });
|
|
1303
|
+
this.abortController?.abort();
|
|
931
1304
|
}
|
|
932
1305
|
finally {
|
|
933
1306
|
this._nudgeInFlight = false;
|
|
@@ -956,6 +1329,9 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
956
1329
|
const userMessage = this.conversations.appendMessage(convId, 'user', content);
|
|
957
1330
|
this.emit({ type: 'message:user', message: userMessage });
|
|
958
1331
|
}
|
|
1332
|
+
const latestUserMessage = content
|
|
1333
|
+
|| [...this.conversations.getMessages(convId)].reverse().find((message) => message.role === 'user')?.content
|
|
1334
|
+
|| '';
|
|
959
1335
|
this.streamNudgeCount = 0;
|
|
960
1336
|
let completionRetryCount = 0;
|
|
961
1337
|
let nextRetryContext = null;
|
|
@@ -965,13 +1341,14 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
965
1341
|
const maxTotalAttempts = 1 + this.maxNudgeRetries + this.maxCompletionRetries;
|
|
966
1342
|
let loopTerminated = false;
|
|
967
1343
|
let nextRetryCause = null;
|
|
1344
|
+
const turnContinuity = this.createTurnContinuityState();
|
|
968
1345
|
try {
|
|
969
1346
|
for (let attempt = 0; attempt < maxTotalAttempts; attempt++) {
|
|
970
1347
|
const abortController = new AbortController();
|
|
971
1348
|
const attemptNumber = attempt + 1;
|
|
972
1349
|
const retryCause = nextRetryCause;
|
|
973
1350
|
nextRetryCause = null;
|
|
974
|
-
const completionState = this.createAttemptCompletionState();
|
|
1351
|
+
const completionState = this.createAttemptCompletionState(turnContinuity);
|
|
975
1352
|
let completionGuardFailure = null;
|
|
976
1353
|
const stallRetryCountAtStart = this.streamNudgeCount;
|
|
977
1354
|
const attemptMessageIds = [];
|
|
@@ -983,6 +1360,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
983
1360
|
attempt: attemptNumber,
|
|
984
1361
|
conversationId: convId,
|
|
985
1362
|
};
|
|
1363
|
+
this.currentPartialContent = '';
|
|
1364
|
+
this.pendingRecoveryDecision = null;
|
|
986
1365
|
log.info('stream_started', {
|
|
987
1366
|
turnId,
|
|
988
1367
|
attempt: attemptNumber,
|
|
@@ -992,7 +1371,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
992
1371
|
retryCause,
|
|
993
1372
|
});
|
|
994
1373
|
this.emit({ type: 'message:cerebrum:start', conversationId: convId });
|
|
995
|
-
this.startStreamWatchdog();
|
|
1374
|
+
this.startStreamWatchdog(latestUserMessage);
|
|
996
1375
|
let messages = this.conversations.getMessages(convId);
|
|
997
1376
|
// On retry: exclude failed attempts' messages from history.
|
|
998
1377
|
// The resume context already summarizes what happened — sending the raw tool calls
|
|
@@ -1061,6 +1440,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1061
1440
|
];
|
|
1062
1441
|
const toolDefs = Object.fromEntries(allTools);
|
|
1063
1442
|
let fullContent = '';
|
|
1443
|
+
let finalDisplayContent = '';
|
|
1444
|
+
let attemptFinishMeta;
|
|
1064
1445
|
const throwIfToolAttemptAborted = () => {
|
|
1065
1446
|
if (!isCurrentAttempt()) {
|
|
1066
1447
|
throw createAbortError('Tool execution aborted');
|
|
@@ -1073,15 +1454,13 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1073
1454
|
if (!isCurrentAttempt() || abortController.signal.aborted)
|
|
1074
1455
|
return;
|
|
1075
1456
|
fullContent += chunk;
|
|
1457
|
+
this.currentPartialContent = fullContent;
|
|
1076
1458
|
this.markStreamWaitingModel();
|
|
1077
1459
|
this.emit({ type: 'message:cerebrum:chunk', chunk });
|
|
1078
1460
|
},
|
|
1079
1461
|
onToolCall: async (toolCall) => {
|
|
1080
1462
|
throwIfToolAttemptAborted();
|
|
1081
|
-
this.logStreamDebug('tool_callback_started',
|
|
1082
|
-
toolName: toolCall.name.trim() || toolCall.name,
|
|
1083
|
-
toolCallId: toolCall.id,
|
|
1084
|
-
});
|
|
1463
|
+
this.logStreamDebug('tool_callback_started', this.buildToolDebugPayload(toolCall));
|
|
1085
1464
|
this.markStreamWaitingTool(toolCall);
|
|
1086
1465
|
const requestedToolName = toolCall.name;
|
|
1087
1466
|
const normalizedToolName = requestedToolName.trim() || requestedToolName;
|
|
@@ -1092,7 +1471,13 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1092
1471
|
else {
|
|
1093
1472
|
completionState.externalToolCallCount++;
|
|
1094
1473
|
this.emit({ type: 'message:cerebrum:toolcall', toolCall: { ...toolCall, name: normalizedToolName } });
|
|
1095
|
-
this.emit({
|
|
1474
|
+
this.emit({
|
|
1475
|
+
type: 'tool:start',
|
|
1476
|
+
callId: toolCall.id,
|
|
1477
|
+
name: normalizedToolName,
|
|
1478
|
+
requestedName: requestedToolName !== normalizedToolName ? requestedToolName : undefined,
|
|
1479
|
+
args: toolCall.args,
|
|
1480
|
+
});
|
|
1096
1481
|
}
|
|
1097
1482
|
const { toolName, result } = await this.toolRuntime.execute({
|
|
1098
1483
|
toolCall,
|
|
@@ -1102,15 +1487,18 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1102
1487
|
scopeKey: convId,
|
|
1103
1488
|
abortSignal: abortController.signal,
|
|
1104
1489
|
});
|
|
1105
|
-
this.logStreamDebug('tool_callback_finished',
|
|
1106
|
-
toolName,
|
|
1107
|
-
toolCallId: toolCall.id,
|
|
1108
|
-
isError: result.isError,
|
|
1109
|
-
});
|
|
1490
|
+
this.logStreamDebug('tool_callback_finished', this.buildToolDebugPayload(toolCall, result, toolName));
|
|
1110
1491
|
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
1111
1492
|
this.markStreamWaitingModel();
|
|
1112
1493
|
if (!isInternalTaskSignal) {
|
|
1113
|
-
this.emit({
|
|
1494
|
+
this.emit({
|
|
1495
|
+
type: 'tool:end',
|
|
1496
|
+
callId: toolCall.id,
|
|
1497
|
+
name: toolName,
|
|
1498
|
+
requestedName: requestedToolName !== toolName ? requestedToolName : undefined,
|
|
1499
|
+
args: toolCall.args,
|
|
1500
|
+
result,
|
|
1501
|
+
});
|
|
1114
1502
|
}
|
|
1115
1503
|
if (!isInternalTaskSignal && !result.isError) {
|
|
1116
1504
|
completionState.successfulExternalToolCount++;
|
|
@@ -1152,7 +1540,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1152
1540
|
}
|
|
1153
1541
|
throwIfToolAttemptAborted();
|
|
1154
1542
|
if (!isInternalTaskSignal) {
|
|
1155
|
-
this.
|
|
1543
|
+
this.recordAttemptToolProgress(completionState, toolName, result);
|
|
1156
1544
|
const toolMsg = this.conversations.appendMessage(convId, 'tool', result.output, {
|
|
1157
1545
|
toolResult: result,
|
|
1158
1546
|
metadata: {
|
|
@@ -1169,6 +1557,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1169
1557
|
return;
|
|
1170
1558
|
this.stopStreamWatchdog();
|
|
1171
1559
|
let displayContent = content;
|
|
1560
|
+
finalDisplayContent = content;
|
|
1561
|
+
attemptFinishMeta = finishMeta;
|
|
1172
1562
|
const visibleToolCalls = toolCalls?.filter((toolCall) => !this.isInternalTaskSignalTool(toolCall.name));
|
|
1173
1563
|
log.info('stream_finish_observed', {
|
|
1174
1564
|
turnId,
|
|
@@ -1189,6 +1579,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1189
1579
|
displayContent = content
|
|
1190
1580
|
.replace(/<discovery_complete>[\s\S]*?<\/discovery_complete>/g, '')
|
|
1191
1581
|
.trim();
|
|
1582
|
+
finalDisplayContent = displayContent;
|
|
1192
1583
|
if (parsed && this.onDiscoveryComplete) {
|
|
1193
1584
|
this.discoveryMode = false;
|
|
1194
1585
|
this.onDiscoveryComplete(parsed);
|
|
@@ -1198,23 +1589,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1198
1589
|
const guardFailure = this.evaluateCompletionGuard(displayContent, finishMeta, completionState);
|
|
1199
1590
|
if (guardFailure) {
|
|
1200
1591
|
completionGuardFailure = guardFailure;
|
|
1201
|
-
|
|
1202
|
-
attempt: attemptNumber,
|
|
1203
|
-
partialContent: fullContent || displayContent,
|
|
1204
|
-
completionState,
|
|
1205
|
-
finishMeta,
|
|
1206
|
-
}));
|
|
1207
|
-
if (nextRetryContext) {
|
|
1208
|
-
log.info('completion_retry_context_prepared', {
|
|
1209
|
-
turnId,
|
|
1210
|
-
attempt: attemptNumber,
|
|
1211
|
-
conversationId: convId,
|
|
1212
|
-
finishReason: finishMeta?.finishReason,
|
|
1213
|
-
rawFinishReason: finishMeta?.rawFinishReason,
|
|
1214
|
-
hasPartialContent: (fullContent || displayContent).trim().length > 0,
|
|
1215
|
-
recentToolSummaries: completionState.recentExternalToolSummaries.length,
|
|
1216
|
-
});
|
|
1217
|
-
}
|
|
1592
|
+
finalDisplayContent = displayContent;
|
|
1218
1593
|
this.emitCompletionTrace('guard_triggered', guardFailure.message, guardFailure.signal, 'warn');
|
|
1219
1594
|
log.warn('completion_guard_triggered', {
|
|
1220
1595
|
turnId,
|
|
@@ -1270,9 +1645,50 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1270
1645
|
const completionFailure = completionGuardFailure;
|
|
1271
1646
|
if (completionFailure !== null) {
|
|
1272
1647
|
const completionSignal = completionFailure.signal;
|
|
1648
|
+
const recoveryRequest = this.buildRecoveryRequest({
|
|
1649
|
+
cause: 'completion',
|
|
1650
|
+
attempt: attemptNumber,
|
|
1651
|
+
partialContent: fullContent || finalDisplayContent,
|
|
1652
|
+
completionState,
|
|
1653
|
+
latestUserMessage,
|
|
1654
|
+
completionRetryCount,
|
|
1655
|
+
finishMeta: attemptFinishMeta,
|
|
1656
|
+
});
|
|
1657
|
+
const { source, assessment } = await this.assessTurnRecovery(recoveryRequest);
|
|
1658
|
+
this.emitRecoveryTrace('completion', source, assessment, assessment.action === 'stop' ? 'warn' : 'info');
|
|
1659
|
+
nextRetryContext = this.buildRetryContextMessage('completion', attemptNumber, assessment.modelMessage, source);
|
|
1660
|
+
log.info('completion_retry_context_prepared', {
|
|
1661
|
+
turnId,
|
|
1662
|
+
attempt: attemptNumber,
|
|
1663
|
+
conversationId: convId,
|
|
1664
|
+
source,
|
|
1665
|
+
action: assessment.action,
|
|
1666
|
+
finishReason: attemptFinishMeta?.finishReason,
|
|
1667
|
+
rawFinishReason: attemptFinishMeta?.rawFinishReason,
|
|
1668
|
+
hasPartialContent: (fullContent || finalDisplayContent).trim().length > 0,
|
|
1669
|
+
progressEntries: completionState.continuity.progressLedger.length,
|
|
1670
|
+
taskCheckpoints: completionState.continuity.taskCheckpoints.length,
|
|
1671
|
+
completedSteps: assessment.completedSteps,
|
|
1672
|
+
nextStep: assessment.nextStep,
|
|
1673
|
+
});
|
|
1674
|
+
if (assessment.action === 'stop') {
|
|
1675
|
+
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
1676
|
+
const diagnosticMessage = this.conversations.appendMessage(convId, 'system', assessment.operatorMessage);
|
|
1677
|
+
this.emit({ type: 'message:system', message: diagnosticMessage });
|
|
1678
|
+
this.emitCompletionTrace('retry_failed', assessment.diagnosis, completionSignal, 'error');
|
|
1679
|
+
this.emit({
|
|
1680
|
+
type: 'error',
|
|
1681
|
+
error: new Error(assessment.diagnosis || 'Turn ended without a valid completion signal or final answer.'),
|
|
1682
|
+
});
|
|
1683
|
+
if (failedAttemptMessageIds.length > 0) {
|
|
1684
|
+
this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
1685
|
+
}
|
|
1686
|
+
loopTerminated = true;
|
|
1687
|
+
break;
|
|
1688
|
+
}
|
|
1273
1689
|
if (completionRetryCount < this.maxCompletionRetries) {
|
|
1274
1690
|
completionRetryCount++;
|
|
1275
|
-
const systemMessage = this.conversations.appendMessage(convId, 'system',
|
|
1691
|
+
const systemMessage = this.conversations.appendMessage(convId, 'system', assessment.operatorMessage);
|
|
1276
1692
|
attemptMessageIds.push(systemMessage.id);
|
|
1277
1693
|
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
1278
1694
|
this.emit({ type: 'message:system', message: systemMessage });
|
|
@@ -1281,12 +1697,14 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1281
1697
|
continue;
|
|
1282
1698
|
}
|
|
1283
1699
|
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
1284
|
-
const diagnosticMessage = this.conversations.appendMessage(convId, 'system',
|
|
1700
|
+
const diagnosticMessage = this.conversations.appendMessage(convId, 'system', source === 'cerebellum'
|
|
1701
|
+
? '[Cerebellum] The turn ended repeatedly without a valid completion signal or final answer.'
|
|
1702
|
+
: '[System fallback] The turn ended repeatedly without a valid completion signal or final answer.');
|
|
1285
1703
|
this.emit({ type: 'message:system', message: diagnosticMessage });
|
|
1286
|
-
this.emitCompletionTrace('retry_failed', `Completion retries exhausted after ${completionRetryCount}/${this.maxCompletionRetries}: ${completionFailure.message}`, completionSignal, 'error');
|
|
1704
|
+
this.emitCompletionTrace('retry_failed', `Completion retries exhausted after ${completionRetryCount}/${this.maxCompletionRetries}: ${assessment.diagnosis || completionFailure.message}`, completionSignal, 'error');
|
|
1287
1705
|
this.emit({
|
|
1288
1706
|
type: 'error',
|
|
1289
|
-
error: new Error('Turn ended without a valid completion signal or final answer.'),
|
|
1707
|
+
error: new Error(assessment.diagnosis || 'Turn ended without a valid completion signal or final answer.'),
|
|
1290
1708
|
});
|
|
1291
1709
|
// Clean up all failed attempt messages on exhaustion
|
|
1292
1710
|
if (failedAttemptMessageIds.length > 0) {
|
|
@@ -1302,21 +1720,17 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1302
1720
|
const failureState = this.getStreamState();
|
|
1303
1721
|
this.stopStreamWatchdog();
|
|
1304
1722
|
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
1305
|
-
|
|
1306
|
-
|
|
1723
|
+
const recoveryDecision = this.pendingRecoveryDecision;
|
|
1724
|
+
this.pendingRecoveryDecision = null;
|
|
1725
|
+
const stallRecovery = recoveryDecision;
|
|
1726
|
+
const isRecoveryRetryAbort = abortController.signal.aborted
|
|
1727
|
+
&& stallRecovery !== null
|
|
1728
|
+
&& stallRecovery.assessment.action === 'retry'
|
|
1307
1729
|
&& this.streamNudgeCount > stallRetryCountAtStart
|
|
1308
1730
|
&& this.streamNudgeCount <= this.maxNudgeRetries;
|
|
1309
|
-
if (
|
|
1310
|
-
nextRetryContext = this.
|
|
1311
|
-
|
|
1312
|
-
phase: failureState.phase,
|
|
1313
|
-
activeToolName: failureState.activeToolName,
|
|
1314
|
-
activeToolCallId: failureState.activeToolCallId,
|
|
1315
|
-
partialContent: fullContent,
|
|
1316
|
-
completionState,
|
|
1317
|
-
}));
|
|
1318
|
-
// Inject nudge message and retry via the loop
|
|
1319
|
-
const systemMessage = this.conversations.appendMessage(convId, 'system', '[Cerebellum] You stopped mid-response. Continue from where you left off.');
|
|
1731
|
+
if (isRecoveryRetryAbort && stallRecovery) {
|
|
1732
|
+
nextRetryContext = this.buildRetryContextMessage('stall', attemptNumber, stallRecovery.assessment.modelMessage, stallRecovery.source);
|
|
1733
|
+
const systemMessage = this.conversations.appendMessage(convId, 'system', stallRecovery.assessment.operatorMessage);
|
|
1320
1734
|
attemptMessageIds.push(systemMessage.id);
|
|
1321
1735
|
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
1322
1736
|
this.emit({ type: 'message:system', message: systemMessage });
|
|
@@ -1324,6 +1738,18 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1324
1738
|
nextRetryCause = 'stall';
|
|
1325
1739
|
continue; // retry loop
|
|
1326
1740
|
}
|
|
1741
|
+
if (abortController.signal.aborted
|
|
1742
|
+
&& stallRecovery !== null
|
|
1743
|
+
&& stallRecovery.assessment.action === 'stop') {
|
|
1744
|
+
const systemMessage = this.conversations.appendMessage(convId, 'system', stallRecovery.assessment.operatorMessage);
|
|
1745
|
+
this.emit({ type: 'message:system', message: systemMessage });
|
|
1746
|
+
this.emit({ type: 'error', error: new Error(stallRecovery.assessment.diagnosis) });
|
|
1747
|
+
if (failedAttemptMessageIds.length > 0) {
|
|
1748
|
+
this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
1749
|
+
}
|
|
1750
|
+
loopTerminated = true;
|
|
1751
|
+
break;
|
|
1752
|
+
}
|
|
1327
1753
|
// Check if Cerebellum dropped mid-stream
|
|
1328
1754
|
if (this.cerebellum && !this.cerebellum.isConnected() && abortController.signal.aborted) {
|
|
1329
1755
|
const err = new Error('Cerebellum disconnected during active response. Restart it with: docker compose up -d cerebellum');
|