@cereworker/core 26.330.3 → 26.403.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/conversation.d.ts +27 -5
- package/dist/conversation.d.ts.map +1 -1
- package/dist/conversation.js +254 -108
- package/dist/conversation.js.map +1 -1
- package/dist/index.d.ts +4 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/legacy-sqlite.d.ts +6 -0
- package/dist/legacy-sqlite.d.ts.map +1 -0
- package/dist/legacy-sqlite.js +90 -0
- package/dist/legacy-sqlite.js.map +1 -0
- package/dist/orchestrator.d.ts +11 -1
- package/dist/orchestrator.d.ts.map +1 -1
- package/dist/orchestrator.js +460 -75
- package/dist/orchestrator.js.map +1 -1
- package/dist/pairing.d.ts +11 -2
- package/dist/pairing.d.ts.map +1 -1
- package/dist/pairing.js +166 -95
- package/dist/pairing.js.map +1 -1
- package/dist/plan-store.d.ts +4 -3
- package/dist/plan-store.d.ts.map +1 -1
- package/dist/plan-store.js +69 -72
- package/dist/plan-store.js.map +1 -1
- package/dist/system-prompt.d.ts.map +1 -1
- package/dist/system-prompt.js +31 -11
- package/dist/system-prompt.js.map +1 -1
- package/dist/text-store.d.ts +18 -0
- package/dist/text-store.d.ts.map +1 -0
- package/dist/text-store.js +212 -0
- package/dist/text-store.js.map +1 -0
- package/dist/types.d.ts +35 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +2 -2
package/dist/orchestrator.js
CHANGED
|
@@ -12,7 +12,11 @@ const log = createLogger('orchestrator');
|
|
|
12
12
|
const TASK_COMPLETE_TOOL = 'task_complete';
|
|
13
13
|
const TASK_BLOCKED_TOOL = 'task_blocked';
|
|
14
14
|
const TASK_CHECKPOINT_TOOL = 'task_checkpoint';
|
|
15
|
-
const INTERNAL_TASK_TOOL_NAMES = new Set([
|
|
15
|
+
const INTERNAL_TASK_TOOL_NAMES = new Set([
|
|
16
|
+
TASK_COMPLETE_TOOL,
|
|
17
|
+
TASK_BLOCKED_TOOL,
|
|
18
|
+
TASK_CHECKPOINT_TOOL,
|
|
19
|
+
]);
|
|
16
20
|
const SYSTEM_FALLBACK_COMPLETION_PROMPT = '[System fallback] The last turn ended without a final answer. Continue from the last verified state and end by calling task_complete or task_blocked before your final answer.';
|
|
17
21
|
const SYSTEM_FALLBACK_STALL_PROMPT = '[System fallback] The stalled turn is being retried from the last verified state.';
|
|
18
22
|
const DEBUG_TOOL_OUTPUT_MAX_CHARS = 8_000;
|
|
@@ -51,9 +55,16 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
51
55
|
fineTuneMethod = 'auto';
|
|
52
56
|
fineTuneSchedule = 'auto';
|
|
53
57
|
fineTuneStatus = {
|
|
54
|
-
status: 'idle',
|
|
55
|
-
|
|
56
|
-
|
|
58
|
+
status: 'idle',
|
|
59
|
+
jobId: '',
|
|
60
|
+
progress: 0,
|
|
61
|
+
currentStep: 0,
|
|
62
|
+
totalSteps: 0,
|
|
63
|
+
currentLoss: 0,
|
|
64
|
+
error: '',
|
|
65
|
+
checkpointPath: '',
|
|
66
|
+
startedAt: 0,
|
|
67
|
+
completedAt: 0,
|
|
57
68
|
};
|
|
58
69
|
_fineTuneHistory = [];
|
|
59
70
|
gatewayMode = 'standalone';
|
|
@@ -71,11 +82,17 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
71
82
|
streamStallThreshold = 30_000;
|
|
72
83
|
maxNudgeRetries = 2;
|
|
73
84
|
maxCompletionRetries = 2;
|
|
85
|
+
turnJournalRetention = {
|
|
86
|
+
maxDays: 30,
|
|
87
|
+
maxFilesPerConversation: 100,
|
|
88
|
+
};
|
|
74
89
|
streamPhase = 'idle';
|
|
75
90
|
activeToolCall = null;
|
|
76
91
|
currentStreamTurn = null;
|
|
77
92
|
currentAttemptCompletionState = null;
|
|
78
93
|
currentPartialContent = '';
|
|
94
|
+
currentLastContentKind = 'empty';
|
|
95
|
+
currentJournaledContentLength = 0;
|
|
79
96
|
pendingRecoveryDecision = null;
|
|
80
97
|
streamAbortGraceMs = 1_000;
|
|
81
98
|
taskConversations = new Map();
|
|
@@ -102,6 +119,12 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
102
119
|
this.maxNudgeRetries = options.maxNudgeRetries;
|
|
103
120
|
this.maxCompletionRetries = options.maxNudgeRetries;
|
|
104
121
|
}
|
|
122
|
+
if (options?.turnJournalRetention) {
|
|
123
|
+
this.turnJournalRetention = {
|
|
124
|
+
...this.turnJournalRetention,
|
|
125
|
+
...options.turnJournalRetention,
|
|
126
|
+
};
|
|
127
|
+
}
|
|
105
128
|
}
|
|
106
129
|
setCerebrum(cerebrum) {
|
|
107
130
|
this.cerebrum = cerebrum;
|
|
@@ -181,9 +204,19 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
181
204
|
parameters: {
|
|
182
205
|
type: 'object',
|
|
183
206
|
properties: {
|
|
184
|
-
step: {
|
|
185
|
-
|
|
186
|
-
|
|
207
|
+
step: {
|
|
208
|
+
type: 'string',
|
|
209
|
+
description: 'Short milestone name, such as "profile continuity checked"',
|
|
210
|
+
},
|
|
211
|
+
status: {
|
|
212
|
+
type: 'string',
|
|
213
|
+
enum: ['done', 'in_progress'],
|
|
214
|
+
description: 'Whether the milestone is done or currently in progress',
|
|
215
|
+
},
|
|
216
|
+
evidence: {
|
|
217
|
+
type: 'string',
|
|
218
|
+
description: 'Concrete evidence showing what happened at this milestone',
|
|
219
|
+
},
|
|
187
220
|
},
|
|
188
221
|
required: ['step', 'status', 'evidence'],
|
|
189
222
|
additionalProperties: false,
|
|
@@ -220,8 +253,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
220
253
|
isError: true,
|
|
221
254
|
};
|
|
222
255
|
}
|
|
223
|
-
const hasVerifiedProgress = state.successfulExternalToolCount > 0
|
|
224
|
-
|
|
256
|
+
const hasVerifiedProgress = state.successfulExternalToolCount > 0 ||
|
|
257
|
+
state.continuity.progressLedger.some((entry) => entry.source === 'tool' && !entry.isError);
|
|
225
258
|
if (!hasVerifiedProgress) {
|
|
226
259
|
return {
|
|
227
260
|
output: 'task_complete requires at least one successful external tool result in this turn.',
|
|
@@ -246,6 +279,22 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
246
279
|
state.summary = undefined;
|
|
247
280
|
state.evidence = evidence;
|
|
248
281
|
}
|
|
282
|
+
const boundarySummary = signal === 'complete'
|
|
283
|
+
? `Task completion recorded: ${state.summary}`
|
|
284
|
+
: `Task blocker recorded: ${state.blocker}`;
|
|
285
|
+
this.recordBoundary(state.continuity, {
|
|
286
|
+
kind: 'completion',
|
|
287
|
+
action: signal === 'complete' ? TASK_COMPLETE_TOOL : TASK_BLOCKED_TOOL,
|
|
288
|
+
summary: boundarySummary,
|
|
289
|
+
stateChanging: true,
|
|
290
|
+
evidence,
|
|
291
|
+
});
|
|
292
|
+
this.appendTurnJournalEntry('completion_signal', boundarySummary, {
|
|
293
|
+
signal,
|
|
294
|
+
evidence,
|
|
295
|
+
summary: state.summary,
|
|
296
|
+
blocker: state.blocker,
|
|
297
|
+
});
|
|
249
298
|
this.emitCompletionTrace('signal_recorded', signal === 'complete'
|
|
250
299
|
? `Recorded task_complete signal with evidence: ${evidence}`
|
|
251
300
|
: `Recorded task_blocked signal with evidence: ${evidence}`, signal, 'info');
|
|
@@ -269,9 +318,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
269
318
|
const step = String(args.step ?? '').trim();
|
|
270
319
|
const evidence = String(args.evidence ?? '').trim();
|
|
271
320
|
const statusValue = String(args.status ?? '').trim();
|
|
272
|
-
const status = statusValue === 'done' || statusValue === 'in_progress'
|
|
273
|
-
? statusValue
|
|
274
|
-
: null;
|
|
321
|
+
const status = statusValue === 'done' || statusValue === 'in_progress' ? statusValue : null;
|
|
275
322
|
if (!step) {
|
|
276
323
|
return {
|
|
277
324
|
output: 'A non-empty step field is required.',
|
|
@@ -460,7 +507,12 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
460
507
|
if (!result.started) {
|
|
461
508
|
throw new Error(result.error || 'Failed to start fine-tuning');
|
|
462
509
|
}
|
|
463
|
-
this.fineTuneStatus = {
|
|
510
|
+
this.fineTuneStatus = {
|
|
511
|
+
...this.fineTuneStatus,
|
|
512
|
+
status: 'running',
|
|
513
|
+
jobId: result.jobId,
|
|
514
|
+
startedAt: Date.now(),
|
|
515
|
+
};
|
|
464
516
|
this.emit({ type: 'finetune:start', jobId: result.jobId });
|
|
465
517
|
log.info('Fine-tuning started', { jobId: result.jobId });
|
|
466
518
|
// 5. Poll for progress
|
|
@@ -499,7 +551,10 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
499
551
|
jobId: status.jobId,
|
|
500
552
|
checkpointPath: status.checkpointPath,
|
|
501
553
|
});
|
|
502
|
-
log.info('Fine-tuning completed', {
|
|
554
|
+
log.info('Fine-tuning completed', {
|
|
555
|
+
jobId: status.jobId,
|
|
556
|
+
checkpoint: status.checkpointPath,
|
|
557
|
+
});
|
|
503
558
|
this.stopFineTunePoller();
|
|
504
559
|
}
|
|
505
560
|
else if (status.status === 'failed') {
|
|
@@ -540,7 +595,10 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
540
595
|
return {
|
|
541
596
|
name: nameMatch?.[1]?.trim() || 'Cere',
|
|
542
597
|
role: roleMatch?.[1]?.trim() || 'general-purpose assistant',
|
|
543
|
-
traits: traitsMatch?.[1]
|
|
598
|
+
traits: traitsMatch?.[1]
|
|
599
|
+
?.split(',')
|
|
600
|
+
.map((t) => t.trim().toLowerCase())
|
|
601
|
+
.filter(Boolean) ?? [],
|
|
544
602
|
};
|
|
545
603
|
}
|
|
546
604
|
getActiveConversationId() {
|
|
@@ -644,7 +702,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
644
702
|
return {
|
|
645
703
|
streaming: this.streamWatchdog !== null,
|
|
646
704
|
lastActivityAt: this.lastStreamActivityAt,
|
|
647
|
-
stallDetected: this.streamWatchdog !== null &&
|
|
705
|
+
stallDetected: this.streamWatchdog !== null && Date.now() - this.lastStreamActivityAt > stallThresholdMs,
|
|
648
706
|
nudgeCount: this.streamNudgeCount,
|
|
649
707
|
phase: this.streamPhase,
|
|
650
708
|
activeToolName: this.activeToolCall?.name,
|
|
@@ -665,9 +723,9 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
665
723
|
}
|
|
666
724
|
markStreamWaitingTool(toolCall, activityAt = Date.now()) {
|
|
667
725
|
const normalizedToolName = toolCall.name.trim() || toolCall.name;
|
|
668
|
-
const phaseChanged = this.streamPhase !== 'waiting_tool'
|
|
669
|
-
|
|
670
|
-
|
|
726
|
+
const phaseChanged = this.streamPhase !== 'waiting_tool' ||
|
|
727
|
+
this.activeToolCall?.id !== toolCall.id ||
|
|
728
|
+
this.activeToolCall?.name !== normalizedToolName;
|
|
671
729
|
this.lastStreamActivityAt = activityAt;
|
|
672
730
|
this.streamPhase = 'waiting_tool';
|
|
673
731
|
this.activeToolCall = {
|
|
@@ -688,6 +746,121 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
688
746
|
this.activeToolCall = null;
|
|
689
747
|
this.streamDeferredUntil = 0;
|
|
690
748
|
this.currentPartialContent = '';
|
|
749
|
+
this.currentLastContentKind = 'empty';
|
|
750
|
+
this.currentJournaledContentLength = 0;
|
|
751
|
+
}
|
|
752
|
+
appendTurnJournalEntry(type, summary, data) {
|
|
753
|
+
if (!this.currentStreamTurn)
|
|
754
|
+
return;
|
|
755
|
+
const entry = {
|
|
756
|
+
turnId: this.currentStreamTurn.turnId,
|
|
757
|
+
attempt: this.currentStreamTurn.attempt,
|
|
758
|
+
timestamp: Date.now(),
|
|
759
|
+
type,
|
|
760
|
+
summary: this.truncateResumeText(summary, 500),
|
|
761
|
+
...(data ? { data } : {}),
|
|
762
|
+
};
|
|
763
|
+
this.conversations.appendTurnJournalEntry(this.currentStreamTurn.conversationId, this.currentStreamTurn.turnId, entry);
|
|
764
|
+
}
|
|
765
|
+
pruneTurnJournals(conversationId) {
|
|
766
|
+
try {
|
|
767
|
+
const result = this.conversations.pruneTurnJournals(conversationId, this.turnJournalRetention);
|
|
768
|
+
if (result.prunedByAge > 0 || result.prunedByCount > 0) {
|
|
769
|
+
log.debug('Pruned turn journals', {
|
|
770
|
+
conversationId,
|
|
771
|
+
prunedByAge: result.prunedByAge,
|
|
772
|
+
prunedByCount: result.prunedByCount,
|
|
773
|
+
remaining: result.remaining,
|
|
774
|
+
});
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
catch (error) {
|
|
778
|
+
log.warn('Failed to prune turn journals', {
|
|
779
|
+
conversationId,
|
|
780
|
+
error: error instanceof Error ? error.message : String(error),
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
persistPartialContentSnapshot(force = false) {
|
|
785
|
+
const normalized = this.currentPartialContent.trim();
|
|
786
|
+
if (!normalized)
|
|
787
|
+
return;
|
|
788
|
+
if (!force && this.currentPartialContent.length - this.currentJournaledContentLength < 200) {
|
|
789
|
+
return;
|
|
790
|
+
}
|
|
791
|
+
this.currentJournaledContentLength = this.currentPartialContent.length;
|
|
792
|
+
this.appendTurnJournalEntry('partial_text', `Assistant partial text: ${this.truncateResumeText(normalized, 220)}`, {
|
|
793
|
+
chars: this.currentPartialContent.length,
|
|
794
|
+
excerpt: this.truncateResumeText(normalized, 1200),
|
|
795
|
+
});
|
|
796
|
+
}
|
|
797
|
+
recordBoundary(continuity, boundary) {
|
|
798
|
+
const createdAt = Date.now();
|
|
799
|
+
const summary = {
|
|
800
|
+
id: nanoid(10),
|
|
801
|
+
createdAt,
|
|
802
|
+
browserState: this.cloneBrowserState(boundary.browserState ?? continuity.browserState),
|
|
803
|
+
...boundary,
|
|
804
|
+
};
|
|
805
|
+
continuity.boundaries.push(summary);
|
|
806
|
+
while (continuity.boundaries.length > 20) {
|
|
807
|
+
continuity.boundaries.shift();
|
|
808
|
+
}
|
|
809
|
+
this.appendTurnJournalEntry('boundary', summary.summary, {
|
|
810
|
+
boundaryId: summary.id,
|
|
811
|
+
kind: summary.kind,
|
|
812
|
+
action: summary.action,
|
|
813
|
+
stateChanging: summary.stateChanging,
|
|
814
|
+
url: summary.url,
|
|
815
|
+
tabId: summary.tabId,
|
|
816
|
+
checkpointStatus: summary.checkpointStatus,
|
|
817
|
+
evidence: summary.evidence,
|
|
818
|
+
browserState: summary.browserState,
|
|
819
|
+
});
|
|
820
|
+
return summary;
|
|
821
|
+
}
|
|
822
|
+
deriveRepetitionSignals(continuity) {
|
|
823
|
+
const signals = [];
|
|
824
|
+
const recentToolEntries = continuity.progressLedger
|
|
825
|
+
.filter((entry) => entry.source === 'tool')
|
|
826
|
+
.slice(-10);
|
|
827
|
+
const recentActions = recentToolEntries.map((entry) => entry.action);
|
|
828
|
+
if (recentActions.length >= 4 && new Set(recentActions.slice(-4)).size <= 2) {
|
|
829
|
+
signals.push(`Recent tool actions are cycling between ${Array.from(new Set(recentActions.slice(-4))).join(', ')}`);
|
|
830
|
+
}
|
|
831
|
+
const summaryCounts = new Map();
|
|
832
|
+
for (const boundary of continuity.boundaries.slice(-12)) {
|
|
833
|
+
if (boundary.kind !== 'tool' || !boundary.stateChanging)
|
|
834
|
+
continue;
|
|
835
|
+
const key = `${boundary.action}|${boundary.summary}`;
|
|
836
|
+
summaryCounts.set(key, (summaryCounts.get(key) ?? 0) + 1);
|
|
837
|
+
}
|
|
838
|
+
for (const [key, count] of summaryCounts.entries()) {
|
|
839
|
+
if (count < 2)
|
|
840
|
+
continue;
|
|
841
|
+
const [, summary] = key.split('|', 2);
|
|
842
|
+
signals.push(`Repeated verified action x${count}: ${summary}`);
|
|
843
|
+
}
|
|
844
|
+
return signals.slice(0, 5);
|
|
845
|
+
}
|
|
846
|
+
classifyTurnOutcome(displayContent, finishMeta, completionState) {
|
|
847
|
+
const trimmed = displayContent.trim();
|
|
848
|
+
const endedOnToolCalls = finishMeta?.finishReason === 'tool-calls' ||
|
|
849
|
+
finishMeta?.stepFinishReasons.at(-1) === 'tool-calls' ||
|
|
850
|
+
finishMeta?.endedWithToolCall === true;
|
|
851
|
+
if (endedOnToolCalls) {
|
|
852
|
+
return 'ended_on_tool_calls';
|
|
853
|
+
}
|
|
854
|
+
if (completionState.externalToolCallCount > 0 && completionState.signal === 'none') {
|
|
855
|
+
return 'completion_signal_missing';
|
|
856
|
+
}
|
|
857
|
+
if (finishMeta?.finishReason === 'error' || finishMeta?.lastContentKind === 'error') {
|
|
858
|
+
return 'protocol_error';
|
|
859
|
+
}
|
|
860
|
+
if (trimmed.length > 0) {
|
|
861
|
+
return 'completed';
|
|
862
|
+
}
|
|
863
|
+
return 'completed_no_text';
|
|
691
864
|
}
|
|
692
865
|
getStreamDiagnostics(elapsedSeconds) {
|
|
693
866
|
return {
|
|
@@ -700,17 +873,13 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
700
873
|
}
|
|
701
874
|
describeStreamLocation(phase = this.streamPhase, activeToolName = this.activeToolCall?.name) {
|
|
702
875
|
if (phase === 'waiting_tool') {
|
|
703
|
-
return activeToolName
|
|
704
|
-
? `waiting_tool/${activeToolName}`
|
|
705
|
-
: 'waiting_tool';
|
|
876
|
+
return activeToolName ? `waiting_tool/${activeToolName}` : 'waiting_tool';
|
|
706
877
|
}
|
|
707
878
|
return phase;
|
|
708
879
|
}
|
|
709
880
|
getCurrentStallThresholdMs(phase = this.streamPhase, nudgeCount = this.streamNudgeCount) {
|
|
710
|
-
const phaseBase = phase === 'waiting_model'
|
|
711
|
-
|
|
712
|
-
: this.streamStallThreshold;
|
|
713
|
-
return phaseBase + (nudgeCount * this.streamStallThreshold);
|
|
881
|
+
const phaseBase = phase === 'waiting_model' ? this.streamStallThreshold * 3 : this.streamStallThreshold;
|
|
882
|
+
return phaseBase + nudgeCount * this.streamStallThreshold;
|
|
714
883
|
}
|
|
715
884
|
logStreamDebug(msg, data) {
|
|
716
885
|
if (!this.currentStreamTurn)
|
|
@@ -793,11 +962,13 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
793
962
|
progressLedger: [],
|
|
794
963
|
taskCheckpoints: [],
|
|
795
964
|
browserState: {},
|
|
965
|
+
boundaries: [],
|
|
796
966
|
};
|
|
797
967
|
}
|
|
798
968
|
buildRecoveryRequest(params) {
|
|
799
969
|
const partialContent = this.truncateResumeText(params.partialContent, 600);
|
|
800
970
|
const continuity = params.completionState.continuity;
|
|
971
|
+
const latestBoundary = continuity.boundaries.at(-1);
|
|
801
972
|
return {
|
|
802
973
|
conversationId: this.currentStreamTurn?.conversationId ?? '',
|
|
803
974
|
turnId: this.currentStreamTurn?.turnId ?? '',
|
|
@@ -809,12 +980,31 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
809
980
|
stallRetryCount: this.streamNudgeCount,
|
|
810
981
|
completionRetryCount: params.completionRetryCount ?? 0,
|
|
811
982
|
finishReason: params.finishMeta?.finishReason ?? params.finishMeta?.stepFinishReasons.at(-1),
|
|
983
|
+
turnOutcome: params.turnOutcome,
|
|
984
|
+
lastContentKind: params.finishMeta?.lastContentKind ?? this.currentLastContentKind,
|
|
812
985
|
elapsedSeconds: params.elapsedSeconds,
|
|
813
986
|
partialContent: partialContent || undefined,
|
|
814
|
-
latestUserMessage: params.latestUserMessage
|
|
987
|
+
latestUserMessage: params.latestUserMessage
|
|
988
|
+
? this.truncateResumeText(params.latestUserMessage, 600)
|
|
989
|
+
: undefined,
|
|
815
990
|
progressEntries: continuity.progressLedger.slice(-50).map((entry) => ({ ...entry })),
|
|
816
991
|
taskCheckpoints: continuity.taskCheckpoints.map((checkpoint) => ({ ...checkpoint })),
|
|
817
992
|
browserState: this.cloneBrowserState(continuity.browserState),
|
|
993
|
+
latestBoundary: latestBoundary
|
|
994
|
+
? {
|
|
995
|
+
...latestBoundary,
|
|
996
|
+
...(latestBoundary.browserState
|
|
997
|
+
? { browserState: this.cloneBrowserState(latestBoundary.browserState) }
|
|
998
|
+
: {}),
|
|
999
|
+
}
|
|
1000
|
+
: undefined,
|
|
1001
|
+
recentBoundaries: continuity.boundaries.slice(-20).map((boundary) => ({
|
|
1002
|
+
...boundary,
|
|
1003
|
+
...(boundary.browserState
|
|
1004
|
+
? { browserState: this.cloneBrowserState(boundary.browserState) }
|
|
1005
|
+
: {}),
|
|
1006
|
+
})),
|
|
1007
|
+
repetitionSignals: this.deriveRepetitionSignals(continuity),
|
|
818
1008
|
};
|
|
819
1009
|
}
|
|
820
1010
|
emitRecoveryTrace(cause, source, assessment, level = 'info') {
|
|
@@ -864,12 +1054,19 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
864
1054
|
stallRetryCount: request.stallRetryCount,
|
|
865
1055
|
completionRetryCount: request.completionRetryCount,
|
|
866
1056
|
finishReason: request.finishReason,
|
|
1057
|
+
turnOutcome: request.turnOutcome,
|
|
1058
|
+
lastContentKind: request.lastContentKind,
|
|
867
1059
|
elapsedSeconds: request.elapsedSeconds,
|
|
868
1060
|
hasPartialContent: Boolean(request.partialContent),
|
|
869
|
-
latestUserMessage: request.latestUserMessage
|
|
1061
|
+
latestUserMessage: request.latestUserMessage
|
|
1062
|
+
? this.truncateResumeText(request.latestUserMessage, 300)
|
|
1063
|
+
: '',
|
|
870
1064
|
browserState: request.browserState,
|
|
871
1065
|
progressEntries: request.progressEntries,
|
|
872
1066
|
taskCheckpoints: request.taskCheckpoints,
|
|
1067
|
+
latestBoundary: request.latestBoundary,
|
|
1068
|
+
recentBoundaries: request.recentBoundaries,
|
|
1069
|
+
repetitionSignals: request.repetitionSignals,
|
|
873
1070
|
});
|
|
874
1071
|
if (this.cerebellum?.isConnected() && this.cerebellum.assessTurnRecovery) {
|
|
875
1072
|
try {
|
|
@@ -905,6 +1102,14 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
905
1102
|
}
|
|
906
1103
|
deriveCompletedSteps(request) {
|
|
907
1104
|
const completed = new Set();
|
|
1105
|
+
for (const boundary of request.recentBoundaries) {
|
|
1106
|
+
if ((boundary.kind === 'tool' ||
|
|
1107
|
+
boundary.kind === 'checkpoint' ||
|
|
1108
|
+
boundary.kind === 'completion') &&
|
|
1109
|
+
boundary.summary) {
|
|
1110
|
+
completed.add(boundary.summary);
|
|
1111
|
+
}
|
|
1112
|
+
}
|
|
908
1113
|
for (const checkpoint of request.taskCheckpoints) {
|
|
909
1114
|
if (checkpoint.status === 'done') {
|
|
910
1115
|
completed.add(checkpoint.summary);
|
|
@@ -924,10 +1129,10 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
924
1129
|
browserHints.push(`Current URL: ${request.browserState.currentUrl}`);
|
|
925
1130
|
if (request.browserState.activeTabId)
|
|
926
1131
|
browserHints.push(`Active tab: ${request.browserState.activeTabId}`);
|
|
927
|
-
const diagnosis = options?.reason
|
|
928
|
-
|
|
1132
|
+
const diagnosis = options?.reason ??
|
|
1133
|
+
(request.cause === 'stall'
|
|
929
1134
|
? `Recovery guidance is unavailable while the stream is stalled in ${this.describeStreamLocation(request.phase, request.activeToolName)}.`
|
|
930
|
-
: `Recovery guidance is unavailable after the turn ended with ${request.finishReason ?? 'no final answer'}.`);
|
|
1135
|
+
: `Recovery guidance is unavailable after the turn ended with ${request.turnOutcome} (${request.finishReason ?? 'no final answer'}).`);
|
|
931
1136
|
const nextStep = request.cause === 'stall'
|
|
932
1137
|
? 'Resume from the last verified browser state and continue with the next unfinished step.'
|
|
933
1138
|
: 'Use the verified progress below to continue from the next unfinished step and avoid repeating confirmed work.';
|
|
@@ -946,6 +1151,14 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
946
1151
|
for (const hint of browserHints)
|
|
947
1152
|
lines.push(`- ${hint}`);
|
|
948
1153
|
}
|
|
1154
|
+
if (request.latestBoundary) {
|
|
1155
|
+
lines.push('', `Latest verified boundary: ${request.latestBoundary.summary}`);
|
|
1156
|
+
}
|
|
1157
|
+
if (request.repetitionSignals.length > 0) {
|
|
1158
|
+
lines.push('', 'Repetition warnings:');
|
|
1159
|
+
for (const signal of request.repetitionSignals)
|
|
1160
|
+
lines.push(`- ${signal}`);
|
|
1161
|
+
}
|
|
949
1162
|
if (request.partialContent) {
|
|
950
1163
|
lines.push('', 'Partial assistant text from the failed attempt:', request.partialContent);
|
|
951
1164
|
}
|
|
@@ -977,9 +1190,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
977
1190
|
};
|
|
978
1191
|
}
|
|
979
1192
|
formatToolOutputPreview(output) {
|
|
980
|
-
return this.truncateResumeText(output, 180)
|
|
981
|
-
.replace(/\s+/g, ' ')
|
|
982
|
-
.trim();
|
|
1193
|
+
return this.truncateResumeText(output, 180).replace(/\s+/g, ' ').trim();
|
|
983
1194
|
}
|
|
984
1195
|
truncateResumeText(text, maxChars) {
|
|
985
1196
|
const normalized = text.replace(/\r\n/g, '\n').trim();
|
|
@@ -988,9 +1199,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
988
1199
|
return `${normalized.slice(0, Math.max(0, maxChars - 1)).trimEnd()}…`;
|
|
989
1200
|
}
|
|
990
1201
|
serializeDebugValue(value, maxChars) {
|
|
991
|
-
const raw = typeof value === 'string'
|
|
992
|
-
? value
|
|
993
|
-
: JSON.stringify(value, null, 2) ?? String(value);
|
|
1202
|
+
const raw = typeof value === 'string' ? value : (JSON.stringify(value, null, 2) ?? String(value));
|
|
994
1203
|
if (raw.length <= maxChars) {
|
|
995
1204
|
return { value: raw, truncated: false };
|
|
996
1205
|
}
|
|
@@ -1031,7 +1240,10 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1031
1240
|
isError: result.isError,
|
|
1032
1241
|
warnings: result.warnings ?? [],
|
|
1033
1242
|
truncated: result.truncated ?? false,
|
|
1034
|
-
debugPayloadTruncated: argsPreview.truncated ||
|
|
1243
|
+
debugPayloadTruncated: argsPreview.truncated ||
|
|
1244
|
+
outputPreview.truncated ||
|
|
1245
|
+
Boolean(detailsPreview?.truncated) ||
|
|
1246
|
+
Boolean(resumePreview?.truncated),
|
|
1035
1247
|
};
|
|
1036
1248
|
}
|
|
1037
1249
|
recordCheckpoint(continuity, step, status, evidence) {
|
|
@@ -1056,6 +1268,19 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1056
1268
|
isError: false,
|
|
1057
1269
|
checkpointStatus: status,
|
|
1058
1270
|
});
|
|
1271
|
+
this.recordBoundary(continuity, {
|
|
1272
|
+
kind: 'checkpoint',
|
|
1273
|
+
action: 'task_checkpoint',
|
|
1274
|
+
summary: checkpoint.summary,
|
|
1275
|
+
stateChanging: status === 'done',
|
|
1276
|
+
evidence,
|
|
1277
|
+
checkpointStatus: status,
|
|
1278
|
+
});
|
|
1279
|
+
this.appendTurnJournalEntry('checkpoint', checkpoint.summary, {
|
|
1280
|
+
step,
|
|
1281
|
+
status,
|
|
1282
|
+
evidence,
|
|
1283
|
+
});
|
|
1059
1284
|
return checkpoint;
|
|
1060
1285
|
}
|
|
1061
1286
|
recordAttemptToolProgress(completionState, toolName, result) {
|
|
@@ -1065,6 +1290,16 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1065
1290
|
return;
|
|
1066
1291
|
this.recordProgressEntry(continuity, entry);
|
|
1067
1292
|
this.updateBrowserState(continuity.browserState, result);
|
|
1293
|
+
if (entry.stateChanging && !entry.isError) {
|
|
1294
|
+
this.recordBoundary(continuity, {
|
|
1295
|
+
kind: 'tool',
|
|
1296
|
+
action: entry.action,
|
|
1297
|
+
summary: entry.summary,
|
|
1298
|
+
stateChanging: true,
|
|
1299
|
+
url: entry.url,
|
|
1300
|
+
tabId: entry.tabId,
|
|
1301
|
+
});
|
|
1302
|
+
}
|
|
1068
1303
|
}
|
|
1069
1304
|
createProgressEntry(toolName, result) {
|
|
1070
1305
|
const resume = this.getBrowserResumeMetadata(result);
|
|
@@ -1094,15 +1329,15 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1094
1329
|
}
|
|
1095
1330
|
recordProgressEntry(continuity, entry) {
|
|
1096
1331
|
const last = continuity.progressLedger.at(-1);
|
|
1097
|
-
if (last
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1332
|
+
if (last &&
|
|
1333
|
+
entry.source === 'tool' &&
|
|
1334
|
+
last.source === 'tool' &&
|
|
1335
|
+
!entry.stateChanging &&
|
|
1336
|
+
!last.stateChanging &&
|
|
1337
|
+
last.action === entry.action &&
|
|
1338
|
+
last.summary === entry.summary &&
|
|
1339
|
+
last.url === entry.url &&
|
|
1340
|
+
last.tabId === entry.tabId) {
|
|
1106
1341
|
return;
|
|
1107
1342
|
}
|
|
1108
1343
|
continuity.progressLedger.push(entry);
|
|
@@ -1124,18 +1359,27 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1124
1359
|
const resume = this.getBrowserResumeMetadata(result);
|
|
1125
1360
|
if (!resume)
|
|
1126
1361
|
return;
|
|
1127
|
-
|
|
1128
|
-
|
|
1362
|
+
const stateDelta = resume.stateDelta && typeof resume.stateDelta === 'object'
|
|
1363
|
+
? resume.stateDelta
|
|
1364
|
+
: null;
|
|
1365
|
+
const deltaUrl = typeof stateDelta?.currentUrl === 'string' ? stateDelta.currentUrl : undefined;
|
|
1366
|
+
const deltaActiveTabId = typeof stateDelta?.activeTabId === 'string' ? stateDelta.activeTabId : undefined;
|
|
1367
|
+
const deltaTabs = Array.isArray(stateDelta?.tabs)
|
|
1368
|
+
? stateDelta.tabs
|
|
1369
|
+
: undefined;
|
|
1370
|
+
if (resume.url ?? deltaUrl) {
|
|
1371
|
+
browserState.currentUrl = resume.url ?? deltaUrl;
|
|
1129
1372
|
}
|
|
1130
|
-
if (resume.activeTabId) {
|
|
1131
|
-
browserState.activeTabId = resume.activeTabId;
|
|
1373
|
+
if (resume.activeTabId ?? deltaActiveTabId) {
|
|
1374
|
+
browserState.activeTabId = resume.activeTabId ?? deltaActiveTabId;
|
|
1132
1375
|
}
|
|
1133
1376
|
else if (resume.tabId && resume.stateChanging) {
|
|
1134
1377
|
browserState.activeTabId = resume.tabId;
|
|
1135
1378
|
}
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1379
|
+
const nextTabs = resume.tabs?.length ? resume.tabs : deltaTabs;
|
|
1380
|
+
if (nextTabs?.length) {
|
|
1381
|
+
browserState.tabs = nextTabs.map((tab) => ({ ...tab }));
|
|
1382
|
+
const active = nextTabs.find((tab) => tab.active);
|
|
1139
1383
|
if (active) {
|
|
1140
1384
|
browserState.activeTabId = active.id;
|
|
1141
1385
|
browserState.currentUrl = active.url;
|
|
@@ -1155,8 +1399,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1155
1399
|
evaluateCompletionGuard(displayContent, finishMeta, completionState) {
|
|
1156
1400
|
const trimmedContent = displayContent.trim();
|
|
1157
1401
|
const hadExternalToolActivity = completionState.externalToolCallCount > 0;
|
|
1158
|
-
const endedOnToolCalls = finishMeta?.finishReason === 'tool-calls'
|
|
1159
|
-
|
|
1402
|
+
const endedOnToolCalls = finishMeta?.finishReason === 'tool-calls' ||
|
|
1403
|
+
finishMeta?.stepFinishReasons.at(-1) === 'tool-calls';
|
|
1160
1404
|
if (trimmedContent.length === 0 && hadExternalToolActivity) {
|
|
1161
1405
|
return {
|
|
1162
1406
|
message: 'Turn ended after tool activity without a final answer.',
|
|
@@ -1259,26 +1503,52 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1259
1503
|
attempt: this.currentStreamTurn.attempt,
|
|
1260
1504
|
partialContent: this.currentPartialContent,
|
|
1261
1505
|
completionState: this.currentAttemptCompletionState,
|
|
1506
|
+
turnOutcome: 'stalled',
|
|
1262
1507
|
latestUserMessage,
|
|
1263
1508
|
elapsedSeconds,
|
|
1264
1509
|
});
|
|
1265
1510
|
const { source, assessment } = await this.assessTurnRecovery(request);
|
|
1266
1511
|
this.emitRecoveryTrace('stall', source, assessment, assessment.action === 'stop' ? 'warn' : 'info');
|
|
1512
|
+
this.appendTurnJournalEntry('recovery', assessment.operatorMessage, {
|
|
1513
|
+
cause: 'stall',
|
|
1514
|
+
source,
|
|
1515
|
+
action: assessment.action,
|
|
1516
|
+
diagnosis: assessment.diagnosis,
|
|
1517
|
+
nextStep: assessment.nextStep,
|
|
1518
|
+
waitSeconds: assessment.waitSeconds,
|
|
1519
|
+
completedSteps: assessment.completedSteps,
|
|
1520
|
+
});
|
|
1267
1521
|
if (assessment.action === 'wait') {
|
|
1268
1522
|
const waitSeconds = Math.max(15, assessment.waitSeconds ?? this.streamStallThreshold / 1000);
|
|
1269
|
-
this.streamDeferredUntil = Date.now() +
|
|
1523
|
+
this.streamDeferredUntil = Date.now() + waitSeconds * 1000;
|
|
1270
1524
|
return;
|
|
1271
1525
|
}
|
|
1272
1526
|
if (assessment.action === 'retry') {
|
|
1273
1527
|
this.streamNudgeCount++;
|
|
1274
1528
|
this.pendingRecoveryDecision = { cause: 'stall', source, assessment };
|
|
1529
|
+
this.recordBoundary(this.currentAttemptCompletionState.continuity, {
|
|
1530
|
+
kind: 'recovery',
|
|
1531
|
+
action: 'stall_retry',
|
|
1532
|
+
summary: assessment.diagnosis,
|
|
1533
|
+
stateChanging: false,
|
|
1534
|
+
});
|
|
1275
1535
|
this.emitWatchdog('nudge_requested', `Cerebellum requested nudge ${this.streamNudgeCount}/${this.maxNudgeRetries} after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'info', elapsedSeconds });
|
|
1276
|
-
this.emit({
|
|
1536
|
+
this.emit({
|
|
1537
|
+
type: 'cerebrum:stall:nudge',
|
|
1538
|
+
attempt: this.streamNudgeCount,
|
|
1539
|
+
...diagnostics,
|
|
1540
|
+
});
|
|
1277
1541
|
this.emitWatchdog('abort_issued', `Aborting stalled stream attempt ${this.currentStreamTurn?.attempt ?? 0}.`, { level: 'warn', elapsedSeconds });
|
|
1278
1542
|
this.abortController?.abort();
|
|
1279
1543
|
return;
|
|
1280
1544
|
}
|
|
1281
1545
|
this.pendingRecoveryDecision = { cause: 'stall', source, assessment };
|
|
1546
|
+
this.recordBoundary(this.currentAttemptCompletionState.continuity, {
|
|
1547
|
+
kind: 'recovery',
|
|
1548
|
+
action: 'stall_stop',
|
|
1549
|
+
summary: assessment.diagnosis,
|
|
1550
|
+
stateChanging: false,
|
|
1551
|
+
});
|
|
1282
1552
|
this.emitWatchdog('abort_issued', 'Aborting stalled stream because recovery guidance requested stop.', { level: 'warn', elapsedSeconds });
|
|
1283
1553
|
this.abortController?.abort();
|
|
1284
1554
|
}
|
|
@@ -1288,6 +1558,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1288
1558
|
attempt: this.currentStreamTurn.attempt,
|
|
1289
1559
|
partialContent: this.currentPartialContent,
|
|
1290
1560
|
completionState: this.currentAttemptCompletionState,
|
|
1561
|
+
turnOutcome: 'stalled',
|
|
1291
1562
|
latestUserMessage,
|
|
1292
1563
|
elapsedSeconds,
|
|
1293
1564
|
});
|
|
@@ -1296,9 +1567,28 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1296
1567
|
});
|
|
1297
1568
|
this.pendingRecoveryDecision = { cause: 'stall', source: 'fallback', assessment };
|
|
1298
1569
|
this.emitRecoveryTrace('stall', 'fallback', assessment, 'warn');
|
|
1570
|
+
this.appendTurnJournalEntry('recovery', assessment.operatorMessage, {
|
|
1571
|
+
cause: 'stall',
|
|
1572
|
+
source: 'fallback',
|
|
1573
|
+
action: assessment.action,
|
|
1574
|
+
diagnosis: assessment.diagnosis,
|
|
1575
|
+
nextStep: assessment.nextStep,
|
|
1576
|
+
waitSeconds: assessment.waitSeconds,
|
|
1577
|
+
completedSteps: assessment.completedSteps,
|
|
1578
|
+
});
|
|
1299
1579
|
this.streamNudgeCount++;
|
|
1580
|
+
this.recordBoundary(this.currentAttemptCompletionState.continuity, {
|
|
1581
|
+
kind: 'recovery',
|
|
1582
|
+
action: 'stall_retry',
|
|
1583
|
+
summary: assessment.diagnosis,
|
|
1584
|
+
stateChanging: false,
|
|
1585
|
+
});
|
|
1300
1586
|
this.emitWatchdog('nudge_requested', `Fallback retry ${this.streamNudgeCount}/${this.maxNudgeRetries} after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'info', elapsedSeconds });
|
|
1301
|
-
this.emit({
|
|
1587
|
+
this.emit({
|
|
1588
|
+
type: 'cerebrum:stall:nudge',
|
|
1589
|
+
attempt: this.streamNudgeCount,
|
|
1590
|
+
...diagnostics,
|
|
1591
|
+
});
|
|
1302
1592
|
this.emitWatchdog('abort_issued', `Aborting stalled stream attempt ${this.currentStreamTurn?.attempt ?? 0}.`, { level: 'warn', elapsedSeconds });
|
|
1303
1593
|
this.abortController?.abort();
|
|
1304
1594
|
}
|
|
@@ -1329,9 +1619,11 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1329
1619
|
const userMessage = this.conversations.appendMessage(convId, 'user', content);
|
|
1330
1620
|
this.emit({ type: 'message:user', message: userMessage });
|
|
1331
1621
|
}
|
|
1332
|
-
const latestUserMessage = content
|
|
1333
|
-
|
|
1334
|
-
|
|
1622
|
+
const latestUserMessage = content ||
|
|
1623
|
+
[...this.conversations.getMessages(convId)]
|
|
1624
|
+
.reverse()
|
|
1625
|
+
.find((message) => message.role === 'user')?.content ||
|
|
1626
|
+
'';
|
|
1335
1627
|
this.streamNudgeCount = 0;
|
|
1336
1628
|
let completionRetryCount = 0;
|
|
1337
1629
|
let nextRetryContext = null;
|
|
@@ -1361,6 +1653,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1361
1653
|
conversationId: convId,
|
|
1362
1654
|
};
|
|
1363
1655
|
this.currentPartialContent = '';
|
|
1656
|
+
this.currentLastContentKind = 'empty';
|
|
1657
|
+
this.currentJournaledContentLength = 0;
|
|
1364
1658
|
this.pendingRecoveryDecision = null;
|
|
1365
1659
|
log.info('stream_started', {
|
|
1366
1660
|
turnId,
|
|
@@ -1370,6 +1664,12 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1370
1664
|
completionRetryCount,
|
|
1371
1665
|
retryCause,
|
|
1372
1666
|
});
|
|
1667
|
+
this.appendTurnJournalEntry('turn_started', `Turn attempt ${attemptNumber} started.`, {
|
|
1668
|
+
retryCause: retryCause ?? null,
|
|
1669
|
+
latestUserMessage: this.truncateResumeText(latestUserMessage, 600),
|
|
1670
|
+
stallRetryCount: this.streamNudgeCount,
|
|
1671
|
+
completionRetryCount,
|
|
1672
|
+
});
|
|
1373
1673
|
this.emit({ type: 'message:cerebrum:start', conversationId: convId });
|
|
1374
1674
|
this.startStreamWatchdog(latestUserMessage);
|
|
1375
1675
|
let messages = this.conversations.getMessages(convId);
|
|
@@ -1455,22 +1755,34 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1455
1755
|
return;
|
|
1456
1756
|
fullContent += chunk;
|
|
1457
1757
|
this.currentPartialContent = fullContent;
|
|
1758
|
+
this.currentLastContentKind = 'text';
|
|
1458
1759
|
this.markStreamWaitingModel();
|
|
1760
|
+
this.persistPartialContentSnapshot();
|
|
1459
1761
|
this.emit({ type: 'message:cerebrum:chunk', chunk });
|
|
1460
1762
|
},
|
|
1461
1763
|
onToolCall: async (toolCall) => {
|
|
1462
1764
|
throwIfToolAttemptAborted();
|
|
1463
1765
|
this.logStreamDebug('tool_callback_started', this.buildToolDebugPayload(toolCall));
|
|
1464
1766
|
this.markStreamWaitingTool(toolCall);
|
|
1767
|
+
this.currentLastContentKind = 'tool-call';
|
|
1465
1768
|
const requestedToolName = toolCall.name;
|
|
1466
1769
|
const normalizedToolName = requestedToolName.trim() || requestedToolName;
|
|
1770
|
+
this.appendTurnJournalEntry('tool_start', `Calling ${normalizedToolName}`, {
|
|
1771
|
+
requestedToolName,
|
|
1772
|
+
toolName: normalizedToolName,
|
|
1773
|
+
callId: toolCall.id,
|
|
1774
|
+
args: toolCall.args,
|
|
1775
|
+
});
|
|
1467
1776
|
const isInternalTaskSignal = this.isInternalTaskSignalTool(normalizedToolName);
|
|
1468
1777
|
if (isInternalTaskSignal) {
|
|
1469
1778
|
completionState.internalToolCallCount++;
|
|
1470
1779
|
}
|
|
1471
1780
|
else {
|
|
1472
1781
|
completionState.externalToolCallCount++;
|
|
1473
|
-
this.emit({
|
|
1782
|
+
this.emit({
|
|
1783
|
+
type: 'message:cerebrum:toolcall',
|
|
1784
|
+
toolCall: { ...toolCall, name: normalizedToolName },
|
|
1785
|
+
});
|
|
1474
1786
|
this.emit({
|
|
1475
1787
|
type: 'tool:start',
|
|
1476
1788
|
callId: toolCall.id,
|
|
@@ -1490,6 +1802,7 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1490
1802
|
this.logStreamDebug('tool_callback_finished', this.buildToolDebugPayload(toolCall, result, toolName));
|
|
1491
1803
|
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
1492
1804
|
this.markStreamWaitingModel();
|
|
1805
|
+
this.currentLastContentKind = result.isError ? 'error' : 'tool-call';
|
|
1493
1806
|
if (!isInternalTaskSignal) {
|
|
1494
1807
|
this.emit({
|
|
1495
1808
|
type: 'tool:end',
|
|
@@ -1504,7 +1817,9 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1504
1817
|
completionState.successfulExternalToolCount++;
|
|
1505
1818
|
}
|
|
1506
1819
|
// Cerebellum verification (non-blocking)
|
|
1507
|
-
if (!isInternalTaskSignal &&
|
|
1820
|
+
if (!isInternalTaskSignal &&
|
|
1821
|
+
this.cerebellum?.isConnected() &&
|
|
1822
|
+
this.verificationEnabled) {
|
|
1508
1823
|
try {
|
|
1509
1824
|
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
1510
1825
|
this.emit({ type: 'verification:start', callId: toolCall.id, toolName });
|
|
@@ -1539,6 +1854,18 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1539
1854
|
}
|
|
1540
1855
|
}
|
|
1541
1856
|
throwIfToolAttemptAborted();
|
|
1857
|
+
const toolJournalEntry = this.createProgressEntry(toolName, result);
|
|
1858
|
+
this.appendTurnJournalEntry('tool_end', toolJournalEntry?.summary ??
|
|
1859
|
+
`${toolName}: ${this.formatToolOutputPreview(result.output)}`, {
|
|
1860
|
+
requestedToolName,
|
|
1861
|
+
toolName,
|
|
1862
|
+
callId: toolCall.id,
|
|
1863
|
+
isError: result.isError,
|
|
1864
|
+
args: toolCall.args,
|
|
1865
|
+
output: this.truncateResumeText(result.output, 2_000),
|
|
1866
|
+
details: result.details,
|
|
1867
|
+
metadata: result.metadata,
|
|
1868
|
+
});
|
|
1542
1869
|
if (!isInternalTaskSignal) {
|
|
1543
1870
|
this.recordAttemptToolProgress(completionState, toolName, result);
|
|
1544
1871
|
const toolMsg = this.conversations.appendMessage(convId, 'tool', result.output, {
|
|
@@ -1559,18 +1886,25 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1559
1886
|
let displayContent = content;
|
|
1560
1887
|
finalDisplayContent = content;
|
|
1561
1888
|
attemptFinishMeta = finishMeta;
|
|
1889
|
+
this.currentLastContentKind =
|
|
1890
|
+
finishMeta?.lastContentKind ??
|
|
1891
|
+
(content.trim() ? 'text' : this.currentLastContentKind);
|
|
1892
|
+
this.persistPartialContentSnapshot(true);
|
|
1562
1893
|
const visibleToolCalls = toolCalls?.filter((toolCall) => !this.isInternalTaskSignalTool(toolCall.name));
|
|
1894
|
+
const turnOutcome = this.classifyTurnOutcome(displayContent, finishMeta, completionState);
|
|
1563
1895
|
log.info('stream_finish_observed', {
|
|
1564
1896
|
turnId,
|
|
1565
1897
|
attempt: attemptNumber,
|
|
1566
1898
|
conversationId: convId,
|
|
1567
1899
|
finishReason: finishMeta?.finishReason,
|
|
1568
1900
|
rawFinishReason: finishMeta?.rawFinishReason,
|
|
1901
|
+
lastContentKind: finishMeta?.lastContentKind,
|
|
1569
1902
|
stepCount: finishMeta?.stepCount ?? 0,
|
|
1570
1903
|
chunkCount: finishMeta?.chunkCount ?? 0,
|
|
1571
1904
|
toolCallCount: finishMeta?.toolCallCount ?? 0,
|
|
1572
1905
|
textChars: finishMeta?.textChars ?? content.length,
|
|
1573
1906
|
completionSignal: completionState.signal,
|
|
1907
|
+
turnOutcome,
|
|
1574
1908
|
});
|
|
1575
1909
|
// Check for discovery completion — parse and strip the tag before storing
|
|
1576
1910
|
if (this.discoveryMode && content.includes('<discovery_complete>')) {
|
|
@@ -1586,6 +1920,17 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1586
1920
|
log.info('Discovery completed', { name: parsed.name });
|
|
1587
1921
|
}
|
|
1588
1922
|
}
|
|
1923
|
+
this.appendTurnJournalEntry('turn_finished', `Turn ended with ${turnOutcome}.`, {
|
|
1924
|
+
finishReason: finishMeta?.finishReason,
|
|
1925
|
+
rawFinishReason: finishMeta?.rawFinishReason,
|
|
1926
|
+
lastContentKind: finishMeta?.lastContentKind ?? this.currentLastContentKind,
|
|
1927
|
+
turnOutcome,
|
|
1928
|
+
textChars: finishMeta?.textChars ?? displayContent.length,
|
|
1929
|
+
toolCallCount: finishMeta?.toolCallCount ?? 0,
|
|
1930
|
+
completionSignal: completionState.signal,
|
|
1931
|
+
finalContent: this.truncateResumeText(displayContent, 2_000),
|
|
1932
|
+
});
|
|
1933
|
+
this.pruneTurnJournals(convId);
|
|
1589
1934
|
const guardFailure = this.evaluateCompletionGuard(displayContent, finishMeta, completionState);
|
|
1590
1935
|
if (guardFailure) {
|
|
1591
1936
|
completionGuardFailure = guardFailure;
|
|
@@ -1597,11 +1942,13 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1597
1942
|
conversationId: convId,
|
|
1598
1943
|
finishReason: finishMeta?.finishReason,
|
|
1599
1944
|
rawFinishReason: finishMeta?.rawFinishReason,
|
|
1945
|
+
lastContentKind: finishMeta?.lastContentKind,
|
|
1600
1946
|
stepCount: finishMeta?.stepCount ?? 0,
|
|
1601
1947
|
chunkCount: finishMeta?.chunkCount ?? 0,
|
|
1602
1948
|
toolCallCount: finishMeta?.toolCallCount ?? 0,
|
|
1603
1949
|
textChars: finishMeta?.textChars ?? displayContent.length,
|
|
1604
1950
|
completionSignal: completionState.signal,
|
|
1951
|
+
turnOutcome,
|
|
1605
1952
|
});
|
|
1606
1953
|
return;
|
|
1607
1954
|
}
|
|
@@ -1610,7 +1957,11 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1610
1957
|
if (failedAttemptMessageIds.length > 0) {
|
|
1611
1958
|
const deleted = this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
1612
1959
|
if (deleted > 0) {
|
|
1613
|
-
log.info('Cleaned up failed attempt messages', {
|
|
1960
|
+
log.info('Cleaned up failed attempt messages', {
|
|
1961
|
+
deleted,
|
|
1962
|
+
convId,
|
|
1963
|
+
attempt: attemptNumber,
|
|
1964
|
+
});
|
|
1614
1965
|
}
|
|
1615
1966
|
}
|
|
1616
1967
|
const cerebrumMessage = this.conversations.appendMessage(convId, 'cerebrum', displayContent, visibleToolCalls?.length ? { toolCalls: visibleToolCalls } : undefined);
|
|
@@ -1644,12 +1995,14 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1644
1995
|
await this.awaitStreamAttempt(streamPromise, abortController);
|
|
1645
1996
|
const completionFailure = completionGuardFailure;
|
|
1646
1997
|
if (completionFailure !== null) {
|
|
1998
|
+
const turnOutcome = this.classifyTurnOutcome(finalDisplayContent, attemptFinishMeta, completionState);
|
|
1647
1999
|
const completionSignal = completionFailure.signal;
|
|
1648
2000
|
const recoveryRequest = this.buildRecoveryRequest({
|
|
1649
2001
|
cause: 'completion',
|
|
1650
2002
|
attempt: attemptNumber,
|
|
1651
2003
|
partialContent: fullContent || finalDisplayContent,
|
|
1652
2004
|
completionState,
|
|
2005
|
+
turnOutcome,
|
|
1653
2006
|
latestUserMessage,
|
|
1654
2007
|
completionRetryCount,
|
|
1655
2008
|
finishMeta: attemptFinishMeta,
|
|
@@ -1657,6 +2010,22 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1657
2010
|
const { source, assessment } = await this.assessTurnRecovery(recoveryRequest);
|
|
1658
2011
|
this.emitRecoveryTrace('completion', source, assessment, assessment.action === 'stop' ? 'warn' : 'info');
|
|
1659
2012
|
nextRetryContext = this.buildRetryContextMessage('completion', attemptNumber, assessment.modelMessage, source);
|
|
2013
|
+
this.appendTurnJournalEntry('recovery', assessment.operatorMessage, {
|
|
2014
|
+
cause: 'completion',
|
|
2015
|
+
source,
|
|
2016
|
+
action: assessment.action,
|
|
2017
|
+
diagnosis: assessment.diagnosis,
|
|
2018
|
+
nextStep: assessment.nextStep,
|
|
2019
|
+
completedSteps: assessment.completedSteps,
|
|
2020
|
+
turnOutcome,
|
|
2021
|
+
finishReason: attemptFinishMeta?.finishReason,
|
|
2022
|
+
});
|
|
2023
|
+
this.recordBoundary(completionState.continuity, {
|
|
2024
|
+
kind: 'recovery',
|
|
2025
|
+
action: assessment.action === 'stop' ? 'completion_stop' : 'completion_retry',
|
|
2026
|
+
summary: assessment.diagnosis,
|
|
2027
|
+
stateChanging: false,
|
|
2028
|
+
});
|
|
1660
2029
|
log.info('completion_retry_context_prepared', {
|
|
1661
2030
|
turnId,
|
|
1662
2031
|
attempt: attemptNumber,
|
|
@@ -1670,6 +2039,10 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1670
2039
|
taskCheckpoints: completionState.continuity.taskCheckpoints.length,
|
|
1671
2040
|
completedSteps: assessment.completedSteps,
|
|
1672
2041
|
nextStep: assessment.nextStep,
|
|
2042
|
+
turnOutcome,
|
|
2043
|
+
lastContentKind: attemptFinishMeta?.lastContentKind,
|
|
2044
|
+
latestBoundary: completionState.continuity.boundaries.at(-1),
|
|
2045
|
+
repetitionSignals: recoveryRequest.repetitionSignals,
|
|
1673
2046
|
});
|
|
1674
2047
|
if (assessment.action === 'stop') {
|
|
1675
2048
|
failedAttemptMessageIds.push(...attemptMessageIds);
|
|
@@ -1678,7 +2051,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1678
2051
|
this.emitCompletionTrace('retry_failed', assessment.diagnosis, completionSignal, 'error');
|
|
1679
2052
|
this.emit({
|
|
1680
2053
|
type: 'error',
|
|
1681
|
-
error: new Error(assessment.diagnosis ||
|
|
2054
|
+
error: new Error(assessment.diagnosis ||
|
|
2055
|
+
'Turn ended without a valid completion signal or final answer.'),
|
|
1682
2056
|
});
|
|
1683
2057
|
if (failedAttemptMessageIds.length > 0) {
|
|
1684
2058
|
this.conversations.deleteMessages(convId, failedAttemptMessageIds);
|
|
@@ -1704,7 +2078,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1704
2078
|
this.emitCompletionTrace('retry_failed', `Completion retries exhausted after ${completionRetryCount}/${this.maxCompletionRetries}: ${assessment.diagnosis || completionFailure.message}`, completionSignal, 'error');
|
|
1705
2079
|
this.emit({
|
|
1706
2080
|
type: 'error',
|
|
1707
|
-
error: new Error(assessment.diagnosis ||
|
|
2081
|
+
error: new Error(assessment.diagnosis ||
|
|
2082
|
+
'Turn ended without a valid completion signal or final answer.'),
|
|
1708
2083
|
});
|
|
1709
2084
|
// Clean up all failed attempt messages on exhaustion
|
|
1710
2085
|
if (failedAttemptMessageIds.length > 0) {
|
|
@@ -1723,11 +2098,11 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1723
2098
|
const recoveryDecision = this.pendingRecoveryDecision;
|
|
1724
2099
|
this.pendingRecoveryDecision = null;
|
|
1725
2100
|
const stallRecovery = recoveryDecision;
|
|
1726
|
-
const isRecoveryRetryAbort = abortController.signal.aborted
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
2101
|
+
const isRecoveryRetryAbort = abortController.signal.aborted &&
|
|
2102
|
+
stallRecovery !== null &&
|
|
2103
|
+
stallRecovery.assessment.action === 'retry' &&
|
|
2104
|
+
this.streamNudgeCount > stallRetryCountAtStart &&
|
|
2105
|
+
this.streamNudgeCount <= this.maxNudgeRetries;
|
|
1731
2106
|
if (isRecoveryRetryAbort && stallRecovery) {
|
|
1732
2107
|
nextRetryContext = this.buildRetryContextMessage('stall', attemptNumber, stallRecovery.assessment.modelMessage, stallRecovery.source);
|
|
1733
2108
|
const systemMessage = this.conversations.appendMessage(convId, 'system', stallRecovery.assessment.operatorMessage);
|
|
@@ -1738,9 +2113,9 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1738
2113
|
nextRetryCause = 'stall';
|
|
1739
2114
|
continue; // retry loop
|
|
1740
2115
|
}
|
|
1741
|
-
if (abortController.signal.aborted
|
|
1742
|
-
|
|
1743
|
-
|
|
2116
|
+
if (abortController.signal.aborted &&
|
|
2117
|
+
stallRecovery !== null &&
|
|
2118
|
+
stallRecovery.assessment.action === 'stop') {
|
|
1744
2119
|
const systemMessage = this.conversations.appendMessage(convId, 'system', stallRecovery.assessment.operatorMessage);
|
|
1745
2120
|
this.emit({ type: 'message:system', message: systemMessage });
|
|
1746
2121
|
this.emit({ type: 'error', error: new Error(stallRecovery.assessment.diagnosis) });
|
|
@@ -1762,6 +2137,16 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
1762
2137
|
break;
|
|
1763
2138
|
}
|
|
1764
2139
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
2140
|
+
this.appendTurnJournalEntry('turn_error', `Turn attempt ${attemptNumber} failed: ${err.message}`, {
|
|
2141
|
+
retryCause,
|
|
2142
|
+
stallRetryCount: this.streamNudgeCount,
|
|
2143
|
+
completionRetryCount,
|
|
2144
|
+
phase: failureState.phase,
|
|
2145
|
+
activeToolName: failureState.activeToolName,
|
|
2146
|
+
activeToolCallId: failureState.activeToolCallId,
|
|
2147
|
+
error: err.message,
|
|
2148
|
+
});
|
|
2149
|
+
this.pruneTurnJournals(convId);
|
|
1765
2150
|
if (retryCause === 'completion') {
|
|
1766
2151
|
this.emitCompletionTrace('retry_failed', `Completion retry attempt ${attemptNumber} failed: ${err.message}`, completionState.signal, 'error');
|
|
1767
2152
|
}
|