@cereworker/core 26.329.23 → 26.329.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/abort.d.ts +5 -0
- package/dist/abort.d.ts.map +1 -0
- package/dist/abort.js +36 -0
- package/dist/abort.js.map +1 -0
- package/dist/events.d.ts +9 -1
- package/dist/events.d.ts.map +1 -1
- package/dist/events.js.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/orchestrator.d.ts +6 -0
- package/dist/orchestrator.d.ts.map +1 -1
- package/dist/orchestrator.js +353 -222
- package/dist/orchestrator.js.map +1 -1
- package/dist/sub-agent-manager.d.ts.map +1 -1
- package/dist/sub-agent-manager.js +4 -7
- package/dist/sub-agent-manager.js.map +1 -1
- package/dist/tool-runtime.d.ts.map +1 -1
- package/dist/tool-runtime.js +2 -33
- package/dist/tool-runtime.js.map +1 -1
- package/package.json +2 -2
package/dist/orchestrator.js
CHANGED
|
@@ -6,6 +6,7 @@ import { createSubAgentTools } from './sub-agent-tools.js';
|
|
|
6
6
|
import { createLogger } from './logger.js';
|
|
7
7
|
import { buildSystemPrompt } from './system-prompt.js';
|
|
8
8
|
import { estimateMessageTokens, shouldCompact, buildCompactionMessages } from './context.js';
|
|
9
|
+
import { createAbortError, throwIfAborted } from './abort.js';
|
|
9
10
|
import { ToolRuntime, } from './tool-runtime.js';
|
|
10
11
|
const log = createLogger('orchestrator');
|
|
11
12
|
export class Orchestrator extends TypedEventEmitter {
|
|
@@ -47,6 +48,8 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
47
48
|
maxNudgeRetries = 2;
|
|
48
49
|
streamPhase = 'idle';
|
|
49
50
|
activeToolCall = null;
|
|
51
|
+
currentStreamTurn = null;
|
|
52
|
+
streamAbortGraceMs = 1_000;
|
|
50
53
|
taskConversations = new Map();
|
|
51
54
|
taskRunning = new Set();
|
|
52
55
|
recurringTasks = [];
|
|
@@ -453,18 +456,35 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
453
456
|
};
|
|
454
457
|
}
|
|
455
458
|
markStreamWaitingModel(activityAt = Date.now()) {
|
|
459
|
+
const phaseChanged = this.streamPhase !== 'waiting_model' || this.activeToolCall !== null;
|
|
456
460
|
this.lastStreamActivityAt = activityAt;
|
|
457
461
|
this.streamPhase = 'waiting_model';
|
|
458
462
|
this.activeToolCall = null;
|
|
463
|
+
if (phaseChanged) {
|
|
464
|
+
this.logStreamDebug('stream_phase_changed', {
|
|
465
|
+
phase: this.streamPhase,
|
|
466
|
+
});
|
|
467
|
+
}
|
|
459
468
|
}
|
|
460
469
|
markStreamWaitingTool(toolCall, activityAt = Date.now()) {
|
|
470
|
+
const normalizedToolName = toolCall.name.trim() || toolCall.name;
|
|
471
|
+
const phaseChanged = this.streamPhase !== 'waiting_tool'
|
|
472
|
+
|| this.activeToolCall?.id !== toolCall.id
|
|
473
|
+
|| this.activeToolCall?.name !== normalizedToolName;
|
|
461
474
|
this.lastStreamActivityAt = activityAt;
|
|
462
475
|
this.streamPhase = 'waiting_tool';
|
|
463
476
|
this.activeToolCall = {
|
|
464
477
|
id: toolCall.id,
|
|
465
|
-
name:
|
|
478
|
+
name: normalizedToolName,
|
|
466
479
|
startedAt: activityAt,
|
|
467
480
|
};
|
|
481
|
+
if (phaseChanged) {
|
|
482
|
+
this.logStreamDebug('stream_phase_changed', {
|
|
483
|
+
phase: this.streamPhase,
|
|
484
|
+
activeToolName: normalizedToolName,
|
|
485
|
+
activeToolCallId: toolCall.id,
|
|
486
|
+
});
|
|
487
|
+
}
|
|
468
488
|
}
|
|
469
489
|
resetStreamState() {
|
|
470
490
|
this.streamPhase = 'idle';
|
|
@@ -479,6 +499,100 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
479
499
|
activeToolStartedAt: this.activeToolCall?.startedAt,
|
|
480
500
|
};
|
|
481
501
|
}
|
|
502
|
+
describeStreamLocation() {
|
|
503
|
+
if (this.streamPhase === 'waiting_tool') {
|
|
504
|
+
return this.activeToolCall?.name
|
|
505
|
+
? `waiting_tool/${this.activeToolCall.name}`
|
|
506
|
+
: 'waiting_tool';
|
|
507
|
+
}
|
|
508
|
+
return this.streamPhase;
|
|
509
|
+
}
|
|
510
|
+
logStreamDebug(msg, data) {
|
|
511
|
+
if (!this.currentStreamTurn)
|
|
512
|
+
return;
|
|
513
|
+
log.debug(msg, {
|
|
514
|
+
turnId: this.currentStreamTurn.turnId,
|
|
515
|
+
attempt: this.currentStreamTurn.attempt,
|
|
516
|
+
conversationId: this.currentStreamTurn.conversationId,
|
|
517
|
+
...data,
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
emitWatchdog(stage, message, options) {
|
|
521
|
+
if (!this.currentStreamTurn)
|
|
522
|
+
return;
|
|
523
|
+
const payload = {
|
|
524
|
+
stage,
|
|
525
|
+
turnId: this.currentStreamTurn.turnId,
|
|
526
|
+
attempt: this.currentStreamTurn.attempt,
|
|
527
|
+
conversationId: this.currentStreamTurn.conversationId,
|
|
528
|
+
message,
|
|
529
|
+
...this.getStreamDiagnostics(options?.elapsedSeconds),
|
|
530
|
+
};
|
|
531
|
+
const level = options?.level ?? 'info';
|
|
532
|
+
switch (level) {
|
|
533
|
+
case 'debug':
|
|
534
|
+
log.debug(`watchdog_${stage}`, payload);
|
|
535
|
+
break;
|
|
536
|
+
case 'warn':
|
|
537
|
+
log.warn(`watchdog_${stage}`, payload);
|
|
538
|
+
break;
|
|
539
|
+
case 'error':
|
|
540
|
+
log.error(`watchdog_${stage}`, payload);
|
|
541
|
+
break;
|
|
542
|
+
default:
|
|
543
|
+
log.info(`watchdog_${stage}`, payload);
|
|
544
|
+
break;
|
|
545
|
+
}
|
|
546
|
+
this.emit({ type: 'cerebrum:watchdog', ...payload });
|
|
547
|
+
}
|
|
548
|
+
async awaitStreamAttempt(streamPromise, abortController) {
|
|
549
|
+
return new Promise((resolve, reject) => {
|
|
550
|
+
let settled = false;
|
|
551
|
+
let abortTimer = null;
|
|
552
|
+
const cleanup = () => {
|
|
553
|
+
abortController.signal.removeEventListener('abort', onAbort);
|
|
554
|
+
if (abortTimer) {
|
|
555
|
+
clearTimeout(abortTimer);
|
|
556
|
+
abortTimer = null;
|
|
557
|
+
}
|
|
558
|
+
};
|
|
559
|
+
const settleResolve = () => {
|
|
560
|
+
if (settled)
|
|
561
|
+
return;
|
|
562
|
+
settled = true;
|
|
563
|
+
cleanup();
|
|
564
|
+
resolve();
|
|
565
|
+
};
|
|
566
|
+
const settleReject = (error) => {
|
|
567
|
+
if (settled)
|
|
568
|
+
return;
|
|
569
|
+
settled = true;
|
|
570
|
+
cleanup();
|
|
571
|
+
reject(error);
|
|
572
|
+
};
|
|
573
|
+
const onAbort = () => {
|
|
574
|
+
this.logStreamDebug('provider_abort_observed', {
|
|
575
|
+
phase: this.streamPhase,
|
|
576
|
+
activeToolName: this.activeToolCall?.name,
|
|
577
|
+
activeToolCallId: this.activeToolCall?.id,
|
|
578
|
+
});
|
|
579
|
+
if (abortTimer)
|
|
580
|
+
return;
|
|
581
|
+
abortTimer = setTimeout(() => {
|
|
582
|
+
if (settled)
|
|
583
|
+
return;
|
|
584
|
+
const elapsedSeconds = Math.max(1, Math.round((Date.now() - this.lastStreamActivityAt) / 1000));
|
|
585
|
+
this.emitWatchdog('teardown_timeout', `Provider did not settle within ${this.streamAbortGraceMs}ms after abort; continuing retry.`, { level: 'warn', elapsedSeconds });
|
|
586
|
+
settleReject(createAbortError('Stream aborted'));
|
|
587
|
+
}, this.streamAbortGraceMs);
|
|
588
|
+
};
|
|
589
|
+
abortController.signal.addEventListener('abort', onAbort, { once: true });
|
|
590
|
+
if (abortController.signal.aborted) {
|
|
591
|
+
onAbort();
|
|
592
|
+
}
|
|
593
|
+
streamPromise.then(settleResolve, settleReject);
|
|
594
|
+
});
|
|
595
|
+
}
|
|
482
596
|
startStreamWatchdog() {
|
|
483
597
|
this.stopStreamWatchdog();
|
|
484
598
|
this.markStreamWaitingModel();
|
|
@@ -492,19 +606,20 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
492
606
|
return;
|
|
493
607
|
const elapsedSeconds = Math.round(elapsed / 1000);
|
|
494
608
|
const diagnostics = this.getStreamDiagnostics(elapsedSeconds);
|
|
495
|
-
|
|
609
|
+
this.emitWatchdog('stalled', `Stalled after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'warn', elapsedSeconds });
|
|
496
610
|
this.emit({ type: 'cerebrum:stall', ...diagnostics });
|
|
497
611
|
if (!this.cerebellum?.isConnected()) {
|
|
498
612
|
// Cerebellum dropped mid-stream — abort the current turn
|
|
499
|
-
|
|
613
|
+
this.emitWatchdog('abort_issued', 'Cerebellum disconnected during an active stream; aborting the turn.', { level: 'warn', elapsedSeconds });
|
|
500
614
|
this.abortController?.abort();
|
|
501
615
|
return;
|
|
502
616
|
}
|
|
503
617
|
this._nudgeInFlight = true;
|
|
504
618
|
const doNudge = () => {
|
|
505
619
|
this.streamNudgeCount++;
|
|
506
|
-
|
|
620
|
+
this.emitWatchdog('nudge_requested', `Cerebellum requested nudge ${this.streamNudgeCount}/${this.maxNudgeRetries} after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'info', elapsedSeconds });
|
|
507
621
|
this.emit({ type: 'cerebrum:stall:nudge', attempt: this.streamNudgeCount, ...diagnostics });
|
|
622
|
+
this.emitWatchdog('abort_issued', `Aborting stalled stream attempt ${this.currentStreamTurn?.attempt ?? 0}.`, { level: 'warn', elapsedSeconds });
|
|
508
623
|
this.abortController?.abort();
|
|
509
624
|
};
|
|
510
625
|
void (async () => {
|
|
@@ -548,239 +663,255 @@ export class Orchestrator extends TypedEventEmitter {
|
|
|
548
663
|
this.emit({ type: 'message:user', message: userMessage });
|
|
549
664
|
}
|
|
550
665
|
this.streamNudgeCount = 0;
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
666
|
+
const turnId = nanoid(10);
|
|
667
|
+
try {
|
|
668
|
+
// Retry loop — nudge aborts land here for retry
|
|
669
|
+
for (let attempt = 0; attempt <= this.maxNudgeRetries; attempt++) {
|
|
670
|
+
const abortController = new AbortController();
|
|
671
|
+
const attemptNumber = attempt + 1;
|
|
672
|
+
const isCurrentAttempt = () => this.abortController === abortController;
|
|
673
|
+
this.abortController = abortController;
|
|
674
|
+
this.currentStreamTurn = {
|
|
675
|
+
turnId,
|
|
676
|
+
attempt: attemptNumber,
|
|
677
|
+
conversationId: convId,
|
|
678
|
+
};
|
|
679
|
+
if (attempt > 0) {
|
|
680
|
+
this.emitWatchdog('retry_started', `Retrying stalled turn with attempt ${attemptNumber}.`, { level: 'info' });
|
|
681
|
+
}
|
|
682
|
+
log.info('stream_started', {
|
|
683
|
+
turnId,
|
|
684
|
+
attempt: attemptNumber,
|
|
559
685
|
conversationId: convId,
|
|
560
686
|
});
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
687
|
+
this.emit({ type: 'message:cerebrum:start', conversationId: convId });
|
|
688
|
+
this.startStreamWatchdog();
|
|
689
|
+
let messages = this.conversations.getMessages(convId);
|
|
690
|
+
// Context window compaction
|
|
691
|
+
if (this.compactionConfig.enabled &&
|
|
692
|
+
this.cerebrum?.summarize &&
|
|
693
|
+
shouldCompact(messages, this.compactionConfig.contextWindow, this.compactionConfig.threshold)) {
|
|
694
|
+
try {
|
|
695
|
+
const keepRecent = this.compactionConfig.keepRecentMessages;
|
|
696
|
+
const olderMessages = messages.slice(0, Math.max(0, messages.length - keepRecent));
|
|
697
|
+
if (olderMessages.length > 0) {
|
|
698
|
+
log.info('Compacting conversation', {
|
|
699
|
+
totalMessages: messages.length,
|
|
700
|
+
compactingMessages: olderMessages.length,
|
|
701
|
+
estimatedTokens: estimateMessageTokens(messages),
|
|
702
|
+
});
|
|
703
|
+
const summary = await this.cerebrum.summarize(olderMessages);
|
|
704
|
+
messages = buildCompactionMessages(messages, summary, keepRecent);
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
catch (error) {
|
|
708
|
+
log.warn('Compaction failed, continuing with full context', {
|
|
709
|
+
error: error instanceof Error ? error.message : String(error),
|
|
577
710
|
});
|
|
578
|
-
const summary = await this.cerebrum.summarize(olderMessages);
|
|
579
|
-
messages = buildCompactionMessages(messages, summary, keepRecent);
|
|
580
711
|
}
|
|
581
712
|
}
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
profile: this.profile,
|
|
598
|
-
finetuneStatus: {
|
|
599
|
-
enabled: !!this.fineTuneDataProvider,
|
|
600
|
-
status: this.fineTuneStatus.status,
|
|
601
|
-
progress: this.fineTuneStatus.progress,
|
|
602
|
-
lastJobId: this.fineTuneStatus.jobId || undefined,
|
|
603
|
-
},
|
|
604
|
-
recurringTasks: this.recurringTasks,
|
|
605
|
-
instanceId: instance?.id,
|
|
606
|
-
instanceCreatedAt: instance?.createdAt,
|
|
607
|
-
finetuneCount: instance?.finetuneLineage.length,
|
|
608
|
-
proactiveEnabled: this.proactiveEnabled,
|
|
609
|
-
discoveryMode: this.discoveryMode,
|
|
610
|
-
});
|
|
611
|
-
const systemParts = [basePrompt];
|
|
612
|
-
if (this.systemContext)
|
|
613
|
-
systemParts.push(this.systemContext);
|
|
614
|
-
const fullSystemPrompt = systemParts.join('\n\n---\n\n');
|
|
615
|
-
const allMessages = [
|
|
616
|
-
{ id: 'system', role: 'system', content: fullSystemPrompt, timestamp: 0 },
|
|
617
|
-
...messages,
|
|
618
|
-
];
|
|
619
|
-
const toolDefs = Object.fromEntries(this.tools);
|
|
620
|
-
let fullContent = '';
|
|
621
|
-
try {
|
|
622
|
-
await this.cerebrum.stream(allMessages, toolDefs, {
|
|
623
|
-
onChunk: (chunk) => {
|
|
624
|
-
if (!isCurrentAttempt() || abortController.signal.aborted)
|
|
625
|
-
return;
|
|
626
|
-
fullContent += chunk;
|
|
627
|
-
this.markStreamWaitingModel();
|
|
628
|
-
this.emit({ type: 'message:cerebrum:chunk', chunk });
|
|
713
|
+
// Build system prompt with runtime state + skills context
|
|
714
|
+
const instance = this.instanceStore?.get();
|
|
715
|
+
const basePrompt = buildSystemPrompt({
|
|
716
|
+
cerebellumConnected: this.cerebellum?.isConnected() ?? false,
|
|
717
|
+
tools: this.tools,
|
|
718
|
+
autoMode: this.autoMode,
|
|
719
|
+
gatewayMode: this.gatewayMode,
|
|
720
|
+
connectedNodes: this.connectedNodes,
|
|
721
|
+
gatewayUrl: this.gatewayUrl,
|
|
722
|
+
profile: this.profile,
|
|
723
|
+
finetuneStatus: {
|
|
724
|
+
enabled: !!this.fineTuneDataProvider,
|
|
725
|
+
status: this.fineTuneStatus.status,
|
|
726
|
+
progress: this.fineTuneStatus.progress,
|
|
727
|
+
lastJobId: this.fineTuneStatus.jobId || undefined,
|
|
629
728
|
},
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
729
|
+
recurringTasks: this.recurringTasks,
|
|
730
|
+
instanceId: instance?.id,
|
|
731
|
+
instanceCreatedAt: instance?.createdAt,
|
|
732
|
+
finetuneCount: instance?.finetuneLineage.length,
|
|
733
|
+
proactiveEnabled: this.proactiveEnabled,
|
|
734
|
+
discoveryMode: this.discoveryMode,
|
|
735
|
+
});
|
|
736
|
+
const systemParts = [basePrompt];
|
|
737
|
+
if (this.systemContext)
|
|
738
|
+
systemParts.push(this.systemContext);
|
|
739
|
+
const fullSystemPrompt = systemParts.join('\n\n---\n\n');
|
|
740
|
+
const allMessages = [
|
|
741
|
+
{ id: 'system', role: 'system', content: fullSystemPrompt, timestamp: 0 },
|
|
742
|
+
...messages,
|
|
743
|
+
];
|
|
744
|
+
const toolDefs = Object.fromEntries(this.tools);
|
|
745
|
+
let fullContent = '';
|
|
746
|
+
const throwIfToolAttemptAborted = () => {
|
|
747
|
+
if (!isCurrentAttempt()) {
|
|
748
|
+
throw createAbortError('Tool execution aborted');
|
|
749
|
+
}
|
|
750
|
+
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
751
|
+
};
|
|
752
|
+
try {
|
|
753
|
+
const streamPromise = this.cerebrum.stream(allMessages, toolDefs, {
|
|
754
|
+
onChunk: (chunk) => {
|
|
755
|
+
if (!isCurrentAttempt() || abortController.signal.aborted)
|
|
756
|
+
return;
|
|
757
|
+
fullContent += chunk;
|
|
758
|
+
this.markStreamWaitingModel();
|
|
759
|
+
this.emit({ type: 'message:cerebrum:chunk', chunk });
|
|
760
|
+
},
|
|
761
|
+
onToolCall: async (toolCall) => {
|
|
762
|
+
throwIfToolAttemptAborted();
|
|
763
|
+
this.logStreamDebug('tool_callback_started', {
|
|
764
|
+
toolName: toolCall.name.trim() || toolCall.name,
|
|
765
|
+
toolCallId: toolCall.id,
|
|
766
|
+
});
|
|
767
|
+
this.markStreamWaitingTool(toolCall);
|
|
768
|
+
const requestedToolName = toolCall.name;
|
|
769
|
+
const normalizedToolName = requestedToolName.trim() || requestedToolName;
|
|
770
|
+
this.emit({ type: 'message:cerebrum:toolcall', toolCall: { ...toolCall, name: normalizedToolName } });
|
|
771
|
+
this.emit({ type: 'tool:start', callId: toolCall.id, name: normalizedToolName });
|
|
772
|
+
const { toolName, result } = await this.toolRuntime.execute({
|
|
773
|
+
toolCall,
|
|
774
|
+
tools: this.tools,
|
|
775
|
+
conversationId: convId,
|
|
776
|
+
sessionKey: 'agent:main',
|
|
777
|
+
scopeKey: convId,
|
|
778
|
+
abortSignal: abortController.signal,
|
|
779
|
+
});
|
|
780
|
+
this.logStreamDebug('tool_callback_finished', {
|
|
781
|
+
toolName,
|
|
782
|
+
toolCallId: toolCall.id,
|
|
783
|
+
isError: result.isError,
|
|
784
|
+
});
|
|
785
|
+
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
786
|
+
this.markStreamWaitingModel();
|
|
787
|
+
this.emit({ type: 'tool:end', result });
|
|
788
|
+
// Cerebellum verification (non-blocking)
|
|
789
|
+
if (this.cerebellum?.isConnected() && this.verificationEnabled) {
|
|
790
|
+
try {
|
|
791
|
+
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
792
|
+
this.emit({ type: 'verification:start', callId: toolCall.id, toolName });
|
|
793
|
+
const toolArgs = {};
|
|
794
|
+
for (const [k, v] of Object.entries(toolCall.args)) {
|
|
795
|
+
toolArgs[k] = String(v);
|
|
796
|
+
}
|
|
797
|
+
const verifyPromise = this.cerebellum.verifyToolResult(toolName, toolArgs, result.output, !result.isError);
|
|
798
|
+
const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve(null), this.verificationTimeoutMs));
|
|
799
|
+
const verification = await Promise.race([verifyPromise, timeoutPromise]);
|
|
800
|
+
throwIfAborted(abortController.signal, 'Tool execution aborted');
|
|
801
|
+
if (verification && !verification.passed) {
|
|
802
|
+
const failedChecks = verification.checks
|
|
803
|
+
.filter((c) => !c.passed)
|
|
804
|
+
.map((c) => c.description)
|
|
805
|
+
.join(', ');
|
|
806
|
+
result.output += `\n[Cerebellum warning: ${failedChecks}]`;
|
|
807
|
+
}
|
|
808
|
+
if (verification) {
|
|
809
|
+
const vResult = {
|
|
810
|
+
passed: verification.passed,
|
|
811
|
+
checks: verification.checks,
|
|
812
|
+
modelVerdict: verification.modelVerdict,
|
|
813
|
+
toolCallId: toolCall.id,
|
|
814
|
+
toolName,
|
|
815
|
+
};
|
|
816
|
+
this.emit({ type: 'verification:end', result: vResult });
|
|
817
|
+
}
|
|
683
818
|
}
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
passed: verification.passed,
|
|
687
|
-
checks: verification.checks,
|
|
688
|
-
modelVerdict: verification.modelVerdict,
|
|
689
|
-
toolCallId: toolCall.id,
|
|
690
|
-
toolName,
|
|
691
|
-
};
|
|
692
|
-
this.emit({ type: 'verification:end', result: vResult });
|
|
819
|
+
catch {
|
|
820
|
+
// Verification failure should never block tool execution
|
|
693
821
|
}
|
|
694
822
|
}
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
displayContent = content
|
|
723
|
-
.replace(/<discovery_complete>[\s\S]*?<\/discovery_complete>/g, '')
|
|
724
|
-
.trim();
|
|
725
|
-
if (parsed && this.onDiscoveryComplete) {
|
|
726
|
-
this.discoveryMode = false;
|
|
727
|
-
this.onDiscoveryComplete(parsed);
|
|
728
|
-
log.info('Discovery completed', { name: parsed.name });
|
|
823
|
+
throwIfToolAttemptAborted();
|
|
824
|
+
this.conversations.appendMessage(convId, 'tool', result.output, {
|
|
825
|
+
toolResult: result,
|
|
826
|
+
metadata: {
|
|
827
|
+
toolName,
|
|
828
|
+
...(requestedToolName !== toolName ? { requestedToolName } : {}),
|
|
829
|
+
},
|
|
830
|
+
});
|
|
831
|
+
return result;
|
|
832
|
+
},
|
|
833
|
+
onFinish: (content, toolCalls) => {
|
|
834
|
+
if (!isCurrentAttempt() || abortController.signal.aborted)
|
|
835
|
+
return;
|
|
836
|
+
this.stopStreamWatchdog();
|
|
837
|
+
let displayContent = content;
|
|
838
|
+
// Check for discovery completion — parse and strip the tag before storing
|
|
839
|
+
if (this.discoveryMode && content.includes('<discovery_complete>')) {
|
|
840
|
+
const parsed = this.parseDiscoveryCompletion(content);
|
|
841
|
+
// Strip the tag block from the displayed/stored content
|
|
842
|
+
displayContent = content
|
|
843
|
+
.replace(/<discovery_complete>[\s\S]*?<\/discovery_complete>/g, '')
|
|
844
|
+
.trim();
|
|
845
|
+
if (parsed && this.onDiscoveryComplete) {
|
|
846
|
+
this.discoveryMode = false;
|
|
847
|
+
this.onDiscoveryComplete(parsed);
|
|
848
|
+
log.info('Discovery completed', { name: parsed.name });
|
|
849
|
+
}
|
|
729
850
|
}
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
attempt,
|
|
851
|
+
const cerebrumMessage = this.conversations.appendMessage(convId, 'cerebrum', displayContent, toolCalls?.length ? { toolCalls } : undefined);
|
|
852
|
+
this.emit({ type: 'message:cerebrum:end', message: cerebrumMessage });
|
|
853
|
+
log.info('stream_finished', {
|
|
854
|
+
turnId,
|
|
855
|
+
attempt: attemptNumber,
|
|
736
856
|
conversationId: convId,
|
|
737
857
|
});
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
this.stopStreamWatchdog();
|
|
755
|
-
// Check if this was a nudge-abort (not emergency stop, not a real error)
|
|
756
|
-
const isNudgeAbort = abortController.signal.aborted && this.streamNudgeCount > 0 && this.streamNudgeCount <= this.maxNudgeRetries;
|
|
757
|
-
if (isNudgeAbort) {
|
|
758
|
-
// Inject nudge message and retry via the for-loop
|
|
759
|
-
const systemMessage = this.conversations.appendMessage(convId, 'system', '[Cerebellum] You stopped mid-response. Continue from where you left off.');
|
|
760
|
-
this.emit({ type: 'message:system', message: systemMessage });
|
|
761
|
-
continue; // retry loop
|
|
858
|
+
if (attempt > 0) {
|
|
859
|
+
this.emitWatchdog('retry_recovered', `Retry attempt ${attemptNumber} recovered the stalled turn.`, { level: 'info' });
|
|
860
|
+
}
|
|
861
|
+
},
|
|
862
|
+
onError: (error) => {
|
|
863
|
+
if (!isCurrentAttempt())
|
|
864
|
+
return;
|
|
865
|
+
this.stopStreamWatchdog();
|
|
866
|
+
// Don't log/emit if the abort was intentional (nudge or Cerebellum disconnect) — catch block handles it
|
|
867
|
+
if (abortController.signal.aborted)
|
|
868
|
+
return;
|
|
869
|
+
log.error('Cerebrum stream error', { error: error.message });
|
|
870
|
+
this.emit({ type: 'error', error });
|
|
871
|
+
},
|
|
872
|
+
}, { abortSignal: abortController.signal });
|
|
873
|
+
await this.awaitStreamAttempt(streamPromise, abortController);
|
|
762
874
|
}
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
875
|
+
catch (error) {
|
|
876
|
+
const failureState = this.getStreamState();
|
|
877
|
+
this.stopStreamWatchdog();
|
|
878
|
+
// Check if this was a nudge-abort (not emergency stop, not a real error)
|
|
879
|
+
const isNudgeAbort = abortController.signal.aborted && this.streamNudgeCount > 0 && this.streamNudgeCount <= this.maxNudgeRetries;
|
|
880
|
+
if (isNudgeAbort) {
|
|
881
|
+
// Inject nudge message and retry via the for-loop
|
|
882
|
+
const systemMessage = this.conversations.appendMessage(convId, 'system', '[Cerebellum] You stopped mid-response. Continue from where you left off.');
|
|
883
|
+
this.emit({ type: 'message:system', message: systemMessage });
|
|
884
|
+
continue; // retry loop
|
|
885
|
+
}
|
|
886
|
+
// Check if Cerebellum dropped mid-stream
|
|
887
|
+
if (this.cerebellum && !this.cerebellum.isConnected() && abortController.signal.aborted) {
|
|
888
|
+
const err = new Error('Cerebellum disconnected during active response. Restart it with: docker compose up -d cerebellum');
|
|
889
|
+
log.error('Cerebellum disconnected mid-stream', { error: err.message });
|
|
890
|
+
this.emit({ type: 'error', error: err });
|
|
891
|
+
break;
|
|
892
|
+
}
|
|
893
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
894
|
+
if (attempt > 0) {
|
|
895
|
+
this.emitWatchdog('retry_failed', `Retry attempt ${attemptNumber} failed: ${err.message}`, { level: 'error' });
|
|
896
|
+
}
|
|
897
|
+
log.error('Send message failed', {
|
|
898
|
+
error: err.message,
|
|
899
|
+
turnId,
|
|
900
|
+
attempt: attemptNumber,
|
|
901
|
+
conversationId: convId,
|
|
902
|
+
phase: failureState.phase,
|
|
903
|
+
activeToolName: failureState.activeToolName,
|
|
904
|
+
activeToolCallId: failureState.activeToolCallId,
|
|
905
|
+
activeToolStartedAt: failureState.activeToolStartedAt,
|
|
906
|
+
});
|
|
767
907
|
this.emit({ type: 'error', error: err });
|
|
768
|
-
break;
|
|
769
908
|
}
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
activeToolName: failureState.activeToolName,
|
|
777
|
-
activeToolCallId: failureState.activeToolCallId,
|
|
778
|
-
activeToolStartedAt: failureState.activeToolStartedAt,
|
|
779
|
-
});
|
|
780
|
-
this.emit({ type: 'error', error: err });
|
|
781
|
-
}
|
|
782
|
-
break; // success — exit retry loop
|
|
783
|
-
} // end retry for-loop
|
|
909
|
+
break; // success — exit retry loop
|
|
910
|
+
} // end retry for-loop
|
|
911
|
+
}
|
|
912
|
+
finally {
|
|
913
|
+
this.currentStreamTurn = null;
|
|
914
|
+
}
|
|
784
915
|
}
|
|
785
916
|
async start() {
|
|
786
917
|
if (!this.activeConversationId) {
|