@cereworker/core 26.329.23 → 26.329.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,6 +6,7 @@ import { createSubAgentTools } from './sub-agent-tools.js';
6
6
  import { createLogger } from './logger.js';
7
7
  import { buildSystemPrompt } from './system-prompt.js';
8
8
  import { estimateMessageTokens, shouldCompact, buildCompactionMessages } from './context.js';
9
+ import { createAbortError, throwIfAborted } from './abort.js';
9
10
  import { ToolRuntime, } from './tool-runtime.js';
10
11
  const log = createLogger('orchestrator');
11
12
  export class Orchestrator extends TypedEventEmitter {
@@ -47,6 +48,8 @@ export class Orchestrator extends TypedEventEmitter {
47
48
  maxNudgeRetries = 2;
48
49
  streamPhase = 'idle';
49
50
  activeToolCall = null;
51
+ currentStreamTurn = null;
52
+ streamAbortGraceMs = 1_000;
50
53
  taskConversations = new Map();
51
54
  taskRunning = new Set();
52
55
  recurringTasks = [];
@@ -453,18 +456,35 @@ export class Orchestrator extends TypedEventEmitter {
453
456
  };
454
457
  }
455
458
  markStreamWaitingModel(activityAt = Date.now()) {
459
+ const phaseChanged = this.streamPhase !== 'waiting_model' || this.activeToolCall !== null;
456
460
  this.lastStreamActivityAt = activityAt;
457
461
  this.streamPhase = 'waiting_model';
458
462
  this.activeToolCall = null;
463
+ if (phaseChanged) {
464
+ this.logStreamDebug('stream_phase_changed', {
465
+ phase: this.streamPhase,
466
+ });
467
+ }
459
468
  }
460
469
  markStreamWaitingTool(toolCall, activityAt = Date.now()) {
470
+ const normalizedToolName = toolCall.name.trim() || toolCall.name;
471
+ const phaseChanged = this.streamPhase !== 'waiting_tool'
472
+ || this.activeToolCall?.id !== toolCall.id
473
+ || this.activeToolCall?.name !== normalizedToolName;
461
474
  this.lastStreamActivityAt = activityAt;
462
475
  this.streamPhase = 'waiting_tool';
463
476
  this.activeToolCall = {
464
477
  id: toolCall.id,
465
- name: toolCall.name.trim() || toolCall.name,
478
+ name: normalizedToolName,
466
479
  startedAt: activityAt,
467
480
  };
481
+ if (phaseChanged) {
482
+ this.logStreamDebug('stream_phase_changed', {
483
+ phase: this.streamPhase,
484
+ activeToolName: normalizedToolName,
485
+ activeToolCallId: toolCall.id,
486
+ });
487
+ }
468
488
  }
469
489
  resetStreamState() {
470
490
  this.streamPhase = 'idle';
@@ -479,6 +499,100 @@ export class Orchestrator extends TypedEventEmitter {
479
499
  activeToolStartedAt: this.activeToolCall?.startedAt,
480
500
  };
481
501
  }
502
+ describeStreamLocation() {
503
+ if (this.streamPhase === 'waiting_tool') {
504
+ return this.activeToolCall?.name
505
+ ? `waiting_tool/${this.activeToolCall.name}`
506
+ : 'waiting_tool';
507
+ }
508
+ return this.streamPhase;
509
+ }
510
+ logStreamDebug(msg, data) {
511
+ if (!this.currentStreamTurn)
512
+ return;
513
+ log.debug(msg, {
514
+ turnId: this.currentStreamTurn.turnId,
515
+ attempt: this.currentStreamTurn.attempt,
516
+ conversationId: this.currentStreamTurn.conversationId,
517
+ ...data,
518
+ });
519
+ }
520
+ emitWatchdog(stage, message, options) {
521
+ if (!this.currentStreamTurn)
522
+ return;
523
+ const payload = {
524
+ stage,
525
+ turnId: this.currentStreamTurn.turnId,
526
+ attempt: this.currentStreamTurn.attempt,
527
+ conversationId: this.currentStreamTurn.conversationId,
528
+ message,
529
+ ...this.getStreamDiagnostics(options?.elapsedSeconds),
530
+ };
531
+ const level = options?.level ?? 'info';
532
+ switch (level) {
533
+ case 'debug':
534
+ log.debug(`watchdog_${stage}`, payload);
535
+ break;
536
+ case 'warn':
537
+ log.warn(`watchdog_${stage}`, payload);
538
+ break;
539
+ case 'error':
540
+ log.error(`watchdog_${stage}`, payload);
541
+ break;
542
+ default:
543
+ log.info(`watchdog_${stage}`, payload);
544
+ break;
545
+ }
546
+ this.emit({ type: 'cerebrum:watchdog', ...payload });
547
+ }
548
+ async awaitStreamAttempt(streamPromise, abortController) {
549
+ return new Promise((resolve, reject) => {
550
+ let settled = false;
551
+ let abortTimer = null;
552
+ const cleanup = () => {
553
+ abortController.signal.removeEventListener('abort', onAbort);
554
+ if (abortTimer) {
555
+ clearTimeout(abortTimer);
556
+ abortTimer = null;
557
+ }
558
+ };
559
+ const settleResolve = () => {
560
+ if (settled)
561
+ return;
562
+ settled = true;
563
+ cleanup();
564
+ resolve();
565
+ };
566
+ const settleReject = (error) => {
567
+ if (settled)
568
+ return;
569
+ settled = true;
570
+ cleanup();
571
+ reject(error);
572
+ };
573
+ const onAbort = () => {
574
+ this.logStreamDebug('provider_abort_observed', {
575
+ phase: this.streamPhase,
576
+ activeToolName: this.activeToolCall?.name,
577
+ activeToolCallId: this.activeToolCall?.id,
578
+ });
579
+ if (abortTimer)
580
+ return;
581
+ abortTimer = setTimeout(() => {
582
+ if (settled)
583
+ return;
584
+ const elapsedSeconds = Math.max(1, Math.round((Date.now() - this.lastStreamActivityAt) / 1000));
585
+ this.emitWatchdog('teardown_timeout', `Provider did not settle within ${this.streamAbortGraceMs}ms after abort; continuing retry.`, { level: 'warn', elapsedSeconds });
586
+ settleReject(createAbortError('Stream aborted'));
587
+ }, this.streamAbortGraceMs);
588
+ };
589
+ abortController.signal.addEventListener('abort', onAbort, { once: true });
590
+ if (abortController.signal.aborted) {
591
+ onAbort();
592
+ }
593
+ streamPromise.then(settleResolve, settleReject);
594
+ });
595
+ }
482
596
  startStreamWatchdog() {
483
597
  this.stopStreamWatchdog();
484
598
  this.markStreamWaitingModel();
@@ -492,19 +606,20 @@ export class Orchestrator extends TypedEventEmitter {
492
606
  return;
493
607
  const elapsedSeconds = Math.round(elapsed / 1000);
494
608
  const diagnostics = this.getStreamDiagnostics(elapsedSeconds);
495
- log.warn('Cerebrum stream stalled', diagnostics);
609
+ this.emitWatchdog('stalled', `Stalled after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'warn', elapsedSeconds });
496
610
  this.emit({ type: 'cerebrum:stall', ...diagnostics });
497
611
  if (!this.cerebellum?.isConnected()) {
498
612
  // Cerebellum dropped mid-stream — abort the current turn
499
- log.warn('Cerebellum disconnected during active stream aborting');
613
+ this.emitWatchdog('abort_issued', 'Cerebellum disconnected during an active stream; aborting the turn.', { level: 'warn', elapsedSeconds });
500
614
  this.abortController?.abort();
501
615
  return;
502
616
  }
503
617
  this._nudgeInFlight = true;
504
618
  const doNudge = () => {
505
619
  this.streamNudgeCount++;
506
- log.info('Cerebellum nudging stalled stream', { attempt: this.streamNudgeCount, ...diagnostics });
620
+ this.emitWatchdog('nudge_requested', `Cerebellum requested nudge ${this.streamNudgeCount}/${this.maxNudgeRetries} after ${elapsedSeconds}s while ${this.describeStreamLocation()}.`, { level: 'info', elapsedSeconds });
507
621
  this.emit({ type: 'cerebrum:stall:nudge', attempt: this.streamNudgeCount, ...diagnostics });
622
+ this.emitWatchdog('abort_issued', `Aborting stalled stream attempt ${this.currentStreamTurn?.attempt ?? 0}.`, { level: 'warn', elapsedSeconds });
508
623
  this.abortController?.abort();
509
624
  };
510
625
  void (async () => {
@@ -548,239 +663,255 @@ export class Orchestrator extends TypedEventEmitter {
548
663
  this.emit({ type: 'message:user', message: userMessage });
549
664
  }
550
665
  this.streamNudgeCount = 0;
551
- // Retry loop — nudge aborts land here for retry
552
- for (let attempt = 0; attempt <= this.maxNudgeRetries; attempt++) {
553
- const abortController = new AbortController();
554
- const isCurrentAttempt = () => this.abortController === abortController;
555
- this.abortController = abortController;
556
- if (attempt > 0) {
557
- log.info('Retrying Cerebrum stream after watchdog nudge', {
558
- attempt,
666
+ const turnId = nanoid(10);
667
+ try {
668
+ // Retry loop nudge aborts land here for retry
669
+ for (let attempt = 0; attempt <= this.maxNudgeRetries; attempt++) {
670
+ const abortController = new AbortController();
671
+ const attemptNumber = attempt + 1;
672
+ const isCurrentAttempt = () => this.abortController === abortController;
673
+ this.abortController = abortController;
674
+ this.currentStreamTurn = {
675
+ turnId,
676
+ attempt: attemptNumber,
677
+ conversationId: convId,
678
+ };
679
+ if (attempt > 0) {
680
+ this.emitWatchdog('retry_started', `Retrying stalled turn with attempt ${attemptNumber}.`, { level: 'info' });
681
+ }
682
+ log.info('stream_started', {
683
+ turnId,
684
+ attempt: attemptNumber,
559
685
  conversationId: convId,
560
686
  });
561
- }
562
- this.emit({ type: 'message:cerebrum:start', conversationId: convId });
563
- this.startStreamWatchdog();
564
- let messages = this.conversations.getMessages(convId);
565
- // Context window compaction
566
- if (this.compactionConfig.enabled &&
567
- this.cerebrum?.summarize &&
568
- shouldCompact(messages, this.compactionConfig.contextWindow, this.compactionConfig.threshold)) {
569
- try {
570
- const keepRecent = this.compactionConfig.keepRecentMessages;
571
- const olderMessages = messages.slice(0, Math.max(0, messages.length - keepRecent));
572
- if (olderMessages.length > 0) {
573
- log.info('Compacting conversation', {
574
- totalMessages: messages.length,
575
- compactingMessages: olderMessages.length,
576
- estimatedTokens: estimateMessageTokens(messages),
687
+ this.emit({ type: 'message:cerebrum:start', conversationId: convId });
688
+ this.startStreamWatchdog();
689
+ let messages = this.conversations.getMessages(convId);
690
+ // Context window compaction
691
+ if (this.compactionConfig.enabled &&
692
+ this.cerebrum?.summarize &&
693
+ shouldCompact(messages, this.compactionConfig.contextWindow, this.compactionConfig.threshold)) {
694
+ try {
695
+ const keepRecent = this.compactionConfig.keepRecentMessages;
696
+ const olderMessages = messages.slice(0, Math.max(0, messages.length - keepRecent));
697
+ if (olderMessages.length > 0) {
698
+ log.info('Compacting conversation', {
699
+ totalMessages: messages.length,
700
+ compactingMessages: olderMessages.length,
701
+ estimatedTokens: estimateMessageTokens(messages),
702
+ });
703
+ const summary = await this.cerebrum.summarize(olderMessages);
704
+ messages = buildCompactionMessages(messages, summary, keepRecent);
705
+ }
706
+ }
707
+ catch (error) {
708
+ log.warn('Compaction failed, continuing with full context', {
709
+ error: error instanceof Error ? error.message : String(error),
577
710
  });
578
- const summary = await this.cerebrum.summarize(olderMessages);
579
- messages = buildCompactionMessages(messages, summary, keepRecent);
580
711
  }
581
712
  }
582
- catch (error) {
583
- log.warn('Compaction failed, continuing with full context', {
584
- error: error instanceof Error ? error.message : String(error),
585
- });
586
- }
587
- }
588
- // Build system prompt with runtime state + skills context
589
- const instance = this.instanceStore?.get();
590
- const basePrompt = buildSystemPrompt({
591
- cerebellumConnected: this.cerebellum?.isConnected() ?? false,
592
- tools: this.tools,
593
- autoMode: this.autoMode,
594
- gatewayMode: this.gatewayMode,
595
- connectedNodes: this.connectedNodes,
596
- gatewayUrl: this.gatewayUrl,
597
- profile: this.profile,
598
- finetuneStatus: {
599
- enabled: !!this.fineTuneDataProvider,
600
- status: this.fineTuneStatus.status,
601
- progress: this.fineTuneStatus.progress,
602
- lastJobId: this.fineTuneStatus.jobId || undefined,
603
- },
604
- recurringTasks: this.recurringTasks,
605
- instanceId: instance?.id,
606
- instanceCreatedAt: instance?.createdAt,
607
- finetuneCount: instance?.finetuneLineage.length,
608
- proactiveEnabled: this.proactiveEnabled,
609
- discoveryMode: this.discoveryMode,
610
- });
611
- const systemParts = [basePrompt];
612
- if (this.systemContext)
613
- systemParts.push(this.systemContext);
614
- const fullSystemPrompt = systemParts.join('\n\n---\n\n');
615
- const allMessages = [
616
- { id: 'system', role: 'system', content: fullSystemPrompt, timestamp: 0 },
617
- ...messages,
618
- ];
619
- const toolDefs = Object.fromEntries(this.tools);
620
- let fullContent = '';
621
- try {
622
- await this.cerebrum.stream(allMessages, toolDefs, {
623
- onChunk: (chunk) => {
624
- if (!isCurrentAttempt() || abortController.signal.aborted)
625
- return;
626
- fullContent += chunk;
627
- this.markStreamWaitingModel();
628
- this.emit({ type: 'message:cerebrum:chunk', chunk });
713
+ // Build system prompt with runtime state + skills context
714
+ const instance = this.instanceStore?.get();
715
+ const basePrompt = buildSystemPrompt({
716
+ cerebellumConnected: this.cerebellum?.isConnected() ?? false,
717
+ tools: this.tools,
718
+ autoMode: this.autoMode,
719
+ gatewayMode: this.gatewayMode,
720
+ connectedNodes: this.connectedNodes,
721
+ gatewayUrl: this.gatewayUrl,
722
+ profile: this.profile,
723
+ finetuneStatus: {
724
+ enabled: !!this.fineTuneDataProvider,
725
+ status: this.fineTuneStatus.status,
726
+ progress: this.fineTuneStatus.progress,
727
+ lastJobId: this.fineTuneStatus.jobId || undefined,
629
728
  },
630
- onToolCall: async (toolCall) => {
631
- if (!isCurrentAttempt() || abortController.signal.aborted) {
632
- const error = new Error('Tool execution aborted');
633
- error.name = 'AbortError';
634
- throw error;
635
- }
636
- this.markStreamWaitingTool(toolCall);
637
- const requestedToolName = toolCall.name;
638
- const normalizedToolName = requestedToolName.trim() || requestedToolName;
639
- this.emit({ type: 'message:cerebrum:toolcall', toolCall: { ...toolCall, name: normalizedToolName } });
640
- this.emit({ type: 'tool:start', callId: toolCall.id, name: normalizedToolName });
641
- const { toolName, result } = await this.toolRuntime.execute({
642
- toolCall,
643
- tools: this.tools,
644
- conversationId: convId,
645
- sessionKey: 'agent:main',
646
- scopeKey: convId,
647
- abortSignal: abortController.signal,
648
- });
649
- if (abortController.signal.aborted) {
650
- const error = new Error('Tool execution aborted');
651
- error.name = 'AbortError';
652
- throw error;
653
- }
654
- this.markStreamWaitingModel();
655
- this.emit({ type: 'tool:end', result });
656
- // Cerebellum verification (non-blocking)
657
- if (this.cerebellum?.isConnected() && this.verificationEnabled) {
658
- try {
659
- if (abortController.signal.aborted) {
660
- const error = new Error('Tool execution aborted');
661
- error.name = 'AbortError';
662
- throw error;
663
- }
664
- this.emit({ type: 'verification:start', callId: toolCall.id, toolName });
665
- const toolArgs = {};
666
- for (const [k, v] of Object.entries(toolCall.args)) {
667
- toolArgs[k] = String(v);
668
- }
669
- const verifyPromise = this.cerebellum.verifyToolResult(toolName, toolArgs, result.output, !result.isError);
670
- const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve(null), this.verificationTimeoutMs));
671
- const verification = await Promise.race([verifyPromise, timeoutPromise]);
672
- if (abortController.signal.aborted) {
673
- const error = new Error('Tool execution aborted');
674
- error.name = 'AbortError';
675
- throw error;
676
- }
677
- if (verification && !verification.passed) {
678
- const failedChecks = verification.checks
679
- .filter((c) => !c.passed)
680
- .map((c) => c.description)
681
- .join(', ');
682
- result.output += `\n[Cerebellum warning: ${failedChecks}]`;
729
+ recurringTasks: this.recurringTasks,
730
+ instanceId: instance?.id,
731
+ instanceCreatedAt: instance?.createdAt,
732
+ finetuneCount: instance?.finetuneLineage.length,
733
+ proactiveEnabled: this.proactiveEnabled,
734
+ discoveryMode: this.discoveryMode,
735
+ });
736
+ const systemParts = [basePrompt];
737
+ if (this.systemContext)
738
+ systemParts.push(this.systemContext);
739
+ const fullSystemPrompt = systemParts.join('\n\n---\n\n');
740
+ const allMessages = [
741
+ { id: 'system', role: 'system', content: fullSystemPrompt, timestamp: 0 },
742
+ ...messages,
743
+ ];
744
+ const toolDefs = Object.fromEntries(this.tools);
745
+ let fullContent = '';
746
+ const throwIfToolAttemptAborted = () => {
747
+ if (!isCurrentAttempt()) {
748
+ throw createAbortError('Tool execution aborted');
749
+ }
750
+ throwIfAborted(abortController.signal, 'Tool execution aborted');
751
+ };
752
+ try {
753
+ const streamPromise = this.cerebrum.stream(allMessages, toolDefs, {
754
+ onChunk: (chunk) => {
755
+ if (!isCurrentAttempt() || abortController.signal.aborted)
756
+ return;
757
+ fullContent += chunk;
758
+ this.markStreamWaitingModel();
759
+ this.emit({ type: 'message:cerebrum:chunk', chunk });
760
+ },
761
+ onToolCall: async (toolCall) => {
762
+ throwIfToolAttemptAborted();
763
+ this.logStreamDebug('tool_callback_started', {
764
+ toolName: toolCall.name.trim() || toolCall.name,
765
+ toolCallId: toolCall.id,
766
+ });
767
+ this.markStreamWaitingTool(toolCall);
768
+ const requestedToolName = toolCall.name;
769
+ const normalizedToolName = requestedToolName.trim() || requestedToolName;
770
+ this.emit({ type: 'message:cerebrum:toolcall', toolCall: { ...toolCall, name: normalizedToolName } });
771
+ this.emit({ type: 'tool:start', callId: toolCall.id, name: normalizedToolName });
772
+ const { toolName, result } = await this.toolRuntime.execute({
773
+ toolCall,
774
+ tools: this.tools,
775
+ conversationId: convId,
776
+ sessionKey: 'agent:main',
777
+ scopeKey: convId,
778
+ abortSignal: abortController.signal,
779
+ });
780
+ this.logStreamDebug('tool_callback_finished', {
781
+ toolName,
782
+ toolCallId: toolCall.id,
783
+ isError: result.isError,
784
+ });
785
+ throwIfAborted(abortController.signal, 'Tool execution aborted');
786
+ this.markStreamWaitingModel();
787
+ this.emit({ type: 'tool:end', result });
788
+ // Cerebellum verification (non-blocking)
789
+ if (this.cerebellum?.isConnected() && this.verificationEnabled) {
790
+ try {
791
+ throwIfAborted(abortController.signal, 'Tool execution aborted');
792
+ this.emit({ type: 'verification:start', callId: toolCall.id, toolName });
793
+ const toolArgs = {};
794
+ for (const [k, v] of Object.entries(toolCall.args)) {
795
+ toolArgs[k] = String(v);
796
+ }
797
+ const verifyPromise = this.cerebellum.verifyToolResult(toolName, toolArgs, result.output, !result.isError);
798
+ const timeoutPromise = new Promise((resolve) => setTimeout(() => resolve(null), this.verificationTimeoutMs));
799
+ const verification = await Promise.race([verifyPromise, timeoutPromise]);
800
+ throwIfAborted(abortController.signal, 'Tool execution aborted');
801
+ if (verification && !verification.passed) {
802
+ const failedChecks = verification.checks
803
+ .filter((c) => !c.passed)
804
+ .map((c) => c.description)
805
+ .join(', ');
806
+ result.output += `\n[Cerebellum warning: ${failedChecks}]`;
807
+ }
808
+ if (verification) {
809
+ const vResult = {
810
+ passed: verification.passed,
811
+ checks: verification.checks,
812
+ modelVerdict: verification.modelVerdict,
813
+ toolCallId: toolCall.id,
814
+ toolName,
815
+ };
816
+ this.emit({ type: 'verification:end', result: vResult });
817
+ }
683
818
  }
684
- if (verification) {
685
- const vResult = {
686
- passed: verification.passed,
687
- checks: verification.checks,
688
- modelVerdict: verification.modelVerdict,
689
- toolCallId: toolCall.id,
690
- toolName,
691
- };
692
- this.emit({ type: 'verification:end', result: vResult });
819
+ catch {
820
+ // Verification failure should never block tool execution
693
821
  }
694
822
  }
695
- catch {
696
- // Verification failure should never block tool execution
697
- }
698
- }
699
- if (!isCurrentAttempt() || abortController.signal.aborted) {
700
- const error = new Error('Tool execution aborted');
701
- error.name = 'AbortError';
702
- throw error;
703
- }
704
- this.conversations.appendMessage(convId, 'tool', result.output, {
705
- toolResult: result,
706
- metadata: {
707
- toolName,
708
- ...(requestedToolName !== toolName ? { requestedToolName } : {}),
709
- },
710
- });
711
- return result;
712
- },
713
- onFinish: (content, toolCalls) => {
714
- if (!isCurrentAttempt() || abortController.signal.aborted)
715
- return;
716
- this.stopStreamWatchdog();
717
- let displayContent = content;
718
- // Check for discovery completion — parse and strip the tag before storing
719
- if (this.discoveryMode && content.includes('<discovery_complete>')) {
720
- const parsed = this.parseDiscoveryCompletion(content);
721
- // Strip the tag block from the displayed/stored content
722
- displayContent = content
723
- .replace(/<discovery_complete>[\s\S]*?<\/discovery_complete>/g, '')
724
- .trim();
725
- if (parsed && this.onDiscoveryComplete) {
726
- this.discoveryMode = false;
727
- this.onDiscoveryComplete(parsed);
728
- log.info('Discovery completed', { name: parsed.name });
823
+ throwIfToolAttemptAborted();
824
+ this.conversations.appendMessage(convId, 'tool', result.output, {
825
+ toolResult: result,
826
+ metadata: {
827
+ toolName,
828
+ ...(requestedToolName !== toolName ? { requestedToolName } : {}),
829
+ },
830
+ });
831
+ return result;
832
+ },
833
+ onFinish: (content, toolCalls) => {
834
+ if (!isCurrentAttempt() || abortController.signal.aborted)
835
+ return;
836
+ this.stopStreamWatchdog();
837
+ let displayContent = content;
838
+ // Check for discovery completion — parse and strip the tag before storing
839
+ if (this.discoveryMode && content.includes('<discovery_complete>')) {
840
+ const parsed = this.parseDiscoveryCompletion(content);
841
+ // Strip the tag block from the displayed/stored content
842
+ displayContent = content
843
+ .replace(/<discovery_complete>[\s\S]*?<\/discovery_complete>/g, '')
844
+ .trim();
845
+ if (parsed && this.onDiscoveryComplete) {
846
+ this.discoveryMode = false;
847
+ this.onDiscoveryComplete(parsed);
848
+ log.info('Discovery completed', { name: parsed.name });
849
+ }
729
850
  }
730
- }
731
- const cerebrumMessage = this.conversations.appendMessage(convId, 'cerebrum', displayContent, toolCalls?.length ? { toolCalls } : undefined);
732
- this.emit({ type: 'message:cerebrum:end', message: cerebrumMessage });
733
- if (attempt > 0) {
734
- log.info('Cerebrum stream recovered after watchdog retry', {
735
- attempt,
851
+ const cerebrumMessage = this.conversations.appendMessage(convId, 'cerebrum', displayContent, toolCalls?.length ? { toolCalls } : undefined);
852
+ this.emit({ type: 'message:cerebrum:end', message: cerebrumMessage });
853
+ log.info('stream_finished', {
854
+ turnId,
855
+ attempt: attemptNumber,
736
856
  conversationId: convId,
737
857
  });
738
- }
739
- },
740
- onError: (error) => {
741
- if (!isCurrentAttempt())
742
- return;
743
- this.stopStreamWatchdog();
744
- // Don't log/emit if the abort was intentional (nudge or Cerebellum disconnect) — catch block handles it
745
- if (abortController.signal.aborted)
746
- return;
747
- log.error('Cerebrum stream error', { error: error.message });
748
- this.emit({ type: 'error', error });
749
- },
750
- }, { abortSignal: abortController.signal });
751
- }
752
- catch (error) {
753
- const failureState = this.getStreamState();
754
- this.stopStreamWatchdog();
755
- // Check if this was a nudge-abort (not emergency stop, not a real error)
756
- const isNudgeAbort = abortController.signal.aborted && this.streamNudgeCount > 0 && this.streamNudgeCount <= this.maxNudgeRetries;
757
- if (isNudgeAbort) {
758
- // Inject nudge message and retry via the for-loop
759
- const systemMessage = this.conversations.appendMessage(convId, 'system', '[Cerebellum] You stopped mid-response. Continue from where you left off.');
760
- this.emit({ type: 'message:system', message: systemMessage });
761
- continue; // retry loop
858
+ if (attempt > 0) {
859
+ this.emitWatchdog('retry_recovered', `Retry attempt ${attemptNumber} recovered the stalled turn.`, { level: 'info' });
860
+ }
861
+ },
862
+ onError: (error) => {
863
+ if (!isCurrentAttempt())
864
+ return;
865
+ this.stopStreamWatchdog();
866
+ // Don't log/emit if the abort was intentional (nudge or Cerebellum disconnect) — catch block handles it
867
+ if (abortController.signal.aborted)
868
+ return;
869
+ log.error('Cerebrum stream error', { error: error.message });
870
+ this.emit({ type: 'error', error });
871
+ },
872
+ }, { abortSignal: abortController.signal });
873
+ await this.awaitStreamAttempt(streamPromise, abortController);
762
874
  }
763
- // Check if Cerebellum dropped mid-stream
764
- if (this.cerebellum && !this.cerebellum.isConnected() && abortController.signal.aborted) {
765
- const err = new Error('Cerebellum disconnected during active response. Restart it with: docker compose up -d cerebellum');
766
- log.error('Cerebellum disconnected mid-stream', { error: err.message });
875
+ catch (error) {
876
+ const failureState = this.getStreamState();
877
+ this.stopStreamWatchdog();
878
+ // Check if this was a nudge-abort (not emergency stop, not a real error)
879
+ const isNudgeAbort = abortController.signal.aborted && this.streamNudgeCount > 0 && this.streamNudgeCount <= this.maxNudgeRetries;
880
+ if (isNudgeAbort) {
881
+ // Inject nudge message and retry via the for-loop
882
+ const systemMessage = this.conversations.appendMessage(convId, 'system', '[Cerebellum] You stopped mid-response. Continue from where you left off.');
883
+ this.emit({ type: 'message:system', message: systemMessage });
884
+ continue; // retry loop
885
+ }
886
+ // Check if Cerebellum dropped mid-stream
887
+ if (this.cerebellum && !this.cerebellum.isConnected() && abortController.signal.aborted) {
888
+ const err = new Error('Cerebellum disconnected during active response. Restart it with: docker compose up -d cerebellum');
889
+ log.error('Cerebellum disconnected mid-stream', { error: err.message });
890
+ this.emit({ type: 'error', error: err });
891
+ break;
892
+ }
893
+ const err = error instanceof Error ? error : new Error(String(error));
894
+ if (attempt > 0) {
895
+ this.emitWatchdog('retry_failed', `Retry attempt ${attemptNumber} failed: ${err.message}`, { level: 'error' });
896
+ }
897
+ log.error('Send message failed', {
898
+ error: err.message,
899
+ turnId,
900
+ attempt: attemptNumber,
901
+ conversationId: convId,
902
+ phase: failureState.phase,
903
+ activeToolName: failureState.activeToolName,
904
+ activeToolCallId: failureState.activeToolCallId,
905
+ activeToolStartedAt: failureState.activeToolStartedAt,
906
+ });
767
907
  this.emit({ type: 'error', error: err });
768
- break;
769
908
  }
770
- const err = error instanceof Error ? error : new Error(String(error));
771
- log.error('Send message failed', {
772
- error: err.message,
773
- attempt,
774
- conversationId: convId,
775
- phase: failureState.phase,
776
- activeToolName: failureState.activeToolName,
777
- activeToolCallId: failureState.activeToolCallId,
778
- activeToolStartedAt: failureState.activeToolStartedAt,
779
- });
780
- this.emit({ type: 'error', error: err });
781
- }
782
- break; // success — exit retry loop
783
- } // end retry for-loop
909
+ break; // success exit retry loop
910
+ } // end retry for-loop
911
+ }
912
+ finally {
913
+ this.currentStreamTurn = null;
914
+ }
784
915
  }
785
916
  async start() {
786
917
  if (!this.activeConversationId) {