dsh-live-voice 0.3.2 → 0.4.0

package/src/modules/core/coordinator.ts

@@ -5,8 +5,10 @@ import {
   filterSpeechOutput,
   hasMinimumWords,
   hasUnclosedCodeFence,
+  hasIncompleteSpeechMarkdown,
   matchVoiceCommand,
   splitSpeechOutput,
+  splitInitialSpeechItem,
 } from './filters.ts';
 
 const message = (error) => error?.message || String(error);
@@ -26,14 +28,16 @@ function cancellable(promise, signal) {
 
 /** Session-owned policy. Input transitions are serialized; generations invalidate late work. */
 export class VoiceCoordinator {
-  constructor({ recognition, engines, meter, composer, settings = {} }) {
-    Object.assign(this, { recognition, engines, meter, composer });
+  constructor({ recognition, engines, meter, composer, settings = {}, persistSettings }) {
+    Object.assign(this, { recognition, engines, meter, composer, persistSettings });
     this.listeners = new Set();
     this.transcript = new TranscriptDraft();
     this.epoch = 0;
     this.speechEpoch = 0;
     this.controlEpoch = 0;
     this.queue = [];
+    this.speechHistory = [];
+    this.speechCursor = -1;
     this.prefetchController = new AbortController();
     this.prefetchItems = new Set();
     this.activeSpeechItem = null;
@@ -56,9 +60,16 @@ export class VoiceCoordinator {
       starting: false,
       error: null,
       activeMessageId: null,
+      speechLoading: false,
+      speechText: null,
+      speechEngine: null,
       answeringQuestion: false,
       autoSendAt: null,
       speechSegmentsRemaining: 0,
+      speechSegmentIndex: 0,
+      speechSegmentsTotal: 0,
+      speechHasPrevious: false,
+      speechRunActive: false,
       capabilities: {},
       settings: normalizeSettings(settings),
     };
@@ -273,7 +284,32 @@ export class VoiceCoordinator {
           throw new Error(capability.reason || 'Local browser recognition is unavailable.');
         let started;
         try {
-          started = await this.meter.start({ signal: controller.signal });
+          const deviceId = this.snapshot.settings.inputDeviceId;
+          try {
+            started = await this.meter.start({ signal: controller.signal });
+          } catch (error) {
+            if (
+              error?.name !== 'OverconstrainedError' ||
+              !deviceId ||
+              !valid() ||
+              controller.signal.aborted ||
+              this.snapshot.settings.inputDeviceId !== deviceId
+            )
+              throw error;
+            // Repair only the obsolete input preference, without ending this input operation.
+            VoiceCoordinator.prototype.updateSettings.call(this, { inputDeviceId: '' });
+            await cancellable(
+              Promise.resolve(this.persistSettings?.({ inputDeviceId: '' })),
+              controller.signal,
+            );
+            if (
+              !valid() ||
+              controller.signal.aborted ||
+              this.snapshot.settings.inputDeviceId !== ''
+            )
+              return;
+            started = await this.meter.start({ signal: controller.signal });
+          }
         } catch (error) {
           if (valid())
             this.patch({
@@ -403,7 +439,7 @@ export class VoiceCoordinator {
     }
   }
 
-  onResult({ final = '', interim = '' }) {
+  onResult({ final = '', interim = '', startedAt }) {
     if (this.disposed || (!this.snapshot.listening && !this.snapshot.starting)) return;
     if (
       this.snapshot.speaking &&
@@ -463,7 +499,7 @@ export class VoiceCoordinator {
         } else if (this.snapshot.muted) return;
         else {
           const sendingMode = command === 'send' ? 'steer' : 'queue';
-          this.updateSettings({ sendingMode });
+          // Spoken delivery is a one-shot action, not an automatic-send preference change.
           if (this.composer.getDraft().trim() && typeof this.composer.submit === 'function')
             this.composer.submit(sendingMode);
         }
@@ -484,8 +520,9 @@ export class VoiceCoordinator {
         this.patch({ recognizing: false });
         return;
       }
-      const next = this.transcript.update(this.composer.getDraft(), final, true);
-      this.composer.setDraft(next);
+      const appended = this.composer.appendFinal?.(final, startedAt);
+      const next = appended ?? this.transcript.update(this.composer.getDraft(), final, true);
+      if (appended == null) this.composer.setDraft(next);
       this.maybeScheduleAutoSend(next);
     }
     if (interim) {
@@ -586,8 +623,10 @@ export class VoiceCoordinator {
     this.prefetchController.abort();
     this.prefetchController = new AbortController();
     for (const item of this.prefetchItems) {
-      item.prepared?.dispose?.();
-      item.prepared = null;
+      if (!this.speechHistory.includes(item)) {
+        item.prepared?.dispose?.();
+        item.prepared = null;
+      }
       item.preparing = null;
     }
     this.prefetchItems.clear();
@@ -610,7 +649,10 @@ export class VoiceCoordinator {
         .prepare(item.text, { ...options, signal })
         .then((prepared) => {
           item.preparing = null;
-          if (signal.aborted || !this.queue.includes(item)) {
+          if (
+            signal.aborted ||
+            (!this.queue.includes(item) && !this.speechHistory.includes(item))
+          ) {
             prepared.dispose?.();
             this.prefetchItems.delete(item);
           } else item.prepared = prepared;
@@ -623,10 +665,46 @@ export class VoiceCoordinator {
     }
   }
 
+  _releaseSpeechHistory() {
+    for (const item of this.speechHistory) item.prepared?.dispose?.();
+    this.speechHistory = [];
+    this.speechCursor = -1;
+  }
+  getSpeechHistory() {
+    const groups = [];
+    for (const item of [...this.speechHistory, ...this.queue]) {
+      let group = groups.at(-1);
+      if (!group || group.messageId !== item.id) {
+        group = { messageId: item.id, segments: [] };
+        groups.push(group);
+      }
+      if (!group.segments.includes(item)) group.segments.push(item);
+    }
+    return groups;
+  }
   _syncSpeechSegments() {
     const speechSegmentsRemaining = this.queue.length + (this.snapshot.speaking ? 1 : 0);
-    if (speechSegmentsRemaining !== this.snapshot.speechSegmentsRemaining)
-      this.patch({ speechSegmentsRemaining });
+    const speechHasPrevious = this.speechCursor > 0;
+    const items = [...new Set([...this.speechHistory, ...this.queue])];
+    const speechSegmentsTotal = items.length;
+    const speechSegmentIndex = this.speechCursor >= 0 ? this.speechCursor + 1 : 0;
+    const speechRunActive =
+      speechSegmentsRemaining > 0 ||
+      this.speechHistory.some((item) => !item.manual && this.unfinished.has(item.id));
+    if (
+      speechSegmentsRemaining !== this.snapshot.speechSegmentsRemaining ||
+      speechHasPrevious !== this.snapshot.speechHasPrevious ||
+      speechRunActive !== this.snapshot.speechRunActive ||
+      speechSegmentsTotal !== this.snapshot.speechSegmentsTotal ||
+      speechSegmentIndex !== this.snapshot.speechSegmentIndex
+    )
+      this.patch({
+        speechSegmentsRemaining,
+        speechHasPrevious,
+        speechRunActive,
+        speechSegmentsTotal,
+        speechSegmentIndex,
+      });
   }
 
   _suppressPending() {
@@ -644,7 +722,15 @@ export class VoiceCoordinator {
     ++this.controlEpoch;
     this._cancelSpeechPrefetch();
     this._suppressPending();
-    this.patch({ speaking: false, paused: false, activeMessageId: null });
+    this._releaseSpeechHistory();
+    this.patch({
+      speaking: false,
+      paused: false,
+      activeMessageId: null,
+      speechLoading: false,
+      speechText: null,
+      speechEngine: null,
+    });
     this._syncSpeechSegments();
     const stopped = this.speechBarrier.then(async () => {
       const results = await Promise.allSettled(
@@ -671,50 +757,57 @@ export class VoiceCoordinator {
     )
       await this._startInput(true);
   }
+  async previousSpeechSegment() {
+    if (this.speechCursor <= 0 || this.disposed) return;
+    const replay = this.speechHistory.slice(this.speechCursor - 1);
+    this.queue = [...replay, ...this.queue.filter((item) => !replay.includes(item))];
+    await this._navigateSpeech();
+  }
   async skipSpeechSegment() {
-    // Unlike Stop, Next discards only the active segment. Queued and future
-    // streaming segments remain eligible for playback.
-    if (this.disposed || this.queue.length === 0) return;
-    // During the intentional inter-segment gap there is no active audio. In
-    // that case Next discards the imminent segment and immediately drains the
-    // following one, so the persistent button remains useful.
+    if (this.disposed || !this.queue.length) return;
     if (!this.snapshot.speaking && !this.snapshot.paused) {
       const skipped = this.queue.shift();
-      skipped?.prepared?.dispose?.();
-      this.prefetchItems.delete(skipped);
-      this._cancelAssistantSpeechTimer();
-      this.assistantSpeechNotBefore = 0;
-      this._syncSpeechSegments();
-      this._prefetchSpeech();
-      this._drain();
-      return;
+      if (!this.speechHistory.includes(skipped)) this.speechHistory.push(skipped);
+      this.speechCursor = this.speechHistory.indexOf(skipped);
     }
+    await this._navigateSpeech();
+  }
+  async _navigateSpeech() {
+    const engine = this.engines[this.snapshot.speechEngine || this.snapshot.settings.engine];
     const epoch = ++this.speechEpoch;
     ++this.controlEpoch;
     this._cancelAssistantSpeechTimer();
     this.assistantSpeechNotBefore = 0;
-    this._cancelSpeechPrefetch();
-    this.patch({ speaking: false, paused: false, activeMessageId: null });
+    this.patch({ speaking: false, paused: false, speechLoading: false });
     this._syncSpeechSegments();
-    const stopped = this.speechBarrier.then(() =>
-      Promise.resolve().then(() => this.engines[this.snapshot.settings.engine]?.stop()),
-    );
+    const stopped = this.speechBarrier.then(() => engine?.stop());
     this.speechBarrier = stopped.catch(() => {});
     try {
       await stopped;
     } catch (error) {
+      this.speechStopError = error;
       if (epoch === this.speechEpoch) this.patch({ error: message(error) });
       return;
     }
     if (epoch !== this.speechEpoch || this.disposed) return;
     this._drain();
   }
+  getSpeechProgress() {
+    return this.engines[this.snapshot.speechEngine]?.getPlaybackProgress?.() ?? null;
+  }
   async pauseSpeech() {
-    if (this.disposed || !this.snapshot.speaking || this.snapshot.paused) return;
+    if (this.disposed || this.snapshot.paused) return;
+    if (!this.snapshot.speaking && this.snapshot.speechRunActive) {
+      this._cancelAssistantSpeechTimer();
+      this.patch({ paused: true });
+      return;
+    }
+    if (!this.snapshot.speaking) return;
     const epoch = this.speechEpoch;
     const control = ++this.controlEpoch;
     try {
-      const result = await this.engines[this.snapshot.settings.engine].pause();
+      const result =
+        await this.engines[this.snapshot.speechEngine || this.snapshot.settings.engine].pause();
       if (result !== false) this._cancelSpeechPrefetch();
       if (
         epoch === this.speechEpoch &&
@@ -729,6 +822,12 @@ export class VoiceCoordinator {
   }
   async resumeSpeech() {
     if (this.disposed || !this.snapshot.paused) return;
+    if (!this.snapshot.speaking) {
+      this.patch({ paused: false });
+      this.assistantSpeechNotBefore = 0;
+      this._drain();
+      return;
+    }
     const epoch = this.speechEpoch;
     const control = ++this.controlEpoch;
     try {
@@ -737,7 +836,8 @@ export class VoiceCoordinator {
         if (this.inputReleaseError) throw this.inputReleaseError;
       }
       if (epoch !== this.speechEpoch || control !== this.controlEpoch) return;
-      const result = await this.engines[this.snapshot.settings.engine].resume();
+      const result =
+        await this.engines[this.snapshot.speechEngine || this.snapshot.settings.engine].resume();
       if (epoch === this.speechEpoch && control === this.controlEpoch && result !== false) {
         this.patch({ paused: false });
         this._prefetchSpeech();
@@ -756,9 +856,13 @@ export class VoiceCoordinator {
       filterCodeBlocks: this.snapshot.settings.outputCodeFilterEnabled,
       codeBlockMaxLines: this.snapshot.settings.outputCodeMaxLines,
       codeBlockNotice: this.snapshot.settings.outputCodeNotice,
+      lang: this.snapshot.settings.lang,
     });
-    const first = segments.shift();
-    if (!first) return;
+    const initial = segments.shift();
+    if (!initial) return;
+    const parts = splitInitialSpeechItem(initial);
+    const first = parts.shift();
+    segments.unshift(...parts);
     this.queue.push(...segments.map((segment) => ({ text: segment, id: messageId, manual: true })));
     this._prefetchSpeech();
     this._syncSpeechSegments();
@@ -772,15 +876,30 @@ export class VoiceCoordinator {
       this._syncSpeechSegments();
       return;
     }
+    const item =
+      this.activeSpeechItem?.text === text
+        ? this.activeSpeechItem
+        : { text, id: messageId, manual };
+    if (!this.speechHistory.includes(item)) this.speechHistory.push(item);
+    this.speechCursor = this.speechHistory.indexOf(item);
     const epoch = ++this.speechEpoch;
-    const engine = this.engines[this.snapshot.settings.engine];
+    const engine = item.engine || this.engines[this.snapshot.settings.engine];
+    item.engine = engine;
     if (!engine) {
       this.queue = [];
       this.patch({ error: 'Speech engine unavailable.' });
       return;
     }
     // Reserve playback synchronously so incoming stream chunks cannot overtake gating.
-    this.patch({ speaking: true, paused: false, activeMessageId: messageId, error: null });
+    this.patch({
+      speaking: true,
+      paused: false,
+      activeMessageId: messageId,
+      speechLoading: true,
+      speechText: text,
+      speechEngine: this.snapshot.settings.engine,
+      error: null,
+    });
     this._syncSpeechSegments();
     let failed = false;
     try {
@@ -792,20 +911,30 @@ export class VoiceCoordinator {
         if (this.inputReleaseError) throw this.inputReleaseError;
       }
       if (epoch !== this.speechEpoch || this.disposed) return;
-      const queued = this.activeSpeechItem;
+      const queued = item;
       if (queued?.preparing) await queued.preparing;
       if (epoch !== this.speechEpoch || this.disposed) return;
       if (queued?.prepareError) throw queued.prepareError;
       const options = {
+        onPlaybackStart: () => {
+          if (epoch === this.speechEpoch && !this.disposed) this.patch({ speechLoading: false });
+        },
         voice: this.snapshot.settings.voice || undefined,
         rate: this.snapshot.settings.rate,
         outputDeviceId: this.snapshot.settings.outputDeviceId,
       };
+      if (!queued.prepared && typeof engine.prepare === 'function') {
+        const prepared = await engine.prepare(text, options);
+        if (epoch !== this.speechEpoch || this.disposed) {
+          prepared.dispose?.();
+          return;
+        }
+        queued.prepared = prepared;
+      }
       if (queued?.prepared && typeof engine.playPrepared === 'function') {
         const prepared = queued.prepared;
-        queued.prepared = null;
         this.prefetchItems.delete(queued);
-        await engine.playPrepared(prepared, options);
+        await engine.playPrepared(prepared, { ...options, retainPrepared: true });
       } else await engine.speak(text, options);
     } catch (error) {
       if (epoch === this.speechEpoch) {
@@ -815,7 +944,21 @@ export class VoiceCoordinator {
       }
     } finally {
       if (epoch === this.speechEpoch && !this.disposed) {
-        this.patch({ speaking: false, paused: false, activeMessageId: null });
+        this.patch({
+          speaking: false,
+          paused: false,
+          activeMessageId: null,
+          speechLoading: false,
+          speechText: this.queue.length || this.unfinished.has(messageId) ? text : null,
+          speechEngine:
+            this.queue.length || this.unfinished.has(messageId) ? this.snapshot.speechEngine : null,
+        });
+        if (
+          failed ||
+          (!this.queue.length &&
+            !this.speechHistory.some((item) => !item.manual && this.unfinished.has(item.id)))
+        )
+          this._releaseSpeechHistory();
         this._syncSpeechSegments();
         if (!failed && this.queue.length) {
           this.assistantSpeechNotBefore = Date.now() + this.snapshot.settings.segmentGapMs;
@@ -839,6 +982,7 @@ export class VoiceCoordinator {
       (!this.snapshot.conversation && !this.queue[0]?.manual) ||
       (!this.snapshot.settings.announceAssistantMessages && !this.queue[0]?.manual) ||
       this.snapshot.speaking ||
+      this.snapshot.paused ||
       this.snapshot.recognizing ||
       this.snapshot.starting ||
       (!this.queue[0]?.manual &&
@@ -862,7 +1006,7 @@ export class VoiceCoordinator {
     this._prefetchSpeech();
     void this.play(next.text, next.id, next.manual === true).finally(() => {
       if (this.activeSpeechItem === next) this.activeSpeechItem = null;
-      next.prepared?.dispose?.();
+      // Audio remains owned by the playback run until completion.
       this.prefetchItems.delete(next);
     });
   }
@@ -871,6 +1015,16 @@ export class VoiceCoordinator {
     if (this.disposed) return;
     if (complete) this.unfinished.delete(id);
     else this.unfinished.add(id);
+    if (
+      complete &&
+      !this.snapshot.speaking &&
+      !this.queue.length &&
+      this.speechHistory.some((item) => item.id === id)
+    ) {
+      this._releaseSpeechHistory();
+      this.patch({ speechText: null, speechEngine: null });
+      this._syncSpeechSegments();
+    }
     if (
       baseline ||
       !this.snapshot.conversation ||
@@ -888,20 +1042,19 @@ export class VoiceCoordinator {
     const remaining = text.slice(offset);
     const boundary = complete ? remaining.length : remaining.lastIndexOf('\n') + 1;
     if (boundary <= 0) return;
-    if (
-      this.snapshot.settings.outputCodeFilterEnabled &&
-      hasUnclosedCodeFence(remaining.slice(0, boundary)) &&
-      !complete
-    )
-      return;
+    if (!complete && hasIncompleteSpeechMarkdown(remaining.slice(0, boundary))) return;
+    if (hasUnclosedCodeFence(remaining.slice(0, boundary)) && !complete) return;
     const chunk = filterSpeechOutput(remaining.slice(0, boundary).trim(), {
       filterCodeBlocks: this.snapshot.settings.outputCodeFilterEnabled,
       codeBlockMaxLines: this.snapshot.settings.outputCodeMaxLines,
       codeBlockNotice: this.snapshot.settings.outputCodeNotice,
+      lang: this.snapshot.settings.lang,
     }).trim();
     this.consumed.set(id, offset + boundary);
     if (chunk) {
-      this.queue.push({ text: chunk, id });
+      const initial = !this.snapshot.speaking && !this.queue.some((item) => item.prepared);
+      const parts = initial ? splitInitialSpeechItem(chunk) : [chunk];
+      this.queue.push(...parts.map((text) => ({ text, id })));
       this._prefetchSpeech();
       this._syncSpeechSegments();
       this._drain();

package/src/modules/core/developerExtension.ts

@@ -0,0 +1,26 @@
+import type { ComponentType } from 'react';
+export const DEVELOPER_KEY = Symbol.for('dsh-live-voice.developer.v1');
+export const DEVELOPER_CHANGED = 'dsh-live-voice:developer-changed';
+export interface DeveloperExtension {
+  version: 1;
+  label(): string;
+  component: ComponentType;
+}
+export function readDeveloperExtension(target: any = globalThis.window): DeveloperExtension | null {
+  const extension = target?.[DEVELOPER_KEY];
+  return extension?.version === 1 ? extension : null;
+}
+export function subscribeDeveloperExtension(listener: () => void, target: any = globalThis.window) {
+  target?.addEventListener(DEVELOPER_CHANGED, listener);
+  return () => target?.removeEventListener(DEVELOPER_CHANGED, listener);
+}
+export function publishDeveloperExtension(target: any, extension: DeveloperExtension) {
+  const ChangedEvent = target.Event ?? Event;
+  target[DEVELOPER_KEY] = extension;
+  target.dispatchEvent(new ChangedEvent(DEVELOPER_CHANGED));
+  return () => {
+    if (target[DEVELOPER_KEY] !== extension) return;
+    delete target[DEVELOPER_KEY];
+    target.dispatchEvent(new ChangedEvent(DEVELOPER_CHANGED));
+  };
+}

package/src/modules/core/diagnostics.ts

@@ -0,0 +1,150 @@
+/** Versioned read-only boundary; debugger UI is never imported by the runtime. */
+export const DIAGNOSTIC_KEY = Symbol.for('dsh-live-voice.diagnostics.v1');
+export const DIAGNOSTIC_CHANGED = 'dsh-live-voice:diagnostics-changed';
+export type DiagnosticValue =
+  null | boolean | number | string | DiagnosticValue[] | { [key: string]: DiagnosticValue };
+export interface DiagnosticSource {
+  version: 1;
+  read(options?: { includeContent?: boolean }): DiagnosticValue;
+  subscribe(listener: () => void): () => void;
+}
+export function publishDiagnosticSource(target: any, source: DiagnosticSource) {
+  target[DIAGNOSTIC_KEY] = source;
+  const ChangedEvent = target.Event ?? Event;
+  target.dispatchEvent?.(new ChangedEvent(DIAGNOSTIC_CHANGED));
+  return () => {
+    if (target[DIAGNOSTIC_KEY] !== source) return;
+    delete target[DIAGNOSTIC_KEY];
+    target.dispatchEvent?.(new ChangedEvent(DIAGNOSTIC_CHANGED));
+  };
+}
+const number = (value: unknown) =>
+  typeof value === 'number' && Number.isFinite(value) ? value : null;
+/** Explicit projection excludes audio, functions, resources and free-form errors. */
+export function readCoordinatorDiagnostics(
+  controller: any,
+  includeContent = false,
+): DiagnosticValue {
+  const state = controller.getSnapshot();
+  const recognition = controller.recognition?.session;
+  const settings: Record<string, DiagnosticValue> = {};
+  const publicSettings = new Set([
+    'engine',
+    'recognitionEngine',
+    'voiceDetectionPreset',
+    'sendingMode',
+    'dshBusyEnterBehavior',
+    'mode',
+    'lang',
+    'recognitionLang',
+  ]);
+  for (const [key, value] of Object.entries(state.settings ?? {})) {
+    settings[key] =
+      typeof value === 'string'
+        ? includeContent || publicSettings.has(key)
+          ? value
+          : { characters: value.length }
+        : typeof value === 'boolean' || typeof value === 'number'
+          ? value
+          : null;
+  }
+  const item = (entry: any): DiagnosticValue => ({
+    messageId: entry.id == null ? null : String(entry.id),
+    manual: entry.manual === true,
+    characters: entry.text?.length ?? 0,
+    ...(includeContent ? { text: String(entry.text ?? '') } : {}),
+    preparing: Boolean(entry.preparing),
+    prepared: Boolean(entry.prepared),
+    failed: Boolean(entry.prepareError),
+  });
+  return {
+    settings,
+    lifecycle: {
+      disposed: Boolean(controller.disposed),
+      conversation: Boolean(state.conversation),
+    },
+    composer: {
+      characters: controller.composer?.getDraft?.()?.length ?? 0,
+      answeringQuestion: Boolean(state.answeringQuestion),
+      ...(includeContent ? { draft: controller.composer?.getDraft?.() ?? '' } : {}),
+    },
+    capture: {
+      starting: Boolean(state.starting),
+      listening: Boolean(state.listening),
+      muted: Boolean(state.muted),
+      generation: number(controller.epoch),
+      level: number(controller.meter?.level?.()),
+      releaseFailed: Boolean(controller.inputReleaseError),
+    },
+    vad: {
+      source: state.settings?.recognitionEngine === 'browser' ? 'native' : 'plugin',
+      activity: Boolean(state.recognizing),
+      threshold: recognition?.chunks ? 0.012 : null,
+      silenceMs:
+        recognition?.chunks && controller.meter?.context?.sampleRate
+          ? (recognition.silence / controller.meter.context.sampleRate) * 1000
+          : null,
+    },
+    chunker: {
+      available: Boolean(recognition?.chunks),
+      bufferedSamples: number(recognition?.samples),
+      bufferedChunks: recognition?.chunks?.length ?? null,
+      containsSpeech: recognition?.voiced ?? null,
+    },
+    recognition: {
+      pendingTranscriptions: number(state.pendingTranscriptions),
+      activeRequest: Boolean(recognition?.activeRequest),
+      queue: (recognition?.transcriptionQueue ?? []).map((chunk: any) => ({
+        samples: (chunk.samples ?? chunk).length,
+        durationMs: controller.meter?.context?.sampleRate
+          ? ((chunk.samples ?? chunk).length / controller.meter.context.sampleRate) * 1000
+          : null,
+      })),
+    },
+    delivery: {
+      autoSendAt: number(state.autoSendAt),
+      countdownActive: controller.autoSendTimer != null,
+      holdToTalkRelease: Boolean(controller.holdToTalkRelease),
+    },
+    speech: {
+      speaking: Boolean(state.speaking),
+      paused: Boolean(state.paused),
+      activeMessageId: state.activeMessageId == null ? null : String(state.activeMessageId),
+      generation: number(controller.speechEpoch),
+      controlGeneration: number(controller.controlEpoch),
+      remaining: number(state.speechSegmentsRemaining),
+      notBefore: number(controller.assistantSpeechNotBefore),
+      waitingForActivity: controller.assistantSpeechNotBefore === Infinity,
+      stopFailed: Boolean(controller.speechStopError),
+      queue: (controller.queue ?? []).map(item),
+      activeItem: controller.activeSpeechItem ? item(controller.activeSpeechItem) : null,
+      prefetchCount: controller.prefetchItems?.size ?? 0,
+      consumed: Object.fromEntries(controller.consumed ?? []),
+      unfinished: [...(controller.unfinished ?? [])].map(String),
+      suppressed: [...(controller.suppressed ?? [])].map(String),
+    },
+    interruption: {
+      transcriptConfirmed: Boolean(controller.interruptionTranscriptConfirmed),
+      pausedSpeech: Boolean(controller.interruptionPausedSpeech),
+      timerActive: controller.interruptionTimer != null,
+    },
+    capabilities: Object.fromEntries(
+      Object.entries(state.capabilities ?? {}).map(([key, cap]: [string, any]) => [
+        key,
+        cap
+          ? {
+              supported: cap.supported === true,
+              local: cap.local ?? null,
+              pause: cap.pause ?? null,
+              resume: cap.resume ?? null,
+              hasReason: Boolean(cap.reason),
+            }
+          : null,
+      ]),
+    ),
+    error: {
+      present: Boolean(state.error),
+      ...(includeContent ? { message: state.error ?? null } : {}),
+    },
+  };
+}

package/src/modules/core/filters.ts

@@ -1,4 +1,6 @@
 // @ts-nocheck
+import { markdownSpeech } from './markdownSpeech.js';
+export { hasIncompleteSpeechMarkdown } from './markdownSpeech.js';
 const words = (text) =>
   String(text || '')
     .trim()
@@ -41,23 +43,26 @@ export function splitSpeechOutput(text, options = {}) {
     .filter(Boolean);
 }
 
+/** Split only the initial item at sentence punctuation or colons, never commas. */
+export function splitInitialSpeechItem(text) {
+  const result = [];
+  let start = 0;
+  // Require whitespace/end after punctuation to preserve decimals, URLs and file names.
+  for (const match of text.matchAll(/[.!?:]+(?:["”’)]*)?(?=\s|$)/gu)) {
+    const end = match.index + match[0].length;
+    const part = text.slice(start, end).trim();
+    if (part) result.push(part);
+    start = end;
+  }
+  const tail = text.slice(start).trim();
+  if (tail) result.push(tail);
+  return result.length ? result : [text];
+}
+
 export function hasUnclosedCodeFence(text) {
   return (String(text || '').match(/```/g) || []).length % 2 === 1;
 }
 
-export function filterSpeechOutput(
-  text,
-  {
-    filterCodeBlocks = true,
-    codeBlockMaxLines = 5,
-    codeBlockNotice = 'Look the code on out conversation',
-  } = {},
-) {
-  const source = String(text || '');
-  if (!filterCodeBlocks) return source;
-  return source.replace(/```[^\n]*\n?([\s\S]*?)```/g, (block, body) => {
-    const normalized = body.replace(/\n$/, '');
-    const lineCount = normalized ? normalized.split(/\r?\n/).length : 0;
-    return lineCount <= codeBlockMaxLines ? normalized : codeBlockNotice;
-  });
+export function filterSpeechOutput(text, options = {}) {
+  return markdownSpeech(text, options);
 }