@livx.cc/agentx 0.99.7 → 0.99.8

package/dist/index.d.ts

@@ -1299,6 +1299,10 @@ interface SttLike {
 interface TtsLike {
     onAudio: (chunk: Uint8Array) => void;
     onDone: () => void;
+    /** Optional word-timestamp seam for karaoke reveal (revealMode==='word'). `start[i]` = seconds
+     *  from turn-audio start. Set `wantTimestamps` to request them from the provider. */
+    onTimestamps?: (words: string[], start: number[]) => void;
+    wantTimestamps?: boolean;
     connect(): Promise<void> | void;
     /** Optional: prime the synthesis pipeline right after connect (throwaway context, audio discarded)
      *  so the FIRST real turn doesn't pay the provider's cold-synthesis spin-up. */
@@ -1409,6 +1413,18 @@ declare class VoiceEngineOptions {
     emotions: boolean;
     /** Show the `[emotion]` tags in the on-screen echo (debug). false = hide (spoken-only). */
     showEmotions: boolean;
+    /**
+     * Progressive text reveal — the "karaoke" capability, opt-in.
+     *   'off'   — no reveal events (CLI default; the host renders text however it likes).
+     *   'delta' — emit `onReveal(fullText)` as each spoken delta lands (near-free; text grows
+     *             in step with the model stream).
+     *   'word'  — reveal word-by-word synced to the TTS playback CLOCK (true karaoke). Requires a
+     *             timestamp-emitting TTS (CartesiaTTS.wantTimestamps) driven over `AudioSink.playedMs`.
+     */
+    revealMode: 'off' | 'delta' | 'word';
+    /** Called with the cumulative revealed text for the current assistant turn (see `revealMode`).
+     *  Reset to '' at the start of each spoken turn. */
+    onReveal: (revealed: string) => void;
     /** Time source for every engine timer/timestamp. Default: real time (identical behavior). */
     clock: EngineClock;
     /** Structured diagnostic event at every decision point (state transitions, dispatch/drop paths,
@@ -1426,6 +1442,10 @@ declare class VoiceEngine {
     private ctxOpen;
     private interrupted;
     private spokeDeltas;
+    private revealText;
+    private wordStarts;
+    private revealedN;
+    private revealPoll;
     private clock;
     private drainTimer;
     private echoWords;
@@ -1490,6 +1510,15 @@ declare class VoiceEngine {
     /** Feed a spoken delta. Returns the on-screen echo text (emotion tags shown/hidden per config) so the
      *  host renders the SAME stream that was parsed for TTS — no second, state-doubling parse. */
     speakDelta(text: string): string;
+    /** Karaoke reveal poll: reveal the display-word prefix up to the count of spoken words whose
+     *  Cartesia start time has been reached on the playback clock (`AudioSink.playedMs`, which excludes
+     *  paused time). Poll-driven (not AudioContext timers) so it's portable across CLI/browser sinks and
+     *  deterministic under the virtual clock. */
+    private startWordReveal;
+    /** Stop the poll. `finalFlush` reveals the whole line once audio has finished, so a timing
+     *  undershoot never leaves the tail unrevealed (skipped after a barge-in — the truncated prefix
+     *  is what was actually spoken). */
+    private stopWordReveal;
     /** close the spoken turn (idempotent); stays audible until ALL audio arrived AND playback drains */
     endSpeech(): void;
     /** text of the reply cut by the last barge-in — consumed by the host to tell the model what
@@ -1673,6 +1702,12 @@ declare class CartesiaTTS {
     ctxId: string;
     onAudio: (chunk: Uint8Array) => void;
     onDone: () => void;
+    /** Word-level timestamps for karaoke reveal: `start[i]` = seconds from turn-audio start (absolute
+     *  across continuation chunks). Fired only when a consumer opts into reveal (see `wantTimestamps`). */
+    onTimestamps: (words: string[], start: number[]) => void;
+    /** Request `add_timestamps` from Cartesia. Off by default (saves bandwidth) — the engine sets it
+     *  when revealMode==='word'. */
+    wantTimestamps: boolean;
     /** Diagnostic tap (provider errors, circuit-breaker open/close, ws reconnects). Fire-and-forget:
      *  a throwing handler disables itself — never affects synthesis. Wired by VoiceIO / the lab bridge. */
     onDiag: (ev: {

package/dist/index.js

@@ -5012,6 +5012,19 @@ var VoiceEngineOptions = class {
   emotions = true;
   /** Show the `[emotion]` tags in the on-screen echo (debug). false = hide (spoken-only). */
   showEmotions = false;
+  /**
+   * Progressive text reveal — the "karaoke" capability, opt-in.
+   *   'off'   — no reveal events (CLI default; the host renders text however it likes).
+   *   'delta' — emit `onReveal(fullText)` as each spoken delta lands (near-free; text grows
+   *             in step with the model stream).
+   *   'word'  — reveal word-by-word synced to the TTS playback CLOCK (true karaoke). Requires a
+   *             timestamp-emitting TTS (CartesiaTTS.wantTimestamps) driven over `AudioSink.playedMs`.
+   */
+  revealMode = "off";
+  /** Called with the cumulative revealed text for the current assistant turn (see `revealMode`).
+   *  Reset to '' at the start of each spoken turn. */
+  onReveal = () => {
+  };
   /** Time source for every engine timer/timestamp. Default: real time (identical behavior). */
   clock = realClock;
   /** Structured diagnostic event at every decision point (state transitions, dispatch/drop paths,
@@ -5034,6 +5047,12 @@ var VoiceEngine = class _VoiceEngine {
   // barge-in latch: drop in-flight deltas until the next legitimate turn
   spokeDeltas = false;
   // a TTS context is open for the current spoken turn
+  revealText = "";
+  // cumulative revealed (on-screen) text for the current turn — see revealMode
+  // 'word' karaoke reveal: spoken-word start times (sec, absolute from turn-audio start) + poll state.
+  wordStarts = [];
+  revealedN = 0;
+  revealPoll = null;
   clock;
   drainTimer = null;
   // heuristic tier state (inert under AEC) — frozen as validated in the experiment
@@ -5121,6 +5140,13 @@ var VoiceEngine = class _VoiceEngine {
     this.tts.onAudio = (c) => {
       if (this.speaking || this.bcActive) this.player.write(c);
     };
+    if (this.options.revealMode === "word") {
+      this.tts.wantTimestamps = true;
+      this.tts.onTimestamps = (_words, start) => {
+        if (this.interrupted) return;
+        for (const s of start) this.wordStarts.push(s);
+      };
+    }
     this.stt.onPartial = (text) => this.handlePartial(text);
     this.stt.onUtterance = (text) => this.handleUtterance(text);
     this.stt.onLevel = (rms) => this.handleLevel(rms);
@@ -5186,6 +5212,10 @@ var VoiceEngine = class _VoiceEngine {
     this.ctxOpen = true;
     this.spokeDeltas = false;
     this.reply = "";
+    this.revealText = "";
+    this.wordStarts = [];
+    this.revealedN = 0;
+    if (this.options.revealMode === "word") this.startWordReveal();
     this.emo = this.options.emotions ? new EmotionStream(this.options.showEmotions) : null;
     this.echoWords = new Set(this.words(this.prevReply));
     this.tts.newContext();
@@ -5234,8 +5264,46 @@ var VoiceEngine = class _VoiceEngine {
     if (!this.spokeDeltas && this.turnStartAt) log10.debug(`ttft: ${Math.round(this.clock.now() - this.turnStartAt)}ms`);
     this.spokeDeltas = true;
     this.setState("speaking");
+    if (this.options.revealMode !== "off" && display) {
+      this.revealText += display;
+      if (this.options.revealMode === "delta") this.options.onReveal(this.revealText);
+    }
     return display;
   }
+  /** Karaoke reveal poll: reveal the display-word prefix up to the count of spoken words whose
+   *  Cartesia start time has been reached on the playback clock (`AudioSink.playedMs`, which excludes
+   *  paused time). Poll-driven (not AudioContext timers) so it's portable across CLI/browser sinks and
+   *  deterministic under the virtual clock. */
+  startWordReveal() {
+    if (this.revealPoll) return;
+    const tick = () => {
+      this.revealPoll = null;
+      if (!this.speaking || this.interrupted || this.options.revealMode !== "word") return;
+      const playedSec = this.player.playedMs() / 1e3;
+      let n = this.revealedN;
+      while (n < this.wordStarts.length && this.wordStarts[n] <= playedSec) n++;
+      if (n !== this.revealedN) {
+        this.revealedN = n;
+        const words = this.revealText.trim().split(/\s+/).filter(Boolean);
+        this.options.onReveal(words.slice(0, n).join(" "));
+      }
+      this.revealPoll = this.clock.setTimeout(tick, 60);
+    };
+    this.revealPoll = this.clock.setTimeout(tick, 60);
+  }
+  /** Stop the poll. `finalFlush` reveals the whole line once audio has finished, so a timing
+   *  undershoot never leaves the tail unrevealed (skipped after a barge-in — the truncated prefix
+   *  is what was actually spoken). */
+  stopWordReveal(finalFlush) {
+    if (this.revealPoll) {
+      this.clock.clearTimeout(this.revealPoll);
+      this.revealPoll = null;
+    }
+    if (finalFlush && this.options.revealMode === "word" && !this.interrupted) {
+      const full = this.revealText.trim();
+      if (full) this.options.onReveal(full);
+    }
+  }
   /** close the spoken turn (idempotent); stays audible until ALL audio arrived AND playback drains */
   endSpeech() {
     this.interrupted = false;
@@ -5261,6 +5329,7 @@ var VoiceEngine = class _VoiceEngine {
       }
       this.drainTimer = null;
       this.speaking = false;
+      this.stopWordReveal(true);
       if (this.turnStartAt) log10.debug(`turn: ${Math.round(this.clock.now() - this.turnStartAt)}ms (incl. playback)`);
       this.echoUntil = this.clock.now() + 2500;
       if (!this.usingAec) this.stt.reset();
@@ -5348,6 +5417,7 @@ var VoiceEngine = class _VoiceEngine {
     const droppedQueued = this.uttQueue.length;
     this.uttQueue = [];
     if (!this.speaking && !this.drainTimer) return;
+    this.stopWordReveal(false);
     this.diag("interrupt", { droppedQueued, ctxOpen: this.ctxOpen, playedMs: Math.round(Math.max(0, this.player.playedMs())) });
     if (this.drainTimer) {
       this.clock.clearTimeout(this.drainTimer);
@@ -7272,6 +7342,13 @@ var CartesiaTTS = class _CartesiaTTS {
   };
   onDone = () => {
   };
+  /** Word-level timestamps for karaoke reveal: `start[i]` = seconds from turn-audio start (absolute
+   *  across continuation chunks). Fired only when a consumer opts into reveal (see `wantTimestamps`). */
+  onTimestamps = () => {
+  };
+  /** Request `add_timestamps` from Cartesia. Off by default (saves bandwidth) — the engine sets it
+   *  when revealMode==='word'. */
+  wantTimestamps = false;
   /** Diagnostic tap (provider errors, circuit-breaker open/close, ws reconnects). Fire-and-forget:
    *  a throwing handler disables itself — never affects synthesis. Wired by VoiceIO / the lab bridge. */
   onDiag = () => {
@@ -7339,6 +7416,9 @@ var CartesiaTTS = class _CartesiaTTS {
         this.consecutiveErrors = 0;
         this.markRecovered();
         this.onDone();
+      } else if (m.type === "timestamps" && m.word_timestamps) {
+        const wt = m.word_timestamps;
+        if (wt.words?.length && wt.start?.length) this.onTimestamps(wt.words, wt.start);
       } else if (m.type === "error") {
         if (/already been cancelled|does not exist/.test(m.message || "")) return;
         this.consecutiveErrors++;
@@ -7395,7 +7475,8 @@ var CartesiaTTS = class _CartesiaTTS {
       voice: { mode: "id", id: this.options.voiceId },
       output_format: { container: "raw", encoding: "pcm_s16le", sample_rate: TTS_SAMPLE_RATE },
       context_id: this.ctxId,
-      continue: cont
+      continue: cont,
+      ...this.wantTimestamps ? { add_timestamps: true } : {}
     });
   }
   speak(text, cont) {