@livx.cc/agentx 0.99.5 → 0.99.7

package/dist/index.d.ts

@@ -974,8 +974,10 @@ declare class DuplexAgent {
      *  left unguarded it polls TaskStatus after a dispatch and/or dispatches silently (dead air).
      *  Like CC's Task tool, a dispatch is "said my piece, now wait for the push" — these enforce that. */
     private turnDispatched;
+    private spokeBeforeDispatch;
     private turnBriefs;
     private spokeThisTurn;
+    private externalSpeech;
     private heldThisTurn;
     private nudging;
     private reflexBuf;
@@ -1001,6 +1003,18 @@ declare class DuplexAgent {
         question: string;
         resolve: (answer: string) => void;
     }>;
+    /** SPECULATIVE REFLEX (VoiceEngine.onSpeculate → speculate()): the reflex turn starts on a stable
+     *  PARTIAL transcript, its spoken output buffered — nothing reaches TTS until the endpointed final
+     *  confirms via send() (which flushes the buffer and adopts the call as THE turn). A divergent final
+     *  aborts it: output dropped, history rolled back, the final dispatches normally. */
+    private spec?;
+    /** Aborted speculative calls (double-billing visibility — each one paid for tokens never spoken). */
+    speculativeAbortedCalls: number;
+    /** Tools the reflex may execute on UNCONFIRMED (speculated) input — read-only lookups and the
+     *  intentionally-silent Hold. Anything else (Act/Think/AnswerTask/CancelTask/ExitSession/memory
+     *  writes/…) is a side effect on input the user may not have said: the guard ABORTS the speculation
+     *  instead (the endpointed final then dispatches normally and may use the tool for real). */
+    private static readonly SPEC_SAFE_TOOLS;
     /** Lazily resolved memory tools (async loadMemory runs in initMemory). */
     private memoryReady;
     constructor(options?: Partial<DuplexAgentOptions>);
@@ -1009,6 +1023,10 @@ declare class DuplexAgent {
     /** Flush any held-back trailing fragment (a possible `[task` opener that never completed) once the
      *  turn's stream is done — so a legit message ending in "[t" isn't silently dropped. */
     private flushHeldReflexTail;
+    /** Remove complete stage-direction parentheticals from the UNFORWARDED reflex text (STAGE_DIRECTION_RE).
+     *  Only the unforwarded region is touched — already-spoken audio can't be unsent, and splicing before
+     *  reflexForwarded would corrupt the forward offset. */
+    private scrubStageDirections;
     /** Clear the per-turn guards. Called at the head of every voice turn (user send + re-voice flush). */
     private resetTurn;
     /** preToolUse guard on the reflex: once it has dispatched this turn, a dispatch is "said my piece,
@@ -1016,6 +1034,11 @@ declare class DuplexAgent {
      *  re-dispatch — so the only remaining move is to voice a short ack and end. A genuinely NEW Act is
      *  still allowed (parallel independent work). During a re-ack pass, block every tool. */
     private dispatchGuard;
+    /** The host spoke on this turn's behalf OUTSIDE the reflex stream (e.g. the voice engine's adaptive
+     *  micro-ack). For a DISPATCHED turn that's a sufficient ack (don't stack a second one); for an
+     *  inline turn whose reply came back empty it is NOT — "One sec." followed by permanent silence is
+     *  still dead air, so silentTurn ignores external speech unless work was dispatched. */
+    noteExternalSpeech(): void;
     /** True when the just-finished turn voiced NOTHING — dead air to repair. Two ways this happens:
      *  (a) a dispatch with no spoken ack, and (b) an INLINE turn whose `final` channel came back empty —
      *  gpt-oss harmony sometimes puts the whole reply in `analysis` (→ thinking_delta, suppressed in
@@ -1026,8 +1049,27 @@ declare class DuplexAgent {
      *  line (no template). If it STILL says nothing, fall back to a minimal line so silence never ships.
      *  Wording adapts to whether work was dispatched (an ack) or the inline reply was simply lost. */
     private ackIfSilent;
-    /** One user turn: the voice agent streams the reply (and may Act/Think). Serialized with re-voice turns. */
+    /** Dead-air fallback pools (see ackIfSilent). Both retry lines keep the 'say that again' phrase —
+     *  hosts/tests key on it. */
+    private static readonly FALLBACK_ACKS;
+    private static readonly FALLBACK_RETRY;
+    private lastFallback;
+    /** One user turn: the voice agent streams the reply (and may Act/Think). Serialized with re-voice turns.
+     *  If a speculative reflex call is pending and this content CONFIRMS it (the final matches the
+     *  speculated partial), the in-flight call is adopted as THE turn: its buffered output flushes to the
+     *  host NOW (this is the latency win) and streaming continues live. Any other content aborts the
+     *  speculation first (rolled back silently) and runs a normal turn behind it. */
     send(content: MessageContent): Promise<RunResult>;
+    /** Start a HELD speculative reflex turn on a stable partial (VoiceEngine.onSpeculate). Its spoken
+     *  output buffers in emitHost; send() later confirms (flush + adopt) or aborts it. The turn holds the
+     *  voice mutex until that decision (bounded by a safety timeout), so no other turn can interleave
+     *  into the transcript between the speculative messages and their rollback. No-op if a speculation
+     *  is already in flight. */
+    speculate(text: string): void;
+    /** Abort a pending speculation (idempotent): kill the in-flight call, drop its buffered output.
+     *  Called on divergence (send), a stale partial (VoiceEngine.onSpeculateAbort), a side-effect tool
+     *  attempt (dispatchGuard), or host paths that will never dispatch a final (commands, voice-off). */
+    abortSpeculation(): void;
     /** Cancel a running background task — shared by the CancelTask tool and the CLI /tasks picker. */
     cancelTask(id: string): string;
     /** Barge-in: the user took the floor while task(s) were running. Suppress those tasks' remaining SPOKEN
@@ -1040,6 +1082,11 @@ declare class DuplexAgent {
     /** Promise-chain mutex: turns run strictly one at a time; a failed turn doesn't poison the chain. */
     private enqueue;
     private notify;
+    /** Host-boundary emit for the reflex's spoken channel: during a PENDING speculation, text_delta and
+     *  hold_filler are BUFFERED (nothing may reach TTS on unconfirmed input); confirm flushes them in
+     *  order, abort drops them silently. Everything else (task_* lifecycle, worker speak_utterance —
+     *  which bypasses this via host.notify directly) passes through untouched. */
+    private emitHost;
     /** Queue a `[task …]` event for re-voicing. Events arriving while the voice is busy coalesce into ONE turn.
      *  `nonClean` (out-of-band boolean, set by the caller that KNOWS this event integrates a NON-CLEAN outcome)
      *  marks the coalesced flush as a non-clean integration turn — turn-eligibility, never inferred from event
@@ -1221,6 +1268,22 @@ interface AudioSink {
 type AuthProvider = string | (() => string | Promise<string>);
 declare function resolveAuth(auth: AuthProvider): Promise<string>;
 
+/** Injectable time source — ALL engine timing (merge windows, barge grace, overlap resume, drain
+ *  settles) routes through it. Defaults to real timers; the deterministic bench swaps a virtual
+ *  clock so timing scenarios run instantly and exactly. */
+interface EngineClock {
+    now(): number;
+    setTimeout(fn: () => void, ms: number): unknown;
+    clearTimeout(handle: unknown): void;
+}
+/** One structured diagnostic event (fire-and-forget, never affects behavior). `t` = engine clock ms.
+ *  Emitted at every DECISION point; kinds + fields are the per-session forensic record
+ *  (CLI: `<session>.voice.jsonl`; lab: `<session>.timeline.jsonl`). See mind/11-voice.md §Diagnostics. */
+interface DiagEvent {
+    t: number;
+    kind: string;
+    [k: string]: unknown;
+}
 /** Structural contracts (satisfied by SonioxSTT/CartesiaTTS or test fakes). */
 interface SttLike {
     usingAec: boolean;
@@ -1237,6 +1300,9 @@ interface TtsLike {
     onAudio: (chunk: Uint8Array) => void;
     onDone: () => void;
     connect(): Promise<void> | void;
+    /** Optional: prime the synthesis pipeline right after connect (throwaway context, audio discarded)
+     *  so the FIRST real turn doesn't pay the provider's cold-synthesis spin-up. */
+    warmup?(): void;
     newContext(): string;
     speak(text: string, cont: boolean): void;
     end(): void;
@@ -1258,6 +1324,12 @@ declare class VoiceEngineOptions {
     onBargeIn: (phase: 'speaking' | 'drain') => void;
     /** spoken micro-ack on utterance endpoint (masks LLM TTFT); '' disables */
     ackPhrase: string;
+    /** ADAPTIVE micro-ack: on an utterance-dispatched turn, speak a short varied ack ONLY if no reflex
+     *  delta has arrived after this many ms (masks a slow TTFT without acking every turn — a fixed
+     *  per-turn ack was rejected as robotic). First delta / interrupt / hold cancels it. 0 = off. */
+    adaptiveAckMs: number;
+    /** the adaptive ack actually fired (host can mark the turn as spoken — e.g. suppress dead-air repair) */
+    onAdaptiveAck: () => void;
     /** Endpoint merge window (ms): hold an endpointed utterance briefly — if speech resumes (spelled
      *  letters, mid-thought pauses), the next utterance MERGES instead of dispatching a truncated one
      *  ("E-L-Y." / "A."). Costs this much latency per turn; 0 disables. */
@@ -1303,11 +1375,46 @@ declare class VoiceEngineOptions {
      *  speech at all, an audible hiccup. Default OFF: the genuine-gated STT partial is the
      *  mechanism-correct pause trigger; enable only if barge-in onset feels sluggish in a clean-AEC room. */
     overlapEnergyHold: boolean;
+    /** SPECULATIVE REFLEX START (the root TTFT fix): a partial transcript that has stopped changing for
+     *  this many ms AND carries ≥ speculativeMinWords is "stable" — `onSpeculate` fires so the host can
+     *  start the reflex EARLY, ~endpoint+merge (500-850ms) before the final would dispatch. The
+     *  speculative call's output is HELD by the host (nothing reaches TTS) until the endpointed final
+     *  confirms it (see speculationConfirms). At most one speculation per turn-in-progress. 0 = off. */
+    speculativeMs: number;
+    /** Minimum word count for a partial to qualify as a speculation trigger. */
+    speculativeMinWords: number;
+    /** A stable partial (speculativeMs) — the host starts a HELD speculative reflex call. */
+    onSpeculate: (text: string) => void;
+    /** AGENT-SIDE BACKCHANNELING (rule-based v1): while LISTENING to a long multi-clause user turn, a
+     *  partial that reaches a clause boundary (trailing [,.;!?] or conjunction/filler) and then stays
+     *  UNCHANGED for this many ms (a micro-pause — before the silence endpoint fires) triggers a short
+     *  quiet TTS blip ("Mm-hm.") on a throwaway context. ZERO floor-claim: no state change, no timers
+     *  touched, no turn context — audio passes a narrow gate bypass and a real turn supersedes it via
+     *  context rotation. Latin-predominant partials only (Hebrew/mixed text never misfires — the
+     *  boundary/conjunction heuristics are English-tuned, so non-Latin turns simply get no blips).
+     *  0 = off (default). ~200-300 recommended: live, Soniox's SEMANTIC endpoint (<end>) lands within
+     *  ~300-400ms of a clause pause — a longer stability window loses the race and never fires. */
+    backchannelMs: number;
+    /** Min gap between blips (rate limit); additionally max 2 blips per user turn-in-progress. */
+    backchannelMinGapMs: number;
+    /** Only multi-clause turns: the partial must carry at least this many words before a blip. */
+    backchannelMinWords: number;
+    /** A backchannel blip was spoken (host renders a timeline event; the blip is NOT a reply). */
+    onBackchannel: (phrase: string) => void;
+    /** The partial outgrew the speculated text (user kept talking) — the host aborts the speculation
+     *  quietly (the endpointed final will also refuse to confirm; this just stops the billing earlier). */
+    onSpeculateAbort: () => void;
     /** Map inline `[emotion]` tags (emitted by the model, prompt-taught) into Cartesia inline emotion
      *  tags in the spoken transcript (sonic-3 stitches the prosody). false = strip them silently. */
     emotions: boolean;
     /** Show the `[emotion]` tags in the on-screen echo (debug). false = hide (spoken-only). */
     showEmotions: boolean;
+    /** Time source for every engine timer/timestamp. Default: real time (identical behavior). */
+    clock: EngineClock;
+    /** Structured diagnostic event at every decision point (state transitions, dispatch/drop paths,
+     *  barge-in, overlap pause/resume, acks, speculation, backchannels, echo swallows). Fire-and-forget:
+     *  a throwing handler is caught once and diagnostics disable — engine behavior is never affected. */
+    onDiag: (ev: DiagEvent) => void;
 }
 declare class VoiceEngine {
     options: VoiceEngineOptions;
@@ -1319,6 +1426,7 @@ declare class VoiceEngine {
     private ctxOpen;
     private interrupted;
     private spokeDeltas;
+    private clock;
     private drainTimer;
     private echoWords;
     private prevReply;
@@ -1328,15 +1436,36 @@ declare class VoiceEngine {
     private hot;
     private suspectUntil;
     private ackAt;
+    private lastAck;
+    private ackTimer;
     private bargeGraceUntil;
     private pendingUtt;
+    private mergePath;
+    private lastGraceDiag;
     private pendingTimer;
+    private lastDispatchFlat;
+    private lastDispatchWords;
+    private lastDispatchAt;
+    private repliedSinceDispatch;
+    private static readonly DUP_FINAL_MS;
     private lastInterrupted;
     private pausedAt;
     private lastResumeAt;
     private lastOverlapPartial;
     private resumeTimer;
     private turnStartAt;
+    private specPartial;
+    private specTimer;
+    private specText;
+    private specSpent;
+    private bcPartial;
+    private bcTimer;
+    private bcCount;
+    private bcActive;
+    private bcActiveTimer;
+    private lastBcAt;
+    private lastBcPhrase;
+    private recentBc;
     private uttQueue;
     private emo;
     constructor(options?: Partial<VoiceEngineOptions>);
@@ -1346,6 +1475,10 @@ declare class VoiceEngine {
     setBargeIn(on: boolean): void;
     /** Show/hide the `[emotion]` debug tags in the echo (next turn's stream picks it up). */
     setShowEmotions(on: boolean): void;
+    /** Diagnostics tap (options.onDiag). Fire-and-forget: a throwing handler disables the tap once —
+     *  it can NEVER perturb engine behavior. Protected so VoiceIO can route provider events through it. */
+    private diagOn;
+    protected diag(kind: string, fields?: Record<string, unknown>): void;
     private idleWaiters;
     private setState;
     /** Resolve when the engine is no longer speaking (immediate if already idle). */
@@ -1374,6 +1507,20 @@ declare class VoiceEngine {
     /** Play the next queued worker utterance as its own one-shot turn (begin → delta → end). Drives the
      *  next one from the settle completion (endSpeech), so utterances serialize without overlap. */
     private pumpQueue;
+    /** Short varied adaptive acks (adaptiveAckMs) — two shape pools picked by the dispatched
+     *  utterance (question → thinking-ish, otherwise neutral/on-it), with anti-repetition (never one
+     *  of the last 4 used). A 3-phrase round-robin sounded synthetic live ("started with 'hmm' too
+     *  many times"). All phrases are sub-second and semantically safe for their shape.
+     *  No 'Okay.': a genuine user "Okay." within the 6s squash window would be swallowed as ack echo. */
+    static readonly ACKS_NEUTRAL: string[];
+    static readonly ACKS_QUESTION: string[];
+    private recentAcks;
+    private pickAck;
+    private lastDispatchWasQuestion;
+    private clearAckTimer;
+    /** Cancel a pending adaptive ack without touching the turn — hosts call this when the turn turns out
+     *  to be a Hold (intentionally silent; an ack would read as the start of an answer). */
+    cancelPendingAck(): void;
     /** barge-in: stop audio NOW, cancel generation, reset for the user's utterance */
     interrupt(): void;
     stop(): void;
@@ -1387,12 +1534,67 @@ declare class VoiceEngine {
      *  longer ones on count. */
     private genuine;
     private handlePartial;
+    /** Speculative-reflex trigger (listening side only — the speaking branch returns before this): a
+     *  partial that hasn't CHANGED for speculativeMs and has ≥ speculativeMinWords is stable → fire
+     *  onSpeculate once. If the user then keeps talking well past the speculated text, onSpeculateAbort
+     *  tells the host to kill the held call early. The confirm/abort DECISION belongs to the host at
+     *  dispatch time (speculationConfirms against the real final) — flushUtterance only resets the
+     *  trigger for the next turn. Merge windows are untouched: no speculation while an endpointed
+     *  utterance is pending (the final would be the MERGED text, which the partial alone never matches). */
+    private trackSpeculation;
+    /** Backchannel blip pool — short, quiet, semantically inert. Chosen to be transcript-safe: if
+     *  imperfect AEC lets a blip reach Soniox MID-user-speech it lands inside their partial stream, so
+     *  every phrase is a word whose accidental presence barely hurts a transcript, and flushUtterance
+     *  strips an isolated echo of the exact phrase at a clause edge within 2s (stripBackchannelEcho).
+     *  'Okay.'/'Right.' are fine here (unlike ACKS_NEUTRAL's no-Okay rule): a standalone user "Okay."
+     *  right after OUR blip is overwhelmingly the blip's echo — the squash guard eating it is the point. */
+    static readonly BACKCHANNELS: string[];
+    /** Backchannel trigger (listening side only — the speaking branch returns before this): a partial
+     *  that reached a clause boundary and then stayed UNCHANGED for backchannelMs (a micro-pause, still
+     *  BEFORE the silence endpoint) fires a blip — if long enough (≥ backchannelMinWords), predominantly
+     *  Latin, rate-limited (backchannelMinGapMs + max 2/turn), and no endpointed text is pending.
+     *  Touches nothing else: merge/endpoint/speculation timers and turn state are never affected. */
+    private trackBackchannel;
+    /** Speak one blip on a THROWAWAY TTS context. Zero floor-claim: no `speaking`, no state change, no
+     *  markTurn, no merge/endpoint/barge timers, no repliedSinceDispatch, no utterance queue. Echo
+     *  pre-seeding happens BEFORE any audio exists: the blip's words join echoWords (its mic echo is
+     *  never "novel"), the ack-squash guard is armed with the exact phrase (a standalone echo final is
+     *  swallowed), and the echo window extends so echo-shaped finals stay gated. */
+    private fireBackchannel;
+    /** A real turn (or shutdown) takes over mid-blip: close the audio bypass + stability timer. */
+    private bcSupersede;
+    /** Strip the mic echo of the LAST blip from a dispatching final, conservatively: only the exact
+     *  phrase, as an isolated token at a clause edge (start/end of utterance or beside punctuation),
+     *  within 2s of the blip. "Right"/"okay" as genuine mid-sentence content words are never touched. */
+    private stripBackchannelEcho;
     /** Merge a resumed utterance into the pending one, deduping any word-overlap. Soniox re-finalizes
      *  overlapping audio when the silence-timer and the semantic `<end>` both endpoint a growing
      *  utterance (or after a reconnect): the next "utterance" repeats the tail of the previous one, and
      *  a naive `${prev} ${next}` produced the live duplication ("Um, I want to check if Um, I want to
      *  check if…"). Find the longest suffix of `prev`'s words that prefixes `next` and drop it. */
     private mergeUtterance;
+    private static normWord;
+    /** Soniox re-finalization of an ALREADY-DISPATCHED utterance, arriving PAST the merge window:
+     *  the new final is a strict word-prefix superset of the last dispatch (the last dispatched word
+     *  may be a char-prefix of the corresponding new word — a mid-word endpoint), and it lands within
+     *  DUP_FINAL_MS of the dispatch. Live: "Hi, please tell me a very short" dispatched, then the full
+     *  "…very short joke." re-finalized ~900ms later → dispatched TWICE (two replies, the second to a
+     *  question already being answered).
+     *  DESIGN — HYBRID at the caller (flushUtterance): this shape check identifies a re-finalization
+     *  (a human physically cannot re-speak a ≥3-word sentence plus extra words within 3s of the
+     *  previous dispatch; a genuine continuation arrives as NEW words, never as a superset), and
+     *  repliedSinceDispatch then picks the action:
+     *  • reply already streaming (the live trace: TTFT ~500ms < the ~900ms re-final) → DROP. True
+     *    "supersede" would mean aborting audible speech mid-word to re-answer nearly the same text —
+     *    worse UX — and needs turn-abort plumbing in every host (the lab bridge has none;
+     *    DuplexAgent.send is a non-cancelable queue, so a second send just stacks a SECOND full reply).
+     *    Soniox's premature endpoint fires at a prosodic boundary, so the loss is trailing word(s).
+     *  • NO reply yet → DISPATCH the fuller text. Live-verified necessity: the reflex Holds on the
+     *    truncated fragment ("Hi, please tell me a very short" → Hold), and dropping the re-final then
+     *    starves the conversation entirely — the fuller final IS the completion the Hold is waiting
+     *    for. A slow-but-answering reflex in this window degrades to today's two-reply behavior (rare
+     *    race), never worse. Fillers ("mhm") deliberately don't count as replies — see speakFiller. */
+    private refinalizes;
     private static readonly TRAIL_RE;
     /** The utterance sounds like the user paused mid-thought (trailing conjunction/filler/comma). */
     private looksIncomplete;
@@ -1430,6 +1632,15 @@ declare class SonioxSTT {
     /** Unrecoverable: the mic source stopped delivering audio (Soniox starves → idle-timeout reconnect
      *  loop). The host tears voice down instead of spinning forever. */
     onFatal: (message: string) => void;
+    /** Diagnostic tap (provider errors, ws reconnects, no-audio watchdog). Fire-and-forget: a throwing
+     *  handler disables itself — never affects transcription. Wired by VoiceIO / the lab bridge. */
+    onDiag: (ev: {
+        t: number;
+        kind: string;
+        [k: string]: unknown;
+    }) => void;
+    private diagOn;
+    private diag;
     private lastChunkAt;
     private startedChunksAt;
     private noAudioTimer;
@@ -1462,6 +1673,15 @@ declare class CartesiaTTS {
     ctxId: string;
     onAudio: (chunk: Uint8Array) => void;
     onDone: () => void;
+    /** Diagnostic tap (provider errors, circuit-breaker open/close, ws reconnects). Fire-and-forget:
+     *  a throwing handler disables itself — never affects synthesis. Wired by VoiceIO / the lab bridge. */
+    onDiag: (ev: {
+        t: number;
+        kind: string;
+        [k: string]: unknown;
+    }) => void;
+    private diagOn;
+    private diag;
     firstAudioAt: number;
     /** Circuit breaker: consecutive error count + down flag. */
     private consecutiveErrors;
@@ -1482,6 +1702,11 @@ declare class CartesiaTTS {
     private markRecovered;
     /** Ensure the WS is open before sending — reconnects if idle-closed. */
     private ensureConnected;
+    /** Prime the synthesis pipeline on a throwaway context right after connect: the FIRST synthesis on
+     *  a fresh WS pays Cartesia's cold spin-up (~1s+ live) — absorb it at mic-on instead of turn one.
+     *  The returned audio is discarded (engine drops onAudio while not speaking; once a real turn calls
+     *  newContext() any stragglers are stale-context-filtered). Respects the circuit breaker. */
+    warmup(): void;
     newContext(): string;
     private frame;
     speak(text: string, cont: boolean): void;