@livx.cc/agentx 0.99.165 → 0.99.166

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -8149,6 +8149,9 @@ var VoiceEngine = class _VoiceEngine {
8149
8149
  // bumps on every aside start/cancel — a late aside of a dead hold never plays
8150
8150
  speaking = false;
8151
8151
  // audible (deltas flowing OR audio draining)
8152
+ speechStartAt = 0;
8153
+ // clock time `speaking` last turned on / off
8154
+ speechEndAt = 0;
8152
8155
  ctxOpen = false;
8153
8156
  // the current TTS context still accepts deltas (false once end-frame sent)
8154
8157
  interrupted = false;
@@ -8346,11 +8349,13 @@ var VoiceEngine = class _VoiceEngine {
8346
8349
  this.handleUtterance(k.text, language ? { [language]: k.text.length } : void 0, k.turns);
8347
8350
  };
8348
8351
  this.stt.onLevel = (rms) => this.handleLevel(rms);
8352
+ this.stt.isAudible = (agoMs = 0) => this.speakingAt(this.clock.now() - agoMs);
8349
8353
  const stt = this.stt;
8350
8354
  this.agentText = stt.agentTextInput && stt.sendAgentText ? new AgentTextFeed({
8351
8355
  clock: this.clock,
8352
8356
  playedMs: () => Math.max(0, this.player.playedMs()),
8353
8357
  send: (text, end) => {
8358
+ this.diag("agent_text", { text: text.slice(0, 80), ...end ? { end: true } : {} });
8354
8359
  try {
8355
8360
  stt.sendAgentText(text, end);
8356
8361
  } catch (e) {
@@ -8442,7 +8447,10 @@ var VoiceEngine = class _VoiceEngine {
8442
8447
  }
8443
8448
  this.diag("tts_turn_begin", { ack, gapless: this.speaking });
8444
8449
  const gapless = this.speaking;
8445
- if (!this.speaking) this.player.markTurn();
8450
+ if (!this.speaking) {
8451
+ this.player.markTurn();
8452
+ this.speechStartAt = this.clock.now();
8453
+ }
8446
8454
  this.speaking = true;
8447
8455
  this.ctxOpen = true;
8448
8456
  this.spokeDeltas = false;
@@ -8667,6 +8675,7 @@ var VoiceEngine = class _VoiceEngine {
8667
8675
  }
8668
8676
  this.drainTimer = null;
8669
8677
  this.speaking = false;
8678
+ this.speechEndAt = this.clock.now();
8670
8679
  this.agentText?.end();
8671
8680
  this.stopWordReveal(true);
8672
8681
  if (this.turnStartAt) log15.debug(`turn: ${Math.round(this.clock.now() - this.turnStartAt)}ms (incl. playback)`);
@@ -8865,6 +8874,7 @@ var VoiceEngine = class _VoiceEngine {
8865
8874
  if (this.reply || droppedQueued) this.lastInterrupted = { full: this.reply, heard, exact, queued, queuedTurn: this.queuedTurn };
8866
8875
  if (heard.length < this.reply.length || droppedQueued) this.bargedSinceDispatch = true;
8867
8876
  this.speaking = false;
8877
+ this.speechEndAt = this.clock.now();
8868
8878
  this.ctxOpen = false;
8869
8879
  this.interrupted = true;
8870
8880
  this.suspectUntil = 0;
@@ -8974,6 +8984,14 @@ var VoiceEngine = class _VoiceEngine {
8974
8984
  novelWords(text) {
8975
8985
  return this.words(text).filter((w) => !this.echoWords.has(w));
8976
8986
  }
8987
+ /** Was the assistant speaking at clock time `t` (+ PHANTOM_TAIL_MS)? Narrower than echoActive's multi-second echo
8988
+ * tail on purpose: the STT phantom guard drops low-confidence short finals here, and a real "Yes." said right after
8989
+ * a question must not land inside it. speechEndAt already trails the audio by the settle margin (~300ms). */
8990
+ speakingAt(t) {
8991
+ if (t < this.speechStartAt) return false;
8992
+ return this.speaking || t < this.speechEndAt + _VoiceEngine.PHANTOM_TAIL_MS;
8993
+ }
8994
+ static PHANTOM_TAIL_MS = 300;
8977
8995
  echoActive() {
8978
8996
  return this.speaking || this.clock.now() < this.echoUntil;
8979
8997
  }
@@ -10043,7 +10061,18 @@ var SonioxSTTOptions = class {
10043
10061
  * same socket (`sendAgentText`, which VoiceEngine drives paced to playback), so Soniox filters the leftover echo
10044
10062
  * of our own voice out of the transcript. Complements AEC — keep echo cancellation on. */
10045
10063
  agentTextInput = false;
10064
+ /** Phantom guard: when its last word fell while the assistant was speaking (`isAudible`, wired by VoiceEngine), drop a
10065
+ * final of at most `phantomMaxWords` words, all listener tokens (`phantomTokens`), whose least-sure token is below
10066
+ * `phantomMinConfidence`. stt-rt-v6 makes up backchannels ("Okay.", "Yeah.", "Mm-hmm.") from echo residue under our
10067
+ * own voice: all at confMin 0.33-0.78 (agentx session 20261111-124649). Confidence alone can't separate them from a
10068
+ * real word said over our voice — `say` clips into live Soniox under a reply's residue: 12/19 soft interjections fell
10069
+ * below 0.8 — so words that act ("Stop.", "No.", "Yes.", "Wait.") always pass; a listener token over our own speech
10070
+ * is a no-op anyway. 0 = off. */
10071
+ phantomMinConfidence = 0.8;
10072
+ phantomMaxWords = 2;
10073
+ phantomTokens = /^(?:okay|ok|yeah|yep|yup|mm-?hmm|mhm|hmm+|mm+|uh-?huh|oh|ah|right)$/i;
10046
10074
  };
10075
+ var STD_TOKEN_KEYS = /* @__PURE__ */ new Set(["text", "start_ms", "end_ms", "confidence", "is_final", "language", "speaker"]);
10047
10076
  var SonioxSTT = class {
10048
10077
  options;
10049
10078
  ws;
@@ -10068,6 +10097,9 @@ var SonioxSTT = class {
10068
10097
  * loop). The host tears voice down instead of spinning forever. */
10069
10098
  onFatal = () => {
10070
10099
  };
10100
+ /** Was the assistant speaking `agoMs` ago (+ a short tail)? Set by VoiceEngine; gates the phantom guard, asked for the
10101
+ * moment the utterance's last word was said (not when its final landed, ~1s later). */
10102
+ isAudible = () => false;
10071
10103
  /** Diagnostic tap (provider errors, ws reconnects, no-audio watchdog). Fire-and-forget: a throwing
10072
10104
  * handler disables itself — never affects transcription. Wired by VoiceIO / the lab bridge. */
10073
10105
  onDiag = () => {
@@ -10102,6 +10134,12 @@ var SonioxSTT = class {
10102
10134
  finalToks = [];
10103
10135
  // diarization only: finalized tokens of the current utterance
10104
10136
  partialToks = [];
10137
+ uttToks = [];
10138
+ // stt_final diag: finalized tokens of the current utterance
10139
+ uttPartials = [];
10140
+ // stt_final diag: its current partial tokens
10141
+ levels = [];
10142
+ // stt_final diag: recent mic RMS per chunk (~12s)
10105
10143
  constructor(options) {
10106
10144
  this.options = { ...new SonioxSTTOptions(), ...options };
10107
10145
  }
@@ -10165,10 +10203,7 @@ var SonioxSTT = class {
10165
10203
  const combined = (this.finalText + this.partialText).trim();
10166
10204
  if (!combined || now() - this.lastChangeAt < this.options.silenceEndpointMs) return;
10167
10205
  if (this.firstTokenAt) log16.debug(`stt: ${Math.round(now() - this.firstTokenAt)}ms first-token\u2192silence-endpoint, "${combined.slice(0, 60)}"`);
10168
- const language = this.language(true);
10169
- const turns = this.turns(true), ago = this.lastWordAgoMs();
10170
- this.reset();
10171
- this.onUtterance(this.stray(combined), now(), language, turns, ago);
10206
+ this.emit(combined, "silence");
10172
10207
  }, 120);
10173
10208
  this.endpointTimer.unref?.();
10174
10209
  this.startedChunksAt = now();
@@ -10194,7 +10229,10 @@ var SonioxSTT = class {
10194
10229
  const v = view.getInt16(i, true);
10195
10230
  sum += v * v;
10196
10231
  }
10197
- this.onLevel(Math.sqrt(sum / (chunk.byteLength / 2)));
10232
+ const rms = Math.sqrt(sum / (chunk.byteLength / 2));
10233
+ this.levels.push({ at: this.lastChunkAt, rms });
10234
+ if (this.levels.length > 120) this.levels.shift();
10235
+ this.onLevel(rms);
10198
10236
  if (this.ws.readyState === WebSocket.OPEN) {
10199
10237
  this.ws.send(chunk);
10200
10238
  const at = this.sentBytes / 2 / STT_SAMPLE_RATE * 1e3;
@@ -10225,6 +10263,7 @@ var SonioxSTT = class {
10225
10263
  if (t.text !== "<end>" && typeof t.end_ms === "number" && String(t.text).trim() && t.end_ms > this.lastWordEndMs) this.lastWordEndMs = t.end_ms;
10226
10264
  if (t.text === "<end>") endpoint = true;
10227
10265
  else if (t.is_final) {
10266
+ this.uttToks.push(t);
10228
10267
  this.finalText += t.text;
10229
10268
  this.countLang(this.finalLangChars, t);
10230
10269
  if (this.options.diarization) this.finalToks.push(t);
@@ -10235,6 +10274,7 @@ var SonioxSTT = class {
10235
10274
  this.partialLangChars = {};
10236
10275
  for (const t of partials) this.countLang(this.partialLangChars, t);
10237
10276
  if (this.options.diarization) this.partialToks = partials;
10277
+ this.uttPartials = partials;
10238
10278
  const combined = this.finalText + this.partialText;
10239
10279
  if (combined !== this.lastCombined) {
10240
10280
  this.lastCombined = combined;
@@ -10245,10 +10285,32 @@ var SonioxSTT = class {
10245
10285
  if (endpoint && this.finalText.trim()) {
10246
10286
  const utterance = this.finalText.trim();
10247
10287
  if (this.firstTokenAt) log16.debug(`stt: ${Math.round(now() - this.firstTokenAt)}ms first-token\u2192endpoint, "${utterance.slice(0, 60)}"`);
10248
- const language = this.language(false);
10249
- const turns = this.turns(false), ago = this.lastWordAgoMs();
10250
- this.reset();
10251
- this.onUtterance(this.stray(utterance), now(), language, turns, ago);
10288
+ this.emit(utterance, "end");
10289
+ }
10290
+ }
10291
+ /** Close the current utterance: diag, reset, then hand it to onUtterance — unless the phantom guard drops it. */
10292
+ emit(text, endpoint) {
10293
+ const withPartials = endpoint === "silence";
10294
+ const language = this.language(withPartials);
10295
+ const turns = this.turns(withPartials), ago = this.lastWordAgoMs();
10296
+ const confMin = this.finalDiag(text, endpoint, ago);
10297
+ this.reset();
10298
+ if (this.phantom(text, confMin, ago)) {
10299
+ this.diag("stt_phantom_dropped", { text: text.slice(0, 80), confMin, lastWordAgoMs: ago });
10300
+ return;
10301
+ }
10302
+ this.onUtterance(this.stray(text), now(), language, turns, ago);
10303
+ }
10304
+ phantom(text, confMin, agoMs) {
10305
+ const { phantomMinConfidence: min, phantomMaxWords: max, phantomTokens: listener } = this.options;
10306
+ if (!min || confMin === void 0 || confMin >= min) return false;
10307
+ const words4 = text.split(/[\s,.!?…]+/).filter(Boolean);
10308
+ if (!words4.length || words4.length > max || !words4.every((w) => listener.test(w))) return false;
10309
+ try {
10310
+ return this.isAudible(agoMs ?? 0);
10311
+ } catch (e) {
10312
+ log16.warn(`isAudible threw: ${e instanceof Error ? e.message : e}`);
10313
+ return false;
10252
10314
  }
10253
10315
  }
10254
10316
  /** How long ago (ms, now) the utterance's last spoken word ended: the audio sent so far minus that token's end_ms on
@@ -10281,7 +10343,29 @@ var SonioxSTT = class {
10281
10343
  const words4 = sonioxWords(withPartials ? [...this.finalToks, ...this.partialToks] : this.finalToks);
10282
10344
  return words4.some((w) => w.spk != null) ? speakerRuns(words4) : void 0;
10283
10345
  }
10346
+ /** One `stt_final` diag per utterance: how sure Soniox was, the mic level over it, and any token fields beyond the
10347
+ * usual ones — to tell words the user said from ones the STT made up (a phantom "Okay." heard 186× over our own
10348
+ * speech, agentx session 20261111-105859). */
10349
+ finalDiag(text, endpoint, lastWordAgoMs) {
10350
+ const toks = [...this.uttToks, ...endpoint === "silence" ? this.uttPartials : []].filter((t) => String(t.text).trim());
10351
+ const conf = toks.map((t) => t.confidence).filter((c) => typeof c === "number");
10352
+ const since = (this.firstTokenAt || now()) - 1500;
10353
+ const rms = this.levels.filter((l) => l.at >= since).map((l) => l.rms).sort((a, b) => a - b);
10354
+ const extra = [...new Set(toks.flatMap((t) => Object.keys(t).filter((k) => !STD_TOKEN_KEYS.has(k))))];
10355
+ this.diag("stt_final", {
10356
+ text: text.slice(0, 80),
10357
+ endpoint,
10358
+ lastWordAgoMs,
10359
+ ...conf.length ? { confMin: +Math.min(...conf).toFixed(3), confAvg: +(conf.reduce((a, b) => a + b, 0) / conf.length).toFixed(3) } : {},
10360
+ ...rms.length ? { rmsPeak: Math.round(rms[rms.length - 1]), rmsMed: Math.round(rms[rms.length >> 1]) } : {},
10361
+ ...toks.length ? { startMs: toks[0].start_ms, endMs: toks[toks.length - 1].end_ms } : {},
10362
+ ...extra.length ? { tokenKeys: extra } : {}
10363
+ });
10364
+ return conf.length ? Math.min(...conf) : void 0;
10365
+ }
10284
10366
  reset() {
10367
+ this.uttToks = [];
10368
+ this.uttPartials = [];
10285
10369
  this.finalToks = [];
10286
10370
  this.partialToks = [];
10287
10371
  this.finalLangChars = {};
@@ -10384,7 +10468,7 @@ var now2 = () => performance.now();
10384
10468
  var CartesiaTTSOptions = class {
10385
10469
  auth = "";
10386
10470
  voiceId = "";
10387
- model = "sonic-3.5";
10471
+ model = "sonic-3.6";
10388
10472
  /** 'apiKey' (server/CLI) → `api_key=` URL param; 'token' (browser, BE-minted) → `access_token=`. */
10389
10473
  authMode = "apiKey";
10390
10474
  /** Cartesia `language` code ('en', 'he', …). Unset → field omitted (provider default). Mutable:
@@ -10486,6 +10570,16 @@ var CartesiaTTS = class _CartesiaTTS {
10486
10570
  };
10487
10571
  this.ws.onmessage = (ev) => {
10488
10572
  const m = JSON.parse(String(ev.data));
10573
+ const pr = m.context_id && this.priming.get(m.context_id);
10574
+ if (pr) {
10575
+ if (m.type === "chunk" && m.data) pr.chunks.push(base64ToBytes(m.data));
10576
+ else if (m.type === "done" || m.type === "error") {
10577
+ this.priming.delete(m.context_id);
10578
+ if (m.type === "done") this.primed.set(pr.key, pr.chunks);
10579
+ else this.primed.delete(pr.key);
10580
+ }
10581
+ return;
10582
+ }
10489
10583
  if (m.context_id && m.context_id !== this.ctxId) return;
10490
10584
  if (m.type === "chunk" && m.data) {
10491
10585
  this.consecutiveErrors = 0;
@@ -10543,7 +10637,40 @@ var CartesiaTTS = class _CartesiaTTS {
10543
10637
  this.newContext();
10544
10638
  if (this.ws?.readyState === WebSocket.OPEN) this.ws.send(this.frame("Ok.", false));
10545
10639
  }
10640
+ /** Phrases synthesized ahead (prime): key → audio (null = still generating). */
10641
+ primed = /* @__PURE__ */ new Map();
10642
+ priming = /* @__PURE__ */ new Map();
10643
+ /** This context began with a primed phrase: its audio played from the cache; more text is synthesized after it. */
10644
+ ctxCached = false;
10645
+ ctxRest = "";
10646
+ ctxSent = false;
10647
+ static key(t) {
10648
+ return t.trim().replace(/[\s.,!?…]+$/u, "").replace(/\s+/g, " ");
10649
+ }
10650
+ /** Synthesize `text` now, in this voice and language, and keep its audio: a later utterance that is exactly this text
10651
+ * plays at once, with no TTS round trip (a call's planned opener — live: ~470ms to its first audio). */
10652
+ prime(text) {
10653
+ const key = _CartesiaTTS.key(text);
10654
+ if (!key || this.primed.has(key) || this.down) return;
10655
+ this.primed.set(key, null);
10656
+ const id = `prime-${++this.ctxSeq}`;
10657
+ const send = () => {
10658
+ const cur = this.ctxId, lang = this.ctxLanguage;
10659
+ this.ctxId = id;
10660
+ this.ctxLanguage = this.options.language;
10661
+ const f = this.frame(text, false);
10662
+ this.ctxId = cur;
10663
+ this.ctxLanguage = lang;
10664
+ this.priming.set(id, { key, chunks: [] });
10665
+ this.ws.send(f);
10666
+ };
10667
+ if (this.ws?.readyState === WebSocket.OPEN) send();
10668
+ else void this.ensureConnected().then(() => this.ws?.readyState === WebSocket.OPEN && send());
10669
+ }
10546
10670
  newContext() {
10671
+ this.ctxCached = false;
10672
+ this.ctxRest = "";
10673
+ this.ctxSent = false;
10547
10674
  this.ctxId = `ctx-${++this.ctxSeq}`;
10548
10675
  this.firstAudioAt = 0;
10549
10676
  this.ctxLanguage = this.options.language;
@@ -10569,6 +10696,19 @@ var CartesiaTTS = class _CartesiaTTS {
10569
10696
  if (this.down) return;
10570
10697
  text = cont ? this.say.feed(text) : this.say.feed(text) + this.say.flush();
10571
10698
  if (cont && !text) return;
10699
+ const hit = !this.firstAudioAt && !this.ctxCached && this.primed.get(_CartesiaTTS.key(text));
10700
+ if (hit) {
10701
+ this.ctxCached = true;
10702
+ this.firstAudioAt = now2();
10703
+ this.diag("tts_primed", { chars: text.length });
10704
+ for (const c of hit) this.onAudio(c);
10705
+ return;
10706
+ }
10707
+ if (this.ctxCached) {
10708
+ this.ctxRest += text;
10709
+ return;
10710
+ }
10711
+ this.ctxSent = true;
10572
10712
  if (this.ws?.readyState === WebSocket.OPEN) this.ws.send(this.frame(text, cont));
10573
10713
  else void this.ensureConnected().then(() => this.ws?.readyState === WebSocket.OPEN && this.ws.send(this.frame(text, cont)));
10574
10714
  }
@@ -10582,11 +10722,27 @@ var CartesiaTTS = class _CartesiaTTS {
10582
10722
  this.ws.send(JSON.stringify({ ...JSON.parse(this.frame("", true)), flush: true }));
10583
10723
  }
10584
10724
  end() {
10585
- const held = this.say.flush();
10725
+ let held = this.say.flush();
10586
10726
  if (this.down) {
10587
10727
  this.onDone();
10588
10728
  return;
10589
10729
  }
10730
+ const hit = !this.ctxSent && !this.ctxCached && !this.firstAudioAt && this.primed.get(_CartesiaTTS.key(held));
10731
+ if (hit) {
10732
+ this.diag("tts_primed", { chars: held.length });
10733
+ this.firstAudioAt = now2();
10734
+ for (const c of hit) this.onAudio(c);
10735
+ queueMicrotask(() => this.onDone());
10736
+ return;
10737
+ }
10738
+ if (this.ctxCached) {
10739
+ held = (this.ctxRest + held).trim();
10740
+ this.ctxRest = "";
10741
+ if (!held) {
10742
+ queueMicrotask(() => this.onDone());
10743
+ return;
10744
+ }
10745
+ }
10590
10746
  if (this.ws?.readyState === WebSocket.OPEN) this.ws.send(this.frame(held, false));
10591
10747
  }
10592
10748
  cancel() {
@@ -10688,8 +10844,6 @@ var REACT = /^(?:(?:oh|and|so|wait|but|ok|okay|yeah)\s+)*(?:wow|whoa|woah|really
10688
10844
  var WH_LEAD = "^(?:(?:oh|and|so|wait|but|ok|okay|um|uh|sorry)\\s+)*";
10689
10845
  var WH_QUESTION = new RegExp(WH_LEAD + "(?:who|whom|whose|where|which|why|how|when|what|\u05DE\u05D9|\u05D0\u05D9\u05E4\u05D4|\u05DC\u05DE\u05D4|\u05D0\u05D9\u05DA|\u05DE\u05EA\u05D9|\u05DE\u05D4)\\s+\\S", "u");
10690
10846
  var WH_EXCLAIM = new RegExp(WH_LEAD + "(?:what an? |how (?:cool|nice|fun|sweet|sad|funny|weird|crazy|strange|interesting|beautiful|lovely|cute|amazing|awesome)\\b)", "u");
10691
- var BARE = /* @__PURE__ */ new Set(["yeah", "yea", "yes", "yep", "yup", "ok", "okay", "k", "right", "sure", "mm", "mmm", "hmm", "mhm", "mm-hm", "mm-hmm", "uh-huh", "um", "uh", "ah", "oh"]);
10692
- var isBareCue = (text) => BARE.has(norm2(text).replace(/^-+|-+$/g, ""));
10693
10847
  var isReaction = (text) => {
10694
10848
  const t = norm2(text);
10695
10849
  return REACT.test(t) || new RegExp("\\s\\p{L}{1,3}$", "u").test(t) && REACT.test(t.replace(new RegExp("\\s\\p{L}{1,3}$", "u"), ""));
@@ -15653,10 +15807,6 @@ var VoiceCall = class {
15653
15807
  this.barge();
15654
15808
  return "cut";
15655
15809
  }
15656
- if (isBareCue(text)) {
15657
- this.o.onMetric({ name: "drop", kind: "bare cue", text });
15658
- return "keep";
15659
- }
15660
15810
  if (this.o.mode === "live" && isReaction(text)) return this.kept(text), "keep";
15661
15811
  const d = await this.judge.judge(text);
15662
15812
  if (d.verdict === "cut") this.barge();
@@ -15701,7 +15851,7 @@ var VoiceCall = class {
15701
15851
  * fill line: never MAIN history. */
15702
15852
  aside(text) {
15703
15853
  const now3 = this.o.now();
15704
- if (isBareCue(text) || now3 - this.lastAside.at < this.o.asideGapMs) return null;
15854
+ if (now3 - this.lastAside.at < this.o.asideGapMs) return null;
15705
15855
  const line = overlapAside(text, this.lang, this.lastAside.text, void 0, this.o.mode === "live");
15706
15856
  if (!line) return null;
15707
15857
  this.lastAside = { text: line, at: now3 };
@@ -15740,10 +15890,7 @@ var VoiceCall = class {
15740
15890
  onTurn(t) {
15741
15891
  this.floor = "";
15742
15892
  this.flushKept(t.text);
15743
- if (this.o.noteTurn && this.o.workReply && t.kind !== "replace" && this.working && !isClosing(t.text)) {
15744
- if (isBareCue(t.text)) return void this.o.onMetric({ name: "drop", id: t.id, kind: "bare cue", text: t.text });
15745
- return void this.overWork(this.cur, t);
15746
- }
15893
+ if (this.o.noteTurn && this.o.workReply && t.kind !== "replace" && this.working && !isClosing(t.text)) return void this.overWork(this.cur, t);
15747
15894
  const prev = this.cur, quick = this.quickOn && !this.working && (t.kind === "new" || prev?.lane !== "deep");
15748
15895
  const now3 = this.o.now(), lastWord = this.lastWordAt !== null;
15749
15896
  const at = this.lastWordAt ?? this.firstFinalAt ?? now3;