@agent-native/core 0.133.3 → 0.134.1

package/corpus/templates/clips/desktop/src/lib/transcription-engine.ts

@@ -30,9 +30,6 @@ export interface SourcedTranscriptSegment extends TranscriptSegment {
   source: TranscriptSource;
 }
 
-const DUPLICATE_TOKEN_OVERLAP = 0.72;
-const DUPLICATE_TIME_OVERLAP = 0.35;
-
 export interface FinalTranscriptEvent {
   /** Raw text (not trimmed); callers decide whether to skip empties. */
   text: string;
@@ -86,133 +83,212 @@ function transcriptWords(text: string): string[] {
   return normalized ? normalized.split(/\s+/) : [];
 }
 
-function tokenOverlap(left: string[], right: string[]): number {
-  if (left.length === 0 || right.length === 0) return 0;
-
-  const rightWords = new Set(right);
-  const sharedWords = new Set(left.filter((word) => rightWords.has(word)));
-  return sharedWords.size / Math.min(new Set(left).size, new Set(right).size);
-}
-
-function timeOverlapRatio(
-  left: SourcedTranscriptSegment,
-  right: SourcedTranscriptSegment,
-): number {
-  const overlapStart = Math.max(left.startMs, right.startMs);
-  const overlapEnd = Math.min(left.endMs, right.endMs);
-  const overlapMs = Math.max(0, overlapEnd - overlapStart);
-  const shorterDuration = Math.max(
-    1,
-    Math.min(left.endMs - left.startMs, right.endMs - right.startMs),
-  );
-
-  return overlapMs / shorterDuration;
+// ---------------------------------------------------------------------------
+// Echo de-duplication
+// ---------------------------------------------------------------------------
+//
+// Without headphones the microphone re-records whatever the speakers play, so
+// the remote side can reach the transcript twice: cleanly on the system stream
+// and, whenever the acoustic guard in `echo_guard.rs` was not confident enough
+// to drop it, again on the mic. The leak only ever runs one way, because a call
+// app never plays the local user back. So when both streams carry the same
+// words at the same time, the system copy is the real one and the mic copy is
+// echo — no matter which stream happened to finalize first.
+
+/** Share of the *longer* of the two word lists that has to match. Scoring the
+ *  longer side is what keeps a brief interjection ("yeah, I think so") alive
+ *  next to a long remote passage that happens to contain those words in order:
+ *  echo repeats a whole utterance, it does not sprinkle a few words into one. */
+const ECHO_MATCH_RATIO = 0.65;
+/** Fewer matched words than this is coincidence, not evidence. */
+const ECHO_MIN_WORDS = 4;
+/** Words that have to agree before a long in-order run counts as echo on its
+ *  own. Two people do not independently say eight of the same words, in the
+ *  same order, at the same moment. */
+const ECHO_LONG_MATCH_WORDS = 8;
+/** How many recently appended lines count as "at the same time".
+ *
+ *  Arrival order, not timestamps: each stream's Whisper timestamps are
+ *  estimates against its own rolling buffer, and the two streams cut those
+ *  buffers at their own silences, so the same words routinely carry stamps
+ *  seconds apart. Both finals still *arrive* within a second or two of each
+ *  other, because they are transcribed from the same sound. Finalized lines also
+ *  get a loose timestamp bound below to reject much later deliberate repeats. */
+const ECHO_RECENT_LINES = 6;
+/** Cross-stream timestamps are approximate, but echo finals should still begin
+ *  within this loose bound of each other. */
+const ECHO_MAX_START_DELTA_MS = 15_000;
+
+/** Length of the longest common subsequence of two word lists. Subsequence
+ *  rather than set intersection because echo repeats the words *in order*,
+ *  while two people using the same vocabulary do not — and rather than exact
+ *  equality because Whisper mangles echo with dropped and substituted words. */
+function commonWordRun(left: string[], right: string[]): number {
+  let previous = new Array<number>(right.length + 1).fill(0);
+  let current = new Array<number>(right.length + 1).fill(0);
+  for (const word of left) {
+    for (let index = 0; index < right.length; index++) {
+      current[index + 1] =
+        word === right[index]
+          ? previous[index] + 1
+          : Math.max(current[index], previous[index + 1]);
+    }
+    [previous, current] = [current, previous];
+  }
+  return previous[right.length];
 }
 
-function isDuplicateTranscriptSegment(
-  existing: SourcedTranscriptSegment,
-  incoming: SourcedTranscriptSegment,
+/** Whether `text` heard on the mic is the speakers bleeding back into it.
+ *
+ *  Exported because the live overlay has to make the same call on in-flight
+ *  partials, which never reach `appendFinalTranscript`.
+ *
+ *  Scored against every contiguous run of the recent system lines, not against
+ *  all of them glued together: the two streams cut speech at different points,
+ *  so one mic line can echo a single system line or straddle two, and gluing an
+ *  unrelated third one in would bury the match. */
+export function isMicEcho(
+  text: string,
+  lines: TranscriptLine[],
+  startMs: number | null = null,
 ): boolean {
-  if (existing.source === incoming.source) return false;
-
-  const existingText = normalizedTranscriptText(existing.text);
-  const incomingText = normalizedTranscriptText(incoming.text);
-  const existingWords = transcriptWords(existing.text);
-  const incomingWords = transcriptWords(incoming.text);
-
-  if (
-    !existingText ||
-    !incomingText ||
-    existingWords.length < 3 ||
-    incomingWords.length < 3
-  ) {
-    return false;
+  const words = transcriptWords(text);
+  if (words.length < ECHO_MIN_WORDS) return false;
+
+  const nearby = lines
+    .slice(-ECHO_RECENT_LINES)
+    .filter(
+      (line) =>
+        line.source === "system" &&
+        !line.historical &&
+        (startMs === null ||
+          line.startMs === null ||
+          Math.abs(startMs - line.startMs) <= ECHO_MAX_START_DELTA_MS),
+    )
+    .map((line) => transcriptWords(line.text));
+
+  for (let start = 0; start < nearby.length; start++) {
+    const run: string[] = [];
+    for (let end = start; end < nearby.length; end++) {
+      run.push(...nearby[end]);
+      const matched = commonWordRun(words, run);
+      if (matched < ECHO_MIN_WORDS) continue;
+      // Either the two are the same length and mostly agree, or they agree on
+      // a stretch long enough that nothing but echo explains it.
+      const sameUtterance =
+        matched / Math.max(words.length, run.length) >= ECHO_MATCH_RATIO;
+      const longRun =
+        matched >= ECHO_LONG_MATCH_WORDS &&
+        matched / words.length >= ECHO_MATCH_RATIO;
+      if (sameUtterance || longRun) return true;
+    }
   }
+  return false;
+}
 
-  if (timeOverlapRatio(existing, incoming) < DUPLICATE_TIME_OVERLAP) {
-    return false;
+/** Drop the recent mic lines that a just-appended system line exposes as echo. */
+function retractMicEcho(lines: TranscriptLine[]): void {
+  const snapshot = [...lines];
+  const oldest = Math.max(0, snapshot.length - ECHO_RECENT_LINES);
+  const removals: number[] = [];
+  for (let index = snapshot.length - 1; index >= oldest; index--) {
+    const line = snapshot[index];
+    if (line.source !== "mic" || line.historical) continue;
+    const evidence = snapshot.slice(index, index + ECHO_RECENT_LINES);
+    if (isMicEcho(line.text, evidence, line.startMs)) removals.push(index);
   }
+  for (const index of removals) lines.splice(index, 1);
+}
 
-  return (
-    existingText === incomingText ||
-    tokenOverlap(existingWords, incomingWords) >= DUPLICATE_TOKEN_OVERLAP
-  );
+// ---------------------------------------------------------------------------
+// Transcript lines
+// ---------------------------------------------------------------------------
+
+/** One speaker-labelled transcript line. `segments` carries the verbatim
+ *  Whisper timings behind the line and is empty for the mic-only fallback
+ *  engines, which report no timestamps. */
+export interface TranscriptLine {
+  source: TranscriptSource;
+  /** Preloaded lines are display/persistence data, not live echo evidence. */
+  historical?: boolean;
+  /** Meeting-timeline position, or null from an engine that reports none.
+   *  Not 0 — the overlay renders a timestamp for every line that has one, and
+   *  "start of the meeting" is a different claim from "unknown". */
+  startMs: number | null;
+  text: string;
+  segments: SourcedTranscriptSegment[];
 }
 
-function isDuplicateTranscriptLine(
-  lines: string[],
-  source: TranscriptSource,
-  text: string,
-): boolean {
-  const words = transcriptWords(text);
-  if (words.length < 4) return false;
+function lineFromSegments(
+  segments: SourcedTranscriptSegment[],
+): TranscriptLine {
+  return {
+    source: segments[0].source,
+    startMs: segments[0].startMs,
+    text: segments.map((segment) => segment.text).join(" "),
+    segments,
+  };
+}
 
-  const speaker = speakerFor(source);
-  return lines.some((line) => {
-    const separatorIndex = line.indexOf(":");
-    if (separatorIndex === -1 || line.slice(0, separatorIndex) === speaker) {
-      return false;
-    }
+/** Rebuild a line from a stored segment, for preloaded transcript history. */
+export function transcriptLineFromSegment(
+  segment: SourcedTranscriptSegment,
+): TranscriptLine {
+  return { ...lineFromSegments([segment]), historical: true };
+}
 
-    const existingText = line.slice(separatorIndex + 1);
-    const existingWords = transcriptWords(existingText);
-    return (
-      normalizedTranscriptText(existingText) ===
-        normalizedTranscriptText(text) ||
-      tokenOverlap(existingWords, words) >= DUPLICATE_TOKEN_OVERLAP
-    );
-  });
+/** Speaker-labelled text, as persisted by `save-browser-transcript`. */
+export function transcriptFullText(lines: TranscriptLine[]): string {
+  return lines
+    .map((line) => `${speakerFor(line.source)}: ${line.text}`)
+    .join("\n\n")
+    .trim();
+}
+
+/** Flattened verbatim segments, as persisted alongside the text. */
+export function transcriptSegments(
+  lines: TranscriptLine[],
+): SourcedTranscriptSegment[] {
+  return lines.flatMap((line) => line.segments);
 }
 
 /**
- * Fold a final-transcript event into a running transcript: appends a
- * speaker-labelled line and the event's (non-empty) segments tagged with the
- * event source. Mutates `lines`/`segments` in place. Returns true if anything
- * was appended (i.e. the event had non-empty text).
+ * Fold a final-transcript event into a running transcript, dropping mic speech
+ * that only echoes the system audio and retracting mic lines that a later
+ * system line exposes as echo. Mutates `lines` in place; returns whether the
+ * transcript changed.
  */
 export function appendFinalTranscript(
   event: FinalTranscriptEvent,
-  lines: string[],
-  segments: SourcedTranscriptSegment[],
+  lines: TranscriptLine[],
 ): boolean {
   const text = event.text.trim();
   if (!text) return false;
 
-  if (event.segments.length === 0) {
-    if (isDuplicateTranscriptLine(lines, event.source, text)) return false;
-
-    lines.push(`${speakerFor(event.source)}: ${text}`);
+  const segments: SourcedTranscriptSegment[] = event.segments
+    .map((segment) => ({
+      startMs: segment.startMs,
+      endMs: segment.endMs,
+      text: segment.text?.trim() ?? "",
+      source: event.source,
+    }))
+    .filter((segment) => segment.text.length > 0);
+
+  // One event is one stream's take on one utterance, so the whole line is the
+  // unit that is or is not echo. Engines without timestamps still produce a
+  // line, just one carrying no segments behind it.
+  const line: TranscriptLine = segments.length
+    ? lineFromSegments(segments)
+    : { source: event.source, startMs: null, text, segments: [] };
+
+  if (line.source === "mic") {
+    if (isMicEcho(line.text, lines, line.startMs)) return false;
+    lines.push(line);
     return true;
   }
 
-  const uniqueSegments = event.segments.filter((segment) => {
-    const segText = segment.text?.trim();
-    if (!segText) return false;
-
-    return !segments.some((existing) =>
-      isDuplicateTranscriptSegment(existing, {
-        ...segment,
-        text: segText,
-        source: event.source,
-      }),
-    );
-  });
-
-  if (uniqueSegments.length === 0) return false;
-
-  lines.push(
-    `${speakerFor(event.source)}: ${uniqueSegments
-      .map((segment) => segment.text.trim())
-      .join(" ")}`,
-  );
-  for (const seg of uniqueSegments) {
-    segments.push({
-      startMs: seg.startMs,
-      endMs: seg.endMs,
-      text: seg.text.trim(),
-      source: event.source,
-    });
-  }
+  lines.push(line);
+  retractMicEcho(lines);
   return true;
 }
 

package/corpus/templates/clips/desktop/src/overlays/live-transcript.tsx

@@ -1,8 +1,11 @@
-import { useEffect, useRef, useState } from "react";
+import { useEffect, useMemo, useRef, useState } from "react";
 
 import {
+  appendFinalTranscript,
+  isMicEcho,
   onFinalTranscript,
   onPartialTranscript,
+  type TranscriptLine,
 } from "../lib/transcription-engine";
 
 type Source = "mic" | "system";
@@ -13,6 +16,17 @@ export interface FinalLine {
   startMs?: number;
 }
 
+/** Preloaded history arrives without its verbatim segments. */
+function historyLine(line: FinalLine): TranscriptLine {
+  return {
+    text: line.text,
+    source: line.source,
+    startMs: line.startMs ?? null,
+    segments: [],
+    historical: true,
+  };
+}
+
 /** Format ms since meeting start as m:ss. */
 function formatTimestamp(ms: number): string {
   const total = Math.floor(ms / 1000);
@@ -38,10 +52,12 @@ export function LiveTranscript({
   onLinesChange,
   initialLines,
 }: {
-  onLinesChange?: (lines: FinalLine[]) => void;
+  onLinesChange?: (lines: TranscriptLine[]) => void;
   initialLines?: FinalLine[];
 } = {}) {
-  const [finals, setFinals] = useState<FinalLine[]>(initialLines ?? []);
+  const [finals, setFinals] = useState<TranscriptLine[]>(
+    () => initialLines?.map(historyLine) ?? [],
+  );
   const [micPartial, setMicPartial] = useState("");
   const [sysPartial, setSysPartial] = useState("");
   const scrollRef = useRef<HTMLDivElement | null>(null);
@@ -69,7 +85,7 @@ export function LiveTranscript({
     // older history), instead of being overwritten.
     if (preloadAppliedRef.current) return;
     preloadAppliedRef.current = true;
-    setFinals((prev) => [...lines, ...prev]);
+    setFinals((prev) => [...lines.map(historyLine), ...prev]);
   }, [initialLines]);
 
   useEffect(() => {
@@ -97,12 +113,13 @@ export function LiveTranscript({
       }),
     );
     trackListen(
-      onFinalTranscript(({ text, source, segments }) => {
-        const txt = text.trim();
-        if (!txt) return;
-        const startMs = segments[0]?.startMs;
-        setFinals((prev) => [...prev, { text: txt, source, startMs }]);
-        if (source === "system") setSysPartial("");
+      onFinalTranscript((event) => {
+        if (!event.text.trim()) return;
+        setFinals((prev) => {
+          const next = [...prev];
+          return appendFinalTranscript(event, next) ? next : prev;
+        });
+        if (event.source === "system") setSysPartial("");
         else setMicPartial("");
       }),
     );
@@ -126,9 +143,21 @@ export function LiveTranscript({
     el.scrollTop = el.scrollHeight;
   }, [finals, micPartial, sysPartial]);
 
+  // Partials never reach `appendFinalTranscript`, so speaker bleed shows up
+  // here as a live "You" bubble mirroring what the remote side is still
+  // saying. Judge the in-flight mic text against the in-flight system text too,
+  // since neither has been committed to a line yet.
+  const spokenMicPartial = useMemo(() => {
+    if (!micPartial) return "";
+    const inFlight: TranscriptLine[] = sysPartial
+      ? [{ source: "system", text: sysPartial, startMs: null, segments: [] }]
+      : [];
+    return isMicEcho(micPartial, [...finals, ...inFlight]) ? "" : micPartial;
+  }, [micPartial, sysPartial, finals]);
+
   return (
     <div ref={scrollRef} className="lt-chat">
-      {finals.length === 0 && !micPartial && !sysPartial ? (
+      {finals.length === 0 && !spokenMicPartial && !sysPartial ? (
         <div className="lt-empty">Listening…</div>
       ) : null}
       {finals.map((line, i) => (
@@ -142,8 +171,8 @@ export function LiveTranscript({
       {sysPartial ? (
         <ChatBubble source="system" text={sysPartial} pending />
       ) : null}
-      {micPartial ? (
-        <ChatBubble source="mic" text={micPartial} pending />
+      {spokenMicPartial ? (
+        <ChatBubble source="mic" text={spokenMicPartial} pending />
       ) : null}
     </div>
   );
@@ -164,7 +193,7 @@ function ChatBubble({
   source: Source;
   text: string;
   pending?: boolean;
-  startMs?: number;
+  startMs?: number | null;
 }) {
   const isYou = source === "mic";
   const label = isYou ? "You" : "Them";

package/corpus/templates/clips/desktop/src/overlays/recording-pill.tsx

@@ -17,7 +17,7 @@ import { getCurrentWindow } from "@tauri-apps/api/window";
 import { useCallback, useEffect, useRef, useState } from "react";
 
 import { isDirectPillClick, type ScreenPoint } from "../lib/pill-interaction";
-import { speakerFor } from "../lib/transcription-engine";
+import { speakerFor, type TranscriptLine } from "../lib/transcription-engine";
 import { LiveAudioBars } from "./live-audio-bars";
 import { LiveTranscript, type FinalLine } from "./live-transcript";
 import { PillLogo } from "./pill-logo";
@@ -52,7 +52,7 @@ export function RecordingPill() {
   );
   const finished = finishedMeetingId !== null;
   const [error, setError] = useState<string | null>(null);
-  const transcriptLinesRef = useRef<FinalLine[]>([]);
+  const transcriptLinesRef = useRef<TranscriptLine[]>([]);
   const [hasTranscriptLines, setHasTranscriptLines] = useState(false);
   const [transcriptCopied, setTranscriptCopied] = useState(false);
   const [preloadedLines, setPreloadedLines] = useState<FinalLine[]>([]);
@@ -269,7 +269,7 @@ export function RecordingPill() {
 
   // Stable callback for LiveTranscript to push locked-in lines up. Stable
   // identity matters — it's a dep of an effect inside LiveTranscript.
-  const handleTranscriptLines = useCallback((lines: FinalLine[]) => {
+  const handleTranscriptLines = useCallback((lines: TranscriptLine[]) => {
     transcriptLinesRef.current = lines;
     setHasTranscriptLines(lines.length > 0);
   }, []);