@agent-native/core 0.94.0 → 0.94.2

package/corpus/core/src/client/AssistantChat.tsx

@@ -146,6 +146,10 @@ import { AgentComposerFrame } from "./composer/AgentComposerFrame.js";
 import { TextAttachmentAdapter } from "./composer/attachment-accept.js";
 import { isPastedTextAttachmentName } from "./composer/pasted-text.js";
 import { PastedTextChip } from "./composer/PastedTextChip.js";
+import {
+  appendRealtimeVoiceTranscriptToRepository,
+  realtimeVoiceTranscriptRegistry,
+} from "./composer/realtime-voice-transcript.js";
 import {
   TiptapComposer,
   type ComposerSubmitIntent,
@@ -354,6 +358,27 @@ function activeRunMatchesThread(
   return Boolean(threadId && state?.threadId === threadId && state.runId);
 }
 
+export function assistantMessageRunId(message: unknown): string | undefined {
+  const metadata = (message as { metadata?: unknown })?.metadata as
+    | { custom?: { runId?: unknown }; runId?: unknown }
+    | undefined;
+  return typeof metadata?.custom?.runId === "string"
+    ? metadata.custom.runId
+    : typeof metadata?.runId === "string"
+      ? metadata.runId
+      : undefined;
+}
+
+export function shouldAcceptRunError(args: {
+  errorRunId?: string;
+  activeRunId?: string;
+  latestAssistantRunId?: string;
+}): boolean {
+  if (!args.errorRunId) return true;
+  const expectedRunId = args.activeRunId ?? args.latestAssistantRunId;
+  return !expectedRunId || args.errorRunId === expectedRunId;
+}
+
 function isAssistantUiDuplicateMessageIdError(error: unknown): boolean {
   const message = error instanceof Error ? error.message : String(error ?? "");
   return (
@@ -2259,6 +2284,32 @@ const AssistantChatInner = forwardRef<
     [threadRuntime],
   );
 
+  const appendRealtimeVoiceTranscript = useCallback(
+    (
+      transcript: Parameters<typeof realtimeVoiceTranscriptRegistry.publish>[0],
+    ) => {
+      if (isRestoring || isRunning) return false;
+      const result = appendRealtimeVoiceTranscriptToRepository(
+        exportCleanThreadRepo(),
+        transcript,
+      );
+      if (!result.appended) return true;
+      threadRuntime.import(ensureMessageMetadata(result.repository));
+      return true;
+    },
+    [exportCleanThreadRepo, isRestoring, isRunning, threadRuntime],
+  );
+
+  useEffect(() => {
+    const transcriptThreadId = threadId ?? tabId;
+    if (!transcriptThreadId) return;
+    return realtimeVoiceTranscriptRegistry.register({
+      threadId: transcriptThreadId,
+      active: isActiveComposer,
+      append: appendRealtimeVoiceTranscript,
+    });
+  }, [appendRealtimeVoiceTranscript, isActiveComposer, tabId, threadId]);
+
   const appendThreadMessage = useCallback(
     (message: Parameters<typeof threadRuntime.append>[0]) => {
       try {
@@ -3260,6 +3311,16 @@ const AssistantChatInner = forwardRef<
     return () => window.removeEventListener("agent-chat:loop-limit", handler);
   }, [tabId]);
 
+  const latestAssistantRunId = useMemo(() => {
+    for (let index = messages.length - 1; index >= 0; index -= 1) {
+      const message = messages[index];
+      if (message?.role !== "assistant") continue;
+      const runId = assistantMessageRunId(message);
+      if (runId) return runId;
+    }
+    return undefined;
+  }, [messages]);
+
   useEffect(() => {
     const handler = (e: Event) => {
       const detail = (e as CustomEvent).detail as RunErrorInfo & {
@@ -3267,6 +3328,19 @@ const AssistantChatInner = forwardRef<
       };
       if (tabId && detail?.tabId && detail.tabId !== tabId) return;
       if (!detail?.message) return;
+      const activeRun = getActiveRun();
+      const activeRunId = activeRunMatchesThread(activeRun, threadId)
+        ? activeRun?.runId
+        : undefined;
+      if (
+        !shouldAcceptRunError({
+          errorRunId: detail.runId,
+          activeRunId,
+          latestAssistantRunId,
+        })
+      ) {
+        return;
+      }
       const stopped = userStoppedRunRef.current;
       if (
         stopped &&
@@ -3286,7 +3360,7 @@ const AssistantChatInner = forwardRef<
     };
     window.addEventListener("agent-chat:run-error", handler);
     return () => window.removeEventListener("agent-chat:run-error", handler);
-  }, [tabId]);
+  }, [latestAssistantRunId, tabId, threadId]);
 
   // Real activity means the next chunk has started. Surface longer-lived
   // activity such as "Still generating image" so active-run reconnects do not

package/corpus/core/src/client/MultiTabAssistantChat.tsx

@@ -2117,13 +2117,31 @@ export function MultiTabAssistantChat({
   useEffect(() => {
     const handleOpenThread = (event: Event) => {
       const detail = (event as CustomEvent).detail as
-        | { threadId?: unknown; newThread?: unknown; openRequestId?: unknown }
+        | {
+            threadId?: unknown;
+            newThread?: unknown;
+            onlyIfActiveThreadId?: unknown;
+            openRequestId?: unknown;
+          }
         | undefined;
       const threadId =
         typeof detail?.threadId === "string" ? detail.threadId : "";
       if (!detail || !threadId) return;
       if (!claimAgentChatOpenRequest(detail.openRequestId)) return;
 
+      const onlyIfActiveThreadId =
+        typeof detail.onlyIfActiveThreadId === "string"
+          ? detail.onlyIfActiveThreadId.trim()
+          : "";
+      const activeThreadId = activeThreadIdRef.current;
+      if (
+        onlyIfActiveThreadId &&
+        activeThreadId &&
+        activeThreadId !== onlyIfActiveThreadId
+      ) {
+        return;
+      }
+
       if (detail?.newThread === true) {
         newThreadIds.current.add(threadId);
         void createThread(threadId).then((createdId) => {

package/corpus/core/src/client/agent-chat.ts

@@ -138,6 +138,12 @@ export interface AgentChatContextState {
 export interface AgentChatOpenThreadRequest {
   threadId: string;
   newThread?: boolean;
+  /**
+   * Open only while this thread is still active (or no thread is active).
+   * This lets transient surfaces restore their own chat without stealing a
+   * thread the user selected in the meantime.
+   */
+  onlyIfActiveThreadId?: string;
   openRequestId?: string;
 }
 

package/corpus/core/src/client/composer/RealtimeVoiceMode.tsx

@@ -6,7 +6,7 @@ import {
   IconPhoneOff,
   IconVolume,
 } from "@tabler/icons-react";
-import { useId, useState } from "react";
+import { useEffect, useId, useState, useSyncExternalStore } from "react";
 
 import {
   Popover,
@@ -19,6 +19,10 @@ import {
   TooltipTrigger,
 } from "../components/ui/tooltip.js";
 import { cn } from "../utils.js";
+import {
+  createRealtimeVoiceAudioLevelStore,
+  type RealtimeVoiceAudioLevelStore,
+} from "./realtime-voice-audio-level.js";
 
 export type RealtimeVoiceModeState =
   | "connecting"
@@ -164,12 +168,55 @@ export interface RealtimeVoiceModeDockProps {
   state: RealtimeVoiceModeState;
   copy: RealtimeVoiceModeCopy;
   chatVisible: boolean;
+  audioLevels?: RealtimeVoiceAudioLevelStore;
   onToggleChat: () => void;
   onEndVoiceMode: () => void;
   errorMessage?: string | null;
   className?: string;
 }
 
+const SILENT_AUDIO_LEVELS = createRealtimeVoiceAudioLevelStore();
+const WAVEFORM_WEIGHTS = [0.55, 0.82, 1, 0.82, 0.55];
+const AUDIO_ACTIVITY_THRESHOLD = 0.035;
+
+function usePrefersReducedMotion(): boolean {
+  const [reduced, setReduced] = useState(false);
+  useEffect(() => {
+    if (typeof window === "undefined" || !window.matchMedia) return;
+    const query = window.matchMedia("(prefers-reduced-motion: reduce)");
+    const update = () => setReduced(query.matches);
+    update();
+    query.addEventListener?.("change", update);
+    return () => query.removeEventListener?.("change", update);
+  }, []);
+  return reduced;
+}
+
+function VoiceWaveform({
+  level,
+  reducedMotion,
+}: {
+  level: number;
+  reducedMotion: boolean;
+}) {
+  const visibleLevel = reducedMotion ? 0.45 : level;
+  return (
+    <span
+      aria-hidden="true"
+      className="flex h-6 items-center justify-center gap-0.5"
+      data-realtime-voice-waveform="true"
+    >
+      {WAVEFORM_WEIGHTS.map((weight, index) => (
+        <span
+          key={index}
+          className="w-0.5 rounded-full bg-current transition-[height] duration-75 ease-out motion-reduce:transition-none"
+          style={{ height: `${4 + visibleLevel * 16 * weight}px` }}
+        />
+      ))}
+    </span>
+  );
+}
+
 const ORB_STATE_CLASSES: Record<RealtimeVoiceModeState, string> = {
   connecting:
     "bg-secondary text-secondary-foreground ring-4 ring-secondary/40 hover:bg-secondary/80",
@@ -212,12 +259,26 @@ export function RealtimeVoiceModeDock({
   state,
   copy,
   chatVisible,
+  audioLevels = SILENT_AUDIO_LEVELS,
   onToggleChat,
   onEndVoiceMode,
   errorMessage,
   className,
 }: RealtimeVoiceModeDockProps) {
   const statusId = useId();
+  const levels = useSyncExternalStore(
+    audioLevels.subscribe,
+    audioLevels.getSnapshot,
+    audioLevels.getSnapshot,
+  );
+  const reducedMotion = usePrefersReducedMotion();
+  const activity =
+    levels.output > AUDIO_ACTIVITY_THRESHOLD
+      ? "assistant"
+      : levels.input > AUDIO_ACTIVITY_THRESHOLD
+        ? "user"
+        : "idle";
+  const activityLevel = activity === "assistant" ? levels.output : levels.input;
   const toggleLabel = chatVisible ? copy.hideChat : copy.showChat;
   const ending = state === "ending";
   const errorDetailVisible = state === "error" && Boolean(errorMessage);
@@ -228,7 +289,9 @@ export function RealtimeVoiceModeDock({
         "pointer-events-none fixed bottom-4 end-4 flex max-w-[calc(100vw-2rem)] flex-col items-end gap-2",
         className,
       )}
+      style={{ zIndex: 270 }}
       data-realtime-voice-state={state}
+      data-realtime-voice-activity={activity}
     >
       {errorDetailVisible ? (
         <div
@@ -283,7 +346,14 @@ export function RealtimeVoiceModeDock({
                 ORB_STATE_CLASSES[state],
               )}
             >
-              <VoiceStateIcon state={state} />
+              {activity === "idle" ? (
+                <VoiceStateIcon state={state} />
+              ) : (
+                <VoiceWaveform
+                  level={activityLevel}
+                  reducedMotion={reducedMotion}
+                />
+              )}
             </Button>
           </TooltipTrigger>
           <TooltipContent>{toggleLabel}</TooltipContent>

package/corpus/core/src/client/composer/realtime-voice-audio-level.ts

@@ -0,0 +1,64 @@
+export interface RealtimeVoiceAudioLevels {
+  input: number;
+  output: number;
+}
+
+export interface RealtimeVoiceAudioLevelStore {
+  getSnapshot: () => RealtimeVoiceAudioLevels;
+  subscribe: (listener: () => void) => () => void;
+  set: (levels: RealtimeVoiceAudioLevels) => void;
+  reset: () => void;
+}
+
+const SILENT_LEVELS: RealtimeVoiceAudioLevels = { input: 0, output: 0 };
+
+function clampLevel(value: number): number {
+  if (!Number.isFinite(value)) return 0;
+  return Math.max(0, Math.min(1, value));
+}
+
+export function normalizeRealtimeVoiceRms(samples: Uint8Array): number {
+  if (samples.length === 0) return 0;
+  let sumSquares = 0;
+  for (const sample of samples) {
+    const normalized = (sample - 128) / 128;
+    sumSquares += normalized * normalized;
+  }
+  const rms = Math.sqrt(sumSquares / samples.length);
+  return clampLevel(Math.max(0, rms - 0.012) * 3.2);
+}
+
+export function smoothRealtimeVoiceLevel(
+  previous: number,
+  next: number,
+): number {
+  const target = clampLevel(next);
+  const factor = target > previous ? 0.55 : 0.2;
+  return clampLevel(previous + (target - previous) * factor);
+}
+
+export function createRealtimeVoiceAudioLevelStore(): RealtimeVoiceAudioLevelStore {
+  let snapshot = SILENT_LEVELS;
+  const listeners = new Set<() => void>();
+
+  const set = (levels: RealtimeVoiceAudioLevels) => {
+    const next = {
+      input: clampLevel(levels.input),
+      output: clampLevel(levels.output),
+    };
+    if (next.input === snapshot.input && next.output === snapshot.output)
+      return;
+    snapshot = next;
+    for (const listener of listeners) listener();
+  };
+
+  return {
+    getSnapshot: () => snapshot,
+    subscribe(listener) {
+      listeners.add(listener);
+      return () => listeners.delete(listener);
+    },
+    set,
+    reset: () => set(SILENT_LEVELS),
+  };
+}

package/corpus/core/src/client/composer/realtime-voice-transcript.ts

@@ -0,0 +1,134 @@
+export type RealtimeVoiceTranscriptRole = "user" | "assistant";
+
+export interface RealtimeVoiceTranscriptMessage {
+  id: string;
+  threadId: string;
+  role: RealtimeVoiceTranscriptRole;
+  text: string;
+  createdAt: string;
+}
+
+export interface RealtimeVoiceTranscriptSink {
+  threadId: string;
+  active: boolean;
+  append: (message: RealtimeVoiceTranscriptMessage) => boolean;
+}
+
+export interface RealtimeVoiceTranscriptRegistry {
+  activeThreadId: () => string | undefined;
+  publish: (message: RealtimeVoiceTranscriptMessage) => void;
+  register: (sink: RealtimeVoiceTranscriptSink) => () => void;
+  pendingCount: () => number;
+}
+
+export function appendRealtimeVoiceTranscriptToRepository(
+  value: unknown,
+  transcript: RealtimeVoiceTranscriptMessage,
+): { appended: boolean; repository: Record<string, unknown> } {
+  const repository =
+    value && typeof value === "object" && !Array.isArray(value)
+      ? (value as Record<string, unknown>)
+      : {};
+  const messages = Array.isArray(repository.messages)
+    ? [...repository.messages]
+    : [];
+  const duplicate = messages.some((entry) => {
+    if (!entry || typeof entry !== "object") return false;
+    const record = entry as Record<string, unknown>;
+    const message =
+      record.message && typeof record.message === "object"
+        ? (record.message as Record<string, unknown>)
+        : record;
+    return message.id === transcript.id;
+  });
+  if (duplicate) return { appended: false, repository };
+
+  const parentId =
+    typeof repository.headId === "string" ? repository.headId : null;
+  const createdAt = new Date(transcript.createdAt);
+  const message = {
+    id: transcript.id,
+    role: transcript.role,
+    content: [{ type: "text", text: transcript.text }],
+    createdAt: Number.isNaN(createdAt.getTime()) ? new Date() : createdAt,
+    metadata: {
+      custom: {
+        source: "realtime-voice",
+      },
+    },
+    ...(transcript.role === "assistant"
+      ? { status: { type: "complete", reason: "stop" } }
+      : {}),
+  };
+
+  return {
+    appended: true,
+    repository: {
+      ...repository,
+      messages: [...messages, { message, parentId }],
+      headId: transcript.id,
+    },
+  };
+}
+
+export function createRealtimeVoiceTranscriptRegistry(): RealtimeVoiceTranscriptRegistry {
+  const sinks = new Map<
+    symbol,
+    RealtimeVoiceTranscriptSink & { order: number }
+  >();
+  const pending = new Map<string, RealtimeVoiceTranscriptMessage>();
+  const delivered = new Set<string>();
+  let order = 0;
+
+  const flush = () => {
+    for (const [id, message] of pending) {
+      let consumed = false;
+      for (const sink of sinks.values()) {
+        if (sink.threadId !== message.threadId) continue;
+        try {
+          consumed = sink.append(message) || consumed;
+        } catch {
+          // Keep the completed transcript queued until the chat can import it.
+        }
+      }
+      if (!consumed) continue;
+      pending.delete(id);
+      delivered.add(id);
+      if (delivered.size > 1_000) {
+        const oldest = delivered.values().next().value;
+        if (typeof oldest === "string") delivered.delete(oldest);
+      }
+    }
+  };
+
+  return {
+    activeThreadId() {
+      let active: (RealtimeVoiceTranscriptSink & { order: number }) | undefined;
+      for (const sink of sinks.values()) {
+        if (!sink.active || (active && sink.order <= active.order)) continue;
+        active = sink;
+      }
+      return active?.threadId;
+    },
+    publish(message) {
+      const id = message.id.trim();
+      const threadId = message.threadId.trim();
+      const text = message.text.trim();
+      if (!id || !threadId || !text || delivered.has(id) || pending.has(id)) {
+        return;
+      }
+      pending.set(id, { ...message, id, threadId, text });
+      flush();
+    },
+    register(sink) {
+      const token = Symbol("realtime-voice-transcript-sink");
+      sinks.set(token, { ...sink, order: ++order });
+      flush();
+      return () => sinks.delete(token);
+    },
+    pendingCount: () => pending.size,
+  };
+}
+
+export const realtimeVoiceTranscriptRegistry =
+  createRealtimeVoiceTranscriptRegistry();