@vivix-ai/ivi-frontend-sdk 0.3.4 → 0.3.6

package/README.md

@@ -380,20 +380,22 @@ function useIviSubtitles(
 ): IviSubtitleItem[];
 ```
 
-基于 runtime 的 conversation / response 事件维护字幕队列，不负责渲染，也不按时间自动消失。
+基于 runtime 的 conversation / response 事件维护字幕队列，不负责渲染，也不按时间自动消失。默认从 conversation 条目聚合字幕；当 `roles` 包含 `"model"` 且开启 `useModelStreamingTranscript` 时，model 字幕改为使用 `response.output_audio_transcript.*` 事件，`response.output_audio_transcript.done` 只更新文本，只有 `response.done` 才表示这一轮 model 字幕结束。
 
 | Option | 类型 | 默认值 | 说明 |
 |--------|------|--------|------|
 | `roles` | `IviSubtitleRole \| IviSubtitleRole[]` | `"user"` | 要收集的发言人角色 |
 | `maxItems` | `number` | `2` | 最多保留的字幕条数，超过后清理最旧条目 |
+| `useModelStreamingTranscript` | `boolean` | `false` | 当 `roles` 包含 `"model"` 时，model 字幕是否改用 `response.output_audio_transcript.*` 事件 |
 
-`IviSubtitleItem` 包含 `id`、`role`、`lifecycle`、`status`、`text`、`transcript`、`displayText`、`timestamp`、`updatedAt`、`content`、`item`。
+`IviSubtitleItem` 包含 `id`、`role`、`lifecycle`、`status`、`text`、`transcript`、`displayText`、`timestamp`、`updatedAt`、`content`、`item`、`source`、`responseId`、`itemId`。
 
 ```tsx
 function CustomSubtitles({ runtime }: { runtime: IviRuntimeCoordinator | null }) {
   const subtitles = useIviSubtitles(runtime, {
     roles: ["user", "model"],
-    maxItems: 5
+    maxItems: 5,
+    useModelStreamingTranscript: true
   });
 
   return (
@@ -464,7 +466,7 @@ function useApplyVolumeToSlot(
 | `showVolumeControl` | `boolean` | — | 是否显示音量控制浮层 |
 | `volumeControlProps` | — | — | 音量控制自定义配置 |
 | `showSubtitle` | `boolean` | — | 是否显示字幕浮层 |
-| `subtitleProps` | `Omit<IVISubtitleOverlayProps, "runtime">` | — | 字幕自定义配置 |
+| `subtitleProps` | `Omit<IVISubtitleOverlayProps, "runtime">` | — | 字幕自定义配置，可传 `roles`、`maxItems`、`useModelStreamingTranscript` 等 |
 | `trtcPlayerProps` | — | — | TRTC 播放器自定义配置 |
 | `livekitPlayerProps` | — | — | LiveKit 播放器自定义配置 |
 | `videoProps` / `imageProps` | — | — | 透传给原生 `<video>` / `<img>` |
@@ -574,6 +576,7 @@ npm install xgplayer xgplayer-flv
 | `roles` | `IviSubtitleRole \| IviSubtitleRole[]` | `"user"` | 要展示的发言人角色 |
 | `maxItems` | `number` | `2` | 最多保留的字幕条数，超过后清理最旧条目 |
 | `maxVisible` | `number` | — | 已废弃，兼容旧字段；请使用 `maxItems` |
+| `useModelStreamingTranscript` | `boolean` | `false` | 当 `roles` 包含 `"model"` 时，model 字幕是否改用 `response.output_audio_transcript.*` 事件 |
 | `subtitleStyle` | `IVISubtitleOverlayStyle` | — | 样式配置 |
 | `className` / `style` | — | — | 样式 |
 

package/dist/index.cjs

@@ -3088,6 +3088,7 @@ function useIviSubtitles(runtime, options = {}) {
     () => new Set(roleKey.split("\0")),
     [roleKey]
   );
+  const useModelStreamingTranscript = options.useModelStreamingTranscript === true && roleSet.has("model");
   const [subtitles, setSubtitles] = react.useState([]);
   const seenIdsRef = react.useRef(/* @__PURE__ */ new Set());
   const initializedRef = react.useRef(false);
@@ -3104,7 +3105,7 @@ function useIviSubtitles(runtime, options = {}) {
       if (!initializedRef.current) {
         initializedRef.current = true;
         for (const item of conversations) {
-          if (item.lifecycle === "done" || !getDisplayText(item) || !roleSet.has(item.role)) {
+          if (item.lifecycle === "done" || !getDisplayText(item) || !shouldUseConversationRole(item.role, roleSet, useModelStreamingTranscript)) {
             seenIds.add(item.id);
           }
         }
@@ -3113,8 +3114,12 @@ function useIviSubtitles(runtime, options = {}) {
         const conversationMap = new Map(conversations.map((item) => [item.id, item]));
         const nextById = /* @__PURE__ */ new Map();
         for (const previousItem of previous) {
+          if (previousItem.source === "response_audio_transcript") {
+            nextById.set(previousItem.id, previousItem);
+            continue;
+          }
           const conversation = conversationMap.get(previousItem.id);
-          if (!conversation || !roleSet.has(conversation.role) || !getDisplayText(conversation)) {
+          if (!conversation || !shouldUseConversationRole(conversation.role, roleSet, useModelStreamingTranscript) || !getDisplayText(conversation)) {
             continue;
           }
           nextById.set(
@@ -3127,7 +3132,7 @@ function useIviSubtitles(runtime, options = {}) {
           );
         }
         for (const conversation of conversations) {
-          if (!roleSet.has(conversation.role) || !getDisplayText(conversation)) {
+          if (!shouldUseConversationRole(conversation.role, roleSet, useModelStreamingTranscript) || !getDisplayText(conversation)) {
             continue;
           }
           if (seenIds.has(conversation.id)) {
@@ -3137,10 +3142,7 @@ function useIviSubtitles(runtime, options = {}) {
           nextById.set(conversation.id, buildSubtitleItem(conversation, now, now));
         }
         const next = Array.from(nextById.values());
-        if (maxItems === 0) {
-          return [];
-        }
-        return next.length > maxItems ? next.slice(next.length - maxItems) : next;
+        return trimSubtitleItems(next, maxItems);
       });
     };
     syncConversations(runtime.getState().conversations);
@@ -3151,12 +3153,20 @@ function useIviSubtitles(runtime, options = {}) {
         setSubtitles([]);
         return;
       }
+      if (useModelStreamingTranscript && isModelStreamingTranscriptEvent(event)) {
+        setSubtitles((previous) => trimSubtitleItems(upsertModelStreamingSubtitle(previous, event), maxItems));
+        return;
+      }
+      if (useModelStreamingTranscript && isResponseDoneEvent(event)) {
+        setSubtitles((previous) => trimSubtitleItems(markModelStreamingSubtitleDone(previous, event), maxItems));
+        return;
+      }
       if (!isSubtitleRelatedEvent(event.type)) {
         return;
       }
       syncConversations(state.conversations);
     });
-  }, [runtime, roleSet, maxItems]);
+  }, [runtime, roleSet, maxItems, useModelStreamingTranscript]);
   return subtitles;
 }
 function normalizeMaxItems(maxItems) {
@@ -3168,6 +3178,19 @@ function normalizeMaxItems(maxItems) {
   }
   return Math.max(0, Math.floor(maxItems));
 }
+function shouldUseConversationRole(role, roleSet, useModelStreamingTranscript) {
+  if (!roleSet.has(role)) {
+    return false;
+  }
+  return !(useModelStreamingTranscript && role === "model");
+}
+function trimSubtitleItems(items, maxItems) {
+  if (maxItems === 0) {
+    return [];
+  }
+  const sorted = [...items].sort((a, b) => a.timestamp - b.timestamp);
+  return sorted.length > maxItems ? sorted.slice(sorted.length - maxItems) : sorted;
+}
 function getDisplayText(item) {
   return item.text || item.transcript;
 }
@@ -3182,16 +3205,109 @@ function buildSubtitleItem(item, timestamp, updatedAt) {
     displayText: getDisplayText(item),
     content: item.content,
     item: item.item,
+    source: "conversation",
     timestamp,
     updatedAt
   };
 }
+function upsertModelStreamingSubtitle(previous, event, now = Date.now()) {
+  const responseId = getEventString(event, "responseId", "response_id");
+  if (!responseId) {
+    return previous;
+  }
+  const displayText = event.type === "response.output_audio_transcript.done" ? event.transcript : event.delta;
+  if (typeof displayText !== "string" || displayText.length === 0) {
+    return previous;
+  }
+  const id = buildModelStreamingSubtitleId(responseId);
+  const itemId = getEventString(event, "itemId", "item_id");
+  const existing = previous.find((item) => item.id === id);
+  const nextItem = {
+    id,
+    role: "model",
+    lifecycle: "added",
+    status: "in_progress",
+    text: "",
+    transcript: displayText,
+    displayText,
+    content: [],
+    item: {
+      id: itemId ?? id,
+      type: "message",
+      role: "model",
+      status: "in_progress",
+      content: [{ type: "audio", transcript: displayText }]
+    },
+    source: "response_audio_transcript",
+    responseId,
+    itemId,
+    timestamp: existing?.timestamp ?? now,
+    updatedAt: hasStreamingSubtitleChanged(existing, displayText) ? now : existing?.updatedAt ?? now
+  };
+  return replaceSubtitleItem(previous, nextItem);
+}
+function markModelStreamingSubtitleDone(previous, event, now = Date.now()) {
+  const responseId = event.response?.id;
+  if (!responseId) {
+    return previous;
+  }
+  const id = buildModelStreamingSubtitleId(responseId);
+  const status = mapResponseStatusToConversationStatus(event.response?.status);
+  return previous.map((item) => {
+    if (item.id !== id || item.source !== "response_audio_transcript") {
+      return item;
+    }
+    return {
+      ...item,
+      lifecycle: "done",
+      status,
+      item: {
+        ...item.item,
+        status
+      },
+      updatedAt: now
+    };
+  });
+}
+function replaceSubtitleItem(previous, nextItem) {
+  const replaced = previous.map((item) => item.id === nextItem.id ? nextItem : item);
+  if (replaced.some((item) => item.id === nextItem.id)) {
+    return replaced;
+  }
+  return [...previous, nextItem];
+}
+function hasStreamingSubtitleChanged(previous, nextDisplayText) {
+  return !previous || previous.displayText !== nextDisplayText || previous.lifecycle !== "added" || previous.status !== "in_progress";
+}
+function buildModelStreamingSubtitleId(responseId) {
+  return `model-response:${responseId}`;
+}
+function getEventString(event, camelKey, snakeKey) {
+  const record = event;
+  const value = record[camelKey] ?? record[snakeKey];
+  return typeof value === "string" && value.length > 0 ? value : void 0;
+}
+function mapResponseStatusToConversationStatus(status) {
+  if (status === "completed") {
+    return "completed";
+  }
+  if (status === "in_progress") {
+    return "in_progress";
+  }
+  return "incomplete";
+}
 function hasSubtitleChanged(previous, next) {
   return previous.text !== next.text || previous.transcript !== next.transcript || previous.lifecycle !== next.lifecycle || previous.status !== next.status || previous.role !== next.role;
 }
 function isSubtitleRelatedEvent(type) {
   return type.startsWith("conversation.") || type.startsWith("response.");
 }
+function isModelStreamingTranscriptEvent(event) {
+  return event.type === "response.output_audio_transcript.delta" || event.type === "response.output_audio_transcript.done";
+}
+function isResponseDoneEvent(event) {
+  return event.type === "response.done";
+}
 var BREATHE_KEYFRAMES = `@keyframes ivi-subtitle-breathe{0%,100%{opacity:1}50%{opacity:.55}}`;
 function IVISubtitleOverlay(props) {
   const {
@@ -3199,13 +3315,15 @@ function IVISubtitleOverlay(props) {
     roles = "user",
     maxItems,
     maxVisible,
+    useModelStreamingTranscript,
     subtitleStyle,
     className,
     style
   } = props;
   const entries = useIviSubtitles(runtime, {
     roles,
-    maxItems: maxItems ?? maxVisible
+    maxItems: maxItems ?? maxVisible,
+    useModelStreamingTranscript
   });
   if (entries.length === 0) return null;
   const fontFamily = subtitleStyle?.fontFamily ?? "system-ui, -apple-system, sans-serif";
@@ -3828,6 +3946,19 @@ function detectMediaVolumeType(source) {
   if (!url) return null;
   return isM3u8Url(url) ? "hls" : "video";
 }
+function getSourceRenderKey(sourceId, source) {
+  const trtc = getTrtcPlayback(source.playback);
+  if (!trtc) {
+    return sourceId;
+  }
+  return [
+    "trtc",
+    trtc.app_id ?? "",
+    trtc.room_id ?? "",
+    trtc.user_id ?? "",
+    trtc.user_sig ?? ""
+  ].join(":");
+}
 function TrackSlotMediaContent(props) {
   const {
     slot,
@@ -3865,7 +3996,7 @@ function TrackSlotMediaContent(props) {
       IVITrtcPlayer,
       {
         trtc,
-        sourceId: source.source.source_id,
+        sourceId: slotSourceId,
         runtime,
         ...trtcPlayerProps,
         muted: trtcMuted,
@@ -4020,26 +4151,35 @@ function SlotVideo(props) {
 // src/react/internal/use-multi-preload-sources.ts
 function useMultiPreloadSources(sources, activeSourceId) {
   return react.useMemo(() => {
-    const entries = [];
-    for (const [id, runtimeSource] of sources) {
-      const isActive = id === activeSourceId;
-      const shouldMount = Boolean(runtimeSource.preload) || isActive;
-      if (!shouldMount) {
-        continue;
-      }
-      const ready = toReadyRuntimeSource(runtimeSource);
-      if (!ready) {
-        continue;
-      }
-      entries.push({
-        sourceId: id,
-        source: ready,
-        isActive
-      });
-    }
-    return entries;
+    return buildPreloadSourceEntries(sources, activeSourceId);
   }, [sources, activeSourceId]);
 }
+function buildPreloadSourceEntries(sources, activeSourceId) {
+  const entriesByRenderKey = /* @__PURE__ */ new Map();
+  for (const [id, runtimeSource] of sources) {
+    const isActive = id === activeSourceId;
+    const shouldMount = Boolean(runtimeSource.preload) || isActive;
+    if (!shouldMount) {
+      continue;
+    }
+    const ready = toReadyRuntimeSource(runtimeSource);
+    if (!ready) {
+      continue;
+    }
+    const renderKey = getSourceRenderKey(id, ready);
+    const entry = {
+      sourceId: id,
+      renderKey,
+      source: ready,
+      isActive
+    };
+    const previous = entriesByRenderKey.get(renderKey);
+    if (!previous || !previous.isActive && isActive) {
+      entriesByRenderKey.set(renderKey, entry);
+    }
+  }
+  return Array.from(entriesByRenderKey.values());
+}
 function IVITrackSlot(props) {
   const {
     slot,
@@ -4129,7 +4269,7 @@ function IVITrackSlot(props) {
                 }
               ) })
             },
-            entry.sourceId
+            entry.renderKey
           );
         }),
         showSubtitle && activeSource && supportsSubtitleOverlay(activeSource) && /* @__PURE__ */ jsxRuntime.jsx("div", { style: SUBTITLE_OVERLAY_STYLE, children: /* @__PURE__ */ jsxRuntime.jsx(