@vivix-ai/ivi-frontend-sdk 0.3.4 → 0.3.6

package/dist/index.d.cts

@@ -627,6 +627,7 @@ interface IVIVolumeControlProps {
 }
 
 type IviSubtitleRole = IviRuntimeConversationItem["role"];
+type IviSubtitleSource = "conversation" | "response_audio_transcript";
 interface IviSubtitleItem {
     id: string;
     role: IviSubtitleRole;
@@ -637,6 +638,12 @@ interface IviSubtitleItem {
     displayText: string;
     content: IviRuntimeConversationItem["content"];
     item: IviRuntimeConversationItem["item"];
+    /** 字幕数据来源。 */
+    source?: IviSubtitleSource;
+    /** 关联的 response_id；仅 response_audio_transcript 来源有值。 */
+    responseId?: string;
+    /** 关联的 item_id；仅 response_audio_transcript 来源有值。 */
+    itemId?: string;
     /** 首次进入字幕队列的时间戳。 */
     timestamp: number;
     /** 最近一次字幕内容或状态更新的时间戳。 */
@@ -647,6 +654,12 @@ interface IviUseSubtitlesOptions {
     roles?: IviSubtitleRole | IviSubtitleRole[];
     /** 最多保留的字幕条数，超过后自动清理最旧条目，默认 2。 */
     maxItems?: number;
+    /**
+     * 当 roles 包含 "model" 时，是否使用 response.output_audio_transcript.* 事件作为 model 字幕来源。
+     * 开启后 model 字幕不再从 conversation.* 聚合；response.output_audio_transcript.done 不视为结束，
+     * 只有 response.done 才会把这一轮 model 字幕标记为 done。
+     */
+    useModelStreamingTranscript?: boolean;
 }
 /**
  * 监听 runtime 中的 conversation/response 事件，并维护当前应展示的字幕队列。
@@ -672,6 +685,10 @@ interface IVISubtitleOverlayProps {
     maxItems?: number;
     /** @deprecated Use `maxItems` instead. */
     maxVisible?: number;
+    /**
+     * 当 roles 包含 "model" 时，是否使用 response.output_audio_transcript.* 事件作为 model 字幕来源。
+     */
+    useModelStreamingTranscript?: boolean;
     /** 样式配置 */
     subtitleStyle?: IVISubtitleOverlayStyle;
     /** 自定义类名 */
@@ -756,4 +773,4 @@ declare function useManagedIviRuntime(config: IviManagedRuntimeConfig): IviRunti
 
 declare function useIviStageView(): IviStageViewContextValue;
 
-export { EMPTY_RUNTIME_STATE, IVILivekitPlayer, type IVILivekitPlayerProps, IVIStageView, type IVIStageViewProps, IVISubtitleOverlay, type IVISubtitleOverlayProps, type IVISubtitleOverlayStyle, IVITrackSlot, type IVITrackSlotProps, IVITrtcPlayer, type IVITrtcPlayerProps, type IviFrontendClientConfig, IviFrontendSdk, type IviManagedRuntimeConfig, type IviManagedRuntimeLogCallback, type IviManagedRuntimeLogEntry, type IviManagedRuntimeLogLevel, type IviManagedRuntimeLogSource, type IviRuntimeConversationItem, type IviRuntimeConversationLifecycle, type IviRuntimeConversationStatus, IviRuntimeCoordinator, type IviRuntimeCoordinatorConfig, IviRuntimeDispatcher, type IviRuntimeDispatcherConfig, type IviRuntimeEventListener, type IviRuntimeLogCallback, type IviRuntimeLogEntry, type IviRuntimeLogLevel, type IviRuntimeSource, type IviRuntimeSourcePreloadState, type IviRuntimeState, type IviRuntimeStatus, type IviRuntimeStream, type IviRuntimeTrtcAIDenoiserMode, type IviRuntimeTrtcAIDenoiserOptions, type IviRuntimeTrtcEvent, type IviRuntimeTrtcEventListener, type IviRuntimeTrtcEventType, type IviRuntimeUserTextToResponseCallbacks, type IviRuntimeUserTextToResponseOptions, type IviRuntimeUserTextToResponseResult, type IviSourcePlaybackLivekit, type IviSourcePlaybackLivekitDescriptor, type IviStageSlotBinding, type IviStageViewContextValue, type IviSubtitleItem, type IviSubtitleRole, type IviUseSubtitlesOptions, LivekitSourceManager, TrtcSourceManager, isLivekitSourcePlayback, isReadyLivekitRuntimeSource, isSameLivekitConfig, useIviStageView, useIviSubtitles, useManagedIviRuntime, useRuntimeState };
+export { EMPTY_RUNTIME_STATE, IVILivekitPlayer, type IVILivekitPlayerProps, IVIStageView, type IVIStageViewProps, IVISubtitleOverlay, type IVISubtitleOverlayProps, type IVISubtitleOverlayStyle, IVITrackSlot, type IVITrackSlotProps, IVITrtcPlayer, type IVITrtcPlayerProps, type IviFrontendClientConfig, IviFrontendSdk, type IviManagedRuntimeConfig, type IviManagedRuntimeLogCallback, type IviManagedRuntimeLogEntry, type IviManagedRuntimeLogLevel, type IviManagedRuntimeLogSource, type IviRuntimeConversationItem, type IviRuntimeConversationLifecycle, type IviRuntimeConversationStatus, IviRuntimeCoordinator, type IviRuntimeCoordinatorConfig, IviRuntimeDispatcher, type IviRuntimeDispatcherConfig, type IviRuntimeEventListener, type IviRuntimeLogCallback, type IviRuntimeLogEntry, type IviRuntimeLogLevel, type IviRuntimeSource, type IviRuntimeSourcePreloadState, type IviRuntimeState, type IviRuntimeStatus, type IviRuntimeStream, type IviRuntimeTrtcAIDenoiserMode, type IviRuntimeTrtcAIDenoiserOptions, type IviRuntimeTrtcEvent, type IviRuntimeTrtcEventListener, type IviRuntimeTrtcEventType, type IviRuntimeUserTextToResponseCallbacks, type IviRuntimeUserTextToResponseOptions, type IviRuntimeUserTextToResponseResult, type IviSourcePlaybackLivekit, type IviSourcePlaybackLivekitDescriptor, type IviStageSlotBinding, type IviStageViewContextValue, type IviSubtitleItem, type IviSubtitleRole, type IviSubtitleSource, type IviUseSubtitlesOptions, LivekitSourceManager, TrtcSourceManager, isLivekitSourcePlayback, isReadyLivekitRuntimeSource, isSameLivekitConfig, useIviStageView, useIviSubtitles, useManagedIviRuntime, useRuntimeState };

package/dist/index.d.ts

@@ -627,6 +627,7 @@ interface IVIVolumeControlProps {
 }
 
 type IviSubtitleRole = IviRuntimeConversationItem["role"];
+type IviSubtitleSource = "conversation" | "response_audio_transcript";
 interface IviSubtitleItem {
     id: string;
     role: IviSubtitleRole;
@@ -637,6 +638,12 @@ interface IviSubtitleItem {
     displayText: string;
     content: IviRuntimeConversationItem["content"];
     item: IviRuntimeConversationItem["item"];
+    /** 字幕数据来源。 */
+    source?: IviSubtitleSource;
+    /** 关联的 response_id；仅 response_audio_transcript 来源有值。 */
+    responseId?: string;
+    /** 关联的 item_id；仅 response_audio_transcript 来源有值。 */
+    itemId?: string;
     /** 首次进入字幕队列的时间戳。 */
     timestamp: number;
     /** 最近一次字幕内容或状态更新的时间戳。 */
@@ -647,6 +654,12 @@ interface IviUseSubtitlesOptions {
     roles?: IviSubtitleRole | IviSubtitleRole[];
     /** 最多保留的字幕条数，超过后自动清理最旧条目，默认 2。 */
     maxItems?: number;
+    /**
+     * 当 roles 包含 "model" 时，是否使用 response.output_audio_transcript.* 事件作为 model 字幕来源。
+     * 开启后 model 字幕不再从 conversation.* 聚合；response.output_audio_transcript.done 不视为结束，
+     * 只有 response.done 才会把这一轮 model 字幕标记为 done。
+     */
+    useModelStreamingTranscript?: boolean;
 }
 /**
  * 监听 runtime 中的 conversation/response 事件，并维护当前应展示的字幕队列。
@@ -672,6 +685,10 @@ interface IVISubtitleOverlayProps {
     maxItems?: number;
     /** @deprecated Use `maxItems` instead. */
     maxVisible?: number;
+    /**
+     * 当 roles 包含 "model" 时，是否使用 response.output_audio_transcript.* 事件作为 model 字幕来源。
+     */
+    useModelStreamingTranscript?: boolean;
     /** 样式配置 */
     subtitleStyle?: IVISubtitleOverlayStyle;
     /** 自定义类名 */
@@ -756,4 +773,4 @@ declare function useManagedIviRuntime(config: IviManagedRuntimeConfig): IviRunti
 
 declare function useIviStageView(): IviStageViewContextValue;
 
-export { EMPTY_RUNTIME_STATE, IVILivekitPlayer, type IVILivekitPlayerProps, IVIStageView, type IVIStageViewProps, IVISubtitleOverlay, type IVISubtitleOverlayProps, type IVISubtitleOverlayStyle, IVITrackSlot, type IVITrackSlotProps, IVITrtcPlayer, type IVITrtcPlayerProps, type IviFrontendClientConfig, IviFrontendSdk, type IviManagedRuntimeConfig, type IviManagedRuntimeLogCallback, type IviManagedRuntimeLogEntry, type IviManagedRuntimeLogLevel, type IviManagedRuntimeLogSource, type IviRuntimeConversationItem, type IviRuntimeConversationLifecycle, type IviRuntimeConversationStatus, IviRuntimeCoordinator, type IviRuntimeCoordinatorConfig, IviRuntimeDispatcher, type IviRuntimeDispatcherConfig, type IviRuntimeEventListener, type IviRuntimeLogCallback, type IviRuntimeLogEntry, type IviRuntimeLogLevel, type IviRuntimeSource, type IviRuntimeSourcePreloadState, type IviRuntimeState, type IviRuntimeStatus, type IviRuntimeStream, type IviRuntimeTrtcAIDenoiserMode, type IviRuntimeTrtcAIDenoiserOptions, type IviRuntimeTrtcEvent, type IviRuntimeTrtcEventListener, type IviRuntimeTrtcEventType, type IviRuntimeUserTextToResponseCallbacks, type IviRuntimeUserTextToResponseOptions, type IviRuntimeUserTextToResponseResult, type IviSourcePlaybackLivekit, type IviSourcePlaybackLivekitDescriptor, type IviStageSlotBinding, type IviStageViewContextValue, type IviSubtitleItem, type IviSubtitleRole, type IviUseSubtitlesOptions, LivekitSourceManager, TrtcSourceManager, isLivekitSourcePlayback, isReadyLivekitRuntimeSource, isSameLivekitConfig, useIviStageView, useIviSubtitles, useManagedIviRuntime, useRuntimeState };
+export { EMPTY_RUNTIME_STATE, IVILivekitPlayer, type IVILivekitPlayerProps, IVIStageView, type IVIStageViewProps, IVISubtitleOverlay, type IVISubtitleOverlayProps, type IVISubtitleOverlayStyle, IVITrackSlot, type IVITrackSlotProps, IVITrtcPlayer, type IVITrtcPlayerProps, type IviFrontendClientConfig, IviFrontendSdk, type IviManagedRuntimeConfig, type IviManagedRuntimeLogCallback, type IviManagedRuntimeLogEntry, type IviManagedRuntimeLogLevel, type IviManagedRuntimeLogSource, type IviRuntimeConversationItem, type IviRuntimeConversationLifecycle, type IviRuntimeConversationStatus, IviRuntimeCoordinator, type IviRuntimeCoordinatorConfig, IviRuntimeDispatcher, type IviRuntimeDispatcherConfig, type IviRuntimeEventListener, type IviRuntimeLogCallback, type IviRuntimeLogEntry, type IviRuntimeLogLevel, type IviRuntimeSource, type IviRuntimeSourcePreloadState, type IviRuntimeState, type IviRuntimeStatus, type IviRuntimeStream, type IviRuntimeTrtcAIDenoiserMode, type IviRuntimeTrtcAIDenoiserOptions, type IviRuntimeTrtcEvent, type IviRuntimeTrtcEventListener, type IviRuntimeTrtcEventType, type IviRuntimeUserTextToResponseCallbacks, type IviRuntimeUserTextToResponseOptions, type IviRuntimeUserTextToResponseResult, type IviSourcePlaybackLivekit, type IviSourcePlaybackLivekitDescriptor, type IviStageSlotBinding, type IviStageViewContextValue, type IviSubtitleItem, type IviSubtitleRole, type IviSubtitleSource, type IviUseSubtitlesOptions, LivekitSourceManager, TrtcSourceManager, isLivekitSourcePlayback, isReadyLivekitRuntimeSource, isSameLivekitConfig, useIviStageView, useIviSubtitles, useManagedIviRuntime, useRuntimeState };

package/dist/index.js

@@ -3086,6 +3086,7 @@ function useIviSubtitles(runtime, options = {}) {
     () => new Set(roleKey.split("\0")),
     [roleKey]
   );
+  const useModelStreamingTranscript = options.useModelStreamingTranscript === true && roleSet.has("model");
   const [subtitles, setSubtitles] = useState([]);
   const seenIdsRef = useRef(/* @__PURE__ */ new Set());
   const initializedRef = useRef(false);
@@ -3102,7 +3103,7 @@ function useIviSubtitles(runtime, options = {}) {
       if (!initializedRef.current) {
         initializedRef.current = true;
         for (const item of conversations) {
-          if (item.lifecycle === "done" || !getDisplayText(item) || !roleSet.has(item.role)) {
+          if (item.lifecycle === "done" || !getDisplayText(item) || !shouldUseConversationRole(item.role, roleSet, useModelStreamingTranscript)) {
             seenIds.add(item.id);
           }
         }
@@ -3111,8 +3112,12 @@ function useIviSubtitles(runtime, options = {}) {
         const conversationMap = new Map(conversations.map((item) => [item.id, item]));
         const nextById = /* @__PURE__ */ new Map();
         for (const previousItem of previous) {
+          if (previousItem.source === "response_audio_transcript") {
+            nextById.set(previousItem.id, previousItem);
+            continue;
+          }
           const conversation = conversationMap.get(previousItem.id);
-          if (!conversation || !roleSet.has(conversation.role) || !getDisplayText(conversation)) {
+          if (!conversation || !shouldUseConversationRole(conversation.role, roleSet, useModelStreamingTranscript) || !getDisplayText(conversation)) {
             continue;
           }
           nextById.set(
@@ -3125,7 +3130,7 @@ function useIviSubtitles(runtime, options = {}) {
           );
         }
         for (const conversation of conversations) {
-          if (!roleSet.has(conversation.role) || !getDisplayText(conversation)) {
+          if (!shouldUseConversationRole(conversation.role, roleSet, useModelStreamingTranscript) || !getDisplayText(conversation)) {
             continue;
           }
           if (seenIds.has(conversation.id)) {
@@ -3135,10 +3140,7 @@ function useIviSubtitles(runtime, options = {}) {
           nextById.set(conversation.id, buildSubtitleItem(conversation, now, now));
         }
         const next = Array.from(nextById.values());
-        if (maxItems === 0) {
-          return [];
-        }
-        return next.length > maxItems ? next.slice(next.length - maxItems) : next;
+        return trimSubtitleItems(next, maxItems);
       });
     };
     syncConversations(runtime.getState().conversations);
@@ -3149,12 +3151,20 @@ function useIviSubtitles(runtime, options = {}) {
         setSubtitles([]);
         return;
       }
+      if (useModelStreamingTranscript && isModelStreamingTranscriptEvent(event)) {
+        setSubtitles((previous) => trimSubtitleItems(upsertModelStreamingSubtitle(previous, event), maxItems));
+        return;
+      }
+      if (useModelStreamingTranscript && isResponseDoneEvent(event)) {
+        setSubtitles((previous) => trimSubtitleItems(markModelStreamingSubtitleDone(previous, event), maxItems));
+        return;
+      }
       if (!isSubtitleRelatedEvent(event.type)) {
         return;
       }
       syncConversations(state.conversations);
     });
-  }, [runtime, roleSet, maxItems]);
+  }, [runtime, roleSet, maxItems, useModelStreamingTranscript]);
   return subtitles;
 }
 function normalizeMaxItems(maxItems) {
@@ -3166,6 +3176,19 @@ function normalizeMaxItems(maxItems) {
   }
   return Math.max(0, Math.floor(maxItems));
 }
+function shouldUseConversationRole(role, roleSet, useModelStreamingTranscript) {
+  if (!roleSet.has(role)) {
+    return false;
+  }
+  return !(useModelStreamingTranscript && role === "model");
+}
+function trimSubtitleItems(items, maxItems) {
+  if (maxItems === 0) {
+    return [];
+  }
+  const sorted = [...items].sort((a, b) => a.timestamp - b.timestamp);
+  return sorted.length > maxItems ? sorted.slice(sorted.length - maxItems) : sorted;
+}
 function getDisplayText(item) {
   return item.text || item.transcript;
 }
@@ -3180,16 +3203,109 @@ function buildSubtitleItem(item, timestamp, updatedAt) {
     displayText: getDisplayText(item),
     content: item.content,
     item: item.item,
+    source: "conversation",
     timestamp,
     updatedAt
   };
 }
+function upsertModelStreamingSubtitle(previous, event, now = Date.now()) {
+  const responseId = getEventString(event, "responseId", "response_id");
+  if (!responseId) {
+    return previous;
+  }
+  const displayText = event.type === "response.output_audio_transcript.done" ? event.transcript : event.delta;
+  if (typeof displayText !== "string" || displayText.length === 0) {
+    return previous;
+  }
+  const id = buildModelStreamingSubtitleId(responseId);
+  const itemId = getEventString(event, "itemId", "item_id");
+  const existing = previous.find((item) => item.id === id);
+  const nextItem = {
+    id,
+    role: "model",
+    lifecycle: "added",
+    status: "in_progress",
+    text: "",
+    transcript: displayText,
+    displayText,
+    content: [],
+    item: {
+      id: itemId ?? id,
+      type: "message",
+      role: "model",
+      status: "in_progress",
+      content: [{ type: "audio", transcript: displayText }]
+    },
+    source: "response_audio_transcript",
+    responseId,
+    itemId,
+    timestamp: existing?.timestamp ?? now,
+    updatedAt: hasStreamingSubtitleChanged(existing, displayText) ? now : existing?.updatedAt ?? now
+  };
+  return replaceSubtitleItem(previous, nextItem);
+}
+function markModelStreamingSubtitleDone(previous, event, now = Date.now()) {
+  const responseId = event.response?.id;
+  if (!responseId) {
+    return previous;
+  }
+  const id = buildModelStreamingSubtitleId(responseId);
+  const status = mapResponseStatusToConversationStatus(event.response?.status);
+  return previous.map((item) => {
+    if (item.id !== id || item.source !== "response_audio_transcript") {
+      return item;
+    }
+    return {
+      ...item,
+      lifecycle: "done",
+      status,
+      item: {
+        ...item.item,
+        status
+      },
+      updatedAt: now
+    };
+  });
+}
+function replaceSubtitleItem(previous, nextItem) {
+  const replaced = previous.map((item) => item.id === nextItem.id ? nextItem : item);
+  if (replaced.some((item) => item.id === nextItem.id)) {
+    return replaced;
+  }
+  return [...previous, nextItem];
+}
+function hasStreamingSubtitleChanged(previous, nextDisplayText) {
+  return !previous || previous.displayText !== nextDisplayText || previous.lifecycle !== "added" || previous.status !== "in_progress";
+}
+function buildModelStreamingSubtitleId(responseId) {
+  return `model-response:${responseId}`;
+}
+function getEventString(event, camelKey, snakeKey) {
+  const record = event;
+  const value = record[camelKey] ?? record[snakeKey];
+  return typeof value === "string" && value.length > 0 ? value : void 0;
+}
+function mapResponseStatusToConversationStatus(status) {
+  if (status === "completed") {
+    return "completed";
+  }
+  if (status === "in_progress") {
+    return "in_progress";
+  }
+  return "incomplete";
+}
 function hasSubtitleChanged(previous, next) {
   return previous.text !== next.text || previous.transcript !== next.transcript || previous.lifecycle !== next.lifecycle || previous.status !== next.status || previous.role !== next.role;
 }
 function isSubtitleRelatedEvent(type) {
   return type.startsWith("conversation.") || type.startsWith("response.");
 }
+function isModelStreamingTranscriptEvent(event) {
+  return event.type === "response.output_audio_transcript.delta" || event.type === "response.output_audio_transcript.done";
+}
+function isResponseDoneEvent(event) {
+  return event.type === "response.done";
+}
 var BREATHE_KEYFRAMES = `@keyframes ivi-subtitle-breathe{0%,100%{opacity:1}50%{opacity:.55}}`;
 function IVISubtitleOverlay(props) {
   const {
@@ -3197,13 +3313,15 @@ function IVISubtitleOverlay(props) {
     roles = "user",
     maxItems,
     maxVisible,
+    useModelStreamingTranscript,
     subtitleStyle,
     className,
     style
   } = props;
   const entries = useIviSubtitles(runtime, {
     roles,
-    maxItems: maxItems ?? maxVisible
+    maxItems: maxItems ?? maxVisible,
+    useModelStreamingTranscript
   });
   if (entries.length === 0) return null;
   const fontFamily = subtitleStyle?.fontFamily ?? "system-ui, -apple-system, sans-serif";
@@ -3826,6 +3944,19 @@ function detectMediaVolumeType(source) {
   if (!url) return null;
   return isM3u8Url(url) ? "hls" : "video";
 }
+function getSourceRenderKey(sourceId, source) {
+  const trtc = getTrtcPlayback(source.playback);
+  if (!trtc) {
+    return sourceId;
+  }
+  return [
+    "trtc",
+    trtc.app_id ?? "",
+    trtc.room_id ?? "",
+    trtc.user_id ?? "",
+    trtc.user_sig ?? ""
+  ].join(":");
+}
 function TrackSlotMediaContent(props) {
   const {
     slot,
@@ -3863,7 +3994,7 @@ function TrackSlotMediaContent(props) {
       IVITrtcPlayer,
       {
         trtc,
-        sourceId: source.source.source_id,
+        sourceId: slotSourceId,
         runtime,
         ...trtcPlayerProps,
         muted: trtcMuted,
@@ -4018,26 +4149,35 @@ function SlotVideo(props) {
 // src/react/internal/use-multi-preload-sources.ts
 function useMultiPreloadSources(sources, activeSourceId) {
   return useMemo(() => {
-    const entries = [];
-    for (const [id, runtimeSource] of sources) {
-      const isActive = id === activeSourceId;
-      const shouldMount = Boolean(runtimeSource.preload) || isActive;
-      if (!shouldMount) {
-        continue;
-      }
-      const ready = toReadyRuntimeSource(runtimeSource);
-      if (!ready) {
-        continue;
-      }
-      entries.push({
-        sourceId: id,
-        source: ready,
-        isActive
-      });
-    }
-    return entries;
+    return buildPreloadSourceEntries(sources, activeSourceId);
   }, [sources, activeSourceId]);
 }
+function buildPreloadSourceEntries(sources, activeSourceId) {
+  const entriesByRenderKey = /* @__PURE__ */ new Map();
+  for (const [id, runtimeSource] of sources) {
+    const isActive = id === activeSourceId;
+    const shouldMount = Boolean(runtimeSource.preload) || isActive;
+    if (!shouldMount) {
+      continue;
+    }
+    const ready = toReadyRuntimeSource(runtimeSource);
+    if (!ready) {
+      continue;
+    }
+    const renderKey = getSourceRenderKey(id, ready);
+    const entry = {
+      sourceId: id,
+      renderKey,
+      source: ready,
+      isActive
+    };
+    const previous = entriesByRenderKey.get(renderKey);
+    if (!previous || !previous.isActive && isActive) {
+      entriesByRenderKey.set(renderKey, entry);
+    }
+  }
+  return Array.from(entriesByRenderKey.values());
+}
 function IVITrackSlot(props) {
   const {
     slot,
@@ -4127,7 +4267,7 @@ function IVITrackSlot(props) {
                 }
               ) })
             },
-            entry.sourceId
+            entry.renderKey
           );
         }),
         showSubtitle && activeSource && supportsSubtitleOverlay(activeSource) && /* @__PURE__ */ jsx("div", { style: SUBTITLE_OVERLAY_STYLE, children: /* @__PURE__ */ jsx(