@vivix-ai/ivi-frontend-sdk 0.2.3 → 0.3.1

package/dist/index.d.ts

@@ -1,12 +1,18 @@
-import { IviRealtimeSession, IviStage, IviTrack, IviSource, IviSourcePlayback, IviSessionSourceFailedPayload, IviStream, IviStreamStatus, IviConversationItemRole, IviConversationItem, IviConversationItemContent, ReceiveSessionEndedEvent, ParsedIviEvent, IviRealtimeResponseCreateParams, ReceiveConversationItemAddedEvent, ReceiveConversationItemDoneEvent, ReceiveResponseCreatedEvent, IviSourcePlaybackTrtc, IviClient, IviClientConfig } from '@vivix/ivi-sdk-ts';
+import { ReceiveSessionSourceReadyPayload, IviStream, IviConversationItem, IviRealtimeSessionConfig, IviStageComposition, IviTrack, IviSourceAsset, ReceiveSessionSourceFailedPayload, IviConversationItemContent, ReceiveSessionEndedEvent, ParsedIviEvent, IviRealtimeResponseCreateParams, ReceiveConversationItemAddedEvent, ReceiveConversationItemDoneEvent, ReceiveResponseCreatedEvent, IviClient, IviClientConfig } from '@vivix-ai/ivi-sdk-ts';
+import { LiveKitBrowserTransportConfig } from '@vivix-ai/ivi-sdk-ts/transports/livekit-browser';
 import { ReactNode, CSSProperties, ReactElement, VideoHTMLAttributes, ImgHTMLAttributes } from 'react';
-import { IviClientLogEntry } from '@vivix/ivi-sdk-ts/client';
+import { IviClientLogEntry } from '@vivix-ai/ivi-sdk-ts/client';
+
+type IviConversationItemRole = NonNullable<IviConversationItem["role"]>;
+type IviSourcePlayback = NonNullable<ReceiveSessionSourceReadyPayload["playback"]>;
+type IviSourcePlaybackTrtc = NonNullable<IviSourcePlayback["trtc"]>;
+type IviStreamStatus = IviStream["status"];
 
 type IviRuntimeStatus = "idle" | "connecting" | "syncing" | "running" | "stopped";
 interface IviRuntimeState {
     status: IviRuntimeStatus;
-    session: IviRealtimeSession | null;
-    stage: IviStage | null;
+    session: IviRealtimeSessionConfig | null;
+    stage: IviStageComposition | null;
     tracks: Map<string, IviTrack>;
     sources: Map<string, IviRuntimeSource>;
     streams: Map<string, IviRuntimeStream>;
@@ -32,14 +38,14 @@ interface IviRuntimeSourcePreloadState {
     autoclearAfterPlay: boolean;
 }
 interface IviRuntimeSource {
-    source: IviSource;
+    source: IviSourceAsset;
     status: "created" | "ready" | "failed";
     playback?: IviSourcePlayback;
     width?: number;
     height?: number;
     durationMs?: number;
     hasAudio?: boolean;
-    error?: IviSessionSourceFailedPayload["error"];
+    error?: ReceiveSessionSourceFailedPayload["error"];
     /**
      * 预加载状态：
      * - 存在：渲染层应对该 source 进行预加载；
@@ -80,7 +86,7 @@ interface IviRuntimeCoordinatorConfig {
      */
     onSessionEnded?: (event: ReceiveSessionEndedEvent) => void;
     /**
-     * Runtime 内部链路日志回调；未传入时仍仅输出原有 console 日志。
+     * Runtime 内部链路日志回调；未传入时不输出日志。
      */
     onLog?: IviRuntimeLogCallback;
 }
@@ -105,6 +111,13 @@ interface IviRuntimeUserTextToResponseOptions {
      * 透传给 response.create 的响应参数（不含 input）。
      */
     response?: Omit<IviRealtimeResponseCreateParams, "input">;
+    /**
+     * 触发 response.create 时如何处理 input。
+     *
+     * - `"context"`（默认）：不显式传 `response.input`，由 IVI 服务端使用内部维护的会话上下文。
+     * - `"item_reference"`：显式传入刚创建的用户 item 引用；服务端会以该 input 为准，不使用默认上下文。
+     */
+    responseInput?: "context" | "item_reference";
     /**
      * 关键节点回调。
      */
@@ -213,6 +226,105 @@ declare class TrtcSourceManager {
     private log;
 }
 
+/**
+ * LiveKit 拉流所需的会话凭证。连接字段复用 `@vivix-ai/ivi-sdk-ts` LiveKit transport 的上游类型。
+ */
+type IviSourcePlaybackLivekit = {
+    ws_url: LiveKitBrowserTransportConfig["url"];
+    token: LiveKitBrowserTransportConfig["token"];
+    room?: string;
+    identity?: string;
+};
+/**
+ * LiveKit 形态的 playback 实体（运行态用）。
+ */
+type IviSourcePlaybackLivekitDescriptor = IviSourcePlayback & {
+    type: "livekit";
+    livekit: IviSourcePlaybackLivekit;
+};
+/**
+ * 判定一个 playback 是否携带合法的 LiveKit 配置。
+ * 兼容上游类型尚未感知 `livekit` 分支的情况，因此使用结构化检查。
+ */
+declare function isLivekitSourcePlayback(playback: IviSourcePlayback | undefined | null): playback is IviSourcePlaybackLivekitDescriptor;
+/**
+ * Runtime 层守卫：source 已 ready 且 playback 是 livekit。
+ */
+declare function isReadyLivekitRuntimeSource(source: IviRuntimeSource | undefined | null): source is IviRuntimeSource & {
+    playback: IviSourcePlaybackLivekitDescriptor;
+};
+/**
+ * 浅比较两份 livekit 配置；用于 SourceManager 决定是否需要重建 Room 连接。
+ */
+declare function isSameLivekitConfig(a: IviSourcePlaybackLivekit, b: IviSourcePlaybackLivekit): boolean;
+
+type LivekitSourceConnectionStatus = "idle" | "connecting" | "connected" | "error";
+interface LivekitSourceSnapshot {
+    status: LivekitSourceConnectionStatus;
+    error?: string;
+}
+type LivekitSourceListener = (snapshot: LivekitSourceSnapshot) => void;
+/**
+ * LiveKit 拉流会话池：
+ * - 每个 `sourceId` 对应一个 LiveKit `Room` 实例，懒连接、单飞建连；
+ * - 远端 track 订阅后按 view 分发到各自容器；
+ * - 接口形状刻意贴近 `TrtcSourceManager`，便于 runtime 协调器与 React 组件镜像复用。
+ */
+declare class LivekitSourceManager {
+    private readonly onLog?;
+    private readonly sessions;
+    private readonly listeners;
+    constructor(onLog?: IviRuntimeLogCallback | undefined);
+    /**
+     * 与 runtime 当前 sources 对齐：
+     * - 对 ready + livekit 的 source 进行 upsert（必要时建立连接）；
+     * - 清理已不在 runtime 中的会话（释放资源）。
+     */
+    syncRuntimeSources(sources: Map<string, IviRuntimeSource>): void;
+    /**
+     * 按 sourceId 注册/更新 LiveKit 配置。
+     * 若 ws_url / token / room / identity 任一发生变化，会先销毁旧会话再按新参数重建连接。
+     */
+    upsertSource(sourceId: string, livekit: IviSourcePlaybackLivekit): void;
+    /**
+     * 删除指定 source 的 LiveKit 会话并释放连接资源。
+     */
+    removeSource(sourceId: string): void;
+    reset(): void;
+    subscribe(sourceId: string, listener: LivekitSourceListener): () => void;
+    getSnapshot(sourceId: string): LivekitSourceSnapshot;
+    hasRemoteVideoAvailable(sourceId: string): boolean;
+    /**
+     * 等待指定 source 的 LiveKit 会话首次拿到远端视频 track。
+     * - 若已收到过，立即返回 true；
+     * - 若会话被销毁或超时，返回 false。
+     */
+    waitForRemoteVideoAvailable(sourceId: string, timeoutMs?: number): Promise<boolean>;
+    /**
+     * 把一个渲染容器绑定到 source 会话：
+     * - 确保已连接；
+     * - 把已订阅的远端 video/audio 回放到该容器；
+     * - 应用初始 muted 策略。
+     */
+    attachView(sourceId: string, viewId: string, container: HTMLElement, muted: boolean): Promise<void>;
+    detachView(sourceId: string, viewId: string): void;
+    updateViewMuted(sourceId: string, viewId: string, muted: boolean): void;
+    private createSession;
+    /**
+     * 懒连接：单飞模式下复用同一个 connectPromise；建连后注册远端事件并把已订阅 track 推给 view。
+     */
+    private ensureConnected;
+    private attachKnownTracksToView;
+    private attachTrackToView;
+    private detachTrackFromBinding;
+    private detachAllElementsFromView;
+    private applyMutedToBinding;
+    private applyMutedToElement;
+    private disposeSession;
+    private emitSnapshot;
+    private log;
+}
+
 /**
  * 协调者负责 runtime 生命周期、状态机和跨域补偿，不参与事件类型分发。
  *
@@ -258,6 +370,7 @@ declare class IviRuntimeCoordinator {
     private readonly sourceManager;
     private readonly streamManager;
     private readonly trtcSourceManager;
+    private readonly livekitSourceManager;
     private readonly conversationManager;
     private readonly sessionHandler;
     private readonly stageHandler;
@@ -289,13 +402,14 @@ declare class IviRuntimeCoordinator {
     getState(): IviRuntimeState;
     onStateChange(listener: (state: IviRuntimeState) => void): () => void;
     onEvent(listener: IviRuntimeEventListener): () => void;
+    emitLog(entry: IviRuntimeLogEntry): void;
     /**
      * 编排一次"用户文本输入 -> 触发模型回复"链路。
      *
      * 流程：
      * 1) 发送 conversation.item.create（可指定 itemId）；
      * 2) 等待 conversation.item.added + conversation.item.done；
-     * 3) 发送 response.create（携带 item_reference，可选绑定 stream）；
+     * 3) 发送 response.create（默认不携带 input，使用 IVI 内部上下文；可选显式绑定 item_reference）；
      * 4) 等待 response.created 并结束。
      */
     sendUserTextAndTriggerResponse(options: IviRuntimeUserTextToResponseOptions): Promise<IviRuntimeUserTextToResponseResult>;
@@ -308,6 +422,7 @@ declare class IviRuntimeCoordinator {
      */
     sendSessionSourcePlaybackCompleted(sourceId: string, trackId?: string): void;
     getTrtcSourceManager(): TrtcSourceManager;
+    getLivekitSourceManager(): LivekitSourceManager;
     private onConnectionChange;
     private validateStageTracks;
     private validateTracksListRefreshed;
@@ -318,6 +433,7 @@ declare class IviRuntimeCoordinator {
     private onTracksChanged;
     private applyLocalTrackTake;
     private deferredTrtcTakeCompleteAndTake;
+    private deferredLivekitTakeCompleteAndTake;
     private sendSessionTrackTake;
     private onSourcesChanged;
     private onStreamsChanged;
@@ -361,9 +477,10 @@ declare class IviRuntimeDispatcher {
     private dispatchEvent;
 }
 
+type IviFrontendClientConfig = IviClientConfig;
 declare class IviFrontendSdk {
-    createClient(config: IviClientConfig): IviClient;
-    createRuntimeCoordinator(clientConfig: IviClientConfig, runtimeConfig?: IviRuntimeCoordinatorConfig): IviRuntimeCoordinator;
+    createClient(config: IviFrontendClientConfig): IviClient;
+    createRuntimeCoordinator(clientConfig: IviFrontendClientConfig, runtimeConfig?: IviRuntimeCoordinatorConfig): IviRuntimeCoordinator;
 }
 
 interface IviStageSlotBinding {
@@ -411,6 +528,27 @@ interface IVITrtcPlayerProps {
     errorFallback?: ReactNode;
     muted?: boolean;
 }
+/**
+ * 默认 TRTC 播放器：挂载即加入房间并拉流，连接由 TrtcSourceManager 统一复用。
+ * muted 变化仅更新音频策略，不触发重连。
+ */
+declare function IVITrtcPlayer(props: IVITrtcPlayerProps): ReactNode;
+
+interface IVILivekitPlayerProps {
+    livekit: IviSourcePlaybackLivekit;
+    sourceId?: string;
+    runtime?: IviRuntimeCoordinator | null;
+    className?: string;
+    style?: CSSProperties;
+    loadingFallback?: ReactNode;
+    errorFallback?: ReactNode;
+    muted?: boolean;
+}
+/**
+ * 默认 LiveKit 播放器：挂载即加入房间并拉流，连接由 LivekitSourceManager 统一复用。
+ * muted 变化仅更新音频静音策略，不触发重连。
+ */
+declare function IVILivekitPlayer(props: IVILivekitPlayerProps): ReactNode;
 
 interface IVIVolumeControlColors {
     /** 容器背景色，默认 rgba(0, 0, 0, 0.55) */
@@ -437,6 +575,33 @@ interface IVIVolumeControlProps {
     style?: CSSProperties;
 }
 
+type IviSubtitleRole = IviRuntimeConversationItem["role"];
+interface IviSubtitleItem {
+    id: string;
+    role: IviSubtitleRole;
+    lifecycle: IviRuntimeConversationItem["lifecycle"];
+    status: IviRuntimeConversationItem["status"];
+    text: string;
+    transcript: string;
+    displayText: string;
+    content: IviRuntimeConversationItem["content"];
+    item: IviRuntimeConversationItem["item"];
+    /** 首次进入字幕队列的时间戳。 */
+    timestamp: number;
+    /** 最近一次字幕内容或状态更新的时间戳。 */
+    updatedAt: number;
+}
+interface IviUseSubtitlesOptions {
+    /** 要收集的发言人角色，默认 ["user"]。传单个字符串或数组均可。 */
+    roles?: IviSubtitleRole | IviSubtitleRole[];
+    /** 最多保留的字幕条数，超过后自动清理最旧条目，默认 2。 */
+    maxItems?: number;
+}
+/**
+ * 监听 runtime 中的 conversation/response 事件，并维护当前应展示的字幕队列。
+ */
+declare function useIviSubtitles(runtime: IviRuntimeCoordinator | null, options?: IviUseSubtitlesOptions): IviSubtitleItem[];
+
 interface IVISubtitleOverlayStyle {
     /** 字体，默认 system-ui */
     fontFamily?: string;
@@ -448,14 +613,14 @@ interface IVISubtitleOverlayStyle {
     background?: string;
 }
 interface IVISubtitleOverlayProps {
-    /** 会话条目列表（由 runtime state 提供） */
-    conversations: IviRuntimeConversationItem[];
+    /** 字幕来源 runtime。 */
+    runtime: IviRuntimeCoordinator | null;
     /** 要显示的发言人角色，默认 ["user"]。传单个字符串或数组均可。 */
-    roles?: IviConversationItemRole | IviConversationItemRole[];
-    /** 同时可见的最大条目数，默认 2 */
+    roles?: IviUseSubtitlesOptions["roles"];
+    /** 最多保留的字幕条数，默认 2。 */
+    maxItems?: number;
+    /** @deprecated Use `maxItems` instead. */
     maxVisible?: number;
-    /** lifecycle 变为 done 后自动消失的毫秒数，默认 5000 */
-    dismissAfterMs?: number;
     /** 样式配置 */
     subtitleStyle?: IVISubtitleOverlayStyle;
     /** 自定义类名 */
@@ -463,6 +628,11 @@ interface IVISubtitleOverlayProps {
     /** 自定义样式 */
     style?: CSSProperties;
 }
+/**
+ * 字幕浮层：基于 useIviSubtitles 的预制 UI，胶囊形半透明黑底白字。
+ * added 阶段背景呼吸闪动；字幕保留与裁剪由 hook 的 maxItems 控制。
+ */
+declare function IVISubtitleOverlay(props: IVISubtitleOverlayProps): ReactNode;
 
 interface IviTrackSlotRenderContext {
     slot: string;
@@ -471,19 +641,6 @@ interface IviTrackSlotRenderContext {
     /** 当前正在 standby 槽位预加载（不可见、已静音） */
     isPreloading?: boolean;
 }
-/**
- * 媒体空白区域（letterbox / pillarbox）背景模式。
- * - `"black"` / `"white"` / `"transparent"`：纯色填充
- * - `{ color: string }`：用户自定义 CSS 颜色
- * - `"blur"`：实时高斯模糊（等同 `{ blur: "live" }`）
- * - `{ blur: "live" }`：实时模糊，背景跟随媒体画面持续更新
- * - `{ blur: "static" }`：静态模糊，仅取首帧进行模糊，性能开销极低
- */
-type IviTrackSlotBackground = "black" | "white" | "transparent" | {
-    color: string;
-} | "blur" | {
-    blur: "live" | "static";
-};
 interface IVITrackSlotProps {
     slot?: string;
     trackId?: string;
@@ -491,22 +648,24 @@ interface IVITrackSlotProps {
     style?: CSSProperties;
     emptyFallback?: ReactNode;
     renderTrtc?: (context: IviTrackSlotRenderContext) => ReactNode;
+    renderLivekit?: (context: IviTrackSlotRenderContext) => ReactNode;
     renderMedia?: (context: IviTrackSlotRenderContext) => ReactNode;
     videoProps?: Omit<VideoHTMLAttributes<HTMLVideoElement>, "src">;
     imageProps?: Omit<ImgHTMLAttributes<HTMLImageElement>, "src">;
     adaptToSourceSize?: boolean;
     fitStrategy?: "contain" | "cover" | "auto";
     trtcPlayerProps?: Omit<IVITrtcPlayerProps, "trtc" | "sourceId" | "runtime">;
+    livekitPlayerProps?: Omit<IVILivekitPlayerProps, "livekit" | "sourceId" | "runtime">;
     /** 是否显示音量控制浮层（仅在播放音视频媒体时生效，图片无音量） */
     showVolumeControl?: boolean;
     /** 音量控制组件的自定义配置（图标、颜色、样式等） */
     volumeControlProps?: Omit<IVIVolumeControlProps, "volume" | "onVolumeChange">;
     /** 是否显示字幕浮层（仅在支持字幕的播放源上生效） */
     showSubtitle?: boolean;
-    /** 字幕组件的自定义配置（字体、颜色、消失延时等） */
-    subtitleProps?: Omit<IVISubtitleOverlayProps, "conversations">;
-    /** 媒体空白区域（letterbox/pillarbox）的背景模式，默认 "black" */
-    background?: IviTrackSlotBackground;
+    /** 字幕组件的自定义配置（角色、最大条数、字体、颜色等） */
+    subtitleProps?: Omit<IVISubtitleOverlayProps, "runtime">;
+    /** TrackSlot 内部媒体链路日志回调；未传入时不输出日志。 */
+    onLog?: IviRuntimeLogCallback;
 }
 /**
  * Track 窗口组件：基于多重预加载策略渲染所有 ready 的 Source。
@@ -530,15 +689,15 @@ interface IviManagedRuntimeLogEntry {
 }
 type IviManagedRuntimeLogCallback = (entry: IviManagedRuntimeLogEntry) => void;
 interface IviManagedRuntimeConfig {
-    clientConfig: IviClientConfig;
+    clientConfig: Partial<IviFrontendClientConfig>;
     autoStart?: boolean;
     runtimeConfig?: IviRuntimeCoordinatorConfig;
     onRuntimeInitError?: (error: unknown) => void;
     /**
      * 统一收集 SDK 内部带 IVI 标记的日志。
      *
-     * - IviClient 的 onLog 会被转换为 `[IVI-SEND]`、`[IVI-WS]`、`[IVI-RECONNECT]`
-     * - runtime / TRTC 等日志会通过 runtime onLog 回调上报，不会通过 patch console 收集
+     * - IviClient 的 onLog 会被转换为 `[IVI-SEND]`、`[IVI-TRANSPORT]`、`[IVI-RECONNECT]`
+     * - runtime / TRTC / LiveKit 等日志会通过 runtime onLog 回调上报，不会通过 patch console 收集
      */
     onLog?: IviManagedRuntimeLogCallback;
 }
@@ -546,4 +705,4 @@ declare function useManagedIviRuntime(config: IviManagedRuntimeConfig): IviRunti
 
 declare function useIviStageView(): IviStageViewContextValue;
 
-export { EMPTY_RUNTIME_STATE, IVIStageView, type IVIStageViewProps, IVITrackSlot, type IVITrackSlotProps, IviFrontendSdk, type IviManagedRuntimeConfig, type IviManagedRuntimeLogCallback, type IviManagedRuntimeLogEntry, type IviManagedRuntimeLogLevel, type IviManagedRuntimeLogSource, type IviRuntimeConversationItem, type IviRuntimeConversationLifecycle, type IviRuntimeConversationStatus, IviRuntimeCoordinator, type IviRuntimeCoordinatorConfig, IviRuntimeDispatcher, type IviRuntimeDispatcherConfig, type IviRuntimeEventListener, type IviRuntimeLogCallback, type IviRuntimeLogEntry, type IviRuntimeLogLevel, type IviRuntimeSource, type IviRuntimeSourcePreloadState, type IviRuntimeState, type IviRuntimeStatus, type IviRuntimeStream, type IviRuntimeUserTextToResponseCallbacks, type IviRuntimeUserTextToResponseOptions, type IviRuntimeUserTextToResponseResult, type IviStageSlotBinding, type IviStageViewContextValue, type IviTrackSlotBackground, useIviStageView, useManagedIviRuntime, useRuntimeState };
+export { EMPTY_RUNTIME_STATE, IVILivekitPlayer, type IVILivekitPlayerProps, IVIStageView, type IVIStageViewProps, IVISubtitleOverlay, type IVISubtitleOverlayProps, type IVISubtitleOverlayStyle, IVITrackSlot, type IVITrackSlotProps, IVITrtcPlayer, type IVITrtcPlayerProps, type IviFrontendClientConfig, IviFrontendSdk, type IviManagedRuntimeConfig, type IviManagedRuntimeLogCallback, type IviManagedRuntimeLogEntry, type IviManagedRuntimeLogLevel, type IviManagedRuntimeLogSource, type IviRuntimeConversationItem, type IviRuntimeConversationLifecycle, type IviRuntimeConversationStatus, IviRuntimeCoordinator, type IviRuntimeCoordinatorConfig, IviRuntimeDispatcher, type IviRuntimeDispatcherConfig, type IviRuntimeEventListener, type IviRuntimeLogCallback, type IviRuntimeLogEntry, type IviRuntimeLogLevel, type IviRuntimeSource, type IviRuntimeSourcePreloadState, type IviRuntimeState, type IviRuntimeStatus, type IviRuntimeStream, type IviRuntimeUserTextToResponseCallbacks, type IviRuntimeUserTextToResponseOptions, type IviRuntimeUserTextToResponseResult, type IviSourcePlaybackLivekit, type IviSourcePlaybackLivekitDescriptor, type IviStageSlotBinding, type IviStageViewContextValue, type IviSubtitleItem, type IviSubtitleRole, type IviUseSubtitlesOptions, LivekitSourceManager, TrtcSourceManager, isLivekitSourcePlayback, isReadyLivekitRuntimeSource, isSameLivekitConfig, useIviStageView, useIviSubtitles, useManagedIviRuntime, useRuntimeState };