@ikonai/sdk 1.0.72 → 1.0.74

package/media/ikon-audio-playback.d.ts

@@ -76,11 +76,6 @@ export declare class IkonAudioPlayback {
     private readonly fallbackQueues;
     private readonly activeStreams;
     private mediaSessionConfig;
-    private readonly streamShapeSets;
-    private readonly currentShapeSetValues;
-    private readonly visemeBuffers;
-    private visemeDelayMs;
-    private readonly _visemeResult;
     constructor(client: IkonClient, config?: IkonAudioPlaybackConfig);
     /**
      * Set Media Session metadata for OS-level media controls (lock screen, notifications).
@@ -136,58 +131,6 @@ export declare class IkonAudioPlayback {
      * Dispose of resources. After calling this, the instance should not be reused.
      */
     dispose(): void;
-    /**
-     * Get the shape set declarations for a stream.
-     * Shape sets are declared at stream begin and contain metadata about the
-     * shape values that will be included in audio frames.
-     *
-     * @param streamId - The stream ID to get shape sets for
-     * @returns Array of shape set declarations, or null if stream not found
-     */
-    getShapeSets(streamId: string): {
-        SetId: number;
-        Name: string;
-        ShapeNames: string[];
-    }[] | null;
-    /**
-     * Get the current shape set values for a named shape set on a stream.
-     * Values are updated with each audio frame and represent the latest
-     * analysis results (e.g., viseme data).
-     *
-     * @param streamId - The stream ID to get values for
-     * @param setName - The name of the shape set (e.g., "Viseme")
-     * @returns Array of float values, or null if not found
-     */
-    getShapeSetValues(streamId: string, setName: string): number[] | null;
-    /**
-     * Get the current shape set values by set ID for a stream.
-     *
-     * @param streamId - The stream ID to get values for
-     * @param setId - The numeric ID of the shape set
-     * @returns Array of float values, or null if not found
-     */
-    getShapeSetValuesById(streamId: string, setId: number): number[] | null;
-    /**
-     * Get all current shape set values for a stream.
-     *
-     * @param streamId - The stream ID to get values for
-     * @returns Map of set ID to values, or null if stream not found
-     */
-    getAllShapeSetValues(streamId: string): Map<number, number[]> | null;
-    /**
-     * Get current viseme values from any active stream.
-     * Uses sample-based timing for precise synchronization with audio playback.
-     * Interpolates between viseme keyframes for smooth animation.
-     *
-     * @returns Object with mouthOpenY (0-1) and mouthForm (-1 to +1), or null if no viseme data
-     */
-    getCurrentVisemeValues(): {
-        mouthOpenY: number;
-        mouthForm: number;
-    } | null;
-    setVisemeDelayMs(delayMs: number): void;
-    getVisemeDelayMs(): number;
-    private processShapeSetValues;
     private resolveDiagnosticsConfig;
     private resolveBackgroundConfig;
     private ensureAudioGraph;

package/media/index.d.ts

@@ -4,6 +4,7 @@ export { IkonAudioPlayback } from './ikon-audio-playback';
 export type { IkonAudioPlaybackConfig } from './ikon-audio-playback';
 export { IkonVideoPlayback } from './ikon-video-playback';
 export type { IkonVideoPlaybackConfig } from './ikon-video-playback';
+export { VisemeTracker } from './viseme-tracker';
 export { isSharedArrayBufferSupported, isAudioWorkletSupported } from './capabilities';
 export { IkonMediaCapture } from './ikon-media-capture';
 export type { IkonMediaCaptureConfig } from './ikon-media-capture';

package/media/viseme-tracker.d.ts

@@ -0,0 +1,53 @@
+import { AudioShapeSet } from '../../../../shared/protocol/src/index.ts';
+import { IkonClient } from '../client/ikon-client';
+/**
+ * Time-synchronized lip-sync / shape-set receiver.
+ *
+ * Server audio analyzers (viseme, sentiment, etc.) declare their shape sets on
+ * {@link AudioStreamBegin} and stream per-frame values as {@link AudioShapeFrame}
+ * messages. Those messages ride the same GROUP_AUDIO stream as audio, but carry no
+ * PCM — so consuming them needs none of the audio playback machinery (decoder,
+ * AudioContext, worklet). This tracker parses them in a dedicated lightweight
+ * worker and exposes interpolated values via {@link getCurrentVisemeValues},
+ * driven by a plain wall clock. It works identically whether audio is delivered
+ * over WebRTC RTP tracks or the SDK's own playback pipeline.
+ */
+export declare class VisemeTracker {
+    private readonly client;
+    private worker;
+    private protocolPort;
+    private readonly streamShapeSets;
+    private readonly currentShapeSetValues;
+    private readonly visemeBuffers;
+    private visemeDelayMs;
+    private readonly _visemeResult;
+    constructor(client: IkonClient);
+    /**
+     * Starts the viseme worker and subscribes to the GROUP_AUDIO protocol stream.
+     * Idempotent; safe to call once audio is known to be in use.
+     */
+    enable(): void;
+    dispose(): void;
+    getShapeSets(streamId: string): AudioShapeSet[] | null;
+    /**
+     * Get the latest values for a named shape set on a stream (e.g. "Viseme").
+     */
+    getShapeSetValues(streamId: string, setName: string): number[] | null;
+    getShapeSetValuesById(streamId: string, setId: number): number[] | null;
+    getAllShapeSetValues(streamId: string): Map<number, number[]> | null;
+    /**
+     * Get current viseme values from any active stream.
+     * Uses sample-based timing for precise synchronization with audio playback.
+     * Interpolates between viseme keyframes for smooth animation.
+     *
+     * @returns Object with mouthOpenY (0-1) and mouthForm (-1 to +1), or null if no viseme data
+     */
+    getCurrentVisemeValues(): {
+        mouthOpenY: number;
+        mouthForm: number;
+    } | null;
+    setVisemeDelayMs(delayMs: number): void;
+    getVisemeDelayMs(): number;
+    private onWorkerMessage;
+    private processShapeSetValues;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ikonai/sdk",
-  "version": "1.0.72",
+  "version": "1.0.74",
   "type": "module",
   "main": "./index.js",
   "types": "./index.d.ts",

package/utils/connect-timeline.d.ts

@@ -0,0 +1,4 @@
+export declare function getConnectTraceId(): string;
+export declare function reportConnectTimeline(opts?: {
+    snapshotSeeded?: boolean;
+}): void;

package/utils/log-sender.d.ts

@@ -1,2 +1,3 @@
 export declare function startLogSender(backendUrl: string, authToken: string, sessionId?: string, channelInstanceId?: string): void;
+export declare function sendAnalyticsEvent(eventName: string, parameters: Record<string, string>): void;
 export declare function stopLogSender(): void;

package/utils/query-params.d.ts

@@ -17,6 +17,7 @@
  * - ikon-audio        Force audio on (true) or off (false)
  * - ikon-video        Force video on (true) or off (false)
  * - ikon-webrtc       Force WebRTC on (true) or off (false)
+ * - ikon-ice-transport Force the WebRTC ICE transport policy: "relay" (TURN-only, useful for testing the relay path) or "all" (default)
  * - ikon-inspect      Enable element inspection overlay (true)
  * - ikon-retry        Enable retry loops in auth/channel connect (default true; set false to fail fast)
  * - ikon-api          Route backend API calls through same-origin /ikon/api (default true on deployed apps; false to force direct calls to api.{env}.ikon.live; always off in local dev)
@@ -33,6 +34,7 @@ export declare const IKON_PARAM_GIT_BRANCH = "ikon-git-branch";
 export declare const IKON_PARAM_AUDIO = "ikon-audio";
 export declare const IKON_PARAM_VIDEO = "ikon-video";
 export declare const IKON_PARAM_WEBRTC = "ikon-webrtc";
+export declare const IKON_PARAM_ICE_TRANSPORT = "ikon-ice-transport";
 export declare const IKON_PARAM_INSPECT = "ikon-inspect";
 export declare const IKON_PARAM_DEBUG_OVERLAY = "ikon-debug-overlay";
 export declare const IKON_PARAM_AUTH = "ikon-auth";
@@ -52,6 +54,13 @@ export declare function getServerUrlParam(): string | null;
 export declare function getAudioParam(): boolean | null;
 export declare function getVideoParam(): boolean | null;
 export declare function getWebRtcParam(): boolean | null;
+/**
+ * Force the WebRTC ICE transport policy. `relay` makes the browser gather only
+ * TURN relay candidates (no host/srflx), forcing media through the TURN server —
+ * useful for verifying the relay path works. `all` is the browser default.
+ * Returns null when unset or unrecognized.
+ */
+export declare function getIceTransportPolicyParam(): RTCIceTransportPolicy | null;
 export declare function getInspectParam(): boolean;
 /**
  * On-screen debug overlay toggle. Distinct from `ikon-debug` (verbose logging / devtools):

package/webrtc/index.d.ts

@@ -1,3 +1,3 @@
 export { WebRTCConnection, type WebRTCConnectionConfig, type WebRTCConnectionState } from './webrtc-connection';
 export { WebRTCSignaling, type WebRTCSignalingConfig } from './webrtc-signaling';
-export { WEBRTC_AUDIO_CODEC, WEBRTC_AUDIO_SAMPLE_RATE, WEBRTC_AUDIO_CHANNELS, WEBRTC_AUDIO_BITRATE, WEBRTC_AUDIO_JITTER_BUFFER_TARGET_MS, WEBRTC_VIDEO_CODEC, WEBRTC_VIDEO_MAX_BITRATE, WEBRTC_VIDEO_MAX_FRAMERATE, WEBRTC_MAX_AUDIO_TRACKS, WEBRTC_MAX_VIDEO_TRACKS, WEBRTC_BUNDLE_POLICY, WEBRTC_RTCP_MUX_POLICY, WEBRTC_DATA_CHANNEL_LABEL, WEBRTC_DATA_CHANNEL_ORDERED, WEBRTC_DATA_CHANNEL_MAX_RETRANSMITS, } from './webrtc-config';
+export { WEBRTC_AUDIO_CODEC, WEBRTC_AUDIO_SAMPLE_RATE, WEBRTC_AUDIO_CHANNELS, WEBRTC_AUDIO_BITRATE, WEBRTC_AUDIO_JITTER_BUFFER_TARGET_MS, WEBRTC_VIDEO_CODEC, WEBRTC_VIDEO_MAX_BITRATE, WEBRTC_VIDEO_MAX_FRAMERATE, WEBRTC_MAX_AUDIO_TRACKS, WEBRTC_MAX_VIDEO_TRACKS, WEBRTC_BUNDLE_POLICY, WEBRTC_RTCP_MUX_POLICY, WEBRTC_DATA_CHANNEL_LABEL, WEBRTC_DATA_CHANNEL_ORDERED, WEBRTC_DATA_CHANNEL_MAX_RETRANSMITS, WEBRTC_CONNECT_TIMEOUT_MS, } from './webrtc-config';

package/webrtc/webrtc-config.d.ts

@@ -9,6 +9,8 @@ export declare const WEBRTC_VIDEO_MAX_FRAMERATE = 30;
 export declare const WEBRTC_VIDEO_PLAYOUT_DELAY_HINT_S = 0.05;
 export declare const WEBRTC_BUNDLE_POLICY: RTCBundlePolicy;
 export declare const WEBRTC_RTCP_MUX_POLICY: RTCRtcpMuxPolicy;
+export declare const WEBRTC_ICE_SERVERS_REQUEST_TIMEOUT_MS = 5000;
+export declare const WEBRTC_CONNECT_TIMEOUT_MS = 8000;
 export declare const WEBRTC_MAX_AUDIO_TRACKS = 16;
 export declare const WEBRTC_MAX_VIDEO_TRACKS = 16;
 export declare const WEBRTC_DATA_CHANNEL_LABEL = "ikon-data";

package/webrtc/webrtc-connection.d.ts

@@ -4,6 +4,7 @@ export interface WebRTCConnectionConfig {
     useVideoTrack: boolean;
     useDataChannel: boolean;
     iceServers?: RTCIceServer[];
+    iceTransportPolicy?: RTCIceTransportPolicy;
     onIceCandidate?: (candidate: RTCIceCandidate | null) => void;
     onStateChange?: (state: WebRTCConnectionState) => void;
     onDataChannelMessage?: (data: ArrayBuffer) => void;

package/webrtc/webrtc-signaling.d.ts

@@ -8,6 +8,7 @@ export interface WebRTCSignalingConfig {
     useVideoTrack: boolean;
     useDataChannel: boolean;
     iceServers?: RTCIceServer[];
+    iceTransportPolicy?: RTCIceTransportPolicy;
     onStateChange?: (state: WebRTCConnectionState) => void;
     onDataChannelMessage?: (data: ArrayBuffer) => void;
     onDataChannelOpen?: () => void;
@@ -23,12 +24,19 @@ export declare class WebRTCSignaling {
     private isConnecting;
     private audioTrackCount;
     private videoTrackCount;
+    private iceServersResolve;
     constructor(config: WebRTCSignalingConfig);
     get state(): WebRTCConnectionState;
     get peerConnection(): RTCPeerConnection | null;
     get isDataChannelOpen(): boolean;
     connect(sendMessage: (message: ProtocolMessage) => void): Promise<void>;
     handleProtocolMessage(message: ProtocolMessage | Uint8Array): boolean;
+    /**
+     * Tell the server we are abandoning WebRTC so it drops the peer immediately
+     * and resumes protocol-channel media, rather than waiting for its liveness
+     * timeout. Must be called before disconnect() (which clears the sender).
+     */
+    notifyServerClose(): void;
     disconnect(): void;
     sendDataChannelMessage(data: ArrayBuffer): void;
     private audioCaptureStream;
@@ -60,6 +68,14 @@ export declare class WebRTCSignaling {
     stopCapture(): Promise<void>;
     private applyEncodingParams;
     private sendOffer;
+    /**
+     * Ask the server for the current TURN/STUN servers (with freshly minted TURN
+     * credentials) and wait for the response before building the peer connection.
+     * Falls back to any app-configured servers if the server doesn't answer in
+     * time, so a missing or older server never hangs the connection.
+     */
+    private requestIceServers;
+    private handleIceServersResponse;
     private handleAnswer;
     private parseVideoCodecFromSdp;
     private handleLocalIceCandidate;

package/worker/audio-playback-worker.d.ts

@@ -1,4 +1,3 @@
-import { AudioShapeSet, AudioShapeSetValues } from '../../../../shared/protocol/src/index.ts';
 export type AudioPlaybackWorkerTransport = 'sab' | 'pcm';
 export type AudioPlaybackWorkerConfigureMessage = {
     type: 'configure';
@@ -26,7 +25,6 @@ export type AudioPlaybackWorkerOut = {
     type: 'streamBegin';
     streamId: string;
     sharedArrayBuffer?: SharedArrayBuffer;
-    shapeSets?: AudioShapeSet[];
 } | {
     type: 'streamEnd';
     streamId: string;
@@ -51,12 +49,4 @@ export type AudioPlaybackWorkerOut = {
     isFirst: boolean;
     isLast: boolean;
     jitterMs: number;
-} | {
-    type: 'shapeFrame';
-    streamId: string;
-    epoch: number;
-    sequence: number;
-    timestampInterleavedSamples: number;
-    frameSizeInterleavedSamples: number;
-    shapeSetValues: AudioShapeSetValues[];
 };

package/worker/viseme-worker.d.ts

@@ -0,0 +1,24 @@
+import { AudioShapeSet, AudioShapeSetValues } from '../../../../shared/protocol/src/index.ts';
+export type VisemeWorkerAttachProtocolPortMessage = {
+    type: 'attachProtocolPort';
+    port: MessagePort;
+};
+export type VisemeWorkerDisposeMessage = {
+    type: 'dispose';
+};
+export type VisemeWorkerIn = VisemeWorkerAttachProtocolPortMessage | VisemeWorkerDisposeMessage;
+export type VisemeWorkerOut = {
+    type: 'streamBegin';
+    streamId: string;
+    shapeSets: AudioShapeSet[];
+} | {
+    type: 'streamEnd';
+    streamId: string;
+} | {
+    type: 'shapeFrame';
+    streamId: string;
+    epoch: number;
+    timestampInterleavedSamples: number;
+    frameSizeInterleavedSamples: number;
+    shapeSetValues: AudioShapeSetValues[];
+};