@ikonai/sdk 0.0.41 → 0.0.43

package/media/audio-constants.d.ts

@@ -26,3 +26,8 @@ export declare const AUDIO_CAPTURE_DEFAULT_BITRATE = 32000;
 export declare const AUDIO_CAPTURE_DEFAULT_COMPLEXITY = 5;
 export declare const AUDIO_CAPTURE_MIN_BITRATE = 8000;
 export declare const AUDIO_CAPTURE_MAX_BITRATE = 128000;
+export declare const VISEME_BUFFER_MAX_FRAMES = 1500;
+export declare const VISEME_BUFFER_DELAY_MS = -20;
+export declare const VISEME_BUFFER_DELAY_MIN_MS = -100;
+export declare const VISEME_BUFFER_DELAY_MAX_MS = 200;
+export declare const VISEME_INTERPOLATION_ENABLED = true;

package/media/ikon-audio-capture.d.ts

@@ -18,6 +18,8 @@ export interface IkonAudioCaptureRequest {
 }
 export interface IkonAudioCaptureHandle {
     captureId: string;
+    /** Whether the capture is still active (not stopped). On iOS, this becomes false after endSegment. */
+    readonly isActive: boolean;
     startSegment(): void;
     endSegment(): void;
     stop(): Promise<void>;

package/media/ikon-audio-playback.d.ts

@@ -76,6 +76,10 @@ export declare class IkonAudioPlayback {
     private readonly fallbackQueues;
     private readonly activeStreams;
     private mediaSessionConfig;
+    private readonly streamShapeSets;
+    private readonly currentShapeSetValues;
+    private readonly visemeBuffers;
+    private readonly _visemeResult;
     constructor(client: IkonClient, config?: IkonAudioPlaybackConfig);
     /**
      * Set Media Session metadata for OS-level media controls (lock screen, notifications).
@@ -115,10 +119,71 @@ export declare class IkonAudioPlayback {
      * Fast operation - reconnects nodes to destination.
      */
     resume(): void;
+    /**
+     * Request recovery of the AudioContext. Call this when external factors
+     * (like audio capture stopping on iOS) may have disrupted playback.
+     *
+     * IMPORTANT: On iOS, this must be called synchronously from a user gesture handler
+     * (e.g., button click/release) to ensure the AudioContext can be recreated.
+     */
+    requestRecovery(): void;
+    /**
+     * Set up audio worklet after recreating AudioContext on iOS.
+     */
+    private setupAudioGraphAfterRecreate;
     /**
      * Dispose of resources. After calling this, the instance should not be reused.
      */
     dispose(): void;
+    /**
+     * Get the shape set declarations for a stream.
+     * Shape sets are declared at stream begin and contain metadata about the
+     * shape values that will be included in audio frames.
+     *
+     * @param streamId - The stream ID to get shape sets for
+     * @returns Array of shape set declarations, or null if stream not found
+     */
+    getShapeSets(streamId: string): {
+        SetId: number;
+        Name: string;
+        ShapeNames: string[];
+    }[] | null;
+    /**
+     * Get the current shape set values for a named shape set on a stream.
+     * Values are updated with each audio frame and represent the latest
+     * analysis results (e.g., viseme data).
+     *
+     * @param streamId - The stream ID to get values for
+     * @param setName - The name of the shape set (e.g., "Viseme")
+     * @returns Array of float values, or null if not found
+     */
+    getShapeSetValues(streamId: string, setName: string): number[] | null;
+    /**
+     * Get the current shape set values by set ID for a stream.
+     *
+     * @param streamId - The stream ID to get values for
+     * @param setId - The numeric ID of the shape set
+     * @returns Array of float values, or null if not found
+     */
+    getShapeSetValuesById(streamId: string, setId: number): number[] | null;
+    /**
+     * Get all current shape set values for a stream.
+     *
+     * @param streamId - The stream ID to get values for
+     * @returns Map of set ID to values, or null if stream not found
+     */
+    getAllShapeSetValues(streamId: string): Map<number, number[]> | null;
+    /**
+     * Get current viseme values from any active stream.
+     * Uses sample-based timing for precise synchronization with audio playback.
+     * Interpolates between viseme keyframes for smooth animation.
+     *
+     * @returns Object with mouthOpenY (0-1) and mouthForm (-1 to +1), or null if no viseme data
+     */
+    getCurrentVisemeValues(): {
+        mouthOpenY: number;
+        mouthForm: number;
+    } | null;
     private resolveDiagnosticsConfig;
     private resolveBackgroundConfig;
     private ensureAudioGraph;

package/media/ikon-video-capture.d.ts

@@ -28,5 +28,4 @@ export declare class IkonVideoCapture {
     startScreen(options?: Omit<IkonVideoCaptureRequest, 'source'>): Promise<IkonVideoCaptureHandle>;
     stop(captureId: string): Promise<boolean>;
     dispose(): void;
-    private pumpFrames;
 }

package/media/ikon-video-playback.d.ts

@@ -46,7 +46,6 @@ export declare class IkonVideoPlayback {
      */
     dispose(): void;
     private canUseOffscreenCanvas;
-    private hasDemand;
     private ensureWorker;
     private onWorkerMessage;
     private syncAllStreamsToWorker;
@@ -55,7 +54,6 @@ export declare class IkonVideoPlayback {
     private parseStreamId;
     private detachProtocolPort;
     private ensureProtocolPort;
-    private maybeStopWorker;
     private stopWorker;
     private ensureStateSubscription;
 }

package/media/video-constants.d.ts

@@ -0,0 +1,42 @@
+export declare const VIDEO_CAPTURE_DEFAULT_WIDTH = 1280;
+export declare const VIDEO_CAPTURE_DEFAULT_HEIGHT = 720;
+export declare const VIDEO_CAPTURE_DEFAULT_FRAMERATE = 30;
+export declare const VIDEO_CAPTURE_DEFAULT_KEY_FRAME_INTERVAL: number;
+export declare const VIDEO_CODEC_STRINGS_H264: string[];
+export declare const VIDEO_CODEC_STRING_H264: string;
+export declare const VIDEO_CODEC_STRING_VP8 = "vp8";
+export declare const VIDEO_CODEC_STRINGS_VP9: string[];
+export declare const VIDEO_CODEC_STRING_VP9: string;
+export declare const VIDEO_CODEC_STRINGS_AV1: string[];
+export declare const VIDEO_CODEC_STRING_AV1: string;
+export declare const VIDEO_DECODE_CODEC_H264 = "avc1.42E01E";
+export declare const VIDEO_DECODE_CODEC_VP8 = "vp8";
+export declare const VIDEO_DECODE_CODEC_VP9 = "vp09.00.10.08";
+export declare const VIDEO_DECODE_CODEC_AV1 = "av01.0.04M.08";
+export declare const VIDEO_CAPTURE_DEFAULT_CODEC_PREFERENCES: readonly ('h264' | 'vp8' | 'vp9' | 'av1')[];
+export declare const VIDEO_RESOLUTION_FALLBACKS: readonly [{
+    readonly width: 1920;
+    readonly height: 1080;
+}, {
+    readonly width: 1280;
+    readonly height: 720;
+}, {
+    readonly width: 960;
+    readonly height: 540;
+}, {
+    readonly width: 640;
+    readonly height: 480;
+}, {
+    readonly width: 640;
+    readonly height: 360;
+}];
+export declare const VIDEO_CAPTURE_BITS_PER_PIXEL = 0.1;
+export declare const VIDEO_CAPTURE_BITRATE_FRAMERATE_BASELINE = 30;
+export declare const VIDEO_CAPTURE_MIN_BITRATE = 50000;
+export declare const VIDEO_CAPTURE_DEFAULT_BITRATE_MIN = 500000;
+export declare const VIDEO_CAPTURE_DEFAULT_BITRATE_MAX = 8000000;
+export declare const VIDEO_ENCODER_MAX_POOLED_BUFFERS = 8;
+export declare const VIDEO_ENCODER_MAX_QUEUE_DEPTH = 3;
+export declare const VIDEO_DECODER_MAX_PENDING_DECODES = 3;
+export declare const IMAGE_CAPTURE_DEFAULT_MIME_TYPE = "image/jpeg";
+export declare const IMAGE_CAPTURE_DEFAULT_JPEG_QUALITY = 0.92;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ikonai/sdk",
-  "version": "0.0.41",
+  "version": "0.0.43",
   "type": "module",
   "main": "./index.js",
   "types": "./index.d.ts",

package/utils/platform.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Detect if running on iOS (iPhone, iPad, iPod).
+ * Also detects iPadOS which reports as "Macintosh" but has touch support.
+ */
+export declare const isIOS: () => boolean;

package/worker/audio-playback-worker.d.ts

@@ -1,3 +1,4 @@
+import { AudioShapeSetValues, AudioShapeSet } from '../../../../shared/protocol/src/index.ts';
 export type AudioPlaybackWorkerTransport = 'sab' | 'pcm';
 export type AudioPlaybackWorkerConfigureMessage = {
     type: 'configure';
@@ -24,6 +25,7 @@ export type AudioPlaybackWorkerOut = {
     type: 'streamBegin';
     streamId: string;
     sharedArrayBuffer?: SharedArrayBuffer;
+    shapeSets?: AudioShapeSet[];
 } | {
     type: 'streamEnd';
     streamId: string;
@@ -48,4 +50,5 @@ export type AudioPlaybackWorkerOut = {
     isFirst: boolean;
     isLast: boolean;
     jitterMs: number;
+    shapeSetValues?: AudioShapeSetValues[];
 };

package/worker/video-capture-worker.d.ts

@@ -1,4 +1,5 @@
-type CodecPreference = 'h264' | 'vp8' | 'hevc';
+type CodecPreference = 'h264' | 'vp8' | 'vp9' | 'av1';
+type HardwareAcceleration = 'preferHardware' | 'preferSoftware';
 export type VideoCaptureWorkerOptions = {
     preferredCodecs?: CodecPreference[];
     width?: number;
@@ -6,5 +7,6 @@ export type VideoCaptureWorkerOptions = {
     framerate?: number;
     bitrate?: number;
     keyFrameIntervalFrames?: number;
+    hardwareAcceleration?: HardwareAcceleration;
 };
 export {};

package/worker/video-playback-worker.d.ts

@@ -56,4 +56,7 @@ export type VideoPlaybackWorkerOut = {
         message: string;
         stack?: string;
     };
+} | {
+    type: 'log';
+    message: string;
 };