@ikonai/sdk 0.0.11 → 0.0.13

package/media/capabilities.d.ts

@@ -0,0 +1,13 @@
+/**
+ * SharedArrayBuffer is the most efficient way to move continuous PCM/video data between threads.
+ *
+ * Browsers gate it behind cross-origin isolation (COOP/COEP) to prevent side-channel attacks.
+ * This helper centralizes the runtime checks so pipelines can prefer SAB but always fall back.
+ */
+export declare function isSharedArrayBufferSupported(): boolean;
+/**
+ * AudioWorklet is the preferred audio rendering path because it runs on the browser's audio thread.
+ *
+ * When unavailable (older browsers or restrictive environments), the SDK must fall back to main-thread audio.
+ */
+export declare function isAudioWorkletSupported(ctx: BaseAudioContext): boolean;

package/media/ikon-audio-capture.d.ts

@@ -0,0 +1,34 @@
+import { IkonClient } from '../client/ikon-client';
+export interface IkonAudioCaptureRequest {
+    userGesture?: boolean;
+    constraints?: MediaTrackConstraints;
+    options?: {
+        bitrate?: number;
+        frameDurationMs?: number;
+    };
+}
+export interface IkonAudioCaptureHandle {
+    captureId: string;
+    stop(): Promise<void>;
+}
+export declare class IkonAudioCapture {
+    private readonly client;
+    private worker;
+    private sendPort;
+    private workletModuleUrl;
+    private workletReady;
+    private workletContext;
+    private audioContext;
+    private nextTrackId;
+    private captures;
+    constructor(client: IkonClient);
+    private allocateTrackId;
+    private ensureWorker;
+    private ensureSendPort;
+    private ensureAudioContext;
+    private acquireMicrophoneStream;
+    private ensureWorklet;
+    startMic(request?: IkonAudioCaptureRequest): Promise<IkonAudioCaptureHandle>;
+    stop(captureId: string): Promise<boolean>;
+    dispose(): void;
+}

package/media/ikon-audio-playback.d.ts

@@ -0,0 +1,157 @@
+import { IkonClient } from '../client/ikon-client';
+export interface IkonAudioOutputConfig {
+    /**
+     * Target output sample rate.
+     *
+     * The browser ultimately controls the AudioContext sample rate, so this is best-effort.
+     * The decode worker resamples into the actual output rate used by the playback graph.
+     */
+    sampleRate?: number;
+    /**
+     * Target output channel count.
+     *
+     * This is used to normalize decoded streams into a single mixing format.
+     */
+    channels?: number;
+    /**
+     * Startup buffer size (seconds).
+     *
+     * This is the minimum buffered audio required before playback begins after a reset.
+     */
+    minBufferSizeSeconds?: number;
+    /**
+     * Maximum buffered audio (seconds).
+     *
+     * Larger buffers improve robustness under jitter but increase latency.
+     */
+    maxBufferSizeSeconds?: number;
+}
+export interface IkonAudioPlaybackConfig {
+    /**
+     * Output format used by the mixing graph.
+     */
+    output?: IkonAudioOutputConfig;
+    /**
+     * Called when the browser requires a user gesture to start audio playback.
+     */
+    onAudioUnlockRequired?: () => void;
+    /**
+     * Worker and transport preferences.
+     *
+     * The SDK prefers:
+     * - decode in a worker
+     * - audio rendering in an AudioWorklet
+     * - PCM transfer through SharedArrayBuffer when cross-origin isolated
+     *
+     * All of these have fallbacks for restrictive embed environments.
+     */
+    threading?: {
+        decodeWorker?: 'auto' | 'disabled';
+        preferSharedArrayBuffer?: boolean;
+        preferAudioWorklet?: boolean;
+        preferWebCodecs?: boolean;
+    };
+}
+/**
+ * Audio playback pipeline for Ikon protocol audio streams.
+ *
+ * The key separation of responsibilities is:
+ * - protocol I/O: handled by IkonClient (optionally in ProtocolWorker)
+ * - decoding: done in a dedicated AudioWorker
+ * - rendering: done in AudioWorklet when available
+ *
+ * This keeps frequent audio frames off the UI thread and avoids forcing the SDK
+ * to invent a parallel “audio message model” alongside the protocol.
+ */
+export declare class IkonAudioPlayback {
+    private readonly client;
+    private readonly config;
+    private enabled;
+    private prepared;
+    private stateUnsubscribe;
+    private unlockRequiredSignaled;
+    private readonly isIos;
+    private recreateOnNextResume;
+    private boundVisibilityChange;
+    private unlockHandler;
+    private awaitingUnlock;
+    private recoveryTask;
+    /** Timeout for AudioContext.resume() - normally resolves instantly, but some browsers hang indefinitely */
+    private static readonly RESUME_TIMEOUT_MS;
+    private audioContext;
+    private audioWorkletNode;
+    private scriptProcessorNode;
+    private workletModuleUrl;
+    private workletReady;
+    private pendingWorkletMessages;
+    private audioWorker;
+    private protocolPort;
+    private readonly fallbackQueues;
+    private readonly activeStreams;
+    constructor(client: IkonClient, config?: IkonAudioPlaybackConfig);
+    /**
+     * Prepare the audio pipeline without attempting to start playback.
+     *
+     * This is meant to reduce time-to-first-audio by:
+     * - creating the AudioContext (it may remain suspended)
+     * - loading the AudioWorklet module (if supported)
+     * - starting the decode worker when the client is connected
+     *
+     * Prewarming does not trigger user-gesture prompts and does not call resume().
+     */
+    prepare(): Promise<void>;
+    /**
+     * Enable or disable audio playback.
+     *
+     * Browsers may require a user gesture to start audio. When that happens, the SDK
+     * keeps the decode pipeline alive but the audio graph remains suspended until
+     * the app calls `enable({ userGesture: true })` or resumes the AudioContext.
+     */
+    setEnabled(enabled: boolean, options?: {
+        userGesture?: boolean;
+    }): Promise<void>;
+    /**
+     * Convenience wrapper for `setEnabled(true)`.
+     */
+    enable(options?: {
+        userGesture?: boolean;
+    }): Promise<void>;
+    /**
+     * Convenience wrapper for `setEnabled(false)`.
+     */
+    disable(): Promise<void>;
+    /**
+     * Dispose of resources. After calling this, the instance should not be reused.
+     */
+    dispose(): void;
+    private resolveOutputConfig;
+    private ensureAudioGraph;
+    private attachAudioContextHealthHandlers;
+    private attachVisibilityHandler;
+    private removeVisibilityHandler;
+    private getContextState;
+    private addUnlockHandler;
+    private removeUnlockHandler;
+    /**
+     * Wraps AudioContext.resume() with a timeout to prevent indefinite hanging.
+     * Some browsers return a Promise that never resolves in certain edge cases.
+     */
+    private resumeWithTimeout;
+    private signalUnlockRequired;
+    private ensureAudioWorklet;
+    private ensureScriptProcessor;
+    private readFromFallbackQueue;
+    private resumeAudioContextIfPossible;
+    private recoverAudioContext;
+    private recoverAudioContextInternal;
+    private recreateAudioGraph;
+    private rebindWorkerTransportIfNeeded;
+    private ensureWorkers;
+    private onAudioWorkerMessage;
+    private postToWorklet;
+    private stopAudioWorklet;
+    private stopScriptProcessor;
+    private stopAudioGraph;
+    private stopWorkers;
+    private ensureStateSubscription;
+}

package/media/ikon-image-capture.d.ts

@@ -0,0 +1,24 @@
+export interface IkonImageCaptureRequest {
+    userGesture?: boolean;
+    constraints?: MediaTrackConstraints;
+    mime?: 'image/jpeg' | 'image/png';
+    quality?: number;
+    width?: number;
+    height?: number;
+}
+export interface IkonImageCaptureResult {
+    mime: string;
+    width: number;
+    height: number;
+    data: Uint8Array;
+}
+export declare class IkonImageCapture {
+    private readonly requestUserGesture;
+    constructor(requestUserGesture: (req: {
+        kind: 'image';
+        resume: () => void;
+        cancel?: () => void;
+    }) => void);
+    private acquireCameraStream;
+    captureFromCamera(request?: IkonImageCaptureRequest): Promise<IkonImageCaptureResult>;
+}

package/media/ikon-media-capture.d.ts

@@ -0,0 +1,11 @@
+import { IkonClient } from '../client/ikon-client';
+import { IkonAudioCapture } from './ikon-audio-capture';
+import { IkonImageCapture } from './ikon-image-capture';
+import { IkonVideoCapture } from './ikon-video-capture';
+export declare class IkonMediaCapture {
+    readonly audio: IkonAudioCapture;
+    readonly video: IkonVideoCapture;
+    readonly image: IkonImageCapture;
+    constructor(client: IkonClient);
+    dispose(): void;
+}

package/media/ikon-media.d.ts

@@ -0,0 +1,27 @@
+import { IkonClient } from '../client/ikon-client';
+import { IkonAudioPlayback, IkonAudioPlaybackConfig } from './ikon-audio-playback';
+import { IkonVideoPlayback, IkonVideoPlaybackConfig } from './ikon-video-playback';
+export interface IkonMediaConfig {
+    audio?: IkonAudioPlaybackConfig;
+    video?: IkonVideoPlaybackConfig;
+}
+/**
+ * Convenience wrapper for SDK-provided media pipelines.
+ *
+ * The SDK keeps protocol messages as the source of truth and moves heavy work
+ * (decode, parsing, buffering) into dedicated workers when possible.
+ *
+ * This wrapper is optional; applications can instantiate `IkonAudioPlayback` and
+ * `IkonVideoPlayback` directly when they want tighter control.
+ */
+export declare class IkonMedia {
+    readonly audio: IkonAudioPlayback;
+    readonly video: IkonVideoPlayback;
+    constructor(client: IkonClient, config?: IkonMediaConfig);
+    /**
+     * Releases media resources (workers, audio graph, decoders).
+     *
+     * This does not disconnect the underlying `IkonClient`.
+     */
+    dispose(): void;
+}

package/media/ikon-video-capture.d.ts

@@ -0,0 +1,33 @@
+import { IkonClient } from '../client/ikon-client';
+import { VideoCaptureWorkerOptions } from '../worker/video-capture-worker';
+export type VideoCaptureSource = 'camera' | 'screen';
+export interface IkonVideoCaptureRequest {
+    source: VideoCaptureSource;
+    options?: VideoCaptureWorkerOptions;
+    userGesture?: boolean;
+    constraints?: MediaTrackConstraints;
+}
+export interface IkonVideoCaptureHandle {
+    captureId: string;
+    stop(): Promise<void>;
+}
+export declare class IkonVideoCapture {
+    private readonly client;
+    private worker;
+    private protocolPort;
+    private sendPort;
+    private nextTrackId;
+    private captures;
+    constructor(client: IkonClient);
+    private allocateTrackId;
+    private ensureWorker;
+    private ensurePorts;
+    private getMediaStream;
+    private acquireStreamWithGestureFallback;
+    start(request: IkonVideoCaptureRequest): Promise<IkonVideoCaptureHandle>;
+    startCamera(options?: Omit<IkonVideoCaptureRequest, 'source'>): Promise<IkonVideoCaptureHandle>;
+    startScreen(options?: Omit<IkonVideoCaptureRequest, 'source'>): Promise<IkonVideoCaptureHandle>;
+    stop(captureId: string): Promise<boolean>;
+    dispose(): void;
+    private pumpFrames;
+}

package/media/ikon-video-playback.d.ts

@@ -0,0 +1,70 @@
+import { IkonClient } from '../client/ikon-client';
+export interface IkonVideoPlaybackConfig {
+    /**
+     * Worker and rendering preferences.
+     *
+     * The preferred path is decode + render inside a dedicated worker using OffscreenCanvas,
+     * which avoids scheduling heavy work on the UI thread during high-frequency streaming.
+     */
+    threading?: {
+        decodeWorker?: 'auto' | 'disabled';
+        preferOffscreenCanvas?: boolean;
+    };
+}
+/**
+ * Video playback pipeline for Ikon protocol video streams.
+ *
+ * This implementation is intentionally canvas-based and renderer-agnostic.
+ * Applications decide how to position and size the canvas; the SDK focuses on
+ * decoding and presenting frames efficiently.
+ */
+export declare class IkonVideoPlayback {
+    private readonly client;
+    private readonly config;
+    private enabled;
+    private prepared;
+    private stateUnsubscribe;
+    private worker;
+    private protocolPort;
+    private readonly surfaces;
+    constructor(client: IkonClient, config?: IkonVideoPlaybackConfig);
+    /**
+     * Prepare the video pipeline. This can reduce time-to-first-frame by creating
+     * the worker and attaching the protocol subscription early.
+     *
+     * Video still requires an explicit canvas surface to actually present frames.
+     */
+    prepare(): Promise<void>;
+    /**
+     * Attach a canvas for a specific video stream id.
+     *
+     * When OffscreenCanvas is available, the SDK transfers the rendering surface to a worker
+     * so the UI thread only participates in layout and input handling.
+     */
+    attachCanvas(streamId: string, canvas: HTMLCanvasElement): Promise<void>;
+    /**
+     * Detach the canvas for the given stream id.
+     */
+    detachCanvas(streamId: string): void;
+    /**
+     * Enable or disable video playback.
+     */
+    setEnabled(enabled: boolean): Promise<void>;
+    /**
+     * Dispose of resources. After calling this, the instance should not be reused.
+     */
+    dispose(): void;
+    private canUseOffscreenCanvas;
+    private hasDemand;
+    private ensureWorker;
+    private onWorkerMessage;
+    private syncAllStreamsToWorker;
+    private syncStreamToWorker;
+    private requestIdrForStream;
+    private parseStreamId;
+    private detachProtocolPort;
+    private ensureProtocolPort;
+    private maybeStopWorker;
+    private stopWorker;
+    private ensureStateSubscription;
+}

package/media/index.d.ts

@@ -0,0 +1,14 @@
+export { IkonMedia } from './ikon-media';
+export type { IkonMediaConfig } from './ikon-media';
+export { IkonAudioPlayback } from './ikon-audio-playback';
+export type { IkonAudioPlaybackConfig, IkonAudioOutputConfig } from './ikon-audio-playback';
+export { IkonVideoPlayback } from './ikon-video-playback';
+export type { IkonVideoPlaybackConfig } from './ikon-video-playback';
+export { isSharedArrayBufferSupported, isAudioWorkletSupported } from './capabilities';
+export { IkonMediaCapture } from './ikon-media-capture';
+export { IkonVideoCapture } from './ikon-video-capture';
+export type { IkonVideoCaptureHandle, IkonVideoCaptureRequest, VideoCaptureSource } from './ikon-video-capture';
+export { IkonAudioCapture } from './ikon-audio-capture';
+export type { IkonAudioCaptureHandle, IkonAudioCaptureRequest } from './ikon-audio-capture';
+export { IkonImageCapture } from './ikon-image-capture';
+export type { IkonImageCaptureRequest, IkonImageCaptureResult } from './ikon-image-capture';

package/media/ring-buffer.d.ts

@@ -0,0 +1,21 @@
+/**
+ * SharedArrayBuffer-backed ring buffer for real-time PCM transport.
+ *
+ * The design goal is predictable, low-latency reads on the AudioWorklet thread.
+ * The ring buffer stores a fixed-size Float32 sample array plus atomic read/write indices.
+ */
+export declare class RingBuffer<TArray extends Float32Array = Float32Array> {
+    static getStorageForCapacity(capacity: number, type: typeof Float32Array): SharedArrayBuffer;
+    private readonly readWrite;
+    private readonly storage;
+    readonly buf: SharedArrayBuffer;
+    constructor(sharedArrayBuffer: SharedArrayBuffer, type: {
+        new (buffer: ArrayBufferLike, byteOffset: number, length: number): TArray;
+    });
+    get capacity(): number;
+    flush(): void;
+    availableRead(): number;
+    availableWrite(): number;
+    push(input: Float32Array): number;
+    pop(target: Float32Array): number;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ikonai/sdk",
-  "version": "0.0.11",
+  "version": "0.0.13",
   "type": "module",
   "main": "./index.js",
   "types": "./index.d.ts",

package/utils/deferred.d.ts

@@ -0,0 +1,6 @@
+export interface Deferred<T> {
+    promise: Promise<T>;
+    resolve: (value: T) => void;
+    reject: (error: Error) => void;
+}
+export declare function createDeferred<T>(): Deferred<T>;

package/utils/opcode-names.d.ts

@@ -0,0 +1 @@
+export declare function getOpcodeName(opcode: number): string;

package/utils/user-gesture.d.ts

@@ -0,0 +1,7 @@
+export type UserGestureKind = 'microphone' | 'camera' | 'screen' | 'image';
+export interface UserGestureRequest {
+    kind: UserGestureKind;
+    resume: () => void;
+    cancel?: () => void;
+}
+export declare function isPermissionOrGestureError(error: unknown): boolean;

package/worker/audio-capture-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/worker/audio-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/worker/protocol-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/worker/video-capture-worker.d.ts

@@ -0,0 +1,10 @@
+type CodecPreference = 'h264' | 'vp8' | 'hevc';
+export type VideoCaptureWorkerOptions = {
+    preferredCodecs?: CodecPreference[];
+    width?: number;
+    height?: number;
+    framerate?: number;
+    bitrate?: number;
+    keyFrameIntervalFrames?: number;
+};
+export {};

package/worker/video-worker.d.ts

@@ -0,0 +1 @@
+export {};