@ikonai/sdk 0.0.11 → 0.0.13

package/channel/channel-manager.d.ts

@@ -27,7 +27,17 @@ export interface ChannelManagerConfig {
     /**
      * Endpoint selector for ordering types.
      */
-    endpointSelector: EndpointSelector;
+    endpointSelector?: EndpointSelector;
+    /**
+     * Ordered list of endpoint types to try.
+     * Useful for worker environments where localStorage might be unavailable.
+     */
+    orderedEndpointTypes?: EntrypointType[];
+    /**
+     * Called when a working endpoint type is discovered.
+     * Typically used by the host to persist endpoint preference.
+     */
+    onRememberWorkingType?: (type: EntrypointType) => void;
     /**
      * Callback when a protocol message is received from any channel.
      */
@@ -47,7 +57,7 @@ export interface ChannelManagerConfig {
  * Features:
  * - Groups entrypoints by type
  * - Tries types in order (WT → WS → WTProxy → WSProxy for prod)
- * - Connects core channel first, then others in parallel
+ * - Connects all channels in parallel
  * - Handles reconnection with type fallback
  * - Routes messages to/from appropriate channels
  */
@@ -88,11 +98,11 @@ export declare class ChannelManager {
     sendToAll(message: ProtocolMessage): void;
     /**
      * Internal connect implementation.
+     * @param isReconnect - If true, don't set offline state on failure (let attemptReconnect handle it)
      */
     private connectInternal;
     /**
-     * Connect all channels for a given endpoint type.
-     * Core channel connects first, then others in parallel.
+     * Connect all channels for a given endpoint type in parallel.
      */
     private connectAllChannels;
     /**

package/client/ikon-client-config.d.ts

@@ -1,6 +1,8 @@
 import { ProtocolMessage } from '../../../../shared/generated/src/index.ts';
 import { BackendType } from '../connection/urls';
 import { ConnectionState } from './connection-state';
+import { IkonAudioOutputConfig, IkonAudioPlaybackConfig } from '../media/ikon-audio-playback';
+import { IkonVideoPlaybackConfig } from '../media/ikon-video-playback';
 export type { BackendType };
 /**
  * Common connection options shared across all modes.
@@ -144,6 +146,21 @@ export interface SessionTokenConfig extends CloudConnectionConfig {
      */
     backendType?: BackendType;
 }
+/**
+ * Debug configuration for the SDK.
+ */
+export interface DebugConfig {
+    /**
+     * Log sent protocol messages.
+     * Format: "Sent {OPCODE_NAME} {senderId} -> {targets}"
+     */
+    logSentMessages?: boolean;
+    /**
+     * Log received protocol messages.
+     * Format: "Received {senderId} -> {OPCODE_NAME}"
+     */
+    logReceivedMessages?: boolean;
+}
 /**
  * Timeout configuration for the SDK.
  */
@@ -199,6 +216,81 @@ export interface IkonClientConfig {
      * Timeout configuration.
      */
     timeouts?: TimeoutConfig;
+    /**
+     * Debug configuration.
+     */
+    debug?: DebugConfig;
+    /**
+     * Threading and worker configuration.
+     */
+    threading?: {
+        /**
+         * Move protocol send/receive to a WebWorker when possible.
+         * - 'auto': try to use a worker and fall back to main thread on failure
+         * - 'disabled': always use main thread transports
+         */
+        protocolWorker?: 'auto' | 'disabled';
+    };
+    /**
+     * Media (audio/video) playback configuration.
+     *
+     * The SDK treats protocol messages as the canonical representation of streams.
+     * This section controls whether the SDK should create and prepare media pipelines
+     * (workers, decoders, AudioWorklet) to reduce time-to-first-audio/video.
+     */
+    media?: {
+        /**
+         * Audio playback pipeline.
+         */
+        audio?: {
+            /**
+             * When enabled, the SDK prepares the audio pipeline and starts decoding as soon as possible.
+             *
+             * Audio output still depends on browser autoplay policies; the app must ensure a user gesture
+             * has occurred before resuming the AudioContext. The SDK surfaces this need via `onAudioUnlockRequired`.
+             */
+            enabled?: boolean;
+            /**
+             * Callback fired when audio playback requires a user gesture (autoplay restriction).
+             */
+            onAudioUnlockRequired?: () => void;
+            /**
+             * Output format for mixing and rendering.
+             */
+            output?: IkonAudioOutputConfig;
+            /**
+             * Threading and capability preferences for audio.
+             */
+            threading?: IkonAudioPlaybackConfig['threading'];
+        };
+        /**
+         * Video playback pipeline.
+         *
+         * Video rendering requires an explicit surface (canvas) provided by the app.
+         * The SDK can still prepare worker/decoder plumbing to reduce time-to-first-frame.
+         */
+        video?: {
+            enabled?: boolean;
+            threading?: IkonVideoPlaybackConfig['threading'];
+        };
+    };
+    /**
+     * Media capture configuration (microphone/camera/screen/image).
+     *
+     * The SDK does not render any UI. When browser policies require a user gesture
+     * to start capture, the SDK notifies the host app via `onUserGestureRequired`.
+     *
+     * The app should show its own overlay and call `request.resume()` directly
+     * from a user gesture handler (e.g. button click) so the browser treats the
+     * subsequent getUserMedia/getDisplayMedia call as user-initiated.
+     */
+    mediaCapture?: {
+        onUserGestureRequired?: (request: {
+            kind: 'microphone' | 'camera' | 'screen' | 'image';
+            resume: () => void;
+            cancel?: () => void;
+        }) => void;
+    };
     /**
      * Callback when a protocol message is received.
      * The SDK handles keepalive internally; other messages are passed to this callback.
@@ -220,12 +312,12 @@ export interface IkonClientConfig {
      */
     onVisibilityChange?: (isVisible: boolean) => void;
     /**
-     * Called when connected to the server (including reconnects).
+     * Called when all endpoints are connected and GlobalState is received (including reconnects).
      * Perform any initialization here. After this callback completes,
      * the SDK sends ClientReady to the server.
      * Can be async (return a Promise).
      */
-    onReady?: () => void | Promise<void>;
+    onJoined?: () => void | Promise<void>;
     /**
      * Disable auto-registration of browser convenience functions.
      * By default, functions like getTheme, setTheme, getLocation, etc. are

package/client/ikon-client.d.ts

@@ -1,11 +1,52 @@
 import { GlobalState, ProtocolMessage } from '../../../../shared/generated/src/index.ts';
 import { FunctionRegistry } from '../functions/function-registry';
+import { IkonMedia } from '../media/ikon-media';
+import { IkonMediaCapture } from '../media/ikon-media-capture';
+import { UserGestureRequest } from '../utils/user-gesture';
 import { ConnectionState } from './connection-state';
 import { IkonClientConfig } from './ikon-client-config';
 /**
  * Protocol message handler callback type for subscribers.
  */
 export type ProtocolMessageHandler = (message: ProtocolMessage) => void;
+/**
+ * A MessagePort that receives raw protocol messages as transferred ArrayBuffers.
+ *
+ * This is intended for advanced integrations where decoding and processing should
+ * happen off the UI thread (e.g. media decode workers). The messages are still the
+ * canonical Teleport protocol payloads.
+ */
+export interface ProtocolMessagePort {
+    port: MessagePort;
+    close: () => void;
+}
+/**
+ * A MessagePort that accepts outbound protocol messages as transferred ArrayBuffers.
+ *
+ * This enables high-throughput producers (e.g. media capture encode workers) to send
+ * protocol messages without bouncing through the main thread.
+ */
+export interface ProtocolSendPort {
+    port: MessagePort;
+    close: () => void;
+}
+export interface ProtocolMessageSubscriptionOptions {
+    /**
+     * Bitmask of opcode groups to receive for this subscriber.
+     *
+     * This exists to keep protocol messages first-class while still allowing
+     * high-throughput clients to reduce work (and message copies) when only a
+     * subset of the protocol is relevant to the current app state.
+     */
+    opcodeGroupsMask?: number;
+    /**
+     * Optional explicit opcode list for this subscriber.
+     *
+     * When provided, the SDK falls back to forwarding all messages from the protocol worker,
+     * because the worker only filters by opcode group to keep routing cheap.
+     */
+    opcodes?: number[];
+}
 /**
  * State handler callback type for state subscribers.
  */
@@ -39,6 +80,8 @@ export type StateHandler = (state: ConnectionState) => void;
  */
 export declare class IkonClient {
     private channelManager;
+    private protocolWorker;
+    private workerManagerState;
     private authResponse;
     private currentState;
     private slowConnectionTimer;
@@ -54,10 +97,32 @@ export declare class IkonClient {
     private boundPageHide;
     private boundVisibilityChange;
     private _globalState;
-    private _initialGlobalStateReceived;
+    private _globalStateReceived;
+    private _channelsConnected;
+    private _joinedHandled;
     private readonly _functionRegistry;
     private unregisterBrowserFunctions;
+    private unregisterMediaCaptureFunctions;
+    private nextProtocolPortId;
+    private nextProtocolSendPortId;
+    private readonly _media;
+    private _mediaCapture;
+    /**
+     * Optional SDK-provided media pipelines (audio/video).
+     *
+     * These are only created when `IkonClientConfig.media` is provided. Keeping media
+     * optional avoids pulling in extra worker graphs and Web APIs for apps that only
+     * need protocol + UI streaming.
+     */
+    get media(): IkonMedia | null;
+    /**
+     * Optional SDK-provided media capture pipelines (audio/video/image).
+     *
+     * These are only available in browser environments.
+     */
+    get mediaCapture(): IkonMediaCapture | null;
     constructor(config: IkonClientConfig);
+    requestUserGesture(request: UserGestureRequest): void;
     private _lastError;
     /**
      * Get the error that caused 'offlineError' state, if any.
@@ -98,11 +163,37 @@ export declare class IkonClient {
      * Routes to appropriate channel based on the message's opcode group.
      */
     sendProtocolMessage(message: ProtocolMessage): void;
+    /**
+     * Create a MessagePort that receives raw protocol message buffers.
+     *
+     * This is primarily used to build high-throughput pipelines (audio/video/UI)
+     * where the heavy work is done in dedicated workers, while keeping the protocol
+     * messages themselves as the source of truth.
+     *
+     * If the client is using a protocol worker, the port is attached directly to it.
+     * Otherwise, the SDK forwards matching messages from the main thread as a fallback.
+     */
+    createProtocolMessagePort(options: {
+        opcodeGroupsMask: number;
+    }): ProtocolMessagePort | null;
+    /**
+     * Create a MessagePort that can send raw protocol messages as transferred ArrayBuffers.
+     *
+     * This is primarily used by worker-based encoders that need to stream large amounts of
+     * data (audio/video frames) while keeping protocol messages as the canonical payload.
+     *
+     * When the SDK is using a protocol worker, the port is attached directly to it.
+     * Otherwise, the SDK listens on the port on the main thread and forwards messages to
+     * the active channel manager.
+     */
+    createProtocolSendPort(options: {
+        opcodeGroupsMask: number;
+    }): ProtocolSendPort | null;
     /**
      * Subscribe to protocol messages.
      * Returns an unsubscribe function.
      */
-    subscribeToProtocolMessages(handler: ProtocolMessageHandler): () => void;
+    subscribeToProtocolMessages(handler: ProtocolMessageHandler, options?: ProtocolMessageSubscriptionOptions): () => void;
     /**
      * Subscribe to connection state changes.
      * Handler is called immediately with current state, then on every change.
@@ -134,6 +225,11 @@ export declare class IkonClient {
      * Use this for unrecoverable SDK internal errors.
      */
     private setErrorState;
+    private connectProtocol;
+    private connectProtocolOnMainThread;
+    private disconnectProtocol;
+    private computeOrderedEndpointTypes;
+    private updateWorkerInterestMask;
     /**
      * Clear all timers.
      */
@@ -167,8 +263,12 @@ export declare class IkonClient {
      */
     private checkConnectionHealth;
     /**
-     * Handle connection ready - attach function registry, call user's onReady callback, then send ClientReady.
+     * Try to handle joined state - only proceeds if both channels are connected and GlobalState is received.
+     */
+    private tryHandleJoined;
+    /**
+     * Handle joined state - attach function registry, call user's onJoined callback, then send ClientReady.
      * Called on every connect (including reconnects).
      */
-    private handleReady;
+    private handleJoined;
 }

package/connection/urls.d.ts

@@ -7,6 +7,11 @@
 export type BackendType = 'production' | 'development';
 export declare const IKON_BACKEND_URL_PROD = "https://api.prod.ikon.live";
 export declare const IKON_BACKEND_URL_DEV = "https://api.dev.ikon.live";
+export declare const IKON_AUTH_URL_PROD = "https://auth.ikonai.com";
+export declare const IKON_AUTH_URL_DEV = "https://auth.dev.ikonai.com";
+/**
+ * @deprecated Use deriveAuthUrl() instead
+ */
 export declare const IKON_AUTH_BASE_URL = "https://auth.ikonai.com";
 /**
  * Get backend URL from BackendType.
@@ -25,6 +30,14 @@ export declare function deriveBackendUrl(): string;
  * - Otherwise → 'production'
  */
 export declare function deriveBackendType(): BackendType;
+/**
+ * Derive auth service URL from environment or window.location.hostname.
+ * - VITE_IKON_AUTH_URL env var takes precedence (for local development)
+ * - localhost/127.0.0.1 → http://localhost:3400
+ * - hostname contains '.dev.' → IKON_AUTH_URL_DEV
+ * - Otherwise → IKON_AUTH_URL_PROD
+ */
+export declare function deriveAuthUrl(): string;
 /**
  * Parse URL query parameters.
  */

package/functions/function-registry.d.ts

@@ -44,7 +44,7 @@ export declare class FunctionRegistry {
      */
     handleProtocolMessage(message: ProtocolMessage, opcode: number): boolean;
     /**
-     * Send registrations for all functions.
+     * Send registrations for all functions as a batch.
      * Called when connection is established or re-established.
      */
     private sendAllRegistrations;

package/functions/index.d.ts

@@ -1,3 +1,4 @@
-export type { ValueDescriptor, NormalizedValueDescriptor, PrimitiveValueKind, FunctionParameterDefinition, FunctionDefinition, FunctionHandler, } from './types';
+export type { ValueDescriptor, NormalizedValueDescriptor, PrimitiveValueKind, FunctionParameterDefinition, FunctionDefinition, FunctionHandler, FunctionResultWithData, } from './types';
+export { withResultData } from './types';
 export { FunctionRegistry, type FunctionRegistryConfig } from './function-registry';
 export { registerBrowserFunctions, isBrowserEnvironment } from './browser-functions';

package/functions/media-capture-functions.d.ts

@@ -0,0 +1,9 @@
+import { IkonClient } from '../client/ikon-client';
+import { FunctionRegistry } from './function-registry';
+export declare const MediaCaptureFunctionNames: {
+    readonly StartVideoCapture: "ikon.client.startVideoCapture";
+    readonly StartAudioCapture: "ikon.client.startAudioCapture";
+    readonly StopCapture: "ikon.client.stopCapture";
+    readonly CaptureImage: "ikon.client.captureImage";
+};
+export declare function registerMediaCaptureFunctions(registry: FunctionRegistry, client: IkonClient): () => void;

package/functions/types.d.ts

@@ -45,6 +45,11 @@ export interface FunctionDefinition {
     parameters?: FunctionParameterDefinition[];
 }
 export type FunctionHandler = (...parameters: unknown[]) => unknown | Promise<unknown>;
+export interface FunctionResultWithData<T = unknown> {
+    value: T;
+    data: Uint8Array;
+}
+export declare function withResultData<T>(value: T, data: Uint8Array): FunctionResultWithData<T>;
 export interface NormalizedFunctionParameter {
     name: string;
     descriptor: NormalizedValueDescriptor;

package/index.d.ts

@@ -1,5 +1,5 @@
-export { IkonClient, type ProtocolMessageHandler, type StateHandler } from './client/ikon-client';
-export type { IkonClientConfig, LocalConfig, ApiKeyConfig, SessionTokenConfig, CloudConnectionConfig, CommonConnectionConfig, TimeoutConfig, BackendType } from './client/ikon-client-config';
+export { IkonClient, type ProtocolMessageHandler, type StateHandler, type ProtocolMessageSubscriptionOptions, type ProtocolMessagePort, type ProtocolSendPort, } from './client/ikon-client';
+export type { IkonClientConfig, LocalConfig, ApiKeyConfig, SessionTokenConfig, CloudConnectionConfig, CommonConnectionConfig, TimeoutConfig, DebugConfig, BackendType } from './client/ikon-client-config';
 export type { ConnectionState } from './client/connection-state';
 export { isConnecting, isConnected, isOffline, isError } from './client/connection-state';
 export { ChannelManager, type ChannelManagerConfig, type ChannelManagerState } from './channel/channel-manager';
@@ -12,9 +12,13 @@ export { teleportReadOpcode as readOpcode, teleportReadOpcodeGroup as readOpcode
 export { isWebTransportSupported } from './transport/web-transport-transport';
 export { setLogLevel, setLogSink, getLogLevel, getLogSink, createLogger, LogLevel } from './utils/logger';
 export type { LogEntry, LogSink, Logger } from './utils/logger';
+export { getOpcodeName } from './utils/opcode-names';
 export { initializeLogSink, setSendLogsCallback, getBufferedLogs, takeBufferedLogs, flushLogs, clearLogBuffer, getLogBufferSize } from './utils/logSink';
 export type { LogSinkConfig } from './utils/logSink';
 export { FunctionRegistry, registerBrowserFunctions, isBrowserEnvironment, type FunctionRegistryConfig } from './functions';
-export type { ValueDescriptor, NormalizedValueDescriptor, PrimitiveValueKind, FunctionParameterDefinition, FunctionDefinition, FunctionHandler } from './functions';
-export { IKON_AUTH_BASE_URL, IKON_BACKEND_URL_PROD, IKON_BACKEND_URL_DEV, deriveBackendUrl, deriveBackendType, parseUrlParams, isCloudEnvironment, type UrlParams } from './connection/urls';
+export { withResultData } from './functions';
+export type { ValueDescriptor, NormalizedValueDescriptor, PrimitiveValueKind, FunctionParameterDefinition, FunctionDefinition, FunctionHandler, FunctionResultWithData, } from './functions';
+export { IKON_AUTH_BASE_URL, IKON_AUTH_URL_PROD, IKON_AUTH_URL_DEV, IKON_BACKEND_URL_PROD, IKON_BACKEND_URL_DEV, deriveAuthUrl, deriveBackendUrl, deriveBackendType, parseUrlParams, isCloudEnvironment, type UrlParams } from './connection/urls';
 export { getOrCreateDeviceId, clearDeviceId, extractUserIdFromToken } from './storage';
+export { IkonMedia, IkonAudioPlayback, IkonVideoPlayback, isSharedArrayBufferSupported, isAudioWorkletSupported, IkonMediaCapture, IkonVideoCapture, IkonAudioCapture, IkonImageCapture, } from './media';
+export type { IkonMediaConfig, IkonAudioPlaybackConfig, IkonAudioOutputConfig, IkonVideoPlaybackConfig, IkonVideoCaptureHandle, IkonVideoCaptureRequest, VideoCaptureSource, IkonAudioCaptureHandle, IkonAudioCaptureRequest, IkonImageCaptureRequest, IkonImageCaptureResult, } from './media';