@interlucent/pixel-stream 0.0.78

package/README.md

@@ -0,0 +1,5 @@
+# @interlucent/pixel-stream
+
+Stream interactive applications to any web browser in real time.
+
+For documentation, visit [interlucent.ai](https://interlucent.ai).

package/dist/pixel-stream.d.ts

@@ -0,0 +1,553 @@
+import type { LogLevel } from '@interlucent/logging';
+
+declare interface AdmissionConfig {
+    /** Required token for session admission */
+    admissionToken: string;
+    /** Optional application identifier (overrides/supplements admission token) */
+    appId?: string;
+    /** Optional application version refinement */
+    appVersion?: string;
+}
+
+declare interface PerformanceMetrics {
+    sessionStarted: number | null;
+    tokenExchangeStart: number | null;
+    tokenExchangeComplete: number | null;
+    peerConnectionCreated: number | null;
+    sessionWsConnectStart: number | null;
+    sessionWsConnectComplete: number | null;
+    sessionReady: number | null;
+    jobRequested: number | null;
+    subscribeStart: number | null;
+    offerReceived: number | null;
+    answerSent: number | null;
+    iceConnected: number | null;
+    firstTrack: number | null;
+    firstFrame: number | null;
+    connectionComplete: number | null;
+}
+
+export declare class PixelStream extends HTMLElement {
+    /** Package version, injected at build time. */
+    static readonly VERSION: string;
+    /** Instance accessor for the package version. */
+    get version(): string;
+    /** Current configuration snapshot. */
+    get config(): PixelStreamConfig;
+    readonly performanceMetrics: PerformanceMetrics;
+    static get observedAttributes(): string[];
+    get nativeTouch(): boolean;
+    set nativeTouch(value: boolean);
+    get pointerLock(): boolean;
+    set pointerLock(value: boolean);
+    get pointerLockRelease(): boolean;
+    set pointerLockRelease(value: boolean);
+    get suppressBrowserKeys(): boolean;
+    set suppressBrowserKeys(value: boolean);
+    /**
+     * Debug mode - shows visual indicators for session connectivity.
+     * When enabled, displays a colored border/glow:
+     * - Gray: idle/no session
+     * - Yellow: connecting/authenticating/reconnecting
+     * - Green: connected
+     * - Red: error
+     */
+    get debug(): boolean;
+    set debug(value: boolean);
+    get logLevel(): LogLevel;
+    set logLevel(value: LogLevel);
+    /**
+     * ICE candidate batching delay in milliseconds.
+     * - 0ms for localhost/LAN (no batching overhead)
+     * - 10-20ms for regional networks
+     * - 50-100ms for high-latency or mobile networks
+     */
+    get iceBatchDelay(): number;
+    set iceBatchDelay(value: number);
+    /**
+     * Required token for session admission.
+     * Setting this triggers an 'admission-token-change' event.
+     */
+    get admissionToken(): string | null;
+    set admissionToken(value: string | null);
+    /**
+     * Optional application identifier (overrides/supplements admission token)
+     */
+    get appId(): string | null;
+    set appId(value: string | null);
+    /**
+     * Optional application version refinement
+     */
+    get appVersion(): string | null;
+    set appVersion(value: string | null);
+    /**
+     * Disable automatic session connection when admission token is set.
+     * By default, the component auto-connects. Set this to opt out and
+     * manage session lifecycle manually via `startSession()`.
+     */
+    get noAutoConnect(): boolean;
+    set noAutoConnect(value: boolean);
+    /**
+     * Mute audio output
+     */
+    get mute(): boolean;
+    set mute(value: boolean);
+    /**
+     * Audio output volume (0.0 silent – 1.0 full).
+     * Applies to both the video and dedicated audio elements.
+     */
+    get volume(): number;
+    set volume(value: number);
+    /**
+     * Milliseconds to wait for a peer streamer before giving up.
+     * If set and no streamer connects within this time, 'rendezvous-timeout' event is fired.
+     */
+    get rendezvousPreference(): number | null;
+    set rendezvousPreference(value: number | null);
+    /**
+     * Milliseconds to remain in session after all other agents leave.
+     * When the last peer leaves, a 'linger-started' event is fired and the timer begins.
+     * If no peer reconnects, 'linger-timeout' event is fired.
+     */
+    get lingerPreference(): number | null;
+    set lingerPreference(value: number | null);
+    /**
+     * Milliseconds grace period before this client appears as left to other agents.
+     * This allows for brief disconnections without triggering peer 'left' events.
+     */
+    get leftGracePeriod(): number | null;
+    set leftGracePeriod(value: number | null);
+    /**
+     * API endpoint for token exchange and WebSocket connections.
+     * Defaults to 'api.interlucent.ai'. Use 'api.dev.interlucent.ai' for dev stack.
+     */
+    get apiEndpoint(): string;
+    set apiEndpoint(value: string);
+    /**
+     * Reconnection mode: 'none', 'recover', or 'always'.
+     * - none: No automatic reconnection (default)
+     * - recover: Reconnect to the same session using existing access token
+     * - always: Always attempt reconnection (for future use)
+     */
+    get reconnectMode(): 'none' | 'recover' | 'always';
+    set reconnectMode(value: 'none' | 'recover' | 'always');
+    /**
+     * Maximum number of reconnection attempts.
+     * Use Infinity or -1 for unlimited attempts.
+     */
+    get reconnectAttempts(): number;
+    set reconnectAttempts(value: number);
+    /**
+     * Reconnection strategy: 'periodic' or 'exponential-backoff'.
+     */
+    get reconnectStrategy(): 'periodic' | 'exponential-backoff';
+    set reconnectStrategy(value: 'periodic' | 'exponential-backoff');
+    /**
+     * Reconnection interval in milliseconds (for periodic strategy).
+     */
+    get reconnectInterval(): number;
+    set reconnectInterval(value: number);
+    /**
+     * Grace period in milliseconds before treating a disconnected WebRTC connection as failed.
+     * During this period, RTP stats are polled to check if media is still flowing.
+     * Default: 9000ms (9 seconds)
+     */
+    get disconnectGraceMs(): number;
+    set disconnectGraceMs(value: number);
+    /**
+     * Resize mode - determines what messages are sent when viewport resizes.
+     * - 'none': No automatic resize messages sent
+     * - 'auto': Auto-detect best format based on UE version/response
+     * - 'pureweb': Pureweb plugin format (UIInteraction with action: '1', width, height)
+     * - 'pre-5.4': Legacy mode - sends UIInteraction with Console command (r.setres WxH)
+     * - '5.4+': Modern mode (default) - sends Command with Resolution.Width/Height
+     */
+    get resizeMode(): 'none' | 'auto' | 'pureweb' | 'pre-5.4' | '5.4+';
+    set resizeMode(value: 'none' | 'auto' | 'pureweb' | 'pre-5.4' | '5.4+');
+    /**
+     * DPR cap - limits devicePixelRatio used when computing resize dimensions.
+     * - 1 (default): Send CSS pixels to the game (matches Epic's PixelStreamingInfrastructure).
+     *   Games handle their own internal resolution scaling.
+     * - Infinity: Send physical device pixels (matches PureWeb SDK behavior).
+     *   Can cause issues on Retina displays (2x/3x) when games can't handle the resolution.
+     * - 1.5, 2, etc.: Allow some Retina benefit with a cap.
+     */
+    get dprCap(): number;
+    set dprCap(value: number);
+    /**
+     * Resolution clamp — caps device-pixel dimensions sent to the server.
+     * Accepts a named preset string (e.g. 'fhd', 'qhd') or a custom { maxWidth, maxHeight } object.
+     * Set to null to disable clamping.
+     */
+    get resolutionClamp(): StreamResolutionClamp | null;
+    set resolutionClamp(value: StreamResolutionClamp | ResolutionClampName | null);
+    /**
+     * Check if the current stream is detected as an XR/stereo stream.
+     * Detection is based on:
+     * - Multiple video tracks with XR-like labels (left/right/stereo)
+     * - Side-by-side stereo aspect ratio (>3.5:1)
+     * - Exactly 2 video tracks (stereo pair)
+     */
+    get isXRStream(): boolean;
+    /**
+     * Current session connection state.
+     * One of: 'idle', 'connecting', 'authenticating', 'connected', 'reconnecting', 'error'
+     */
+    get sessionState(): string;
+    /**
+     * Current WebRTC media pipeline state.
+     * One of: 'idle', 'starting', 'streaming', 'interrupted', 'recovering', 'failed'
+     */
+    get streamState(): string;
+    /**
+     * Derived meta-status combining session, job, and stream states into a
+     * single user-facing progression.
+     */
+    get status(): PixelStreamStatus;
+    /**
+     * When status is 'failed', the reason for the failure (e.g.
+     * 'queue_wait_timeout', 'media_pipeline_failure'). Null otherwise.
+     */
+    get failureReason(): string | null;
+    /**
+     * Timestamp (Date.now()) of the last user input sent to the streamer.
+     * Returns 0 if no input has been sent yet. Consumers can poll this to
+     * implement their own idle timeouts.
+     */
+    get lastUserInteraction(): number;
+    /**
+     * Timestamp (Date.now()) of when the stream began.
+     * Returns 0 if not currently streaming.
+     */
+    get streamStartedAt(): number;
+    /**
+     * Access token obtained from token exchange (read-only)
+     */
+    get accessToken(): string | null;
+    /**
+     * Session ID obtained from token exchange (read-only)
+     */
+    get sessionId(): string | null;
+    /**
+     * Agent ID obtained from token exchange (read-only)
+     */
+    get agentId(): string | null;
+    /**
+     * ICE servers obtained from token exchange (read-only)
+     */
+    get iceServers(): RTCIceServer[];
+    /**
+     * Get the current admission configuration
+     */
+    getAdmissionConfig(): AdmissionConfig | null;
+    /**
+     * Get the current session preferences
+     */
+    getSessionPreferences(): SessionPreferences;
+    /**
+     * Check if the player is currently admitted to a session
+     */
+    get isAdmitted(): boolean;
+    constructor();
+    attributeChangedCallback(name: string, oldValue: string | null, newValue: string | null): void;
+    connectedCallback(): void;
+    disconnectedCallback(): void;
+    startXRSession(): Promise<void>;
+    endXRSession(): void;
+    static isXRSupported(): Promise<boolean>;
+    resetPerformanceMetrics(): void;
+    getPerformanceMetrics(): PerformanceMetrics;
+    collectAndDisplaySessionStats(): Promise<SessionStats | null>;
+    /**
+     * Handle InitialSettings message to extract UE version hints for resize mode selection.
+     * Called from webrtc.ts when InitialSettings is received.
+     */
+    _handleInitialSettingsForResize(settings: any): void;
+    /**
+     * Detect if the current stream is an XR/stereo stream based on track heuristics.
+     * Updates _isXRStream and dispatches 'xr-stream-detected' event if detected.
+     */
+    _detectXRStream(): void;
+    /** Send message to UE via WebRTC data channel */
+    sendToStreamer(type: string, data?: Array<number | string>): void;
+    /** Send raw binary data over the data channel */
+    sendRawBinary(buffer: ArrayBuffer | Uint8Array): void;
+    sendCommand(command: object | string): void;
+    sendUIInteraction(descriptor: object | string): void;
+    requestIFrame(): void;
+    requestKeyFrame(): void;
+    requestQualityControl(): void;
+    requestFps(fps: number): void;
+    requestBitrate(): void;
+    startStreaming(): void;
+    stopStreaming(): void;
+    sendTestEcho(message: string): void;
+    checkPermission(permission: 'microphone' | 'camera'): Promise<string>;
+    checkPermissions(permissions: Array<'microphone' | 'camera'>): Promise<Record<string, string>>;
+    watchPermission(permission: 'microphone' | 'camera'): Promise<void>;
+    requestMicrophone(): Promise<boolean>;
+    enableMicrophone(): Promise<void>;
+    disableMicrophone(): void;
+    setMicrophoneMuted(muted: boolean): void;
+    get isMicrophoneEnabled(): boolean;
+    get isMicrophoneMuted(): boolean;
+    getVideoTracks(): Array<{
+        trackId: string;
+        label: string;
+        isActive: boolean;
+    }>;
+    switchVideoTrack(trackId: string): void;
+    getActiveVideoTrack(): {
+        trackId: string;
+        label: string;
+    } | null;
+    /**
+     * Get the last received file from UE (if available)
+     * Returns null if no file has been received or file is incomplete
+     */
+    getLastReceivedFile(): ReceivedFile | null;
+    /**
+     * Wait for the next file to be received from UE
+     * Returns a promise that resolves when a complete file is received
+     *
+     * @example
+     * const file = await player.waitForFile();
+     * const url = URL.createObjectURL(file.blob);
+     * const a = document.createElement('a');
+     * a.href = url;
+     * a.download = `file.${file.extension}`;
+     * a.click();
+     */
+    waitForFile(): Promise<ReceivedFile>;
+    /**
+     * Set the video source (for testing or manual stream setup)
+     */
+    setVideoSource(srcObject: MediaStream): void;
+    /**
+     * Start a session by exchanging the admission token for an access token,
+     * then connecting the WebSocket for signalling.
+     */
+    startSession(): Promise<void>;
+    /**
+     * Send a message through the session WebSocket.
+     */
+    sendSessionMessage(message: object): void;
+    /**
+     * Request a marketplace job for this session.
+     * @deprecated Use {@link play} instead — it handles all states including autoplay.
+     */
+    requestJob(): Promise<void>;
+    /**
+     * Start or resume streaming — the primary user-facing action.
+     *
+     * Handles all states:
+     * - Autoplay blocked → resumes video playback
+     * - Session connected, job unsubmitted → sends `request-job`
+     * - Session connecting → waits for connect hook
+     * - Session idle → starts session, connect hook sends `request-job`
+     * - Error/ended → clears error state and restarts
+     *
+     * Must be triggered by a user gesture for browser autoplay policy.
+     */
+    play(): Promise<void>;
+    /**
+     * Cancel a pending job request.
+     */
+    cancel(): void;
+    /**
+     * Request a session invite token.
+     *
+     * Sends a `request-invite` message to the session. The server responds with
+     * an `invite-token` event containing a short-lived JWT that can be passed as
+     * the `admission-token` to a second `<pixel-stream>` element, allowing it to
+     * join the same session as a `session-collaborator`.
+     *
+     * Only the session owner (browser-agent) may call this. Requires an active
+     * WebSocket connection — call after the `session-connected` event.
+     */
+    requestInvite(): void;
+    /**
+     * Stop the session gracefully.
+     */
+    stop(reason?: string): void;
+    /** @deprecated Use {@link play}. */
+    resumePlayback(): Promise<void>;
+    /** @deprecated Use {@link cancel}. */
+    cancelJob(): void;
+    /** @deprecated Use {@link stop}. */
+    leaveSession(reason?: string): void;
+    subscribeToStreamer(streamerId: string): void;
+    get availableStreamers(): string[];
+    /**
+     * Update the debug visual indicator based on current session state.
+     * Only applies when the 'debug' attribute is set.
+     */
+    _updateDebugIndicator(): void;
+    /**
+     * Force-close the session without notifying the server.
+     * Prefer {@link stop} for graceful departure.
+     */
+    terminateSession(): void;
+}
+
+declare interface PixelStreamConfig {
+    nativeTouch: boolean;
+    pointerLocked: boolean;
+    pointerLockRelease: boolean;
+    suppressBrowserKeys: boolean;
+    apiEndpoint: string;
+    admissionToken: string | null;
+    appId: string | null;
+    appVersion: string | null;
+    noAutoConnect: boolean;
+    mute: boolean;
+    volume: number;
+    rendezvousPreference: number | null;
+    lingerPreference: number | null;
+    leftGracePeriod: number | null;
+    reconnectMode: 'none' | 'recover' | 'always';
+    reconnectAttempts: number;
+    reconnectStrategy: 'periodic' | 'exponential-backoff';
+    reconnectInterval: number;
+    disconnectGraceMs: number;
+    resizeMode: 'none' | 'auto' | 'pureweb' | 'pre-5.4' | '5.4+';
+    dprCap: number;
+    resolutionClamp: {
+        maxWidth: number;
+        maxHeight: number;
+    } | null;
+    queueWaitTolerance: number | null;
+    webrtcNegotiationTolerance: number | null;
+    debug: boolean;
+    controls: boolean;
+    swiftJobRequest: boolean;
+    streamerId: string | null;
+    lat: number | null;
+    lng: number | null;
+    forceRelay: boolean;
+    useMic: boolean;
+    forceMonoAudio: boolean;
+    iceBatchDelay: number;
+    streamQuality: 'low-latency' | 'balanced' | 'quality';
+    videoBitrateMin: number | null;
+    videoBitrateStart: number | null;
+    videoBitrateMax: number | null;
+    logLevel: LogLevel;
+}
+
+export declare function PixelStreamResizeObserverForVideo(opts: PixelStreamResizeObserverOptions): ResizeObserver;
+
+export declare type PixelStreamResizeObserverOptions = {
+    el: HTMLVideoElement;
+    onUpdate: (u: PixelStreamResizeUpdate) => void;
+    settleMs?: number;
+    dprCap?: number | (() => number);
+    even?: boolean;
+    cssJitterPx?: number;
+};
+
+/**
+ * PixelStreamResizeObserverForVideo
+ *
+ * Hybrid ResizeObserver wrapper for a <video> element WITHOUT a wrapper/shell:
+ *  - Width/height delta filtering
+ *  - requestAnimationFrame coalescing (live updates: at most once per frame)
+ *  - Debounced "final" pass (semantic commit when resizing settles)
+ *  - Measures the *content box* via clientWidth/clientHeight (so borders on <video> do NOT pollute sizes)
+ *  - Computes desired stream resolution in *device pixels* (CSS px * DPR), with optional DPR cap + even sizing
+ *
+ * Usage:
+ *   this.resizeObserver = PixelStreamResizeObserverForVideo({
+ *     el: this.videoEl,
+ *     settleMs: 200,
+ *     even: true,
+ *     dprCap: 2, // optional
+ *     onUpdate: ({ final, cssW, cssH, devW, devH, dpr }) => {
+ *       // live: cheap UI math
+ *       this.updateCoordTranslator({ final, cssW, cssH, devW, devH, dpr });
+ *
+ *       // final: expensive / stateful: tell streamer desired resolution
+ *       if (final) this.requestStreamResize(devW, devH);
+ *     },
+ *   });
+ *
+ *   // later
+ *   (this.resizeObserver as any).dispose?.(); // or this.resizeObserver.disconnect()
+ */
+export declare type PixelStreamResizeUpdate = {
+    final: boolean;
+    cssW: number;
+    cssH: number;
+    dpr: number;
+    devW: number;
+    devH: number;
+};
+
+export declare type PixelStreamStatus = 'idle' | 'connecting' | 'authenticating' | 'connected' | 'queued' | 'rendezvoused' | 'negotiating' | 'streaming' | 'ready' | 'interrupted' | 'recovering' | 'failed' | 'ended';
+
+declare interface ReceivedFile {
+    blob: Blob;
+    extension: string;
+    mimeType: string;
+    chunks: number;
+}
+
+export declare type ResolutionClampName = 'sxga' | 'hd' | 'hdplus' | 'fhd' | 'wuxga' | 'qhd' | 'wqhd' | 'uhd';
+
+declare interface SessionPreferences {
+    /** Seconds to wait for a peer streamer before giving up */
+    rendezvousPreference?: number;
+    /** Seconds to remain in session after all other agents leave */
+    lingerPreference?: number;
+    /** Seconds grace period before this client appears as left to other agents */
+    leftGracePeriod?: number;
+}
+
+declare interface SessionStats {
+    duration: number;
+    video: {
+        codec: string | null;
+        mimeType: string | null;
+        bytesReceived: number;
+        packetsReceived: number;
+        packetsLost: number;
+        framesReceived: number;
+        framesDecoded: number;
+        framesDropped: number;
+        frameRate: number;
+        bitrate: number;
+        resolution: {
+            width: number;
+            height: number;
+        };
+        jitter: number;
+        pliCount: number;
+        nackCount: number;
+    };
+    audio: {
+        codec: string | null;
+        mimeType: string | null;
+        bytesReceived: number;
+        packetsReceived: number;
+        packetsLost: number;
+        bitrate: number;
+        jitter: number;
+    };
+    connection: {
+        iceLocalCandidateType: string | null;
+        iceRemoteCandidateType: string | null;
+        turnUsed: boolean;
+        rtt: number;
+        availableOutgoingBitrate: number;
+        protocol: string | null;
+        localAddress: string | null;
+        remoteAddress: string | null;
+    };
+}
+
+export declare interface StreamResolutionClamp {
+    maxWidth: number;
+    maxHeight: number;
+}
+
+export { }