@furious.luke/argus-js 0.4.0 → 0.5.1

package/dist/index.d.cts

@@ -1,3 +1,39 @@
+/**
+ * The logical purpose of a track within a stream. A stream may carry more than
+ * one track — camera and screen-share video, plus a single `audio` microphone
+ * track. Reads and change notifications address a specific video track by type;
+ * the `audio` track feeds server-side transcription rather than the frame path.
+ */
+type TrackType = "camera" | "screen" | "audio";
+/**
+ * The video track types. The video publish surface (start, publish, unpublish,
+ * replaceStream) is typed to this rather than {@link TrackType}, so `audio`
+ * cannot be passed to a video call — the microphone has its own dedicated API
+ * ({@link Publisher.startAudioOnly}, {@link Publisher.publishMicrophone}). A
+ * session may also begin without media via {@link Publisher.startTextOnly}.
+ */
+type VideoTrackType = "camera" | "screen";
+/**
+ * Declares the logical type and identity of a single published track.
+ *
+ * `mid` is the negotiated media-section id — the ROUTING key the server matches a
+ * track to its type by. It is used rather than the browser `MediaStreamTrack.id`
+ * because it is the only track identifier both peers agree on: Firefox mints an
+ * SDP msid track id unrelated to the `MediaStreamTrack.id`, so routing keyed on
+ * the latter cannot be matched server-side.
+ *
+ * `id` is the browser `MediaStreamTrack.id` — the GENERATION identity. The mid is
+ * stable across renegotiation and is reused when a track of the same type is
+ * republished on the same m-line, so it cannot distinguish a delayed end of a
+ * removed track from a failure of its replacement; the id changes with each new
+ * capture and can. The server echoes it in `media_track_ended`, and the publisher
+ * correlates that against the id it intentionally removed.
+ */
+interface TrackLabel {
+    mid: string;
+    id: string;
+    type: TrackType;
+}
 /**
  * Message types exchanged over the WebSocket signaling channel.
  */
@@ -5,10 +41,14 @@ type SignalMessage = {
     type: "offer";
     sdp: string;
     sdp_type: "offer";
+    tracks?: TrackLabel[];
+    negotiation_id?: string;
+    speech_enabled?: boolean;
 } | {
     type: "answer";
     sdp: string;
     sdp_type: "answer";
+    negotiation_id?: string;
 } | {
     type: "ice_candidate";
     candidate: string;
@@ -18,27 +58,38 @@ type SignalMessage = {
 } | {
     type: "connection_state";
     state: string;
+} | {
+    type: "ice_path";
+    local_candidate_type: string;
+    local_protocol?: string;
+    remote_candidate_type?: string;
+    remote_protocol?: string;
+    relay_protocol?: string;
+    turn_url?: string;
 } | {
     type: "media_stall";
-    track: string;
+    track: TrackType;
     frame_age_ms: number;
 } | {
     type: "media_resumed";
-    track: string;
+    track: TrackType;
     duration_ms: number;
 } | {
     type: "media_track_ended";
-    track: string;
+    track: TrackType;
+    track_id?: string;
     reason?: string;
 } | {
     type: "recovery_event";
     event: "recovery_started" | "recovery_retry" | "recovery_failed";
-    track: string;
+    track: TrackType;
     action?: "sender_restart" | "ice_restart";
     reason?: "capture_ended" | "automatic_recovery_failed";
 } | {
     type: "error";
     error: string;
+    negotiation_id?: string;
+    fatal?: boolean;
 } | {
     type: "resumed";
 } | {
@@ -66,14 +117,31 @@ interface PublisherCallbacks {
     onRecoveryStateChange?: (event: PublisherRecoveryEvent) => void;
     /** Called when recovery requires the host application to obtain a new screen share. */
     onRecoveryRequired?: (event: PublisherRecoveryEvent) => void;
+    /** Called when paced assistant text arrives alongside synthesized speech. */
+    onAssistantText?: (event: AssistantTextEvent) => void;
+    /** Called when the server accepts or rejects typed user input. */
+    onUserTextResult?: (event: UserTextResultEvent) => void;
+    /** Called once the explicitly requested outbound speech track arrives. */
+    onSpeechTrack?: (track: MediaStreamTrack, streams: readonly MediaStream[]) => void;
+}
+interface AssistantTextEvent {
+    utteranceId: string;
+    text: string;
+}
+interface UserTextResultEvent {
+    messageId: string;
+    accepted: boolean;
+    reason?: string;
 }
 type PublisherRecoveryState = "recovering" | "recovered" | "failed";
 type PublisherRecoveryAction = "sender_restart" | "ice_restart";
 type PublisherRecoveryFailureReason = "capture_ended" | "automatic_recovery_failed";
+/** Restricts gateway-advertised TURN URLs used for ICE candidate gathering. */
+type TurnTransportPolicy = "all" | "udp" | "tls";
 /** A transition in the publisher's fixed automatic media-recovery ladder. */
 interface PublisherRecoveryEvent {
     state: PublisherRecoveryState;
-    track: string;
+    track: TrackType;
     action?: PublisherRecoveryAction;
     reason?: PublisherRecoveryFailureReason;
 }
@@ -81,7 +149,7 @@ interface PublisherRecoveryEvent {
  * TURN and read-token info delivered in the gateway `ready` message.
  */
 interface GatewayReadyInfo {
-    /** Ordered TURN URLs (UDP first, then TCP) so relay can fall back to TCP on UDP-blocking networks. */
+    /** Bounded TURN URLs with UDP and TCP transports for each selected relay server. */
     turn_urls?: string[];
     turn_username?: string;
     turn_credential?: string;
@@ -98,6 +166,29 @@ interface PublisherOptions {
     token: string;
     /** Optional extra ICE servers (e.g. STUN). TURN is supplied by the winning gateway. */
     iceServers?: RTCIceServer[];
+    /**
+     * ICE transport policy passed to the underlying `RTCPeerConnection`. Defaults
+     * to `"all"`, letting ICE pick the best path (usually direct). Set to
+     * `"relay"` to force media through the TURN relay only — no host or
+     * server-reflexive candidates — which is how you verify the TURN path end to
+     * end rather than have ICE silently bypass it.
+     */
+    iceTransportPolicy?: RTCIceTransportPolicy;
+    /**
+     * Restricts TURN URLs supplied by the winning gateway. Defaults to `"all"`.
+     * `"tls"` admits only secure `turns:` URLs and `"udp"` admits only TURN/UDP.
+     * Combine an explicit policy with `iceTransportPolicy: "relay"` when proving
+     * a particular relay path end to end rather than merely making it available.
+     */
+    turnTransportPolicy?: TurnTransportPolicy;
+    /**
+     * Overall deadline for the initial accepted/proceed/ready gateway race.
+     * Unaccepted sockets are replaced after three seconds within this deadline.
+     * Defaults to 20 seconds.
+     */
+    gatewayHandshakeTimeoutMs?: number;
+    /** Deadline after the initial offer for WebRTC to reach connected. Defaults to 30 seconds. */
+    peerConnectionTimeoutMs?: number;
     /** How long to retry a dropped signaling connection to the selected gateway. Defaults to 20 seconds. */
     signalingReconnectTimeoutMs?: number;
     /** Callbacks for lifecycle events. */
@@ -105,11 +196,12 @@ interface PublisherOptions {
 }
 
 /**
- * Publisher streams a browser {@link MediaStream} to an Argus media server over
- * WebRTC. Given the `gateway_urls` and `token` from a join-token response, it
+ * Publisher establishes a browser WebRTC session with an Argus media server.
+ * The session may start with camera, microphone, or only its reliable text data
+ * channel. Given the `gateway_urls` and `token` from a join-token response, it
  * races the candidate gateways to the fastest one, completes the two-phase
- * signaling handshake, and manages the peer connection — offer/answer exchange,
- * ICE candidate trickling, and track (re)negotiation.
+ * signaling handshake, and manages offer/answer exchange, ICE candidate
+ * trickling, and track (re)negotiation.
  *
  * After {@link Publisher.start} resolves, {@link Publisher.frameReadToken} holds
  * the token your application server needs to fetch frames for this stream.
@@ -132,18 +224,44 @@ declare class Publisher {
     private pc;
     private hasAnswer;
     private pendingRemoteCandidates;
-    private localStream;
+    private pendingLocalCandidates;
+    private localCandidateOfferSent;
+    private retainedLocalCandidates;
+    private retainedLocalCandidateKeys;
+    private localCandidateGeneration;
+    private remoteCandidateKeys;
+    private remoteCandidateOrder;
+    private remoteCandidateGeneration;
     private readToken;
     private gatewayURL;
+    private lastReportedICEPath;
+    private watchedICETransports;
     private stopped;
+    private lifecycleGeneration;
+    private runAbort;
+    private peerConnectionTimer;
     private reconnecting;
     private reconnectGeneration;
     private resumeSocket;
-    private recoveryGeneration;
-    private recoveringMedia;
-    private recoveryRequired;
-    private recoveryAction;
+    private signalingWaiters;
+    private pendingOffer;
+    private recoverySequence;
+    private recoveryStates;
+    private iceRestartSequence;
+    private iceRestartAttempt;
     private trackEndHandlers;
+    private published;
+    private publishedStreams;
+    private typeSenders;
+    private intentionalTrackEnds;
+    private negotiationChain;
+    private pendingAnswer;
+    private negotiationSeq;
+    private textChannel;
+    private speechEnabled;
+    private speechPending;
+    private speechTransceiver;
+    private microphoneTransceiver;
     constructor(opts: PublisherOptions);
     /** The read token used for frame fetches and signaling resume in the selected region. */
     get frameReadToken(): string | null;
@@ -153,41 +271,218 @@ declare class Publisher {
      * region for frame reads and change-notification subscriptions.
      */
     get selectedGatewayURL(): string | null;
+    /** Requests the persistent outbound `speech` track. This is explicit user
+     * opt-in and renegotiates only once; the track remains silent between turns. */
+    enableSpeech(): Promise<void>;
+    /** Sends typed input over the reliable ordered Argus text channel. */
+    sendUserText(messageId: string, text: string): void;
     /**
      * Starts the publisher: races all gateways to find the fastest, completes
      * the two-phase handshake, creates the peer connection, and sends the SDP
      * offer. Resolves when the offer has been sent (not when ICE completes —
      * use onConnected for that).
+     *
+     * The stream's single video track is published under `type` (default `"camera"`),
+     * declared to the server so reads and change notifications can address them by
+     * type. Add or remove further tracks live with {@link Publisher.publish} and
+     * {@link Publisher.unpublish}.
+     */
+    start(stream: MediaStream, type?: VideoTrackType): Promise<void>;
+    /**
+     * Starts the publisher with a microphone track and no video — a fully valid
+     * audio-only stream, the natural starting point for a voice agent. Exactly one
+     * audio track must be present in `stream`. Video can be added later with
+     * {@link Publisher.publish}; a stream carries at most one microphone track.
+     *
+     * Like {@link Publisher.start} it races the gateways, completes the handshake,
+     * and sends the offer; it resolves once the offer is sent. The microphone is not
+     * subject to the video recovery ladder — a mic that stops simply ends
+     * transcription for the stream.
+     */
+    startAudioOnly(stream: MediaStream): Promise<void>;
+    /**
+     * Starts a WebRTC session with only the ordered `argus.text` data channel.
+     * This is the natural entry point for a typed, text-only agent: it requests no
+     * camera or microphone permission and publishes no media. Camera, screen, or
+     * microphone tracks can be added later with {@link Publisher.publish} or
+     * {@link Publisher.publishMicrophone}; {@link Publisher.enableSpeech} can add
+     * the optional inbound speech track independently.
+     */
+    startTextOnly(): Promise<void>;
+    /**
+     * Shared startup for video, audio-only, and text-only entry points: race the
+     * gateways, build the peer connection and text channel, optionally add an
+     * initial media track, and send the first offer.
+     */
+    private startSession;
+    /**
+     * Adds the single video track from `stream` to the live session under `type`,
+     * renegotiating so the media server begins ingesting them. Use this to add a
+     * track after {@link Publisher.start} — for example to begin a screen share on
+     * top of a live camera.
+     *
+     * Exactly one video track must be present in `stream`. If a track of `type` is already
+     * live it is removed and replaced (a "screen" published while another "screen"
+     * is live supersedes it).
+     */
+    publish(stream: MediaStream, type: VideoTrackType): Promise<void>;
+    /**
+     * Removes the live track(s) of the given type, stops their local capture, and
+     * renegotiates so the media server ends ingestion for that track. A no-op if
+     * no track of that type is published.
+     */
+    unpublish(type: VideoTrackType): Promise<void>;
+    /**
+     * Adds the microphone (audio) track from `stream` to the live session and
+     * renegotiates, so the media server begins transcribing it. Exactly one audio
+     * track must be present in `stream`. Publishing a microphone while one is
+     * already live replaces it.
+     *
+     * The audio track feeds server-side speech-to-text; its transcripts are
+     * delivered to the customer server over the change-notification subscription,
+     * not to the browser. Audio is not subject to the video recovery ladder — a
+     * mic that stops is simply removed.
+     */
+    publishMicrophone(stream: MediaStream): Promise<void>;
+    /**
+     * Removes the live microphone track, stops its local capture, and
+     * renegotiates so the media server ends transcription. A no-op if no
+     * microphone is published.
+     */
+    unpublishMicrophone(): Promise<void>;
+    /**
+     * Replaces the published track of a single type with a new stream and
+     * renegotiates in place — e.g. to swap to a freshly reacquired screen share
+     * after {@link PublisherCallbacks.onRecoveryRequired}. Defaults to the
+     * `"camera"` type. This is a convenience over {@link Publisher.publish}, which
+     * it delegates to (publishing one track per type replaces any existing track
+     * of that type).
      */
-    start(stream: MediaStream): Promise<void>;
-    /** Replaces the currently published stream with a new one. */
-    replaceStream(stream: MediaStream): Promise<void>;
+    replaceStream(stream: MediaStream, type?: VideoTrackType): Promise<void>;
     /** Stops publishing and tears down the peer connection. */
     stop(): void;
+    private rejectPendingAnswer;
     /** Returns the current RTCPeerConnection, or null if not started. */
     get peerConnection(): RTCPeerConnection | null;
     /** Returns true if the peer connection is in the "connected" state. */
     get isConnected(): boolean;
     private raceGateways;
     private installSignaling;
+    private awaitSignaling;
+    private resolveSignalingWaiters;
+    private rejectSignalingWaiters;
+    private sendWhenSignalingAvailable;
+    private sendOffer;
     private resumeSignaling;
     private openResumeSocket;
     private wait;
+    private isActiveRun;
+    private assertActiveRun;
+    private armPeerConnectionTimeout;
+    private clearPeerConnectionTimeout;
     private terminateWithError;
     private handleSignal;
     private beginMediaRecovery;
     private restartSenders;
-    private renegotiate;
+    /**
+     * Adds a recovery renegotiation to the same queue as user operations. The
+     * recovery ladder awaits its completion before starting the stage observation
+     * window. If recovery has completed by the time this reaches the head of the
+     * queue, it is skipped.
+     */
+    /** Returns the peer-wide ICE attempt shared by all active track recoveries. */
+    private sharedIceRestart;
+    /**
+     * Serializes a renegotiation onto the shared chain: it waits for any prior
+     * negotiation to finish (its answer applied), sends a fresh offer, and resolves
+     * only once this offer's answer has been applied. This prevents overlapping
+     * offers and answers being applied to the wrong offer.
+     */
+    private enqueueNegotiation;
+    private negotiateOnce;
+    /**
+     * Registers interest in the answer for the offer identified by `id` and returns
+     * a promise for its SDP. A second pending answer is an invariant violation: all
+     * offer creation, including recovery, must pass through negotiationChain.
+     */
+    private nextNegotiationId;
+    private handleTextMessage;
+    private awaitAnswer;
+    private negotiationAnswerTimeoutMs;
+    private handleLocalICECandidate;
+    private candidateKey;
+    private retainLocalCandidate;
+    private retainRemoteCandidate;
+    private clearRetainedICECandidates;
+    private beginLocalCandidateBatch;
+    private releaseLocalCandidateBatch;
+    private applyAnswered;
+    private watchSelectedICEPairChanges;
+    private reportSelectedICEPath;
     private completeMediaRecovery;
     private failMediaRecovery;
+    private recoveryState;
     private cancelMediaRecovery;
+    private cancelAllMediaRecovery;
     private isCurrentRecovery;
+    private hasActiveMediaRecovery;
+    private clearSharedIceRestartIfIdle;
     private emitRecoveryTransition;
     private sendRecoveryDiagnostic;
-    private watchStreamTracks;
+    private requireSingleVideoTrack;
+    private requireLiveVideoTrack;
+    private requireLiveTrack;
+    private requireSingleAudioTrack;
+    /**
+     * Stages the microphone track onto the peer connection. Unlike video, audio
+     * has no recovery ladder and no SSIM/frame semantics, so this simply adds (or
+     * replaces) the single audio sender and declares the updated labels.
+     */
+    private stagePublishAudio;
+    /**
+     * Gives the microphone its own sendonly transceiver. addTrack() may reuse an
+     * existing compatible recvonly transceiver, which would collapse the
+     * microphone and assistant speech roles when speech was enabled first.
+     * addTransceiver() always creates a distinct m-line and its direction keeps
+     * the server's outbound speech sender on the dedicated speech transceiver.
+     */
+    private addMicrophoneTrack;
+    /** Registers one physical video track under its logical type and source stream. */
+    private registerTrack;
+    /** The live video tracks currently published under the given type. */
+    private tracksOfType;
+    /** Builds the id → type label array declared to the server on every offer. */
+    private buildTrackLabels;
+    private labelsReplacingType;
+    /**
+     * The negotiated mid of the transceiver currently sending `track`, or null if
+     * none is found or it has not been negotiated yet. The mid is the identifier
+     * both peers agree on; it is assigned once setLocalDescription runs, which the
+     * publisher always does before sending an offer's labels.
+     */
+    private midForTrack;
+    private stagePublish;
+    private stageUnpublish;
+    /** Stops every published local track and clears the published map. */
+    private stopPublishedTracks;
+    private expectIntentionalTrackEnd;
+    private forgetIntentionalTrackEnds;
+    private consumeIntentionalTrackEnd;
+    private clearIntentionalTrackEnds;
+    /**
+     * Watches a track's "ended" event so an involuntary capture stop (the user
+     * revokes a screen share, a device unplugs) reports as a recovery failure for
+     * that track's actual type. Intentional removals unwatch first.
+     */
+    private watchTrack;
+    /**
+     * Watches a microphone only for lifecycle removal. An ended microphone is not
+     * recovered like video; it is negotiated away so the media server can flush
+     * the utterance and release transcription resources.
+     */
+    private watchMicrophone;
+    private unwatchTrack;
     private unwatchStreamTracks;
-    /** Waits for ICE gathering to reach the "complete" state. */
-    private gatherComplete;
 }
 
 /**
@@ -306,5 +601,36 @@ declare function captureCamera(opts?: CaptureCameraOptions): Promise<MediaStream
  * ```
  */
 declare function captureScreen(opts?: CaptureScreenOptions): Promise<MediaStream>;
+/**
+ * Options for {@link captureMicrophone}.
+ */
+interface CaptureMicrophoneOptions {
+    /**
+     * Audio constraints, or `true`. Defaults to enabling the browser's echo
+     * cancellation, noise suppression, and auto gain control — the settings that
+     * give a speech-to-text engine the cleanest signal.
+     */
+    audio?: MediaTrackConstraints | boolean;
+    /**
+     * The `MediaDevices` instance to capture from. Defaults to the global
+     * `navigator.mediaDevices`.
+     */
+    mediaDevices?: MediaDevices;
+}
+/**
+ * Captures the user's microphone via `navigator.mediaDevices.getUserMedia`,
+ * applying defaults tuned for speech-to-text. Pair with
+ * {@link Publisher.publishMicrophone} to add transcription to a stream.
+ *
+ * Defaults:
+ * - `audio`: `{ echoCancellation: true, noiseSuppression: true, autoGainControl: true }`.
+ *
+ * @example
+ * ```ts
+ * const mic = await captureMicrophone();
+ * await publisher.publishMicrophone(mic);
+ * ```
+ */
+declare function captureMicrophone(opts?: CaptureMicrophoneOptions): Promise<MediaStream>;
 
-export { type CaptureCameraOptions, type CaptureScreenOptions, type GatewayReadyInfo, Publisher, type PublisherCallbacks, type PublisherOptions, type PublisherRecoveryAction, type PublisherRecoveryEvent, type PublisherRecoveryFailureReason, type PublisherRecoveryState, type SignalMessage, captureCamera, captureScreen };
+export { type AssistantTextEvent, type CaptureCameraOptions, type CaptureMicrophoneOptions, type CaptureScreenOptions, type GatewayReadyInfo, Publisher, type PublisherCallbacks, type PublisherOptions, type PublisherRecoveryAction, type PublisherRecoveryEvent, type PublisherRecoveryFailureReason, type PublisherRecoveryState, type SignalMessage, type TrackLabel, type TrackType, type TurnTransportPolicy, type UserTextResultEvent, type VideoTrackType, captureCamera, captureMicrophone, captureScreen };