@memberjunction/ng-conversations 6.2.0-edge.4 → 6.2.0-edge.5

package/dist/lib/services/realtime-audio-recorder.d.ts

@@ -1,53 +1,53 @@
+import { IRealtimeSessionRecorder } from '@memberjunction/realtime-runtime';
+import { IOpusPacketEncoder } from './recording/opus-packet-encoder';
 /**
- * Browser-side audio recorder for a CLIENT-DIRECT realtime voice session.
+ * Browser-side audio recorder for a CLIENT-DIRECT realtime voice session, implementing the
+ * runtime's {@link IRealtimeSessionRecorder} contract directly.
  *
- * Mixes the user's microphone stream with the agent's remote-audio stream (every driver exposes
- * one via `GetRemoteMediaStream()`: WebRTC drivers publish their remote track, PCM drivers —
- * Gemini, ElevenLabs, AssemblyAI, xAI, HuggingFace — publish their playout engine's output) into a
- * single mono **WAV** (16-bit PCM) blob via the Web Audio API. Rather than a `MediaRecorder` (which only emits webm/opus —
- * header-less, no duration/cues, so HTTP-range seeking is unreliable), the mix is routed into an
- * {@link AudioWorkletNode} (preferred) or a {@link ScriptProcessorNode} (fallback) that captures
- * Float32 PCM frames. PCM is accumulated in memory and encoded as a seekable RIFF/WAVE file at
- * {@link Stop}. WAV's header carries the exact duration and maps byte-offset → time linearly, so
- * range seeking is exact and waveform peaks are computed cheaply during capture. The tradeoff is
- * size (PCM WAV is larger than opus) — acceptable for these short session recordings.
+ * Mixes the user's microphone with the agent's remote-audio stream (every driver exposes one via
+ * `GetRemoteMediaStream()`: WebRTC drivers publish their remote track, PCM drivers — Gemini,
+ * ElevenLabs, AssemblyAI, xAI, HuggingFace — publish their playout engine's output) in a Web Audio
+ * graph, captures the mix as mono Float32 frames through an {@link AudioWorkletNode} (preferred) or a
+ * {@link ScriptProcessorNode} (fallback), and feeds them to ONE continuous Opus encoder for the whole
+ * call. The runtime pulls the encoded packets with {@link TakeEncodedPackets}, cuts them into shards
+ * and uploads them as the call goes, so nothing is held in memory until the end.
  *
- * Robust + MVP by design — every step degrades gracefully:
+ * Timing is the 48 kHz sample clock: the graph asks for a 48 kHz context, and when the browser
+ * refuses (or reports another rate) every frame is resampled to 48 kHz before encoding. The
+ * captured-sample count drives {@link NowOffsetMs} and is what {@link Stop} resolves.
+ *
+ * Robust by design — every step degrades gracefully:
  * - No `AudioContext` (server-side render / unsupported env) → the recorder disables itself
- *   ({@link IsRecording} stays `false`, {@link Stop} resolves `null`); the voice call proceeds.
+ *   ({@link IsRecording} stays `false`, {@link Stop} resolves 0); the voice call proceeds.
  * - No `AudioWorkletNode` AND no `ScriptProcessorNode` → disables itself rather than throwing.
+ * - A forced-48 kHz context that cannot take the microphone (Firefox rejects a source whose rate
+ *   differs from its context's) → rebuilt once at the device rate with resampling.
  * - No remote stream yet (e.g. the WebRTC track hasn't landed) → records mic only until it does.
- * - A remote stream that can't be connected (e.g. a browser that rejects cross-sample-rate
- *   `createMediaStreamSource` — the PCM playout context runs at 24 kHz, this one at the device
- *   rate) → warns and records mic only; the mic recording is never lost to the agent stream.
- * - Any construction/start error is logged and disables the recorder rather than throwing into
- *   the session-start path.
+ * - A remote stream that can't be connected (e.g. a cross-sample-rate `createMediaStreamSource`
+ *   rejection) → warns and records mic only; the mic recording is never lost to the agent stream.
+ * - No usable encoder → logged, recording disabled for the call.
  *
- * Not an Angular service: it owns transient per-session media plumbing the
- * {@link RealtimeSessionService} creates and discards per call, so a plain class is the right
- * shape (no DI, no singleton).
+ * Not an Angular service: it owns transient per-session media plumbing the runtime creates and
+ * discards per call, so a plain class is the right shape (no DI, no singleton).
  */
-export declare class RealtimeAudioRecorder {
-    /** The container/codec we now produce — a seekable mono 16-bit PCM WAV. */
-    private static readonly wavMimeType;
-    /** Target waveform-peak resolution (buckets) across the whole recording. */
-    private static readonly peakBuckets;
+export declare class RealtimeAudioRecorder implements IRealtimeSessionRecorder {
+    private readonly selectEncoder;
     /** PCM frame size for the ScriptProcessor fallback (worklet uses its native 128-frame quantum). */
     private static readonly scriptProcessorBufferSize;
+    /**
+     * Most 48 kHz audio held while the encoder is still being selected / loaded (10 s). Selection is
+     * normally well under a second; past this the encoder is stuck, and later frames are dropped
+     * (see {@link captureFrame} for why they are not counted either).
+     */
+    private static readonly maxPendingSamples;
     /** Inline AudioWorklet processor: forwards each 128-frame mono input block to the main thread. */
     private static readonly workletProcessorSource;
     private audioContext;
     private destination;
     private workletNode;
     private scriptNode;
-    /** Captured mono Float32 PCM frames (in capture order); concatenated + encoded to WAV at Stop. */
-    private pcmFrames;
-    /** Total captured sample count across {@link pcmFrames} (kept in step to avoid re-summing). */
-    private totalSamples;
-    /** Streaming waveform-peak accumulator — bounded regardless of recording length. */
-    private peaks;
-    /** Final normalized peaks, snapshotted at {@link Stop} so {@link GetPeaks} survives cleanup. */
-    private finalPeaks;
+    /** Converts the context's rate to 48 kHz; `null` when the context already runs at 48 kHz. */
+    private resampler;
     private recording;
     /** True once the agent's remote stream has been wired into the mix (prevents double-attach). */
     private remoteAttached;
@@ -57,37 +57,33 @@ export declare class RealtimeAudioRecorder {
     private micStream;
     /** The source node reading the microphone, swapped by ReplaceMicrophone. */
     private micSource;
-    /** Sample index up to which crash-recovery shards have already been emitted (the segment cursor). */
-    private flushedSampleCount;
-    /** Wall-clock ms at recording start; anchors {@link NowOffsetMs}. 0 before {@link Start}. */
-    private startedAtMs;
-    /** Sample rate of the capturing {@link AudioContext}; 0 before {@link Start}. Needed to wrap shards. */
-    private sampleRate;
+    /** 48 kHz samples captured (post-resample) — the recording's clock. */
+    private capturedSamples;
+    /** The call's encoder once selected; `null` while selecting, after a failure, or when not recording. */
+    private encoder;
+    /** Resolves to the encoder (or `null` on failure) — Stop awaits it so buffered audio still gets encoded. */
+    private encoderReady;
+    /** 48 kHz frames captured before the encoder was ready, fed to it in order once it is. */
+    private pendingFrames;
+    private pendingSamples;
+    private pendingOverflowWarned;
+    /** The first Stop's promise; later calls return it (idempotent). */
+    private stopping;
     /**
-     * Bumped by {@link cleanup}. The async setup snapshots it and, after each await, gives up if it moved:
-     * a {@link Stop} that lands mid-setup can only release what setup has already published, so setup
-     * must release the rest itself rather than build a live graph on a stopped recorder.
+     * Bumped whenever the audio graph is released. The async graph setup snapshots it and, after each
+     * await, gives up if it moved: a {@link Stop} that lands mid-setup can only release what setup has
+     * already published, so setup must release the rest itself rather than build a live graph on a
+     * stopped recorder.
      */
     private setupGeneration;
+    /** @param selectEncoder picks the call's Opus encoder; injectable for tests. */
+    constructor(selectEncoder?: () => Promise<IOpusPacketEncoder>);
     /** True while a recorder is actively capturing. */
     get IsRecording(): boolean;
-    /** Wall-clock ms when recording started (0 before {@link Start}). */
-    get StartedAtMs(): number;
-    /**
-     * The container/codec the active recording uses. Always `'audio/wav'` once recording (empty
-     * string before {@link Start} / when disabled) — the consolidated upload and the WAV-wrapped
-     * crash-recovery shards are all 16-bit PCM WAV.
-     */
-    get MimeType(): string;
-    /**
-     * The captured AudioContext sample rate (0 before {@link Start} / when disabled). Exposed so a
-     * recovery path can WAV-wrap the raw-PCM shards with the correct rate.
-     */
-    get SampleRate(): number;
     /**
      * Begins recording a mix of `micStream` and (when present) `remoteStream`. Best-effort:
      * a missing `AudioContext` / no worklet+scriptprocessor support / any setup failure disables
-     * the recorder silently (logged) so the live call is never blocked.
+     * the recorder (logged) so the live call is never blocked.
      *
      * @param micStream The user's microphone capture stream (required).
      * @param remoteStream The agent's remote-audio stream, or `null` for mic-only capture.
@@ -107,31 +103,37 @@ export declare class RealtimeAudioRecorder {
      * stream, and Start replaces it); a stream without an audio track keeps the current one.
      */
     ReplaceMicrophone(micStream: MediaStream): void;
-    /**
-     * Returns a Blob of the raw PCM captured since the last call (a window of header-less,
-     * little-endian PCM16 bytes), advancing the segment cursor. `null` when nothing new. Used to
-     * upload crash-recovery shards mid-call. NOTE: shards are header-less raw PCM (NOT individually
-     * playable WAV) — recovery is concatenation in order, then WAV-wrapping with {@link SampleRate}.
-     * The canonical consolidated file is still authored as a single WAV from full memory at {@link Stop}.
-     */
-    SnapshotNewSegment(): Blob | null;
-    /** Ms elapsed since recording started (>= 0); 0 when not recording. */
+    /** Milliseconds of audio captured so far, on the 48 kHz sample clock (never wall clock). */
     NowOffsetMs(): number;
+    /** The Opus packets the encoder produced since the last call (also its final packets after {@link Stop}). */
+    TakeEncodedPackets(): Uint8Array[];
     /**
-     * The downsampled waveform peaks (each 0..1, max-abs per bucket, ~{@link RealtimeAudioRecorder.PeakBuckets}
-     * buckets across the whole recording). Computed incrementally during capture so it's cheap and
-     * bounded regardless of length. Returns `[]` before any audio / for a silent recording. Safe to
-     * call after {@link Stop} — the peaks are snapshotted there before {@link cleanup} clears state.
+     * Stops capture, waits for the encoder (encoding any audio buffered while it loaded), flushes and
+     * closes it, and resolves the 48 kHz samples captured. The final packets stay available to
+     * {@link TakeEncodedPackets}. Idempotent: every call resolves the same count; 0 when the recorder
+     * never started.
      */
-    GetPeaks(): number[];
+    Stop(): Promise<number>;
+    private stopOnce;
     /**
-     * Stops recording and resolves the captured audio as a single mono 16-bit PCM **WAV**
-     * {@link Blob}, then releases the audio graph. Resolves `null` when nothing was recorded
-     * (recorder disabled, or no samples captured). Safe to call when not recording.
+     * Selects the encoder and feeds it what was captured meanwhile. A selection failure disables the
+     * recorder: with no encoder nothing captured could ever reach the server.
      */
-    Stop(): Promise<Blob | null>;
-    /** Wires up the Web Audio mix graph + PCM capture node (worklet preferred) and starts capture. */
+    private prepareEncoder;
+    /** Builds the mix graph + capture node, retrying once at the device rate if 48 kHz cannot take the mic. */
     private startMixedRecording;
+    /**
+     * Wires up the Web Audio mix graph + PCM capture node (worklet preferred) and starts capture.
+     * Resolves `'mic-rejected'` when a forced-48 kHz context refused the microphone source (the graph
+     * is released so the caller can rebuild at the device rate); otherwise `'done'`.
+     */
+    private buildGraph;
+    /**
+     * A 48 kHz context when asked for and the browser allows it, else the device-rate default. A
+     * browser that silently picks another rate is fine too: the resampler is chosen from the rate the
+     * context actually reports.
+     */
+    private createAudioContext;
     /**
      * Connects the agent stream, degrading to mic-only on failure. Unlike the mic (whose failure
      * rightly disables the recorder), the agent stream is best-effort: a browser may reject it
@@ -152,20 +154,26 @@ export declare class RealtimeAudioRecorder {
     private createScriptProcessorNode;
     /** The active capture node (worklet or script processor), or `null` before setup. */
     private captureNode;
-    /** Absorbs one captured PCM frame: accumulate samples + update the bounded peak array. */
+    /**
+     * Absorbs one captured frame: resample to 48 kHz, count it, and hand it to the encoder (or buffer
+     * it while the encoder loads).
+     *
+     * Past the pending-buffer cap a frame is dropped AND not counted. Keeping the count equal to the
+     * audio actually handed to the encoder is what lets the runtime trust `packets × 960 ≥ captured`
+     * when it trims the final shard; a few lost seconds at the start of a call whose encoder never
+     * loaded is the smaller harm.
+     */
     private captureFrame;
     /** Routes one stream's audio tracks into the capture node; returns the source, or null when track-less / no node. */
     private connectStream;
-    /** Concatenates a half-open sample range [from, to) across the captured frames into one Float32Array. */
-    private collectSamples;
     /**
-     * Converts a mono Float32 buffer to a little-endian PCM16 `ArrayBuffer` (for raw crash-recovery
-     * shards). Returns the backing `ArrayBuffer` directly so it's a valid `BlobPart`.
+     * Turns recording off after a setup failure and releases the graph. Audio already captured stays
+     * queued for the encoder, so a graph failure mid-call still keeps what was recorded before it.
      */
-    private floatToPcm16Bytes;
-    /** Encodes all accumulated PCM into a single seekable WAV blob, or `null` when nothing captured. */
-    private encodeWavBlob;
-    /** Releases the audio graph + capture node and resets state. Idempotent. */
-    private cleanup;
+    private disable;
+    /** Releases the audio graph + capture node. Idempotent; leaves the encoder and the sample count alone. */
+    private releaseGraph;
+    private disconnectNode;
+    private closeContext;
 }
 //# sourceMappingURL=realtime-audio-recorder.d.ts.map

package/dist/lib/services/realtime-audio-recorder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"realtime-audio-recorder.d.ts","sourceRoot":"","sources":["../../../src/lib/services/realtime-audio-recorder.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,qBAAqB;IAC9B,2EAA2E;IAC3E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAe;IAClD,4EAA4E;IAC5E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAO;IAC1C,mGAAmG;IACnG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAQ;IACzD,kGAAkG;IAClG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAY5C;IAEF,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,WAAW,CAAgD;IACnE,OAAO,CAAC,WAAW,CAAiC;IACpD,OAAO,CAAC,UAAU,CAAoC;IACtD,kGAAkG;IAClG,OAAO,CAAC,SAAS,CAAsB;IACvC,+FAA+F;IAC/F,OAAO,CAAC,YAAY,CAAK;IACzB,oFAAoF;IACpF,OAAO,CAAC,KAAK,CAA0D;IACvE,gGAAgG;IAChG,OAAO,CAAC,UAAU,CAAgB;IAClC,OAAO,CAAC,SAAS,CAAS;IAE1B,gGAAgG;IAChG,OAAO,CAAC,cAAc,CAAS;IAC/B,qGAAqG;IACrG,OAAO,CAAC,mBAAmB,CAA4B;IACvD,0GAA0G;IAC1G,OAAO,CAAC,SAAS,CAA4B;IAC7C,4EAA4E;IAC5E,OAAO,CAAC,SAAS,CAA2C;IAE5D,qGAAqG;IACrG,OAAO,CAAC,kBAAkB,CAAK;IAE/B,6FAA6F;IAC7F,OAAO,CAAC,WAAW,CAAK;IAExB,wGAAwG;IACxG,OAAO,CAAC,UAAU,CAAK;IAEvB;;;;OAIG;IACH,OAAO,CAAC,eAAe,CAAK;IAE5B,mDAAmD;IACnD,IAAW,WAAW,IAAI,OAAO,CAEhC;IAED,qEAAqE;IACrE,IAAW,WAAW,IAAI,MAAM,CAE/B;IAED;;;;OAIG;IACH,IAAW,QAAQ,IAAI,MAAM,CAE5B;IAED;;;OAGG;IACH,IAAW,UAAU,IAAI,MAAM,CAE9B;IAED;;;;;;;OAOG;IACI,KAAK,CAAC,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,GAAG,IAAI,GAAG,IAAI;IAwB5E;;;;;OAKG;IACI,kBAAkB,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IAYpD;;;;;OAKG;IACI,iBAAiB,CAAC,SAAS,EAAE,WAAW,GAAG,IAAI;IAYtD;;;;;;OAMG;IACI,kBAAkB,IAAI,IAAI,GAAG,IAAI;IAaxC,uEAAuE;IAChE,WAAW,IAAI,MAAM;IAO5B;;;;;OAKG;IACI,QAAQ,IAAI,MAAM,EAAE;IAI3B;;;;OAIG;IACU,IAAI,IAAI,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC;IAczC,kGAAkG;YACpF,mBAAmB;IAuCjC;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;IAU3B;;;;;OAKG;YACW,iBAAiB;IAO/B,oGAAoG;YACtF,oBAAoB;IAwBlC,sFAAsF;IACtF,OAAO,CAAC,yBAAyB;IAcjC,qFAAqF;IACrF,OAAO,CAAC,WAAW;IAInB,0FAA0F;IAC1F,OAAO,CAAC,YAAY;IAWpB,qHAAqH;IACrH,OAAO,CAAC,aAAa;IAUrB,yGAAyG;IACzG,OAAO,CAAC,cAAc;IA0BtB;;;OAGG;IACH,OAAO,CAAC,iBAAiB;IAUzB,oGAAoG;IACpG,OAAO,CAAC,aAAa;IASrB,4EAA4E;IAC5E,OAAO,CAAC,OAAO;CA4BlB"}
+{"version":3,"file":"realtime-audio-recorder.d.ts","sourceRoot":"","sources":["../../../src/lib/services/realtime-audio-recorder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAE5E,OAAO,EAAE,kBAAkB,EAA2B,MAAM,iCAAiC,CAAC;AAE9F;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,qBAAsB,YAAW,wBAAwB;IA+DtD,OAAO,CAAC,QAAQ,CAAC,aAAa;IA9D1C,mGAAmG;IACnG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAQ;IACzD;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,CAA8B;IACvE,kGAAkG;IAClG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAY5C;IAEF,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,WAAW,CAAgD;IACnE,OAAO,CAAC,WAAW,CAAiC;IACpD,OAAO,CAAC,UAAU,CAAoC;IACtD,6FAA6F;IAC7F,OAAO,CAAC,SAAS,CAAgC;IACjD,OAAO,CAAC,SAAS,CAAS;IAE1B,gGAAgG;IAChG,OAAO,CAAC,cAAc,CAAS;IAC/B,qGAAqG;IACrG,OAAO,CAAC,mBAAmB,CAA4B;IACvD,0GAA0G;IAC1G,OAAO,CAAC,SAAS,CAA4B;IAC7C,4EAA4E;IAC5E,OAAO,CAAC,SAAS,CAA2C;IAE5D,uEAAuE;IACvE,OAAO,CAAC,eAAe,CAAK;IAC5B,wGAAwG;IACxG,OAAO,CAAC,OAAO,CAAmC;IAClD,4GAA4G;IAC5G,OAAO,CAAC,YAAY,CAAmD;IACvE,0FAA0F;IAC1F,OAAO,CAAC,aAAa,CAAsB;IAC3C,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,qBAAqB,CAAS;IACtC,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAgC;IAEhD;;;;;OAKG;IACH,OAAO,CAAC,eAAe,CAAK;IAE5B,gFAAgF;gBACnD,aAAa,GAAE,MAAM,OAAO,CAAC,kBAAkB,CAAmC;IAE/G,mDAAmD;IACnD,IAAW,WAAW,IAAI,OAAO,CAEhC;IAED;;;;;;;OAOG;IACI,KAAK,CAAC,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,WAAW,GAAG,IAAI,GAAG,IAAI;IAwB5E;;;;;OAKG;IACI,kBAAkB,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IAYpD;;;;;OAKG;IACI,iBAAiB,CAAC,SAAS,EAAE,WAAW,GAAG,IAAI;IAYtD,4FAA4F;IACrF,WAAW,IAAI,MAAM;IAI5B,6GAA6G;IACtG,kBAAkB,IAAI,UAAU,EAAE;IAIzC;;;;;OAKG;IACI,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;YAOhB,QAAQ;IAetB;;;OAGG;YACW,cAAc;IAsB5B,2GAA2G;YAC7F,mBAAmB;IAOjC;;;;OAIG;YACW,UAAU;IAuDxB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAW1B;;;;;OAKG;IACH,OAAO,CAAC,mBAAmB;IAU3B;;;;;OAKG;YACW,iBAAiB;IAO/B,oGAAoG;YACtF,oBAAoB;IAwBlC,sFAAsF;IACtF,OAAO,CAAC,yBAAyB;IAWjC,qFAAqF;IACrF,OAAO,CAAC,WAAW;IAInB;;;;;;;;OAQG;IACH,OAAO,CAAC,YAAY;IAwBpB,qHAAqH;IACrH,OAAO,CAAC,aAAa;IAUrB;;;OAGG;IACH,OAAO,CAAC,OAAO;IAKf,0GAA0G;IAC1G,OAAO,CAAC,YAAY;IAwBpB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,YAAY;CAMvB"}