@memberjunction/ng-conversations 6.2.0-edge.4 → 6.2.0-edge.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. package/README.md +29 -0
  2. package/dist/lib/components/attachment/image-viewer.component.d.ts +3 -1
  3. package/dist/lib/components/attachment/image-viewer.component.d.ts.map +1 -1
  4. package/dist/lib/components/attachment/image-viewer.component.js +5 -1
  5. package/dist/lib/components/attachment/image-viewer.component.js.map +1 -1
  6. package/dist/lib/components/conversation/conversation-agent-picker.component.d.ts +1 -1
  7. package/dist/lib/components/conversation/conversation-agent-picker.component.js +3 -3
  8. package/dist/lib/components/conversation/conversation-agent-picker.component.js.map +1 -1
  9. package/dist/lib/components/conversation/conversation-chat-area.component.d.ts +88 -20
  10. package/dist/lib/components/conversation/conversation-chat-area.component.d.ts.map +1 -1
  11. package/dist/lib/components/conversation/conversation-chat-area.component.js +785 -387
  12. package/dist/lib/components/conversation/conversation-chat-area.component.js.map +1 -1
  13. package/dist/lib/components/message/message-input.component.d.ts +60 -20
  14. package/dist/lib/components/message/message-input.component.d.ts.map +1 -1
  15. package/dist/lib/components/message/message-input.component.js +290 -389
  16. package/dist/lib/components/message/message-input.component.js.map +1 -1
  17. package/dist/lib/components/message/message-item.component.d.ts +153 -17
  18. package/dist/lib/components/message/message-item.component.d.ts.map +1 -1
  19. package/dist/lib/components/message/message-item.component.js +1400 -299
  20. package/dist/lib/components/message/message-item.component.js.map +1 -1
  21. package/dist/lib/components/message/message-list.component.d.ts +25 -2
  22. package/dist/lib/components/message/message-list.component.d.ts.map +1 -1
  23. package/dist/lib/components/message/message-list.component.js +79 -40
  24. package/dist/lib/components/message/message-list.component.js.map +1 -1
  25. package/dist/lib/components/overlay/chat-overlay.component.d.ts +19 -0
  26. package/dist/lib/components/overlay/chat-overlay.component.d.ts.map +1 -1
  27. package/dist/lib/components/overlay/chat-overlay.component.js +125 -13
  28. package/dist/lib/components/overlay/chat-overlay.component.js.map +1 -1
  29. package/dist/lib/components/project/project-form-modal.component.js +2 -2
  30. package/dist/lib/components/project/project-form-modal.component.js.map +1 -1
  31. package/dist/lib/components/realtime/realtime-agent-banner.component.d.ts +6 -1
  32. package/dist/lib/components/realtime/realtime-agent-banner.component.d.ts.map +1 -1
  33. package/dist/lib/components/realtime/realtime-agent-banner.component.js +110 -82
  34. package/dist/lib/components/realtime/realtime-agent-banner.component.js.map +1 -1
  35. package/dist/lib/components/realtime/realtime-call-failure-notice.d.ts +13 -0
  36. package/dist/lib/components/realtime/realtime-call-failure-notice.d.ts.map +1 -0
  37. package/dist/lib/components/realtime/realtime-call-failure-notice.js +21 -0
  38. package/dist/lib/components/realtime/realtime-call-failure-notice.js.map +1 -0
  39. package/dist/lib/components/realtime/realtime-composer.component.js +114 -102
  40. package/dist/lib/components/realtime/realtime-composer.component.js.map +1 -1
  41. package/dist/lib/components/realtime/realtime-session-overlay.component.d.ts +21 -2
  42. package/dist/lib/components/realtime/realtime-session-overlay.component.d.ts.map +1 -1
  43. package/dist/lib/components/realtime/realtime-session-overlay.component.js +113 -76
  44. package/dist/lib/components/realtime/realtime-session-overlay.component.js.map +1 -1
  45. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.d.ts.map +1 -1
  46. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.js +7 -1
  47. package/dist/lib/components/realtime/remote-browser/remote-browser-channel.js.map +1 -1
  48. package/dist/lib/components/sidebar/routines-section.component.js +1 -1
  49. package/dist/lib/components/sidebar/routines-section.component.js.map +1 -1
  50. package/dist/lib/components/task/tasks-full-view.component.d.ts +152 -31
  51. package/dist/lib/components/task/tasks-full-view.component.d.ts.map +1 -1
  52. package/dist/lib/components/task/tasks-full-view.component.js +2227 -214
  53. package/dist/lib/components/task/tasks-full-view.component.js.map +1 -1
  54. package/dist/lib/components/tasks/tasks-dropdown.component.js +4 -4
  55. package/dist/lib/components/tasks/tasks-dropdown.component.js.map +1 -1
  56. package/dist/lib/components/workspace/conversation-workspace.component.d.ts +11 -1
  57. package/dist/lib/components/workspace/conversation-workspace.component.d.ts.map +1 -1
  58. package/dist/lib/components/workspace/conversation-workspace.component.js +25 -28
  59. package/dist/lib/components/workspace/conversation-workspace.component.js.map +1 -1
  60. package/dist/lib/services/agent-state.service.d.ts.map +1 -1
  61. package/dist/lib/services/agent-state.service.js +8 -8
  62. package/dist/lib/services/agent-state.service.js.map +1 -1
  63. package/dist/lib/services/browser-realtime-media-host.d.ts +7 -7
  64. package/dist/lib/services/browser-realtime-media-host.js +8 -78
  65. package/dist/lib/services/browser-realtime-media-host.js.map +1 -1
  66. package/dist/lib/services/conversation-agent.service.d.ts +53 -20
  67. package/dist/lib/services/conversation-agent.service.d.ts.map +1 -1
  68. package/dist/lib/services/conversation-agent.service.js +108 -54
  69. package/dist/lib/services/conversation-agent.service.js.map +1 -1
  70. package/dist/lib/services/conversation-detail-window.store.d.ts +5 -0
  71. package/dist/lib/services/conversation-detail-window.store.d.ts.map +1 -1
  72. package/dist/lib/services/conversation-detail-window.store.js +52 -5
  73. package/dist/lib/services/conversation-detail-window.store.js.map +1 -1
  74. package/dist/lib/services/conversation-naming.d.ts.map +1 -1
  75. package/dist/lib/services/conversation-naming.js +9 -2
  76. package/dist/lib/services/conversation-naming.js.map +1 -1
  77. package/dist/lib/services/conversation-tasks-filter.d.ts +15 -0
  78. package/dist/lib/services/conversation-tasks-filter.d.ts.map +1 -0
  79. package/dist/lib/services/conversation-tasks-filter.js +69 -0
  80. package/dist/lib/services/conversation-tasks-filter.js.map +1 -0
  81. package/dist/lib/services/realtime-audio-recorder.d.ts +94 -86
  82. package/dist/lib/services/realtime-audio-recorder.d.ts.map +1 -1
  83. package/dist/lib/services/realtime-audio-recorder.js +222 -195
  84. package/dist/lib/services/realtime-audio-recorder.js.map +1 -1
  85. package/dist/lib/services/realtime-session.service.d.ts +30 -2
  86. package/dist/lib/services/realtime-session.service.d.ts.map +1 -1
  87. package/dist/lib/services/realtime-session.service.js +35 -7
  88. package/dist/lib/services/realtime-session.service.js.map +1 -1
  89. package/dist/lib/services/recording/libopus-wasm-binary.d.ts +3 -0
  90. package/dist/lib/services/recording/libopus-wasm-binary.d.ts.map +1 -0
  91. package/dist/lib/services/recording/libopus-wasm-binary.js +489 -0
  92. package/dist/lib/services/recording/libopus-wasm-binary.js.map +1 -0
  93. package/dist/lib/services/recording/libopus-worker-source.d.ts +48 -0
  94. package/dist/lib/services/recording/libopus-worker-source.d.ts.map +1 -0
  95. package/dist/lib/services/recording/libopus-worker-source.js +170 -0
  96. package/dist/lib/services/recording/libopus-worker-source.js.map +1 -0
  97. package/dist/lib/services/recording/linear-resampler.d.ts +45 -0
  98. package/dist/lib/services/recording/linear-resampler.d.ts.map +1 -0
  99. package/dist/lib/services/recording/linear-resampler.js +85 -0
  100. package/dist/lib/services/recording/linear-resampler.js.map +1 -0
  101. package/dist/lib/services/recording/opus-packet-encoder.d.ts +50 -0
  102. package/dist/lib/services/recording/opus-packet-encoder.d.ts.map +1 -0
  103. package/dist/lib/services/recording/opus-packet-encoder.js +41 -0
  104. package/dist/lib/services/recording/opus-packet-encoder.js.map +1 -0
  105. package/dist/lib/services/recording/wasm-opus-encoder.d.ts +86 -0
  106. package/dist/lib/services/recording/wasm-opus-encoder.d.ts.map +1 -0
  107. package/dist/lib/services/recording/wasm-opus-encoder.js +232 -0
  108. package/dist/lib/services/recording/wasm-opus-encoder.js.map +1 -0
  109. package/dist/lib/services/recording/webcodecs-opus-encoder.d.ts +39 -0
  110. package/dist/lib/services/recording/webcodecs-opus-encoder.d.ts.map +1 -0
  111. package/dist/lib/services/recording/webcodecs-opus-encoder.js +101 -0
  112. package/dist/lib/services/recording/webcodecs-opus-encoder.js.map +1 -0
  113. package/dist/lib/services/search.service.d.ts.map +1 -1
  114. package/dist/lib/services/search.service.js +3 -16
  115. package/dist/lib/services/search.service.js.map +1 -1
  116. package/dist/lib/utils/agent-failure-display.d.ts +14 -0
  117. package/dist/lib/utils/agent-failure-display.d.ts.map +1 -0
  118. package/dist/lib/utils/agent-failure-display.js +26 -0
  119. package/dist/lib/utils/agent-failure-display.js.map +1 -0
  120. package/dist/lib/utils/distinct-artifacts.d.ts +9 -3
  121. package/dist/lib/utils/distinct-artifacts.d.ts.map +1 -1
  122. package/dist/lib/utils/distinct-artifacts.js +24 -5
  123. package/dist/lib/utils/distinct-artifacts.js.map +1 -1
  124. package/dist/lib/utils/message-list-order.d.ts +13 -0
  125. package/dist/lib/utils/message-list-order.d.ts.map +1 -0
  126. package/dist/lib/utils/message-list-order.js +31 -0
  127. package/dist/lib/utils/message-list-order.js.map +1 -0
  128. package/dist/lib/utils/pin-state.d.ts +11 -0
  129. package/dist/lib/utils/pin-state.d.ts.map +1 -0
  130. package/dist/lib/utils/pin-state.js +9 -0
  131. package/dist/lib/utils/pin-state.js.map +1 -0
  132. package/dist/public-api.d.ts +1 -0
  133. package/dist/public-api.d.ts.map +1 -1
  134. package/dist/public-api.js +1 -0
  135. package/dist/public-api.js.map +1 -1
  136. package/package.json +36 -36
  137. package/dist/lib/services/browser-recording-unload-guard.d.ts +0 -30
  138. package/dist/lib/services/browser-recording-unload-guard.d.ts.map +0 -1
  139. package/dist/lib/services/browser-recording-unload-guard.js +0 -43
  140. package/dist/lib/services/browser-recording-unload-guard.js.map +0 -1
  141. package/dist/lib/services/realtime-pcm-wav.d.ts +0 -79
  142. package/dist/lib/services/realtime-pcm-wav.d.ts.map +0 -1
  143. package/dist/lib/services/realtime-pcm-wav.js +0 -213
  144. package/dist/lib/services/realtime-pcm-wav.js.map +0 -1
@@ -1,40 +1,46 @@
1
- import { EncodePcm16Wav, PeakAccumulator } from './realtime-pcm-wav';
1
+ import { RECORDING_SAMPLE_RATE } from '@memberjunction/ai-core-plus';
2
+ import { LinearResampler } from './recording/linear-resampler';
3
+ import { SelectOpusPacketEncoder } from './recording/opus-packet-encoder';
2
4
  /**
3
- * Browser-side audio recorder for a CLIENT-DIRECT realtime voice session.
5
+ * Browser-side audio recorder for a CLIENT-DIRECT realtime voice session, implementing the
6
+ * runtime's {@link IRealtimeSessionRecorder} contract directly.
4
7
  *
5
- * Mixes the user's microphone stream with the agent's remote-audio stream (every driver exposes
6
- * one via `GetRemoteMediaStream()`: WebRTC drivers publish their remote track, PCM drivers —
7
- * Gemini, ElevenLabs, AssemblyAI, xAI, HuggingFace — publish their playout engine's output) into a
8
- * single mono **WAV** (16-bit PCM) blob via the Web Audio API. Rather than a `MediaRecorder` (which only emits webm/opus —
9
- * header-less, no duration/cues, so HTTP-range seeking is unreliable), the mix is routed into an
10
- * {@link AudioWorkletNode} (preferred) or a {@link ScriptProcessorNode} (fallback) that captures
11
- * Float32 PCM frames. PCM is accumulated in memory and encoded as a seekable RIFF/WAVE file at
12
- * {@link Stop}. WAV's header carries the exact duration and maps byte-offset → time linearly, so
13
- * range seeking is exact and waveform peaks are computed cheaply during capture. The tradeoff is
14
- * size (PCM WAV is larger than opus) — acceptable for these short session recordings.
8
+ * Mixes the user's microphone with the agent's remote-audio stream (every driver exposes one via
9
+ * `GetRemoteMediaStream()`: WebRTC drivers publish their remote track, PCM drivers — Gemini,
10
+ * ElevenLabs, AssemblyAI, xAI, HuggingFace — publish their playout engine's output) in a Web Audio
11
+ * graph, captures the mix as mono Float32 frames through an {@link AudioWorkletNode} (preferred) or a
12
+ * {@link ScriptProcessorNode} (fallback), and feeds them to ONE continuous Opus encoder for the whole
13
+ * call. The runtime pulls the encoded packets with {@link TakeEncodedPackets}, cuts them into shards
14
+ * and uploads them as the call goes, so nothing is held in memory until the end.
15
15
  *
16
- * Robust + MVP by design — every step degrades gracefully:
16
+ * Timing is the 48 kHz sample clock: the graph asks for a 48 kHz context, and when the browser
17
+ * refuses (or reports another rate) every frame is resampled to 48 kHz before encoding. The
18
+ * captured-sample count drives {@link NowOffsetMs} and is what {@link Stop} resolves.
19
+ *
20
+ * Robust by design — every step degrades gracefully:
17
21
  * - No `AudioContext` (server-side render / unsupported env) → the recorder disables itself
18
- * ({@link IsRecording} stays `false`, {@link Stop} resolves `null`); the voice call proceeds.
22
+ * ({@link IsRecording} stays `false`, {@link Stop} resolves 0); the voice call proceeds.
19
23
  * - No `AudioWorkletNode` AND no `ScriptProcessorNode` → disables itself rather than throwing.
24
+ * - A forced-48 kHz context that cannot take the microphone (Firefox rejects a source whose rate
25
+ * differs from its context's) → rebuilt once at the device rate with resampling.
20
26
  * - No remote stream yet (e.g. the WebRTC track hasn't landed) → records mic only until it does.
21
- * - A remote stream that can't be connected (e.g. a browser that rejects cross-sample-rate
22
- * `createMediaStreamSource` — the PCM playout context runs at 24 kHz, this one at the device
23
- * rate) → warns and records mic only; the mic recording is never lost to the agent stream.
24
- * - Any construction/start error is logged and disables the recorder rather than throwing into
25
- * the session-start path.
27
+ * - A remote stream that can't be connected (e.g. a cross-sample-rate `createMediaStreamSource`
28
+ * rejection) → warns and records mic only; the mic recording is never lost to the agent stream.
29
+ * - No usable encoder → logged, recording disabled for the call.
26
30
  *
27
- * Not an Angular service: it owns transient per-session media plumbing the
28
- * {@link RealtimeSessionService} creates and discards per call, so a plain class is the right
29
- * shape (no DI, no singleton).
31
+ * Not an Angular service: it owns transient per-session media plumbing the runtime creates and
32
+ * discards per call, so a plain class is the right shape (no DI, no singleton).
30
33
  */
31
34
  export class RealtimeAudioRecorder {
32
- /** The container/codec we now produce — a seekable mono 16-bit PCM WAV. */
33
- static wavMimeType = 'audio/wav';
34
- /** Target waveform-peak resolution (buckets) across the whole recording. */
35
- static peakBuckets = 600;
35
+ selectEncoder;
36
36
  /** PCM frame size for the ScriptProcessor fallback (worklet uses its native 128-frame quantum). */
37
37
  static scriptProcessorBufferSize = 4096;
38
+ /**
39
+ * Most 48 kHz audio held while the encoder is still being selected / loaded (10 s). Selection is
40
+ * normally well under a second; past this the encoder is stuck, and later frames are dropped
41
+ * (see {@link captureFrame} for why they are not counted either).
42
+ */
43
+ static maxPendingSamples = 10 * RECORDING_SAMPLE_RATE;
38
44
  /** Inline AudioWorklet processor: forwards each 128-frame mono input block to the main thread. */
39
45
  static workletProcessorSource = `
40
46
  class MJPcmCaptureProcessor extends AudioWorkletProcessor {
@@ -53,14 +59,8 @@ export class RealtimeAudioRecorder {
53
59
  destination = null;
54
60
  workletNode = null;
55
61
  scriptNode = null;
56
- /** Captured mono Float32 PCM frames (in capture order); concatenated + encoded to WAV at Stop. */
57
- pcmFrames = [];
58
- /** Total captured sample count across {@link pcmFrames} (kept in step to avoid re-summing). */
59
- totalSamples = 0;
60
- /** Streaming waveform-peak accumulator — bounded regardless of recording length. */
61
- peaks = new PeakAccumulator(RealtimeAudioRecorder.peakBuckets);
62
- /** Final normalized peaks, snapshotted at {@link Stop} so {@link GetPeaks} survives cleanup. */
63
- finalPeaks = [];
62
+ /** Converts the context's rate to 48 kHz; `null` when the context already runs at 48 kHz. */
63
+ resampler = null;
64
64
  recording = false;
65
65
  /** True once the agent's remote stream has been wired into the mix (prevents double-attach). */
66
66
  remoteAttached = false;
@@ -70,45 +70,37 @@ export class RealtimeAudioRecorder {
70
70
  micStream = null;
71
71
  /** The source node reading the microphone, swapped by ReplaceMicrophone. */
72
72
  micSource = null;
73
- /** Sample index up to which crash-recovery shards have already been emitted (the segment cursor). */
74
- flushedSampleCount = 0;
75
- /** Wall-clock ms at recording start; anchors {@link NowOffsetMs}. 0 before {@link Start}. */
76
- startedAtMs = 0;
77
- /** Sample rate of the capturing {@link AudioContext}; 0 before {@link Start}. Needed to wrap shards. */
78
- sampleRate = 0;
73
+ /** 48 kHz samples captured (post-resample) — the recording's clock. */
74
+ capturedSamples = 0;
75
+ /** The call's encoder once selected; `null` while selecting, after a failure, or when not recording. */
76
+ encoder = null;
77
+ /** Resolves to the encoder (or `null` on failure) — Stop awaits it so buffered audio still gets encoded. */
78
+ encoderReady = null;
79
+ /** 48 kHz frames captured before the encoder was ready, fed to it in order once it is. */
80
+ pendingFrames = [];
81
+ pendingSamples = 0;
82
+ pendingOverflowWarned = false;
83
+ /** The first Stop's promise; later calls return it (idempotent). */
84
+ stopping = null;
79
85
  /**
80
- * Bumped by {@link cleanup}. The async setup snapshots it and, after each await, gives up if it moved:
81
- * a {@link Stop} that lands mid-setup can only release what setup has already published, so setup
82
- * must release the rest itself rather than build a live graph on a stopped recorder.
86
+ * Bumped whenever the audio graph is released. The async graph setup snapshots it and, after each
87
+ * await, gives up if it moved: a {@link Stop} that lands mid-setup can only release what setup has
88
+ * already published, so setup must release the rest itself rather than build a live graph on a
89
+ * stopped recorder.
83
90
  */
84
91
  setupGeneration = 0;
92
+ /** @param selectEncoder picks the call's Opus encoder; injectable for tests. */
93
+ constructor(selectEncoder = () => SelectOpusPacketEncoder()) {
94
+ this.selectEncoder = selectEncoder;
95
+ }
85
96
  /** True while a recorder is actively capturing. */
86
97
  get IsRecording() {
87
98
  return this.recording;
88
99
  }
89
- /** Wall-clock ms when recording started (0 before {@link Start}). */
90
- get StartedAtMs() {
91
- return this.startedAtMs;
92
- }
93
- /**
94
- * The container/codec the active recording uses. Always `'audio/wav'` once recording (empty
95
- * string before {@link Start} / when disabled) — the consolidated upload and the WAV-wrapped
96
- * crash-recovery shards are all 16-bit PCM WAV.
97
- */
98
- get MimeType() {
99
- return this.recording ? RealtimeAudioRecorder.wavMimeType : '';
100
- }
101
- /**
102
- * The captured AudioContext sample rate (0 before {@link Start} / when disabled). Exposed so a
103
- * recovery path can WAV-wrap the raw-PCM shards with the correct rate.
104
- */
105
- get SampleRate() {
106
- return this.sampleRate;
107
- }
108
100
  /**
109
101
  * Begins recording a mix of `micStream` and (when present) `remoteStream`. Best-effort:
110
102
  * a missing `AudioContext` / no worklet+scriptprocessor support / any setup failure disables
111
- * the recorder silently (logged) so the live call is never blocked.
103
+ * the recorder (logged) so the live call is never blocked.
112
104
  *
113
105
  * @param micStream The user's microphone capture stream (required).
114
106
  * @param remoteStream The agent's remote-audio stream, or `null` for mic-only capture.
@@ -121,19 +113,19 @@ export class RealtimeAudioRecorder {
121
113
  console.warn('[RealtimeAudioRecorder] AudioContext unavailable — recording disabled for this session.');
122
114
  return;
123
115
  }
124
- // Mark recording + stamp t0 SYNCHRONOUSLY so a caller that checks IsRecording immediately after
125
- // Start() (the session service does) sees `true`, even though the audio-graph setup
126
- // (AudioContext.resume + worklet module load) completes asynchronously below. Without this the
127
- // caller discards the recorder before it finishes starting and nothing is ever uploaded.
128
- this.startedAtMs = Date.now();
129
- this.finalPeaks = [];
116
+ // Mark recording SYNCHRONOUSLY so a caller that checks IsRecording immediately after Start()
117
+ // (the runtime does) sees `true`, even though the audio graph and the encoder are set up
118
+ // asynchronously below.
130
119
  this.recording = true;
120
+ this.stopping = null;
121
+ this.capturedSamples = 0;
122
+ this.encoderReady = this.prepareEncoder();
131
123
  this.micStream = micStream;
132
124
  // Fire-and-forget the async setup (it must `await audioContext.resume()` + worklet load); any
133
125
  // failure disables the recorder rather than throwing into the session-start path.
134
126
  void this.startMixedRecording(micStream, remoteStream).catch((error) => {
135
127
  console.warn('[RealtimeAudioRecorder] Failed to start recording — disabling for this session:', error);
136
- this.cleanup();
128
+ this.disable();
137
129
  });
138
130
  }
139
131
  /**
@@ -171,63 +163,82 @@ export class RealtimeAudioRecorder {
171
163
  this.micSource = replacement;
172
164
  }
173
165
  }
166
+ /** Milliseconds of audio captured so far, on the 48 kHz sample clock (never wall clock). */
167
+ NowOffsetMs() {
168
+ return (this.capturedSamples * 1000) / RECORDING_SAMPLE_RATE;
169
+ }
170
+ /** The Opus packets the encoder produced since the last call (also its final packets after {@link Stop}). */
171
+ TakeEncodedPackets() {
172
+ return this.encoder ? this.encoder.TakePackets() : [];
173
+ }
174
174
  /**
175
- * Returns a Blob of the raw PCM captured since the last call (a window of header-less,
176
- * little-endian PCM16 bytes), advancing the segment cursor. `null` when nothing new. Used to
177
- * upload crash-recovery shards mid-call. NOTE: shards are header-less raw PCM (NOT individually
178
- * playable WAV) — recovery is concatenation in order, then WAV-wrapping with {@link SampleRate}.
179
- * The canonical consolidated file is still authored as a single WAV from full memory at {@link Stop}.
175
+ * Stops capture, waits for the encoder (encoding any audio buffered while it loaded), flushes and
176
+ * closes it, and resolves the 48 kHz samples captured. The final packets stay available to
177
+ * {@link TakeEncodedPackets}. Idempotent: every call resolves the same count; 0 when the recorder
178
+ * never started.
180
179
  */
181
- SnapshotNewSegment() {
182
- if (!this.recording || this.flushedSampleCount >= this.totalSamples) {
183
- return null;
180
+ Stop() {
181
+ if (!this.stopping) {
182
+ this.stopping = this.stopOnce();
184
183
  }
185
- const fresh = this.collectSamples(this.flushedSampleCount, this.totalSamples);
186
- this.flushedSampleCount = this.totalSamples;
187
- if (fresh.length === 0) {
188
- return null;
189
- }
190
- const pcm16 = this.floatToPcm16Bytes(fresh);
191
- return new Blob([pcm16], { type: 'application/octet-stream' });
184
+ return this.stopping;
192
185
  }
193
- /** Ms elapsed since recording started (>= 0); 0 when not recording. */
194
- NowOffsetMs() {
195
- if (this.startedAtMs === 0) {
196
- return 0;
186
+ async stopOnce() {
187
+ this.recording = false;
188
+ this.releaseGraph();
189
+ const encoderReady = this.encoderReady;
190
+ if (!encoderReady) {
191
+ return this.capturedSamples;
197
192
  }
198
- return Math.max(0, Date.now() - this.startedAtMs);
199
- }
200
- /**
201
- * The downsampled waveform peaks (each 0..1, max-abs per bucket, ~{@link RealtimeAudioRecorder.PeakBuckets}
202
- * buckets across the whole recording). Computed incrementally during capture so it's cheap and
203
- * bounded regardless of length. Returns `[]` before any audio / for a silent recording. Safe to
204
- * call after {@link Stop} — the peaks are snapshotted there before {@link cleanup} clears state.
205
- */
206
- GetPeaks() {
207
- return this.recording ? this.peaks.Normalize() : this.finalPeaks;
193
+ const encoder = await encoderReady;
194
+ if (encoder) {
195
+ await encoder.Flush();
196
+ encoder.Close();
197
+ }
198
+ return this.capturedSamples;
208
199
  }
209
200
  /**
210
- * Stops recording and resolves the captured audio as a single mono 16-bit PCM **WAV**
211
- * {@link Blob}, then releases the audio graph. Resolves `null` when nothing was recorded
212
- * (recorder disabled, or no samples captured). Safe to call when not recording.
201
+ * Selects the encoder and feeds it what was captured meanwhile. A selection failure disables the
202
+ * recorder: with no encoder nothing captured could ever reach the server.
213
203
  */
214
- async Stop() {
215
- if (!this.recording) {
216
- this.cleanup();
204
+ async prepareEncoder() {
205
+ let encoder;
206
+ try {
207
+ encoder = await this.selectEncoder();
208
+ }
209
+ catch (error) {
210
+ console.warn('[RealtimeAudioRecorder] No Opus encoder available — recording disabled for this session:', error);
211
+ this.disable();
212
+ // Nothing captured can be encoded now, so report nothing recorded rather than a count with no audio.
213
+ this.pendingFrames = [];
214
+ this.pendingSamples = 0;
215
+ this.capturedSamples = 0;
217
216
  return null;
218
217
  }
219
- // Detach the capture node so no further frames arrive while we assemble the file. Snapshot the
220
- // peaks BEFORE cleanup so GetPeaks() still returns them for the post-Stop upload.
221
- this.recording = false;
222
- this.finalPeaks = this.peaks.Normalize();
223
- const blob = this.encodeWavBlob();
224
- this.cleanup();
225
- return blob;
218
+ this.encoder = encoder;
219
+ for (const frame of this.pendingFrames) {
220
+ encoder.Encode(frame);
221
+ }
222
+ this.pendingFrames = [];
223
+ this.pendingSamples = 0;
224
+ return encoder;
226
225
  }
227
- /** Wires up the Web Audio mix graph + PCM capture node (worklet preferred) and starts capture. */
226
+ /** Builds the mix graph + capture node, retrying once at the device rate if 48 kHz cannot take the mic. */
228
227
  async startMixedRecording(micStream, remoteStream) {
228
+ const built = await this.buildGraph(micStream, remoteStream, true);
229
+ if (built === 'mic-rejected') {
230
+ await this.buildGraph(micStream, remoteStream, false);
231
+ }
232
+ }
233
+ /**
234
+ * Wires up the Web Audio mix graph + PCM capture node (worklet preferred) and starts capture.
235
+ * Resolves `'mic-rejected'` when a forced-48 kHz context refused the microphone source (the graph
236
+ * is released so the caller can rebuild at the device rate); otherwise `'done'`.
237
+ */
238
+ async buildGraph(micStream, remoteStream, prefer48k) {
229
239
  const generation = this.setupGeneration;
230
- const audioContext = new AudioContext();
240
+ const audioContext = this.createAudioContext(prefer48k);
241
+ const forced48k = prefer48k && audioContext.sampleRate === RECORDING_SAMPLE_RATE;
231
242
  // CRITICAL: browsers create an AudioContext in the `suspended` state under the autoplay policy.
232
243
  // Without resuming it, the mix graph never processes audio and we capture only silence. Starting
233
244
  // a voice call is a user gesture, so resume() succeeds here.
@@ -235,24 +246,39 @@ export class RealtimeAudioRecorder {
235
246
  await audioContext.resume();
236
247
  }
237
248
  if (generation !== this.setupGeneration) {
238
- // Stopped while resuming: the context was never published, so cleanup() could not close it.
239
- void audioContext.close().catch(() => { });
240
- return;
249
+ // Stopped while resuming: the context was never published, so releaseGraph() could not close it.
250
+ this.closeContext(audioContext);
251
+ return 'done';
241
252
  }
242
253
  const destination = audioContext.createMediaStreamDestination();
243
- // Publish the context + sample rate BEFORE wiring the remote so a concurrent AttachRemoteStream()
244
- // (the agent track can land during the awaits above) sees a ready graph to connect to.
254
+ // Publish the context BEFORE wiring the remote so a concurrent AttachRemoteStream() (the agent
255
+ // track can land during the awaits above) sees a ready graph to connect to.
245
256
  this.audioContext = audioContext;
246
257
  this.destination = destination;
247
- this.sampleRate = audioContext.sampleRate;
258
+ this.resampler = audioContext.sampleRate === RECORDING_SAMPLE_RATE ? null : new LinearResampler(audioContext.sampleRate, RECORDING_SAMPLE_RATE);
248
259
  // Build the PCM capture node: AudioWorklet preferred, ScriptProcessor fallback. Either way the
249
260
  // node both pulls audio from the mix AND drives the destination so the graph keeps processing.
250
261
  await this.createCaptureNode(audioContext, destination);
251
262
  if (generation !== this.setupGeneration) {
252
- return; // Stopped while the capture node loaded: cleanup() already closed the published context.
263
+ return 'done'; // Stopped while the capture node loaded: releaseGraph() already closed the published context.
264
+ }
265
+ try {
266
+ // The microphone may have been replaced while the graph was being built.
267
+ this.micSource = this.connectStream(audioContext, this.micStream ?? micStream);
268
+ }
269
+ catch (error) {
270
+ if (!forced48k) {
271
+ throw error;
272
+ }
273
+ console.warn('[RealtimeAudioRecorder] The 48 kHz context rejected the microphone — rebuilding at the device rate with resampling:', error);
274
+ // releaseGraph() forgets a stashed agent stream and a replaced microphone; the rebuild still has to connect both.
275
+ const stashedRemote = this.pendingRemoteStream;
276
+ const stashedMic = this.micStream;
277
+ this.releaseGraph();
278
+ this.pendingRemoteStream = stashedRemote;
279
+ this.micStream = stashedMic;
280
+ return 'mic-rejected';
253
281
  }
254
- // The microphone may have been replaced while the graph was being built.
255
- this.micSource = this.connectStream(audioContext, this.micStream ?? micStream);
256
282
  // Connect the agent's stream if it was passed now, or if AttachRemoteStream() stashed one while
257
283
  // we were setting up. Either way, capture the agent voice — not just the mic.
258
284
  const remote = remoteStream ?? this.pendingRemoteStream;
@@ -260,6 +286,23 @@ export class RealtimeAudioRecorder {
260
286
  if (remote && remote.getAudioTracks().length > 0) {
261
287
  this.remoteAttached = this.connectRemoteStream(audioContext, remote);
262
288
  }
289
+ return 'done';
290
+ }
291
+ /**
292
+ * A 48 kHz context when asked for and the browser allows it, else the device-rate default. A
293
+ * browser that silently picks another rate is fine too: the resampler is chosen from the rate the
294
+ * context actually reports.
295
+ */
296
+ createAudioContext(prefer48k) {
297
+ if (prefer48k) {
298
+ try {
299
+ return new AudioContext({ sampleRate: RECORDING_SAMPLE_RATE });
300
+ }
301
+ catch (error) {
302
+ console.warn('[RealtimeAudioRecorder] 48 kHz AudioContext refused — capturing at the device rate with resampling:', error);
303
+ }
304
+ }
305
+ return new AudioContext();
263
306
  }
264
307
  /**
265
308
  * Connects the agent stream, degrading to mic-only on failure. Unlike the mic (whose failure
@@ -319,10 +362,7 @@ export class RealtimeAudioRecorder {
319
362
  throw new Error('Neither AudioWorklet nor ScriptProcessor is supported');
320
363
  }
321
364
  const node = audioContext.createScriptProcessor(RealtimeAudioRecorder.scriptProcessorBufferSize, 1, 1);
322
- node.onaudioprocess = (event) => {
323
- // Copy out of the reused input buffer before retaining.
324
- this.captureFrame(event.inputBuffer.getChannelData(0).slice(0));
325
- };
365
+ node.onaudioprocess = (event) => this.captureFrame(event.inputBuffer.getChannelData(0));
326
366
  // ScriptProcessor only fires onaudioprocess while connected to the destination.
327
367
  node.connect(destination);
328
368
  this.scriptNode = node;
@@ -331,16 +371,39 @@ export class RealtimeAudioRecorder {
331
371
  captureNode() {
332
372
  return this.workletNode ?? this.scriptNode;
333
373
  }
334
- /** Absorbs one captured PCM frame: accumulate samples + update the bounded peak array. */
374
+ /**
375
+ * Absorbs one captured frame: resample to 48 kHz, count it, and hand it to the encoder (or buffer
376
+ * it while the encoder loads).
377
+ *
378
+ * Past the pending-buffer cap a frame is dropped AND not counted. Keeping the count equal to the
379
+ * audio actually handed to the encoder is what lets the runtime trust `packets × 960 ≥ captured`
380
+ * when it trims the final shard; a few lost seconds at the start of a call whose encoder never
381
+ * loaded is the smaller harm.
382
+ */
335
383
  captureFrame(frame) {
336
384
  if (!this.recording || frame.length === 0) {
337
385
  return;
338
386
  }
339
- // Defensive copy — worklet/script buffers are reused by the engine.
340
- const copy = frame.slice(0);
341
- this.pcmFrames.push(copy);
342
- this.totalSamples += copy.length;
343
- this.peaks.Push(copy);
387
+ // Copy out of the engine's reused buffer (a resampler output is already a fresh array).
388
+ const samples = this.resampler ? this.resampler.Process(frame) : frame.slice(0);
389
+ if (samples.length === 0) {
390
+ return;
391
+ }
392
+ if (this.encoder) {
393
+ this.encoder.Encode(samples);
394
+ }
395
+ else if (this.pendingSamples + samples.length <= RealtimeAudioRecorder.maxPendingSamples) {
396
+ this.pendingFrames.push(samples);
397
+ this.pendingSamples += samples.length;
398
+ }
399
+ else {
400
+ if (!this.pendingOverflowWarned) {
401
+ this.pendingOverflowWarned = true;
402
+ console.warn(`[RealtimeAudioRecorder] Opus encoder still not ready after ${RealtimeAudioRecorder.maxPendingSamples / RECORDING_SAMPLE_RATE} s of audio — dropping audio until it is.`);
403
+ }
404
+ return;
405
+ }
406
+ this.capturedSamples += samples.length;
344
407
  }
345
408
  /** Routes one stream's audio tracks into the capture node; returns the source, or null when track-less / no node. */
346
409
  connectStream(audioContext, stream) {
@@ -352,88 +415,52 @@ export class RealtimeAudioRecorder {
352
415
  source.connect(node);
353
416
  return source;
354
417
  }
355
- /** Concatenates a half-open sample range [from, to) across the captured frames into one Float32Array. */
356
- collectSamples(from, to) {
357
- const out = new Float32Array(Math.max(0, to - from));
358
- if (out.length === 0) {
359
- return out;
360
- }
361
- let cursor = 0; // absolute sample index at the start of the current frame
362
- let written = 0;
363
- for (const frame of this.pcmFrames) {
364
- const frameStart = cursor;
365
- const frameEnd = cursor + frame.length;
366
- cursor = frameEnd;
367
- if (frameEnd <= from) {
368
- continue; // entirely before the window
369
- }
370
- if (frameStart >= to) {
371
- break; // entirely after the window
372
- }
373
- const sliceStart = Math.max(0, from - frameStart);
374
- const sliceEnd = Math.min(frame.length, to - frameStart);
375
- for (let i = sliceStart; i < sliceEnd; i++) {
376
- out[written++] = frame[i];
377
- }
378
- }
379
- return written === out.length ? out : out.subarray(0, written);
380
- }
381
418
  /**
382
- * Converts a mono Float32 buffer to a little-endian PCM16 `ArrayBuffer` (for raw crash-recovery
383
- * shards). Returns the backing `ArrayBuffer` directly so it's a valid `BlobPart`.
419
+ * Turns recording off after a setup failure and releases the graph. Audio already captured stays
420
+ * queued for the encoder, so a graph failure mid-call still keeps what was recorded before it.
384
421
  */
385
- floatToPcm16Bytes(samples) {
386
- const buffer = new ArrayBuffer(samples.length * 2);
387
- const view = new DataView(buffer);
388
- for (let i = 0; i < samples.length; i++) {
389
- const s = samples[i] < -1 ? -1 : samples[i] > 1 ? 1 : samples[i];
390
- view.setInt16(i * 2, s < 0 ? s * 0x8000 : s * 0x7fff, true);
391
- }
392
- return buffer;
393
- }
394
- /** Encodes all accumulated PCM into a single seekable WAV blob, or `null` when nothing captured. */
395
- encodeWavBlob() {
396
- if (this.totalSamples === 0 || this.sampleRate === 0) {
397
- return null;
398
- }
399
- const all = this.collectSamples(0, this.totalSamples);
400
- const wav = EncodePcm16Wav(all, this.sampleRate);
401
- return new Blob([wav], { type: RealtimeAudioRecorder.wavMimeType });
422
+ disable() {
423
+ this.recording = false;
424
+ this.releaseGraph();
402
425
  }
403
- /** Releases the audio graph + capture node and resets state. Idempotent. */
404
- cleanup() {
426
+ /** Releases the audio graph + capture node. Idempotent; leaves the encoder and the sample count alone. */
427
+ releaseGraph() {
405
428
  this.setupGeneration++;
406
- this.recording = false;
407
- this.pcmFrames = [];
408
- this.totalSamples = 0;
409
- this.flushedSampleCount = 0;
410
- this.sampleRate = 0;
411
- this.peaks = new PeakAccumulator(RealtimeAudioRecorder.peakBuckets);
412
429
  this.remoteAttached = false;
413
430
  this.pendingRemoteStream = null;
414
431
  this.micStream = null;
415
432
  this.micSource = null;
416
433
  if (this.workletNode) {
417
434
  this.workletNode.port.onmessage = null;
418
- try {
419
- this.workletNode.disconnect();
420
- }
421
- catch { /* already disconnected */ }
435
+ this.disconnectNode(this.workletNode);
422
436
  this.workletNode = null;
423
437
  }
424
438
  if (this.scriptNode) {
425
439
  this.scriptNode.onaudioprocess = null;
426
- try {
427
- this.scriptNode.disconnect();
428
- }
429
- catch { /* already disconnected */ }
440
+ this.disconnectNode(this.scriptNode);
430
441
  this.scriptNode = null;
431
442
  }
432
443
  this.destination = null;
444
+ this.resampler = null;
433
445
  if (this.audioContext) {
434
- void this.audioContext.close().catch(() => { });
446
+ this.closeContext(this.audioContext);
435
447
  this.audioContext = null;
436
448
  }
437
449
  }
450
+ disconnectNode(node) {
451
+ try {
452
+ node.disconnect();
453
+ }
454
+ catch (error) {
455
+ // Expected when the node was never connected or is already disconnected; nothing to release.
456
+ console.debug('[RealtimeAudioRecorder] Capture node disconnect skipped:', error);
457
+ }
458
+ }
459
+ closeContext(audioContext) {
460
+ void audioContext.close().catch((error) => {
461
+ // A context that is already closed rejects here; the graph is released either way.
462
+ console.debug('[RealtimeAudioRecorder] AudioContext close skipped:', error);
463
+ });
464
+ }
438
465
  }
439
466
  //# sourceMappingURL=realtime-audio-recorder.js.map