@alexkroman1/aai-ui 1.14.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audio.d.ts +43 -5
- package/dist/audio.js +39 -20
- package/dist/chat-view-DKFhxMAT.js +129 -0
- package/dist/client-config.d.ts +7 -6
- package/dist/components/chat-view.d.ts +1 -8
- package/dist/components/chat-view.js +2 -2
- package/dist/components/console-shell.d.ts +37 -0
- package/dist/components/controls.js +1 -1
- package/dist/components/url-chips.d.ts +1 -1
- package/dist/components/workflow-view.d.ts +13 -0
- package/dist/{controls-BbZcmnJf.js → controls-4OJoekj6.js} +1 -1
- package/dist/default-client/assets/audio-Cgviqo9t.js +1 -0
- package/dist/default-client/assets/capture-processor-DLHxAIfT.js +90 -0
- package/dist/default-client/assets/index-BgbIWnfG.css +2 -0
- package/dist/default-client/assets/index-DbKKR3UE.js +62 -0
- package/dist/default-client/assets/playback-processor-bMTdFp-8.js +271 -0
- package/dist/default-client/assets/rolldown-runtime-BpQH8Ho1.js +1 -0
- package/dist/default-client/assets/types-Bpg3ZIZK.js +64 -0
- package/dist/default-client/index.html +5 -2
- package/dist/define-client-yoGybEYR.js +710 -0
- package/dist/define-client.d.ts +0 -9
- package/dist/define-client.js +1 -1
- package/dist/index.d.ts +3 -4
- package/dist/index.js +6 -5
- package/dist/{session-core-B64kau_v.js → session-core-BLhiQ18c.js} +6 -0
- package/dist/session-core.js +1 -1
- package/dist/sync-mic.d.ts +30 -43
- package/dist/types.d.ts +24 -1
- package/dist/types.js +32 -2
- package/dist/worklets/capture-processor.d.ts +1 -1
- package/dist/worklets/capture-processor.js +32 -49
- package/dist/worklets/playback-processor.d.ts +1 -1
- package/dist/worklets/playback-processor.js +125 -9
- package/package.json +2 -2
- package/dist/chat-view-gi6FccZq.js +0 -193
- package/dist/components/sync-chat-view.d.ts +0 -18
- package/dist/components/text-controls.d.ts +0 -14
- package/dist/default-client/assets/audio-CTPVKBQ_.js +0 -1
- package/dist/default-client/assets/capture-processor-UlKEKyIW.js +0 -108
- package/dist/default-client/assets/index-CF3RKVUo.css +0 -2
- package/dist/default-client/assets/index-I5mZ1vB1.js +0 -121
- package/dist/default-client/assets/playback-processor-C5HVRVbu.js +0 -156
- package/dist/define-client-Dx4JaWzL.js +0 -881
- package/dist/sync-vad.d.ts +0 -54
package/dist/define-client.d.ts
CHANGED
|
@@ -21,15 +21,6 @@ type BaseOptions = {
|
|
|
21
21
|
resumeSessionId?: string;
|
|
22
22
|
/** WebSocket constructor override. Passed through to session options. */
|
|
23
23
|
WebSocket?: WebSocketConstructor;
|
|
24
|
-
/**
|
|
25
|
-
* Transport the UI talks over. Unset (the default) asks the server via
|
|
26
|
-
* `GET client-config`, so `agent({ transport })` decides — an agent that
|
|
27
|
-
* declared `transport: "sync"` gets the sync shell (HTTP turns, no
|
|
28
|
-
* WebSocket) with no custom client needed. An explicit value here skips
|
|
29
|
-
* the lookup. Only the config tier branches on it; a custom `component`
|
|
30
|
-
* owns its own transport.
|
|
31
|
-
*/
|
|
32
|
-
transport?: "websocket" | "sync";
|
|
33
24
|
};
|
|
34
25
|
/**
|
|
35
26
|
* Tier 1: Config-only options. Renders the default shell (StartScreen + ChatView).
|
package/dist/define-client.js
CHANGED
package/dist/index.d.ts
CHANGED
|
@@ -5,11 +5,10 @@ export { Controls } from "./components/controls.tsx";
|
|
|
5
5
|
export { MessageList } from "./components/message-list.tsx";
|
|
6
6
|
export { SidebarLayout } from "./components/sidebar-layout.tsx";
|
|
7
7
|
export { StartScreen } from "./components/start-screen.tsx";
|
|
8
|
-
export { SyncChatView } from "./components/sync-chat-view.tsx";
|
|
9
|
-
export { TextControls } from "./components/text-controls.tsx";
|
|
10
8
|
export type { ToolDisplayConfig } from "./components/tool-config-context.ts";
|
|
11
9
|
export { ToolConfigContext } from "./components/tool-config-context.ts";
|
|
12
10
|
export { ApiUrlChip, SessionUrlChips, UiUrlChip } from "./components/url-chips.tsx";
|
|
11
|
+
export { WorkflowView } from "./components/workflow-view.tsx";
|
|
13
12
|
export type { Session } from "./context.ts";
|
|
14
13
|
export { SessionProvider, ThemeProvider, useSession, useSessionSelector, useTheme, } from "./context.ts";
|
|
15
14
|
export type { ClientConfig, ClientHandle, } from "./define-client.tsx";
|
|
@@ -17,7 +16,7 @@ export { client } from "./define-client.tsx";
|
|
|
17
16
|
export { useEvent, useToolCallStart, useToolResult } from "./hooks.ts";
|
|
18
17
|
export type { CustomEvent, SessionCore, SessionCoreOptions, SessionSnapshot, } from "./session-core.ts";
|
|
19
18
|
export { createSessionCore } from "./session-core.ts";
|
|
20
|
-
export { CAPTURE_WORKLET_MODULE_URL, createPttRecorder, DEFAULT_SYNC_MIC_SAMPLE_RATE, floatToPcm16, type PttRecorder,
|
|
19
|
+
export { CAPTURE_WORKLET_MODULE_URL, createPttRecorder, DEFAULT_SYNC_MIC_SAMPLE_RATE, floatToPcm16, type PttRecorder, } from "./sync-mic.ts";
|
|
21
20
|
export { base64ToPcm16, createSyncSession, pcm16ToBase64, type SyncSession, type SyncSessionOptions, type SyncTurnResult, } from "./sync-session.ts";
|
|
22
|
-
export { createUtteranceDetector, type UtteranceDetector, type UtteranceDetectorOptions, } from "./sync-vad.ts";
|
|
23
21
|
export type { AgentState, ChatMessage, ClientTheme, SessionError, SessionErrorCode, ToolCallInfo, VoiceSessionOptions, WebSocketConstructor, } from "./types.ts";
|
|
22
|
+
export { VOICE_CAPTURE_CONSTRAINTS } from "./types.ts";
|
package/dist/index.js
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
|
-
import { a as pcm16ToBase64, c as createPttRecorder, d as
|
|
1
|
+
import { a as pcm16ToBase64, c as createPttRecorder, d as fetchClientConfig, i as createSyncSession, l as floatToPcm16, n as WorkflowView, o as CAPTURE_WORKLET_MODULE_URL, r as base64ToPcm16, s as DEFAULT_SYNC_MIC_SAMPLE_RATE, t as client, u as buildAgentUrl } from "./define-client-yoGybEYR.js";
|
|
2
2
|
import { SessionProvider, ThemeProvider, useSession, useSessionSelector, useTheme } from "./context.js";
|
|
3
3
|
import { Button } from "./components/button.js";
|
|
4
|
-
import {
|
|
4
|
+
import { t as ChatView } from "./chat-view-DKFhxMAT.js";
|
|
5
|
+
import { i as UiUrlChip, n as ApiUrlChip, r as SessionUrlChips, t as Controls } from "./controls-4OJoekj6.js";
|
|
5
6
|
import { n as ToolConfigContext } from "./tool-call-block-DIxpG8GM.js";
|
|
6
7
|
import { MessageList } from "./components/message-list.js";
|
|
7
|
-
import { r as TextControls, t as ChatView } from "./chat-view-gi6FccZq.js";
|
|
8
8
|
import { SidebarLayout } from "./components/sidebar-layout.js";
|
|
9
9
|
import { StartScreen } from "./components/start-screen.js";
|
|
10
|
-
import {
|
|
10
|
+
import { VOICE_CAPTURE_CONSTRAINTS } from "./types.js";
|
|
11
|
+
import { t as createSessionCore } from "./session-core-BLhiQ18c.js";
|
|
11
12
|
import { useEvent, useToolCallStart, useToolResult } from "./hooks.js";
|
|
12
|
-
export { ApiUrlChip, Button, CAPTURE_WORKLET_MODULE_URL, ChatView, Controls, DEFAULT_SYNC_MIC_SAMPLE_RATE, MessageList, SessionProvider, SessionUrlChips, SidebarLayout, StartScreen,
|
|
13
|
+
export { ApiUrlChip, Button, CAPTURE_WORKLET_MODULE_URL, ChatView, Controls, DEFAULT_SYNC_MIC_SAMPLE_RATE, MessageList, SessionProvider, SessionUrlChips, SidebarLayout, StartScreen, ThemeProvider, ToolConfigContext, UiUrlChip, VOICE_CAPTURE_CONSTRAINTS, WorkflowView, base64ToPcm16, buildAgentUrl, client, createPttRecorder, createSessionCore, createSyncSession, fetchClientConfig, floatToPcm16, pcm16ToBase64, useEvent, useSession, useSessionSelector, useTheme, useToolCallStart, useToolResult };
|
|
@@ -74,6 +74,12 @@ async function initAudioCapture(conn, msg, deps, fatal) {
|
|
|
74
74
|
console.debug("[aai-ui] sendAudio dropped: connection closed");
|
|
75
75
|
}
|
|
76
76
|
},
|
|
77
|
+
onPlaybackStats: (stats) => {
|
|
78
|
+
console.warn("[aai-ui] playback concealed a gap in this turn", stats);
|
|
79
|
+
},
|
|
80
|
+
onMicSilent: () => {
|
|
81
|
+
console.warn("[aai-ui] microphone is delivering only silence — check the selected input device");
|
|
82
|
+
},
|
|
77
83
|
onError: (err) => {
|
|
78
84
|
if (conn.generation !== gen) return;
|
|
79
85
|
reportAudioFailure(err.message);
|
package/dist/session-core.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import { t as createSessionCore } from "./session-core-
|
|
1
|
+
import { t as createSessionCore } from "./session-core-BLhiQ18c.js";
|
|
2
2
|
export { createSessionCore };
|
package/dist/sync-mic.d.ts
CHANGED
|
@@ -1,23 +1,39 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* WebRTC
|
|
2
|
+
* WebRTC push-to-talk capture for the workflow run surface.
|
|
3
3
|
*
|
|
4
|
-
* Captures voice through `getUserMedia`
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* the sync session as one HTTP turn. No WebSocket anywhere on the path.
|
|
4
|
+
* Captures voice through `getUserMedia` under
|
|
5
|
+
* {@link VOICE_CAPTURE_CONSTRAINTS} — echo cancellation only — and runs an
|
|
6
|
+
* AudioWorklet that batches raw frames to the main thread; the caller's
|
|
7
|
+
* button is the endpointing. Each recording becomes one `POST /sync` run.
|
|
8
|
+
* No WebSocket anywhere on the path.
|
|
10
9
|
*
|
|
11
10
|
* The worklet module ships inline as a blob URL (same pattern as the
|
|
12
|
-
* WebSocket path's worklets), so
|
|
11
|
+
* WebSocket path's worklets), so this path needs no separately-served
|
|
13
12
|
* processor file. A blob URL rather than a data URI because the agent
|
|
14
13
|
* page's CSP allows `script-src blob:` but not `data:` — a data-URI
|
|
15
14
|
* module fails `addModule` with "Unable to load a worklet's module".
|
|
16
15
|
*/
|
|
17
|
-
import type { SyncSession } from "./sync-session.ts";
|
|
18
|
-
import { type UtteranceDetectorOptions } from "./sync-vad.ts";
|
|
19
16
|
/** Default capture rate — what the STT providers expect. */
|
|
20
17
|
export declare const DEFAULT_SYNC_MIC_SAMPLE_RATE = 16000;
|
|
18
|
+
/** ~128 ms at 16 kHz: few messages per second, fine-enough VAD granularity. */
|
|
19
|
+
export declare const CAPTURE_BATCH_SAMPLES = 2048;
|
|
20
|
+
/**
|
|
21
|
+
* The capture processor: coalesces 128-sample render quanta into
|
|
22
|
+
* {@link CAPTURE_BATCH_SAMPLES} batches and posts them (transferred, so no
|
|
23
|
+
* per-batch copy). Inlined as source because it must be stringified into a
|
|
24
|
+
* blob URL.
|
|
25
|
+
*
|
|
26
|
+
* `batch` is held as a field rather than re-read from the posted view:
|
|
27
|
+
* `postMessage` with a transfer list detaches the buffer, so `out.length` is
|
|
28
|
+
* 0 by the time the next buffer is allocated. Allocating a zero-length `buf`
|
|
29
|
+
* from it made `n` 0 forever, so `read` stopped advancing and the render
|
|
30
|
+
* thread spun inside `process()` posting empty chunks — the mic went
|
|
31
|
+
* permanently deaf on its first flush.
|
|
32
|
+
*
|
|
33
|
+
* Exported for the worklet unit tests (`sync-mic-worklet.test.ts`), which
|
|
34
|
+
* evaluate this source directly; it is not part of the package surface.
|
|
35
|
+
*/
|
|
36
|
+
export declare const CAPTURE_PROCESSOR_SRC = "\nregisterProcessor(\"aai-sync-capture\", class extends AudioWorkletProcessor {\n constructor(options) {\n super();\n const batch = (options && options.processorOptions && options.processorOptions.batchSamples) || 2048;\n this.batch = batch;\n this.buf = new Float32Array(batch);\n this.len = 0;\n }\n process(inputs) {\n const ch = inputs[0] && inputs[0][0];\n if (!ch) return true;\n let read = 0;\n while (read < ch.length) {\n const n = Math.min(ch.length - read, this.buf.length - this.len);\n this.buf.set(ch.subarray(read, read + n), this.len);\n this.len += n;\n read += n;\n if (this.len === this.batch) {\n const out = this.buf;\n // Size the next buffer from this.batch, never from `out`: the\n // transfer below detaches out.buffer, so out.length reads 0 here.\n this.buf = new Float32Array(this.batch);\n this.len = 0;\n this.port.postMessage({ event: \"chunk\", samples: out }, [out.buffer]);\n }\n }\n return true;\n }\n});\n";
|
|
21
37
|
/**
|
|
22
38
|
* Blob-URL module for the capture processor (no served asset). Satisfies the
|
|
23
39
|
* agent page's `script-src blob:` CSP, which rejects data-URI modules.
|
|
@@ -25,28 +41,6 @@ export declare const DEFAULT_SYNC_MIC_SAMPLE_RATE = 16000;
|
|
|
25
41
|
export declare const CAPTURE_WORKLET_MODULE_URL: string;
|
|
26
42
|
/** Clamp-and-convert one Float32 capture batch to PCM16. */
|
|
27
43
|
export declare function floatToPcm16(samples: Float32Array): Int16Array;
|
|
28
|
-
/** Configuration for {@link startSyncMicrophone}. */
|
|
29
|
-
export type SyncMicrophoneOptions = {
|
|
30
|
-
/** The session each completed utterance is sent through. */
|
|
31
|
-
session: Pick<SyncSession, "sendPcm16">;
|
|
32
|
-
/** Capture/STT sample rate. Defaults to {@link DEFAULT_SYNC_MIC_SAMPLE_RATE}. */
|
|
33
|
-
sampleRate?: number | undefined;
|
|
34
|
-
/** VAD tuning overrides (see {@link UtteranceDetectorOptions}). */
|
|
35
|
-
vad?: Omit<UtteranceDetectorOptions, "sampleRate"> | undefined;
|
|
36
|
-
/** Speech onset confirmed — a turn will follow once the user pauses. */
|
|
37
|
-
onSpeechStart?: (() => void) | undefined;
|
|
38
|
-
/** An utterance was endpointed and its turn dispatched. */
|
|
39
|
-
onSpeechEnd?: (() => void) | undefined;
|
|
40
|
-
/** Capture or turn failure (the mic keeps running unless stopped). */
|
|
41
|
-
onError?: ((err: Error) => void) | undefined;
|
|
42
|
-
};
|
|
43
|
-
/** Live microphone handle returned by {@link startSyncMicrophone}. */
|
|
44
|
-
export type SyncMicrophone = {
|
|
45
|
-
/** True while the detector is inside an utterance. */
|
|
46
|
-
readonly speaking: boolean;
|
|
47
|
-
/** Release the mic, the AudioContext, and flush a trailing utterance. */
|
|
48
|
-
stop(): Promise<void>;
|
|
49
|
-
};
|
|
50
44
|
/** Hold-to-record handle returned by {@link createPttRecorder}. */
|
|
51
45
|
export type PttRecorder = {
|
|
52
46
|
/** Open the mic (first call) and start collecting frames. */
|
|
@@ -57,18 +51,11 @@ export type PttRecorder = {
|
|
|
57
51
|
close(): Promise<void>;
|
|
58
52
|
};
|
|
59
53
|
/**
|
|
60
|
-
* Push-to-talk recorder
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
* across presses until `close()`.
|
|
54
|
+
* Push-to-talk recorder: `getUserMedia` voice capture feeding the capture
|
|
55
|
+
* worklet, with the caller's button as the endpointing. Recording runs
|
|
56
|
+
* exactly between `start()` and `stop()`; the mic stays open across presses
|
|
57
|
+
* until `close()`.
|
|
65
58
|
*
|
|
66
59
|
* @public
|
|
67
60
|
*/
|
|
68
61
|
export declare function createPttRecorder(sampleRate?: number): PttRecorder;
|
|
69
|
-
/**
|
|
70
|
-
* Open the microphone and stream endpointed utterances into a sync session.
|
|
71
|
-
*
|
|
72
|
-
* @throws If microphone access is denied or worklet registration fails.
|
|
73
|
-
*/
|
|
74
|
-
export declare function startSyncMicrophone(opts: SyncMicrophoneOptions): Promise<SyncMicrophone>;
|
package/dist/types.d.ts
CHANGED
|
@@ -1,5 +1,28 @@
|
|
|
1
1
|
import type { SessionErrorCode } from "@alexkroman1/aai/protocol";
|
|
2
|
-
export { FILE_SEND_BACKOFF_MS, MIC_BUFFER_SECONDS, MIC_SEND_MAX_BUFFERED_BYTES, } from "@alexkroman1/aai";
|
|
2
|
+
export { FILE_SEND_BACKOFF_MS, MIC_BUFFER_SECONDS, MIC_SEND_MAX_BUFFERED_BYTES, MIC_SILENCE_PROBE_MS, PLAYBACK_CONCEAL_FADE_MS, PLAYBACK_CONCEAL_FLOOR, PLAYBACK_JITTER_MS, PLAYBACK_REFILL_MS, } from "@alexkroman1/aai";
|
|
3
|
+
/**
|
|
4
|
+
* `getUserMedia` audio constraints for every capture path in this package.
|
|
5
|
+
*
|
|
6
|
+
* Defined once because four copies of this object drifted apart trivially, and
|
|
7
|
+
* the flags are not cosmetic — each one rewrites the signal before STT (and
|
|
8
|
+
* before the sync path's energy VAD) ever sees it:
|
|
9
|
+
*
|
|
10
|
+
* - **`autoGainControl: false`** — AGC continuously retargets level, which
|
|
11
|
+
* means riding the noise floor up through silence. An energy VAD calibrated
|
|
12
|
+
* against a moving floor is calibrated against nothing.
|
|
13
|
+
* - **`noiseSuppression: false`** / **`voiceIsolation: false`** — both discard
|
|
14
|
+
* signal to make speech sound cleaner to a human, and both can gate a quiet
|
|
15
|
+
* room to *exact* zeros, which is also what a dead microphone looks like
|
|
16
|
+
* (see `MIC_SILENCE_PROBE_MS`).
|
|
17
|
+
* - **`echoCancellation: true`** — this one stays on. The mic is open while
|
|
18
|
+
* the agent speaks (barge-in needs it), so without AEC the agent hears
|
|
19
|
+
* itself and interrupts its own reply.
|
|
20
|
+
*
|
|
21
|
+
* Cast because `voiceIsolation` is newer than TypeScript's DOM lib.
|
|
22
|
+
*
|
|
23
|
+
* @public
|
|
24
|
+
*/
|
|
25
|
+
export declare const VOICE_CAPTURE_CONSTRAINTS: MediaTrackConstraints;
|
|
3
26
|
/**
|
|
4
27
|
* Current state of the voice agent session.
|
|
5
28
|
*
|
package/dist/types.js
CHANGED
|
@@ -1,2 +1,32 @@
|
|
|
1
|
-
import { FILE_SEND_BACKOFF_MS, MIC_BUFFER_SECONDS, MIC_SEND_MAX_BUFFERED_BYTES } from "@alexkroman1/aai";
|
|
2
|
-
|
|
1
|
+
import { FILE_SEND_BACKOFF_MS, MIC_BUFFER_SECONDS, MIC_SEND_MAX_BUFFERED_BYTES, MIC_SILENCE_PROBE_MS, PLAYBACK_CONCEAL_FADE_MS, PLAYBACK_CONCEAL_FLOOR, PLAYBACK_JITTER_MS, PLAYBACK_REFILL_MS } from "@alexkroman1/aai";
|
|
2
|
+
//#region types.ts
|
|
3
|
+
/**
|
|
4
|
+
* `getUserMedia` audio constraints for every capture path in this package.
|
|
5
|
+
*
|
|
6
|
+
* Defined once because four copies of this object drifted apart trivially, and
|
|
7
|
+
* the flags are not cosmetic — each one rewrites the signal before STT (and
|
|
8
|
+
* before the sync path's energy VAD) ever sees it:
|
|
9
|
+
*
|
|
10
|
+
* - **`autoGainControl: false`** — AGC continuously retargets level, which
|
|
11
|
+
* means riding the noise floor up through silence. An energy VAD calibrated
|
|
12
|
+
* against a moving floor is calibrated against nothing.
|
|
13
|
+
* - **`noiseSuppression: false`** / **`voiceIsolation: false`** — both discard
|
|
14
|
+
* signal to make speech sound cleaner to a human, and both can gate a quiet
|
|
15
|
+
* room to *exact* zeros, which is also what a dead microphone looks like
|
|
16
|
+
* (see `MIC_SILENCE_PROBE_MS`).
|
|
17
|
+
* - **`echoCancellation: true`** — this one stays on. The mic is open while
|
|
18
|
+
* the agent speaks (barge-in needs it), so without AEC the agent hears
|
|
19
|
+
* itself and interrupts its own reply.
|
|
20
|
+
*
|
|
21
|
+
* Cast because `voiceIsolation` is newer than TypeScript's DOM lib.
|
|
22
|
+
*
|
|
23
|
+
* @public
|
|
24
|
+
*/
|
|
25
|
+
const VOICE_CAPTURE_CONSTRAINTS = {
|
|
26
|
+
echoCancellation: true,
|
|
27
|
+
noiseSuppression: false,
|
|
28
|
+
autoGainControl: false,
|
|
29
|
+
voiceIsolation: false
|
|
30
|
+
};
|
|
31
|
+
//#endregion
|
|
32
|
+
export { FILE_SEND_BACKOFF_MS, MIC_BUFFER_SECONDS, MIC_SEND_MAX_BUFFERED_BYTES, MIC_SILENCE_PROBE_MS, PLAYBACK_CONCEAL_FADE_MS, PLAYBACK_CONCEAL_FLOOR, PLAYBACK_JITTER_MS, PLAYBACK_REFILL_MS, VOICE_CAPTURE_CONSTRAINTS };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
/** Raw worklet source — exported so tests can evaluate the processor directly. */
|
|
2
|
-
export declare const captureProcessorSource = "\nclass CaptureProcessor extends AudioWorkletProcessor {\n constructor(options) {\n super();\n this.recording = false;\n const opts = options.processorOptions || {};\n
|
|
2
|
+
export declare const captureProcessorSource = "\nclass CaptureProcessor extends AudioWorkletProcessor {\n constructor(options) {\n super();\n this.recording = false;\n const opts = options.processorOptions || {};\n // The context runs at the STT rate, so this is both the input and the\n // output rate \u2014 there is nothing to convert.\n this.rate = opts.sampleRate || sampleRate;\n // Int16 accumulation buffer: flushed to the main thread as one transferred\n // ArrayBuffer once ~bufferSeconds of samples are batched. Sized 2x the\n // flush target so a whole render quantum always fits before flushing.\n this.targetSamples = Math.max(1, Math.round(this.rate * (opts.bufferSeconds || 0.1)));\n this.pending = new Int16Array(this.targetSamples * 2);\n this.pendingLen = 0;\n // Dead-mic probe: samples left to inspect before concluding the device\n // delivers nothing but digital silence. Only consumed while recording, so\n // the cost disappears after the window (or after the first real sample).\n this.probeSamplesLeft = Math.round(\n (this.rate * (opts.silenceProbeMs ?? 1500)) / 1000,\n );\n this.port.onmessage = (e) => {\n if (e.data.event === 'start') this.recording = true;\n else if (e.data.event === 'stop') {\n // Final flush so the tail of speech isn't dropped on close, then ack\n // so the host knows the tail chunk (if any) has been posted and it is\n // safe to tear the context down.\n this.flush();\n this.recording = false;\n this.port.postMessage({ event: 'stopped' });\n }\n };\n }\n\n // Convert Float32 -> Int16 and append to the pending batch. Writes through\n // an Int16Array directly (assignment truncates like DataView.setInt16).\n accumulate(samples) {\n let buf = this.pending;\n if (this.pendingLen + samples.length > buf.length) {\n // Defensive: only reachable if a render quantum outproduces the 1x\n // headroom above the flush target (never with 128-sample quanta).\n const grown = new Int16Array((this.pendingLen + samples.length) * 2);\n grown.set(buf.subarray(0, this.pendingLen));\n this.pending = grown;\n buf = grown;\n }\n for (let i = 0; i < samples.length; i++) {\n const s = Math.max(-1, Math.min(1, samples[i]));\n buf[this.pendingLen++] = s < 0 ? s * 0x8000 : s * 0x7fff;\n }\n }\n\n // Post the batched samples as one transferred ArrayBuffer and reset.\n flush() {\n if (this.pendingLen === 0) return;\n const buffer = this.pending.buffer.slice(0, this.pendingLen * 2);\n this.pendingLen = 0;\n this.port.postMessage({ event: 'chunk', buffer }, [buffer]);\n }\n\n // Watch the first window of input for any nonzero sample. One is enough to\n // prove the device is live \u2014 a real mic in a quiet room still carries a\n // noise floor, so all-zeros means muted, wrong input, or no input at all.\n probeForSilence(channel) {\n for (let i = 0; i < channel.length; i++) {\n if (channel[i] !== 0) {\n this.probeSamplesLeft = 0;\n return;\n }\n }\n this.probeSamplesLeft -= channel.length;\n if (this.probeSamplesLeft <= 0) {\n this.port.postMessage({ event: 'silent' });\n }\n }\n\n process(inputs) {\n const input = inputs[0];\n if (!input || !input[0] || !this.recording) return true;\n\n if (this.probeSamplesLeft > 0) this.probeForSilence(input[0]);\n\n this.accumulate(input[0]);\n if (this.pendingLen >= this.targetSamples) this.flush();\n return true;\n }\n}\n\nregisterProcessor('capture-processor', CaptureProcessor);\n";
|
|
3
3
|
declare const src: string;
|
|
4
4
|
export default src;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { MIC_SILENCE_PROBE_MS } from "../types.js";
|
|
1
2
|
//#region worklets/capture-processor.ts
|
|
2
3
|
const CaptureProcessorWorklet = `
|
|
3
4
|
class CaptureProcessor extends AudioWorkletProcessor {
|
|
@@ -5,27 +6,21 @@ class CaptureProcessor extends AudioWorkletProcessor {
|
|
|
5
6
|
super();
|
|
6
7
|
this.recording = false;
|
|
7
8
|
const opts = options.processorOptions || {};
|
|
8
|
-
this
|
|
9
|
-
|
|
10
|
-
this.
|
|
11
|
-
this.needsResample = this.fromRate !== this.toRate;
|
|
12
|
-
// Streaming-resampler state carried across process() blocks so the output
|
|
13
|
-
// clock doesn't drift and there's no discontinuity at 128-sample block
|
|
14
|
-
// boundaries. \`prev\` is the previous block's last input sample (used to
|
|
15
|
-
// interpolate across the boundary); \`pos\` is the fractional read position,
|
|
16
|
-
// in input samples, relative to an extended [prev, ...input] frame. Start
|
|
17
|
-
// at 1 so the first output is input[0] (not the bogus initial prev).
|
|
18
|
-
this.prev = 0;
|
|
19
|
-
this.pos = 1;
|
|
20
|
-
// Output buffer reused across process() calls -- the render quantum is a
|
|
21
|
-
// fixed size, so allocating per call would churn the realtime audio thread.
|
|
22
|
-
this.resampleBuf = null;
|
|
9
|
+
// The context runs at the STT rate, so this is both the input and the
|
|
10
|
+
// output rate — there is nothing to convert.
|
|
11
|
+
this.rate = opts.sampleRate || sampleRate;
|
|
23
12
|
// Int16 accumulation buffer: flushed to the main thread as one transferred
|
|
24
|
-
// ArrayBuffer once ~bufferSeconds of
|
|
25
|
-
//
|
|
26
|
-
this.targetSamples = Math.max(1, Math.round(this.
|
|
13
|
+
// ArrayBuffer once ~bufferSeconds of samples are batched. Sized 2x the
|
|
14
|
+
// flush target so a whole render quantum always fits before flushing.
|
|
15
|
+
this.targetSamples = Math.max(1, Math.round(this.rate * (opts.bufferSeconds || 0.1)));
|
|
27
16
|
this.pending = new Int16Array(this.targetSamples * 2);
|
|
28
17
|
this.pendingLen = 0;
|
|
18
|
+
// Dead-mic probe: samples left to inspect before concluding the device
|
|
19
|
+
// delivers nothing but digital silence. Only consumed while recording, so
|
|
20
|
+
// the cost disappears after the window (or after the first real sample).
|
|
21
|
+
this.probeSamplesLeft = Math.round(
|
|
22
|
+
(this.rate * (opts.silenceProbeMs ?? ${MIC_SILENCE_PROBE_MS})) / 1000,
|
|
23
|
+
);
|
|
29
24
|
this.port.onmessage = (e) => {
|
|
30
25
|
if (e.data.event === 'start') this.recording = true;
|
|
31
26
|
else if (e.data.event === 'stop') {
|
|
@@ -39,35 +34,6 @@ class CaptureProcessor extends AudioWorkletProcessor {
|
|
|
39
34
|
};
|
|
40
35
|
}
|
|
41
36
|
|
|
42
|
-
resample(input) {
|
|
43
|
-
const ratio = this.ratio;
|
|
44
|
-
const n = input.length;
|
|
45
|
-
if (n === 0) return new Float32Array(0);
|
|
46
|
-
// At most ceil(n / ratio) + 1 outputs per block (pos starts in [0, ratio)).
|
|
47
|
-
const maxLen = Math.ceil(n / ratio) + 1;
|
|
48
|
-
if (this.resampleBuf === null || this.resampleBuf.length < maxLen) {
|
|
49
|
-
this.resampleBuf = new Float32Array(maxLen);
|
|
50
|
-
}
|
|
51
|
-
const out = this.resampleBuf;
|
|
52
|
-
// Extended frame: index 0 = prev (previous block's last sample),
|
|
53
|
-
// index k>=1 = input[k-1]. Interpolate at fractional positions stepping by
|
|
54
|
-
// ratio, carrying \`pos\` across calls to preserve the sample clock.
|
|
55
|
-
let count = 0;
|
|
56
|
-
let pos = this.pos;
|
|
57
|
-
while (pos < n) {
|
|
58
|
-
const idx = pos | 0;
|
|
59
|
-
const frac = pos - idx;
|
|
60
|
-
const a = idx === 0 ? this.prev : input[idx - 1];
|
|
61
|
-
const b = input[idx];
|
|
62
|
-
out[count++] = a + frac * (b - a);
|
|
63
|
-
pos += ratio;
|
|
64
|
-
}
|
|
65
|
-
// Shift the origin to this block's last sample for the next call.
|
|
66
|
-
this.prev = input[n - 1];
|
|
67
|
-
this.pos = pos - n;
|
|
68
|
-
return out.subarray(0, count);
|
|
69
|
-
}
|
|
70
|
-
|
|
71
37
|
// Convert Float32 -> Int16 and append to the pending batch. Writes through
|
|
72
38
|
// an Int16Array directly (assignment truncates like DataView.setInt16).
|
|
73
39
|
accumulate(samples) {
|
|
@@ -94,12 +60,29 @@ class CaptureProcessor extends AudioWorkletProcessor {
|
|
|
94
60
|
this.port.postMessage({ event: 'chunk', buffer }, [buffer]);
|
|
95
61
|
}
|
|
96
62
|
|
|
63
|
+
// Watch the first window of input for any nonzero sample. One is enough to
|
|
64
|
+
// prove the device is live — a real mic in a quiet room still carries a
|
|
65
|
+
// noise floor, so all-zeros means muted, wrong input, or no input at all.
|
|
66
|
+
probeForSilence(channel) {
|
|
67
|
+
for (let i = 0; i < channel.length; i++) {
|
|
68
|
+
if (channel[i] !== 0) {
|
|
69
|
+
this.probeSamplesLeft = 0;
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
this.probeSamplesLeft -= channel.length;
|
|
74
|
+
if (this.probeSamplesLeft <= 0) {
|
|
75
|
+
this.port.postMessage({ event: 'silent' });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
97
79
|
process(inputs) {
|
|
98
80
|
const input = inputs[0];
|
|
99
81
|
if (!input || !input[0] || !this.recording) return true;
|
|
100
82
|
|
|
101
|
-
|
|
102
|
-
|
|
83
|
+
if (this.probeSamplesLeft > 0) this.probeForSilence(input[0]);
|
|
84
|
+
|
|
85
|
+
this.accumulate(input[0]);
|
|
103
86
|
if (this.pendingLen >= this.targetSamples) this.flush();
|
|
104
87
|
return true;
|
|
105
88
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
/** Raw worklet source — exported so tests can evaluate the processor directly. */
|
|
2
|
-
export declare const playbackProcessorSource = "\nclass PlaybackProcessor extends AudioWorkletProcessor {\n constructor(options) {\n super();\n const rate = options.processorOptions?.sampleRate ?? 24000;\n // Wait for ~400ms of audio before starting.\n // If 'done' arrives first (short utterance), start immediately.\n this.jitterSamples = Math.floor(rate * 0.4);\n // Float32 ring buffer \u2014 60s at the context sample rate. Allocated once for\n // the node's lifetime; per-turn state resets via resetTurn(). writePos and\n // readPos are absolute (monotonic) sample counts; the buffer is indexed\n // modulo capacity so a reply longer than 60s keeps playing instead of\n // writing past the end and going silent.\n this.capacity = rate * 60;\n this.samples = new Float32Array(this.capacity);\n // Platform endianness probe: the wire format is PCM16 little-endian, so\n // the Int16Array fast path in ingestBytes is only valid on LE hosts\n // (every shipping browser target; the DataView path is the fallback).\n this.littleEndian = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;\n this.resetTurn();\n\n this.port.onmessage = (e) => {\n const d = e.data;\n if (d.event === 'write') {\n this.ingestBytes(d.buffer);\n } else if (d.event === 'interrupt') {\n this.interrupted = true;\n } else if (d.event === 'done') {\n this.isDone = true;\n }\n };\n }\n\n // Reset per-turn state so the node is reusable across replies without\n // reallocating the sample buffer or re-instantiating the worklet.\n resetTurn() {\n this.interrupted = false;\n this.isDone = false;\n this.playing = false;\n // Carry-over byte for split samples across chunks\n this.carry = null;\n this.writePos = 0;\n this.readPos = 0;\n }\n\n // End the current turn: notify the host and rearm for the next reply.\n // Must NOT return false from process() \u2014 a processor that stops is dead\n // for good, forcing a new node (and buffer) per reply.\n // `reason` ('interrupt' | 'done') tells the host which turn boundary this\n // stop belongs to: interrupt-stops are dropped host-side (flush() already\n // settled that turn), so they can never resolve a later turn's done() early.\n stopTurn(reason) {\n this.port.postMessage({ event: 'stop', reason });\n this.resetTurn();\n }\n\n ingestBytes(uint8) {\n let bytes = uint8;\n\n if (this.carry !== null) {\n const merged = new Uint8Array(1 + bytes.length);\n merged[0] = this.carry;\n merged.set(bytes, 1);\n bytes = merged;\n this.carry = null;\n }\n\n if (bytes.length % 2 !== 0) {\n this.carry = bytes[bytes.length - 1];\n bytes = bytes.subarray(0, bytes.length - 1);\n }\n\n if (bytes.length === 0) return;\n const numSamples = bytes.length / 2;\n const cap = this.capacity;\n const samples = this.samples;\n if (this.littleEndian && (bytes.byteOffset & 1) === 0) {\n // Fast path: 2-byte-aligned LE bytes wrap directly as an Int16Array;\n // copy wrap-aware in at most two runs with no per-sample DataView call\n // or modulo. This runs on the realtime audio thread.\n const int16 = new Int16Array(bytes.buffer, bytes.byteOffset, numSamples);\n let src = 0;\n let dst = this.writePos % cap;\n while (src < numSamples) {\n const run = Math.min(numSamples - src, cap - dst);\n for (let j = 0; j < run; j++) {\n samples[dst + j] = int16[src + j] / 0x8000;\n }\n src += run;\n dst = 0;\n }\n } else {\n // Odd byte offset (or big-endian host): fall back to per-sample reads.\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.length);\n let dst = this.writePos % cap;\n for (let i = 0; i < numSamples; i++) {\n samples[dst] = view.getInt16(i * 2, true) / 0x8000;\n dst++;\n if (dst === cap) dst = 0;\n }\n }\n this.writePos += numSamples;\n // If the producer outran the consumer by more than the buffer holds, drop\n // the oldest unplayed audio rather than reading samples we've overwritten.\n if (this.writePos - this.readPos > this.capacity) {\n this.readPos = this.writePos - this.capacity;\n }\n }\n\n process(inputs, outputs) {\n // No output wired up yet \u2014 nothing to render this quantum. Throwing here\n // would permanently kill the processor (the node is persistent per\n // session), so guard like the capture processor does.\n if (!outputs[0] || !outputs[0][0]) return true;\n const out = outputs[0][0];\n if (this.interrupted) {\n out.fill(0);\n this.stopTurn('interrupt');\n return true;\n }\n\n const avail = this.writePos - this.readPos;\n\n // Wait for jitter buffer to fill, unless done (short utterance)\n if (!this.playing) {\n if (avail >= this.jitterSamples || this.isDone) {\n this.playing = true;\n } else {\n out.fill(0);\n return true;\n }\n }\n\n if (avail > 0) {\n const n = Math.min(avail, out.length);\n // Copy from the ring buffer, splitting across the wrap boundary.\n const start = this.readPos % this.capacity;\n const first = Math.min(n, this.capacity - start);\n out.set(this.samples.subarray(start, start + first), 0);\n if (n > first) out.set(this.samples.subarray(0, n - first), first);\n this.readPos += n;\n out.fill(0, n);\n return true;\n }\n\n // No data: output silence, end the turn only when done\n out.fill(0);\n if (this.isDone) {\n this.stopTurn('done');\n }\n return true;\n }\n}\n\nregisterProcessor('playback-processor', PlaybackProcessor);\n";
|
|
2
|
+
export declare const playbackProcessorSource = "\nclass PlaybackProcessor extends AudioWorkletProcessor {\n constructor(options) {\n super();\n const opts = options.processorOptions || {};\n const rate = opts.sampleRate ?? 24000;\n // Fill target for the start of a turn. If 'done' arrives first (short\n // utterance), start immediately instead of waiting for audio that is\n // never coming.\n this.jitterSamples = Math.floor((rate * (opts.jitterMs ?? 400)) / 1000);\n // Fill target after an underrun \u2014 see PLAYBACK_REFILL_MS.\n this.refillSamples = Math.floor((rate * (opts.refillMs ?? 200)) / 1000);\n // Concealment source: a ring of the most recently played samples, looped\n // under a decaying gain to cover a gap. Sized to the fade window, with a\n // per-sample decay that reaches the floor exactly at its end.\n this.concealCapacity = Math.max(1, Math.floor((rate * 40) / 1000));\n this.concealBuf = new Float32Array(this.concealCapacity);\n this.concealDecay = Math.exp(Math.log(0.001) / this.concealCapacity);\n // Float32 ring buffer \u2014 60s at the context sample rate. Allocated once for\n // the node's lifetime; per-turn state resets via resetTurn(). writePos and\n // readPos are absolute (monotonic) sample counts; the buffer is indexed\n // modulo capacity so a reply longer than 60s keeps playing instead of\n // writing past the end and going silent.\n this.capacity = rate * 60;\n this.samples = new Float32Array(this.capacity);\n // Platform endianness probe: the wire format is PCM16 little-endian, so\n // the Int16Array fast path in ingestBytes is only valid on LE hosts\n // (every shipping browser target; the DataView path is the fallback).\n this.littleEndian = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;\n this.resetTurn();\n\n this.port.onmessage = (e) => {\n const d = e.data;\n if (d.event === 'write') {\n this.ingestBytes(d.buffer);\n } else if (d.event === 'interrupt') {\n this.interrupted = true;\n } else if (d.event === 'done') {\n this.isDone = true;\n }\n };\n }\n\n // Reset per-turn state so the node is reusable across replies without\n // reallocating the sample buffer or re-instantiating the worklet.\n resetTurn() {\n this.interrupted = false;\n this.isDone = false;\n this.playing = false;\n // Whether any real audio has been rendered this turn. Separates a turn's\n // pre-roll (nothing to extrapolate from, and not a defect) from a\n // mid-turn underrun.\n this.hasPlayed = false;\n this.fillTarget = this.jitterSamples;\n // Carry-over byte for split samples across chunks\n this.carry = null;\n this.writePos = 0;\n this.readPos = 0;\n // Concealment ring state and the current fade position.\n this.concealLen = 0;\n this.concealWrite = 0;\n this.concealPos = 0;\n this.concealGain = 1;\n // Episode flags, so a multi-quantum gap counts as one event.\n this.concealing = false;\n this.concealedSilence = false;\n // Reported to the host on 'stop'. A fresh object per turn: the one just\n // posted must not be mutated by the next turn.\n this.stats = {\n concealedSamples: 0,\n silentConcealedSamples: 0,\n concealmentEvents: 0,\n silentConcealmentEvents: 0,\n };\n }\n\n // End the current turn: notify the host and rearm for the next reply.\n // Must NOT return false from process() \u2014 a processor that stops is dead\n // for good, forcing a new node (and buffer) per reply.\n // `reason` ('interrupt' | 'done') tells the host which turn boundary this\n // stop belongs to: interrupt-stops are dropped host-side (flush() already\n // settled that turn), so they can never resolve a later turn's done() early.\n stopTurn(reason) {\n this.port.postMessage({ event: 'stop', reason, stats: this.stats });\n this.resetTurn();\n }\n\n // Cover a quantum (from `start`) where real audio should have been.\n //\n // Before the turn's first samples there is nothing to extrapolate from, so\n // the gap is plain silence and counted as nothing \u2014 WebRTC likewise only\n // counts concealment once playout has begun. After that, loop the retained\n // tail under a decaying gain: a hard zero-fill is a discontinuity mid-word,\n // which is the click that makes a brief stall sound like breakage.\n coverGap(out, start) {\n if (!this.hasPlayed) {\n out.fill(0, start);\n return;\n }\n if (!this.concealing) {\n this.concealing = true;\n this.concealedSilence = false;\n this.stats.concealmentEvents++;\n }\n const total = out.length - start;\n const len = this.concealLen;\n let silent = 0;\n if (len === 0) {\n out.fill(0, start);\n silent = total;\n } else {\n let g = this.concealGain;\n for (let i = start; i < out.length; i++) {\n if (g < 0.001) {\n // The fade has run out: keep counting the gap, but stop looping a\n // fragment that is now inaudible anyway.\n out[i] = 0;\n silent++;\n continue;\n }\n out[i] = this.concealBuf[this.concealPos] * g;\n this.concealPos = this.concealPos + 1 === len ? 0 : this.concealPos + 1;\n g *= this.concealDecay;\n }\n this.concealGain = g;\n }\n this.stats.concealedSamples += total;\n if (silent > 0) {\n this.stats.silentConcealedSamples += silent;\n if (!this.concealedSilence) {\n this.concealedSilence = true;\n this.stats.silentConcealmentEvents++;\n }\n }\n }\n\n // Retain the tail of a rendered quantum as the next gap's concealment\n // source, and close any episode the real audio just ended.\n rememberTail(out, n) {\n const cap = this.concealCapacity;\n const take = Math.min(n, cap);\n for (let i = n - take; i < n; i++) {\n this.concealBuf[this.concealWrite] = out[i];\n this.concealWrite = this.concealWrite + 1 === cap ? 0 : this.concealWrite + 1;\n }\n this.concealLen = Math.min(cap, this.concealLen + take);\n // Read the loop oldest-first; once the ring is full the write cursor is\n // the oldest retained sample.\n this.concealPos = this.concealLen === cap ? this.concealWrite : 0;\n this.concealing = false;\n this.concealGain = 1;\n }\n\n ingestBytes(uint8) {\n let bytes = uint8;\n\n if (this.carry !== null) {\n const merged = new Uint8Array(1 + bytes.length);\n merged[0] = this.carry;\n merged.set(bytes, 1);\n bytes = merged;\n this.carry = null;\n }\n\n if (bytes.length % 2 !== 0) {\n this.carry = bytes[bytes.length - 1];\n bytes = bytes.subarray(0, bytes.length - 1);\n }\n\n if (bytes.length === 0) return;\n const numSamples = bytes.length / 2;\n const cap = this.capacity;\n const samples = this.samples;\n if (this.littleEndian && (bytes.byteOffset & 1) === 0) {\n // Fast path: 2-byte-aligned LE bytes wrap directly as an Int16Array;\n // copy wrap-aware in at most two runs with no per-sample DataView call\n // or modulo. This runs on the realtime audio thread.\n const int16 = new Int16Array(bytes.buffer, bytes.byteOffset, numSamples);\n let src = 0;\n let dst = this.writePos % cap;\n while (src < numSamples) {\n const run = Math.min(numSamples - src, cap - dst);\n for (let j = 0; j < run; j++) {\n samples[dst + j] = int16[src + j] / 0x8000;\n }\n src += run;\n dst = 0;\n }\n } else {\n // Odd byte offset (or big-endian host): fall back to per-sample reads.\n const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.length);\n let dst = this.writePos % cap;\n for (let i = 0; i < numSamples; i++) {\n samples[dst] = view.getInt16(i * 2, true) / 0x8000;\n dst++;\n if (dst === cap) dst = 0;\n }\n }\n this.writePos += numSamples;\n // If the producer outran the consumer by more than the buffer holds, drop\n // the oldest unplayed audio rather than reading samples we've overwritten.\n if (this.writePos - this.readPos > this.capacity) {\n this.readPos = this.writePos - this.capacity;\n }\n }\n\n process(inputs, outputs) {\n // No output wired up yet \u2014 nothing to render this quantum. Throwing here\n // would permanently kill the processor (the node is persistent per\n // session), so guard like the capture processor does.\n if (!outputs[0] || !outputs[0][0]) return true;\n const out = outputs[0][0];\n if (this.interrupted) {\n out.fill(0);\n this.stopTurn('interrupt');\n return true;\n }\n\n const avail = this.writePos - this.readPos;\n\n // Filling: wait for the target. 'done' short-circuits it \u2014 what is\n // buffered is all there will be, so there is nothing left to wait for.\n if (!this.playing) {\n if (avail >= this.fillTarget || this.isDone) {\n this.playing = true;\n } else {\n this.coverGap(out, 0);\n return true;\n }\n }\n\n // Underrun: this quantum cannot be filled and more audio is still coming.\n // Go back to filling (at the refill target) and cover the gap, leaving\n // readPos untouched \u2014 the fragment stays buffered and plays intact once\n // the buffer recovers, instead of being dribbled out a few samples at a\n // time for the rest of the turn.\n if (avail < out.length && !this.isDone) {\n this.playing = false;\n this.fillTarget = this.refillSamples;\n this.coverGap(out, 0);\n return true;\n }\n\n if (avail > 0) {\n const n = Math.min(avail, out.length);\n // Copy from the ring buffer, splitting across the wrap boundary.\n const start = this.readPos % this.capacity;\n const first = Math.min(n, this.capacity - start);\n out.set(this.samples.subarray(start, start + first), 0);\n if (n > first) out.set(this.samples.subarray(0, n - first), first);\n this.readPos += n;\n // Only reachable with n < out.length on the turn's final partial\n // quantum (the underrun branch above catches every other case).\n out.fill(0, n);\n this.hasPlayed = true;\n this.rememberTail(out, n);\n return true;\n }\n\n // Drained and done: end the turn. Not reachable mid-turn \u2014 an empty\n // buffer with audio still coming is the underrun branch above.\n out.fill(0);\n if (this.isDone) {\n this.stopTurn('done');\n }\n return true;\n }\n}\n\nregisterProcessor('playback-processor', PlaybackProcessor);\n";
|
|
3
3
|
declare const src: string;
|
|
4
4
|
export default src;
|
|
@@ -1,12 +1,23 @@
|
|
|
1
|
+
import { PLAYBACK_CONCEAL_FADE_MS, PLAYBACK_CONCEAL_FLOOR, PLAYBACK_JITTER_MS, PLAYBACK_REFILL_MS } from "../types.js";
|
|
1
2
|
//#region worklets/playback-processor.ts
|
|
2
3
|
const PlaybackProcessorWorklet = `
|
|
3
4
|
class PlaybackProcessor extends AudioWorkletProcessor {
|
|
4
5
|
constructor(options) {
|
|
5
6
|
super();
|
|
6
|
-
const
|
|
7
|
-
|
|
8
|
-
// If 'done' arrives first (short
|
|
9
|
-
|
|
7
|
+
const opts = options.processorOptions || {};
|
|
8
|
+
const rate = opts.sampleRate ?? 24000;
|
|
9
|
+
// Fill target for the start of a turn. If 'done' arrives first (short
|
|
10
|
+
// utterance), start immediately instead of waiting for audio that is
|
|
11
|
+
// never coming.
|
|
12
|
+
this.jitterSamples = Math.floor((rate * (opts.jitterMs ?? ${PLAYBACK_JITTER_MS})) / 1000);
|
|
13
|
+
// Fill target after an underrun — see PLAYBACK_REFILL_MS.
|
|
14
|
+
this.refillSamples = Math.floor((rate * (opts.refillMs ?? ${PLAYBACK_REFILL_MS})) / 1000);
|
|
15
|
+
// Concealment source: a ring of the most recently played samples, looped
|
|
16
|
+
// under a decaying gain to cover a gap. Sized to the fade window, with a
|
|
17
|
+
// per-sample decay that reaches the floor exactly at its end.
|
|
18
|
+
this.concealCapacity = Math.max(1, Math.floor((rate * ${PLAYBACK_CONCEAL_FADE_MS}) / 1000));
|
|
19
|
+
this.concealBuf = new Float32Array(this.concealCapacity);
|
|
20
|
+
this.concealDecay = Math.exp(Math.log(${PLAYBACK_CONCEAL_FLOOR}) / this.concealCapacity);
|
|
10
21
|
// Float32 ring buffer — 60s at the context sample rate. Allocated once for
|
|
11
22
|
// the node's lifetime; per-turn state resets via resetTurn(). writePos and
|
|
12
23
|
// readPos are absolute (monotonic) sample counts; the buffer is indexed
|
|
@@ -38,10 +49,31 @@ class PlaybackProcessor extends AudioWorkletProcessor {
|
|
|
38
49
|
this.interrupted = false;
|
|
39
50
|
this.isDone = false;
|
|
40
51
|
this.playing = false;
|
|
52
|
+
// Whether any real audio has been rendered this turn. Separates a turn's
|
|
53
|
+
// pre-roll (nothing to extrapolate from, and not a defect) from a
|
|
54
|
+
// mid-turn underrun.
|
|
55
|
+
this.hasPlayed = false;
|
|
56
|
+
this.fillTarget = this.jitterSamples;
|
|
41
57
|
// Carry-over byte for split samples across chunks
|
|
42
58
|
this.carry = null;
|
|
43
59
|
this.writePos = 0;
|
|
44
60
|
this.readPos = 0;
|
|
61
|
+
// Concealment ring state and the current fade position.
|
|
62
|
+
this.concealLen = 0;
|
|
63
|
+
this.concealWrite = 0;
|
|
64
|
+
this.concealPos = 0;
|
|
65
|
+
this.concealGain = 1;
|
|
66
|
+
// Episode flags, so a multi-quantum gap counts as one event.
|
|
67
|
+
this.concealing = false;
|
|
68
|
+
this.concealedSilence = false;
|
|
69
|
+
// Reported to the host on 'stop'. A fresh object per turn: the one just
|
|
70
|
+
// posted must not be mutated by the next turn.
|
|
71
|
+
this.stats = {
|
|
72
|
+
concealedSamples: 0,
|
|
73
|
+
silentConcealedSamples: 0,
|
|
74
|
+
concealmentEvents: 0,
|
|
75
|
+
silentConcealmentEvents: 0,
|
|
76
|
+
};
|
|
45
77
|
}
|
|
46
78
|
|
|
47
79
|
// End the current turn: notify the host and rearm for the next reply.
|
|
@@ -51,10 +83,76 @@ class PlaybackProcessor extends AudioWorkletProcessor {
|
|
|
51
83
|
// stop belongs to: interrupt-stops are dropped host-side (flush() already
|
|
52
84
|
// settled that turn), so they can never resolve a later turn's done() early.
|
|
53
85
|
stopTurn(reason) {
|
|
54
|
-
this.port.postMessage({ event: 'stop', reason });
|
|
86
|
+
this.port.postMessage({ event: 'stop', reason, stats: this.stats });
|
|
55
87
|
this.resetTurn();
|
|
56
88
|
}
|
|
57
89
|
|
|
90
|
+
// Cover a quantum (from \`start\`) where real audio should have been.
|
|
91
|
+
//
|
|
92
|
+
// Before the turn's first samples there is nothing to extrapolate from, so
|
|
93
|
+
// the gap is plain silence and counted as nothing — WebRTC likewise only
|
|
94
|
+
// counts concealment once playout has begun. After that, loop the retained
|
|
95
|
+
// tail under a decaying gain: a hard zero-fill is a discontinuity mid-word,
|
|
96
|
+
// which is the click that makes a brief stall sound like breakage.
|
|
97
|
+
coverGap(out, start) {
|
|
98
|
+
if (!this.hasPlayed) {
|
|
99
|
+
out.fill(0, start);
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
if (!this.concealing) {
|
|
103
|
+
this.concealing = true;
|
|
104
|
+
this.concealedSilence = false;
|
|
105
|
+
this.stats.concealmentEvents++;
|
|
106
|
+
}
|
|
107
|
+
const total = out.length - start;
|
|
108
|
+
const len = this.concealLen;
|
|
109
|
+
let silent = 0;
|
|
110
|
+
if (len === 0) {
|
|
111
|
+
out.fill(0, start);
|
|
112
|
+
silent = total;
|
|
113
|
+
} else {
|
|
114
|
+
let g = this.concealGain;
|
|
115
|
+
for (let i = start; i < out.length; i++) {
|
|
116
|
+
if (g < ${PLAYBACK_CONCEAL_FLOOR}) {
|
|
117
|
+
// The fade has run out: keep counting the gap, but stop looping a
|
|
118
|
+
// fragment that is now inaudible anyway.
|
|
119
|
+
out[i] = 0;
|
|
120
|
+
silent++;
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
out[i] = this.concealBuf[this.concealPos] * g;
|
|
124
|
+
this.concealPos = this.concealPos + 1 === len ? 0 : this.concealPos + 1;
|
|
125
|
+
g *= this.concealDecay;
|
|
126
|
+
}
|
|
127
|
+
this.concealGain = g;
|
|
128
|
+
}
|
|
129
|
+
this.stats.concealedSamples += total;
|
|
130
|
+
if (silent > 0) {
|
|
131
|
+
this.stats.silentConcealedSamples += silent;
|
|
132
|
+
if (!this.concealedSilence) {
|
|
133
|
+
this.concealedSilence = true;
|
|
134
|
+
this.stats.silentConcealmentEvents++;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Retain the tail of a rendered quantum as the next gap's concealment
|
|
140
|
+
// source, and close any episode the real audio just ended.
|
|
141
|
+
rememberTail(out, n) {
|
|
142
|
+
const cap = this.concealCapacity;
|
|
143
|
+
const take = Math.min(n, cap);
|
|
144
|
+
for (let i = n - take; i < n; i++) {
|
|
145
|
+
this.concealBuf[this.concealWrite] = out[i];
|
|
146
|
+
this.concealWrite = this.concealWrite + 1 === cap ? 0 : this.concealWrite + 1;
|
|
147
|
+
}
|
|
148
|
+
this.concealLen = Math.min(cap, this.concealLen + take);
|
|
149
|
+
// Read the loop oldest-first; once the ring is full the write cursor is
|
|
150
|
+
// the oldest retained sample.
|
|
151
|
+
this.concealPos = this.concealLen === cap ? this.concealWrite : 0;
|
|
152
|
+
this.concealing = false;
|
|
153
|
+
this.concealGain = 1;
|
|
154
|
+
}
|
|
155
|
+
|
|
58
156
|
ingestBytes(uint8) {
|
|
59
157
|
let bytes = uint8;
|
|
60
158
|
|
|
@@ -122,16 +220,29 @@ class PlaybackProcessor extends AudioWorkletProcessor {
|
|
|
122
220
|
|
|
123
221
|
const avail = this.writePos - this.readPos;
|
|
124
222
|
|
|
125
|
-
//
|
|
223
|
+
// Filling: wait for the target. 'done' short-circuits it — what is
|
|
224
|
+
// buffered is all there will be, so there is nothing left to wait for.
|
|
126
225
|
if (!this.playing) {
|
|
127
|
-
if (avail >= this.
|
|
226
|
+
if (avail >= this.fillTarget || this.isDone) {
|
|
128
227
|
this.playing = true;
|
|
129
228
|
} else {
|
|
130
|
-
|
|
229
|
+
this.coverGap(out, 0);
|
|
131
230
|
return true;
|
|
132
231
|
}
|
|
133
232
|
}
|
|
134
233
|
|
|
234
|
+
// Underrun: this quantum cannot be filled and more audio is still coming.
|
|
235
|
+
// Go back to filling (at the refill target) and cover the gap, leaving
|
|
236
|
+
// readPos untouched — the fragment stays buffered and plays intact once
|
|
237
|
+
// the buffer recovers, instead of being dribbled out a few samples at a
|
|
238
|
+
// time for the rest of the turn.
|
|
239
|
+
if (avail < out.length && !this.isDone) {
|
|
240
|
+
this.playing = false;
|
|
241
|
+
this.fillTarget = this.refillSamples;
|
|
242
|
+
this.coverGap(out, 0);
|
|
243
|
+
return true;
|
|
244
|
+
}
|
|
245
|
+
|
|
135
246
|
if (avail > 0) {
|
|
136
247
|
const n = Math.min(avail, out.length);
|
|
137
248
|
// Copy from the ring buffer, splitting across the wrap boundary.
|
|
@@ -140,11 +251,16 @@ class PlaybackProcessor extends AudioWorkletProcessor {
|
|
|
140
251
|
out.set(this.samples.subarray(start, start + first), 0);
|
|
141
252
|
if (n > first) out.set(this.samples.subarray(0, n - first), first);
|
|
142
253
|
this.readPos += n;
|
|
254
|
+
// Only reachable with n < out.length on the turn's final partial
|
|
255
|
+
// quantum (the underrun branch above catches every other case).
|
|
143
256
|
out.fill(0, n);
|
|
257
|
+
this.hasPlayed = true;
|
|
258
|
+
this.rememberTail(out, n);
|
|
144
259
|
return true;
|
|
145
260
|
}
|
|
146
261
|
|
|
147
|
-
//
|
|
262
|
+
// Drained and done: end the turn. Not reachable mid-turn — an empty
|
|
263
|
+
// buffer with audio still coming is the underrun branch above.
|
|
148
264
|
out.fill(0);
|
|
149
265
|
if (this.isDone) {
|
|
150
266
|
this.stopTurn('done');
|