@burdenoff/fe-libs 2026.1013.13 → 2026.1013.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/shared/assistant/avatar/live/LiveAvatarStage.d.ts +15 -0
- package/dist/shared/assistant/avatar/live/LiveAvatarStage.js +139 -0
- package/dist/shared/assistant/avatar/live/avatarAudioDecode.d.ts +40 -0
- package/dist/shared/assistant/avatar/live/avatarAudioDecode.js +72 -0
- package/dist/shared/assistant/avatar/live/avatarAudioErrors.d.ts +25 -0
- package/dist/shared/assistant/avatar/live/avatarAudioErrors.js +27 -0
- package/dist/shared/assistant/avatar/live/avatarAudioMeter.d.ts +37 -0
- package/dist/shared/assistant/avatar/live/avatarAudioMeter.js +62 -0
- package/dist/shared/assistant/avatar/live/avatarMp3Inspect.d.ts +30 -0
- package/dist/shared/assistant/avatar/live/avatarMp3Inspect.js +168 -0
- package/dist/shared/assistant/avatar/live/avatarPcm16.d.ts +21 -0
- package/dist/shared/assistant/avatar/live/avatarPcm16.js +54 -0
- package/dist/shared/assistant/avatar/live/avatarPlaybackTracker.d.ts +181 -0
- package/dist/shared/assistant/avatar/live/avatarPlaybackTracker.js +221 -0
- package/dist/shared/assistant/avatar/live/avatarPorts.d.ts +66 -0
- package/dist/shared/assistant/avatar/live/avatarPorts.js +44 -0
- package/dist/shared/assistant/avatar/live/avatarProtocol.d.ts +22 -0
- package/dist/shared/assistant/avatar/live/avatarProtocol.js +27 -0
- package/dist/shared/assistant/avatar/live/avatarReplyOutputs.d.ts +170 -0
- package/dist/shared/assistant/avatar/live/avatarReplyOutputs.js +525 -0
- package/dist/shared/assistant/avatar/live/avatarSessionClient.d.ts +96 -0
- package/dist/shared/assistant/avatar/live/avatarSessionClient.js +261 -0
- package/dist/shared/assistant/avatar/live/avatarTransport.d.ts +270 -0
- package/dist/shared/assistant/avatar/live/avatarTransport.js +333 -0
- package/dist/shared/assistant/avatar/live/avatarTransportPort.d.ts +17 -0
- package/dist/shared/assistant/avatar/live/avatarTransportPort.js +27 -0
- package/dist/shared/assistant/avatar/live/createLiveAvatarSeam.d.ts +44 -0
- package/dist/shared/assistant/avatar/live/createLiveAvatarSeam.js +124 -0
- package/dist/shared/assistant/avatar/live/liveAvatarController.d.ts +133 -0
- package/dist/shared/assistant/avatar/live/liveAvatarController.js +488 -0
- package/dist/shared/assistant/avatar/live/liveAvatarStatus.d.ts +4 -0
- package/dist/shared/assistant/avatar/live/liveAvatarStatus.js +14 -0
- package/dist/shared/assistant/avatar/live/liveKitAvatarRoom.d.ts +6 -0
- package/dist/shared/assistant/avatar/live/liveKitAvatarRoom.js +86 -0
- package/dist/shared/assistant/avatar/useAvatarCopy.d.ts +2 -0
- package/dist/shared/assistant/avatar/useAvatarCopy.js +1 -0
- package/dist/shared/assistant/i18nManifest.d.ts +1 -0
- package/dist/shared/assistant/i18nManifest.js +1 -0
- package/dist/shared/assistant/index.d.ts +17 -0
- package/dist/shared/assistant/speech/hostedTtsSpeechEngine.d.ts +37 -0
- package/dist/shared/assistant/speech/hostedTtsSpeechEngine.js +118 -63
- package/dist/shared/assistant/speech/types.d.ts +14 -2
- package/dist/shared-assistant.js +250 -233
- package/package.json +1 -1
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { FC, ReactNode } from 'react';
|
|
2
|
+
import { LiveAvatarController } from './liveAvatarController';
|
|
3
|
+
export interface LiveAvatarStageViewProps {
|
|
4
|
+
readonly controller: LiveAvatarController;
|
|
5
|
+
/** Full-screen (true) or the in-panel stage (false). */
|
|
6
|
+
readonly expanded: boolean;
|
|
7
|
+
/** The waveform: what the stage shows while there is no avatar to show. */
|
|
8
|
+
readonly fallback: ReactNode;
|
|
9
|
+
/**
|
|
10
|
+
* Display names for avatar ids. The server sends ids only (names and pictures are the client's, BACKEND-DESIGN §3.3); without one the
|
|
11
|
+
* chooser numbers them.
|
|
12
|
+
*/
|
|
13
|
+
readonly avatarNames?: Readonly<Record<string, string>>;
|
|
14
|
+
}
|
|
15
|
+
export declare const LiveAvatarStageView: FC<LiveAvatarStageViewProps>;
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { useAvatarCopy as e } from "../useAvatarCopy.js";
|
|
2
|
+
import { liveAvatarStatusText as t } from "./liveAvatarStatus.js";
|
|
3
|
+
import { useCallback as n, useEffect as r, useId as i, useRef as a, useSyncExternalStore as o } from "react";
|
|
4
|
+
import { Video as s, VideoOff as c, Volume2 as l } from "lucide-react";
|
|
5
|
+
import { jsx as u, jsxs as d } from "react/jsx-runtime";
|
|
6
|
+
//#region src/shared/assistant/avatar/live/LiveAvatarStage.tsx
|
|
7
|
+
var f = "inline-flex h-8 items-center gap-1.5 rounded-full border border-border-default bg-bg-elevated px-3 text-xs font-medium text-text-secondary transition-colors hover:bg-bg-raised hover:text-text-primary focus:outline-none focus-visible:ring-2 focus-visible:ring-border-strong aria-disabled:cursor-default aria-disabled:opacity-60", p = "inline-flex h-8 cursor-pointer items-center gap-1.5 rounded-full border px-3 text-xs font-medium transition-colors focus-within:ring-2 focus-within:ring-border-strong", m = "inline-flex h-8 items-center rounded-full bg-action-primary-bg px-3 text-xs font-medium text-action-primary-text transition-colors focus:outline-none focus-visible:ring-2 focus-visible:ring-border-strong", h = ({ controller: h, expanded: g, fallback: _, avatarNames: v }) => {
|
|
8
|
+
let y = e(), b = i(), x = i(), S = a(null), C = a(null), w = n((e) => {
|
|
9
|
+
e?.focus();
|
|
10
|
+
}, []), T = n((e) => (C.current = e, () => {
|
|
11
|
+
C.current = null, document.activeElement === e && S.current?.focus();
|
|
12
|
+
}), []), E = o(h.subscribe, h.getSnapshot, h.getSnapshot);
|
|
13
|
+
r(() => (h.setStageMounted(!0), () => h.setStageMounted(!1)), [h]);
|
|
14
|
+
let { phase: D } = E, O = D === "starting" || D === "connecting" || D === "ready" || D === "ending", k = D === "checking" || D === "ending", A = t(E, y), j = (e, t) => v?.[e] ?? `${y.chooserLabel} ${t + 1}`;
|
|
15
|
+
return /* @__PURE__ */ d("div", {
|
|
16
|
+
ref: S,
|
|
17
|
+
tabIndex: -1,
|
|
18
|
+
className: "flex flex-col items-center gap-2 focus:outline-none",
|
|
19
|
+
"data-testid": "assistant-live-avatar",
|
|
20
|
+
children: [
|
|
21
|
+
O ? /* @__PURE__ */ d("div", {
|
|
22
|
+
className: `relative overflow-hidden rounded-2xl border border-border-subtle bg-bg-sunken ${g ? "size-64" : "size-40"}`,
|
|
23
|
+
"data-testid": "assistant-live-avatar-frame",
|
|
24
|
+
"data-phase": D,
|
|
25
|
+
children: [
|
|
26
|
+
/* @__PURE__ */ u("video", {
|
|
27
|
+
ref: h.attachVideo,
|
|
28
|
+
className: `size-full object-cover ${E.videoReady ? "" : "invisible"}`,
|
|
29
|
+
"aria-label": y.stageLabel,
|
|
30
|
+
autoPlay: !0,
|
|
31
|
+
muted: !0,
|
|
32
|
+
playsInline: !0,
|
|
33
|
+
"data-testid": "assistant-live-avatar-video"
|
|
34
|
+
}),
|
|
35
|
+
E.videoReady ? null : /* @__PURE__ */ u("p", {
|
|
36
|
+
className: "absolute inset-0 flex items-center justify-center p-3 text-center text-xs text-text-muted",
|
|
37
|
+
children: D === "ending" ? y.stateEnding : D === "connecting" ? y.stageWaiting : y.stateStarting
|
|
38
|
+
}),
|
|
39
|
+
E.audioBlocked ? /* @__PURE__ */ d("button", {
|
|
40
|
+
type: "button",
|
|
41
|
+
onClick: () => h.resumeAudio(),
|
|
42
|
+
className: "absolute inset-x-2 bottom-2 inline-flex items-center justify-center gap-1.5 rounded-xl bg-bg-elevated px-2 py-1.5 text-xs font-medium text-text-primary focus:outline-none focus-visible:ring-2 focus-visible:ring-border-strong",
|
|
43
|
+
"data-testid": "assistant-live-avatar-sound",
|
|
44
|
+
children: [/* @__PURE__ */ u(l, {
|
|
45
|
+
className: "size-4 shrink-0",
|
|
46
|
+
"aria-hidden": "true"
|
|
47
|
+
}), /* @__PURE__ */ u("span", { children: y.problem("playback-blocked") })]
|
|
48
|
+
}) : null
|
|
49
|
+
]
|
|
50
|
+
}) : _,
|
|
51
|
+
E.offered || O ? /* @__PURE__ */ d("button", {
|
|
52
|
+
ref: T,
|
|
53
|
+
type: "button",
|
|
54
|
+
onClick: () => {
|
|
55
|
+
k || (O ? h.switchOff() : D === "choosing" ? h.cancelChoice() : h.requestOn());
|
|
56
|
+
},
|
|
57
|
+
className: f,
|
|
58
|
+
"aria-pressed": O,
|
|
59
|
+
title: y.toggleHint,
|
|
60
|
+
"aria-disabled": k,
|
|
61
|
+
"data-testid": "assistant-live-avatar-toggle",
|
|
62
|
+
children: [u(O ? c : s, {
|
|
63
|
+
className: "size-4",
|
|
64
|
+
"aria-hidden": "true"
|
|
65
|
+
}), /* @__PURE__ */ u("span", { children: y.toggle })]
|
|
66
|
+
}) : null,
|
|
67
|
+
D === "choosing" ? /* @__PURE__ */ d("fieldset", {
|
|
68
|
+
className: "flex max-w-xs flex-col gap-2 rounded-xl border border-border-default bg-bg-elevated p-3 text-start",
|
|
69
|
+
"data-testid": "assistant-live-avatar-consent",
|
|
70
|
+
children: [
|
|
71
|
+
/* @__PURE__ */ u("legend", {
|
|
72
|
+
className: "sr-only",
|
|
73
|
+
children: y.toggle
|
|
74
|
+
}),
|
|
75
|
+
E.avatars.length > 1 ? /* @__PURE__ */ d("fieldset", {
|
|
76
|
+
className: "flex flex-wrap gap-1.5",
|
|
77
|
+
"data-testid": "assistant-live-avatar-chooser",
|
|
78
|
+
children: [/* @__PURE__ */ u("legend", {
|
|
79
|
+
className: "sr-only",
|
|
80
|
+
children: y.chooserTitle
|
|
81
|
+
}), E.avatars.map((e, t) => {
|
|
82
|
+
let n = e === E.selectedAvatarId, r = j(e, t);
|
|
83
|
+
return /* @__PURE__ */ d("label", {
|
|
84
|
+
className: `${p} ${n ? "border-transparent bg-action-primary-bg text-action-primary-text" : "border-border-default bg-bg-elevated text-text-secondary hover:bg-bg-raised hover:text-text-primary"}`,
|
|
85
|
+
title: n ? y.chooserSelected(r) : y.chooserOption(r),
|
|
86
|
+
children: [/* @__PURE__ */ u("input", {
|
|
87
|
+
type: "radio",
|
|
88
|
+
name: x,
|
|
89
|
+
value: e,
|
|
90
|
+
checked: n,
|
|
91
|
+
onChange: () => h.selectAvatar(e),
|
|
92
|
+
className: "sr-only",
|
|
93
|
+
"data-avatar-id": e
|
|
94
|
+
}), r]
|
|
95
|
+
}, e);
|
|
96
|
+
})]
|
|
97
|
+
}) : null,
|
|
98
|
+
/* @__PURE__ */ u("p", {
|
|
99
|
+
ref: w,
|
|
100
|
+
id: b,
|
|
101
|
+
tabIndex: -1,
|
|
102
|
+
className: "text-xs text-text-secondary focus:outline-none",
|
|
103
|
+
"data-testid": "assistant-live-avatar-disclosure",
|
|
104
|
+
children: y.consent
|
|
105
|
+
}),
|
|
106
|
+
/* @__PURE__ */ d("div", {
|
|
107
|
+
className: "flex justify-end gap-2",
|
|
108
|
+
children: [/* @__PURE__ */ u("button", {
|
|
109
|
+
type: "button",
|
|
110
|
+
onClick: () => {
|
|
111
|
+
C.current?.focus(), h.cancelChoice();
|
|
112
|
+
},
|
|
113
|
+
className: f,
|
|
114
|
+
children: y.consentCancel
|
|
115
|
+
}), /* @__PURE__ */ u("button", {
|
|
116
|
+
type: "button",
|
|
117
|
+
onClick: () => {
|
|
118
|
+
C.current?.focus(), h.confirm();
|
|
119
|
+
},
|
|
120
|
+
"aria-describedby": b,
|
|
121
|
+
className: m,
|
|
122
|
+
disabled: E.selectedAvatarId === null,
|
|
123
|
+
"data-testid": "assistant-live-avatar-confirm",
|
|
124
|
+
children: y.consentConfirm
|
|
125
|
+
})]
|
|
126
|
+
})
|
|
127
|
+
]
|
|
128
|
+
}) : null,
|
|
129
|
+
/* @__PURE__ */ u("p", {
|
|
130
|
+
className: A === null ? "sr-only" : "max-w-xs text-center text-xs text-text-muted",
|
|
131
|
+
role: "status",
|
|
132
|
+
"data-testid": "assistant-live-avatar-status",
|
|
133
|
+
children: A ?? ""
|
|
134
|
+
})
|
|
135
|
+
]
|
|
136
|
+
});
|
|
137
|
+
};
|
|
138
|
+
//#endregion
|
|
139
|
+
export { h as LiveAvatarStageView };
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
export interface AudioBufferLike {
|
|
2
|
+
readonly sampleRate: number;
|
|
3
|
+
readonly length: number;
|
|
4
|
+
readonly numberOfChannels: number;
|
|
5
|
+
getChannelData(channel: number): Float32Array;
|
|
6
|
+
}
|
|
7
|
+
/** The two browser operations this module needs, injectable so the logic is testable without a browser. */
|
|
8
|
+
export interface AudioDecodeEnv {
|
|
9
|
+
/** decodeAudioData on a fresh OfflineAudioContext of `contextRate` Hz (the browser resamples the result to that rate). */
|
|
10
|
+
decode(data: ArrayBuffer, contextRate: number): Promise<AudioBufferLike>;
|
|
11
|
+
/** Render `buffer` through a mono OfflineAudioContext of `rate` Hz (graph resample + downmix). */
|
|
12
|
+
renderMono(buffer: AudioBufferLike, rate: number): Promise<Float32Array>;
|
|
13
|
+
}
|
|
14
|
+
export type AvatarDecodeStrategy = 'decode-at-target' | 'render';
|
|
15
|
+
export declare const DEFAULT_AVATAR_DECODE_STRATEGY: AvatarDecodeStrategy;
|
|
16
|
+
/** Refuse inputs larger than this before touching the decoder (a hosted TTS sentence is a few hundred KB). */
|
|
17
|
+
export declare const DEFAULT_AVATAR_DECODE_MAX_BYTES: number;
|
|
18
|
+
export interface AvatarDecodeOptions {
|
|
19
|
+
signal?: AbortSignal | undefined;
|
|
20
|
+
strategy?: AvatarDecodeStrategy | undefined;
|
|
21
|
+
maxBytes?: number | undefined;
|
|
22
|
+
env?: AudioDecodeEnv | undefined;
|
|
23
|
+
}
|
|
24
|
+
export interface DecodedPcm16 {
|
|
25
|
+
/** PCM16 little-endian mono at 16 kHz. */
|
|
26
|
+
pcm: Uint8Array;
|
|
27
|
+
durationSec: number;
|
|
28
|
+
sampleCount: number;
|
|
29
|
+
}
|
|
30
|
+
/** The real browser environment. Throws a typed error (not a TypeError) when there is no Web Audio. */
|
|
31
|
+
export declare function browserAvatarAudioEnv(): AudioDecodeEnv;
|
|
32
|
+
/**
|
|
33
|
+
* Decode MP3 bytes to PCM16 LE mono 16 kHz.
|
|
34
|
+
* - invalid / empty / non-MP3 -> InvalidAudioError
|
|
35
|
+
* - last frame cut -> TruncatedAudioError (decodeAudioData alone would silently return the partial audio)
|
|
36
|
+
* - browser decoder rejects -> AudioDecodeError('decode_failed')
|
|
37
|
+
* - signal aborted before or during -> AudioDecodeAbortedError, and no result is ever delivered afterwards
|
|
38
|
+
* The caller's `bytes` are never detached: decodeAudioData transfers its buffer, so a copy is handed to it.
|
|
39
|
+
*/
|
|
40
|
+
export declare function decodeMp3ToPcm16k(bytes: Uint8Array, options?: AvatarDecodeOptions): Promise<DecodedPcm16>;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { AudioDecodeAbortedError as e, AudioDecodeError as t, InvalidAudioError as n, TruncatedAudioError as r } from "./avatarAudioErrors.js";
|
|
2
|
+
import { inspectMp3 as i } from "./avatarMp3Inspect.js";
|
|
3
|
+
import { AVATAR_SAMPLE_RATE as a, downmixToMono as o, floatToPcm16Bytes as s } from "./avatarPcm16.js";
|
|
4
|
+
//#region src/shared/assistant/avatar/live/avatarAudioDecode.ts
|
|
5
|
+
var c = "decode-at-target", l = 16 * 1024 * 1024, u = 48e3;
|
|
6
|
+
function d() {
|
|
7
|
+
let e = globalThis;
|
|
8
|
+
return e.OfflineAudioContext ?? e.webkitOfflineAudioContext ?? null;
|
|
9
|
+
}
|
|
10
|
+
function f() {
|
|
11
|
+
let e = d();
|
|
12
|
+
if (!e) throw new t("unsupported_environment", "OfflineAudioContext is not available in this environment");
|
|
13
|
+
return {
|
|
14
|
+
decode(t, n) {
|
|
15
|
+
return new e(1, 1, n).decodeAudioData(t);
|
|
16
|
+
},
|
|
17
|
+
async renderMono(t, n) {
|
|
18
|
+
let r = new e(1, Math.max(1, Math.ceil(t.length * (n / t.sampleRate))), n), i = r.createBufferSource();
|
|
19
|
+
return i.buffer = t, i.connect(r.destination), i.start(0), (await r.startRendering()).getChannelData(0);
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
function p(t) {
|
|
24
|
+
if (t?.aborted) throw new e();
|
|
25
|
+
}
|
|
26
|
+
function m(t, n) {
|
|
27
|
+
return n ? new Promise((r, i) => {
|
|
28
|
+
let a = () => i(new e());
|
|
29
|
+
if (n.aborted) {
|
|
30
|
+
a(), t.catch(() => void 0);
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
n.addEventListener("abort", a, { once: !0 }), t.then((t) => {
|
|
34
|
+
n.removeEventListener("abort", a), n.aborted ? i(new e()) : r(t);
|
|
35
|
+
}, (t) => {
|
|
36
|
+
n.removeEventListener("abort", a), i(n.aborted ? new e() : t);
|
|
37
|
+
});
|
|
38
|
+
}) : t;
|
|
39
|
+
}
|
|
40
|
+
async function h(e, c = {}) {
|
|
41
|
+
let { signal: l } = c;
|
|
42
|
+
p(l);
|
|
43
|
+
let d = c.maxBytes ?? 16777216;
|
|
44
|
+
if (e.byteLength === 0) throw new n("empty audio");
|
|
45
|
+
if (e.byteLength > d) throw new t("too_large", `audio is ${e.byteLength} bytes, limit is ${d}`);
|
|
46
|
+
let h = i(e);
|
|
47
|
+
if (!h) throw new n("no MP3 frames found");
|
|
48
|
+
if (h.truncated) throw new r(h.danglingBytes, h.durationSec);
|
|
49
|
+
let g = c.env ?? f(), _ = c.strategy ?? "decode-at-target", v = e.slice().buffer, y = await m((async () => {
|
|
50
|
+
try {
|
|
51
|
+
if (_ === "render") {
|
|
52
|
+
let e = await g.decode(v, u);
|
|
53
|
+
return await g.renderMono(e, a);
|
|
54
|
+
}
|
|
55
|
+
let e = await g.decode(v, a);
|
|
56
|
+
if (e.sampleRate !== 16e3) throw new t("decode_failed", `browser returned ${e.sampleRate} Hz, expected ${a}`);
|
|
57
|
+
let n = [];
|
|
58
|
+
for (let t = 0; t < e.numberOfChannels; t += 1) n.push(e.getChannelData(t));
|
|
59
|
+
return o(n);
|
|
60
|
+
} catch (e) {
|
|
61
|
+
throw e instanceof t ? e : new t("decode_failed", `browser could not decode the audio: ${e instanceof Error ? e.message : String(e)}`, { cause: e });
|
|
62
|
+
}
|
|
63
|
+
})(), l);
|
|
64
|
+
if (p(l), y.length === 0) throw new n("decoder produced no samples");
|
|
65
|
+
return {
|
|
66
|
+
pcm: s(y),
|
|
67
|
+
durationSec: y.length / a,
|
|
68
|
+
sampleCount: y.length
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
//#endregion
|
|
72
|
+
export { l as DEFAULT_AVATAR_DECODE_MAX_BYTES, c as DEFAULT_AVATAR_DECODE_STRATEGY, f as browserAvatarAudioEnv, h as decodeMp3ToPcm16k };
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export type AudioDecodeErrorCode = 'invalid_audio' | 'truncated_audio' | 'decode_failed' | 'too_large' | 'aborted' | 'unsupported_environment';
|
|
2
|
+
export declare class AudioDecodeError extends Error {
|
|
3
|
+
readonly code: AudioDecodeErrorCode;
|
|
4
|
+
constructor(code: AudioDecodeErrorCode, message: string, options?: {
|
|
5
|
+
cause?: unknown;
|
|
6
|
+
});
|
|
7
|
+
}
|
|
8
|
+
export declare class InvalidAudioError extends AudioDecodeError {
|
|
9
|
+
constructor(message: string, options?: {
|
|
10
|
+
cause?: unknown;
|
|
11
|
+
});
|
|
12
|
+
}
|
|
13
|
+
export declare class TruncatedAudioError extends AudioDecodeError {
|
|
14
|
+
/** Bytes of the incomplete last frame. */
|
|
15
|
+
readonly danglingBytes: number;
|
|
16
|
+
/** Duration of the complete frames that WERE present. */
|
|
17
|
+
readonly completeFramesDurationSec: number;
|
|
18
|
+
constructor(danglingBytes: number, completeFramesDurationSec: number);
|
|
19
|
+
}
|
|
20
|
+
export declare class AudioDecodeAbortedError extends AudioDecodeError {
|
|
21
|
+
constructor();
|
|
22
|
+
}
|
|
23
|
+
export declare class PcmFormatError extends Error {
|
|
24
|
+
constructor(message: string);
|
|
25
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
//#region src/shared/assistant/avatar/live/avatarAudioErrors.ts
|
|
2
|
+
var e = class extends Error {
|
|
3
|
+
code;
|
|
4
|
+
constructor(e, t, n) {
|
|
5
|
+
super(t, n), this.name = "AudioDecodeError", this.code = e;
|
|
6
|
+
}
|
|
7
|
+
}, t = class extends e {
|
|
8
|
+
constructor(e, t) {
|
|
9
|
+
super("invalid_audio", e, t), this.name = "InvalidAudioError";
|
|
10
|
+
}
|
|
11
|
+
}, n = class extends e {
|
|
12
|
+
danglingBytes;
|
|
13
|
+
completeFramesDurationSec;
|
|
14
|
+
constructor(e, t) {
|
|
15
|
+
super("truncated_audio", `MP3 ends inside a frame (${e} dangling bytes after ${t.toFixed(3)} s of complete frames)`), this.name = "TruncatedAudioError", this.danglingBytes = e, this.completeFramesDurationSec = t;
|
|
16
|
+
}
|
|
17
|
+
}, r = class extends e {
|
|
18
|
+
constructor() {
|
|
19
|
+
super("aborted", "audio decode aborted"), this.name = "AudioDecodeAbortedError";
|
|
20
|
+
}
|
|
21
|
+
}, i = class extends Error {
|
|
22
|
+
constructor(e) {
|
|
23
|
+
super(e), this.name = "PcmFormatError";
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
//#endregion
|
|
27
|
+
export { r as AudioDecodeAbortedError, e as AudioDecodeError, t as InvalidAudioError, i as PcmFormatError, n as TruncatedAudioError };
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
export interface AvatarLevelDetectorConfig {
|
|
2
|
+
/** RMS (0..1) above which audio counts as audible (about -50 dBFS). */
|
|
3
|
+
readonly onThreshold: number;
|
|
4
|
+
/** RMS below which audio counts as quiet again (hysteresis, about -55 dBFS). */
|
|
5
|
+
readonly offThreshold: number;
|
|
6
|
+
/** Must stay above onThreshold this long before "audible" is reported. */
|
|
7
|
+
readonly attackMs: number;
|
|
8
|
+
/** Must stay below offThreshold this long before "quiet" is reported (bridges the gaps between words). */
|
|
9
|
+
readonly hangoverMs: number;
|
|
10
|
+
}
|
|
11
|
+
export declare const DEFAULT_AVATAR_LEVEL_CONFIG: AvatarLevelDetectorConfig;
|
|
12
|
+
/** Pure hysteresis detector: feed it RMS samples with timestamps. */
|
|
13
|
+
export declare class AvatarLevelDetector {
|
|
14
|
+
private readonly config;
|
|
15
|
+
private audible;
|
|
16
|
+
private aboveSince;
|
|
17
|
+
private belowSince;
|
|
18
|
+
constructor(config?: AvatarLevelDetectorConfig);
|
|
19
|
+
get isAudible(): boolean;
|
|
20
|
+
/** Returns the new state when it changed, otherwise null. */
|
|
21
|
+
push(rms: number, nowMs: number): boolean | null;
|
|
22
|
+
reset(): void;
|
|
23
|
+
}
|
|
24
|
+
export declare function rmsOfFloat32(samples: Float32Array): number;
|
|
25
|
+
export interface AvatarMeterHandle {
|
|
26
|
+
stop(): void;
|
|
27
|
+
/** True while the meter can really hear: not stopped and its AudioContext is running (a suspended context reads silence). */
|
|
28
|
+
isRunning(): boolean;
|
|
29
|
+
/** Resume a suspended context. Call it from the same user gesture that resumes the avatar's audio. */
|
|
30
|
+
resume(): Promise<void>;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* WebAudio tap: MediaStream -> AnalyserNode, polled at about 50 Hz. The tap is NOT connected to the destination,
|
|
34
|
+
* so it adds no second audible copy. Whether Chrome delivers non-zero samples for a REMOTE track depends on the
|
|
35
|
+
* track also being attached to a media element (Gate 1 package L, test f, measures this).
|
|
36
|
+
*/
|
|
37
|
+
export declare function startAvatarAudioMeter(stream: MediaStream, onChange: (audible: boolean) => void, config?: AvatarLevelDetectorConfig, pollMs?: number): AvatarMeterHandle;
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
//#region src/shared/assistant/avatar/live/avatarAudioMeter.ts
|
|
2
|
+
var e = {
|
|
3
|
+
onThreshold: .003,
|
|
4
|
+
offThreshold: .0018,
|
|
5
|
+
attackMs: 60,
|
|
6
|
+
hangoverMs: 400
|
|
7
|
+
}, t = class {
|
|
8
|
+
config;
|
|
9
|
+
audible = !1;
|
|
10
|
+
aboveSince = null;
|
|
11
|
+
belowSince = null;
|
|
12
|
+
constructor(t = e) {
|
|
13
|
+
this.config = t;
|
|
14
|
+
}
|
|
15
|
+
get isAudible() {
|
|
16
|
+
return this.audible;
|
|
17
|
+
}
|
|
18
|
+
push(e, t) {
|
|
19
|
+
if (!this.audible) {
|
|
20
|
+
if (e >= this.config.onThreshold) {
|
|
21
|
+
if (this.aboveSince ??= t, t - this.aboveSince >= this.config.attackMs) return this.audible = !0, this.aboveSince = null, this.belowSince = null, !0;
|
|
22
|
+
} else this.aboveSince = null;
|
|
23
|
+
return null;
|
|
24
|
+
}
|
|
25
|
+
if (e <= this.config.offThreshold) {
|
|
26
|
+
if (this.belowSince ??= t, t - this.belowSince >= this.config.hangoverMs) return this.audible = !1, this.belowSince = null, this.aboveSince = null, !1;
|
|
27
|
+
} else this.belowSince = null;
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
reset() {
|
|
31
|
+
this.audible = !1, this.aboveSince = null, this.belowSince = null;
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
function n(e) {
|
|
35
|
+
if (e.length === 0) return 0;
|
|
36
|
+
let t = 0;
|
|
37
|
+
for (let n = 0; n < e.length; n += 1) {
|
|
38
|
+
let r = e[n] ?? 0;
|
|
39
|
+
t += r * r;
|
|
40
|
+
}
|
|
41
|
+
return Math.sqrt(t / e.length);
|
|
42
|
+
}
|
|
43
|
+
function r(r, i, a = e, o = 20) {
|
|
44
|
+
let s = globalThis.AudioContext, c = new s(), l = c.createMediaStreamSource(r), u = c.createAnalyser();
|
|
45
|
+
u.fftSize = 1024, l.connect(u);
|
|
46
|
+
let d = new Float32Array(u.fftSize), f = new t(a), p = setInterval(() => {
|
|
47
|
+
u.getFloatTimeDomainData(d);
|
|
48
|
+
let e = f.push(n(d), performance.now());
|
|
49
|
+
e !== null && i(e);
|
|
50
|
+
}, o), m = !1;
|
|
51
|
+
return {
|
|
52
|
+
isRunning: () => !m && c.state === "running",
|
|
53
|
+
resume: async () => {
|
|
54
|
+
!m && c.state === "suspended" && await c.resume().catch(() => void 0);
|
|
55
|
+
},
|
|
56
|
+
stop() {
|
|
57
|
+
m || (m = !0, clearInterval(p), l.disconnect(), c.state !== "closed" && c.close().catch(() => void 0));
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
//#endregion
|
|
62
|
+
export { t as AvatarLevelDetector, e as DEFAULT_AVATAR_LEVEL_CONFIG, n as rmsOfFloat32, r as startAvatarAudioMeter };
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export interface Mp3Info {
|
|
2
|
+
frames: number;
|
|
3
|
+
sampleRate: number;
|
|
4
|
+
channels: number;
|
|
5
|
+
samplesPerFrame: number;
|
|
6
|
+
/** frames * samplesPerFrame / sampleRate: what a decoder without delay compensation outputs. */
|
|
7
|
+
durationSec: number;
|
|
8
|
+
firstFrameOffset: number;
|
|
9
|
+
/** The last frame is cut short. */
|
|
10
|
+
truncated: boolean;
|
|
11
|
+
/** Bytes of the incomplete last frame (0 unless truncated). */
|
|
12
|
+
danglingBytes: number;
|
|
13
|
+
/** Non-frame bytes after the last complete frame (ID3v1 tag, padding, junk), excluding the dangling frame. */
|
|
14
|
+
trailingBytes: number;
|
|
15
|
+
}
|
|
16
|
+
interface FrameHeader {
|
|
17
|
+
length: number;
|
|
18
|
+
sampleRate: number;
|
|
19
|
+
channels: number;
|
|
20
|
+
samplesPerFrame: number;
|
|
21
|
+
}
|
|
22
|
+
/** Parse the 4-byte header at `at`, or null when it is not a valid frame header. */
|
|
23
|
+
export declare function parseFrameHeader(bytes: Uint8Array, at: number): FrameHeader | null;
|
|
24
|
+
/**
|
|
25
|
+
* Walk the frames. Returns null when no valid MP3 frame sequence starts in the first 64 KiB (garbage, empty, wrong format).
|
|
26
|
+
* A candidate first frame is accepted only when the NEXT header also parses at the computed offset (or the file ends there),
|
|
27
|
+
* which makes a random 0xFF 0xFx byte pair in garbage vanishingly unlikely to be taken for MP3.
|
|
28
|
+
*/
|
|
29
|
+
export declare function inspectMp3(bytes: Uint8Array): Mp3Info | null;
|
|
30
|
+
export {};
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
//#region src/shared/assistant/avatar/live/avatarMp3Inspect.ts
|
|
2
|
+
var e = [
|
|
3
|
+
0,
|
|
4
|
+
32,
|
|
5
|
+
64,
|
|
6
|
+
96,
|
|
7
|
+
128,
|
|
8
|
+
160,
|
|
9
|
+
192,
|
|
10
|
+
224,
|
|
11
|
+
256,
|
|
12
|
+
288,
|
|
13
|
+
320,
|
|
14
|
+
352,
|
|
15
|
+
384,
|
|
16
|
+
416,
|
|
17
|
+
448
|
|
18
|
+
], t = [
|
|
19
|
+
0,
|
|
20
|
+
32,
|
|
21
|
+
48,
|
|
22
|
+
56,
|
|
23
|
+
64,
|
|
24
|
+
80,
|
|
25
|
+
96,
|
|
26
|
+
112,
|
|
27
|
+
128,
|
|
28
|
+
160,
|
|
29
|
+
192,
|
|
30
|
+
224,
|
|
31
|
+
256,
|
|
32
|
+
320,
|
|
33
|
+
384
|
|
34
|
+
], n = [
|
|
35
|
+
0,
|
|
36
|
+
32,
|
|
37
|
+
40,
|
|
38
|
+
48,
|
|
39
|
+
56,
|
|
40
|
+
64,
|
|
41
|
+
80,
|
|
42
|
+
96,
|
|
43
|
+
112,
|
|
44
|
+
128,
|
|
45
|
+
160,
|
|
46
|
+
192,
|
|
47
|
+
224,
|
|
48
|
+
256,
|
|
49
|
+
320
|
|
50
|
+
], r = [
|
|
51
|
+
0,
|
|
52
|
+
32,
|
|
53
|
+
48,
|
|
54
|
+
56,
|
|
55
|
+
64,
|
|
56
|
+
80,
|
|
57
|
+
96,
|
|
58
|
+
112,
|
|
59
|
+
128,
|
|
60
|
+
144,
|
|
61
|
+
160,
|
|
62
|
+
176,
|
|
63
|
+
192,
|
|
64
|
+
224,
|
|
65
|
+
256
|
|
66
|
+
], i = [
|
|
67
|
+
0,
|
|
68
|
+
8,
|
|
69
|
+
16,
|
|
70
|
+
24,
|
|
71
|
+
32,
|
|
72
|
+
40,
|
|
73
|
+
48,
|
|
74
|
+
56,
|
|
75
|
+
64,
|
|
76
|
+
80,
|
|
77
|
+
96,
|
|
78
|
+
112,
|
|
79
|
+
128,
|
|
80
|
+
144,
|
|
81
|
+
160
|
|
82
|
+
], a = {
|
|
83
|
+
3: [
|
|
84
|
+
44100,
|
|
85
|
+
48e3,
|
|
86
|
+
32e3
|
|
87
|
+
],
|
|
88
|
+
2: [
|
|
89
|
+
22050,
|
|
90
|
+
24e3,
|
|
91
|
+
16e3
|
|
92
|
+
],
|
|
93
|
+
0: [
|
|
94
|
+
11025,
|
|
95
|
+
12e3,
|
|
96
|
+
8e3
|
|
97
|
+
]
|
|
98
|
+
};
|
|
99
|
+
function o(o, s) {
|
|
100
|
+
if (s + 4 > o.length) return null;
|
|
101
|
+
let c = o[s] ?? 0, l = o[s + 1] ?? 0, u = o[s + 2] ?? 0, d = o[s + 3] ?? 0;
|
|
102
|
+
if (c !== 255 || (l & 224) != 224) return null;
|
|
103
|
+
let f = l >> 3 & 3, p = l >> 1 & 3;
|
|
104
|
+
if (f === 1 || p === 0) return null;
|
|
105
|
+
let m = u >> 4 & 15, h = u >> 2 & 3;
|
|
106
|
+
if (m === 0 || m === 15 || h === 3) return null;
|
|
107
|
+
let g = a[f]?.[h];
|
|
108
|
+
if (!g) return null;
|
|
109
|
+
let _ = u >> 1 & 1, v = f === 3, y, b;
|
|
110
|
+
p === 3 ? (y = v ? e : r, b = 384) : p === 2 ? (y = v ? t : i, b = 1152) : (y = v ? n : i, b = v ? 1152 : 576);
|
|
111
|
+
let x = y[m];
|
|
112
|
+
if (!x) return null;
|
|
113
|
+
let S = x * 1e3, C = p === 3 ? (Math.floor(12 * S / g) + _) * 4 : Math.floor(b / 8 * S / g) + _;
|
|
114
|
+
return C < 4 ? null : {
|
|
115
|
+
length: C,
|
|
116
|
+
sampleRate: g,
|
|
117
|
+
channels: (d >> 6 & 3) == 3 ? 1 : 2,
|
|
118
|
+
samplesPerFrame: b
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
function s(e) {
|
|
122
|
+
if (e.length >= 10 && e[0] === 73 && e[1] === 68 && e[2] === 51) {
|
|
123
|
+
let t = ((e[6] ?? 0) & 127) * 2097152 + ((e[7] ?? 0) & 127) * 16384 + ((e[8] ?? 0) & 127) * 128 + ((e[9] ?? 0) & 127), n = (e[5] ?? 0) & 16 ? 10 : 0;
|
|
124
|
+
return 10 + t + n;
|
|
125
|
+
}
|
|
126
|
+
return 0;
|
|
127
|
+
}
|
|
128
|
+
function c(e) {
|
|
129
|
+
let t = s(e), n = Math.min(e.length, t + 65536), r = -1, i = null;
|
|
130
|
+
for (let a = t; a < n; a += 1) {
|
|
131
|
+
let t = o(e, a);
|
|
132
|
+
if (!t) continue;
|
|
133
|
+
let n = a + t.length, s = n >= e.length, c = !s && o(e, n) !== null;
|
|
134
|
+
if (s || c) {
|
|
135
|
+
r = a, i = t;
|
|
136
|
+
break;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (r < 0 || !i) return null;
|
|
140
|
+
let a = r, c = 0, l = !1, u = 0, { sampleRate: d, channels: f, samplesPerFrame: p } = i;
|
|
141
|
+
for (;;) {
|
|
142
|
+
let t = o(e, a);
|
|
143
|
+
if (!t || t.sampleRate !== d) {
|
|
144
|
+
let t = e.length - a;
|
|
145
|
+
t > 0 && t < 4 && (e[a] ?? 0) === 255 && (l = !0, u = t, a = e.length);
|
|
146
|
+
break;
|
|
147
|
+
}
|
|
148
|
+
if (a + t.length > e.length) {
|
|
149
|
+
l = !0, u = e.length - a, a = e.length;
|
|
150
|
+
break;
|
|
151
|
+
}
|
|
152
|
+
c += 1, a += t.length;
|
|
153
|
+
}
|
|
154
|
+
let m = l ? e.length - u : a;
|
|
155
|
+
return {
|
|
156
|
+
frames: c,
|
|
157
|
+
sampleRate: d,
|
|
158
|
+
channels: f,
|
|
159
|
+
samplesPerFrame: p,
|
|
160
|
+
durationSec: c * p / d,
|
|
161
|
+
firstFrameOffset: r,
|
|
162
|
+
truncated: l,
|
|
163
|
+
danglingBytes: u,
|
|
164
|
+
trailingBytes: l ? 0 : e.length - m
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
//#endregion
|
|
168
|
+
export { c as inspectMp3, o as parseFrameHeader };
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export declare const AVATAR_SAMPLE_RATE = 16000;
|
|
2
|
+
export declare const PCM16_BYTES_PER_SAMPLE = 2;
|
|
3
|
+
/** float [-1, 1] -> int16, clamped. Negative scales by 32768 and positive by 32767 so +1.0 does not wrap; NaN becomes silence. */
|
|
4
|
+
export declare function floatToInt16(samples: Float32Array): Int16Array;
|
|
5
|
+
/** int16 samples -> little-endian bytes. Uses DataView so it is correct on any host byte order. */
|
|
6
|
+
export declare function int16ToBytesLE(samples: Int16Array): Uint8Array;
|
|
7
|
+
export declare function floatToPcm16Bytes(samples: Float32Array): Uint8Array;
|
|
8
|
+
/** Bytes -> int16 samples (for tests and verification). Throws on an odd byte count. */
|
|
9
|
+
export declare function bytesToInt16LE(pcm: Uint8Array): Int16Array;
|
|
10
|
+
/** Seconds of audio in mono PCM16 bytes. Throws on an odd byte count rather than rounding it away. */
|
|
11
|
+
export declare function pcm16DurationOf(pcm: Uint8Array | number, sampleRate?: number): number;
|
|
12
|
+
/** Samples in one chunk of `ms` milliseconds (at least 1, so tiny `ms` still makes progress). */
|
|
13
|
+
export declare function pcm16SamplesPerChunk(ms: number, sampleRate?: number): number;
|
|
14
|
+
/**
|
|
15
|
+
* Cut mono PCM16 into slices of `ms` milliseconds on whole-sample boundaries. The slices are zero-copy views of `pcm`
|
|
16
|
+
* (callers that keep them past mutation of `pcm` must copy). Every slice but possibly the last has exactly
|
|
17
|
+
* pcm16SamplesPerChunk(ms) samples; the last holds the remainder. An odd byte length is rejected: a half sample is corruption.
|
|
18
|
+
*/
|
|
19
|
+
export declare function chunkPcm16(pcm: Uint8Array, ms: number, sampleRate?: number): Uint8Array[];
|
|
20
|
+
/** Mean of channels into one mono channel. */
|
|
21
|
+
export declare function downmixToMono(channels: readonly Float32Array[]): Float32Array;
|