@alia.onl/sdk 15.7.0 → 15.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +17 -3
- package/package.json +3 -1
- package/src/csp.ts +50 -0
- package/src/hooks/useAliaActivity.ts +1 -1
- package/src/hooks/useAliaChat.ts +1 -1
- package/src/hooks/useDictation.ts +2 -3
- package/src/hooks/useTTS.ts +2 -3
- package/src/hooks/useVoiceRoom.ts +2 -3
- package/src/lib/character-runtime.ts +6 -4
- package/src/lib/origins.ts +15 -0
- package/src/lib/voice-audio-engine.ts +14 -158
- package/src/lib/voice-worklets.ts +19 -0
- package/worklets/voice-processors.js +152 -0
- package/src/lib/api-url.ts +0 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# @alia.onl/sdk
|
|
2
2
|
|
|
3
|
+
## 15.8.0
|
|
4
|
+
|
|
5
|
+
- **The SDK declares its CSP, and the voice call needs no `blob:` script.** In an app with a
|
|
6
|
+
Content-Security-Policy (Mention) the call failed twice over: `connect-src` lacked
|
|
7
|
+
`wss://api.alia.onl`, and the worklets were `blob:` URLs that `script-src` blocks
|
|
8
|
+
(`Failed to load worklet module script`). `@alia.onl/sdk/csp` exports `ALIA_SDK_CSP` and
|
|
9
|
+
`withAliaSdkCsp()` (`OxyCspExtensions` of `@oxy.so/core`), derived from the same origins the
|
|
10
|
+
SDK's URLs start from (`lib/origins.ts`); a test fails if the source names an origin it does not
|
|
11
|
+
declare. The worklets are one plain module, `worklets/voice-processors.js`, taking their numbers
|
|
12
|
+
as `processorOptions` and served by Alia's web app under its content hash
|
|
13
|
+
(`https://alia.onl/sdk/voice-worklets/<revision>.js`, CORS open, every revision kept), under the
|
|
14
|
+
`script-src https://alia.onl` the avatar already needs. The API's URL is defined once (the
|
|
15
|
+
dictation, read-aloud and voice hooks each had their own copy).
|
|
16
|
+
|
|
3
17
|
## 15.7.0
|
|
4
18
|
|
|
5
19
|
- **`AliaAvatarIcon` is her still until she moves.** At icon size it drew Alia through the live 3D
|
package/README.md
CHANGED
|
@@ -28,12 +28,25 @@ The chat is built from Bloom (`@oxy.so/bloom`, a peer dependency): mount Bloom's
|
|
|
28
28
|
Alia's avatar is Bloom's 3D character (`AgentAvatar`). It draws through a runtime of
|
|
29
29
|
static files (about 214 MB), which the chat loads from Alia's own origin
|
|
30
30
|
(`ALIA_CHARACTER_RUNTIME_URL`, served with CORS open), so the app hosts nothing and mounts
|
|
31
|
-
no avatar provider.
|
|
32
|
-
`connect-src` (and `'wasm-unsafe-eval'`). To serve your own copy instead, serve the Bloom
|
|
31
|
+
no avatar provider. To serve your own copy instead, serve the Bloom
|
|
33
32
|
package's `assets/character-runtime/` directory unchanged and pass the URL of `runtime.mjs`
|
|
34
33
|
as `characterRuntimeUrl` to `AliaChatSheet` or `AliaChatScreen`; native avatars run in a
|
|
35
34
|
WebView, so there the URL must be absolute and reachable from the device.
|
|
36
35
|
|
|
36
|
+
A web app with a Content-Security-Policy composes the SDK's into its own instead of listing
|
|
37
|
+
Alia's origins by hand: `@alia.onl/sdk/csp` exports `ALIA_SDK_CSP` and `withAliaSdkCsp(extensions)`
|
|
38
|
+
in the shape of `@oxy.so/core`'s `OxyCspExtensions` (a leaf entry, safe on a server):
|
|
39
|
+
|
|
40
|
+
```ts
|
|
41
|
+
import { withAliaSdkCsp } from '@alia.onl/sdk/csp';
|
|
42
|
+
app.use(createOxySecurityHeaders({ csp: withAliaSdkCsp(APP_CSP_EXTENSIONS) }));
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
It declares the API and its WebSocket (`connect-src`), the avatar runtime and the voice call's
|
|
46
|
+
worklets on `https://alia.onl` (`script-src`, with `'wasm-unsafe-eval'` for the runtime's
|
|
47
|
+
WebAssembly), attachments and read-aloud clips (`img-src`, `media-src`). No `blob:` script:
|
|
48
|
+
the worklets are a content-addressed file on `alia.onl`, not a blob URL.
|
|
49
|
+
|
|
37
50
|
The empty chat greets the signed-in person by name ("Hi Nate, what's on your mind?"); pass
|
|
38
51
|
`welcomeGreeting` to say something else, or `labels.greeting` / `labels.greetingNamed` to translate it.
|
|
39
52
|
|
|
@@ -44,9 +57,10 @@ The chat's own copy is English. Pass `labels` to `AliaChatSheet` or
|
|
|
44
57
|
`AliaChatScreen` to translate it (`AliaChatLabels`; anything omitted stays
|
|
45
58
|
English). Bloom's own strings follow `BloomProvider`'s `locale`.
|
|
46
59
|
|
|
47
|
-
|
|
60
|
+
Entry points:
|
|
48
61
|
|
|
49
62
|
- `@alia.onl/sdk` — text chat, dictation, read-aloud, catalogue, primitives.
|
|
63
|
+
- `@alia.onl/sdk/csp` — the CSP the SDK needs from a host page (above).
|
|
50
64
|
- `@alia.onl/sdk/voice` — `VoiceSession` and `useVoiceRoom`, the voice call.
|
|
51
65
|
Pass `VoiceSession` to the shell as `voiceSession` to offer calls; leave it
|
|
52
66
|
out for text-only chat and the call is never compiled.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@alia.onl/sdk",
|
|
3
|
-
"version": "15.
|
|
3
|
+
"version": "15.8.0",
|
|
4
4
|
"main": "src/index.ts",
|
|
5
5
|
"react-native": "src/index.ts",
|
|
6
6
|
"source": "src/index.ts",
|
|
@@ -9,12 +9,14 @@
|
|
|
9
9
|
".": "./src/index.ts",
|
|
10
10
|
"./avatar": "./src/avatar.ts",
|
|
11
11
|
"./content": "./src/lib/content-utils.ts",
|
|
12
|
+
"./csp": "./src/csp.ts",
|
|
12
13
|
"./tools": "./src/tools.ts",
|
|
13
14
|
"./voice": "./src/voice.ts",
|
|
14
15
|
"./package.json": "./package.json"
|
|
15
16
|
},
|
|
16
17
|
"files": [
|
|
17
18
|
"src",
|
|
19
|
+
"worklets",
|
|
18
20
|
"NOTICE",
|
|
19
21
|
"tsconfig.json",
|
|
20
22
|
"CHANGELOG.md"
|
package/src/csp.ts
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `@alia.onl/sdk/csp` — the Content-Security-Policy the SDK needs from a host page, in the
|
|
3
|
+
* shape of `@oxy.so/core`'s `OxyCspExtensions`, so a host composes it into its own policy
|
|
4
|
+
* instead of copying Alia's origins by hand:
|
|
5
|
+
*
|
|
6
|
+
* ```ts
|
|
7
|
+
* import { withAliaSdkCsp } from '@alia.onl/sdk/csp';
|
|
8
|
+
* createOxySecurityHeaders({ csp: withAliaSdkCsp(APP_CSP_EXTENSIONS) });
|
|
9
|
+
* ```
|
|
10
|
+
*
|
|
11
|
+
* Every source is derived from `lib/origins.ts`, the values the SDK's own URLs start from, and a
|
|
12
|
+
* test fails if the SDK's source names an origin this does not declare. A leaf entry: no React,
|
|
13
|
+
* no React Native, safe to import from a server.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
import type { OxyCspDirective, OxyCspExtensions } from '@oxy.so/core/server';
|
|
17
|
+
import { ALIA_API_URL, ALIA_WEB_ORIGIN } from './lib/origins';
|
|
18
|
+
import { voiceSessionUrl } from './lib/voice-protocol';
|
|
19
|
+
|
|
20
|
+
const api = new URL(ALIA_API_URL).origin;
|
|
21
|
+
const voiceSocket = new URL(voiceSessionUrl(ALIA_API_URL)).origin;
|
|
22
|
+
|
|
23
|
+
/** What the SDK loads, by directive, and why. */
|
|
24
|
+
export const ALIA_SDK_CSP = Object.freeze({
|
|
25
|
+
// The API (chat, dictation, speech, the activity socket's polling) and its WebSocket (the
|
|
26
|
+
// voice call, socket.io); the avatar runtime fetches its WebAssembly and data from the web app.
|
|
27
|
+
connectSrc: Object.freeze([api, voiceSocket, ALIA_WEB_ORIGIN]),
|
|
28
|
+
// The avatar runtime and the voice call's worklets are modules served by the web app; the
|
|
29
|
+
// runtime compiles WebAssembly (`'wasm-unsafe-eval'` allows that and nothing else: no eval).
|
|
30
|
+
scriptSrc: Object.freeze([ALIA_WEB_ORIGIN, "'wasm-unsafe-eval'"]),
|
|
31
|
+
// Stored attachments come from the API; an attachment's preview before upload is a blob.
|
|
32
|
+
imgSrc: Object.freeze([api, 'blob:']),
|
|
33
|
+
// Read-aloud clips are played from the API.
|
|
34
|
+
mediaSrc: Object.freeze([api]),
|
|
35
|
+
} satisfies OxyCspExtensions);
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* The host's own CSP extensions with the SDK's added: each directive is the union, the host's
|
|
39
|
+
* sources first. `@oxy.so/core` dedupes when it builds the policy.
|
|
40
|
+
*/
|
|
41
|
+
export function withAliaSdkCsp(host: OxyCspExtensions = {}): OxyCspExtensions {
|
|
42
|
+
const merged: Partial<Record<OxyCspDirective, readonly string[]>> = { ...host };
|
|
43
|
+
for (const [directive, sources] of Object.entries(ALIA_SDK_CSP) as [
|
|
44
|
+
OxyCspDirective,
|
|
45
|
+
readonly string[],
|
|
46
|
+
][]) {
|
|
47
|
+
merged[directive] = [...(host[directive] ?? []), ...sources];
|
|
48
|
+
}
|
|
49
|
+
return merged;
|
|
50
|
+
}
|
|
@@ -3,7 +3,7 @@ import { useOxy } from '@oxy.so/services/ui/client';
|
|
|
3
3
|
import type { AliaAvatarState } from '../components/AliaAvatar';
|
|
4
4
|
import { selectAliaActivity, useAliaActivityStore } from '../lib/activity-store';
|
|
5
5
|
import { acquireActivitySocket } from '../lib/activity-socket';
|
|
6
|
-
import { ALIA_API_URL } from '../lib/
|
|
6
|
+
import { ALIA_API_URL } from '../lib/origins';
|
|
7
7
|
|
|
8
8
|
export interface UseAliaActivityOptions {
|
|
9
9
|
/** The API the activity socket connects to. Defaults like `useAliaChat`'s `apiUrl`. */
|
package/src/hooks/useAliaChat.ts
CHANGED
|
@@ -7,7 +7,7 @@ import { agentMessageId, delegatedAgent } from '../lib/delegation';
|
|
|
7
7
|
import { streamAliaChat } from '../lib/chat-transport';
|
|
8
8
|
import type { AliaChatStreamEvent } from '../lib/chat-stream';
|
|
9
9
|
|
|
10
|
-
import { ALIA_API_URL } from '../lib/
|
|
10
|
+
import { ALIA_API_URL } from '../lib/origins';
|
|
11
11
|
import { useAliaActivityStore } from '../lib/activity-store';
|
|
12
12
|
import { turnActivityState } from '../lib/turn-state';
|
|
13
13
|
|
|
@@ -28,8 +28,7 @@ import {
|
|
|
28
28
|
import { DictationRecorderError, useDictationRecorder } from '../lib/dictation-recorder';
|
|
29
29
|
import { defaultSpeechLanguage } from '../lib/speech-language';
|
|
30
30
|
import { VOICE_ERROR_MESSAGES, type VoiceErrorCode } from '../lib/speech-messages';
|
|
31
|
-
|
|
32
|
-
const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
|
|
31
|
+
import { ALIA_API_URL } from '../lib/origins';
|
|
33
32
|
|
|
34
33
|
/** The remaining time is shown from here on. */
|
|
35
34
|
const COUNTDOWN_FROM_MS = 30_000;
|
|
@@ -132,7 +131,7 @@ export function useDictation(options: UseDictationOptions = {}) {
|
|
|
132
131
|
abortRef.current = controller;
|
|
133
132
|
try {
|
|
134
133
|
const result = await requestTranscription({
|
|
135
|
-
apiUrl: optionsRef.current.apiUrl ||
|
|
134
|
+
apiUrl: optionsRef.current.apiUrl || ALIA_API_URL,
|
|
136
135
|
token,
|
|
137
136
|
clip,
|
|
138
137
|
language: optionsRef.current.lang ?? defaultSpeechLanguage(),
|
package/src/hooks/useTTS.ts
CHANGED
|
@@ -6,8 +6,7 @@ import { errorMessage } from '../lib/utils';
|
|
|
6
6
|
import { create } from 'zustand';
|
|
7
7
|
import { createAudioLevelMeter } from '../lib/audio-level';
|
|
8
8
|
import { requestSpeechClip } from '../lib/speech-synthesis';
|
|
9
|
-
|
|
10
|
-
const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
|
|
9
|
+
import { ALIA_API_URL } from '../lib/origins';
|
|
11
10
|
|
|
12
11
|
// ============== OPTIONS ==============
|
|
13
12
|
|
|
@@ -83,7 +82,7 @@ useTTSStore.subscribe((state, previous) => {
|
|
|
83
82
|
// ============== HOOK ==============
|
|
84
83
|
|
|
85
84
|
export function useTTS(options: UseTTSOptions = {}) {
|
|
86
|
-
const apiUrl = options.apiUrl ||
|
|
85
|
+
const apiUrl = options.apiUrl || ALIA_API_URL;
|
|
87
86
|
const voicePref = options.voice ?? 'female';
|
|
88
87
|
const voiceModel = options.model;
|
|
89
88
|
const tonePref = options.tone ?? 'default';
|
|
@@ -24,8 +24,7 @@ import type { VoiceAudioEngine } from '../lib/voice-audio-engine-types';
|
|
|
24
24
|
import { VoiceCall, type VoiceCallSnapshot, type VoiceSocketFactory } from '../lib/voice-call';
|
|
25
25
|
import { VoiceLevelChannel, type VoiceLevelSource } from '../lib/voice-levels';
|
|
26
26
|
import type { ProductVoice, VoiceCloseReason } from '../lib/voice-protocol';
|
|
27
|
-
|
|
28
|
-
const API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
|
|
27
|
+
import { ALIA_API_URL } from '../lib/origins';
|
|
29
28
|
|
|
30
29
|
export interface UseVoiceRoomOptions {
|
|
31
30
|
apiUrl?: string;
|
|
@@ -120,7 +119,7 @@ export function useVoiceRoom(options: UseVoiceRoomOptions = {}) {
|
|
|
120
119
|
if (live !== undefined && live !== 'ended' && live !== 'error') return;
|
|
121
120
|
const current = optionsRef.current;
|
|
122
121
|
const call = new VoiceCall({
|
|
123
|
-
apiUrl: current.apiUrl ||
|
|
122
|
+
apiUrl: current.apiUrl || ALIA_API_URL,
|
|
124
123
|
getToken,
|
|
125
124
|
conversationId: current.conversationId ?? newConversationId(),
|
|
126
125
|
...(current.agentId === undefined ? {} : { agentId: current.agentId }),
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
+
import { ALIA_WEB_ORIGIN } from './origins';
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* Where the 3D character runtime (Alia's avatar) is served from when the host does not say.
|
|
3
5
|
*
|
|
4
6
|
* The runtime is ~214 MB of engine and data, so the chat loads it from Alia's own web origin
|
|
5
7
|
* instead of asking every app to host a copy. That origin serves it with CORS open
|
|
6
|
-
* (`packages/app/worker/index.js`), which a cross-origin module import needs;
|
|
7
|
-
*
|
|
8
|
-
*
|
|
8
|
+
* (`packages/app/worker/index.js`), which a cross-origin module import needs; a host with a
|
|
9
|
+
* CSP allows it with `@alia.onl/sdk/csp`. A host that serves its own copy passes
|
|
10
|
+
* `characterRuntimeUrl`.
|
|
9
11
|
*/
|
|
10
|
-
export const ALIA_CHARACTER_RUNTIME_URL =
|
|
12
|
+
export const ALIA_CHARACTER_RUNTIME_URL = `${ALIA_WEB_ORIGIN}/bloom-character/runtime.mjs`;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two origins the SDK talks to, defined once. Every URL it builds starts from one of them,
|
|
3
|
+
* and `@alia.onl/sdk/csp` (`src/csp.ts`) derives the host's CSP from these same values, so an
|
|
4
|
+
* origin the SDK uses is an origin it declares.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/** Alia's API: `EXPO_PUBLIC_ALIA_API_URL`, then production. Chat, dictation, speech, voice. */
|
|
8
|
+
export const ALIA_API_URL = process.env.EXPO_PUBLIC_ALIA_API_URL ?? 'https://api.alia.onl';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Alia's web app, which serves the files the SDK shares with every host: Bloom's character
|
|
12
|
+
* runtime (`/bloom-character/`) and the voice call's worklets (`/sdk/voice-worklets/`), both
|
|
13
|
+
* with CORS open (`packages/app/worker/index.js`).
|
|
14
|
+
*/
|
|
15
|
+
export const ALIA_WEB_ORIGIN = 'https://alia.onl';
|
|
@@ -11,12 +11,14 @@
|
|
|
11
11
|
* playback through a local `RTCPeerConnection` loopback) is not built until the
|
|
12
12
|
* measurement says it is needed.
|
|
13
13
|
*
|
|
14
|
-
* The worklets are
|
|
15
|
-
*
|
|
14
|
+
* The worklets are one module served by Alia's web app (`./voice-worklets.ts`),
|
|
15
|
+
* never a `blob:` URL a host's CSP would have to allow as script. Their numbers
|
|
16
|
+
* (stream rate, frame size, prebuffer) are passed as `processorOptions`.
|
|
16
17
|
*/
|
|
17
18
|
|
|
18
19
|
import { levelOfPcm16, pcm16ToFloat } from './pcm';
|
|
19
20
|
import { VOICE_FRAME_SAMPLES, VOICE_SAMPLE_RATE } from './voice-protocol';
|
|
21
|
+
import { VOICE_WORKLETS_URL } from './voice-worklets';
|
|
20
22
|
import {
|
|
21
23
|
VoiceAudioEngineError,
|
|
22
24
|
type VoiceAudioEngine,
|
|
@@ -30,155 +32,6 @@ const PREBUFFER_MS = 80;
|
|
|
30
32
|
/** A burst shorter than the prebuffer still plays once nothing more arrives for this long. */
|
|
31
33
|
const PREBUFFER_IDLE_MS = 100;
|
|
32
34
|
|
|
33
|
-
const CAPTURE_PROCESSOR = `
|
|
34
|
-
class AliaVoiceCapture extends AudioWorkletProcessor {
|
|
35
|
-
constructor() {
|
|
36
|
-
super();
|
|
37
|
-
this.ratio = sampleRate / ${VOICE_SAMPLE_RATE};
|
|
38
|
-
this.t = 0;
|
|
39
|
-
this.prev = 0;
|
|
40
|
-
this.frame = new Int16Array(${VOICE_FRAME_SAMPLES});
|
|
41
|
-
this.filled = 0;
|
|
42
|
-
}
|
|
43
|
-
process(inputs) {
|
|
44
|
-
const input = inputs[0] && inputs[0][0];
|
|
45
|
-
if (!input || input.length === 0) return true;
|
|
46
|
-
const n = input.length;
|
|
47
|
-
while (this.t < n - 1) {
|
|
48
|
-
const i = Math.floor(this.t);
|
|
49
|
-
const f = this.t - i;
|
|
50
|
-
const a = i < 0 ? this.prev : input[i];
|
|
51
|
-
const b = input[i + 1];
|
|
52
|
-
let s = a + (b - a) * f;
|
|
53
|
-
s = s < -1 ? -1 : s > 1 ? 1 : s;
|
|
54
|
-
this.frame[this.filled++] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
|
55
|
-
if (this.filled === this.frame.length) {
|
|
56
|
-
const out = this.frame;
|
|
57
|
-
this.port.postMessage(out.buffer, [out.buffer]);
|
|
58
|
-
this.frame = new Int16Array(${VOICE_FRAME_SAMPLES});
|
|
59
|
-
this.filled = 0;
|
|
60
|
-
}
|
|
61
|
-
this.t += this.ratio;
|
|
62
|
-
}
|
|
63
|
-
this.t -= n;
|
|
64
|
-
this.prev = input[n - 1];
|
|
65
|
-
return true;
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
registerProcessor('alia-voice-capture', AliaVoiceCapture);
|
|
69
|
-
`;
|
|
70
|
-
|
|
71
|
-
const PLAYBACK_PROCESSOR = `
|
|
72
|
-
class AliaVoicePlayback extends AudioWorkletProcessor {
|
|
73
|
-
constructor() {
|
|
74
|
-
super();
|
|
75
|
-
this.step = ${VOICE_SAMPLE_RATE} / sampleRate;
|
|
76
|
-
this.queue = [];
|
|
77
|
-
this.queued = 0;
|
|
78
|
-
this.pos = 0;
|
|
79
|
-
this.buffering = true;
|
|
80
|
-
this.lastChunkAt = 0;
|
|
81
|
-
this.item = null;
|
|
82
|
-
this.played = 0;
|
|
83
|
-
this.sinceReport = 0;
|
|
84
|
-
this.sumSquares = 0;
|
|
85
|
-
this.count = 0;
|
|
86
|
-
this.port.onmessage = (event) => {
|
|
87
|
-
const message = event.data;
|
|
88
|
-
if (message.type === 'chunk') {
|
|
89
|
-
this.queue.push({ itemId: message.itemId, samples: message.samples });
|
|
90
|
-
this.queued += message.samples.length;
|
|
91
|
-
this.lastChunkAt = currentTime;
|
|
92
|
-
} else if (message.type === 'flush') {
|
|
93
|
-
const playedMs = this.item === null ? null : (this.played / ${VOICE_SAMPLE_RATE}) * 1000;
|
|
94
|
-
this.port.postMessage({ type: 'flushed', itemId: this.item, playedMs });
|
|
95
|
-
this.queue = [];
|
|
96
|
-
this.queued = 0;
|
|
97
|
-
this.pos = 0;
|
|
98
|
-
this.buffering = true;
|
|
99
|
-
this.item = null;
|
|
100
|
-
this.played = 0;
|
|
101
|
-
}
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
report(force) {
|
|
105
|
-
if (!force && this.sinceReport < sampleRate / 20) return;
|
|
106
|
-
const rms = this.count === 0 ? 0 : Math.sqrt(this.sumSquares / this.count);
|
|
107
|
-
this.port.postMessage({
|
|
108
|
-
type: 'progress',
|
|
109
|
-
itemId: this.item,
|
|
110
|
-
playedMs: (this.played / ${VOICE_SAMPLE_RATE}) * 1000,
|
|
111
|
-
rms,
|
|
112
|
-
});
|
|
113
|
-
this.sinceReport = 0;
|
|
114
|
-
this.sumSquares = 0;
|
|
115
|
-
this.count = 0;
|
|
116
|
-
}
|
|
117
|
-
process(_inputs, outputs) {
|
|
118
|
-
const out = outputs[0][0];
|
|
119
|
-
if (this.buffering) {
|
|
120
|
-
const ready =
|
|
121
|
-
this.queued >= ${(VOICE_SAMPLE_RATE * PREBUFFER_MS) / 1000} ||
|
|
122
|
-
(this.queued > 0 && currentTime - this.lastChunkAt > ${PREBUFFER_IDLE_MS / 1000});
|
|
123
|
-
if (!ready) {
|
|
124
|
-
out.fill(0);
|
|
125
|
-
for (let c = 1; c < outputs[0].length; c++) outputs[0][c].fill(0);
|
|
126
|
-
return true;
|
|
127
|
-
}
|
|
128
|
-
this.buffering = false;
|
|
129
|
-
}
|
|
130
|
-
for (let k = 0; k < out.length; k++) {
|
|
131
|
-
const head = this.queue[0];
|
|
132
|
-
if (!head) {
|
|
133
|
-
out[k] = 0;
|
|
134
|
-
if (!this.buffering) {
|
|
135
|
-
this.buffering = true;
|
|
136
|
-
this.report(true);
|
|
137
|
-
this.port.postMessage({ type: 'idle' });
|
|
138
|
-
}
|
|
139
|
-
continue;
|
|
140
|
-
}
|
|
141
|
-
if (head.itemId !== this.item) {
|
|
142
|
-
this.item = head.itemId;
|
|
143
|
-
this.played = 0;
|
|
144
|
-
this.port.postMessage({ type: 'started', itemId: head.itemId });
|
|
145
|
-
}
|
|
146
|
-
const i = Math.floor(this.pos);
|
|
147
|
-
const f = this.pos - i;
|
|
148
|
-
const a = head.samples[i];
|
|
149
|
-
const b = i + 1 < head.samples.length ? head.samples[i + 1] : a;
|
|
150
|
-
const s = a + (b - a) * f;
|
|
151
|
-
out[k] = s;
|
|
152
|
-
this.sumSquares += s * s;
|
|
153
|
-
this.count++;
|
|
154
|
-
this.pos += this.step;
|
|
155
|
-
const consumed = Math.floor(this.pos);
|
|
156
|
-
if (consumed > 0) {
|
|
157
|
-
const taken = Math.min(consumed, head.samples.length);
|
|
158
|
-
this.played += taken;
|
|
159
|
-
this.queued -= taken;
|
|
160
|
-
if (consumed >= head.samples.length) {
|
|
161
|
-
this.queue.shift();
|
|
162
|
-
this.pos = 0;
|
|
163
|
-
} else {
|
|
164
|
-
head.samples = head.samples.subarray(consumed);
|
|
165
|
-
this.pos -= consumed;
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
for (let c = 1; c < outputs[0].length; c++) outputs[0][c].set(out);
|
|
170
|
-
this.sinceReport += out.length;
|
|
171
|
-
this.report(false);
|
|
172
|
-
return true;
|
|
173
|
-
}
|
|
174
|
-
}
|
|
175
|
-
registerProcessor('alia-voice-playback', AliaVoicePlayback);
|
|
176
|
-
`;
|
|
177
|
-
|
|
178
|
-
function moduleUrl(source: string): string {
|
|
179
|
-
return URL.createObjectURL(new Blob([source], { type: 'application/javascript' }));
|
|
180
|
-
}
|
|
181
|
-
|
|
182
35
|
function deviceError(caught: unknown): VoiceAudioEngineError {
|
|
183
36
|
const name = (caught as { name?: unknown } | null)?.name;
|
|
184
37
|
if (name === 'NotAllowedError' || name === 'SecurityError' || name === 'PermissionDeniedError')
|
|
@@ -229,16 +82,10 @@ export function createVoiceAudioEngine(): VoiceAudioEngine {
|
|
|
229
82
|
}
|
|
230
83
|
|
|
231
84
|
context = new AudioContextCtor({ latencyHint: 'interactive' });
|
|
232
|
-
const captureUrl = moduleUrl(CAPTURE_PROCESSOR);
|
|
233
|
-
const playbackUrl = moduleUrl(PLAYBACK_PROCESSOR);
|
|
234
85
|
try {
|
|
235
|
-
await context.audioWorklet.addModule(
|
|
236
|
-
await context.audioWorklet.addModule(playbackUrl);
|
|
86
|
+
await context.audioWorklet.addModule(VOICE_WORKLETS_URL);
|
|
237
87
|
} catch {
|
|
238
88
|
throw new VoiceAudioEngineError('voice-unavailable');
|
|
239
|
-
} finally {
|
|
240
|
-
URL.revokeObjectURL(captureUrl);
|
|
241
|
-
URL.revokeObjectURL(playbackUrl);
|
|
242
89
|
}
|
|
243
90
|
await context.resume().catch(() => undefined);
|
|
244
91
|
|
|
@@ -247,6 +94,10 @@ export function createVoiceAudioEngine(): VoiceAudioEngine {
|
|
|
247
94
|
numberOfInputs: 1,
|
|
248
95
|
numberOfOutputs: 0,
|
|
249
96
|
channelCount: 1,
|
|
97
|
+
processorOptions: {
|
|
98
|
+
streamSampleRate: VOICE_SAMPLE_RATE,
|
|
99
|
+
frameSamples: VOICE_FRAME_SAMPLES,
|
|
100
|
+
},
|
|
250
101
|
});
|
|
251
102
|
capture.port.onmessage = (event: MessageEvent<ArrayBuffer>) => {
|
|
252
103
|
if (stopped) return;
|
|
@@ -260,6 +111,11 @@ export function createVoiceAudioEngine(): VoiceAudioEngine {
|
|
|
260
111
|
numberOfInputs: 0,
|
|
261
112
|
numberOfOutputs: 1,
|
|
262
113
|
outputChannelCount: [context.destination.channelCount || 2],
|
|
114
|
+
processorOptions: {
|
|
115
|
+
streamSampleRate: VOICE_SAMPLE_RATE,
|
|
116
|
+
prebufferMs: PREBUFFER_MS,
|
|
117
|
+
prebufferIdleMs: PREBUFFER_IDLE_MS,
|
|
118
|
+
},
|
|
263
119
|
});
|
|
264
120
|
playback.port.onmessage = (event: MessageEvent) => {
|
|
265
121
|
const message = event.data as
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { ALIA_WEB_ORIGIN } from './origins';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The voice call's AudioWorklet processors (`worklets/voice-processors.js`), as the web engine
|
|
5
|
+
* loads them.
|
|
6
|
+
*
|
|
7
|
+
* Not a `blob:` URL: a worklet module is a script, so a blob would need `script-src blob:` in
|
|
8
|
+
* every host's CSP, which would let any blob the page makes run as script. Alia's web app serves
|
|
9
|
+
* the file instead, under `script-src https://alia.onl` that the avatar already needs
|
|
10
|
+
* (`@alia.onl/sdk/csp`), with CORS open for the cross-origin `addModule`.
|
|
11
|
+
*
|
|
12
|
+
* The name is the file's content hash, so an SDK embedded in another app (on its own release
|
|
13
|
+
* cycle) always loads the bytes it was written against: `packages/app/public/sdk/voice-worklets/`
|
|
14
|
+
* keeps every revision ever published, and a test fails if this revision is not the hash of the
|
|
15
|
+
* file or is not served.
|
|
16
|
+
*/
|
|
17
|
+
export const VOICE_WORKLETS_REVISION = '543d4a8e42937e99';
|
|
18
|
+
|
|
19
|
+
export const VOICE_WORKLETS_URL = `${ALIA_WEB_ORIGIN}/sdk/voice-worklets/${VOICE_WORKLETS_REVISION}.js`;
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
// The web voice call's two AudioWorklet processors (`src/lib/voice-audio-engine.ts`).
|
|
2
|
+
//
|
|
3
|
+
// Served from Alia's web origin, content-addressed (`/sdk/voice-worklets/<revision>.js`,
|
|
4
|
+
// `src/lib/voice-worklets.ts`), so a host page loads it under `script-src https://alia.onl`
|
|
5
|
+
// instead of a `blob:` URL its CSP would have to allow. Plain JavaScript with no imports: a
|
|
6
|
+
// worklet module runs in the AudioWorkletGlobalScope. Every number it needs arrives in
|
|
7
|
+
// `processorOptions`, so the bytes stay the same while the engine's settings change.
|
|
8
|
+
|
|
9
|
+
class AliaVoiceCapture extends AudioWorkletProcessor {
|
|
10
|
+
constructor(options) {
|
|
11
|
+
super();
|
|
12
|
+
const { streamSampleRate, frameSamples } = options.processorOptions;
|
|
13
|
+
this.ratio = sampleRate / streamSampleRate;
|
|
14
|
+
this.frameSamples = frameSamples;
|
|
15
|
+
this.t = 0;
|
|
16
|
+
this.prev = 0;
|
|
17
|
+
this.frame = new Int16Array(frameSamples);
|
|
18
|
+
this.filled = 0;
|
|
19
|
+
}
|
|
20
|
+
process(inputs) {
|
|
21
|
+
const input = inputs[0]?.[0];
|
|
22
|
+
if (!input || input.length === 0) return true;
|
|
23
|
+
const n = input.length;
|
|
24
|
+
while (this.t < n - 1) {
|
|
25
|
+
const i = Math.floor(this.t);
|
|
26
|
+
const f = this.t - i;
|
|
27
|
+
const a = i < 0 ? this.prev : input[i];
|
|
28
|
+
const b = input[i + 1];
|
|
29
|
+
let s = a + (b - a) * f;
|
|
30
|
+
s = s < -1 ? -1 : s > 1 ? 1 : s;
|
|
31
|
+
this.frame[this.filled++] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
|
32
|
+
if (this.filled === this.frame.length) {
|
|
33
|
+
const out = this.frame;
|
|
34
|
+
this.port.postMessage(out.buffer, [out.buffer]);
|
|
35
|
+
this.frame = new Int16Array(this.frameSamples);
|
|
36
|
+
this.filled = 0;
|
|
37
|
+
}
|
|
38
|
+
this.t += this.ratio;
|
|
39
|
+
}
|
|
40
|
+
this.t -= n;
|
|
41
|
+
this.prev = input[n - 1];
|
|
42
|
+
return true;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
registerProcessor('alia-voice-capture', AliaVoiceCapture);
|
|
46
|
+
|
|
47
|
+
class AliaVoicePlayback extends AudioWorkletProcessor {
|
|
48
|
+
constructor(options) {
|
|
49
|
+
super();
|
|
50
|
+
const { streamSampleRate, prebufferMs, prebufferIdleMs } = options.processorOptions;
|
|
51
|
+
this.streamSampleRate = streamSampleRate;
|
|
52
|
+
this.prebufferSamples = (streamSampleRate * prebufferMs) / 1000;
|
|
53
|
+
this.prebufferIdleSeconds = prebufferIdleMs / 1000;
|
|
54
|
+
this.step = streamSampleRate / sampleRate;
|
|
55
|
+
this.queue = [];
|
|
56
|
+
this.queued = 0;
|
|
57
|
+
this.pos = 0;
|
|
58
|
+
this.buffering = true;
|
|
59
|
+
this.lastChunkAt = 0;
|
|
60
|
+
this.item = null;
|
|
61
|
+
this.played = 0;
|
|
62
|
+
this.sinceReport = 0;
|
|
63
|
+
this.sumSquares = 0;
|
|
64
|
+
this.count = 0;
|
|
65
|
+
this.port.onmessage = (event) => {
|
|
66
|
+
const message = event.data;
|
|
67
|
+
if (message.type === 'chunk') {
|
|
68
|
+
this.queue.push({ itemId: message.itemId, samples: message.samples });
|
|
69
|
+
this.queued += message.samples.length;
|
|
70
|
+
this.lastChunkAt = currentTime;
|
|
71
|
+
} else if (message.type === 'flush') {
|
|
72
|
+
const playedMs = this.item === null ? null : this.playedMs();
|
|
73
|
+
this.port.postMessage({ type: 'flushed', itemId: this.item, playedMs });
|
|
74
|
+
this.queue = [];
|
|
75
|
+
this.queued = 0;
|
|
76
|
+
this.pos = 0;
|
|
77
|
+
this.buffering = true;
|
|
78
|
+
this.item = null;
|
|
79
|
+
this.played = 0;
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
playedMs() {
|
|
84
|
+
return (this.played / this.streamSampleRate) * 1000;
|
|
85
|
+
}
|
|
86
|
+
report(force) {
|
|
87
|
+
if (!force && this.sinceReport < sampleRate / 20) return;
|
|
88
|
+
const rms = this.count === 0 ? 0 : Math.sqrt(this.sumSquares / this.count);
|
|
89
|
+
this.port.postMessage({ type: 'progress', itemId: this.item, playedMs: this.playedMs(), rms });
|
|
90
|
+
this.sinceReport = 0;
|
|
91
|
+
this.sumSquares = 0;
|
|
92
|
+
this.count = 0;
|
|
93
|
+
}
|
|
94
|
+
process(_inputs, outputs) {
|
|
95
|
+
const out = outputs[0][0];
|
|
96
|
+
if (this.buffering) {
|
|
97
|
+
const ready =
|
|
98
|
+
this.queued >= this.prebufferSamples ||
|
|
99
|
+
(this.queued > 0 && currentTime - this.lastChunkAt > this.prebufferIdleSeconds);
|
|
100
|
+
if (!ready) {
|
|
101
|
+
out.fill(0);
|
|
102
|
+
for (let c = 1; c < outputs[0].length; c++) outputs[0][c].fill(0);
|
|
103
|
+
return true;
|
|
104
|
+
}
|
|
105
|
+
this.buffering = false;
|
|
106
|
+
}
|
|
107
|
+
for (let k = 0; k < out.length; k++) {
|
|
108
|
+
const head = this.queue[0];
|
|
109
|
+
if (!head) {
|
|
110
|
+
out[k] = 0;
|
|
111
|
+
if (!this.buffering) {
|
|
112
|
+
this.buffering = true;
|
|
113
|
+
this.report(true);
|
|
114
|
+
this.port.postMessage({ type: 'idle' });
|
|
115
|
+
}
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
if (head.itemId !== this.item) {
|
|
119
|
+
this.item = head.itemId;
|
|
120
|
+
this.played = 0;
|
|
121
|
+
this.port.postMessage({ type: 'started', itemId: head.itemId });
|
|
122
|
+
}
|
|
123
|
+
const i = Math.floor(this.pos);
|
|
124
|
+
const f = this.pos - i;
|
|
125
|
+
const a = head.samples[i];
|
|
126
|
+
const b = i + 1 < head.samples.length ? head.samples[i + 1] : a;
|
|
127
|
+
const s = a + (b - a) * f;
|
|
128
|
+
out[k] = s;
|
|
129
|
+
this.sumSquares += s * s;
|
|
130
|
+
this.count++;
|
|
131
|
+
this.pos += this.step;
|
|
132
|
+
const consumed = Math.floor(this.pos);
|
|
133
|
+
if (consumed > 0) {
|
|
134
|
+
const taken = Math.min(consumed, head.samples.length);
|
|
135
|
+
this.played += taken;
|
|
136
|
+
this.queued -= taken;
|
|
137
|
+
if (consumed >= head.samples.length) {
|
|
138
|
+
this.queue.shift();
|
|
139
|
+
this.pos = 0;
|
|
140
|
+
} else {
|
|
141
|
+
head.samples = head.samples.subarray(consumed);
|
|
142
|
+
this.pos -= consumed;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
for (let c = 1; c < outputs[0].length; c++) outputs[0][c].set(out);
|
|
147
|
+
this.sinceReport += out.length;
|
|
148
|
+
this.report(false);
|
|
149
|
+
return true;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
registerProcessor('alia-voice-playback', AliaVoicePlayback);
|
package/src/lib/api-url.ts
DELETED