@olares/lares-core 0.15.33 → 0.15.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/voice/pcm-player.js +58 -15
- package/voice/pcm-worklet.js +0 -1
- package/voice/playout-clock.js +10 -8
- package/voice/stream-client.js +3 -3
- package/voice/unified-conversation.js +1 -1
package/package.json
CHANGED
package/voice/pcm-player.js
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { PCM_PLAYER_WORKLET } from "./pcm-worklet.js";
|
|
2
2
|
import { pcmSamples } from "./realtime-audio.js";
|
|
3
3
|
|
|
4
|
+
/** Unread audio the speaker may be holding. A longer reply waits here; it is not a failed call. */
|
|
5
|
+
export const PLAYBACK_QUEUE_SECONDS = 4;
|
|
6
|
+
|
|
4
7
|
let primed = null;
|
|
5
8
|
const worklets = new WeakMap();
|
|
6
9
|
let watchingVisibility = false;
|
|
@@ -43,7 +46,10 @@ export function primePlaybackContext(Context = globalThis.AudioContext || global
|
|
|
43
46
|
}
|
|
44
47
|
|
|
45
48
|
export function createPcmStreamPlayer({ onProgress = () => {}, onMetric = () => {}, context: Context = globalThis.AudioContext || globalThis.webkitAudioContext } = {}) {
|
|
46
|
-
let context, node, resolve, reject, played = 0,
|
|
49
|
+
let context, node, resolve, reject, played = 0, accepted = 0, rate = 0, generation = 0, pendingByte = null, paused = false, owned = false;
|
|
50
|
+
let tail = Promise.resolve();
|
|
51
|
+
const waiters = new Set();
|
|
52
|
+
const wake = () => { for (const listen of [...waiters]) listen(); };
|
|
47
53
|
/**
|
|
48
54
|
* The page shares one primed context. iOS interrupts it (a call, the lock
|
|
49
55
|
* screen) and it never resumes on its own, so a reply that starts after that
|
|
@@ -85,11 +91,14 @@ export function createPcmStreamPlayer({ onProgress = () => {}, onMetric = () =>
|
|
|
85
91
|
owned = false;
|
|
86
92
|
if (closing && closing.state !== "closed") void closing.close().catch(() => {});
|
|
87
93
|
context = null; resolve?.(); resolve = reject = null; pendingByte = null;
|
|
94
|
+
wake();
|
|
95
|
+
tail = Promise.resolve();
|
|
88
96
|
};
|
|
89
97
|
return {
|
|
90
98
|
async start({ sampleRate, channels, encoding }) {
|
|
91
|
-
stop();
|
|
99
|
+
stop(); played = 0; accepted = 0;
|
|
92
100
|
if (!Context || channels !== 1 || encoding !== "pcm_s16le" || !Number.isInteger(sampleRate) || sampleRate < 8000 || sampleRate > 192000) throw new Error("voice_audio_format");
|
|
101
|
+
rate = sampleRate;
|
|
93
102
|
const epoch = generation;
|
|
94
103
|
const shared = primed && primed.state !== "closed" ? primed : null;
|
|
95
104
|
let opened;
|
|
@@ -109,27 +118,61 @@ export function createPcmStreamPlayer({ onProgress = () => {}, onMetric = () =>
|
|
|
109
118
|
});
|
|
110
119
|
node.port.onmessage = ({ data }) => {
|
|
111
120
|
if (epoch !== generation) return;
|
|
112
|
-
if (data.type === "progress") { played = data.frames; onProgress(played); }
|
|
121
|
+
if (data.type === "progress") { played = data.frames; onProgress(played); wake(); }
|
|
113
122
|
if (data.type === "started" || data.type === "underrun") onMetric({ stage: `playback_${data.type}` });
|
|
114
123
|
if (data.type === "done") { resolve?.(); resolve = reject = null; }
|
|
115
|
-
if (data.type === "error") { failed = new Error("voice_audio_backpressure"); reject?.(failed); }
|
|
116
124
|
};
|
|
117
125
|
node.connect(opened.destination);
|
|
118
126
|
},
|
|
119
127
|
write(bytes) {
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
128
|
+
const epoch = generation;
|
|
129
|
+
const run = () => {
|
|
130
|
+
if (epoch !== generation || !node) throw new Error("voice_cancelled");
|
|
131
|
+
let chunk = bytes;
|
|
132
|
+
if (pendingByte !== null) { chunk = new Uint8Array(bytes.length + 1); chunk[0] = pendingByte; chunk.set(bytes, 1); pendingByte = null; }
|
|
133
|
+
if (chunk.length % 2) { pendingByte = chunk[chunk.length - 1]; chunk = chunk.subarray(0, -1); }
|
|
134
|
+
if (!chunk.length) return;
|
|
135
|
+
const source = pcmSamples(chunk);
|
|
136
|
+
const budget = rate * PLAYBACK_QUEUE_SECONDS;
|
|
137
|
+
let offset = 0;
|
|
138
|
+
const post = (from, count) => {
|
|
139
|
+
const samples = count === source.length && from === 0 ? source : source.slice(from, from + count);
|
|
140
|
+
accepted += samples.length;
|
|
141
|
+
node.port.postMessage({ type: "audio", samples }, [samples.buffer]);
|
|
142
|
+
};
|
|
143
|
+
const pump = () => {
|
|
144
|
+
while (offset < source.length) {
|
|
145
|
+
const room = budget - (accepted - played);
|
|
146
|
+
if (room <= 0) return false;
|
|
147
|
+
const count = Math.min(source.length - offset, room);
|
|
148
|
+
post(offset, count);
|
|
149
|
+
offset += count;
|
|
150
|
+
}
|
|
151
|
+
return true;
|
|
152
|
+
};
|
|
153
|
+
if (pump()) return;
|
|
154
|
+
// The rest of a long reply plays as the speaker catches up.
|
|
155
|
+
return new Promise((done, fail) => {
|
|
156
|
+
const listen = () => {
|
|
157
|
+
if (epoch !== generation || !node) { waiters.delete(listen); fail(new Error("voice_cancelled")); return; }
|
|
158
|
+
if (!pump()) return;
|
|
159
|
+
waiters.delete(listen);
|
|
160
|
+
done();
|
|
161
|
+
};
|
|
162
|
+
waiters.add(listen);
|
|
163
|
+
});
|
|
164
|
+
};
|
|
165
|
+
const pending = tail.then(run, run);
|
|
166
|
+
tail = pending.catch(() => {});
|
|
167
|
+
return pending;
|
|
127
168
|
},
|
|
128
169
|
finish() {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
170
|
+
const epoch = generation;
|
|
171
|
+
return tail.catch(() => {}).then(() => {
|
|
172
|
+
if (epoch !== generation || !node) return;
|
|
173
|
+
if (pendingByte !== null) throw new Error("voice_audio_format");
|
|
174
|
+
return new Promise((done, fail) => { resolve = done; reject = fail; node.port.postMessage({ type: "finish" }); });
|
|
175
|
+
});
|
|
133
176
|
},
|
|
134
177
|
canPause: true,
|
|
135
178
|
pause() { paused = true; node?.port.postMessage({ type: "pause" }); },
|
package/voice/pcm-worklet.js
CHANGED
|
@@ -10,7 +10,6 @@ class LaresPcmPlayer extends AudioWorkletProcessor {
|
|
|
10
10
|
this.paused = false; this.queued = 0; this.finished = false; this.started = false; this.starved = false;
|
|
11
11
|
this.port.onmessage = ({ data }) => {
|
|
12
12
|
if (data.type === 'audio') {
|
|
13
|
-
if (this.queued + data.samples.length > this.rate * 4) { this.port.postMessage({ type: 'error' }); return; }
|
|
14
13
|
this.queue.push(data.samples); this.queued += data.samples.length;
|
|
15
14
|
} else if (data.type === 'finish') this.finished = true;
|
|
16
15
|
else if (data.type === 'pause') this.paused = true;
|
package/voice/playout-clock.js
CHANGED
|
@@ -3,8 +3,10 @@
|
|
|
3
3
|
* means the client has stopped reading. Both gates share this so a dropped
|
|
4
4
|
* progress tick cannot freeze one path while the other keeps going.
|
|
5
5
|
*
|
|
6
|
-
* The
|
|
7
|
-
*
|
|
6
|
+
* The player holds about four seconds. While reports are still arriving, the
|
|
7
|
+
* wall clock may lead the speaker by one send-ahead and no further: a long
|
|
8
|
+
* reply is not already played just because time passed. A report that stops
|
|
9
|
+
* falls back to the wall clock, so one missed tick does not end the reply.
|
|
8
10
|
*/
|
|
9
11
|
export const PLAYOUT_LEAD_MS = 2_000;
|
|
10
12
|
export const SOCKET_FUSE_BYTES = 1024 * 1024;
|
|
@@ -12,12 +14,10 @@ export const SOCKET_FUSE_BYTES = 1024 * 1024;
|
|
|
12
14
|
/**
|
|
13
15
|
* Credit for one reply. The playhead moves at 1x and stops when it catches
|
|
14
16
|
* the audio already handed to the client. A gap while the next sentence is
|
|
15
|
-
* synthesized does not pre-pay a burst
|
|
16
|
-
* hold the rest: time keeps the playhead moving up to what was sent. Pause
|
|
17
|
-
* freezes it.
|
|
17
|
+
* synthesized does not pre-pay a burst. Pause freezes the clock.
|
|
18
18
|
*/
|
|
19
19
|
export function createPlayoutClock({ leadMs = PLAYOUT_LEAD_MS, now = Date.now } = {}) {
|
|
20
|
-
let heard = 0, heldAt = 0, playhead = 0, markedAt = 0, sent = 0;
|
|
20
|
+
let heard = 0, heardAt = 0, heldAt = 0, playhead = 0, markedAt = 0, sent = 0;
|
|
21
21
|
const stamp = () => heldAt || now();
|
|
22
22
|
const advance = (sentMs) => {
|
|
23
23
|
if (Number.isFinite(sentMs) && sentMs > sent) sent = sentMs;
|
|
@@ -27,7 +27,7 @@ export function createPlayoutClock({ leadMs = PLAYOUT_LEAD_MS, now = Date.now }
|
|
|
27
27
|
markedAt = t;
|
|
28
28
|
};
|
|
29
29
|
return {
|
|
30
|
-
heard(ms) { if (Number.isFinite(ms) && ms > heard) heard = ms; },
|
|
30
|
+
heard(ms) { if (Number.isFinite(ms) && ms > heard) { heard = ms; heardAt = stamp(); } },
|
|
31
31
|
pause() {
|
|
32
32
|
if (heldAt) return;
|
|
33
33
|
advance(sent);
|
|
@@ -42,7 +42,9 @@ export function createPlayoutClock({ leadMs = PLAYOUT_LEAD_MS, now = Date.now }
|
|
|
42
42
|
start() { if (!markedAt) markedAt = now(); },
|
|
43
43
|
credit(sentMs) {
|
|
44
44
|
advance(sentMs);
|
|
45
|
-
|
|
45
|
+
const wall = Math.max(heard, playhead);
|
|
46
|
+
if (heardAt && stamp() - heardAt <= leadMs) return Math.min(wall, heard + leadMs);
|
|
47
|
+
return wall;
|
|
46
48
|
},
|
|
47
49
|
blocked(sentMs) { return sentMs - this.credit(sentMs) > leadMs; },
|
|
48
50
|
wait(sentMs) {
|
package/voice/stream-client.js
CHANGED
|
@@ -126,7 +126,7 @@ export async function playAudioStream(options) {
|
|
|
126
126
|
}
|
|
127
127
|
if (!progressNoted) { progressNoted = true; streamMetric(options, { stage: "playback_progress_sent", frames }); }
|
|
128
128
|
};
|
|
129
|
-
const writeAudio = (bytes) => {
|
|
129
|
+
const writeAudio = async (bytes) => {
|
|
130
130
|
let chunk = bytes;
|
|
131
131
|
if (skipBytes > 0) {
|
|
132
132
|
const drop = Math.min(skipBytes, chunk.length);
|
|
@@ -136,7 +136,7 @@ export async function playAudioStream(options) {
|
|
|
136
136
|
chunk = chunk.subarray(drop);
|
|
137
137
|
if (!chunk.length) return;
|
|
138
138
|
}
|
|
139
|
-
player.write(chunk);
|
|
139
|
+
await player.write(chunk);
|
|
140
140
|
};
|
|
141
141
|
const gapWatch = setInterval(() => {
|
|
142
142
|
if (!ready || paused || complete || gapReported || Date.now() - lastProgressAt < 4000) return;
|
|
@@ -168,7 +168,7 @@ export async function playAudioStream(options) {
|
|
|
168
168
|
}
|
|
169
169
|
chain = chain.then(async () => {
|
|
170
170
|
if (options.signal?.aborted) throw new Error("voice_cancelled");
|
|
171
|
-
if (typeof data !== "string") { writeAudio(new Uint8Array(data)); return; }
|
|
171
|
+
if (typeof data !== "string") { await writeAudio(new Uint8Array(data)); return; }
|
|
172
172
|
if (event.type === "segment") {
|
|
173
173
|
if (event.index !== segmentIndex + 1) throw new Error("voice_reply_sequence");
|
|
174
174
|
segmentIndex = event.index;
|
|
@@ -64,7 +64,7 @@ export function createUnifiedConversation({ request, socket: openSocket = browse
|
|
|
64
64
|
return;
|
|
65
65
|
}
|
|
66
66
|
if (!current.audio) { current.audio = true; current.chain = current.chain.then(() => player.start({ sampleRate: RATE, channels: 1, encoding: "pcm_s16le" })); controller.observePlayback(true); }
|
|
67
|
-
current.chain = current.chain.then(() => { if (reply === current) player.write(decodeBase64(delta)); });
|
|
67
|
+
current.chain = current.chain.then(() => { if (reply === current) return player.write(decodeBase64(delta)); });
|
|
68
68
|
current.chain.catch(() => {});
|
|
69
69
|
},
|
|
70
70
|
async done() {
|