@m4trix/core 0.8.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +325 -1685
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +1 -3
- package/dist/index.js +325 -1679
- package/dist/index.js.map +1 -1
- package/dist/matrix/index.cjs +325 -58
- package/dist/matrix/index.cjs.map +1 -1
- package/dist/matrix/index.d.ts +121 -30
- package/dist/matrix/index.js +325 -60
- package/dist/matrix/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1,26 +1,9 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
var react = require('react');
|
|
4
|
-
var socket_ioClient = require('socket.io-client');
|
|
5
|
-
var lit = require('lit');
|
|
6
|
-
var decorators_js = require('lit/decorators.js');
|
|
7
|
-
var ref_js = require('lit/directives/ref.js');
|
|
8
|
-
var animejs = require('animejs');
|
|
9
3
|
var messages = require('@langchain/core/messages');
|
|
10
4
|
var effect = require('effect');
|
|
11
5
|
var crypto$1 = require('crypto');
|
|
12
6
|
|
|
13
|
-
var __defProp = Object.defineProperty;
|
|
14
|
-
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
15
|
-
var __decorateClass = (decorators, target, key, kind) => {
|
|
16
|
-
var result = kind > 1 ? void 0 : kind ? __getOwnPropDesc(target, key) : target;
|
|
17
|
-
for (var i = decorators.length - 1, decorator; i >= 0; i--)
|
|
18
|
-
if (decorator = decorators[i])
|
|
19
|
-
result = (kind ? decorator(target, key, result) : decorator(result)) || result;
|
|
20
|
-
if (kind && result)
|
|
21
|
-
__defProp(target, key, result);
|
|
22
|
-
return result;
|
|
23
|
-
};
|
|
24
7
|
var __accessCheck = (obj, member, msg) => {
|
|
25
8
|
if (!member.has(obj))
|
|
26
9
|
throw TypeError("Cannot " + msg);
|
|
@@ -40,1608 +23,6 @@ var __privateSet = (obj, member, value, setter) => {
|
|
|
40
23
|
return value;
|
|
41
24
|
};
|
|
42
25
|
|
|
43
|
-
// src/utility/Logger.ts
|
|
44
|
-
var _Logger = class _Logger {
|
|
45
|
-
constructor(namespace = "") {
|
|
46
|
-
this.namespace = namespace;
|
|
47
|
-
}
|
|
48
|
-
static enableGlobalLogging() {
|
|
49
|
-
_Logger.globalEnabled = true;
|
|
50
|
-
}
|
|
51
|
-
static disableGlobalLogging() {
|
|
52
|
-
_Logger.globalEnabled = false;
|
|
53
|
-
}
|
|
54
|
-
formatPrefix() {
|
|
55
|
-
return this.namespace ? `[${this.namespace}]` : "";
|
|
56
|
-
}
|
|
57
|
-
logIfEnabled(level, ...args) {
|
|
58
|
-
if (!_Logger.globalEnabled)
|
|
59
|
-
return;
|
|
60
|
-
const prefix = this.formatPrefix();
|
|
61
|
-
if (prefix) {
|
|
62
|
-
console[level](prefix, ...args);
|
|
63
|
-
} else {
|
|
64
|
-
console[level](...args);
|
|
65
|
-
}
|
|
66
|
-
}
|
|
67
|
-
log(...args) {
|
|
68
|
-
this.logIfEnabled("log", ...args);
|
|
69
|
-
}
|
|
70
|
-
debug(...args) {
|
|
71
|
-
this.logIfEnabled("debug", ...args);
|
|
72
|
-
}
|
|
73
|
-
info(...args) {
|
|
74
|
-
this.logIfEnabled("info", ...args);
|
|
75
|
-
}
|
|
76
|
-
warn(...args) {
|
|
77
|
-
this.logIfEnabled("warn", ...args);
|
|
78
|
-
}
|
|
79
|
-
error(...args) {
|
|
80
|
-
this.logIfEnabled("error", ...args);
|
|
81
|
-
}
|
|
82
|
-
};
|
|
83
|
-
_Logger.globalEnabled = false;
|
|
84
|
-
var Logger = _Logger;
|
|
85
|
-
|
|
86
|
-
// src/react/adapter/VoiceEndpointAdapter.ts
|
|
87
|
-
var VoiceEndpointAdapter = class {
|
|
88
|
-
constructor(config) {
|
|
89
|
-
this.logger = new Logger("SuTr > EndpointAdapter");
|
|
90
|
-
this.config = config;
|
|
91
|
-
}
|
|
92
|
-
};
|
|
93
|
-
var BaseVoiceEndpointAdapter = class extends VoiceEndpointAdapter {
|
|
94
|
-
constructor(config) {
|
|
95
|
-
super(config);
|
|
96
|
-
}
|
|
97
|
-
/**
|
|
98
|
-
* Send a voice file to the API endpoint and return a Pump stream of audio chunks
|
|
99
|
-
*/
|
|
100
|
-
async sendVoiceFile({
|
|
101
|
-
blob,
|
|
102
|
-
metadata
|
|
103
|
-
}) {
|
|
104
|
-
const formData = new FormData();
|
|
105
|
-
formData.append("audio", blob);
|
|
106
|
-
if (metadata) {
|
|
107
|
-
formData.append("metadata", JSON.stringify(metadata));
|
|
108
|
-
}
|
|
109
|
-
this.logger.debug("Sending voice file to", this.config.endpoint, formData);
|
|
110
|
-
const response = await fetch(
|
|
111
|
-
`${this.config.baseUrl || ""}${this.config.endpoint}`,
|
|
112
|
-
{
|
|
113
|
-
method: "POST",
|
|
114
|
-
headers: this.config.headers,
|
|
115
|
-
body: formData
|
|
116
|
-
}
|
|
117
|
-
);
|
|
118
|
-
if (!response.ok) {
|
|
119
|
-
throw new Error(`API error: ${response.status} ${await response.text()}`);
|
|
120
|
-
}
|
|
121
|
-
if (!response.body) {
|
|
122
|
-
throw new Error("No response body");
|
|
123
|
-
}
|
|
124
|
-
return response;
|
|
125
|
-
}
|
|
126
|
-
};
|
|
127
|
-
|
|
128
|
-
// src/react/utility/audio/InputAudioController.ts
|
|
129
|
-
var InputAudioController = class {
|
|
130
|
-
constructor() {
|
|
131
|
-
this.logger = new Logger("@m4trix/core > InputAudioController");
|
|
132
|
-
}
|
|
133
|
-
};
|
|
134
|
-
|
|
135
|
-
// src/react/utility/audio/WebAudioInputAudioController.ts
|
|
136
|
-
var DEFAULT_SLICING_INTERVAL = 3e3;
|
|
137
|
-
var WebAudioInputAudioController = class extends InputAudioController {
|
|
138
|
-
constructor(audioConfig = {}) {
|
|
139
|
-
super();
|
|
140
|
-
this.audioConfig = audioConfig;
|
|
141
|
-
// ─── Recording state ─────────────────────────────────────────────────────
|
|
142
|
-
this.audioContextState = {
|
|
143
|
-
context: null,
|
|
144
|
-
source: null,
|
|
145
|
-
analyser: null
|
|
146
|
-
};
|
|
147
|
-
this.mediaRecorder = null;
|
|
148
|
-
this.recordedChunks = [];
|
|
149
|
-
this.recordingStream = null;
|
|
150
|
-
}
|
|
151
|
-
get audioContext() {
|
|
152
|
-
return this.audioContextState.context;
|
|
153
|
-
}
|
|
154
|
-
async createAudioContext() {
|
|
155
|
-
const context = new AudioContext({
|
|
156
|
-
sampleRate: this.audioConfig.sampleRate || 16e3,
|
|
157
|
-
latencyHint: "interactive"
|
|
158
|
-
});
|
|
159
|
-
const analyser = context.createAnalyser();
|
|
160
|
-
analyser.fftSize = 2048;
|
|
161
|
-
return { context, source: null, analyser };
|
|
162
|
-
}
|
|
163
|
-
async cleanupAudioContext() {
|
|
164
|
-
this.logger.debug("Cleaning up audio context");
|
|
165
|
-
const { source, context } = this.audioContextState;
|
|
166
|
-
if (source)
|
|
167
|
-
source.disconnect();
|
|
168
|
-
if (context)
|
|
169
|
-
await context.close();
|
|
170
|
-
this.audioContextState = { context: null, source: null, analyser: null };
|
|
171
|
-
}
|
|
172
|
-
async startRecording({
|
|
173
|
-
onRecordedChunk,
|
|
174
|
-
onError
|
|
175
|
-
} = {}) {
|
|
176
|
-
try {
|
|
177
|
-
this.logger.debug("Starting recording");
|
|
178
|
-
this.recordedChunks = [];
|
|
179
|
-
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
|
180
|
-
this.recordingStream = stream;
|
|
181
|
-
if (!this.audioContextState.context) {
|
|
182
|
-
this.audioContextState = await this.createAudioContext();
|
|
183
|
-
}
|
|
184
|
-
this.mediaRecorder = new MediaRecorder(stream, {
|
|
185
|
-
mimeType: "audio/webm;codecs=opus"
|
|
186
|
-
});
|
|
187
|
-
this.mediaRecorder.ondataavailable = (e) => {
|
|
188
|
-
if (e.data.size > 0) {
|
|
189
|
-
this.recordedChunks.push(e.data);
|
|
190
|
-
onRecordedChunk?.(e.data);
|
|
191
|
-
this.logger.debug("Recorded chunk", e.data.size);
|
|
192
|
-
}
|
|
193
|
-
};
|
|
194
|
-
this.mediaRecorder.start(DEFAULT_SLICING_INTERVAL);
|
|
195
|
-
this.logger.debug("MediaRecorder started");
|
|
196
|
-
} catch (err) {
|
|
197
|
-
const error = err instanceof Error ? err : new Error("Failed to start recording");
|
|
198
|
-
this.logger.error(error);
|
|
199
|
-
onError?.(error);
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
async stopRecording({
|
|
203
|
-
onRecordingCompleted
|
|
204
|
-
} = {}) {
|
|
205
|
-
this.logger.debug("Stopping recording");
|
|
206
|
-
if (!this.mediaRecorder || this.mediaRecorder.state === "inactive")
|
|
207
|
-
return;
|
|
208
|
-
await new Promise((resolve) => {
|
|
209
|
-
this.mediaRecorder.onstop = async () => {
|
|
210
|
-
if (this.recordedChunks.length) {
|
|
211
|
-
const blob = new Blob(this.recordedChunks, { type: "audio/webm" });
|
|
212
|
-
onRecordingCompleted?.(blob);
|
|
213
|
-
this.logger.debug("Recording completed", blob.size);
|
|
214
|
-
}
|
|
215
|
-
this.recordingStream?.getTracks().forEach((t) => t.stop());
|
|
216
|
-
this.recordingStream = null;
|
|
217
|
-
await this.cleanupAudioContext();
|
|
218
|
-
resolve();
|
|
219
|
-
};
|
|
220
|
-
this.mediaRecorder.stop();
|
|
221
|
-
});
|
|
222
|
-
}
|
|
223
|
-
/**
|
|
224
|
-
* Cleans up all audio recording resources.
|
|
225
|
-
*/
|
|
226
|
-
cleanup() {
|
|
227
|
-
this.cleanupAudioContext();
|
|
228
|
-
if (this.mediaRecorder && this.mediaRecorder.state !== "inactive") {
|
|
229
|
-
this.mediaRecorder.stop();
|
|
230
|
-
}
|
|
231
|
-
if (this.recordingStream) {
|
|
232
|
-
this.recordingStream.getTracks().forEach((t) => t.stop());
|
|
233
|
-
this.recordingStream = null;
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
};
|
|
237
|
-
|
|
238
|
-
// src/react/utility/audio/OutputAudioController.ts
|
|
239
|
-
var OutputAudioController = class {
|
|
240
|
-
constructor(loggerName) {
|
|
241
|
-
this.logger = new Logger(loggerName);
|
|
242
|
-
}
|
|
243
|
-
};
|
|
244
|
-
|
|
245
|
-
// src/react/utility/audio/AudioElementOutputAudioController.ts
|
|
246
|
-
var AudioElementOutputAudioController = class extends OutputAudioController {
|
|
247
|
-
constructor() {
|
|
248
|
-
super("@m4trix/core > WebApiOutputAudioController");
|
|
249
|
-
// ─── Playback state ──────────────────────────────────────────────────────
|
|
250
|
-
this.currentHtmlAudio = null;
|
|
251
|
-
this.currentAudioUrl = null;
|
|
252
|
-
}
|
|
253
|
-
// ─── One-shot playback ────────────────────────────────────────────────────
|
|
254
|
-
/**
|
|
255
|
-
* Play either a Blob or a URL string.
|
|
256
|
-
* Uses <audio> under the hood for maximum browser compatibility.
|
|
257
|
-
*/
|
|
258
|
-
async playAudio({
|
|
259
|
-
source,
|
|
260
|
-
onComplete
|
|
261
|
-
}) {
|
|
262
|
-
if (this.currentHtmlAudio) {
|
|
263
|
-
this.currentHtmlAudio.pause();
|
|
264
|
-
this.currentHtmlAudio.src = "";
|
|
265
|
-
if (this.currentAudioUrl && source instanceof Blob) {
|
|
266
|
-
URL.revokeObjectURL(this.currentAudioUrl);
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
const audio = new Audio();
|
|
270
|
-
this.currentHtmlAudio = audio;
|
|
271
|
-
let url;
|
|
272
|
-
if (source instanceof Blob) {
|
|
273
|
-
url = URL.createObjectURL(source);
|
|
274
|
-
this.currentAudioUrl = url;
|
|
275
|
-
audio.onended = () => {
|
|
276
|
-
URL.revokeObjectURL(url);
|
|
277
|
-
onComplete?.();
|
|
278
|
-
};
|
|
279
|
-
} else {
|
|
280
|
-
url = source;
|
|
281
|
-
}
|
|
282
|
-
audio.src = url;
|
|
283
|
-
try {
|
|
284
|
-
await audio.play();
|
|
285
|
-
} catch (err) {
|
|
286
|
-
this.logger.error("Playback failed, user gesture may be required", err);
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
// ─── Streaming playback ──────────────────────────────────────────────────
|
|
290
|
-
/**
|
|
291
|
-
* Stream audio from a Response via MediaSource Extensions.
|
|
292
|
-
* @param params.response The fetch Response whose body is an audio stream
|
|
293
|
-
* @param params.mimeCodec MIME type+codec string, e.g. 'audio/mpeg'
|
|
294
|
-
* @param params.onComplete Optional callback once the stream ends
|
|
295
|
-
*/
|
|
296
|
-
async playAudioStream({
|
|
297
|
-
response,
|
|
298
|
-
mimeCodec = "audio/mpeg",
|
|
299
|
-
onComplete
|
|
300
|
-
}) {
|
|
301
|
-
if (!response.ok || !response.body) {
|
|
302
|
-
throw new Error(`Invalid response (${response.status})`);
|
|
303
|
-
}
|
|
304
|
-
if (typeof MediaSource === "undefined" || !MediaSource.isTypeSupported(mimeCodec)) {
|
|
305
|
-
throw new Error(`Unsupported MIME type or codec: ${mimeCodec}`);
|
|
306
|
-
}
|
|
307
|
-
await this.stopPlayback();
|
|
308
|
-
const mediaSource = new MediaSource();
|
|
309
|
-
const url = URL.createObjectURL(mediaSource);
|
|
310
|
-
this.currentAudioUrl = url;
|
|
311
|
-
const audio = new Audio(url);
|
|
312
|
-
this.currentHtmlAudio = audio;
|
|
313
|
-
audio.autoplay = true;
|
|
314
|
-
audio.onended = () => {
|
|
315
|
-
URL.revokeObjectURL(url);
|
|
316
|
-
this.currentAudioUrl = null;
|
|
317
|
-
onComplete?.();
|
|
318
|
-
};
|
|
319
|
-
mediaSource.addEventListener(
|
|
320
|
-
"sourceopen",
|
|
321
|
-
() => {
|
|
322
|
-
const sourceBuffer = mediaSource.addSourceBuffer(mimeCodec);
|
|
323
|
-
const reader = response.body.getReader();
|
|
324
|
-
const pump = async () => {
|
|
325
|
-
const { done, value } = await reader.read();
|
|
326
|
-
if (done) {
|
|
327
|
-
mediaSource.endOfStream();
|
|
328
|
-
return;
|
|
329
|
-
}
|
|
330
|
-
if (value) {
|
|
331
|
-
sourceBuffer.appendBuffer(value);
|
|
332
|
-
}
|
|
333
|
-
if (sourceBuffer.updating) {
|
|
334
|
-
sourceBuffer.addEventListener("updateend", pump, { once: true });
|
|
335
|
-
} else {
|
|
336
|
-
pump();
|
|
337
|
-
}
|
|
338
|
-
};
|
|
339
|
-
pump();
|
|
340
|
-
},
|
|
341
|
-
{ once: true }
|
|
342
|
-
);
|
|
343
|
-
try {
|
|
344
|
-
await audio.play();
|
|
345
|
-
} catch (err) {
|
|
346
|
-
this.logger.error(
|
|
347
|
-
"Streaming playback failed, user gesture may be required",
|
|
348
|
-
err
|
|
349
|
-
);
|
|
350
|
-
}
|
|
351
|
-
}
|
|
352
|
-
// ─── Chunk-based streaming playback ─────────────────────────────────────
|
|
353
|
-
/**
|
|
354
|
-
* Initialize a streaming audio context for chunk-based playback.
|
|
355
|
-
* This creates the necessary MediaSource and SourceBuffer for subsequent chunk additions.
|
|
356
|
-
* Returns functions to add chunks and end the stream, encapsulated in a closure.
|
|
357
|
-
*
|
|
358
|
-
* @param mimeCodec MIME type+codec string, e.g. 'audio/mpeg'
|
|
359
|
-
* @param onComplete Optional callback once the stream ends
|
|
360
|
-
* @returns Object containing functions to add chunks and end the stream
|
|
361
|
-
*/
|
|
362
|
-
async initializeChunkStream({
|
|
363
|
-
onComplete,
|
|
364
|
-
mimeCodec = "audio/mpeg"
|
|
365
|
-
}) {
|
|
366
|
-
this.logger.debug(`Initializing chunk stream with codec: ${mimeCodec}`);
|
|
367
|
-
if (typeof MediaSource === "undefined") {
|
|
368
|
-
throw new Error("MediaSource API is not supported in this browser");
|
|
369
|
-
}
|
|
370
|
-
if (!MediaSource.isTypeSupported(mimeCodec)) {
|
|
371
|
-
this.logger.warn(
|
|
372
|
-
`Codec ${mimeCodec} not supported, falling back to standard audio/mpeg`
|
|
373
|
-
);
|
|
374
|
-
mimeCodec = "audio/mpeg";
|
|
375
|
-
if (!MediaSource.isTypeSupported(mimeCodec)) {
|
|
376
|
-
throw new Error(
|
|
377
|
-
"Neither the specified codec nor the fallback codec are supported"
|
|
378
|
-
);
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
await this.stopPlayback();
|
|
382
|
-
const mediaSource = new MediaSource();
|
|
383
|
-
let sourceBuffer = null;
|
|
384
|
-
const url = URL.createObjectURL(mediaSource);
|
|
385
|
-
this.currentAudioUrl = url;
|
|
386
|
-
const audio = new Audio(url);
|
|
387
|
-
this.currentHtmlAudio = audio;
|
|
388
|
-
audio.autoplay = false;
|
|
389
|
-
audio.controls = true;
|
|
390
|
-
audio.style.display = "none";
|
|
391
|
-
document.body.appendChild(audio);
|
|
392
|
-
let playbackStarted = false;
|
|
393
|
-
let hasReceivedFirstChunk = false;
|
|
394
|
-
let receivedChunksCount = 0;
|
|
395
|
-
const pendingChunks = [];
|
|
396
|
-
let isProcessingQueue = false;
|
|
397
|
-
this.logger.debug("Waiting for MediaSource to open...");
|
|
398
|
-
await new Promise((resolve, reject) => {
|
|
399
|
-
const timeout = setTimeout(() => {
|
|
400
|
-
reject(new Error("MediaSource failed to open (timeout)"));
|
|
401
|
-
}, 5e3);
|
|
402
|
-
mediaSource.addEventListener(
|
|
403
|
-
"sourceopen",
|
|
404
|
-
() => {
|
|
405
|
-
clearTimeout(timeout);
|
|
406
|
-
this.logger.debug("MediaSource open event received");
|
|
407
|
-
try {
|
|
408
|
-
sourceBuffer = mediaSource.addSourceBuffer(mimeCodec);
|
|
409
|
-
if (mediaSource.duration === Infinity || isNaN(mediaSource.duration)) {
|
|
410
|
-
mediaSource.duration = 1e3;
|
|
411
|
-
}
|
|
412
|
-
this.logger.debug("SourceBuffer created successfully");
|
|
413
|
-
resolve();
|
|
414
|
-
} catch (err) {
|
|
415
|
-
reject(new Error(`Failed to create SourceBuffer: ${err}`));
|
|
416
|
-
}
|
|
417
|
-
},
|
|
418
|
-
{ once: true }
|
|
419
|
-
);
|
|
420
|
-
});
|
|
421
|
-
const logger = this.logger;
|
|
422
|
-
const processQueue = async () => {
|
|
423
|
-
if (!sourceBuffer || pendingChunks.length === 0 || isProcessingQueue) {
|
|
424
|
-
return;
|
|
425
|
-
}
|
|
426
|
-
isProcessingQueue = true;
|
|
427
|
-
try {
|
|
428
|
-
while (pendingChunks.length > 0) {
|
|
429
|
-
if (sourceBuffer.updating) {
|
|
430
|
-
await new Promise((resolve) => {
|
|
431
|
-
sourceBuffer.addEventListener("updateend", () => resolve(), {
|
|
432
|
-
once: true
|
|
433
|
-
});
|
|
434
|
-
});
|
|
435
|
-
}
|
|
436
|
-
const nextChunk = pendingChunks.shift();
|
|
437
|
-
if (!nextChunk)
|
|
438
|
-
continue;
|
|
439
|
-
try {
|
|
440
|
-
sourceBuffer.appendBuffer(nextChunk);
|
|
441
|
-
logger.debug(
|
|
442
|
-
`Processed queued chunk of size ${nextChunk.byteLength}`
|
|
443
|
-
);
|
|
444
|
-
if (!playbackStarted && hasReceivedFirstChunk) {
|
|
445
|
-
await tryStartPlayback();
|
|
446
|
-
}
|
|
447
|
-
await new Promise((resolve) => {
|
|
448
|
-
sourceBuffer.addEventListener("updateend", () => resolve(), {
|
|
449
|
-
once: true
|
|
450
|
-
});
|
|
451
|
-
});
|
|
452
|
-
} catch (err) {
|
|
453
|
-
logger.error("Error appending queued chunk to source buffer", err);
|
|
454
|
-
}
|
|
455
|
-
}
|
|
456
|
-
} finally {
|
|
457
|
-
isProcessingQueue = false;
|
|
458
|
-
}
|
|
459
|
-
};
|
|
460
|
-
const tryStartPlayback = async () => {
|
|
461
|
-
if (playbackStarted)
|
|
462
|
-
return;
|
|
463
|
-
playbackStarted = true;
|
|
464
|
-
logger.debug("Attempting to start audio playback...");
|
|
465
|
-
if (receivedChunksCount < 3 && audio.buffered.length > 0 && audio.buffered.end(0) < 0.5) {
|
|
466
|
-
logger.debug("Not enough data buffered yet, delaying playback");
|
|
467
|
-
return;
|
|
468
|
-
}
|
|
469
|
-
try {
|
|
470
|
-
if (audio.readyState === 0) {
|
|
471
|
-
logger.debug(
|
|
472
|
-
"Audio element not ready yet, waiting for canplay event"
|
|
473
|
-
);
|
|
474
|
-
await new Promise((resolve) => {
|
|
475
|
-
audio.addEventListener("canplay", () => resolve(), { once: true });
|
|
476
|
-
});
|
|
477
|
-
}
|
|
478
|
-
await audio.play();
|
|
479
|
-
logger.debug("Successfully started audio playback");
|
|
480
|
-
} catch (err) {
|
|
481
|
-
logger.error("Failed to start playback", err);
|
|
482
|
-
document.addEventListener(
|
|
483
|
-
"click",
|
|
484
|
-
async () => {
|
|
485
|
-
try {
|
|
486
|
-
await audio.play();
|
|
487
|
-
logger.debug("Started playback after user interaction");
|
|
488
|
-
} catch (innerErr) {
|
|
489
|
-
logger.error(
|
|
490
|
-
"Still failed to play after user interaction",
|
|
491
|
-
innerErr
|
|
492
|
-
);
|
|
493
|
-
}
|
|
494
|
-
},
|
|
495
|
-
{ once: true }
|
|
496
|
-
);
|
|
497
|
-
}
|
|
498
|
-
};
|
|
499
|
-
const addChunkToStream = async (chunk) => {
|
|
500
|
-
if (!sourceBuffer) {
|
|
501
|
-
throw new Error(
|
|
502
|
-
"Streaming context was closed or not properly initialized."
|
|
503
|
-
);
|
|
504
|
-
}
|
|
505
|
-
let arrayBufferChunk;
|
|
506
|
-
if (chunk instanceof Blob) {
|
|
507
|
-
logger.debug("Converting Blob to ArrayBuffer");
|
|
508
|
-
arrayBufferChunk = await chunk.arrayBuffer();
|
|
509
|
-
} else {
|
|
510
|
-
arrayBufferChunk = chunk;
|
|
511
|
-
}
|
|
512
|
-
if (!arrayBufferChunk || arrayBufferChunk.byteLength === 0) {
|
|
513
|
-
logger.warn("Received empty chunk, skipping");
|
|
514
|
-
return;
|
|
515
|
-
}
|
|
516
|
-
if (!hasReceivedFirstChunk) {
|
|
517
|
-
hasReceivedFirstChunk = true;
|
|
518
|
-
logger.debug(
|
|
519
|
-
`First chunk received, size: ${arrayBufferChunk.byteLength} bytes`
|
|
520
|
-
);
|
|
521
|
-
}
|
|
522
|
-
receivedChunksCount++;
|
|
523
|
-
pendingChunks.push(arrayBufferChunk);
|
|
524
|
-
logger.debug(
|
|
525
|
-
`Added chunk #${receivedChunksCount} to queue (size: ${arrayBufferChunk.byteLength} bytes)`
|
|
526
|
-
);
|
|
527
|
-
await processQueue();
|
|
528
|
-
if (!playbackStarted && hasReceivedFirstChunk && receivedChunksCount >= 3) {
|
|
529
|
-
await tryStartPlayback();
|
|
530
|
-
}
|
|
531
|
-
};
|
|
532
|
-
const endChunkStream = () => {
|
|
533
|
-
if (mediaSource && mediaSource.readyState === "open") {
|
|
534
|
-
try {
|
|
535
|
-
if (pendingChunks.length > 0 || sourceBuffer && sourceBuffer.updating) {
|
|
536
|
-
logger.debug("Waiting for pending chunks before ending stream");
|
|
537
|
-
setTimeout(() => endChunkStream(), 200);
|
|
538
|
-
return;
|
|
539
|
-
}
|
|
540
|
-
if (hasReceivedFirstChunk) {
|
|
541
|
-
mediaSource.endOfStream();
|
|
542
|
-
logger.debug("MediaSource stream ended successfully");
|
|
543
|
-
} else {
|
|
544
|
-
logger.warn("Stream ended without receiving any chunks");
|
|
545
|
-
}
|
|
546
|
-
} catch (err) {
|
|
547
|
-
logger.error("Error ending MediaSource stream", err);
|
|
548
|
-
}
|
|
549
|
-
}
|
|
550
|
-
audio.onended = null;
|
|
551
|
-
if (audio.parentNode) {
|
|
552
|
-
audio.parentNode.removeChild(audio);
|
|
553
|
-
}
|
|
554
|
-
if (this.currentAudioUrl === url) {
|
|
555
|
-
this.currentAudioUrl = null;
|
|
556
|
-
URL.revokeObjectURL(url);
|
|
557
|
-
}
|
|
558
|
-
sourceBuffer = null;
|
|
559
|
-
};
|
|
560
|
-
audio.onended = () => {
|
|
561
|
-
logger.debug("Audio playback completed");
|
|
562
|
-
endChunkStream();
|
|
563
|
-
onComplete?.();
|
|
564
|
-
};
|
|
565
|
-
return {
|
|
566
|
-
addChunkToStream,
|
|
567
|
-
endChunkStream
|
|
568
|
-
};
|
|
569
|
-
}
|
|
570
|
-
/**
|
|
571
|
-
* Stop any ongoing HTMLAudioElement playback.
|
|
572
|
-
*/
|
|
573
|
-
async stopPlayback() {
|
|
574
|
-
if (this.currentHtmlAudio) {
|
|
575
|
-
try {
|
|
576
|
-
this.currentHtmlAudio.pause();
|
|
577
|
-
this.currentHtmlAudio.src = "";
|
|
578
|
-
} catch (err) {
|
|
579
|
-
this.logger.error("Error stopping playback", err);
|
|
580
|
-
}
|
|
581
|
-
this.currentHtmlAudio = null;
|
|
582
|
-
}
|
|
583
|
-
if (this.currentAudioUrl) {
|
|
584
|
-
URL.revokeObjectURL(this.currentAudioUrl);
|
|
585
|
-
this.currentAudioUrl = null;
|
|
586
|
-
}
|
|
587
|
-
}
|
|
588
|
-
/**
|
|
589
|
-
* Cleans up all audio playback resources.
|
|
590
|
-
*/
|
|
591
|
-
cleanup() {
|
|
592
|
-
this.stopPlayback();
|
|
593
|
-
}
|
|
594
|
-
};
|
|
595
|
-
|
|
596
|
-
// src/react/hooks/use-conversation/useConversation.ts
|
|
597
|
-
Logger.enableGlobalLogging();
|
|
598
|
-
function useConversation(endpoint, {
|
|
599
|
-
onStartRecording,
|
|
600
|
-
onStopRecording,
|
|
601
|
-
onReceive,
|
|
602
|
-
autoPlay = true,
|
|
603
|
-
downstreamMode = "STREAM",
|
|
604
|
-
onError,
|
|
605
|
-
audioConfig = {},
|
|
606
|
-
requestData = {},
|
|
607
|
-
endpointConfig = {}
|
|
608
|
-
}) {
|
|
609
|
-
const { current: logger } = react.useRef(
|
|
610
|
-
new Logger("@m4trix/core > useConversation")
|
|
611
|
-
);
|
|
612
|
-
const inputAudioControllerRef = react.useRef(void 0);
|
|
613
|
-
const outputAudioControllerRef = react.useRef(
|
|
614
|
-
void 0
|
|
615
|
-
);
|
|
616
|
-
const endpointAdapterRef = react.useRef(
|
|
617
|
-
void 0
|
|
618
|
-
);
|
|
619
|
-
const [voiceAgentState, setVoiceAgentState] = react.useState("READY");
|
|
620
|
-
const [error, setError] = react.useState(null);
|
|
621
|
-
const handleError = react.useCallback(
|
|
622
|
-
(state2, err) => {
|
|
623
|
-
setError(err);
|
|
624
|
-
logger.error(`Error during ${state2}:`, err);
|
|
625
|
-
onError?.(state2, err);
|
|
626
|
-
},
|
|
627
|
-
[onError]
|
|
628
|
-
);
|
|
629
|
-
const startRecording = react.useCallback(() => {
|
|
630
|
-
if (inputAudioControllerRef.current) {
|
|
631
|
-
try {
|
|
632
|
-
logger.debug("Starting recording");
|
|
633
|
-
setVoiceAgentState("RECORDING");
|
|
634
|
-
inputAudioControllerRef.current.startRecording({
|
|
635
|
-
onError: (err) => {
|
|
636
|
-
handleError("RECORDING", err);
|
|
637
|
-
}
|
|
638
|
-
});
|
|
639
|
-
onStartRecording?.();
|
|
640
|
-
} catch (err) {
|
|
641
|
-
if (err instanceof Error) {
|
|
642
|
-
handleError("RECORDING", err);
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
|
-
}
|
|
646
|
-
}, [onStartRecording, handleError]);
|
|
647
|
-
const stopRecording = react.useCallback(async () => {
|
|
648
|
-
if (inputAudioControllerRef.current) {
|
|
649
|
-
try {
|
|
650
|
-
logger.debug("Stopping recording");
|
|
651
|
-
await inputAudioControllerRef.current.stopRecording({
|
|
652
|
-
onRecordingCompleted: async (allData) => {
|
|
653
|
-
setVoiceAgentState("PROCESSING");
|
|
654
|
-
try {
|
|
655
|
-
const response = await endpointAdapterRef.current?.sendVoiceFile({
|
|
656
|
-
blob: allData,
|
|
657
|
-
metadata: requestData
|
|
658
|
-
});
|
|
659
|
-
if (!response) {
|
|
660
|
-
throw new Error("No response received from endpoint");
|
|
661
|
-
}
|
|
662
|
-
setVoiceAgentState("RESPONDING");
|
|
663
|
-
if (autoPlay) {
|
|
664
|
-
if (downstreamMode === "STREAM") {
|
|
665
|
-
await outputAudioControllerRef.current?.playAudioStream({
|
|
666
|
-
response,
|
|
667
|
-
onComplete: () => {
|
|
668
|
-
setVoiceAgentState("READY");
|
|
669
|
-
}
|
|
670
|
-
});
|
|
671
|
-
} else if (downstreamMode === "DOWNLOAD") {
|
|
672
|
-
const responseBlob = await response.blob();
|
|
673
|
-
await outputAudioControllerRef.current?.playAudio({
|
|
674
|
-
source: responseBlob,
|
|
675
|
-
onComplete: () => {
|
|
676
|
-
setVoiceAgentState("READY");
|
|
677
|
-
}
|
|
678
|
-
});
|
|
679
|
-
}
|
|
680
|
-
} else {
|
|
681
|
-
setVoiceAgentState("READY");
|
|
682
|
-
}
|
|
683
|
-
onReceive?.(
|
|
684
|
-
allData,
|
|
685
|
-
async () => {
|
|
686
|
-
if (outputAudioControllerRef.current) {
|
|
687
|
-
if (downstreamMode === "STREAM") {
|
|
688
|
-
return outputAudioControllerRef.current.playAudioStream({
|
|
689
|
-
response,
|
|
690
|
-
onComplete: () => {
|
|
691
|
-
setVoiceAgentState("READY");
|
|
692
|
-
}
|
|
693
|
-
});
|
|
694
|
-
} else {
|
|
695
|
-
const responseBlob = await response.blob();
|
|
696
|
-
return outputAudioControllerRef.current.playAudio({
|
|
697
|
-
source: responseBlob,
|
|
698
|
-
onComplete: () => {
|
|
699
|
-
setVoiceAgentState("READY");
|
|
700
|
-
}
|
|
701
|
-
});
|
|
702
|
-
}
|
|
703
|
-
}
|
|
704
|
-
},
|
|
705
|
-
async () => {
|
|
706
|
-
if (outputAudioControllerRef.current) {
|
|
707
|
-
return outputAudioControllerRef.current.stopPlayback();
|
|
708
|
-
}
|
|
709
|
-
}
|
|
710
|
-
);
|
|
711
|
-
} catch (err) {
|
|
712
|
-
if (err instanceof Error) {
|
|
713
|
-
handleError("PROCESSING", err);
|
|
714
|
-
}
|
|
715
|
-
setVoiceAgentState("READY");
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
});
|
|
719
|
-
onStopRecording?.();
|
|
720
|
-
} catch (err) {
|
|
721
|
-
if (err instanceof Error) {
|
|
722
|
-
handleError("RECORDING", err);
|
|
723
|
-
}
|
|
724
|
-
}
|
|
725
|
-
}
|
|
726
|
-
}, [
|
|
727
|
-
onStopRecording,
|
|
728
|
-
requestData,
|
|
729
|
-
autoPlay,
|
|
730
|
-
downstreamMode,
|
|
731
|
-
handleError,
|
|
732
|
-
onReceive
|
|
733
|
-
]);
|
|
734
|
-
react.useEffect(() => {
|
|
735
|
-
if (endpointAdapterRef.current) {
|
|
736
|
-
return;
|
|
737
|
-
}
|
|
738
|
-
try {
|
|
739
|
-
const endpointAdapter = endpointConfig.endpointAdapter ? endpointConfig.endpointAdapter : new BaseVoiceEndpointAdapter({
|
|
740
|
-
baseUrl: endpointConfig.baseUrl,
|
|
741
|
-
endpoint,
|
|
742
|
-
headers: endpointConfig.headers
|
|
743
|
-
});
|
|
744
|
-
endpointAdapterRef.current = endpointAdapter;
|
|
745
|
-
if (!inputAudioControllerRef.current) {
|
|
746
|
-
inputAudioControllerRef.current = new WebAudioInputAudioController(
|
|
747
|
-
audioConfig
|
|
748
|
-
);
|
|
749
|
-
}
|
|
750
|
-
if (!outputAudioControllerRef.current) {
|
|
751
|
-
outputAudioControllerRef.current = new AudioElementOutputAudioController();
|
|
752
|
-
}
|
|
753
|
-
} catch (err) {
|
|
754
|
-
if (err instanceof Error) {
|
|
755
|
-
handleError("READY", err);
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
}, [endpoint, endpointConfig, audioConfig, handleError]);
|
|
759
|
-
react.useEffect(() => {
|
|
760
|
-
return () => {
|
|
761
|
-
inputAudioControllerRef.current?.cleanup();
|
|
762
|
-
outputAudioControllerRef.current?.cleanup();
|
|
763
|
-
};
|
|
764
|
-
}, []);
|
|
765
|
-
return {
|
|
766
|
-
startRecording,
|
|
767
|
-
stopRecording,
|
|
768
|
-
voiceAgentState,
|
|
769
|
-
error,
|
|
770
|
-
audioContext: inputAudioControllerRef.current?.audioContext || null
|
|
771
|
-
};
|
|
772
|
-
}
|
|
773
|
-
|
|
774
|
-
// src/react/adapter/socket/VoiceSocketAdapter.ts
|
|
775
|
-
var VoiceSocketAdapter = class {
|
|
776
|
-
constructor(config) {
|
|
777
|
-
this._isConnected = false;
|
|
778
|
-
this.logger = new Logger("@m4trix/core > VoiceSocketAdapter");
|
|
779
|
-
this.emitter = new Emitter();
|
|
780
|
-
this.config = config;
|
|
781
|
-
}
|
|
782
|
-
on(event, listener) {
|
|
783
|
-
this.emitter.on(event, listener);
|
|
784
|
-
}
|
|
785
|
-
off(event, listener) {
|
|
786
|
-
this.emitter.off(event, listener);
|
|
787
|
-
}
|
|
788
|
-
once(event, listener) {
|
|
789
|
-
this.emitter.once(event, listener);
|
|
790
|
-
}
|
|
791
|
-
emit(event, data) {
|
|
792
|
-
this.emitter.emit(event, data);
|
|
793
|
-
}
|
|
794
|
-
isConnected() {
|
|
795
|
-
return this._isConnected;
|
|
796
|
-
}
|
|
797
|
-
};
|
|
798
|
-
var Emitter = class {
|
|
799
|
-
constructor() {
|
|
800
|
-
this.target = new EventTarget();
|
|
801
|
-
}
|
|
802
|
-
on(type, listener) {
|
|
803
|
-
this.target.addEventListener(type, listener);
|
|
804
|
-
}
|
|
805
|
-
off(type, listener) {
|
|
806
|
-
this.target.removeEventListener(type, listener);
|
|
807
|
-
}
|
|
808
|
-
once(type, listener) {
|
|
809
|
-
const wrapper = (event) => {
|
|
810
|
-
this.off(type, wrapper);
|
|
811
|
-
listener(event.detail);
|
|
812
|
-
};
|
|
813
|
-
this.on(type, wrapper);
|
|
814
|
-
}
|
|
815
|
-
emit(type, detail) {
|
|
816
|
-
this.target.dispatchEvent(new CustomEvent(type, { detail }));
|
|
817
|
-
}
|
|
818
|
-
};
|
|
819
|
-
var VoiceSocketIOAdapter = class extends VoiceSocketAdapter {
|
|
820
|
-
constructor(config) {
|
|
821
|
-
super(config);
|
|
822
|
-
this.socket = null;
|
|
823
|
-
}
|
|
824
|
-
async connect() {
|
|
825
|
-
return new Promise((resolve, reject) => {
|
|
826
|
-
if (!this.socket) {
|
|
827
|
-
this.socket = socket_ioClient.io(this.config.baseUrl, {
|
|
828
|
-
extraHeaders: this.config.headers,
|
|
829
|
-
autoConnect: true
|
|
830
|
-
});
|
|
831
|
-
}
|
|
832
|
-
this.socket.on("connect", () => {
|
|
833
|
-
this._isConnected = true;
|
|
834
|
-
this.logger.debug("Connected to socket");
|
|
835
|
-
this.emit("connect");
|
|
836
|
-
resolve();
|
|
837
|
-
});
|
|
838
|
-
this.socket.on("disconnect", () => {
|
|
839
|
-
this._isConnected = false;
|
|
840
|
-
this.emit("disconnect");
|
|
841
|
-
this.logger.debug("Disconnected from socket");
|
|
842
|
-
if (this.config.autoReconnect)
|
|
843
|
-
this.connect();
|
|
844
|
-
});
|
|
845
|
-
this.socket.on("connect_error", (error) => {
|
|
846
|
-
this.logger.error("Error connecting to socket", error);
|
|
847
|
-
this.emit("error", error);
|
|
848
|
-
reject(error);
|
|
849
|
-
});
|
|
850
|
-
this.socket.on("voice:chunk_received", (chunk) => {
|
|
851
|
-
this.logger.debug("Received voice chunk", chunk.byteLength);
|
|
852
|
-
this.onVoiceChunkReceived(chunk);
|
|
853
|
-
});
|
|
854
|
-
this.socket.on("voice:received_end_of_response_stream", () => {
|
|
855
|
-
this.logger.debug("Received end of response stream");
|
|
856
|
-
this.onReceivedEndOfResponseStream();
|
|
857
|
-
});
|
|
858
|
-
this.socket.on("voice:file_received", (blob) => {
|
|
859
|
-
this.logger.debug("Received voice file");
|
|
860
|
-
this.onVoiceFileReceived(blob);
|
|
861
|
-
});
|
|
862
|
-
this.socket.on("control-message", (message) => {
|
|
863
|
-
this.logger.debug("Received control message", message);
|
|
864
|
-
this.emit("control-message", message);
|
|
865
|
-
});
|
|
866
|
-
});
|
|
867
|
-
}
|
|
868
|
-
disconnect() {
|
|
869
|
-
this.socket?.disconnect();
|
|
870
|
-
this.socket = null;
|
|
871
|
-
this._isConnected = false;
|
|
872
|
-
}
|
|
873
|
-
exposeSocket() {
|
|
874
|
-
return this.socket;
|
|
875
|
-
}
|
|
876
|
-
async sendVoiceChunk(chunk, metadata) {
|
|
877
|
-
this.logger.debug(
|
|
878
|
-
"Sending voice chunk %i",
|
|
879
|
-
chunk instanceof Blob ? chunk.size : chunk.byteLength
|
|
880
|
-
);
|
|
881
|
-
if (!this.socket || !this.isConnected)
|
|
882
|
-
throw new Error("Socket not connected");
|
|
883
|
-
let chunkToSend;
|
|
884
|
-
if (chunk instanceof Blob) {
|
|
885
|
-
chunkToSend = await chunk.arrayBuffer();
|
|
886
|
-
} else {
|
|
887
|
-
chunkToSend = chunk;
|
|
888
|
-
}
|
|
889
|
-
this.logger.debug("[Socket] Sending voice chunk", chunkToSend.byteLength);
|
|
890
|
-
this.socket.emit("voice:send_chunk", chunkToSend, metadata);
|
|
891
|
-
this.emit("chunk_sent", chunk);
|
|
892
|
-
}
|
|
893
|
-
sendVoiceFile(blob, metadata) {
|
|
894
|
-
this.logger.debug("Sending voice file", blob, metadata);
|
|
895
|
-
if (!this.socket || !this.isConnected)
|
|
896
|
-
throw new Error("Socket not connected");
|
|
897
|
-
this.socket.emit("voice:send_file", blob, metadata);
|
|
898
|
-
this.emit("file-sent", blob);
|
|
899
|
-
}
|
|
900
|
-
commitVoiceMessage() {
|
|
901
|
-
if (!this.socket || !this.isConnected)
|
|
902
|
-
throw new Error("Socket not connected");
|
|
903
|
-
this.socket.emit("voice:commit");
|
|
904
|
-
}
|
|
905
|
-
onVoiceChunkReceived(chunk) {
|
|
906
|
-
this.emit("chunk-received", chunk);
|
|
907
|
-
}
|
|
908
|
-
onVoiceFileReceived(blob) {
|
|
909
|
-
this.emit("file-received", blob);
|
|
910
|
-
}
|
|
911
|
-
onReceivedEndOfResponseStream() {
|
|
912
|
-
this.emit("received-end-of-response-stream");
|
|
913
|
-
}
|
|
914
|
-
};
|
|
915
|
-
|
|
916
|
-
// src/react/utility/audio/WebAudioOutputAudioController.ts
|
|
917
|
-
var STREAM_SAMPLE_RATE = 24e3;
|
|
918
|
-
var CHANNELS = 1;
|
|
919
|
-
var SLICE_DURATION_S = 0.25;
|
|
920
|
-
var FRAMES_PER_SLICE = Math.floor(STREAM_SAMPLE_RATE * SLICE_DURATION_S);
|
|
921
|
-
var BYTES_PER_SLICE = FRAMES_PER_SLICE * 2;
|
|
922
|
-
var SCHED_TOLERANCE = 0.05;
|
|
923
|
-
var WebAudioOutputAudioController = class extends OutputAudioController {
|
|
924
|
-
constructor() {
|
|
925
|
-
super("@m4trix/core > WebAudioOutputAudioController");
|
|
926
|
-
this.audioCtx = new AudioContext();
|
|
927
|
-
this.gain = this.audioCtx.createGain();
|
|
928
|
-
this.nextPlayTime = 0;
|
|
929
|
-
this.activeSources = /* @__PURE__ */ new Set();
|
|
930
|
-
this.userGestureHookAttached = false;
|
|
931
|
-
this.gain.connect(this.audioCtx.destination);
|
|
932
|
-
this.resetScheduler();
|
|
933
|
-
}
|
|
934
|
-
// ─────────────────────────────────────────────────────────────────────
|
|
935
|
-
// One‑shot playback
|
|
936
|
-
// ─────────────────────────────────────────────────────────────────────
|
|
937
|
-
async playAudio({
|
|
938
|
-
source,
|
|
939
|
-
onComplete
|
|
940
|
-
}) {
|
|
941
|
-
await this.stopPlayback();
|
|
942
|
-
const buf = await this.sourceToArrayBuffer(source);
|
|
943
|
-
const decoded = await this.decode(buf);
|
|
944
|
-
await this.ensureContextRunning();
|
|
945
|
-
const src = this.createSource(decoded, this.audioCtx.currentTime);
|
|
946
|
-
src.onended = () => {
|
|
947
|
-
this.activeSources.delete(src);
|
|
948
|
-
onComplete?.();
|
|
949
|
-
};
|
|
950
|
-
}
|
|
951
|
-
async playAudioStream() {
|
|
952
|
-
}
|
|
953
|
-
// ─────────────────────────────────────────────────────────────────────
|
|
954
|
-
// PCM streaming
|
|
955
|
-
// ─────────────────────────────────────────────────────────────────────
|
|
956
|
-
async initializeChunkStream({
|
|
957
|
-
onComplete
|
|
958
|
-
}) {
|
|
959
|
-
await this.stopPlayback();
|
|
960
|
-
await this.ensureContextRunning();
|
|
961
|
-
this.resetScheduler();
|
|
962
|
-
let streamEnded = false;
|
|
963
|
-
let pending = new Uint8Array(0);
|
|
964
|
-
const addChunkToStream = async (pkt) => {
|
|
965
|
-
if (streamEnded) {
|
|
966
|
-
this.logger.warn("Attempt to add chunk after stream ended \u2013 ignoring.");
|
|
967
|
-
return;
|
|
968
|
-
}
|
|
969
|
-
const bytes = new Uint8Array(
|
|
970
|
-
pkt instanceof Blob ? await pkt.arrayBuffer() : pkt
|
|
971
|
-
);
|
|
972
|
-
if (bytes.length === 0)
|
|
973
|
-
return;
|
|
974
|
-
const merged = new Uint8Array(pending.length + bytes.length);
|
|
975
|
-
merged.set(pending);
|
|
976
|
-
merged.set(bytes, pending.length);
|
|
977
|
-
pending = merged;
|
|
978
|
-
if (pending.length % 2 === 1)
|
|
979
|
-
return;
|
|
980
|
-
while (pending.length >= BYTES_PER_SLICE) {
|
|
981
|
-
const sliceBytes = pending.slice(0, BYTES_PER_SLICE);
|
|
982
|
-
pending = pending.slice(BYTES_PER_SLICE);
|
|
983
|
-
const aligned = sliceBytes.buffer.slice(
|
|
984
|
-
sliceBytes.byteOffset,
|
|
985
|
-
sliceBytes.byteOffset + sliceBytes.byteLength
|
|
986
|
-
);
|
|
987
|
-
const int16 = new Int16Array(aligned);
|
|
988
|
-
const buf = this.audioCtx.createBuffer(
|
|
989
|
-
CHANNELS,
|
|
990
|
-
int16.length,
|
|
991
|
-
STREAM_SAMPLE_RATE
|
|
992
|
-
);
|
|
993
|
-
const data = buf.getChannelData(0);
|
|
994
|
-
for (let i = 0; i < int16.length; i++)
|
|
995
|
-
data[i] = int16[i] / 32768;
|
|
996
|
-
this.scheduleBuffer(buf);
|
|
997
|
-
}
|
|
998
|
-
};
|
|
999
|
-
const endChunkStream = () => {
|
|
1000
|
-
if (streamEnded)
|
|
1001
|
-
return;
|
|
1002
|
-
streamEnded = true;
|
|
1003
|
-
if (onComplete) {
|
|
1004
|
-
if (this.activeSources.size === 0)
|
|
1005
|
-
onComplete();
|
|
1006
|
-
else {
|
|
1007
|
-
const last = Array.from(this.activeSources).pop();
|
|
1008
|
-
if (last) {
|
|
1009
|
-
const prev = last.onended;
|
|
1010
|
-
last.onended = (e) => {
|
|
1011
|
-
if (prev)
|
|
1012
|
-
prev.call(last, e);
|
|
1013
|
-
onComplete();
|
|
1014
|
-
};
|
|
1015
|
-
}
|
|
1016
|
-
}
|
|
1017
|
-
}
|
|
1018
|
-
};
|
|
1019
|
-
return { addChunkToStream, endChunkStream };
|
|
1020
|
-
}
|
|
1021
|
-
// ─────────────────────────────────────────────────────────────────────
|
|
1022
|
-
// Buffer scheduling helpers
|
|
1023
|
-
// ─────────────────────────────────────────────────────────────────────
|
|
1024
|
-
scheduleBuffer(buf) {
|
|
1025
|
-
if (this.nextPlayTime < this.audioCtx.currentTime + SCHED_TOLERANCE) {
|
|
1026
|
-
this.nextPlayTime = this.audioCtx.currentTime + SCHED_TOLERANCE;
|
|
1027
|
-
}
|
|
1028
|
-
this.createSource(buf, this.nextPlayTime);
|
|
1029
|
-
this.nextPlayTime += buf.duration;
|
|
1030
|
-
}
|
|
1031
|
-
createSource(buf, when) {
|
|
1032
|
-
const src = this.audioCtx.createBufferSource();
|
|
1033
|
-
src.buffer = buf;
|
|
1034
|
-
src.connect(this.gain);
|
|
1035
|
-
src.start(when);
|
|
1036
|
-
this.activeSources.add(src);
|
|
1037
|
-
src.onended = () => {
|
|
1038
|
-
this.activeSources.delete(src);
|
|
1039
|
-
};
|
|
1040
|
-
return src;
|
|
1041
|
-
}
|
|
1042
|
-
resetScheduler() {
|
|
1043
|
-
this.nextPlayTime = this.audioCtx.currentTime;
|
|
1044
|
-
}
|
|
1045
|
-
// ─── External resource helpers ───────────────────────────────────────
|
|
1046
|
-
sourceToArrayBuffer(src) {
|
|
1047
|
-
return typeof src === "string" ? fetch(src).then((r) => {
|
|
1048
|
-
if (!r.ok)
|
|
1049
|
-
throw new Error(`${r.status}`);
|
|
1050
|
-
return r.arrayBuffer();
|
|
1051
|
-
}) : src.arrayBuffer();
|
|
1052
|
-
}
|
|
1053
|
-
decode(buf) {
|
|
1054
|
-
return new Promise(
|
|
1055
|
-
(res, rej) => this.audioCtx.decodeAudioData(buf, res, rej)
|
|
1056
|
-
);
|
|
1057
|
-
}
|
|
1058
|
-
// ─── Lifecycle methods ───────────────────────────────────────────────
|
|
1059
|
-
async stopPlayback() {
|
|
1060
|
-
for (const src of this.activeSources) {
|
|
1061
|
-
try {
|
|
1062
|
-
src.stop();
|
|
1063
|
-
} catch {
|
|
1064
|
-
}
|
|
1065
|
-
src.disconnect();
|
|
1066
|
-
}
|
|
1067
|
-
this.activeSources.clear();
|
|
1068
|
-
this.resetScheduler();
|
|
1069
|
-
}
|
|
1070
|
-
cleanup() {
|
|
1071
|
-
this.stopPlayback();
|
|
1072
|
-
if (this.audioCtx.state !== "closed")
|
|
1073
|
-
this.audioCtx.close();
|
|
1074
|
-
}
|
|
1075
|
-
// ─── Autoplay‑policy helper ──────────────────────────────────────────
|
|
1076
|
-
async ensureContextRunning() {
|
|
1077
|
-
if (this.audioCtx.state !== "suspended")
|
|
1078
|
-
return;
|
|
1079
|
-
try {
|
|
1080
|
-
await this.audioCtx.resume();
|
|
1081
|
-
} catch {
|
|
1082
|
-
}
|
|
1083
|
-
if (this.audioCtx.state === "running")
|
|
1084
|
-
return;
|
|
1085
|
-
if (!this.userGestureHookAttached) {
|
|
1086
|
-
this.userGestureHookAttached = true;
|
|
1087
|
-
const resume = async () => {
|
|
1088
|
-
try {
|
|
1089
|
-
await this.audioCtx.resume();
|
|
1090
|
-
} catch {
|
|
1091
|
-
}
|
|
1092
|
-
if (this.audioCtx.state === "running")
|
|
1093
|
-
document.removeEventListener("click", resume);
|
|
1094
|
-
};
|
|
1095
|
-
document.addEventListener("click", resume);
|
|
1096
|
-
}
|
|
1097
|
-
}
|
|
1098
|
-
};
|
|
1099
|
-
|
|
1100
|
-
// src/react/hooks/use-conversation/useSocketConversation.ts
|
|
1101
|
-
Logger.enableGlobalLogging();
|
|
1102
|
-
function useSocketConversation({
|
|
1103
|
-
scope,
|
|
1104
|
-
onStartRecording,
|
|
1105
|
-
onStopRecording,
|
|
1106
|
-
onReceive,
|
|
1107
|
-
upstreamMode = "STREAM_WHILE_TALK",
|
|
1108
|
-
onError,
|
|
1109
|
-
audioConfig = {},
|
|
1110
|
-
socketConfig = {}
|
|
1111
|
-
}) {
|
|
1112
|
-
const { current: logger } = react.useRef(
|
|
1113
|
-
new Logger("SuTr > useSocketConversation")
|
|
1114
|
-
);
|
|
1115
|
-
const inputAudioControllerRef = react.useRef(void 0);
|
|
1116
|
-
const outputAudioControllerRef = react.useRef(
|
|
1117
|
-
void 0
|
|
1118
|
-
);
|
|
1119
|
-
const socketAdapterRef = react.useRef(void 0);
|
|
1120
|
-
const [socket, setSocket] = react.useState(null);
|
|
1121
|
-
const [voiceAgentState, setVoiceAgentState] = react.useState("READY");
|
|
1122
|
-
const [error, setError] = react.useState(null);
|
|
1123
|
-
const shouldStreamWhileTalk = upstreamMode === "STREAM_WHILE_TALK";
|
|
1124
|
-
const handleError = react.useCallback(
|
|
1125
|
-
(state2, err) => {
|
|
1126
|
-
setError(err);
|
|
1127
|
-
logger.error(`Error during ${state2}:`, err);
|
|
1128
|
-
onError?.(state2, err);
|
|
1129
|
-
},
|
|
1130
|
-
[onError]
|
|
1131
|
-
);
|
|
1132
|
-
const subscribeToSocketEventsForChunkDownstreaming = react.useCallback(
|
|
1133
|
-
async (socketAdapter) => {
|
|
1134
|
-
logger.debug("Setting up audio stream for receiving chunks");
|
|
1135
|
-
try {
|
|
1136
|
-
const { addChunkToStream, endChunkStream } = await outputAudioControllerRef.current.initializeChunkStream({
|
|
1137
|
-
mimeCodec: "audio/mpeg",
|
|
1138
|
-
onComplete: () => {
|
|
1139
|
-
logger.debug("Audio stream playback completed");
|
|
1140
|
-
setVoiceAgentState("READY");
|
|
1141
|
-
}
|
|
1142
|
-
});
|
|
1143
|
-
let chunkCount = 0;
|
|
1144
|
-
const chunkReceivedEmitter = async (chunk) => {
|
|
1145
|
-
if (chunk instanceof ArrayBuffer) {
|
|
1146
|
-
chunkCount++;
|
|
1147
|
-
logger.debug(
|
|
1148
|
-
`Received voice chunk #${chunkCount} from socket, size: ${chunk.byteLength} bytes`
|
|
1149
|
-
);
|
|
1150
|
-
if (!chunk || chunk.byteLength === 0) {
|
|
1151
|
-
logger.warn("Received empty chunk, skipping");
|
|
1152
|
-
return;
|
|
1153
|
-
}
|
|
1154
|
-
try {
|
|
1155
|
-
await addChunkToStream(chunk);
|
|
1156
|
-
logger.debug(
|
|
1157
|
-
`Successfully added chunk #${chunkCount} to audio stream`
|
|
1158
|
-
);
|
|
1159
|
-
} catch (err) {
|
|
1160
|
-
logger.error(
|
|
1161
|
-
`Failed to add chunk #${chunkCount} to audio stream`,
|
|
1162
|
-
err
|
|
1163
|
-
);
|
|
1164
|
-
if (err instanceof Error) {
|
|
1165
|
-
handleError("DOWNSTREAMING", err);
|
|
1166
|
-
}
|
|
1167
|
-
}
|
|
1168
|
-
}
|
|
1169
|
-
};
|
|
1170
|
-
socketAdapter.on("chunk-received", chunkReceivedEmitter);
|
|
1171
|
-
const endOfStreamEmitter = () => {
|
|
1172
|
-
logger.debug(
|
|
1173
|
-
`Received end of stream signal after ${chunkCount} chunks, ending chunk stream`
|
|
1174
|
-
);
|
|
1175
|
-
endChunkStream();
|
|
1176
|
-
setVoiceAgentState("READY");
|
|
1177
|
-
};
|
|
1178
|
-
socketAdapter.on("received-end-of-response-stream", endOfStreamEmitter);
|
|
1179
|
-
return () => {
|
|
1180
|
-
logger.debug("Cleaning up socket event listeners");
|
|
1181
|
-
socketAdapter.off("chunk-received", chunkReceivedEmitter);
|
|
1182
|
-
socketAdapter.off(
|
|
1183
|
-
"received-end-of-response-stream",
|
|
1184
|
-
endOfStreamEmitter
|
|
1185
|
-
);
|
|
1186
|
-
endChunkStream();
|
|
1187
|
-
};
|
|
1188
|
-
} catch (err) {
|
|
1189
|
-
if (err instanceof Error) {
|
|
1190
|
-
handleError("DOWNSTREAMING", err);
|
|
1191
|
-
}
|
|
1192
|
-
return () => {
|
|
1193
|
-
};
|
|
1194
|
-
}
|
|
1195
|
-
},
|
|
1196
|
-
[handleError]
|
|
1197
|
-
);
|
|
1198
|
-
const hookupSocketAdapter = react.useCallback(
|
|
1199
|
-
async (socketAdapter) => {
|
|
1200
|
-
logger.debug("Connecting to socket...");
|
|
1201
|
-
try {
|
|
1202
|
-
await socketAdapter.connect();
|
|
1203
|
-
socketAdapter.on("connect", () => {
|
|
1204
|
-
logger.debug("Socket adapter connected");
|
|
1205
|
-
setVoiceAgentState("READY");
|
|
1206
|
-
});
|
|
1207
|
-
socketAdapter.on("disconnect", () => {
|
|
1208
|
-
logger.debug("Socket adapter disconnected");
|
|
1209
|
-
});
|
|
1210
|
-
socketAdapter.on("error", (err) => {
|
|
1211
|
-
if (err instanceof Error) {
|
|
1212
|
-
handleError(voiceAgentState, err);
|
|
1213
|
-
} else {
|
|
1214
|
-
handleError(voiceAgentState, new Error("Unknown error"));
|
|
1215
|
-
}
|
|
1216
|
-
});
|
|
1217
|
-
setSocket(socketAdapter.exposeSocket());
|
|
1218
|
-
} catch (err) {
|
|
1219
|
-
if (err instanceof Error) {
|
|
1220
|
-
handleError("READY", err);
|
|
1221
|
-
}
|
|
1222
|
-
}
|
|
1223
|
-
},
|
|
1224
|
-
[handleError, voiceAgentState]
|
|
1225
|
-
);
|
|
1226
|
-
const startRecording = react.useCallback(() => {
|
|
1227
|
-
if (inputAudioControllerRef.current) {
|
|
1228
|
-
try {
|
|
1229
|
-
logger.debug("Starting recording");
|
|
1230
|
-
setVoiceAgentState("RECORDING");
|
|
1231
|
-
inputAudioControllerRef.current.startRecording({
|
|
1232
|
-
onRecordedChunk: async (chunk) => {
|
|
1233
|
-
if (shouldStreamWhileTalk) {
|
|
1234
|
-
try {
|
|
1235
|
-
await socketAdapterRef.current?.sendVoiceChunk(chunk);
|
|
1236
|
-
} catch (err) {
|
|
1237
|
-
if (err instanceof Error) {
|
|
1238
|
-
handleError("RECORDING", err);
|
|
1239
|
-
}
|
|
1240
|
-
}
|
|
1241
|
-
}
|
|
1242
|
-
}
|
|
1243
|
-
});
|
|
1244
|
-
onStartRecording?.();
|
|
1245
|
-
} catch (err) {
|
|
1246
|
-
if (err instanceof Error) {
|
|
1247
|
-
handleError("RECORDING", err);
|
|
1248
|
-
}
|
|
1249
|
-
}
|
|
1250
|
-
}
|
|
1251
|
-
}, [onStartRecording, shouldStreamWhileTalk, handleError]);
|
|
1252
|
-
const stopRecording = react.useCallback(async () => {
|
|
1253
|
-
if (inputAudioControllerRef.current) {
|
|
1254
|
-
try {
|
|
1255
|
-
logger.debug("Stopping recording");
|
|
1256
|
-
await inputAudioControllerRef.current.stopRecording({
|
|
1257
|
-
onRecordingCompleted: async (allData) => {
|
|
1258
|
-
setVoiceAgentState("PROCESSING");
|
|
1259
|
-
try {
|
|
1260
|
-
if (shouldStreamWhileTalk) {
|
|
1261
|
-
logger.debug("Committing voice message");
|
|
1262
|
-
await socketAdapterRef.current?.commitVoiceMessage();
|
|
1263
|
-
} else {
|
|
1264
|
-
await socketAdapterRef.current?.sendVoiceFile(allData);
|
|
1265
|
-
}
|
|
1266
|
-
setVoiceAgentState("DOWNSTREAMING");
|
|
1267
|
-
await subscribeToSocketEventsForChunkDownstreaming(
|
|
1268
|
-
socketAdapterRef.current
|
|
1269
|
-
);
|
|
1270
|
-
onReceive?.(
|
|
1271
|
-
allData,
|
|
1272
|
-
async () => {
|
|
1273
|
-
if (outputAudioControllerRef.current) {
|
|
1274
|
-
return outputAudioControllerRef.current.stopPlayback();
|
|
1275
|
-
}
|
|
1276
|
-
},
|
|
1277
|
-
async () => {
|
|
1278
|
-
if (outputAudioControllerRef.current) {
|
|
1279
|
-
return outputAudioControllerRef.current.stopPlayback();
|
|
1280
|
-
}
|
|
1281
|
-
}
|
|
1282
|
-
);
|
|
1283
|
-
} catch (err) {
|
|
1284
|
-
if (err instanceof Error) {
|
|
1285
|
-
handleError("PROCESSING", err);
|
|
1286
|
-
}
|
|
1287
|
-
}
|
|
1288
|
-
}
|
|
1289
|
-
});
|
|
1290
|
-
onStopRecording?.();
|
|
1291
|
-
} catch (err) {
|
|
1292
|
-
if (err instanceof Error) {
|
|
1293
|
-
handleError("RECORDING", err);
|
|
1294
|
-
}
|
|
1295
|
-
}
|
|
1296
|
-
}
|
|
1297
|
-
}, [
|
|
1298
|
-
onStopRecording,
|
|
1299
|
-
handleError,
|
|
1300
|
-
subscribeToSocketEventsForChunkDownstreaming,
|
|
1301
|
-
onReceive
|
|
1302
|
-
]);
|
|
1303
|
-
react.useEffect(() => {
|
|
1304
|
-
if (socketAdapterRef.current) {
|
|
1305
|
-
return;
|
|
1306
|
-
}
|
|
1307
|
-
try {
|
|
1308
|
-
const socketAdapter = socketConfig.socketAdapter ? socketConfig.socketAdapter : new VoiceSocketIOAdapter({
|
|
1309
|
-
scope,
|
|
1310
|
-
baseUrl: socketConfig.baseUrl || "",
|
|
1311
|
-
headers: socketConfig.headers
|
|
1312
|
-
});
|
|
1313
|
-
socketAdapterRef.current = socketAdapter;
|
|
1314
|
-
if (!socketAdapter.isConnected()) {
|
|
1315
|
-
hookupSocketAdapter(socketAdapter);
|
|
1316
|
-
}
|
|
1317
|
-
if (!inputAudioControllerRef.current) {
|
|
1318
|
-
inputAudioControllerRef.current = new WebAudioInputAudioController(
|
|
1319
|
-
audioConfig
|
|
1320
|
-
);
|
|
1321
|
-
}
|
|
1322
|
-
if (!outputAudioControllerRef.current) {
|
|
1323
|
-
outputAudioControllerRef.current = new WebAudioOutputAudioController();
|
|
1324
|
-
}
|
|
1325
|
-
} catch (err) {
|
|
1326
|
-
if (err instanceof Error) {
|
|
1327
|
-
handleError("READY", err);
|
|
1328
|
-
}
|
|
1329
|
-
}
|
|
1330
|
-
}, [scope, socketConfig, hookupSocketAdapter, audioConfig, handleError]);
|
|
1331
|
-
react.useEffect(() => {
|
|
1332
|
-
return () => {
|
|
1333
|
-
inputAudioControllerRef.current?.cleanup();
|
|
1334
|
-
outputAudioControllerRef.current?.cleanup();
|
|
1335
|
-
if (socketAdapterRef.current) {
|
|
1336
|
-
socketAdapterRef.current.disconnect();
|
|
1337
|
-
socketAdapterRef.current = void 0;
|
|
1338
|
-
}
|
|
1339
|
-
};
|
|
1340
|
-
}, []);
|
|
1341
|
-
return {
|
|
1342
|
-
startRecording,
|
|
1343
|
-
stopRecording,
|
|
1344
|
-
voiceAgentState,
|
|
1345
|
-
error,
|
|
1346
|
-
audioContext: inputAudioControllerRef.current?.audioContext || null,
|
|
1347
|
-
socket
|
|
1348
|
-
};
|
|
1349
|
-
}
|
|
1350
|
-
var AiCursorComponentStyle = lit.css`
|
|
1351
|
-
:host {
|
|
1352
|
-
--ai-local-cursor-size: var(--sk-ai-cursor-size, 1rem);
|
|
1353
|
-
--ai-local-cursor-label-padding: var(
|
|
1354
|
-
--sk-ai-cursor-label-padding,
|
|
1355
|
-
0.25rem 0.25rem
|
|
1356
|
-
);
|
|
1357
|
-
--ai-local-cursor-border-radius: var(--sk-ai-cursor-border-radius, 0.25rem);
|
|
1358
|
-
--ai-local-label-offset: var(--sk-ai-cursor-label-offset, 1rem);
|
|
1359
|
-
|
|
1360
|
-
--ai-local-label-font-size: var(--sk-ai-cursor-label-font-size, 12px);
|
|
1361
|
-
--ai-local-label-font-weight: var(--sk-ai-cursor-label-font-weight, bold);
|
|
1362
|
-
--ai-local-label-color: var(--sk-ai-cursor-label-color, white);
|
|
1363
|
-
--ai-local-label-background-color: var(
|
|
1364
|
-
--sk-ai-cursor-label-background-color,
|
|
1365
|
-
black
|
|
1366
|
-
);
|
|
1367
|
-
--ai-local-label-border-color: var(
|
|
1368
|
-
--sk-ai-cursor-label-border-color,
|
|
1369
|
-
white
|
|
1370
|
-
);
|
|
1371
|
-
--ai-local-label-border-width: var(
|
|
1372
|
-
--sk-ai-cursor-label-border-width,
|
|
1373
|
-
0.1rem
|
|
1374
|
-
);
|
|
1375
|
-
|
|
1376
|
-
color: black;
|
|
1377
|
-
stroke: white;
|
|
1378
|
-
position: absolute;
|
|
1379
|
-
/* Insetting in the parent element (body) */
|
|
1380
|
-
top: 0;
|
|
1381
|
-
left: 0;
|
|
1382
|
-
bottom: 0;
|
|
1383
|
-
right: 0;
|
|
1384
|
-
pointer-events: none;
|
|
1385
|
-
width: var(--ai-local-cursor-size);
|
|
1386
|
-
height: var(--ai-local-cursor-size);
|
|
1387
|
-
}
|
|
1388
|
-
|
|
1389
|
-
#cursor-graphic-parent {
|
|
1390
|
-
position: absolute;
|
|
1391
|
-
top: 0;
|
|
1392
|
-
left: 0;
|
|
1393
|
-
}
|
|
1394
|
-
|
|
1395
|
-
#label-text {
|
|
1396
|
-
position: absolute;
|
|
1397
|
-
color: white;
|
|
1398
|
-
font-size: 12px;
|
|
1399
|
-
font-weight: bold;
|
|
1400
|
-
padding: var(--ai-local-cursor-label-padding);
|
|
1401
|
-
border-radius: var(--ai-local-cursor-border-radius);
|
|
1402
|
-
|
|
1403
|
-
white-space: nowrap;
|
|
1404
|
-
overflow: hidden;
|
|
1405
|
-
text-overflow: ellipsis;
|
|
1406
|
-
|
|
1407
|
-
width: fit-content;
|
|
1408
|
-
min-width: fit-content;
|
|
1409
|
-
top: var(--ai-local-label-offset);
|
|
1410
|
-
left: var(--ai-local-label-offset);
|
|
1411
|
-
|
|
1412
|
-
border: var(--ai-local-label-border-width) solid
|
|
1413
|
-
var(--ai-local-label-border-color);
|
|
1414
|
-
background-color: var(--ai-local-label-background-color);
|
|
1415
|
-
color: var(--ai-local-label-color);
|
|
1416
|
-
font-size: var(--ai-local-label-font-size);
|
|
1417
|
-
font-weight: var(--ai-local-label-font-weight);
|
|
1418
|
-
}
|
|
1419
|
-
`;
|
|
1420
|
-
|
|
1421
|
-
// src/ui/ai-cursor/rendering/AiCursorComponent.ts
|
|
1422
|
-
var AiCursorComponent = class extends lit.LitElement {
|
|
1423
|
-
constructor() {
|
|
1424
|
-
super();
|
|
1425
|
-
this.eventHooks = {
|
|
1426
|
-
defineSetPosition: () => {
|
|
1427
|
-
},
|
|
1428
|
-
defineAddPositionToQueue: () => {
|
|
1429
|
-
},
|
|
1430
|
-
definePlayQueue: () => {
|
|
1431
|
-
},
|
|
1432
|
-
defineSetShowCursor: () => {
|
|
1433
|
-
}
|
|
1434
|
-
};
|
|
1435
|
-
this.isShowingCursor = true;
|
|
1436
|
-
this.labelText = "AI Cursor";
|
|
1437
|
-
this.cursorPosition = [0, 0];
|
|
1438
|
-
this._cursorRef = ref_js.createRef();
|
|
1439
|
-
this._labelRef = ref_js.createRef();
|
|
1440
|
-
}
|
|
1441
|
-
updated(_changedProperties) {
|
|
1442
|
-
if (_changedProperties.has("_cursorRef")) {
|
|
1443
|
-
if (this._cursorRef.value) {
|
|
1444
|
-
this.hookUpCallbacks();
|
|
1445
|
-
} else {
|
|
1446
|
-
this._timeline?.pause();
|
|
1447
|
-
this._timeline?.refresh();
|
|
1448
|
-
}
|
|
1449
|
-
}
|
|
1450
|
-
super.updated(_changedProperties);
|
|
1451
|
-
}
|
|
1452
|
-
render() {
|
|
1453
|
-
const cursorSvg = lit.html`
|
|
1454
|
-
<svg
|
|
1455
|
-
width=${24}
|
|
1456
|
-
height=${24}
|
|
1457
|
-
viewBox="0 0 100 100"
|
|
1458
|
-
fill="none"
|
|
1459
|
-
xmlns="http://www.w3.org/2000/svg"
|
|
1460
|
-
>
|
|
1461
|
-
<g clip-path="url(#clip0_3576_285)">
|
|
1462
|
-
<path
|
|
1463
|
-
class="cursor-path"
|
|
1464
|
-
d="M2.14849 7.04749C1.35153 4.07321 4.07319 1.35155 7.04747 2.14851L77.3148 20.9766C80.2891 21.7735 81.2853 25.4914 79.108 27.6687L27.6687 79.108C25.4914 81.2853 21.7735 80.2891 20.9766 77.3149L2.14849 7.04749Z"
|
|
1465
|
-
fill="currentColor"
|
|
1466
|
-
/>
|
|
1467
|
-
</g>
|
|
1468
|
-
<defs>
|
|
1469
|
-
<clipPath id="clip0_3576_285">
|
|
1470
|
-
<rect width="100" height="100" fill="white" />
|
|
1471
|
-
</clipPath>
|
|
1472
|
-
</defs>
|
|
1473
|
-
</svg>
|
|
1474
|
-
`;
|
|
1475
|
-
return lit.html`
|
|
1476
|
-
<span
|
|
1477
|
-
id="cursor-graphic-parent"
|
|
1478
|
-
${ref_js.ref(this._cursorRef)}
|
|
1479
|
-
?hidden=${!this.isShowingCursor}
|
|
1480
|
-
>
|
|
1481
|
-
${cursorSvg}
|
|
1482
|
-
<span
|
|
1483
|
-
${ref_js.ref(this._labelRef)}
|
|
1484
|
-
id="label-text"
|
|
1485
|
-
?hidden=${!this.isShowingCursor}
|
|
1486
|
-
>${this.labelText}</span
|
|
1487
|
-
>
|
|
1488
|
-
</span>
|
|
1489
|
-
`;
|
|
1490
|
-
}
|
|
1491
|
-
// private methods
|
|
1492
|
-
/**
|
|
1493
|
-
* The primary way to control the cursor is using an external API.
|
|
1494
|
-
* This interface exposes controlling methods. The Lit Component itself is
|
|
1495
|
-
* intended to be a controlled component.
|
|
1496
|
-
*/
|
|
1497
|
-
hookUpCallbacks() {
|
|
1498
|
-
const animationTarget = this._cursorRef.value;
|
|
1499
|
-
if (!animationTarget) {
|
|
1500
|
-
return;
|
|
1501
|
-
}
|
|
1502
|
-
this._timeline = animejs.createTimeline({ defaults: { duration: 750 } });
|
|
1503
|
-
if (!this._timeline) {
|
|
1504
|
-
return;
|
|
1505
|
-
}
|
|
1506
|
-
this.eventHooks.defineSetPosition((position) => {
|
|
1507
|
-
this._timeline?.add(animationTarget, {
|
|
1508
|
-
translateX: position[0],
|
|
1509
|
-
translateY: position[1],
|
|
1510
|
-
duration: 1
|
|
1511
|
-
});
|
|
1512
|
-
this._timeline?.play();
|
|
1513
|
-
});
|
|
1514
|
-
this.eventHooks.defineAddPositionToQueue((position) => {
|
|
1515
|
-
this._timeline?.add(animationTarget, {
|
|
1516
|
-
translateX: position[0],
|
|
1517
|
-
translateY: position[1],
|
|
1518
|
-
duration: 1e3
|
|
1519
|
-
});
|
|
1520
|
-
});
|
|
1521
|
-
this.eventHooks.defineSetShowCursor((show) => {
|
|
1522
|
-
this.isShowingCursor = show;
|
|
1523
|
-
});
|
|
1524
|
-
this.eventHooks.definePlayQueue(() => {
|
|
1525
|
-
this._timeline?.play();
|
|
1526
|
-
});
|
|
1527
|
-
}
|
|
1528
|
-
// Getters
|
|
1529
|
-
get cursorRef() {
|
|
1530
|
-
return this._cursorRef.value;
|
|
1531
|
-
}
|
|
1532
|
-
get labelRef() {
|
|
1533
|
-
return this._labelRef.value;
|
|
1534
|
-
}
|
|
1535
|
-
};
|
|
1536
|
-
// Define scoped styles right with your component, in plain CSS
|
|
1537
|
-
AiCursorComponent.styles = AiCursorComponentStyle;
|
|
1538
|
-
__decorateClass([
|
|
1539
|
-
decorators_js.property({
|
|
1540
|
-
type: Object
|
|
1541
|
-
})
|
|
1542
|
-
], AiCursorComponent.prototype, "eventHooks", 2);
|
|
1543
|
-
__decorateClass([
|
|
1544
|
-
decorators_js.property({ type: Boolean })
|
|
1545
|
-
], AiCursorComponent.prototype, "isShowingCursor", 2);
|
|
1546
|
-
__decorateClass([
|
|
1547
|
-
decorators_js.property({ type: String })
|
|
1548
|
-
], AiCursorComponent.prototype, "labelText", 2);
|
|
1549
|
-
__decorateClass([
|
|
1550
|
-
decorators_js.property({ type: Array })
|
|
1551
|
-
], AiCursorComponent.prototype, "cursorPosition", 2);
|
|
1552
|
-
__decorateClass([
|
|
1553
|
-
decorators_js.state()
|
|
1554
|
-
], AiCursorComponent.prototype, "_cursorRef", 2);
|
|
1555
|
-
__decorateClass([
|
|
1556
|
-
decorators_js.state()
|
|
1557
|
-
], AiCursorComponent.prototype, "_labelRef", 2);
|
|
1558
|
-
AiCursorComponent = __decorateClass([
|
|
1559
|
-
decorators_js.customElement("ai-cursor")
|
|
1560
|
-
], AiCursorComponent);
|
|
1561
|
-
|
|
1562
|
-
// src/ui/ai-cursor/rendering/index.ts
|
|
1563
|
-
var mountAiCursor = (aiCursorProps) => {
|
|
1564
|
-
const root = document.body;
|
|
1565
|
-
const cursor = document.createElement("ai-cursor");
|
|
1566
|
-
cursor.eventHooks = aiCursorProps.eventHooks;
|
|
1567
|
-
root.appendChild(cursor);
|
|
1568
|
-
};
|
|
1569
|
-
|
|
1570
|
-
// src/ui/ai-cursor/AiCursor.ts
|
|
1571
|
-
var AiCursor = class _AiCursor {
|
|
1572
|
-
constructor() {
|
|
1573
|
-
}
|
|
1574
|
-
// Static constructors
|
|
1575
|
-
static spawn() {
|
|
1576
|
-
const newCursor = new _AiCursor();
|
|
1577
|
-
newCursor.mount();
|
|
1578
|
-
return newCursor;
|
|
1579
|
-
}
|
|
1580
|
-
jumpTo(target) {
|
|
1581
|
-
const position = targetToPosition(target);
|
|
1582
|
-
if (position) {
|
|
1583
|
-
this.setPosition?.(position);
|
|
1584
|
-
}
|
|
1585
|
-
}
|
|
1586
|
-
moveTo(target) {
|
|
1587
|
-
const position = targetToPosition(target);
|
|
1588
|
-
if (position) {
|
|
1589
|
-
this.addPositionToQueue?.(position);
|
|
1590
|
-
this.playQueue?.();
|
|
1591
|
-
}
|
|
1592
|
-
}
|
|
1593
|
-
scheduleMoves(targets) {
|
|
1594
|
-
targets.forEach((target) => {
|
|
1595
|
-
const position = targetToPosition(target);
|
|
1596
|
-
if (position) {
|
|
1597
|
-
this.addPositionToQueue?.(position);
|
|
1598
|
-
}
|
|
1599
|
-
});
|
|
1600
|
-
this.playQueue?.();
|
|
1601
|
-
}
|
|
1602
|
-
show() {
|
|
1603
|
-
this.setShowCursor?.(true);
|
|
1604
|
-
}
|
|
1605
|
-
hide() {
|
|
1606
|
-
this.setShowCursor?.(false);
|
|
1607
|
-
}
|
|
1608
|
-
mount() {
|
|
1609
|
-
mountAiCursor({
|
|
1610
|
-
eventHooks: {
|
|
1611
|
-
defineSetPosition: (callback) => {
|
|
1612
|
-
this.setPosition = callback;
|
|
1613
|
-
},
|
|
1614
|
-
defineAddPositionToQueue: (callback) => {
|
|
1615
|
-
this.addPositionToQueue = callback;
|
|
1616
|
-
},
|
|
1617
|
-
definePlayQueue: (callback) => {
|
|
1618
|
-
this.playQueue = callback;
|
|
1619
|
-
},
|
|
1620
|
-
defineSetShowCursor: (callback) => {
|
|
1621
|
-
this.setShowCursor = callback;
|
|
1622
|
-
}
|
|
1623
|
-
}
|
|
1624
|
-
});
|
|
1625
|
-
}
|
|
1626
|
-
};
|
|
1627
|
-
function calculateClickPositionFromElement(element) {
|
|
1628
|
-
const rect = element.getBoundingClientRect();
|
|
1629
|
-
return [rect.left + rect.width / 2, rect.top + rect.height / 2];
|
|
1630
|
-
}
|
|
1631
|
-
function targetToPosition(target) {
|
|
1632
|
-
if (Array.isArray(target) && target.length === 2 && typeof target[0] === "number" && typeof target[1] === "number") {
|
|
1633
|
-
return target;
|
|
1634
|
-
} else if (target instanceof HTMLElement) {
|
|
1635
|
-
return calculateClickPositionFromElement(target);
|
|
1636
|
-
} else if (typeof target === "string") {
|
|
1637
|
-
const element = document.querySelector(target);
|
|
1638
|
-
if (element) {
|
|
1639
|
-
return calculateClickPositionFromElement(element);
|
|
1640
|
-
}
|
|
1641
|
-
}
|
|
1642
|
-
return void 0;
|
|
1643
|
-
}
|
|
1644
|
-
|
|
1645
26
|
// src/stream/Pump.ts
|
|
1646
27
|
var Pump = class _Pump {
|
|
1647
28
|
constructor(src) {
|
|
@@ -2749,8 +1130,66 @@ var Channel = {
|
|
|
2749
1130
|
};
|
|
2750
1131
|
}
|
|
2751
1132
|
};
|
|
1133
|
+
|
|
1134
|
+
// src/helper/types/noop.ts
|
|
1135
|
+
var asyncNoop = async () => {
|
|
1136
|
+
};
|
|
1137
|
+
|
|
1138
|
+
// src/matrix/agent-network/stores/inmemory-network-store.ts
|
|
1139
|
+
var createInMemoryNetworkStore = () => {
|
|
1140
|
+
const store = /* @__PURE__ */ new Map();
|
|
1141
|
+
return {
|
|
1142
|
+
storeEvent: (contextId, runId, event) => {
|
|
1143
|
+
let byRun = store.get(contextId);
|
|
1144
|
+
if (!byRun) {
|
|
1145
|
+
byRun = /* @__PURE__ */ new Map();
|
|
1146
|
+
store.set(contextId, byRun);
|
|
1147
|
+
}
|
|
1148
|
+
let events = byRun.get(runId);
|
|
1149
|
+
if (!events) {
|
|
1150
|
+
events = [];
|
|
1151
|
+
byRun.set(runId, events);
|
|
1152
|
+
}
|
|
1153
|
+
events.push(event);
|
|
1154
|
+
},
|
|
1155
|
+
getEvents: (contextId, runId) => {
|
|
1156
|
+
const events = store.get(contextId)?.get(runId);
|
|
1157
|
+
return events ? [...events] : [];
|
|
1158
|
+
},
|
|
1159
|
+
getContextEvents: (contextId) => {
|
|
1160
|
+
const byRun = store.get(contextId);
|
|
1161
|
+
const result = /* @__PURE__ */ new Map();
|
|
1162
|
+
if (byRun) {
|
|
1163
|
+
for (const [runId, events] of byRun) {
|
|
1164
|
+
result.set(runId, [...events]);
|
|
1165
|
+
}
|
|
1166
|
+
}
|
|
1167
|
+
return result;
|
|
1168
|
+
},
|
|
1169
|
+
getFullStore: () => {
|
|
1170
|
+
const result = /* @__PURE__ */ new Map();
|
|
1171
|
+
for (const [contextId, byRun] of store) {
|
|
1172
|
+
const contextMap = /* @__PURE__ */ new Map();
|
|
1173
|
+
for (const [runId, events] of byRun) {
|
|
1174
|
+
contextMap.set(runId, [...events]);
|
|
1175
|
+
}
|
|
1176
|
+
result.set(contextId, contextMap);
|
|
1177
|
+
}
|
|
1178
|
+
return result;
|
|
1179
|
+
},
|
|
1180
|
+
persist: () => asyncNoop(),
|
|
1181
|
+
load: () => asyncNoop()
|
|
1182
|
+
};
|
|
1183
|
+
};
|
|
1184
|
+
|
|
1185
|
+
// src/matrix/agent-network/event-plane.ts
|
|
2752
1186
|
var DEFAULT_CAPACITY = 16;
|
|
2753
|
-
var createEventPlane = (
|
|
1187
|
+
var createEventPlane = (options) => effect.Effect.gen(function* () {
|
|
1188
|
+
const {
|
|
1189
|
+
network,
|
|
1190
|
+
capacity = DEFAULT_CAPACITY,
|
|
1191
|
+
store = createInMemoryNetworkStore()
|
|
1192
|
+
} = options;
|
|
2754
1193
|
const channels = network.getChannels();
|
|
2755
1194
|
const pubsubs = /* @__PURE__ */ new Map();
|
|
2756
1195
|
for (const channel of channels.values()) {
|
|
@@ -2763,12 +1202,59 @@ var createEventPlane = (network, capacity = DEFAULT_CAPACITY) => effect.Effect.g
|
|
|
2763
1202
|
throw new Error(`Channel not found: ${channel}`);
|
|
2764
1203
|
return p;
|
|
2765
1204
|
};
|
|
2766
|
-
const
|
|
2767
|
-
|
|
2768
|
-
|
|
2769
|
-
|
|
2770
|
-
)
|
|
1205
|
+
const recordEvent = (envelope) => {
|
|
1206
|
+
const { contextId, runId } = envelope.meta;
|
|
1207
|
+
store.storeEvent(contextId, runId, envelope);
|
|
1208
|
+
};
|
|
1209
|
+
const publishToPubSub = (channel, envelope) => effect.PubSub.publish(getPubsub(channel), envelope);
|
|
1210
|
+
const publish = (channel, envelope) => effect.Effect.sync(() => recordEvent(envelope)).pipe(
|
|
1211
|
+
effect.Effect.flatMap(() => publishToPubSub(channel, envelope)),
|
|
1212
|
+
effect.Effect.withSpan("event.publish", {
|
|
1213
|
+
attributes: {
|
|
1214
|
+
"event.name": envelope.name,
|
|
1215
|
+
"event.payload": payloadForSpan(envelope.payload),
|
|
1216
|
+
channel,
|
|
1217
|
+
runId: envelope.meta.runId,
|
|
1218
|
+
contextId: envelope.meta.contextId
|
|
1219
|
+
}
|
|
1220
|
+
})
|
|
1221
|
+
);
|
|
1222
|
+
const publishToChannels = (targetChannels, envelope) => effect.Effect.sync(() => recordEvent(envelope)).pipe(
|
|
1223
|
+
effect.Effect.flatMap(
|
|
1224
|
+
() => effect.Effect.all(
|
|
1225
|
+
targetChannels.map((c) => publishToPubSub(c.name, envelope)),
|
|
1226
|
+
{ concurrency: "unbounded" }
|
|
1227
|
+
)
|
|
1228
|
+
),
|
|
1229
|
+
effect.Effect.map((results) => results.every(Boolean)),
|
|
1230
|
+
effect.Effect.withSpan("event.publish", {
|
|
1231
|
+
attributes: {
|
|
1232
|
+
"event.name": envelope.name,
|
|
1233
|
+
"event.payload": payloadForSpan(envelope.payload),
|
|
1234
|
+
runId: envelope.meta.runId,
|
|
1235
|
+
contextId: envelope.meta.contextId
|
|
1236
|
+
}
|
|
1237
|
+
})
|
|
1238
|
+
);
|
|
2771
1239
|
const subscribe = (channel) => effect.PubSub.subscribe(getPubsub(channel));
|
|
1240
|
+
const getRunEvents = (runId, contextId) => {
|
|
1241
|
+
return store.getEvents(contextId, runId).slice();
|
|
1242
|
+
};
|
|
1243
|
+
const getContextEvents = (contextId) => {
|
|
1244
|
+
const byRun = store.getContextEvents(contextId);
|
|
1245
|
+
const map = /* @__PURE__ */ new Map();
|
|
1246
|
+
const all = [];
|
|
1247
|
+
for (const [runId, events] of byRun) {
|
|
1248
|
+
const readonlyEvents = events.slice();
|
|
1249
|
+
map.set(runId, readonlyEvents);
|
|
1250
|
+
all.push(...readonlyEvents);
|
|
1251
|
+
}
|
|
1252
|
+
return {
|
|
1253
|
+
all,
|
|
1254
|
+
byRun: (runId) => map.get(runId) ?? [],
|
|
1255
|
+
map
|
|
1256
|
+
};
|
|
1257
|
+
};
|
|
2772
1258
|
const shutdown = effect.Effect.all([...pubsubs.values()].map(effect.PubSub.shutdown), {
|
|
2773
1259
|
concurrency: "unbounded"
|
|
2774
1260
|
}).pipe(effect.Effect.asVoid);
|
|
@@ -2776,42 +1262,77 @@ var createEventPlane = (network, capacity = DEFAULT_CAPACITY) => effect.Effect.g
|
|
|
2776
1262
|
publish,
|
|
2777
1263
|
publishToChannels,
|
|
2778
1264
|
subscribe,
|
|
1265
|
+
getRunEvents,
|
|
1266
|
+
getContextEvents,
|
|
2779
1267
|
shutdown
|
|
2780
1268
|
};
|
|
2781
1269
|
});
|
|
2782
|
-
|
|
1270
|
+
function payloadForSpan(payload, maxLen = 500) {
|
|
1271
|
+
try {
|
|
1272
|
+
const s = JSON.stringify(payload);
|
|
1273
|
+
return s.length > maxLen ? `${s.slice(0, maxLen)}...` : s;
|
|
1274
|
+
} catch {
|
|
1275
|
+
return String(payload);
|
|
1276
|
+
}
|
|
1277
|
+
}
|
|
1278
|
+
var runSubscriber = (agent, publishesTo, dequeue, plane, emitQueue, channelName) => effect.Effect.gen(function* () {
|
|
2783
1279
|
const listensTo = agent.getListensTo?.() ?? [];
|
|
1280
|
+
const agentId = agent.getId();
|
|
2784
1281
|
const processOne = () => effect.Effect.gen(function* () {
|
|
2785
1282
|
const envelope = yield* effect.Queue.take(dequeue);
|
|
2786
1283
|
if (listensTo.length > 0 && !listensTo.includes(envelope.name)) {
|
|
2787
1284
|
return;
|
|
2788
1285
|
}
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
|
|
2792
|
-
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
2799
|
-
|
|
2800
|
-
|
|
2801
|
-
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
|
|
2805
|
-
|
|
2806
|
-
|
|
2807
|
-
effect.Effect.runFork(
|
|
2808
|
-
plane.publishToChannels(publishesTo, fullEnvelope)
|
|
2809
|
-
);
|
|
2810
|
-
}
|
|
1286
|
+
const runEvents = plane.getRunEvents(
|
|
1287
|
+
envelope.meta.runId,
|
|
1288
|
+
envelope.meta.contextId
|
|
1289
|
+
);
|
|
1290
|
+
const contextEvents = plane.getContextEvents(envelope.meta.contextId);
|
|
1291
|
+
yield* effect.Effect.withSpan("agent.listen", {
|
|
1292
|
+
attributes: {
|
|
1293
|
+
agentId,
|
|
1294
|
+
"event.name": envelope.name,
|
|
1295
|
+
"event.payload": payloadForSpan(envelope.payload),
|
|
1296
|
+
...channelName !== void 0 && { channel: channelName }
|
|
1297
|
+
}
|
|
1298
|
+
})(
|
|
1299
|
+
effect.Effect.withSpan("agent.invoke", {
|
|
1300
|
+
attributes: {
|
|
1301
|
+
agentId,
|
|
1302
|
+
"event.name": envelope.name,
|
|
1303
|
+
"event.payload": payloadForSpan(envelope.payload)
|
|
2811
1304
|
}
|
|
2812
|
-
})
|
|
2813
|
-
|
|
2814
|
-
|
|
1305
|
+
})(
|
|
1306
|
+
effect.Effect.tryPromise({
|
|
1307
|
+
try: () => agent.invoke({
|
|
1308
|
+
triggerEvent: envelope,
|
|
1309
|
+
emit: (userEvent) => {
|
|
1310
|
+
const fullEnvelope = {
|
|
1311
|
+
name: userEvent.name,
|
|
1312
|
+
meta: envelope.meta,
|
|
1313
|
+
payload: userEvent.payload
|
|
1314
|
+
};
|
|
1315
|
+
if (emitQueue) {
|
|
1316
|
+
effect.Effect.runPromise(
|
|
1317
|
+
effect.Queue.offer(emitQueue, {
|
|
1318
|
+
channels: publishesTo,
|
|
1319
|
+
envelope: fullEnvelope
|
|
1320
|
+
})
|
|
1321
|
+
).catch(() => {
|
|
1322
|
+
});
|
|
1323
|
+
} else {
|
|
1324
|
+
effect.Effect.runFork(
|
|
1325
|
+
plane.publishToChannels(publishesTo, fullEnvelope)
|
|
1326
|
+
);
|
|
1327
|
+
}
|
|
1328
|
+
},
|
|
1329
|
+
runEvents,
|
|
1330
|
+
contextEvents
|
|
1331
|
+
}),
|
|
1332
|
+
catch: (e) => e
|
|
1333
|
+
})
|
|
1334
|
+
)
|
|
1335
|
+
);
|
|
2815
1336
|
}).pipe(
|
|
2816
1337
|
effect.Effect.catchAllCause(
|
|
2817
1338
|
(cause) => effect.Cause.isInterrupted(cause) ? effect.Effect.void : effect.Effect.sync(() => {
|
|
@@ -2833,7 +1354,8 @@ var run = (network, plane, options) => effect.Effect.gen(function* () {
|
|
|
2833
1354
|
reg.publishesTo,
|
|
2834
1355
|
dequeue,
|
|
2835
1356
|
plane,
|
|
2836
|
-
emitQueue
|
|
1357
|
+
emitQueue,
|
|
1358
|
+
channel.name
|
|
2837
1359
|
);
|
|
2838
1360
|
}
|
|
2839
1361
|
}
|
|
@@ -2902,7 +1424,16 @@ function streamFromDequeue(take, signal, eventFilter) {
|
|
|
2902
1424
|
};
|
|
2903
1425
|
}
|
|
2904
1426
|
function expose(network, options) {
|
|
2905
|
-
const {
|
|
1427
|
+
const {
|
|
1428
|
+
auth,
|
|
1429
|
+
select,
|
|
1430
|
+
plane: providedPlane,
|
|
1431
|
+
onRequest,
|
|
1432
|
+
triggerEvents,
|
|
1433
|
+
tracingLayer
|
|
1434
|
+
} = options;
|
|
1435
|
+
const triggerEventDef = triggerEvents?.[0];
|
|
1436
|
+
const triggerEventName = triggerEventDef?.name ?? "request";
|
|
2906
1437
|
const channels = resolveChannels(network, select);
|
|
2907
1438
|
const eventFilter = select?.events;
|
|
2908
1439
|
const mainChannel = network.getMainChannel();
|
|
@@ -2913,7 +1444,7 @@ function expose(network, options) {
|
|
|
2913
1444
|
const payload = await extractPayload(req);
|
|
2914
1445
|
const signal = req.request?.signal;
|
|
2915
1446
|
const program = effect.Effect.gen(function* () {
|
|
2916
|
-
const plane = providedPlane ?? (yield* createEventPlane(network));
|
|
1447
|
+
const plane = providedPlane ?? (yield* createEventPlane({ network, store: network.getStore() }));
|
|
2917
1448
|
if (!providedPlane) {
|
|
2918
1449
|
const emitQueue = yield* effect.Queue.unbounded();
|
|
2919
1450
|
yield* effect.Effect.fork(
|
|
@@ -2929,25 +1460,46 @@ function expose(network, options) {
|
|
|
2929
1460
|
yield* effect.Effect.sleep("10 millis");
|
|
2930
1461
|
}
|
|
2931
1462
|
const targetChannel = mainChannel?.name ?? channels[0];
|
|
2932
|
-
|
|
2933
|
-
|
|
1463
|
+
let runId = req.runId ?? crypto.randomUUID();
|
|
1464
|
+
let contextId = req.contextId ?? crypto.randomUUID();
|
|
1465
|
+
const setRunId = (id) => {
|
|
1466
|
+
runId = id;
|
|
1467
|
+
};
|
|
1468
|
+
const setContextId = (id) => {
|
|
1469
|
+
contextId = id;
|
|
1470
|
+
};
|
|
1471
|
+
const emitStartEvent = (opts) => {
|
|
1472
|
+
const meta = {
|
|
1473
|
+
runId: opts.runId,
|
|
1474
|
+
contextId: opts.contextId
|
|
1475
|
+
};
|
|
2934
1476
|
const envelope = {
|
|
2935
|
-
name:
|
|
2936
|
-
meta
|
|
2937
|
-
payload:
|
|
1477
|
+
name: opts.event.name,
|
|
1478
|
+
meta,
|
|
1479
|
+
payload: opts.event.payload
|
|
2938
1480
|
};
|
|
2939
|
-
effect.Effect.runPromise(plane.publish(targetChannel, envelope)).catch(
|
|
2940
|
-
|
|
1481
|
+
effect.Effect.runPromise(plane.publish(targetChannel, envelope)).catch(
|
|
1482
|
+
() => {
|
|
1483
|
+
}
|
|
1484
|
+
);
|
|
2941
1485
|
};
|
|
2942
1486
|
const dequeue = yield* plane.subscribe(channels[0]);
|
|
2943
1487
|
if (onRequest) {
|
|
2944
1488
|
yield* effect.Effect.tryPromise(
|
|
2945
|
-
() => Promise.resolve(
|
|
1489
|
+
() => Promise.resolve(
|
|
1490
|
+
onRequest({
|
|
1491
|
+
setRunId,
|
|
1492
|
+
setContextId,
|
|
1493
|
+
emitStartEvent,
|
|
1494
|
+
req,
|
|
1495
|
+
payload
|
|
1496
|
+
})
|
|
1497
|
+
)
|
|
2946
1498
|
);
|
|
2947
1499
|
} else if (!providedPlane) {
|
|
2948
1500
|
const envelope = {
|
|
2949
|
-
name:
|
|
2950
|
-
meta: { runId
|
|
1501
|
+
name: triggerEventName,
|
|
1502
|
+
meta: { runId, contextId },
|
|
2951
1503
|
payload
|
|
2952
1504
|
};
|
|
2953
1505
|
yield* plane.publish(targetChannel, envelope);
|
|
@@ -2960,7 +1512,8 @@ function expose(network, options) {
|
|
|
2960
1512
|
}
|
|
2961
1513
|
return stream;
|
|
2962
1514
|
});
|
|
2963
|
-
|
|
1515
|
+
const runnable = tracingLayer ? program.pipe(effect.Effect.provide(tracingLayer), effect.Effect.scoped) : program.pipe(effect.Effect.scoped);
|
|
1516
|
+
return effect.Effect.runPromise(runnable);
|
|
2964
1517
|
};
|
|
2965
1518
|
return {
|
|
2966
1519
|
protocol: "sse",
|
|
@@ -2992,6 +1545,7 @@ var AgentNetwork = class _AgentNetwork {
|
|
|
2992
1545
|
this.channels = /* @__PURE__ */ new Map();
|
|
2993
1546
|
this.agentRegistrations = /* @__PURE__ */ new Map();
|
|
2994
1547
|
this.spawnerRegistrations = [];
|
|
1548
|
+
this._store = createInMemoryNetworkStore();
|
|
2995
1549
|
}
|
|
2996
1550
|
/* ─── Public Static Factory ─── */
|
|
2997
1551
|
static setup(callback) {
|
|
@@ -3079,6 +1633,10 @@ var AgentNetwork = class _AgentNetwork {
|
|
|
3079
1633
|
getSpawnerRegistrations() {
|
|
3080
1634
|
return this.spawnerRegistrations;
|
|
3081
1635
|
}
|
|
1636
|
+
/** Store defined at network setup time. Shared across all event planes created for this network. */
|
|
1637
|
+
getStore() {
|
|
1638
|
+
return this._store;
|
|
1639
|
+
}
|
|
3082
1640
|
/**
|
|
3083
1641
|
* Expose the network as a streamable API (e.g. SSE). Returns an ExposedAPI
|
|
3084
1642
|
* that adapters (NextEndpoint, ExpressEndpoint) consume to produce streamed
|
|
@@ -3086,7 +1644,7 @@ var AgentNetwork = class _AgentNetwork {
|
|
|
3086
1644
|
*
|
|
3087
1645
|
* @example
|
|
3088
1646
|
* const api = network.expose({ protocol: "sse", auth, select });
|
|
3089
|
-
* export const GET = NextEndpoint.from(api).handler();
|
|
1647
|
+
* export const GET = NextEndpoint.from(api, { requestToContextId, requestToRunId }).handler();
|
|
3090
1648
|
*/
|
|
3091
1649
|
expose(options) {
|
|
3092
1650
|
return expose(this, options);
|
|
@@ -3104,7 +1662,11 @@ var AgentNetwork = class _AgentNetwork {
|
|
|
3104
1662
|
}
|
|
3105
1663
|
runScoped(network, capacity) {
|
|
3106
1664
|
return effect.Effect.gen(function* () {
|
|
3107
|
-
const plane = yield* createEventPlane(
|
|
1665
|
+
const plane = yield* createEventPlane({
|
|
1666
|
+
network,
|
|
1667
|
+
capacity,
|
|
1668
|
+
store: network.getStore()
|
|
1669
|
+
});
|
|
3108
1670
|
yield* effect.Effect.fork(run(network, plane));
|
|
3109
1671
|
return plane;
|
|
3110
1672
|
});
|
|
@@ -3112,7 +1674,7 @@ var AgentNetwork = class _AgentNetwork {
|
|
|
3112
1674
|
};
|
|
3113
1675
|
var EventMetaSchema = effect.Schema.Struct({
|
|
3114
1676
|
runId: effect.Schema.String,
|
|
3115
|
-
contextId: effect.Schema.
|
|
1677
|
+
contextId: effect.Schema.String,
|
|
3116
1678
|
correlationId: effect.Schema.optional(effect.Schema.String),
|
|
3117
1679
|
causationId: effect.Schema.optional(effect.Schema.String),
|
|
3118
1680
|
ts: effect.Schema.optional(effect.Schema.Number)
|
|
@@ -3135,9 +1697,7 @@ var AgentNetworkEvent = {
|
|
|
3135
1697
|
const makeBound = (meta, payload2) => effect.Effect.runSync(
|
|
3136
1698
|
decodeEnvelope({ name, meta, payload: payload2 })
|
|
3137
1699
|
);
|
|
3138
|
-
const makeEffect = (payload2) => decodePayload(payload2).pipe(
|
|
3139
|
-
effect.Effect.map((p) => ({ name, payload: p }))
|
|
3140
|
-
);
|
|
1700
|
+
const makeEffect = (payload2) => decodePayload(payload2).pipe(effect.Effect.map((p) => ({ name, payload: p })));
|
|
3141
1701
|
const makeBoundEffect = (meta, payload2) => decodeEnvelope({ name, meta, payload: payload2 });
|
|
3142
1702
|
const is = effect.Schema.is(envelopeSchema);
|
|
3143
1703
|
return {
|
|
@@ -3170,13 +1730,19 @@ var Agent = class {
|
|
|
3170
1730
|
return __privateGet(this, _listensTo);
|
|
3171
1731
|
}
|
|
3172
1732
|
async invoke(options) {
|
|
3173
|
-
const { triggerEvent, emit } = options ?? {};
|
|
1733
|
+
const { triggerEvent, emit, runEvents, contextEvents } = options ?? {};
|
|
3174
1734
|
const emitFn = emit ?? ((_event) => {
|
|
3175
1735
|
});
|
|
3176
1736
|
await __privateGet(this, _logic).call(this, {
|
|
3177
1737
|
params: __privateGet(this, _params),
|
|
3178
1738
|
triggerEvent: triggerEvent ?? void 0,
|
|
3179
|
-
emit: emitFn
|
|
1739
|
+
emit: emitFn,
|
|
1740
|
+
runEvents: runEvents ?? [],
|
|
1741
|
+
contextEvents: contextEvents ?? {
|
|
1742
|
+
all: [],
|
|
1743
|
+
byRun: () => [],
|
|
1744
|
+
map: /* @__PURE__ */ new Map()
|
|
1745
|
+
}
|
|
3180
1746
|
});
|
|
3181
1747
|
}
|
|
3182
1748
|
getId() {
|
|
@@ -3289,14 +1855,19 @@ function toSSEStream(source, signal) {
|
|
|
3289
1855
|
|
|
3290
1856
|
// src/matrix/io/adapters/next-endpoint.ts
|
|
3291
1857
|
var NextEndpoint = {
|
|
3292
|
-
from(api) {
|
|
1858
|
+
from(api, options) {
|
|
3293
1859
|
if (api.protocol !== "sse") {
|
|
3294
1860
|
throw new Error(`NextEndpoint: unsupported protocol "${api.protocol}"`);
|
|
3295
1861
|
}
|
|
1862
|
+
const { requestToContextId, requestToRunId } = options;
|
|
3296
1863
|
return {
|
|
3297
1864
|
handler() {
|
|
3298
1865
|
return async (request) => {
|
|
3299
|
-
const req = {
|
|
1866
|
+
const req = {
|
|
1867
|
+
request,
|
|
1868
|
+
contextId: requestToContextId(request),
|
|
1869
|
+
runId: requestToRunId(request)
|
|
1870
|
+
};
|
|
3300
1871
|
try {
|
|
3301
1872
|
const encoder = new TextEncoder();
|
|
3302
1873
|
const { readable, writable } = new TransformStream();
|
|
@@ -3341,12 +1912,13 @@ var NextEndpoint = {
|
|
|
3341
1912
|
|
|
3342
1913
|
// src/matrix/io/adapters/express-endpoint.ts
|
|
3343
1914
|
var ExpressEndpoint = {
|
|
3344
|
-
from(api) {
|
|
1915
|
+
from(api, options) {
|
|
3345
1916
|
if (api.protocol !== "sse") {
|
|
3346
1917
|
throw new Error(
|
|
3347
1918
|
`ExpressEndpoint: unsupported protocol "${api.protocol}"`
|
|
3348
1919
|
);
|
|
3349
1920
|
}
|
|
1921
|
+
const { requestToContextId, requestToRunId } = options;
|
|
3350
1922
|
return {
|
|
3351
1923
|
handler() {
|
|
3352
1924
|
return async (req, res) => {
|
|
@@ -3355,7 +1927,9 @@ var ExpressEndpoint = {
|
|
|
3355
1927
|
const exposeReq = {
|
|
3356
1928
|
request: { signal: controller.signal },
|
|
3357
1929
|
req,
|
|
3358
|
-
res
|
|
1930
|
+
res,
|
|
1931
|
+
contextId: requestToContextId(req),
|
|
1932
|
+
runId: requestToRunId(req)
|
|
3359
1933
|
};
|
|
3360
1934
|
try {
|
|
3361
1935
|
const encoder = new TextEncoder();
|
|
@@ -3387,6 +1961,78 @@ var ExpressEndpoint = {
|
|
|
3387
1961
|
};
|
|
3388
1962
|
}
|
|
3389
1963
|
};
|
|
1964
|
+
var randomHexString = (length) => {
|
|
1965
|
+
const chars = "abcdef0123456789";
|
|
1966
|
+
let result = "";
|
|
1967
|
+
for (let i = 0; i < length; i++) {
|
|
1968
|
+
result += chars.charAt(Math.floor(Math.random() * chars.length));
|
|
1969
|
+
}
|
|
1970
|
+
return result;
|
|
1971
|
+
};
|
|
1972
|
+
var ConsoleSpan = class {
|
|
1973
|
+
constructor(name, parent, context, links, startTime, kind, depth) {
|
|
1974
|
+
this.name = name;
|
|
1975
|
+
this.parent = parent;
|
|
1976
|
+
this.context = context;
|
|
1977
|
+
this.startTime = startTime;
|
|
1978
|
+
this.kind = kind;
|
|
1979
|
+
this.depth = depth;
|
|
1980
|
+
this._tag = "Span";
|
|
1981
|
+
this.sampled = true;
|
|
1982
|
+
this.attributes = /* @__PURE__ */ new Map();
|
|
1983
|
+
this.links = [];
|
|
1984
|
+
this.traceId = parent._tag === "Some" ? parent.value.traceId : randomHexString(32);
|
|
1985
|
+
this.spanId = randomHexString(16);
|
|
1986
|
+
this.links = Array.from(links);
|
|
1987
|
+
this.status = { _tag: "Started", startTime };
|
|
1988
|
+
}
|
|
1989
|
+
end(endTime, exit) {
|
|
1990
|
+
if (this.status._tag === "Ended")
|
|
1991
|
+
return;
|
|
1992
|
+
const startTime = this.status.startTime;
|
|
1993
|
+
const durationNs = endTime - startTime;
|
|
1994
|
+
const durationMs = Number(durationNs) / 1e6;
|
|
1995
|
+
const indent = " ".repeat(this.depth);
|
|
1996
|
+
const attrs = Object.fromEntries(this.attributes);
|
|
1997
|
+
const status = effect.Exit.isSuccess(exit) ? "ok" : "error";
|
|
1998
|
+
console.log(
|
|
1999
|
+
`${indent}[trace] ${this.name} ${durationMs.toFixed(2)}ms (${status})`,
|
|
2000
|
+
Object.keys(attrs).length > 0 ? attrs : ""
|
|
2001
|
+
);
|
|
2002
|
+
this.status = { _tag: "Ended", startTime, endTime, exit };
|
|
2003
|
+
}
|
|
2004
|
+
attribute(key, value) {
|
|
2005
|
+
this.attributes.set(key, value);
|
|
2006
|
+
}
|
|
2007
|
+
event(_name, _startTime, _attributes) {
|
|
2008
|
+
}
|
|
2009
|
+
addLinks(links) {
|
|
2010
|
+
this.links.push(...links);
|
|
2011
|
+
}
|
|
2012
|
+
};
|
|
2013
|
+
function getDepth(parent) {
|
|
2014
|
+
if (parent._tag === "None")
|
|
2015
|
+
return 0;
|
|
2016
|
+
const p = parent.value;
|
|
2017
|
+
if (p._tag === "ExternalSpan")
|
|
2018
|
+
return 0;
|
|
2019
|
+
return 1 + getDepth(p.parent);
|
|
2020
|
+
}
|
|
2021
|
+
var consoleTracer = effect.Tracer.make({
|
|
2022
|
+
span: (name, parent, context, links, startTime, kind) => new ConsoleSpan(
|
|
2023
|
+
name,
|
|
2024
|
+
parent,
|
|
2025
|
+
context,
|
|
2026
|
+
links,
|
|
2027
|
+
startTime,
|
|
2028
|
+
kind,
|
|
2029
|
+
getDepth(parent)
|
|
2030
|
+
),
|
|
2031
|
+
context: (f) => f()
|
|
2032
|
+
});
|
|
2033
|
+
var consoleTracerLayer = effect.Layer.setTracer(
|
|
2034
|
+
consoleTracer
|
|
2035
|
+
);
|
|
3390
2036
|
|
|
3391
2037
|
Object.defineProperty(exports, 'S', {
|
|
3392
2038
|
enumerable: true,
|
|
@@ -3396,29 +2042,23 @@ exports.Agent = Agent;
|
|
|
3396
2042
|
exports.AgentFactory = AgentFactory;
|
|
3397
2043
|
exports.AgentNetwork = AgentNetwork;
|
|
3398
2044
|
exports.AgentNetworkEvent = AgentNetworkEvent;
|
|
3399
|
-
exports.AiCursor = AiCursor;
|
|
3400
|
-
exports.BaseVoiceEndpointAdapter = BaseVoiceEndpointAdapter;
|
|
3401
2045
|
exports.Channel = Channel;
|
|
3402
2046
|
exports.ChannelName = ChannelName;
|
|
3403
2047
|
exports.ConfiguredChannel = ConfiguredChannel;
|
|
3404
|
-
exports.Emitter = Emitter;
|
|
3405
2048
|
exports.EventMetaSchema = EventMetaSchema;
|
|
3406
2049
|
exports.ExposeAuthError = ExposeAuthError;
|
|
3407
2050
|
exports.ExpressEndpoint = ExpressEndpoint;
|
|
3408
|
-
exports.InputAudioController = InputAudioController;
|
|
3409
2051
|
exports.NextEndpoint = NextEndpoint;
|
|
3410
2052
|
exports.Pump = Pump;
|
|
3411
2053
|
exports.Sink = Sink;
|
|
3412
2054
|
exports.SocketIoFactory = SocketIoFactory;
|
|
3413
2055
|
exports.TransformMessages = TransformMessages;
|
|
3414
|
-
exports.
|
|
3415
|
-
exports.
|
|
2056
|
+
exports.consoleTracer = consoleTracer;
|
|
2057
|
+
exports.consoleTracerLayer = consoleTracerLayer;
|
|
3416
2058
|
exports.ensureFullWords = ensureFullWords;
|
|
3417
2059
|
exports.formatSSE = formatSSE;
|
|
3418
2060
|
exports.httpStreamResponse = httpStreamResponse;
|
|
3419
2061
|
exports.isHttpStreamSink = isHttpStreamSink;
|
|
3420
2062
|
exports.toSSEStream = toSSEStream;
|
|
3421
|
-
exports.useConversation = useConversation;
|
|
3422
|
-
exports.useSocketConversation = useSocketConversation;
|
|
3423
2063
|
//# sourceMappingURL=out.js.map
|
|
3424
2064
|
//# sourceMappingURL=index.cjs.map
|