@livekit/agents 1.0.48 → 1.0.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cpu.cjs +189 -0
- package/dist/cpu.cjs.map +1 -0
- package/dist/cpu.d.cts +24 -0
- package/dist/cpu.d.ts +24 -0
- package/dist/cpu.d.ts.map +1 -0
- package/dist/cpu.js +152 -0
- package/dist/cpu.js.map +1 -0
- package/dist/cpu.test.cjs +227 -0
- package/dist/cpu.test.cjs.map +1 -0
- package/dist/cpu.test.js +204 -0
- package/dist/cpu.test.js.map +1 -0
- package/dist/inference/llm.cjs.map +1 -1
- package/dist/inference/llm.d.cts +1 -1
- package/dist/inference/llm.d.ts +1 -1
- package/dist/inference/llm.d.ts.map +1 -1
- package/dist/inference/llm.js.map +1 -1
- package/dist/inference/tts.cjs.map +1 -1
- package/dist/inference/tts.d.cts +6 -0
- package/dist/inference/tts.d.ts +6 -0
- package/dist/inference/tts.d.ts.map +1 -1
- package/dist/inference/tts.js.map +1 -1
- package/dist/version.cjs +1 -1
- package/dist/version.js +1 -1
- package/dist/voice/agent_activity.cjs +36 -8
- package/dist/voice/agent_activity.cjs.map +1 -1
- package/dist/voice/agent_activity.d.ts.map +1 -1
- package/dist/voice/agent_activity.js +37 -9
- package/dist/voice/agent_activity.js.map +1 -1
- package/dist/voice/agent_session.cjs +27 -1
- package/dist/voice/agent_session.cjs.map +1 -1
- package/dist/voice/agent_session.d.cts +6 -0
- package/dist/voice/agent_session.d.ts +6 -0
- package/dist/voice/agent_session.d.ts.map +1 -1
- package/dist/voice/agent_session.js +27 -1
- package/dist/voice/agent_session.js.map +1 -1
- package/dist/worker.cjs +6 -29
- package/dist/worker.cjs.map +1 -1
- package/dist/worker.d.ts.map +1 -1
- package/dist/worker.js +6 -19
- package/dist/worker.js.map +1 -1
- package/package.json +1 -1
- package/src/cpu.test.ts +239 -0
- package/src/cpu.ts +173 -0
- package/src/inference/llm.ts +2 -0
- package/src/inference/tts.ts +8 -1
- package/src/voice/agent_activity.ts +58 -10
- package/src/voice/agent_session.ts +33 -2
- package/src/worker.ts +34 -50
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"agent_activity.d.ts","sourceRoot":"","sources":["../../src/voice/agent_activity.ts"],"names":[],"mappings":";;AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAIpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,cAAc,
|
|
1
|
+
{"version":3,"file":"agent_activity.d.ts","sourceRoot":"","sources":["../../src/voice/agent_activity.ts"],"names":[],"mappings":";;AAIA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAIpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAmB,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAAE,KAAK,WAAW,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACvE,OAAO,EAIL,KAAK,sBAAsB,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,uBAAuB,EAC5B,KAAK,2BAA2B,EAChC,GAAG,EACH,aAAa,EAEb,KAAK,eAAe,EACpB,KAAK,UAAU,EACf,KAAK,WAAW,EAEjB,MAAM,iBAAiB,CAAC;AAazB,OAAO,EAAE,GAAG,EAAiB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EAAE,GAAG,EAAiB,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,MAAM,EAAE,IAAI,EAA0B,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,GAAG,EAAE,KAAK,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,KAAK,EAAiB,MAAM,YAAY,CAAC;AAQvD,OAAO,EAAE,KAAK,YAAY,EAAE,KAAK,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC/E,OAAO,EAEL,KAAK,aAAa,EAClB,KAAK,wBAAwB,EAC7B,KAAK,gBAAgB,EAEtB,MAAM,wBAAwB,CAAC;AAsBhC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAGlD,eAAO,MAAM,oBAAoB,kCAAyC,CAAC;AAC3E,eAAO,MAAM,cAAc,gCAAuC,CAAC;AAEnE,UAAU,WAAW;IACnB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,KAAK,CAAC;CACd;AAYD,qBAAa,aAAc,YAAW,gBAAgB;IACpD,KAAK,EAAE,KAAK,CAAC;IACb,YAAY,EAAE,YAAY,CAAC;IAE3B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,yBAAyB,CAAQ;IAEzD,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,gBAAgB,CAAC,CAAmB;IAC5C,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,aAAa,CAAC,CAAoB;IAC1C,OAAO,CAAC,iBAAiB,CAAC,CAA4C;IACtE,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,iBAAiB,CAAQ;IACjC,OAAO,CAAC,kBAAkB,CAAmB;IAC7C,OAAO,CAAC,cAAc,CAAC,CAAe;IACtC,OAAO,CAAC,WAAW,CAAuC;IAC1D,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,WAAW,CAA8B;IACjD,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,WAAW,CAAsC;IACzD,OAAO,CAAC,aAAa,CAAC,CAAS;IAG/B,OAAO,CAAC,UAAU,CAA2B;IAC7C,OAAO,CAAC,qBAAqB,CAAC,CAAuB;IAErD,gBAAgB;IAChB,SAAS,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,YAAY,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,WAAW,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,sBAAsB,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IAEpC,OAAO,CAAC,QAAQ,CAAC,2BAA2B,CACb;IAC/B,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CACb;IAChC,OAAO,CAAC,QAAQ,CAAC,4BAA4B,CACb;IAChC,OAAO,CAAC,QAAQ,CAAC,0CAA0C,CACb;IAC9C,OAAO,CAAC,QAAQ,CAAC,YAAY,CACV;gBACP,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY;IA8F9C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAStB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;YASf,aAAa;IA2H3B,IAAI,aAAa,IAAI,YAAY,GAAG,SAAS,CAE5C;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,OAAO,CAAC,cAAc;IAWtB,IAAI,GAAG,IAAI,GAAG,GAAG,aAAa,GAAG,SAAS,CAEzC;IAED,IAAI,GAAG,IAAI,GAAG,GAAG,SAAS,CAEzB;IAED,IAAI,KAAK,IAAI,WAAW,CAEvB;IAED,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED,IAAI,kBAAkB,IAAI,eAAe,GAAG,SAAS,CAEpD;IAED,IAAI,kBAAkB,IAAI,OAAO,CAGhC;IAED,IAAI,uBAAuB,IAAI,OAAO,CAGrC;IAED,IAAI,aAAa,IAAI,iBAAiB,GAAG,SAAS,CAGjD;IAED,IAAI,OAAO,IAAI,WAAW,CAEzB;IAEK,aAAa,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBlD,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAapD,aAAa,CAAC,EAAE,UAAU,EAAE,EAAE;QAAE,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI;IAUvE,gBAAgB,CAAC,WAAW,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI;IAqC/D,gBAAgB,IAAI,IAAI;IAUxB,cAAc,CACZ,OAAO,GAAE;QACP,aAAa,CAAC,EAAE,OAAO,CAAC;QACxB,eAAe,CAAC,EAAE,OAAO,CAAC;KACtB;IAaR,aAAa;IAKb,GAAG,CACD,IAAI,EAAE,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,EACrC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACnC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,YAAY,CAAC,EAAE,OAAO,CAAC;KACxB,GACA,YAAY;IAuDf,OAAO,CAAC,kBAAkB,CAqBxB;IAEF,OAAO,CAAC,OAAO;IAoBf,oBAAoB,CAAC,GAAG,EAAE,uBAAuB,GAAG,IAAI;IAmBxD,oBAAoB,CAAC,EAAE,EAAE,uBAAuB,GAAG,IAAI;IAkBvD,kCAAkC,CAAC,EAAE,EAAE,2BAA2B,GAAG,IAAI;IAoBzE,mBAAmB,CAAC,EAAE,EAAE,sBAAsB,GAAG,IAAI;IAoCrD,eAAe,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAQnC,aAAa,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAQjC,kBAAkB,CAAC,EAAE,EAAE,QAAQ,GAAG,IAAI;IAWtC,OAAO,CAAC,wBAAwB;IA8ChC,mBAAmB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IAqB1C,iBAAiB,CAAC,EAAE,EAAE,WAAW,GAAG,IAAI;IA+BxC,sBAAsB,CAAC,IAAI,EAAE,wBAAwB,GAAG,IAAI;IA0C5D,OAAO,CAAC,0BAA0B;IAOlC,OAAO,CAAC,gBAAgB;IAiDlB,WAAW,CAAC,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC,OAAO,CAAC;IA6CxD,eAAe,IAAI,WAAW;YAIhB,QAAQ;IAyCtB,OAAO,CAAC,0BAA0B;IAiClC,OAAO,CAAC,cAAc;IAItB,aAAa,CAAC,OAAO,EAAE;QACrB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,OAAO,CAAC,EAAE,WAAW,CAAC;QACtB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC;QAC/B,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GAAG,YAAY;IAkHhB,SAAS,CAAC,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,MAAM,CAAC,IAAI,CAAC;IA8C1D,OAAO,CAAC,mBAAmB;YAMb,iBAAiB;YAyHjB,OAAO;IAmIrB,OAAO,CAAC,sBAAsB,CAqY5B;IAEF,OAAO,CAAC,iBAAiB,CA2BrB;YAEU,sBAAsB;YAsBtB,2BAA2B;IA4azC,OAAO,CAAC,4BAA4B;YA+CtB,iBAAiB;IAqD/B,OAAO,CAAC,cAAc;YAiBR,oBAAoB;IAelC,OAAO,CAAC,qBAAqB;IAOvB,KAAK,CAAC,OAAO,GAAE;QAAE,YAAY,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA;KAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAoBlE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAQd,UAAU;IA0BlB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAuBd,sBAAsB;CAwCrC"}
|
|
@@ -2,7 +2,7 @@ import { Mutex } from "@livekit/mutex";
|
|
|
2
2
|
import { ROOT_CONTEXT, context as otelContext, trace } from "@opentelemetry/api";
|
|
3
3
|
import { Heap } from "heap-js";
|
|
4
4
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
5
|
-
import { ReadableStream } from "node:stream/web";
|
|
5
|
+
import { ReadableStream, TransformStream } from "node:stream/web";
|
|
6
6
|
import { ChatMessage } from "../llm/chat_context.js";
|
|
7
7
|
import {
|
|
8
8
|
LLM,
|
|
@@ -343,13 +343,27 @@ class AgentActivity {
|
|
|
343
343
|
attachAudioInput(audioStream) {
|
|
344
344
|
void this.audioStream.close();
|
|
345
345
|
this.audioStream = new MultiInputStream();
|
|
346
|
+
const aecWarmupAudioFilter = new TransformStream({
|
|
347
|
+
transform: (frame, controller) => {
|
|
348
|
+
const shouldDiscardForAecWarmup = this.agentSession.agentState === "speaking" && this.agentSession._aecWarmupRemaining > 0;
|
|
349
|
+
if (!shouldDiscardForAecWarmup) {
|
|
350
|
+
controller.enqueue(frame);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
});
|
|
346
354
|
this.audioStreamId = this.audioStream.addInputStream(audioStream);
|
|
347
|
-
|
|
348
|
-
|
|
355
|
+
if (this.realtimeSession && this.audioRecognition) {
|
|
356
|
+
const [realtimeAudioStream, recognitionAudioStream] = this.audioStream.stream.pipeThrough(aecWarmupAudioFilter).tee();
|
|
349
357
|
this.realtimeSession.setInputAudioStream(realtimeAudioStream);
|
|
350
|
-
}
|
|
351
|
-
if (this.audioRecognition) {
|
|
352
358
|
this.audioRecognition.setInputAudioStream(recognitionAudioStream);
|
|
359
|
+
} else if (this.realtimeSession) {
|
|
360
|
+
this.realtimeSession.setInputAudioStream(
|
|
361
|
+
this.audioStream.stream.pipeThrough(aecWarmupAudioFilter)
|
|
362
|
+
);
|
|
363
|
+
} else if (this.audioRecognition) {
|
|
364
|
+
this.audioRecognition.setInputAudioStream(
|
|
365
|
+
this.audioStream.stream.pipeThrough(aecWarmupAudioFilter)
|
|
366
|
+
);
|
|
353
367
|
}
|
|
354
368
|
}
|
|
355
369
|
detachAudioInput() {
|
|
@@ -545,6 +559,9 @@ class AgentActivity {
|
|
|
545
559
|
}
|
|
546
560
|
interruptByAudioActivity() {
|
|
547
561
|
var _a, _b;
|
|
562
|
+
if (this.agentSession._aecWarmupRemaining > 0) {
|
|
563
|
+
return;
|
|
564
|
+
}
|
|
548
565
|
if (this.llm instanceof RealtimeModel && this.llm.capabilities.turnDetection) {
|
|
549
566
|
return;
|
|
550
567
|
}
|
|
@@ -863,7 +880,16 @@ ${instructions}`;
|
|
|
863
880
|
speech.interrupt(force);
|
|
864
881
|
}
|
|
865
882
|
(_a = this.realtimeSession) == null ? void 0 : _a.interrupt();
|
|
866
|
-
if (
|
|
883
|
+
if (force) {
|
|
884
|
+
for (const task of this.speechTasks) {
|
|
885
|
+
task.cancel();
|
|
886
|
+
}
|
|
887
|
+
if (currentSpeech && !currentSpeech.done()) {
|
|
888
|
+
currentSpeech._markDone();
|
|
889
|
+
}
|
|
890
|
+
this.speechQueue.clear();
|
|
891
|
+
future.resolve();
|
|
892
|
+
} else if (currentSpeech === void 0) {
|
|
867
893
|
future.resolve();
|
|
868
894
|
} else {
|
|
869
895
|
currentSpeech.addDoneCallback(() => {
|
|
@@ -1231,9 +1257,7 @@ ${instructions}`;
|
|
|
1231
1257
|
audioOutput.clearBuffer();
|
|
1232
1258
|
}
|
|
1233
1259
|
replyAbortController.abort();
|
|
1234
|
-
await
|
|
1235
|
-
tasks.map((task) => task.cancelAndWait(AgentActivity.REPLY_TASK_CANCEL_TIMEOUT))
|
|
1236
|
-
);
|
|
1260
|
+
await cancelAndWait(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
|
|
1237
1261
|
let forwardedText = (textOut == null ? void 0 : textOut.text) || "";
|
|
1238
1262
|
if (audioOutput) {
|
|
1239
1263
|
const playbackEv = await audioOutput.waitForPlayout();
|
|
@@ -1863,6 +1887,10 @@ ${instructions}`;
|
|
|
1863
1887
|
const unlock = await this.lock.lock();
|
|
1864
1888
|
try {
|
|
1865
1889
|
this.cancelPreemptiveGeneration();
|
|
1890
|
+
await cancelAndWait(Array.from(this.speechTasks), AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
|
|
1891
|
+
if (this._currentSpeech && !this._currentSpeech.done()) {
|
|
1892
|
+
this._currentSpeech._markDone();
|
|
1893
|
+
}
|
|
1866
1894
|
await this._closeSessionResources();
|
|
1867
1895
|
if (this._mainTask) {
|
|
1868
1896
|
await this._mainTask.cancelAndWait();
|