@plumbus/voice-livekit 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +26 -0
- package/README.md +146 -0
- package/dist/.tsbuildinfo +1 -0
- package/dist/client/client-noise-cancellation.d.ts +15 -0
- package/dist/client/client-noise-cancellation.d.ts.map +1 -0
- package/dist/client/client-noise-cancellation.js +54 -0
- package/dist/client/client-noise-cancellation.js.map +1 -0
- package/dist/client/livekit-session.d.ts +21 -0
- package/dist/client/livekit-session.d.ts.map +1 -0
- package/dist/client/livekit-session.js +206 -0
- package/dist/client/livekit-session.js.map +1 -0
- package/dist/client/pcm-resample.d.ts +7 -0
- package/dist/client/pcm-resample.d.ts.map +1 -0
- package/dist/client/pcm-resample.js +87 -0
- package/dist/client/pcm-resample.js.map +1 -0
- package/dist/client/session-helpers.d.ts +19 -0
- package/dist/client/session-helpers.d.ts.map +1 -0
- package/dist/client/session-helpers.js +70 -0
- package/dist/client/session-helpers.js.map +1 -0
- package/dist/client.d.ts +5 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +4 -0
- package/dist/client.js.map +1 -0
- package/dist/cost/record-livekit-transport.d.ts +9 -0
- package/dist/cost/record-livekit-transport.d.ts.map +1 -0
- package/dist/cost/record-livekit-transport.js +29 -0
- package/dist/cost/record-livekit-transport.js.map +1 -0
- package/dist/credentials.d.ts +3 -0
- package/dist/credentials.d.ts.map +1 -0
- package/dist/credentials.js +8 -0
- package/dist/credentials.js.map +1 -0
- package/dist/descriptor.d.ts +3 -0
- package/dist/descriptor.d.ts.map +1 -0
- package/dist/descriptor.js +14 -0
- package/dist/descriptor.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +12 -0
- package/dist/index.js.map +1 -0
- package/dist/noise-cancellation/create-inbound-audio-stream.d.ts +9 -0
- package/dist/noise-cancellation/create-inbound-audio-stream.d.ts.map +1 -0
- package/dist/noise-cancellation/create-inbound-audio-stream.js +13 -0
- package/dist/noise-cancellation/create-inbound-audio-stream.js.map +1 -0
- package/dist/noise-cancellation/oss/client-rnnoise-processor.d.ts +2 -0
- package/dist/noise-cancellation/oss/client-rnnoise-processor.d.ts.map +1 -0
- package/dist/noise-cancellation/oss/client-rnnoise-processor.js +2 -0
- package/dist/noise-cancellation/oss/client-rnnoise-processor.js.map +1 -0
- package/dist/noise-cancellation/oss/dtln-frame-processor.d.ts +10 -0
- package/dist/noise-cancellation/oss/dtln-frame-processor.d.ts.map +1 -0
- package/dist/noise-cancellation/oss/dtln-frame-processor.js +118 -0
- package/dist/noise-cancellation/oss/dtln-frame-processor.js.map +1 -0
- package/dist/noise-cancellation/oss/rnnoise-frame-processor.d.ts +9 -0
- package/dist/noise-cancellation/oss/rnnoise-frame-processor.d.ts.map +1 -0
- package/dist/noise-cancellation/oss/rnnoise-frame-processor.js +149 -0
- package/dist/noise-cancellation/oss/rnnoise-frame-processor.js.map +1 -0
- package/dist/noise-cancellation/resolve-noise-cancellation.d.ts +5 -0
- package/dist/noise-cancellation/resolve-noise-cancellation.d.ts.map +1 -0
- package/dist/noise-cancellation/resolve-noise-cancellation.js +38 -0
- package/dist/noise-cancellation/resolve-noise-cancellation.js.map +1 -0
- package/dist/pricing.d.ts +8 -0
- package/dist/pricing.d.ts.map +1 -0
- package/dist/pricing.js +19 -0
- package/dist/pricing.js.map +1 -0
- package/dist/runtime/consume-audio-stream.d.ts +3 -0
- package/dist/runtime/consume-audio-stream.d.ts.map +1 -0
- package/dist/runtime/consume-audio-stream.js +15 -0
- package/dist/runtime/consume-audio-stream.js.map +1 -0
- package/dist/runtime/livekit-agent-worker.d.ts +32 -0
- package/dist/runtime/livekit-agent-worker.d.ts.map +1 -0
- package/dist/runtime/livekit-agent-worker.js +491 -0
- package/dist/runtime/livekit-agent-worker.js.map +1 -0
- package/dist/runtime/optional-cloud-registrations.d.ts +12 -0
- package/dist/runtime/optional-cloud-registrations.d.ts.map +1 -0
- package/dist/runtime/optional-cloud-registrations.js +67 -0
- package/dist/runtime/optional-cloud-registrations.js.map +1 -0
- package/dist/runtime/parse-participant-context.d.ts +16 -0
- package/dist/runtime/parse-participant-context.d.ts.map +1 -0
- package/dist/runtime/parse-participant-context.js +61 -0
- package/dist/runtime/parse-participant-context.js.map +1 -0
- package/dist/runtime/voice-room-worker.d.ts +55 -0
- package/dist/runtime/voice-room-worker.d.ts.map +1 -0
- package/dist/runtime/voice-room-worker.js +151 -0
- package/dist/runtime/voice-room-worker.js.map +1 -0
- package/dist/transport/livekit-transport.d.ts +37 -0
- package/dist/transport/livekit-transport.d.ts.map +1 -0
- package/dist/transport/livekit-transport.js +324 -0
- package/dist/transport/livekit-transport.js.map +1 -0
- package/dist/types.d.ts +77 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/dist/worker.d.ts +4 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +3 -0
- package/dist/worker.js.map +1 -0
- package/instructions/README.md +27 -0
- package/instructions/agent-worker.md +56 -0
- package/instructions/client-session.md +46 -0
- package/instructions/framework.md +111 -0
- package/instructions/noise-cancellation.md +55 -0
- package/package.json +83 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
function parseMetadataString(value) {
|
|
2
|
+
if (!value) {
|
|
3
|
+
return {};
|
|
4
|
+
}
|
|
5
|
+
try {
|
|
6
|
+
const parsed = JSON.parse(value);
|
|
7
|
+
if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
|
|
8
|
+
return parsed;
|
|
9
|
+
}
|
|
10
|
+
}
|
|
11
|
+
catch {
|
|
12
|
+
// ignore malformed metadata
|
|
13
|
+
}
|
|
14
|
+
return {};
|
|
15
|
+
}
|
|
16
|
+
function readStringField(sources, key) {
|
|
17
|
+
for (const source of sources) {
|
|
18
|
+
const value = source[key];
|
|
19
|
+
if (typeof value === 'string' && value.length > 0) {
|
|
20
|
+
return value;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
return undefined;
|
|
24
|
+
}
|
|
25
|
+
export function parseLiveKitParticipantContext(args) {
|
|
26
|
+
const metadata = parseMetadataString(args.participantMetadata);
|
|
27
|
+
const attributeRecord = args.participantAttributes ?? {};
|
|
28
|
+
const sources = [metadata, attributeRecord];
|
|
29
|
+
const sessionId = readStringField(sources, 'sessionId') ??
|
|
30
|
+
(args.roomName.length > 0 ? args.roomName : args.participantIdentity);
|
|
31
|
+
const userId = readStringField(sources, 'userId') ?? args.participantIdentity;
|
|
32
|
+
const tenantId = readStringField(sources, 'tenantId');
|
|
33
|
+
const projectId = readStringField(sources, 'projectId');
|
|
34
|
+
const language = readStringField(sources, 'language');
|
|
35
|
+
return {
|
|
36
|
+
userId,
|
|
37
|
+
tenantId,
|
|
38
|
+
projectId,
|
|
39
|
+
sessionId,
|
|
40
|
+
language,
|
|
41
|
+
metadata: {
|
|
42
|
+
...metadata,
|
|
43
|
+
...(tenantId ? { tenantId } : {}),
|
|
44
|
+
...(projectId ? { projectId } : {}),
|
|
45
|
+
sessionId,
|
|
46
|
+
userId,
|
|
47
|
+
...(language ? { language } : {}),
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
export function buildBrainInputFromParticipantContext(context) {
|
|
52
|
+
const brainInput = { sessionId: context.sessionId };
|
|
53
|
+
if (context.projectId) {
|
|
54
|
+
brainInput.projectId = context.projectId;
|
|
55
|
+
}
|
|
56
|
+
if (context.language) {
|
|
57
|
+
brainInput.language = context.language;
|
|
58
|
+
}
|
|
59
|
+
return brainInput;
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=parse-participant-context.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-participant-context.js","sourceRoot":"","sources":["../../src/runtime/parse-participant-context.ts"],"names":[],"mappings":"AASA,SAAS,mBAAmB,CAAC,KAAyB;IACpD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAY,CAAC;QAC5C,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,OAAO,MAAiC,CAAC;QAC3C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,4BAA4B;IAC9B,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,eAAe,CAAC,OAAkC,EAAE,GAAW;IACtE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,8BAA8B,CAAC,IAK9C;IACC,MAAM,QAAQ,GAAG,mBAAmB,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAC/D,MAAM,eAAe,GAAG,IAAI,CAAC,qBAAqB,IAAI,EAAE,CAAC;IACzD,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAE5C,MAAM,SAAS,GACb,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC;QACrC,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IAExE,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,IAAI,CAAC,mBAAmB,CAAC;IAE9E,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IACtD,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IACxD,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,UAAU,CAAC,CAAC;IAEtD,OAAO;QACL,MAAM;QACN,QAAQ;QACR,SAAS;QACT,SAAS;QACT,QAAQ;QACR,QAAQ,EAAE;YACR,GAAG,QAAQ;YACX,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,SAAS;YACT,MAAM;YACN,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qCAAqC,CACnD,OAAkC;IAElC,MAAM,UAAU,GAA4B,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;IAC7E,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;QACtB,UAAU,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAC3C,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,UAAU,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IACzC,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { ExecutionContext } from '@plumbus/core';
|
|
2
|
+
import { mergeRoomBrainInput, type VoiceProviderRegistry, VoiceSessionController } from '@plumbus/voice';
|
|
3
|
+
import { type VoiceDefinition, type VoiceProvidersConfig, type VoiceSessionBudgetConfig } from '@plumbus/voice/provider-kit';
|
|
4
|
+
import type { ConnectLiveKitWorkerArgs, LiveKitTransportProvider, LiveKitWorkerConnection } from '../types.js';
|
|
5
|
+
export { mergeRoomBrainInput };
|
|
6
|
+
export interface JoinVoiceRoomSessionOptions {
|
|
7
|
+
voice: VoiceDefinition;
|
|
8
|
+
providers: VoiceProvidersConfig;
|
|
9
|
+
roomName: string;
|
|
10
|
+
sessionId?: string;
|
|
11
|
+
createExecutionContext: (args: {
|
|
12
|
+
voiceName: string;
|
|
13
|
+
sessionId: string;
|
|
14
|
+
userId?: string;
|
|
15
|
+
tenantId?: string;
|
|
16
|
+
metadata?: Record<string, unknown>;
|
|
17
|
+
}) => ExecutionContext;
|
|
18
|
+
metadata?: Record<string, unknown>;
|
|
19
|
+
userId?: string;
|
|
20
|
+
tenantId?: string;
|
|
21
|
+
brainInput?: Record<string, unknown>;
|
|
22
|
+
signal?: AbortSignal;
|
|
23
|
+
registry?: VoiceProviderRegistry;
|
|
24
|
+
sessionBudget?: VoiceSessionBudgetConfig;
|
|
25
|
+
connectLiveKitWorker?: (transport: LiveKitTransportProvider, args: ConnectLiveKitWorkerArgs) => Promise<LiveKitWorkerConnection>;
|
|
26
|
+
}
|
|
27
|
+
export interface VoiceRoomSessionHandle {
|
|
28
|
+
sessionId: string;
|
|
29
|
+
controller: VoiceSessionController;
|
|
30
|
+
stop(): Promise<void>;
|
|
31
|
+
}
|
|
32
|
+
export declare function joinVoiceRoomSession(options: JoinVoiceRoomSessionOptions): Promise<VoiceRoomSessionHandle>;
|
|
33
|
+
export interface StartVoiceWorkerOptions {
|
|
34
|
+
voices: VoiceDefinition[];
|
|
35
|
+
providers: VoiceProvidersConfig;
|
|
36
|
+
createExecutionContext: JoinVoiceRoomSessionOptions['createExecutionContext'];
|
|
37
|
+
roomName?: string;
|
|
38
|
+
sessionId?: string;
|
|
39
|
+
voiceName?: string;
|
|
40
|
+
metadata?: Record<string, unknown>;
|
|
41
|
+
userId?: string;
|
|
42
|
+
tenantId?: string;
|
|
43
|
+
brainInput?: Record<string, unknown>;
|
|
44
|
+
signal?: AbortSignal;
|
|
45
|
+
registry?: VoiceProviderRegistry;
|
|
46
|
+
sessionBudget?: VoiceSessionBudgetConfig;
|
|
47
|
+
connectLiveKitWorker?: (transport: LiveKitTransportProvider, args: ConnectLiveKitWorkerArgs) => Promise<LiveKitWorkerConnection>;
|
|
48
|
+
}
|
|
49
|
+
export interface VoiceWorkerHandle {
|
|
50
|
+
sessionId: string;
|
|
51
|
+
stop(): Promise<void>;
|
|
52
|
+
}
|
|
53
|
+
/** @deprecated Prefer joinVoiceRoomSession or startVoiceAgentWorker for per-room dispatch. */
|
|
54
|
+
export declare function startVoiceWorker(options: StartVoiceWorkerOptions): Promise<VoiceWorkerHandle>;
|
|
55
|
+
//# sourceMappingURL=voice-room-worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-room-worker.d.ts","sourceRoot":"","sources":["../../src/runtime/voice-room-worker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEtD,OAAO,EAIL,mBAAmB,EACnB,KAAK,qBAAqB,EAC1B,sBAAsB,EAEvB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,oBAAoB,EACzB,KAAK,wBAAwB,EAC9B,MAAM,6BAA6B,CAAC;AAGrC,OAAO,KAAK,EACV,wBAAwB,EACxB,wBAAwB,EACxB,uBAAuB,EACxB,MAAM,aAAa,CAAC;AAErB,OAAO,EAAE,mBAAmB,EAAE,CAAC;AAE/B,MAAM,WAAW,2BAA2B;IAC1C,KAAK,EAAE,eAAe,CAAC;IACvB,SAAS,EAAE,oBAAoB,CAAC;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,sBAAsB,EAAE,CAAC,IAAI,EAAE;QAC7B,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,KAAK,gBAAgB,CAAC;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,aAAa,CAAC,EAAE,wBAAwB,CAAC;IACzC,oBAAoB,CAAC,EAAE,CACrB,SAAS,EAAE,wBAAwB,EACnC,IAAI,EAAE,wBAAwB,KAC3B,OAAO,CAAC,uBAAuB,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,sBAAsB,CAAC;IACnC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB;AAuBD,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,2BAA2B,GACnC,OAAO,CAAC,sBAAsB,CAAC,CAuGjC;AASD,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,SAAS,EAAE,oBAAoB,CAAC;IAChC,sBAAsB,EAAE,2BAA2B,CAAC,wBAAwB,CAAC,CAAC;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,qBAAqB,CAAC;IACjC,aAAa,CAAC,EAAE,wBAAwB,CAAC;IACzC,oBAAoB,CAAC,EAAE,CACrB,SAAS,EAAE,wBAAwB,EACnC,IAAI,EAAE,wBAAwB,KAC3B,OAAO,CAAC,uBAAuB,CAAC,CAAC;CACvC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACvB;AAED,8FAA8F;AAC9F,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,uBAAuB,GAC/B,OAAO,CAAC,iBAAiB,CAAC,CAsC5B"}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { ErrorCode, PlumbusError } from '@plumbus/core';
|
|
3
|
+
import { createSTTProvider, createTransportProvider, createTTSProvider, mergeRoomBrainInput, VoiceSessionController, validateVoiceProviders, } from '@plumbus/voice';
|
|
4
|
+
import { createVoiceSessionBudget, } from '@plumbus/voice/provider-kit';
|
|
5
|
+
import { DEFAULT_AGENT_AUDIO_TRACK_NAME } from '../client/session-helpers.js';
|
|
6
|
+
import { recordLiveKitTransportCost } from '../cost/record-livekit-transport.js';
|
|
7
|
+
export { mergeRoomBrainInput };
|
|
8
|
+
function assertLiveKitTransport(registry) {
|
|
9
|
+
if (registry.transport.has('livekit')) {
|
|
10
|
+
return registry;
|
|
11
|
+
}
|
|
12
|
+
throw new PlumbusError(ErrorCode.DependencyViolation, 'LiveKit room session registry must include transport.livekit (LIVEKIT_TRANSPORT_REGISTRATION)', { installPackage: '@plumbus/voice-livekit' });
|
|
13
|
+
}
|
|
14
|
+
function resolveRoomRegistry(existing) {
|
|
15
|
+
if (!existing) {
|
|
16
|
+
throw new PlumbusError(ErrorCode.DependencyViolation, 'LiveKit room session requires an explicit registry (pass LIVEKIT_TRANSPORT_REGISTRATION and STT/TTS registrations via createProviderRegistry, or export voiceProviderRegistry from app/voice/registry.ts)');
|
|
17
|
+
}
|
|
18
|
+
return assertLiveKitTransport(existing);
|
|
19
|
+
}
|
|
20
|
+
export async function joinVoiceRoomSession(options) {
|
|
21
|
+
const registry = resolveRoomRegistry(options.registry);
|
|
22
|
+
const validation = validateVoiceProviders({
|
|
23
|
+
voices: [options.voice],
|
|
24
|
+
providers: options.providers,
|
|
25
|
+
registry,
|
|
26
|
+
});
|
|
27
|
+
if (!validation.ok) {
|
|
28
|
+
const detail = validation.issues.map((issue) => issue.message).join('; ');
|
|
29
|
+
throw new PlumbusError(ErrorCode.Validation, `Voice room session cannot start with invalid providers: ${detail}`);
|
|
30
|
+
}
|
|
31
|
+
const sessionId = options.sessionId ?? options.roomName;
|
|
32
|
+
const connectedAt = new Date();
|
|
33
|
+
const metadata = options.metadata ?? {};
|
|
34
|
+
const userId = options.userId ??
|
|
35
|
+
(typeof metadata.userId === 'string' ? metadata.userId : undefined) ??
|
|
36
|
+
'voice-worker';
|
|
37
|
+
const ctx = options.createExecutionContext({
|
|
38
|
+
voiceName: options.voice.name,
|
|
39
|
+
sessionId,
|
|
40
|
+
userId,
|
|
41
|
+
tenantId: options.tenantId ?? (typeof metadata.tenantId === 'string' ? metadata.tenantId : undefined),
|
|
42
|
+
metadata,
|
|
43
|
+
});
|
|
44
|
+
const transport = createTransportProvider({
|
|
45
|
+
registry,
|
|
46
|
+
providers: options.providers,
|
|
47
|
+
voiceSlice: options.voice.transport,
|
|
48
|
+
});
|
|
49
|
+
const sttProvider = createSTTProvider({
|
|
50
|
+
registry,
|
|
51
|
+
providers: options.providers,
|
|
52
|
+
voiceSlice: options.voice.stt,
|
|
53
|
+
});
|
|
54
|
+
const ttsProvider = createTTSProvider({
|
|
55
|
+
registry,
|
|
56
|
+
providers: options.providers,
|
|
57
|
+
voiceSlice: options.voice.tts,
|
|
58
|
+
});
|
|
59
|
+
const budget = createVoiceSessionBudget(options.sessionBudget);
|
|
60
|
+
const controller = new VoiceSessionController({
|
|
61
|
+
voice: options.voice,
|
|
62
|
+
sessionId,
|
|
63
|
+
userId,
|
|
64
|
+
ctx,
|
|
65
|
+
sttProvider,
|
|
66
|
+
ttsProvider,
|
|
67
|
+
transportProvider: transport,
|
|
68
|
+
budget,
|
|
69
|
+
brainInput: mergeRoomBrainInput(options.brainInput, metadata),
|
|
70
|
+
onEvent: async (event) => {
|
|
71
|
+
await transport.sendData(event);
|
|
72
|
+
},
|
|
73
|
+
onAudioChunk: async (chunk) => {
|
|
74
|
+
await transport.publishAudio(chunk);
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
const workerArgs = {
|
|
78
|
+
voiceName: options.voice.name,
|
|
79
|
+
room: options.roomName,
|
|
80
|
+
identity: `voice-agent:${options.voice.name}`,
|
|
81
|
+
metadata: JSON.stringify(metadata),
|
|
82
|
+
agentAudioTrackName: resolveAgentTrackName(options.voice),
|
|
83
|
+
signal: options.signal,
|
|
84
|
+
onAudio: async (audio) => {
|
|
85
|
+
await controller.handleAudioChunk(audio, options.voice.transport.audioFormat);
|
|
86
|
+
},
|
|
87
|
+
onData: async (payload) => {
|
|
88
|
+
if (!payload || typeof payload !== 'object')
|
|
89
|
+
return;
|
|
90
|
+
await controller.handleControlMessage(payload);
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
const connection = options.connectLiveKitWorker
|
|
94
|
+
? await options.connectLiveKitWorker(transport, workerArgs)
|
|
95
|
+
: await transport.connectWorker(workerArgs);
|
|
96
|
+
await controller.hello();
|
|
97
|
+
return {
|
|
98
|
+
sessionId,
|
|
99
|
+
controller,
|
|
100
|
+
async stop() {
|
|
101
|
+
const disconnectedAt = new Date();
|
|
102
|
+
await recordLiveKitTransportCost(ctx, {
|
|
103
|
+
sessionId,
|
|
104
|
+
connectedAt,
|
|
105
|
+
disconnectedAt,
|
|
106
|
+
});
|
|
107
|
+
await controller.dispose();
|
|
108
|
+
await connection.disconnect();
|
|
109
|
+
},
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
function resolveAgentTrackName(voice) {
|
|
113
|
+
const configured = voice.transport.options?.agentAudioTrackName;
|
|
114
|
+
return typeof configured === 'string' && configured.length > 0
|
|
115
|
+
? configured
|
|
116
|
+
: DEFAULT_AGENT_AUDIO_TRACK_NAME;
|
|
117
|
+
}
|
|
118
|
+
/** @deprecated Prefer joinVoiceRoomSession or startVoiceAgentWorker for per-room dispatch. */
|
|
119
|
+
export async function startVoiceWorker(options) {
|
|
120
|
+
const livekitVoices = options.voices.filter((voice) => voice.transport.provider === 'livekit');
|
|
121
|
+
if (livekitVoices.length === 0) {
|
|
122
|
+
throw new PlumbusError(ErrorCode.Validation, 'startVoiceWorker requires at least one livekit voice');
|
|
123
|
+
}
|
|
124
|
+
const selected = (options.voiceName
|
|
125
|
+
? livekitVoices.find((candidate) => candidate.name === options.voiceName)
|
|
126
|
+
: undefined) ?? livekitVoices[0];
|
|
127
|
+
if (!selected) {
|
|
128
|
+
throw new PlumbusError(ErrorCode.Validation, 'startVoiceWorker could not resolve a voice');
|
|
129
|
+
}
|
|
130
|
+
const roomName = options.roomName ?? `voice-${randomUUID()}`;
|
|
131
|
+
const handle = await joinVoiceRoomSession({
|
|
132
|
+
voice: selected,
|
|
133
|
+
providers: options.providers,
|
|
134
|
+
roomName,
|
|
135
|
+
sessionId: options.sessionId ?? roomName,
|
|
136
|
+
createExecutionContext: options.createExecutionContext,
|
|
137
|
+
metadata: options.metadata,
|
|
138
|
+
userId: options.userId,
|
|
139
|
+
tenantId: options.tenantId,
|
|
140
|
+
brainInput: options.brainInput,
|
|
141
|
+
signal: options.signal,
|
|
142
|
+
registry: options.registry,
|
|
143
|
+
sessionBudget: options.sessionBudget,
|
|
144
|
+
connectLiveKitWorker: options.connectLiveKitWorker,
|
|
145
|
+
});
|
|
146
|
+
return {
|
|
147
|
+
sessionId: handle.sessionId,
|
|
148
|
+
stop: () => handle.stop(),
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
//# sourceMappingURL=voice-room-worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"voice-room-worker.js","sourceRoot":"","sources":["../../src/runtime/voice-room-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,EACL,iBAAiB,EACjB,uBAAuB,EACvB,iBAAiB,EACjB,mBAAmB,EAEnB,sBAAsB,EACtB,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,wBAAwB,GAIzB,MAAM,6BAA6B,CAAC;AACrC,OAAO,EAAE,8BAA8B,EAAE,MAAM,8BAA8B,CAAC;AAC9E,OAAO,EAAE,0BAA0B,EAAE,MAAM,qCAAqC,CAAC;AAOjF,OAAO,EAAE,mBAAmB,EAAE,CAAC;AAiC/B,SAAS,sBAAsB,CAAC,QAA+B;IAC7D,IAAI,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,mBAAmB,EAC7B,+FAA+F,EAC/F,EAAE,cAAc,EAAE,wBAAwB,EAAE,CAC7C,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,QAAgC;IAC3D,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,mBAAmB,EAC7B,2MAA2M,CAC5M,CAAC;IACJ,CAAC;IACD,OAAO,sBAAsB,CAAC,QAAQ,CAAC,CAAC;AAC1C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,OAAoC;IAEpC,MAAM,QAAQ,GAAG,mBAAmB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,sBAAsB,CAAC;QACxC,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC;QACvB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,QAAQ;KACT,CAAC,CAAC;IACH,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,CAAC;QACnB,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1E,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,UAAU,EACpB,2DAA2D,MAAM,EAAE,CACpE,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,QAAQ,CAAC;IACxD,MAAM,WAAW,GAAG,IAAI,IAAI,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IACxC,MAAM,MAAM,GACV,OAAO,CAAC,MAAM;QACd,CAAC,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QACnE,cAAc,CAAC;IAEjB,MAAM,GAAG,GAAG,OAAO,CAAC,sBAAsB,CAAC;QACzC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI;QAC7B,SAAS;QACT,MAAM;QACN,QAAQ,EACN,OAAO,CAAC,QAAQ,IAAI,CAAC,OAAO,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7F,QAAQ;KACT,CAAC,CAAC;IAEH,MAAM,SAAS,GAAG,uBAAuB,CAAC;QACxC,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS;KACpC,CAA6B,CAAC;IAE/B,MAAM,WAAW,GAAG,iBAAiB,CAAC;QACpC,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG;KAC9B,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,iBAAiB,CAAC;QACpC,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,UAAU,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG;KAC9B,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,wBAAwB,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC;IAE/D,MAAM,UAAU,GAAG,IAAI,sBAAsB,CAAC;QAC5C,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS;QACT,MAAM;QACN,GAAG;QACH,WAAW;QACX,WAAW;QACX,iBAAiB,EAAE,SAAS;QAC5B,MAAM;QACN,UAAU,EAAE,mBAAmB,CAAC,OAAO,CAAC,UAAU,EAAE,QAAQ,CAAC;QAC7D,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YACvB,MAAM,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;QACD,YAAY,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAC5B,MAAM,SAAS,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;KACF,CAAC,CAAC;IAEH,MAAM,UAAU,GAA6B;QAC3C,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI;QAC7B,IAAI,EAAE,OAAO,CAAC,QAAQ;QACtB,QAAQ,EAAE,eAAe,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE;QAC7C,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC;QAClC,mBAAmB,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAAC;QACzD,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YACvB,MAAM,UAAU,CAAC,gBAAgB,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAChF,CAAC;QACD,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACxB,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,OAAO;YACpD,MAAM,UAAU,CAAC,oBAAoB,CAAC,OAAkC,CAAC,CAAC;QAC5E,CAAC;KACF,CAAC;IAEF,MAAM,UAAU,GAAG,OAAO,CAAC,oBAAoB;QAC7C,CAAC,CAAC,MAAM,OAAO,CAAC,oBAAoB,CAAC,SAAS,EAAE,UAAU,CAAC;QAC3D,CAAC,CAAC,MAAM,SAAS,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;IAE9C,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;IAEzB,OAAO;QACL,SAAS;QACT,UAAU;QACV,KAAK,CAAC,IAAI;YACR,MAAM,cAAc,GAAG,IAAI,IAAI,EAAE,CAAC;YAClC,MAAM,0BAA0B,CAAC,GAAG,EAAE;gBACpC,SAAS;gBACT,WAAW;gBACX,cAAc;aACf,CAAC,CAAC;YACH,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;YAC3B,MAAM,UAAU,CAAC,UAAU,EAAE,CAAC;QAChC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAsB;IACnD,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,mBAAmB,CAAC;IAChE,OAAO,OAAO,UAAU,KAAK,QAAQ,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAC5D,CAAC,CAAC,UAAU;QACZ,CAAC,CAAC,8BAA8B,CAAC;AACrC,CAAC;AA2BD,8FAA8F;AAC9F,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAgC;IAEhC,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC;IAC/F,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,UAAU,EACpB,sDAAsD,CACvD,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GACZ,CAAC,OAAO,CAAC,SAAS;QAChB,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,OAAO,CAAC,SAAS,CAAC;QACzE,CAAC,CAAC,SAAS,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC,CAAC;IACrC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,UAAU,EAAE,4CAA4C,CAAC,CAAC;IAC7F,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,SAAS,UAAU,EAAE,EAAE,CAAC;IAC7D,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC;QACxC,KAAK,EAAE,QAAQ;QACf,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,QAAQ;QACR,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,QAAQ;QACxC,sBAAsB,EAAE,OAAO,CAAC,sBAAsB;QACtD,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,oBAAoB,EAAE,OAAO,CAAC,oBAAoB;KACnD,CAAC,CAAC;IAEH,OAAO;QACL,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,IAAI,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE;KAC1B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { type TransportProviderRegistration, type TransportProviderSession, type VoiceProviderCredentials, type VoiceTransportConfig } from '@plumbus/voice/provider-kit';
|
|
2
|
+
import { LIVEKIT_TRANSPORT_DESCRIPTOR } from '../descriptor.js';
|
|
3
|
+
import type { ConnectLiveKitWorkerArgs, LiveKitTransportProvider as LiveKitTransportProviderContract, LiveKitWorkerConnection, MintLiveKitParticipantTokenArgs } from '../types.js';
|
|
4
|
+
export type { ConnectLiveKitWorkerArgs, LiveKitSessionMetadata, LiveKitWorkerConnection, MintLiveKitParticipantTokenArgs, } from '../types.js';
|
|
5
|
+
export { LIVEKIT_TRANSPORT_DESCRIPTOR };
|
|
6
|
+
export interface MintLiveKitSessionArgs {
|
|
7
|
+
voiceName: string;
|
|
8
|
+
userId?: string;
|
|
9
|
+
sessionId?: string;
|
|
10
|
+
roomName?: string;
|
|
11
|
+
metadata?: Record<string, unknown>;
|
|
12
|
+
attributes?: Record<string, string>;
|
|
13
|
+
tokenTtlSeconds?: number;
|
|
14
|
+
identity?: string;
|
|
15
|
+
}
|
|
16
|
+
export type LiveKitRoomResolver = (args: {
|
|
17
|
+
voiceName: string;
|
|
18
|
+
userId?: string;
|
|
19
|
+
sessionId: string;
|
|
20
|
+
}) => string;
|
|
21
|
+
export declare function mintLiveKitParticipantToken(args: MintLiveKitParticipantTokenArgs): Promise<string>;
|
|
22
|
+
export declare class LiveKitTransportProvider implements LiveKitTransportProviderContract {
|
|
23
|
+
private readonly credentials;
|
|
24
|
+
private readonly voiceSlice;
|
|
25
|
+
private activeConnection?;
|
|
26
|
+
constructor(credentials: VoiceProviderCredentials, voiceSlice: VoiceTransportConfig);
|
|
27
|
+
mintSession(args: MintLiveKitSessionArgs): Promise<TransportProviderSession>;
|
|
28
|
+
connectWorker(args: ConnectLiveKitWorkerArgs): Promise<LiveKitWorkerConnection>;
|
|
29
|
+
publishAudio(audio: Uint8Array): Promise<void>;
|
|
30
|
+
subscribeRemote(onAudio: (audio: Uint8Array) => Promise<void> | void): void;
|
|
31
|
+
sendData(payload: unknown): Promise<void>;
|
|
32
|
+
disconnect(): Promise<void>;
|
|
33
|
+
private createSessionMetadata;
|
|
34
|
+
private forwardRemoteAudio;
|
|
35
|
+
}
|
|
36
|
+
export declare const LIVEKIT_TRANSPORT_REGISTRATION: TransportProviderRegistration;
|
|
37
|
+
//# sourceMappingURL=livekit-transport.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"livekit-transport.d.ts","sourceRoot":"","sources":["../../src/transport/livekit-transport.ts"],"names":[],"mappings":"AAYA,OAAO,EAEL,KAAK,6BAA6B,EAClC,KAAK,wBAAwB,EAC7B,KAAK,wBAAwB,EAC7B,KAAK,oBAAoB,EAC1B,MAAM,6BAA6B,CAAC;AAErC,OAAO,EAAE,4BAA4B,EAAE,MAAM,kBAAkB,CAAC;AAIhE,OAAO,KAAK,EACV,wBAAwB,EAExB,wBAAwB,IAAI,gCAAgC,EAC5D,uBAAuB,EACvB,+BAA+B,EAChC,MAAM,aAAa,CAAC;AAErB,YAAY,EACV,wBAAwB,EACxB,sBAAsB,EACtB,uBAAuB,EACvB,+BAA+B,GAChC,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,4BAA4B,EAAE,CAAC;AASxC,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,mBAAmB,GAAG,CAAC,IAAI,EAAE;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB,KAAK,MAAM,CAAC;AAEb,wBAAsB,2BAA2B,CAC/C,IAAI,EAAE,+BAA+B,GACpC,OAAO,CAAC,MAAM,CAAC,CA0BjB;AAED,qBAAa,wBAAyB,YAAW,gCAAgC;IAU7E,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAV7B,OAAO,CAAC,gBAAgB,CAAC,CAMvB;gBAGiB,WAAW,EAAE,wBAAwB,EACrC,UAAU,EAAE,oBAAoB;IAG7C,WAAW,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,wBAAwB,CAAC;IAqB5E,aAAa,CAAC,IAAI,EAAE,wBAAwB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAmE/E,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAcpD,eAAe,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,IAAI;IAMrE,QAAQ,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC;IAWzC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAUnB,qBAAqB;YAuDrB,kBAAkB;CAWjC;AAED,eAAO,MAAM,8BAA8B,EAAE,6BAuB5C,CAAC"}
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto';
|
|
2
|
+
import { AudioFrame, AudioSource, LocalAudioTrack, Room, RoomEvent, TrackKind, TrackPublishOptions, } from '@livekit/rtc-node';
|
|
3
|
+
import { ErrorCode, PlumbusError } from '@plumbus/core';
|
|
4
|
+
import { readNoiseCancellationFromTransportOptions, } from '@plumbus/voice/provider-kit';
|
|
5
|
+
import { AccessToken, RoomAgentDispatch, RoomConfiguration } from 'livekit-server-sdk';
|
|
6
|
+
import { LIVEKIT_TRANSPORT_DESCRIPTOR } from '../descriptor.js';
|
|
7
|
+
import { createInboundAudioStream } from '../noise-cancellation/create-inbound-audio-stream.js';
|
|
8
|
+
import { LIVEKIT_VOICE_PRICING } from '../pricing.js';
|
|
9
|
+
import { consumeAudioStream } from '../runtime/consume-audio-stream.js';
|
|
10
|
+
export { LIVEKIT_TRANSPORT_DESCRIPTOR };
|
|
11
|
+
const DEFAULT_AUDIO_FORMAT = 'pcm16;rate=16000;channels=1';
|
|
12
|
+
const DEFAULT_DATA_TOPIC = 'voice.events';
|
|
13
|
+
const DEFAULT_SESSION_TTL_SECONDS = 3600;
|
|
14
|
+
const MAX_SESSION_TTL_SECONDS = 7200;
|
|
15
|
+
const DEFAULT_AGENT_AUDIO_TRACK_NAME = 'agent-voice';
|
|
16
|
+
const DEFAULT_ROOM_USER = 'voiceUser';
|
|
17
|
+
export async function mintLiveKitParticipantToken(args) {
|
|
18
|
+
const accessToken = new AccessToken(args.apiKey, args.apiSecret, {
|
|
19
|
+
identity: args.identity,
|
|
20
|
+
name: args.participantName ?? args.identity,
|
|
21
|
+
ttl: clampTokenTtlSeconds(args.ttlSeconds ?? DEFAULT_SESSION_TTL_SECONDS),
|
|
22
|
+
metadata: args.metadata,
|
|
23
|
+
attributes: sanitizeAttributes(args.attributes),
|
|
24
|
+
});
|
|
25
|
+
accessToken.addGrant({
|
|
26
|
+
room: args.room,
|
|
27
|
+
roomJoin: true,
|
|
28
|
+
canPublish: true,
|
|
29
|
+
canSubscribe: true,
|
|
30
|
+
canPublishData: true,
|
|
31
|
+
});
|
|
32
|
+
if (args.agentName) {
|
|
33
|
+
accessToken.roomConfig = new RoomConfiguration({
|
|
34
|
+
agents: [
|
|
35
|
+
new RoomAgentDispatch({
|
|
36
|
+
agentName: args.agentName,
|
|
37
|
+
metadata: args.agentMetadata ?? args.metadata ?? '',
|
|
38
|
+
}),
|
|
39
|
+
],
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
return accessToken.toJwt();
|
|
43
|
+
}
|
|
44
|
+
export class LiveKitTransportProvider {
|
|
45
|
+
credentials;
|
|
46
|
+
voiceSlice;
|
|
47
|
+
activeConnection;
|
|
48
|
+
constructor(credentials, voiceSlice) {
|
|
49
|
+
this.credentials = credentials;
|
|
50
|
+
this.voiceSlice = voiceSlice;
|
|
51
|
+
}
|
|
52
|
+
async mintSession(args) {
|
|
53
|
+
const sessionId = args.sessionId ?? `livekit:${args.voiceName}:${args.userId ?? 'anonymous'}:${randomUUID()}`;
|
|
54
|
+
const serializedMetadata = serializeMetadata(args.metadata);
|
|
55
|
+
const metadata = await this.createSessionMetadata(args.voiceName, args.userId, {
|
|
56
|
+
room: args.roomName,
|
|
57
|
+
identity: args.identity,
|
|
58
|
+
ttlSeconds: args.tokenTtlSeconds,
|
|
59
|
+
metadata: serializedMetadata,
|
|
60
|
+
attributes: args.attributes,
|
|
61
|
+
sessionId,
|
|
62
|
+
agentName: args.voiceName,
|
|
63
|
+
agentMetadata: serializedMetadata,
|
|
64
|
+
});
|
|
65
|
+
return {
|
|
66
|
+
sessionId,
|
|
67
|
+
transport: 'livekit',
|
|
68
|
+
metadata: metadata,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
async connectWorker(args) {
|
|
72
|
+
if (args.signal?.aborted) {
|
|
73
|
+
throw new PlumbusError(ErrorCode.Cancelled, 'LiveKit worker connection aborted before connect');
|
|
74
|
+
}
|
|
75
|
+
const metadata = await this.createSessionMetadata(args.voiceName, args.identity, {
|
|
76
|
+
room: args.room,
|
|
77
|
+
identity: args.identity,
|
|
78
|
+
audioTrackName: args.agentAudioTrackName ?? args.audioTrackName,
|
|
79
|
+
ttlSeconds: resolveWorkerTokenTtlSeconds(this.voiceSlice.options),
|
|
80
|
+
token: args.token,
|
|
81
|
+
participantName: args.participantName,
|
|
82
|
+
metadata: args.metadata,
|
|
83
|
+
attributes: args.attributes,
|
|
84
|
+
});
|
|
85
|
+
const room = new Room();
|
|
86
|
+
await room.connect(metadata.url, metadata.token);
|
|
87
|
+
const audioSource = createAudioSource(metadata.audioFormat);
|
|
88
|
+
const localTrack = LocalAudioTrack.createAudioTrack(metadata.audioTrackName, audioSource);
|
|
89
|
+
const publishOptions = new TrackPublishOptions();
|
|
90
|
+
await room.localParticipant?.publishTrack(localTrack, publishOptions);
|
|
91
|
+
const activeConnection = {
|
|
92
|
+
room,
|
|
93
|
+
audioSource,
|
|
94
|
+
localTrack,
|
|
95
|
+
dataTopic: args.dataTopic ?? DEFAULT_DATA_TOPIC,
|
|
96
|
+
onAudio: args.onAudio,
|
|
97
|
+
};
|
|
98
|
+
this.activeConnection = activeConnection;
|
|
99
|
+
if (args.signal) {
|
|
100
|
+
args.signal.addEventListener('abort', () => {
|
|
101
|
+
void this.disconnect();
|
|
102
|
+
}, { once: true });
|
|
103
|
+
}
|
|
104
|
+
room.on(RoomEvent.TrackSubscribed, (track) => {
|
|
105
|
+
if (!this.activeConnection)
|
|
106
|
+
return;
|
|
107
|
+
const isAudio = track.kind === TrackKind.KIND_AUDIO || track.kind === 1;
|
|
108
|
+
if (!isAudio || !this.activeConnection.onAudio)
|
|
109
|
+
return;
|
|
110
|
+
void this.forwardRemoteAudio(track, this.activeConnection.onAudio);
|
|
111
|
+
});
|
|
112
|
+
room.on(RoomEvent.DataReceived, (payload) => {
|
|
113
|
+
if (!args.onData)
|
|
114
|
+
return;
|
|
115
|
+
void args.onData(parseLiveKitDataPayload(payload));
|
|
116
|
+
});
|
|
117
|
+
return {
|
|
118
|
+
room,
|
|
119
|
+
disconnect: async () => {
|
|
120
|
+
await this.disconnect();
|
|
121
|
+
},
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
async publishAudio(audio) {
|
|
125
|
+
const connection = this.activeConnection;
|
|
126
|
+
if (!connection)
|
|
127
|
+
return;
|
|
128
|
+
const format = parsePcmFormat(this.voiceSlice.audioFormat);
|
|
129
|
+
const audioFrame = new AudioFrame(pcmBytesToInt16(audio), format.sampleRate, format.channels, Math.max(1, audio.byteLength / (2 * format.channels)));
|
|
130
|
+
await connection.audioSource.captureFrame(audioFrame);
|
|
131
|
+
}
|
|
132
|
+
subscribeRemote(onAudio) {
|
|
133
|
+
if (this.activeConnection) {
|
|
134
|
+
this.activeConnection.onAudio = onAudio;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
async sendData(payload) {
|
|
138
|
+
const connection = this.activeConnection;
|
|
139
|
+
if (!connection?.room.localParticipant)
|
|
140
|
+
return;
|
|
141
|
+
const encoded = new TextEncoder().encode(JSON.stringify(payload));
|
|
142
|
+
await connection.room.localParticipant.publishData(encoded, {
|
|
143
|
+
reliable: true,
|
|
144
|
+
topic: connection.dataTopic,
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
async disconnect() {
|
|
148
|
+
const connection = this.activeConnection;
|
|
149
|
+
this.activeConnection = undefined;
|
|
150
|
+
if (!connection)
|
|
151
|
+
return;
|
|
152
|
+
await connection.localTrack.close();
|
|
153
|
+
await connection.audioSource.close();
|
|
154
|
+
await connection.room.disconnect();
|
|
155
|
+
}
|
|
156
|
+
async createSessionMetadata(voiceName, userId, overrides = {}) {
|
|
157
|
+
const url = requireString(this.credentials.url, 'LiveKit url');
|
|
158
|
+
const apiKey = requireString(this.credentials.apiKey, 'LiveKit apiKey');
|
|
159
|
+
const apiSecret = requireString(this.credentials.apiSecret, 'LiveKit apiSecret');
|
|
160
|
+
const sessionId = overrides.sessionId ?? `livekit:${voiceName}:${userId ?? 'anonymous'}:${randomUUID()}`;
|
|
161
|
+
const room = overrides.room ?? resolveRoomName(this.voiceSlice, voiceName, userId, sessionId);
|
|
162
|
+
const identity = overrides.identity ?? resolveIdentity(voiceName, userId);
|
|
163
|
+
const audioFormat = normalizeAudioFormat(this.voiceSlice.audioFormat);
|
|
164
|
+
const audioTrackName = overrides.audioTrackName ?? resolveAgentAudioTrackName(this.voiceSlice, voiceName, identity);
|
|
165
|
+
let token = overrides.token;
|
|
166
|
+
if (!token) {
|
|
167
|
+
token = await mintLiveKitParticipantToken({
|
|
168
|
+
apiKey,
|
|
169
|
+
apiSecret,
|
|
170
|
+
room,
|
|
171
|
+
identity,
|
|
172
|
+
participantName: overrides.participantName ?? identity,
|
|
173
|
+
ttlSeconds: overrides.ttlSeconds ?? resolveLiveKitTokenTtlSeconds(this.voiceSlice.options),
|
|
174
|
+
metadata: overrides.metadata,
|
|
175
|
+
attributes: sanitizeAttributes(overrides.attributes),
|
|
176
|
+
agentName: overrides.agentName,
|
|
177
|
+
agentMetadata: overrides.agentMetadata ?? overrides.metadata,
|
|
178
|
+
});
|
|
179
|
+
}
|
|
180
|
+
return {
|
|
181
|
+
url,
|
|
182
|
+
room,
|
|
183
|
+
token,
|
|
184
|
+
identity,
|
|
185
|
+
audioFormat,
|
|
186
|
+
audioTrackName,
|
|
187
|
+
mode: this.voiceSlice.mode ?? 'pushToTalk',
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
async forwardRemoteAudio(track, onAudio) {
|
|
191
|
+
const format = parsePcmFormat(this.voiceSlice.audioFormat);
|
|
192
|
+
const noiseCancellation = readNoiseCancellationFromTransportOptions(this.voiceSlice.options);
|
|
193
|
+
const stream = createInboundAudioStream(track, format, noiseCancellation);
|
|
194
|
+
await consumeAudioStream(stream, async (frame) => {
|
|
195
|
+
await onAudio(int16ToBytes(frame.data));
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
export const LIVEKIT_TRANSPORT_REGISTRATION = {
|
|
200
|
+
descriptor: LIVEKIT_TRANSPORT_DESCRIPTOR,
|
|
201
|
+
pricing: LIVEKIT_VOICE_PRICING,
|
|
202
|
+
create(credentials, voiceSlice) {
|
|
203
|
+
return new LiveKitTransportProvider(credentials, voiceSlice);
|
|
204
|
+
},
|
|
205
|
+
toClientSessionPayload(session) {
|
|
206
|
+
const metadata = session.metadata ?? {};
|
|
207
|
+
const audioTrackName = typeof metadata.audioTrackName === 'string' ? metadata.audioTrackName : undefined;
|
|
208
|
+
return {
|
|
209
|
+
transport: 'livekit',
|
|
210
|
+
url: metadata.url,
|
|
211
|
+
token: metadata.token,
|
|
212
|
+
room: metadata.room,
|
|
213
|
+
identity: metadata.identity,
|
|
214
|
+
audioTrackName,
|
|
215
|
+
agentAudioTrackName: audioTrackName,
|
|
216
|
+
audioFormat: metadata.audioFormat,
|
|
217
|
+
mode: metadata.mode,
|
|
218
|
+
sessionId: session.sessionId,
|
|
219
|
+
};
|
|
220
|
+
},
|
|
221
|
+
};
|
|
222
|
+
function createAudioSource(audioFormat) {
|
|
223
|
+
const format = parsePcmFormat(audioFormat);
|
|
224
|
+
return new AudioSource(format.sampleRate, format.channels);
|
|
225
|
+
}
|
|
226
|
+
function parsePcmFormat(audioFormat) {
|
|
227
|
+
const normalized = normalizeAudioFormat(audioFormat);
|
|
228
|
+
const rateMatch = normalized.match(/rate=(\d+)/);
|
|
229
|
+
const channelsMatch = normalized.match(/channels=(\d+)/);
|
|
230
|
+
const frameSizeMatch = normalized.match(/frame=(\d+)/);
|
|
231
|
+
return {
|
|
232
|
+
sampleRate: Number(rateMatch?.[1] ?? 16000),
|
|
233
|
+
channels: Number(channelsMatch?.[1] ?? 1),
|
|
234
|
+
frameSizeMs: Number(frameSizeMatch?.[1] ?? 20),
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
function normalizeAudioFormat(audioFormat) {
|
|
238
|
+
if (!audioFormat)
|
|
239
|
+
return DEFAULT_AUDIO_FORMAT;
|
|
240
|
+
if (audioFormat.includes('rate='))
|
|
241
|
+
return audioFormat;
|
|
242
|
+
const short = audioFormat.toLowerCase();
|
|
243
|
+
if (short === 'pcm16-16k') {
|
|
244
|
+
return DEFAULT_AUDIO_FORMAT;
|
|
245
|
+
}
|
|
246
|
+
if (short === 'pcm16-24k') {
|
|
247
|
+
return 'pcm16;rate=24000;channels=1';
|
|
248
|
+
}
|
|
249
|
+
return audioFormat;
|
|
250
|
+
}
|
|
251
|
+
function resolveRoomName(voiceSlice, voiceName, userId, sessionId) {
|
|
252
|
+
const resolver = voiceSlice.options?.roomResolver;
|
|
253
|
+
if (typeof resolver === 'function') {
|
|
254
|
+
return resolver({
|
|
255
|
+
voiceName,
|
|
256
|
+
userId: userId ?? DEFAULT_ROOM_USER,
|
|
257
|
+
sessionId,
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
const configuredRoom = voiceSlice.options?.room;
|
|
261
|
+
if (typeof configuredRoom === 'string' && configuredRoom.length > 0) {
|
|
262
|
+
return configuredRoom;
|
|
263
|
+
}
|
|
264
|
+
return `${voiceName}-${userId ?? DEFAULT_ROOM_USER}`;
|
|
265
|
+
}
|
|
266
|
+
function resolveIdentity(voiceName, userId) {
|
|
267
|
+
return `${voiceName}:${userId ?? `anonymous:${randomUUID()}`}`;
|
|
268
|
+
}
|
|
269
|
+
function resolveAgentAudioTrackName(voiceSlice, _voiceName, _identity) {
|
|
270
|
+
const configured = voiceSlice.options?.agentAudioTrackName;
|
|
271
|
+
if (typeof configured === 'string' && configured.length > 0) {
|
|
272
|
+
return configured;
|
|
273
|
+
}
|
|
274
|
+
return DEFAULT_AGENT_AUDIO_TRACK_NAME;
|
|
275
|
+
}
|
|
276
|
+
function resolveLiveKitTokenTtlSeconds(options) {
|
|
277
|
+
const value = options?.tokenTtlSeconds;
|
|
278
|
+
return typeof value === 'number' && value > 0
|
|
279
|
+
? clampTokenTtlSeconds(value)
|
|
280
|
+
: DEFAULT_SESSION_TTL_SECONDS;
|
|
281
|
+
}
|
|
282
|
+
function resolveWorkerTokenTtlSeconds(options) {
|
|
283
|
+
const value = options?.workerTokenTtlSeconds;
|
|
284
|
+
return typeof value === 'number' && value > 0
|
|
285
|
+
? clampTokenTtlSeconds(value)
|
|
286
|
+
: DEFAULT_SESSION_TTL_SECONDS;
|
|
287
|
+
}
|
|
288
|
+
function clampTokenTtlSeconds(value) {
|
|
289
|
+
return Math.min(Math.max(1, value), MAX_SESSION_TTL_SECONDS);
|
|
290
|
+
}
|
|
291
|
+
function requireString(value, label) {
|
|
292
|
+
if (!value) {
|
|
293
|
+
throw new PlumbusError(ErrorCode.Validation, `${label} is required`);
|
|
294
|
+
}
|
|
295
|
+
return value;
|
|
296
|
+
}
|
|
297
|
+
function sanitizeAttributes(attributes) {
|
|
298
|
+
if (!attributes)
|
|
299
|
+
return undefined;
|
|
300
|
+
return Object.fromEntries(Object.entries(attributes).filter((entry) => typeof entry[0] === 'string' && typeof entry[1] === 'string'));
|
|
301
|
+
}
|
|
302
|
+
function serializeMetadata(metadata) {
|
|
303
|
+
if (!metadata)
|
|
304
|
+
return undefined;
|
|
305
|
+
return JSON.stringify(metadata);
|
|
306
|
+
}
|
|
307
|
+
function pcmBytesToInt16(audio) {
|
|
308
|
+
const buffer = audio.byteOffset === 0 && audio.byteLength === audio.buffer.byteLength
|
|
309
|
+
? audio.buffer
|
|
310
|
+
: audio.buffer.slice(audio.byteOffset, audio.byteOffset + audio.byteLength);
|
|
311
|
+
return new Int16Array(buffer);
|
|
312
|
+
}
|
|
313
|
+
function int16ToBytes(frame) {
|
|
314
|
+
return new Uint8Array(frame.buffer.slice(frame.byteOffset, frame.byteOffset + frame.byteLength));
|
|
315
|
+
}
|
|
316
|
+
function parseLiveKitDataPayload(payload) {
|
|
317
|
+
try {
|
|
318
|
+
return JSON.parse(Buffer.from(payload).toString('utf-8'));
|
|
319
|
+
}
|
|
320
|
+
catch {
|
|
321
|
+
return { type: 'error', code: 'voice.invalid_message', message: 'Expected JSON control frame' };
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
//# sourceMappingURL=livekit-transport.js.map
|