@craftedxp/voice-js 0.6.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONSUMING.md +6 -2
- package/README.md +31 -4
- package/dist/assistant.d.mts +32 -0
- package/dist/assistant.d.ts +32 -0
- package/dist/assistant.js +1241 -0
- package/dist/assistant.js.map +1 -0
- package/dist/assistant.mjs +23 -0
- package/dist/assistant.mjs.map +1 -0
- package/dist/browser.d.mts +12 -573
- package/dist/browser.d.ts +12 -573
- package/dist/browser.js +222 -183
- package/dist/browser.js.map +1 -1
- package/dist/browser.mjs +23 -1373
- package/dist/browser.mjs.map +1 -1
- package/dist/chunk-LV7JGPYW.mjs +200 -0
- package/dist/chunk-LV7JGPYW.mjs.map +1 -0
- package/dist/chunk-ZW22Y67M.mjs +1208 -0
- package/dist/chunk-ZW22Y67M.mjs.map +1 -0
- package/dist/config-D2TbvIqT.d.mts +297 -0
- package/dist/config-D2TbvIqT.d.ts +297 -0
- package/dist/embed.iife.js +5 -42
- package/dist/incomingCall-CfRRzj2P.d.mts +103 -0
- package/dist/incomingCall-CfRRzj2P.d.ts +103 -0
- package/dist/node.d.mts +19 -152
- package/dist/node.d.ts +19 -152
- package/dist/node.js +19 -0
- package/dist/node.js.map +1 -1
- package/dist/node.mjs +19 -0
- package/dist/node.mjs.map +1 -1
- package/dist/room.d.mts +156 -0
- package/dist/room.d.ts +156 -0
- package/dist/room.js +236 -0
- package/dist/room.js.map +1 -0
- package/dist/room.mjs +7 -0
- package/dist/room.mjs.map +1 -0
- package/dist/transcribe.d.mts +14 -0
- package/dist/transcribe.d.ts +14 -0
- package/dist/transcribe.js +1213 -0
- package/dist/transcribe.js.map +1 -0
- package/dist/transcribe.mjs +18 -0
- package/dist/transcribe.mjs.map +1 -0
- package/package.json +21 -4
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
type OnChunk = (pcm: ArrayBuffer) => void;
|
|
2
|
+
type OnVolume$1 = (rms01: number) => void;
|
|
3
|
+
type OnError = (err: Error) => void;
|
|
4
|
+
interface CaptureOptions {
|
|
5
|
+
onChunk: OnChunk;
|
|
6
|
+
onVolume?: OnVolume$1;
|
|
7
|
+
onError?: OnError;
|
|
8
|
+
}
|
|
9
|
+
interface CaptureController {
|
|
10
|
+
start: () => Promise<void>;
|
|
11
|
+
stop: () => void;
|
|
12
|
+
mute: (muted: boolean) => void;
|
|
13
|
+
isCapturing: () => boolean;
|
|
14
|
+
}
|
|
15
|
+
declare const createAudioCapture: (options: CaptureOptions) => CaptureController;
|
|
16
|
+
|
|
17
|
+
type OnVolume = (rms01: number) => void;
|
|
18
|
+
type OnAgentSpeakingChange = (speaking: boolean) => void;
|
|
19
|
+
interface PlaybackOptions {
|
|
20
|
+
sampleRate?: number;
|
|
21
|
+
onVolume?: OnVolume;
|
|
22
|
+
onSpeakingChange?: OnAgentSpeakingChange;
|
|
23
|
+
}
|
|
24
|
+
interface PlaybackController {
|
|
25
|
+
enqueue: (pcm: ArrayBuffer) => void;
|
|
26
|
+
flush: () => void;
|
|
27
|
+
close: () => void;
|
|
28
|
+
resume: () => Promise<void>;
|
|
29
|
+
}
|
|
30
|
+
declare const createAudioPlayback: (options?: PlaybackOptions) => PlaybackController;
|
|
31
|
+
|
|
32
|
+
type RWSEvent = {
|
|
33
|
+
type: 'open';
|
|
34
|
+
} | {
|
|
35
|
+
type: 'reconnected';
|
|
36
|
+
} | {
|
|
37
|
+
type: 'message';
|
|
38
|
+
data: string | ArrayBuffer;
|
|
39
|
+
} | {
|
|
40
|
+
type: 'close';
|
|
41
|
+
code: number;
|
|
42
|
+
reason: string;
|
|
43
|
+
permanent: boolean;
|
|
44
|
+
} | {
|
|
45
|
+
type: 'error';
|
|
46
|
+
error: Error;
|
|
47
|
+
};
|
|
48
|
+
interface WebSocketLike {
|
|
49
|
+
binaryType: string;
|
|
50
|
+
readyState: number;
|
|
51
|
+
onopen: ((ev: unknown) => void) | null;
|
|
52
|
+
onmessage: ((ev: {
|
|
53
|
+
data: string | ArrayBuffer;
|
|
54
|
+
}) => void) | null;
|
|
55
|
+
onerror: ((ev: unknown) => void) | null;
|
|
56
|
+
onclose: ((ev: {
|
|
57
|
+
code: number;
|
|
58
|
+
reason: string;
|
|
59
|
+
}) => void) | null;
|
|
60
|
+
send: (data: string | ArrayBuffer | ArrayBufferView) => void;
|
|
61
|
+
close: (code?: number, reason?: string) => void;
|
|
62
|
+
}
|
|
63
|
+
type WebSocketFactory = (url: string) => WebSocketLike;
|
|
64
|
+
interface RWSOptions {
|
|
65
|
+
url: string;
|
|
66
|
+
wsFactory: WebSocketFactory;
|
|
67
|
+
maxRetries?: number;
|
|
68
|
+
initialBackoffMs?: number;
|
|
69
|
+
maxBackoffMs?: number;
|
|
70
|
+
}
|
|
71
|
+
declare const createReconnectingWebSocket: (options: RWSOptions, onEvent: (ev: RWSEvent) => void) => {
|
|
72
|
+
send: (data: string | ArrayBuffer | ArrayBufferView) => void;
|
|
73
|
+
close: (code?: number, reason?: string) => void;
|
|
74
|
+
readyState: () => number;
|
|
75
|
+
};
|
|
76
|
+
type ReconnectingWebSocket = ReturnType<typeof createReconnectingWebSocket>;
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Canonical payload a tenant places in their VoIP/FCM push so an
|
|
80
|
+
* agent-initiated call can connect. It is the `callTokens.mint` result
|
|
81
|
+
* (token + transport) plus two optional display fields for the native
|
|
82
|
+
* incoming-call UI. The host receives the push, runs `parseIncomingCall`,
|
|
83
|
+
* and on accept passes `token` (+ transport / webrtcGatewayBase) into the
|
|
84
|
+
* voice client. Web background-wake is best-effort (Web Push); native is
|
|
85
|
+
* the real target. See docs/sdks.md "Agent-initiated calls".
|
|
86
|
+
*/
|
|
87
|
+
interface IncomingCallPayload {
|
|
88
|
+
token: string;
|
|
89
|
+
agentId: string;
|
|
90
|
+
transport: 'ws' | 'webrtc';
|
|
91
|
+
webrtcGatewayBase?: string;
|
|
92
|
+
expiresAt?: number;
|
|
93
|
+
agentName?: string;
|
|
94
|
+
agentAvatarUrl?: string;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Validate + normalise a raw push payload into an IncomingCallPayload.
|
|
98
|
+
* Throws synchronously on malformed input. Unknown transports fall back to
|
|
99
|
+
* 'ws'; webrtcGatewayBase is ignored unless transport === 'webrtc'.
|
|
100
|
+
*/
|
|
101
|
+
declare const parseIncomingCall: (raw: unknown) => IncomingCallPayload;
|
|
102
|
+
|
|
103
|
+
export { type CaptureController as C, type IncomingCallPayload as I, type OnAgentSpeakingChange as O, type PlaybackController as P, type RWSEvent as R, type WebSocketFactory as W, type CaptureOptions as a, type OnChunk as b, type OnError as c, type OnVolume$1 as d, type PlaybackOptions as e, type RWSOptions as f, type ReconnectingWebSocket as g, type WebSocketLike as h, createAudioCapture as i, createAudioPlayback as j, createReconnectingWebSocket as k, parseIncomingCall as p };
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
type OnChunk = (pcm: ArrayBuffer) => void;
|
|
2
|
+
type OnVolume$1 = (rms01: number) => void;
|
|
3
|
+
type OnError = (err: Error) => void;
|
|
4
|
+
interface CaptureOptions {
|
|
5
|
+
onChunk: OnChunk;
|
|
6
|
+
onVolume?: OnVolume$1;
|
|
7
|
+
onError?: OnError;
|
|
8
|
+
}
|
|
9
|
+
interface CaptureController {
|
|
10
|
+
start: () => Promise<void>;
|
|
11
|
+
stop: () => void;
|
|
12
|
+
mute: (muted: boolean) => void;
|
|
13
|
+
isCapturing: () => boolean;
|
|
14
|
+
}
|
|
15
|
+
declare const createAudioCapture: (options: CaptureOptions) => CaptureController;
|
|
16
|
+
|
|
17
|
+
type OnVolume = (rms01: number) => void;
|
|
18
|
+
type OnAgentSpeakingChange = (speaking: boolean) => void;
|
|
19
|
+
interface PlaybackOptions {
|
|
20
|
+
sampleRate?: number;
|
|
21
|
+
onVolume?: OnVolume;
|
|
22
|
+
onSpeakingChange?: OnAgentSpeakingChange;
|
|
23
|
+
}
|
|
24
|
+
interface PlaybackController {
|
|
25
|
+
enqueue: (pcm: ArrayBuffer) => void;
|
|
26
|
+
flush: () => void;
|
|
27
|
+
close: () => void;
|
|
28
|
+
resume: () => Promise<void>;
|
|
29
|
+
}
|
|
30
|
+
declare const createAudioPlayback: (options?: PlaybackOptions) => PlaybackController;
|
|
31
|
+
|
|
32
|
+
type RWSEvent = {
|
|
33
|
+
type: 'open';
|
|
34
|
+
} | {
|
|
35
|
+
type: 'reconnected';
|
|
36
|
+
} | {
|
|
37
|
+
type: 'message';
|
|
38
|
+
data: string | ArrayBuffer;
|
|
39
|
+
} | {
|
|
40
|
+
type: 'close';
|
|
41
|
+
code: number;
|
|
42
|
+
reason: string;
|
|
43
|
+
permanent: boolean;
|
|
44
|
+
} | {
|
|
45
|
+
type: 'error';
|
|
46
|
+
error: Error;
|
|
47
|
+
};
|
|
48
|
+
interface WebSocketLike {
|
|
49
|
+
binaryType: string;
|
|
50
|
+
readyState: number;
|
|
51
|
+
onopen: ((ev: unknown) => void) | null;
|
|
52
|
+
onmessage: ((ev: {
|
|
53
|
+
data: string | ArrayBuffer;
|
|
54
|
+
}) => void) | null;
|
|
55
|
+
onerror: ((ev: unknown) => void) | null;
|
|
56
|
+
onclose: ((ev: {
|
|
57
|
+
code: number;
|
|
58
|
+
reason: string;
|
|
59
|
+
}) => void) | null;
|
|
60
|
+
send: (data: string | ArrayBuffer | ArrayBufferView) => void;
|
|
61
|
+
close: (code?: number, reason?: string) => void;
|
|
62
|
+
}
|
|
63
|
+
type WebSocketFactory = (url: string) => WebSocketLike;
|
|
64
|
+
interface RWSOptions {
|
|
65
|
+
url: string;
|
|
66
|
+
wsFactory: WebSocketFactory;
|
|
67
|
+
maxRetries?: number;
|
|
68
|
+
initialBackoffMs?: number;
|
|
69
|
+
maxBackoffMs?: number;
|
|
70
|
+
}
|
|
71
|
+
declare const createReconnectingWebSocket: (options: RWSOptions, onEvent: (ev: RWSEvent) => void) => {
|
|
72
|
+
send: (data: string | ArrayBuffer | ArrayBufferView) => void;
|
|
73
|
+
close: (code?: number, reason?: string) => void;
|
|
74
|
+
readyState: () => number;
|
|
75
|
+
};
|
|
76
|
+
type ReconnectingWebSocket = ReturnType<typeof createReconnectingWebSocket>;
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* Canonical payload a tenant places in their VoIP/FCM push so an
|
|
80
|
+
* agent-initiated call can connect. It is the `callTokens.mint` result
|
|
81
|
+
* (token + transport) plus two optional display fields for the native
|
|
82
|
+
* incoming-call UI. The host receives the push, runs `parseIncomingCall`,
|
|
83
|
+
* and on accept passes `token` (+ transport / webrtcGatewayBase) into the
|
|
84
|
+
* voice client. Web background-wake is best-effort (Web Push); native is
|
|
85
|
+
* the real target. See docs/sdks.md "Agent-initiated calls".
|
|
86
|
+
*/
|
|
87
|
+
interface IncomingCallPayload {
|
|
88
|
+
token: string;
|
|
89
|
+
agentId: string;
|
|
90
|
+
transport: 'ws' | 'webrtc';
|
|
91
|
+
webrtcGatewayBase?: string;
|
|
92
|
+
expiresAt?: number;
|
|
93
|
+
agentName?: string;
|
|
94
|
+
agentAvatarUrl?: string;
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Validate + normalise a raw push payload into an IncomingCallPayload.
|
|
98
|
+
* Throws synchronously on malformed input. Unknown transports fall back to
|
|
99
|
+
* 'ws'; webrtcGatewayBase is ignored unless transport === 'webrtc'.
|
|
100
|
+
*/
|
|
101
|
+
declare const parseIncomingCall: (raw: unknown) => IncomingCallPayload;
|
|
102
|
+
|
|
103
|
+
export { type CaptureController as C, type IncomingCallPayload as I, type OnAgentSpeakingChange as O, type PlaybackController as P, type RWSEvent as R, type WebSocketFactory as W, type CaptureOptions as a, type OnChunk as b, type OnError as c, type OnVolume$1 as d, type PlaybackOptions as e, type RWSOptions as f, type ReconnectingWebSocket as g, type WebSocketLike as h, createAudioCapture as i, createAudioPlayback as j, createReconnectingWebSocket as k, parseIncomingCall as p };
|
package/dist/node.d.mts
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import { RemoteTrack, LocalVideoTrack } from 'livekit-client';
|
|
2
|
-
|
|
3
1
|
interface ClientTool {
|
|
4
2
|
description: string;
|
|
5
3
|
parameters: Record<string, unknown>;
|
|
@@ -81,145 +79,6 @@ interface BuildWsUrlArgs {
|
|
|
81
79
|
}
|
|
82
80
|
declare function buildWsUrl(args: BuildWsUrlArgs): string;
|
|
83
81
|
|
|
84
|
-
type SystemMessage = {
|
|
85
|
-
kind: 'room.starting';
|
|
86
|
-
at: string;
|
|
87
|
-
} | {
|
|
88
|
-
kind: 'room.ending.soon';
|
|
89
|
-
minutesRemaining: 5 | 1;
|
|
90
|
-
} | {
|
|
91
|
-
kind: 'room.ended';
|
|
92
|
-
reason: 'duration_reached' | 'manual' | 'empty';
|
|
93
|
-
} | {
|
|
94
|
-
kind: 'role.promoted';
|
|
95
|
-
participantId: string;
|
|
96
|
-
name: string;
|
|
97
|
-
} | {
|
|
98
|
-
kind: 'role.demoted';
|
|
99
|
-
participantId: string;
|
|
100
|
-
name: string;
|
|
101
|
-
} | {
|
|
102
|
-
kind: 'participant.removed';
|
|
103
|
-
participantId: string;
|
|
104
|
-
name: string;
|
|
105
|
-
byHost?: string;
|
|
106
|
-
} | {
|
|
107
|
-
kind: 'notetaker.connected';
|
|
108
|
-
} | {
|
|
109
|
-
kind: 'notetaker.disconnected';
|
|
110
|
-
} | {
|
|
111
|
-
kind: 'notetaker.partial_degraded';
|
|
112
|
-
participantId: string;
|
|
113
|
-
};
|
|
114
|
-
type TranscriptMessage = {
|
|
115
|
-
kind: 'partial';
|
|
116
|
-
participantId: string;
|
|
117
|
-
speakerName: string;
|
|
118
|
-
text: string;
|
|
119
|
-
startedAt: string;
|
|
120
|
-
};
|
|
121
|
-
|
|
122
|
-
interface JoinRoomOptions {
|
|
123
|
-
/** Full HTTPS URL of the Voissia server. Same shape as VoiceClientConfig.apiBase. */
|
|
124
|
-
apiBase: string;
|
|
125
|
-
/** Server-generated room id (`rm_…`). */
|
|
126
|
-
roomId: string;
|
|
127
|
-
/** Shared room join token from the invite link. Mints a fresh participant each call. */
|
|
128
|
-
joinCode: string;
|
|
129
|
-
/** Display name the joiner registers under for this participant. */
|
|
130
|
-
name: string;
|
|
131
|
-
}
|
|
132
|
-
interface RoomParticipantInfo {
|
|
133
|
-
/** Stable participant id (`p_…`); strips any `guest:` LiveKit identity prefix. */
|
|
134
|
-
participantId: string;
|
|
135
|
-
/** Display name as the worker registered it; may be empty. */
|
|
136
|
-
name: string;
|
|
137
|
-
}
|
|
138
|
-
type RoomTrackKind = 'audio' | 'video';
|
|
139
|
-
/** What a track is — lets consumers tell a camera apart from a screen share
|
|
140
|
-
* (a participant can publish both at once). Mirrors livekit `Track.Source`. */
|
|
141
|
-
type RoomTrackSource = 'camera' | 'microphone' | 'screen_share' | 'screen_share_audio' | 'unknown';
|
|
142
|
-
interface RoomTrackEvent {
|
|
143
|
-
/** Stable participant id (`p_…`); `guest:` prefix stripped. */
|
|
144
|
-
participantId: string;
|
|
145
|
-
kind: RoomTrackKind;
|
|
146
|
-
/** Distinguishes camera vs screen_share so each can render as its own tile. */
|
|
147
|
-
source: RoomTrackSource;
|
|
148
|
-
/** livekit-client track — call `.attach(el)` / `.detach()` to render. */
|
|
149
|
-
track: RemoteTrack;
|
|
150
|
-
}
|
|
151
|
-
type RoomEventName = 'participant.joined' | 'participant.left' | 'transcript.partial' | 'transcript.final' | 'system.message' | 'room.ended' | 'track.subscribed' | 'track.unsubscribed' | 'active.speakers';
|
|
152
|
-
interface RoomEventPayloads {
|
|
153
|
-
'participant.joined': RoomParticipantInfo;
|
|
154
|
-
'participant.left': RoomParticipantInfo;
|
|
155
|
-
'transcript.partial': TranscriptMessage;
|
|
156
|
-
/**
|
|
157
|
-
* Reserved — the worker currently emits only partials over the transcript
|
|
158
|
-
* topic. Final-utterance events will land in Phase 8 once the worker
|
|
159
|
-
* publishes a `final` kind; the SDK keeps the slot reserved so consumers
|
|
160
|
-
* can register handlers today.
|
|
161
|
-
*/
|
|
162
|
-
'transcript.final': {
|
|
163
|
-
participantId: string;
|
|
164
|
-
speakerName: string;
|
|
165
|
-
text: string;
|
|
166
|
-
startedAt: string;
|
|
167
|
-
};
|
|
168
|
-
'system.message': SystemMessage;
|
|
169
|
-
'room.ended': undefined;
|
|
170
|
-
'track.subscribed': RoomTrackEvent;
|
|
171
|
-
'track.unsubscribed': RoomTrackEvent;
|
|
172
|
-
/** participantIds currently speaking (drives an active-speaker UI). */
|
|
173
|
-
'active.speakers': string[];
|
|
174
|
-
}
|
|
175
|
-
type Handler<E extends RoomEventName> = (payload: RoomEventPayloads[E]) => void;
|
|
176
|
-
interface RoomSession {
|
|
177
|
-
/** This session's own stable participant id (`p_…`). Useful to filter
|
|
178
|
-
* yourself out of `active.speakers`, which includes the local participant. */
|
|
179
|
-
readonly participantId: string;
|
|
180
|
-
/** Snapshot of the remote participants currently connected. */
|
|
181
|
-
readonly participants: RoomParticipantInfo[];
|
|
182
|
-
/** Subscribe to a typed event. No unsubscribe surface yet (mirrors `Call.onX`). */
|
|
183
|
-
on<E extends RoomEventName>(event: E, handler: Handler<E>): void;
|
|
184
|
-
/** Publish the local mic track. Resolves once the track is live on LiveKit. */
|
|
185
|
-
publishMic(): Promise<void>;
|
|
186
|
-
/** Publish the local camera track. */
|
|
187
|
-
publishCamera(): Promise<void>;
|
|
188
|
-
/** Mid-call mute/unmute of the local mic. */
|
|
189
|
-
setMicEnabled(on: boolean): Promise<void>;
|
|
190
|
-
/** Mid-call camera on/off. */
|
|
191
|
-
setCameraEnabled(on: boolean): Promise<void>;
|
|
192
|
-
/** Current local mic state (for toggle UI). */
|
|
193
|
-
isMicEnabled(): boolean;
|
|
194
|
-
/** Current local camera state (for toggle UI). */
|
|
195
|
-
isCameraEnabled(): boolean;
|
|
196
|
-
/** The local camera track for self-view, or null before publishCamera resolves. */
|
|
197
|
-
getLocalCameraTrack(): LocalVideoTrack | null;
|
|
198
|
-
/**
|
|
199
|
-
* Remote tracks already subscribed at this moment. A late joiner misses the
|
|
200
|
-
* live `track.subscribed` events for tracks published before it connected
|
|
201
|
-
* (LiveKit delivers them during `connect`, before consumer listeners attach).
|
|
202
|
-
* Call this right after registering `track.subscribed` to backfill them.
|
|
203
|
-
*/
|
|
204
|
-
getRemoteTracks(): RoomTrackEvent[];
|
|
205
|
-
/**
|
|
206
|
-
* Start/stop sharing the screen (via `getDisplayMedia`). Pass `{ audio: true }`
|
|
207
|
-
* to also capture shared/system audio where the browser allows it (Chrome:
|
|
208
|
-
* tab or system audio; macOS Chrome is tab-audio only; Safari/Firefox don't
|
|
209
|
-
* capture share audio). Publishes a `screen_share` video track (+ optional
|
|
210
|
-
* `screen_share_audio`); remote peers receive them via `track.subscribed`.
|
|
211
|
-
*/
|
|
212
|
-
setScreenShareEnabled(on: boolean, opts?: {
|
|
213
|
-
audio?: boolean;
|
|
214
|
-
}): Promise<void>;
|
|
215
|
-
/** Current local screen-share state (for toggle UI). */
|
|
216
|
-
isScreenShareEnabled(): boolean;
|
|
217
|
-
/** The local screen-share video track for self-preview, or null when off. */
|
|
218
|
-
getLocalScreenTrack(): LocalVideoTrack | null;
|
|
219
|
-
/** Disconnect from LiveKit. Idempotent. Triggers `room.ended` via Disconnected. */
|
|
220
|
-
leave(): Promise<void>;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
82
|
/**
|
|
224
83
|
* Browser-friendly text-channel chat session. Mint a `ct_` token with
|
|
225
84
|
* `channel: 'text'` on your backend, then call `startTextSession({...})`
|
|
@@ -365,7 +224,7 @@ interface StartCallOptions {
|
|
|
365
224
|
* the LLM through the existing call WebSocket. Schema and handler
|
|
366
225
|
* colocate. Validated synchronously at startCall — bad input throws.
|
|
367
226
|
*
|
|
368
|
-
* See docs/
|
|
227
|
+
* See docs/sdks.md ("Client tools") for the wire protocol and the
|
|
369
228
|
* server-side guarantees.
|
|
370
229
|
*/
|
|
371
230
|
clientTools?: ClientToolMap;
|
|
@@ -423,16 +282,6 @@ interface VoiceClientFactory {
|
|
|
423
282
|
* don't reject this promise.
|
|
424
283
|
*/
|
|
425
284
|
startCall: (options: StartCallOptions) => Promise<Call>;
|
|
426
|
-
/**
|
|
427
|
-
* Phase 7 (multi-party rooms). Browser only. Exchange a single-use
|
|
428
|
-
* joinCode for a LiveKit JWT and connect to the room. The returned
|
|
429
|
-
* `RoomSession` exposes a typed event surface
|
|
430
|
-
* (participant.joined / participant.left / transcript.partial /
|
|
431
|
-
* transcript.final / system.message / room.ended) plus
|
|
432
|
-
* publishMic / publishCamera / leave. The Node bundle does NOT
|
|
433
|
-
* implement this — livekit-client is a browser-only WebRTC client.
|
|
434
|
-
*/
|
|
435
|
-
joinRoom?: (options: Omit<JoinRoomOptions, 'apiBase'>) => Promise<RoomSession>;
|
|
436
285
|
/**
|
|
437
286
|
* Open a text-channel chat session (no microphone / audio required).
|
|
438
287
|
* Mint a `ct_` token with `channel: 'text'` server-side, then call
|
|
@@ -509,6 +358,24 @@ interface NodeCall extends Call {
|
|
|
509
358
|
* back-pressure or drop).
|
|
510
359
|
*/
|
|
511
360
|
sendAudioChunk: (pcm: ArrayBuffer | ArrayBufferView) => boolean;
|
|
361
|
+
/**
|
|
362
|
+
* Push a short advisory context line to the live call (e.g. a browser
|
|
363
|
+
* action the user just took). The server buffers these and shows them
|
|
364
|
+
* to the LLM ahead of the next turn — they never trigger a response
|
|
365
|
+
* by themselves. Text is trimmed and truncated to 400 chars.
|
|
366
|
+
* Returns `false` if the WS isn't open yet or text is empty.
|
|
367
|
+
*/
|
|
368
|
+
sendClientEvent: (text: string) => boolean;
|
|
369
|
+
/**
|
|
370
|
+
* Send a typed user turn (text/multimodal sessions — token minted
|
|
371
|
+
* with `channel:'text'`). Sends `{type:'user_text', text}`. The
|
|
372
|
+
* server accepts `user_text` only on text-channel sessions and
|
|
373
|
+
* rejects it on voice (anti-injection), so this is a no-op in
|
|
374
|
+
* practice on voice calls beyond the frame being ignored
|
|
375
|
+
* server-side. Text is trimmed. Returns `false` (and sends nothing)
|
|
376
|
+
* if the WS isn't open or text is empty/whitespace. Never throws.
|
|
377
|
+
*/
|
|
378
|
+
sendText: (text: string) => boolean;
|
|
512
379
|
}
|
|
513
380
|
/**
|
|
514
381
|
* Node bundle's analog of `VoiceClientFactory`. Same shape but
|
package/dist/node.d.ts
CHANGED
|
@@ -1,5 +1,3 @@
|
|
|
1
|
-
import { RemoteTrack, LocalVideoTrack } from 'livekit-client';
|
|
2
|
-
|
|
3
1
|
interface ClientTool {
|
|
4
2
|
description: string;
|
|
5
3
|
parameters: Record<string, unknown>;
|
|
@@ -81,145 +79,6 @@ interface BuildWsUrlArgs {
|
|
|
81
79
|
}
|
|
82
80
|
declare function buildWsUrl(args: BuildWsUrlArgs): string;
|
|
83
81
|
|
|
84
|
-
type SystemMessage = {
|
|
85
|
-
kind: 'room.starting';
|
|
86
|
-
at: string;
|
|
87
|
-
} | {
|
|
88
|
-
kind: 'room.ending.soon';
|
|
89
|
-
minutesRemaining: 5 | 1;
|
|
90
|
-
} | {
|
|
91
|
-
kind: 'room.ended';
|
|
92
|
-
reason: 'duration_reached' | 'manual' | 'empty';
|
|
93
|
-
} | {
|
|
94
|
-
kind: 'role.promoted';
|
|
95
|
-
participantId: string;
|
|
96
|
-
name: string;
|
|
97
|
-
} | {
|
|
98
|
-
kind: 'role.demoted';
|
|
99
|
-
participantId: string;
|
|
100
|
-
name: string;
|
|
101
|
-
} | {
|
|
102
|
-
kind: 'participant.removed';
|
|
103
|
-
participantId: string;
|
|
104
|
-
name: string;
|
|
105
|
-
byHost?: string;
|
|
106
|
-
} | {
|
|
107
|
-
kind: 'notetaker.connected';
|
|
108
|
-
} | {
|
|
109
|
-
kind: 'notetaker.disconnected';
|
|
110
|
-
} | {
|
|
111
|
-
kind: 'notetaker.partial_degraded';
|
|
112
|
-
participantId: string;
|
|
113
|
-
};
|
|
114
|
-
type TranscriptMessage = {
|
|
115
|
-
kind: 'partial';
|
|
116
|
-
participantId: string;
|
|
117
|
-
speakerName: string;
|
|
118
|
-
text: string;
|
|
119
|
-
startedAt: string;
|
|
120
|
-
};
|
|
121
|
-
|
|
122
|
-
interface JoinRoomOptions {
|
|
123
|
-
/** Full HTTPS URL of the Voissia server. Same shape as VoiceClientConfig.apiBase. */
|
|
124
|
-
apiBase: string;
|
|
125
|
-
/** Server-generated room id (`rm_…`). */
|
|
126
|
-
roomId: string;
|
|
127
|
-
/** Shared room join token from the invite link. Mints a fresh participant each call. */
|
|
128
|
-
joinCode: string;
|
|
129
|
-
/** Display name the joiner registers under for this participant. */
|
|
130
|
-
name: string;
|
|
131
|
-
}
|
|
132
|
-
interface RoomParticipantInfo {
|
|
133
|
-
/** Stable participant id (`p_…`); strips any `guest:` LiveKit identity prefix. */
|
|
134
|
-
participantId: string;
|
|
135
|
-
/** Display name as the worker registered it; may be empty. */
|
|
136
|
-
name: string;
|
|
137
|
-
}
|
|
138
|
-
type RoomTrackKind = 'audio' | 'video';
|
|
139
|
-
/** What a track is — lets consumers tell a camera apart from a screen share
|
|
140
|
-
* (a participant can publish both at once). Mirrors livekit `Track.Source`. */
|
|
141
|
-
type RoomTrackSource = 'camera' | 'microphone' | 'screen_share' | 'screen_share_audio' | 'unknown';
|
|
142
|
-
interface RoomTrackEvent {
|
|
143
|
-
/** Stable participant id (`p_…`); `guest:` prefix stripped. */
|
|
144
|
-
participantId: string;
|
|
145
|
-
kind: RoomTrackKind;
|
|
146
|
-
/** Distinguishes camera vs screen_share so each can render as its own tile. */
|
|
147
|
-
source: RoomTrackSource;
|
|
148
|
-
/** livekit-client track — call `.attach(el)` / `.detach()` to render. */
|
|
149
|
-
track: RemoteTrack;
|
|
150
|
-
}
|
|
151
|
-
type RoomEventName = 'participant.joined' | 'participant.left' | 'transcript.partial' | 'transcript.final' | 'system.message' | 'room.ended' | 'track.subscribed' | 'track.unsubscribed' | 'active.speakers';
|
|
152
|
-
interface RoomEventPayloads {
|
|
153
|
-
'participant.joined': RoomParticipantInfo;
|
|
154
|
-
'participant.left': RoomParticipantInfo;
|
|
155
|
-
'transcript.partial': TranscriptMessage;
|
|
156
|
-
/**
|
|
157
|
-
* Reserved — the worker currently emits only partials over the transcript
|
|
158
|
-
* topic. Final-utterance events will land in Phase 8 once the worker
|
|
159
|
-
* publishes a `final` kind; the SDK keeps the slot reserved so consumers
|
|
160
|
-
* can register handlers today.
|
|
161
|
-
*/
|
|
162
|
-
'transcript.final': {
|
|
163
|
-
participantId: string;
|
|
164
|
-
speakerName: string;
|
|
165
|
-
text: string;
|
|
166
|
-
startedAt: string;
|
|
167
|
-
};
|
|
168
|
-
'system.message': SystemMessage;
|
|
169
|
-
'room.ended': undefined;
|
|
170
|
-
'track.subscribed': RoomTrackEvent;
|
|
171
|
-
'track.unsubscribed': RoomTrackEvent;
|
|
172
|
-
/** participantIds currently speaking (drives an active-speaker UI). */
|
|
173
|
-
'active.speakers': string[];
|
|
174
|
-
}
|
|
175
|
-
type Handler<E extends RoomEventName> = (payload: RoomEventPayloads[E]) => void;
|
|
176
|
-
interface RoomSession {
|
|
177
|
-
/** This session's own stable participant id (`p_…`). Useful to filter
|
|
178
|
-
* yourself out of `active.speakers`, which includes the local participant. */
|
|
179
|
-
readonly participantId: string;
|
|
180
|
-
/** Snapshot of the remote participants currently connected. */
|
|
181
|
-
readonly participants: RoomParticipantInfo[];
|
|
182
|
-
/** Subscribe to a typed event. No unsubscribe surface yet (mirrors `Call.onX`). */
|
|
183
|
-
on<E extends RoomEventName>(event: E, handler: Handler<E>): void;
|
|
184
|
-
/** Publish the local mic track. Resolves once the track is live on LiveKit. */
|
|
185
|
-
publishMic(): Promise<void>;
|
|
186
|
-
/** Publish the local camera track. */
|
|
187
|
-
publishCamera(): Promise<void>;
|
|
188
|
-
/** Mid-call mute/unmute of the local mic. */
|
|
189
|
-
setMicEnabled(on: boolean): Promise<void>;
|
|
190
|
-
/** Mid-call camera on/off. */
|
|
191
|
-
setCameraEnabled(on: boolean): Promise<void>;
|
|
192
|
-
/** Current local mic state (for toggle UI). */
|
|
193
|
-
isMicEnabled(): boolean;
|
|
194
|
-
/** Current local camera state (for toggle UI). */
|
|
195
|
-
isCameraEnabled(): boolean;
|
|
196
|
-
/** The local camera track for self-view, or null before publishCamera resolves. */
|
|
197
|
-
getLocalCameraTrack(): LocalVideoTrack | null;
|
|
198
|
-
/**
|
|
199
|
-
* Remote tracks already subscribed at this moment. A late joiner misses the
|
|
200
|
-
* live `track.subscribed` events for tracks published before it connected
|
|
201
|
-
* (LiveKit delivers them during `connect`, before consumer listeners attach).
|
|
202
|
-
* Call this right after registering `track.subscribed` to backfill them.
|
|
203
|
-
*/
|
|
204
|
-
getRemoteTracks(): RoomTrackEvent[];
|
|
205
|
-
/**
|
|
206
|
-
* Start/stop sharing the screen (via `getDisplayMedia`). Pass `{ audio: true }`
|
|
207
|
-
* to also capture shared/system audio where the browser allows it (Chrome:
|
|
208
|
-
* tab or system audio; macOS Chrome is tab-audio only; Safari/Firefox don't
|
|
209
|
-
* capture share audio). Publishes a `screen_share` video track (+ optional
|
|
210
|
-
* `screen_share_audio`); remote peers receive them via `track.subscribed`.
|
|
211
|
-
*/
|
|
212
|
-
setScreenShareEnabled(on: boolean, opts?: {
|
|
213
|
-
audio?: boolean;
|
|
214
|
-
}): Promise<void>;
|
|
215
|
-
/** Current local screen-share state (for toggle UI). */
|
|
216
|
-
isScreenShareEnabled(): boolean;
|
|
217
|
-
/** The local screen-share video track for self-preview, or null when off. */
|
|
218
|
-
getLocalScreenTrack(): LocalVideoTrack | null;
|
|
219
|
-
/** Disconnect from LiveKit. Idempotent. Triggers `room.ended` via Disconnected. */
|
|
220
|
-
leave(): Promise<void>;
|
|
221
|
-
}
|
|
222
|
-
|
|
223
82
|
/**
|
|
224
83
|
* Browser-friendly text-channel chat session. Mint a `ct_` token with
|
|
225
84
|
* `channel: 'text'` on your backend, then call `startTextSession({...})`
|
|
@@ -365,7 +224,7 @@ interface StartCallOptions {
|
|
|
365
224
|
* the LLM through the existing call WebSocket. Schema and handler
|
|
366
225
|
* colocate. Validated synchronously at startCall — bad input throws.
|
|
367
226
|
*
|
|
368
|
-
* See docs/
|
|
227
|
+
* See docs/sdks.md ("Client tools") for the wire protocol and the
|
|
369
228
|
* server-side guarantees.
|
|
370
229
|
*/
|
|
371
230
|
clientTools?: ClientToolMap;
|
|
@@ -423,16 +282,6 @@ interface VoiceClientFactory {
|
|
|
423
282
|
* don't reject this promise.
|
|
424
283
|
*/
|
|
425
284
|
startCall: (options: StartCallOptions) => Promise<Call>;
|
|
426
|
-
/**
|
|
427
|
-
* Phase 7 (multi-party rooms). Browser only. Exchange a single-use
|
|
428
|
-
* joinCode for a LiveKit JWT and connect to the room. The returned
|
|
429
|
-
* `RoomSession` exposes a typed event surface
|
|
430
|
-
* (participant.joined / participant.left / transcript.partial /
|
|
431
|
-
* transcript.final / system.message / room.ended) plus
|
|
432
|
-
* publishMic / publishCamera / leave. The Node bundle does NOT
|
|
433
|
-
* implement this — livekit-client is a browser-only WebRTC client.
|
|
434
|
-
*/
|
|
435
|
-
joinRoom?: (options: Omit<JoinRoomOptions, 'apiBase'>) => Promise<RoomSession>;
|
|
436
285
|
/**
|
|
437
286
|
* Open a text-channel chat session (no microphone / audio required).
|
|
438
287
|
* Mint a `ct_` token with `channel: 'text'` server-side, then call
|
|
@@ -509,6 +358,24 @@ interface NodeCall extends Call {
|
|
|
509
358
|
* back-pressure or drop).
|
|
510
359
|
*/
|
|
511
360
|
sendAudioChunk: (pcm: ArrayBuffer | ArrayBufferView) => boolean;
|
|
361
|
+
/**
|
|
362
|
+
* Push a short advisory context line to the live call (e.g. a browser
|
|
363
|
+
* action the user just took). The server buffers these and shows them
|
|
364
|
+
* to the LLM ahead of the next turn — they never trigger a response
|
|
365
|
+
* by themselves. Text is trimmed and truncated to 400 chars.
|
|
366
|
+
* Returns `false` if the WS isn't open yet or text is empty.
|
|
367
|
+
*/
|
|
368
|
+
sendClientEvent: (text: string) => boolean;
|
|
369
|
+
/**
|
|
370
|
+
* Send a typed user turn (text/multimodal sessions — token minted
|
|
371
|
+
* with `channel:'text'`). Sends `{type:'user_text', text}`. The
|
|
372
|
+
* server accepts `user_text` only on text-channel sessions and
|
|
373
|
+
* rejects it on voice (anti-injection), so this is a no-op in
|
|
374
|
+
* practice on voice calls beyond the frame being ignored
|
|
375
|
+
* server-side. Text is trimmed. Returns `false` (and sends nothing)
|
|
376
|
+
* if the WS isn't open or text is empty/whitespace. Never throws.
|
|
377
|
+
*/
|
|
378
|
+
sendText: (text: string) => boolean;
|
|
512
379
|
}
|
|
513
380
|
/**
|
|
514
381
|
* Node bundle's analog of `VoiceClientFactory`. Same shape but
|
package/dist/node.js
CHANGED
|
@@ -454,6 +454,7 @@ var createClientMarksBuffer = (args) => {
|
|
|
454
454
|
};
|
|
455
455
|
|
|
456
456
|
// src/NodeVoiceClient.ts
|
|
457
|
+
var READYSTATE_OPEN2 = 1;
|
|
457
458
|
var NodeVoiceClient = class {
|
|
458
459
|
constructor(args) {
|
|
459
460
|
this.rws = null;
|
|
@@ -484,6 +485,24 @@ var NodeVoiceClient = class {
|
|
|
484
485
|
this.rws.send(pcm);
|
|
485
486
|
return true;
|
|
486
487
|
};
|
|
488
|
+
this.sendClientEvent = (text) => {
|
|
489
|
+
if (!this.rws) return false;
|
|
490
|
+
const t = text.trim().slice(0, 400);
|
|
491
|
+
if (!t) return false;
|
|
492
|
+
this.rws.send(JSON.stringify({ type: "client_event", text: t }));
|
|
493
|
+
return true;
|
|
494
|
+
};
|
|
495
|
+
// Typed user turn (text/multimodal sessions — token minted with
|
|
496
|
+
// channel:'text'). The server accepts user_text only on text-channel
|
|
497
|
+
// sessions and rejects it on voice (anti-injection), so this is a
|
|
498
|
+
// no-op in practice on voice calls beyond the frame being ignored
|
|
499
|
+
// server-side.
|
|
500
|
+
this.sendText = (text) => {
|
|
501
|
+
const t = text.trim();
|
|
502
|
+
if (!t || !this.rws || this.rws.readyState() !== READYSTATE_OPEN2) return false;
|
|
503
|
+
this.rws.send(JSON.stringify({ type: "user_text", text: t }));
|
|
504
|
+
return true;
|
|
505
|
+
};
|
|
487
506
|
// ---------------------------------------------------------------
|
|
488
507
|
// Internal
|
|
489
508
|
// ---------------------------------------------------------------
|