@memberjunction/ai 5.41.0 → 5.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -121,7 +121,47 @@ export declare abstract class BaseRealtimeModel extends BaseModel {
|
|
|
121
121
|
* @throws Always, unless overridden by a provider that supports client-direct sessions.
|
|
122
122
|
*/
|
|
123
123
|
CreateClientSession(_params: RealtimeSessionParams): Promise<ClientRealtimeSessionConfig>;
|
|
124
|
+
/**
|
|
125
|
+
* Whether this driver's sessions carry a **video** track in addition to audio — i.e. the model
|
|
126
|
+
* accepts video input (it can "see" the user's camera) and/or emits video output (a talking-head
|
|
127
|
+
* avatar / generated video), in sync with audio.
|
|
128
|
+
*
|
|
129
|
+
* Defaults to `false` (audio-only — today's realtime models). Video-capable drivers (a native
|
|
130
|
+
* multimodal realtime model, or an avatar provider) override this to `true`. The session's media
|
|
131
|
+
* plane is media-tagged ({@link IRealtimeSession.SendInput} takes a {@link RealtimeMediaKind};
|
|
132
|
+
* {@link IRealtimeSession.OnVideoOutput} delivers video-out), so a video session reuses the entire
|
|
133
|
+
* realtime contract — only the media frames gain a `video` kind. Resolution prefers a video-capable
|
|
134
|
+
* model when an agent requests video, and degrades to audio-only otherwise.
|
|
135
|
+
*
|
|
136
|
+
* @returns `true` if sessions can carry video; `false` (audio-only) otherwise.
|
|
137
|
+
*/
|
|
138
|
+
get SupportsVideo(): boolean;
|
|
139
|
+
/**
|
|
140
|
+
* The provider-native voice ids this model can speak with (e.g. OpenAI `alloy`/`echo`/`shimmer`). The
|
|
141
|
+
* model/driver is the authoritative owner of "what voices do I support", so each driver declares its
|
|
142
|
+
* own — used to populate the dev voice picker. Default empty (a driver that hasn't declared voices
|
|
143
|
+
* yields no picker options, falling back to the configured/default voice).
|
|
144
|
+
*
|
|
145
|
+
* NOTE: this is the near-term, driver-owned source of truth. Long term this should move to metadata so
|
|
146
|
+
* providers that let users add their OWN voices (e.g. ElevenLabs) can be enumerated dynamically.
|
|
147
|
+
*
|
|
148
|
+
* @returns The supported voice ids (id + human label), or `[]` when none are declared.
|
|
149
|
+
*/
|
|
150
|
+
get SupportedVoices(): RealtimeVoiceOption[];
|
|
151
|
+
}
|
|
152
|
+
/** A selectable provider-native voice — `ID` is sent to the provider, `Name` is the human label. */
|
|
153
|
+
export interface RealtimeVoiceOption {
|
|
154
|
+
/** The provider-native voice id (e.g. `echo`) — what gets written to the session config. */
|
|
155
|
+
ID: string;
|
|
156
|
+
/** The human-friendly label for the picker (e.g. `Echo`). */
|
|
157
|
+
Name: string;
|
|
124
158
|
}
|
|
159
|
+
/**
|
|
160
|
+
* The media plane a realtime frame belongs to. The realtime contract is otherwise media-agnostic — a
|
|
161
|
+
* `video` session reuses every method (tools, transcript, usage, turn-taking); only the media frames
|
|
162
|
+
* carry this tag so audio and video can be disambiguated on the same session.
|
|
163
|
+
*/
|
|
164
|
+
export type RealtimeMediaKind = 'audio' | 'video';
|
|
125
165
|
/**
|
|
126
166
|
* The server-minted configuration a browser needs to open a **client-direct** realtime session.
|
|
127
167
|
*
|
|
@@ -174,13 +214,31 @@ export interface ClientRealtimeSessionConfig {
|
|
|
174
214
|
*/
|
|
175
215
|
export interface IRealtimeSession {
|
|
176
216
|
/**
|
|
177
|
-
*
|
|
217
|
+
* The PCM sample rate (Hz) this model **consumes** on {@link IRealtimeSession.SendInput} — its audio
|
|
218
|
+
* INPUT format. Optional; consumers default to 24000 (OpenAI Realtime). **Gemini Live = 16000.** A
|
|
219
|
+
* server-bridged host (LiveKit/Zoom/Teams) MUST resample inbound room audio to this rate or the model
|
|
220
|
+
* receives mis-rated audio it can't parse (the symptom: the agent never responds on the bridge while
|
|
221
|
+
* the same model works client-direct, where the browser negotiates the rate itself).
|
|
222
|
+
*/
|
|
223
|
+
InputSampleRate?: number;
|
|
224
|
+
/**
|
|
225
|
+
* The PCM sample rate (Hz) this model **emits** on {@link IRealtimeSession.OnOutput} — its audio OUTPUT
|
|
226
|
+
* format. Optional; consumers default to 24000 (both OpenAI and Gemini Live emit 24 kHz today).
|
|
227
|
+
*/
|
|
228
|
+
OutputSampleRate?: number;
|
|
229
|
+
/**
|
|
230
|
+
* Sends a client media frame to the model.
|
|
178
231
|
*
|
|
179
|
-
* Fire-and-forget: frames are streamed straight to the provider with no JSON intermediation.
|
|
232
|
+
* Fire-and-forget: frames are streamed straight to the provider with no JSON intermediation. The
|
|
233
|
+
* optional `kind` tags the media plane — `'audio'` (default, back-compatible: existing callers and
|
|
234
|
+
* audio-only drivers need not pass or read it) or `'video'` for a camera frame to a video-capable
|
|
235
|
+
* model (one that {@link BaseRealtimeModel.SupportsVideo}). Audio-only drivers ignore `'video'`
|
|
236
|
+
* frames.
|
|
180
237
|
*
|
|
181
238
|
* @param chunk A raw media frame as an `ArrayBuffer`.
|
|
239
|
+
* @param kind The media plane the frame belongs to. Defaults to `'audio'`.
|
|
182
240
|
*/
|
|
183
|
-
SendInput(chunk: ArrayBuffer): void;
|
|
241
|
+
SendInput(chunk: ArrayBuffer, kind?: RealtimeMediaKind): void;
|
|
184
242
|
/**
|
|
185
243
|
* Registers the set of tools the model may call, translating them into the provider's
|
|
186
244
|
* native function-calling format.
|
|
@@ -207,11 +265,23 @@ export interface IRealtimeSession {
|
|
|
207
265
|
*/
|
|
208
266
|
RegisterTools(tools: RealtimeToolDefinition[]): Promise<void>;
|
|
209
267
|
/**
|
|
210
|
-
* Registers a handler for model
|
|
268
|
+
* Registers a handler for model **audio** output frames (the audio media plane).
|
|
211
269
|
*
|
|
212
|
-
* @param handler Invoked with each output
|
|
270
|
+
* @param handler Invoked with each output audio frame as an `ArrayBuffer`.
|
|
213
271
|
*/
|
|
214
272
|
OnOutput(handler: (chunk: ArrayBuffer) => void): void;
|
|
273
|
+
/**
|
|
274
|
+
* Registers a handler for model **video** output frames — the talking-head avatar / generated
|
|
275
|
+
* video a video-capable model emits, in sync with {@link IRealtimeSession.OnOutput}'s audio.
|
|
276
|
+
*
|
|
277
|
+
* Optional: audio-only drivers (the default) don't implement it, and consumers must call it
|
|
278
|
+
* null-safely (`session.OnVideoOutput?.(...)`). A video-capable driver
|
|
279
|
+
* ({@link BaseRealtimeModel.SupportsVideo}) implements it; the consumer (bridge / client) maps these
|
|
280
|
+
* frames onto its `video-out` track exactly as it maps audio.
|
|
281
|
+
*
|
|
282
|
+
* @param handler Invoked with each output video frame as an `ArrayBuffer`.
|
|
283
|
+
*/
|
|
284
|
+
OnVideoOutput?(handler: (chunk: ArrayBuffer) => void): void;
|
|
215
285
|
/**
|
|
216
286
|
* Registers a handler for transcript events (the text stream).
|
|
217
287
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"baseRealtime.d.ts","sourceRoot":"","sources":["../../src/generic/baseRealtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC;;;GAGG;AACH,MAAM,MAAM,SAAS,GACf,MAAM,GACN,MAAM,GACN,OAAO,GACP,IAAI,GACJ,SAAS,EAAE,GACX;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAEnC;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAEtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AACH,8BAAsB,iBAAkB,SAAQ,SAAS;IACrD;;;;;;;;;;OAUG;aACa,YAAY,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAEtF;;;;;;;;;;;OAWG;IACH,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACU,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"baseRealtime.d.ts","sourceRoot":"","sources":["../../src/generic/baseRealtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC;;;GAGG;AACH,MAAM,MAAM,SAAS,GACf,MAAM,GACN,MAAM,GACN,OAAO,GACP,IAAI,GACJ,SAAS,EAAE,GACX;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAEnC;;;GAGG;AACH,MAAM,MAAM,UAAU,GAAG;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC;AAEtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AACH,8BAAsB,iBAAkB,SAAQ,SAAS;IACrD;;;;;;;;;;OAUG;aACa,YAAY,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAEtF;;;;;;;;;;;OAWG;IACH,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACU,mBAAmB,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,2BAA2B,CAAC;IAItG;;;;;;;;;;;;;OAaG;IACH,IAAW,aAAa,IAAI,OAAO,CAElC;IAED;;;;;;;;;;OAUG;IACH,IAAW,eAAe,IAAI,mBAAmB,EAAE,CAElD;CACJ;AAED,oGAAoG;AACpG,MAAM,WAAW,mBAAmB;IAChC,4FAA4F;IAC5F,EAAE,EAAE,MAAM,CAAC;IACX,6DAA6D;IAC7D,IAAI,EAAE,MAAM,CAAC;CAChB;AAED;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,GAAG,OAAO,GAAG,OAAO,CAAC;AAElD;;;;;;;;;;;;;GAaG;AACH,MAAM,WAAW,2BAA2B;IACxC;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;OAEG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;;OAGG;IACH,cAAc,EAAE,MAAM,CAAC;IAEvB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAC;IAElB;;;;;;;OAOG;IACH,aAAa,EAAE,UAAU,CAAC;CAC7B;AAED;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;;;;;OAMG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAE1B;;;;;;;;;;;OAWG;IACH,SAAS,CAAC,KAAK,EAAE,WAAW,EAAE,IAAI,CAAC,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAE9D;;;;;;;;;;;;;;;;;;;;;;;OAuBG;IACH,aAAa,CAAC,KAAK,EAAE,sBAAsB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9D;;;;OAIG;IACH,QAAQ,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI,CAAC;IAEtD;;;;;;;;;;OAUG;IACH,aAAa,CAAC,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,GAAG,IAAI,CAAC;IAE5D;;;;;;;OAOG;IACH,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,kBAAkB,KAAK,IAAI,GAAG,IAAI,CAAC;IAE7D;;;;;;;OAOG;IACH,UAAU,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,gBAAgB,KAAK,IAAI,GAAG,IAAI,CAAC;IAE5D;;;;;;;;OAQG;IACH,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE9D;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAErC;;;;;;;;;;;;;;;;;OAiBG;IACH,mBAAmB,CAAC,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAEjD;;;;;;;;;;;;;;OAcG;IACH,cAAc,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI,CAAC;IAE1C;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,oBAAoB,KAAK,IAAI,GAAG,IAAI,CAAC;IAE9D;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,IAAI,GAAG,IAAI,CAAC;IAEpC;;;;;;;OAOG;IACH,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,IAAI,GAAG,IAAI,CAAC;IAEnD;;;;OAIG;IACH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IAClC;;;OAGG;IACH,KAAK,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;IAErB;;;;OAIG;IACH,KAAK,CAAC,EAAE,sBAAsB,EAAE,CAAC;IAEjC;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;OAIG;IACH,MAAM,CAAC,EAAE,UAAU,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IAC/B;;OAEG;IACH,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;IAE3B;;;;OAIG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;OAEG;IACH,OAAO,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACjC,oCAAoC;IACpC,OAAO,EAAE,MAAM,CAAC;IAEhB,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd,gDAAgD;IAChD,KAAK,EAAE,OAAO,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC7B;;OAEG;IACH,MAAM,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,SAAS,EAAE,MAAM,CAAC;CACrB;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,aAAa;IAC1B;;OAEG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;OAEG;IACH,YAAY,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,sBAAsB;IACnC;;OAEG;IACH,IAAI,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,WAAW,EAAE,MAAM,CAAC;IAEpB;;;OAGG;IACH,gBAAgB,EAAE,UAAU,CAAC;CAChC"}
|
|
@@ -99,5 +99,36 @@ export class BaseRealtimeModel extends BaseModel {
|
|
|
99
99
|
async CreateClientSession(_params) {
|
|
100
100
|
throw new Error(`${this.constructor.name} does not support client-direct realtime sessions`);
|
|
101
101
|
}
|
|
102
|
+
/**
|
|
103
|
+
* Whether this driver's sessions carry a **video** track in addition to audio — i.e. the model
|
|
104
|
+
* accepts video input (it can "see" the user's camera) and/or emits video output (a talking-head
|
|
105
|
+
* avatar / generated video), in sync with audio.
|
|
106
|
+
*
|
|
107
|
+
* Defaults to `false` (audio-only — today's realtime models). Video-capable drivers (a native
|
|
108
|
+
* multimodal realtime model, or an avatar provider) override this to `true`. The session's media
|
|
109
|
+
* plane is media-tagged ({@link IRealtimeSession.SendInput} takes a {@link RealtimeMediaKind};
|
|
110
|
+
* {@link IRealtimeSession.OnVideoOutput} delivers video-out), so a video session reuses the entire
|
|
111
|
+
* realtime contract — only the media frames gain a `video` kind. Resolution prefers a video-capable
|
|
112
|
+
* model when an agent requests video, and degrades to audio-only otherwise.
|
|
113
|
+
*
|
|
114
|
+
* @returns `true` if sessions can carry video; `false` (audio-only) otherwise.
|
|
115
|
+
*/
|
|
116
|
+
get SupportsVideo() {
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* The provider-native voice ids this model can speak with (e.g. OpenAI `alloy`/`echo`/`shimmer`). The
|
|
121
|
+
* model/driver is the authoritative owner of "what voices do I support", so each driver declares its
|
|
122
|
+
* own — used to populate the dev voice picker. Default empty (a driver that hasn't declared voices
|
|
123
|
+
* yields no picker options, falling back to the configured/default voice).
|
|
124
|
+
*
|
|
125
|
+
* NOTE: this is the near-term, driver-owned source of truth. Long term this should move to metadata so
|
|
126
|
+
* providers that let users add their OWN voices (e.g. ElevenLabs) can be enumerated dynamically.
|
|
127
|
+
*
|
|
128
|
+
* @returns The supported voice ids (id + human label), or `[]` when none are declared.
|
|
129
|
+
*/
|
|
130
|
+
get SupportedVoices() {
|
|
131
|
+
return [];
|
|
132
|
+
}
|
|
102
133
|
}
|
|
103
134
|
//# sourceMappingURL=baseRealtime.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"baseRealtime.js","sourceRoot":"","sources":["../../src/generic/baseRealtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAoBxC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AACH,MAAM,OAAgB,iBAAkB,SAAQ,SAAS;IAcrD;;;;;;;;;;;OAWG;IACH,IAAW,oBAAoB;QAC3B,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACI,KAAK,CAAC,mBAAmB,CAAC,OAA8B;QAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,mDAAmD,CAAC,CAAC;IACjG,CAAC;CACJ"}
|
|
1
|
+
{"version":3,"file":"baseRealtime.js","sourceRoot":"","sources":["../../src/generic/baseRealtime.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAoBxC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6DG;AACH,MAAM,OAAgB,iBAAkB,SAAQ,SAAS;IAcrD;;;;;;;;;;;OAWG;IACH,IAAW,oBAAoB;QAC3B,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACI,KAAK,CAAC,mBAAmB,CAAC,OAA8B;QAC3D,MAAM,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,mDAAmD,CAAC,CAAC;IACjG,CAAC;IAED;;;;;;;;;;;;;OAaG;IACH,IAAW,aAAa;QACpB,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;;;;;;;;;OAUG;IACH,IAAW,eAAe;QACtB,OAAO,EAAE,CAAC;IACd,CAAC;CACJ"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@memberjunction/ai",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "5.
|
|
4
|
+
"version": "5.42.0",
|
|
5
5
|
"description": "MemberJunction: AI - core components for abstracting LLMs and other AI model types that are usable anywhere without ANY other MJ dependencies past @memberjunction/global which itself has zero additional dependencies.",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"types": "dist/index.d.ts",
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
"author": "MemberJunction.com",
|
|
18
18
|
"license": "ISC",
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"@memberjunction/global": "5.
|
|
20
|
+
"@memberjunction/global": "5.42.0",
|
|
21
21
|
"dotenv": "^17.2.4",
|
|
22
22
|
"rxjs": "^7.8.2"
|
|
23
23
|
},
|