nexus-ai-pro 2.20.2 → 2.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/API_STABILITY.md +93 -0
- package/CHANGELOG.md +208 -1
- package/README.md +11 -6
- package/SECURITY.md +16 -2
- package/dist/adapter-kit/gather.d.ts +76 -0
- package/dist/adapter-kit/gather.js +147 -0
- package/dist/adapter-kit/index.d.ts +1 -0
- package/dist/adapter-kit/index.js +1 -0
- package/dist/adapter-kit/record.d.ts +3 -1
- package/dist/adapter-kit/record.js +20 -5
- package/dist/agent/openapi.d.ts +94 -8
- package/dist/agent/openapi.js +528 -32
- package/dist/cli/adapters.d.ts +4 -3
- package/dist/cli/adapters.js +71 -2
- package/dist/cli/doctor.js +3 -0
- package/dist/cli.js +3 -1
- package/dist/core/config-builder.d.ts +1 -0
- package/dist/core/config-builder.js +1 -0
- package/dist/core/defaults.d.ts +34 -0
- package/dist/core/defaults.js +39 -0
- package/dist/core/secure-stream.js +4 -1
- package/dist/decisions/completion.d.ts +10 -7
- package/dist/decisions/completion.js +82 -8
- package/dist/decisions/core.d.ts +88 -43
- package/dist/decisions/core.js +115 -28
- package/dist/decisions/errors.d.ts +25 -0
- package/dist/decisions/errors.js +12 -0
- package/dist/decisions/index.d.ts +6 -5
- package/dist/decisions/index.js +2 -1
- package/dist/decisions/integrations.d.ts +14 -5
- package/dist/decisions/integrations.js +16 -1
- package/dist/decisions/mock.d.ts +5 -3
- package/dist/decisions/mock.js +4 -1
- package/dist/decisions/pictures.d.ts +43 -0
- package/dist/decisions/pictures.js +103 -0
- package/dist/decisions/typesafe.d.ts +33 -5
- package/dist/decisions/typesafe.js +121 -8
- package/dist/doctor/index.d.ts +6 -9
- package/dist/doctor/index.js +41 -0
- package/dist/embeddings/adapters.d.ts +1 -1
- package/dist/embeddings/adapters.js +3 -3
- package/dist/embeddings/batch-google.d.ts +27 -8
- package/dist/embeddings/batch-google.js +112 -36
- package/dist/embeddings/batch-mock.js +1 -1
- package/dist/embeddings/batch.js +10 -4
- package/dist/embeddings/engine.js +14 -13
- package/dist/embeddings/index.d.ts +1 -1
- package/dist/embeddings/inputs.d.ts +26 -3
- package/dist/embeddings/inputs.js +84 -5
- package/dist/embeddings/mock.js +1 -1
- package/dist/embeddings/models.js +39 -4
- package/dist/loaders/csv.d.ts +23 -1
- package/dist/loaders/csv.js +125 -72
- package/dist/loaders/index.d.ts +7 -2
- package/dist/loaders/input.d.ts +6 -0
- package/dist/loaders/input.js +42 -1
- package/dist/loaders/json.d.ts +14 -3
- package/dist/loaders/json.js +171 -20
- package/dist/loaders/web.d.ts +55 -0
- package/dist/loaders/web.js +20 -1
- package/dist/mcp/client.d.ts +54 -5
- package/dist/mcp/client.js +171 -20
- package/dist/mcp/http.d.ts +45 -0
- package/dist/mcp/http.js +233 -0
- package/dist/mcp/index.d.ts +4 -2
- package/dist/mcp/index.js +3 -1
- package/dist/mcp/protocol.d.ts +107 -0
- package/dist/mcp/sampling.d.ts +41 -0
- package/dist/mcp/sampling.js +52 -0
- package/dist/mcp/server.d.ts +96 -14
- package/dist/mcp/server.js +274 -59
- package/dist/models/data.js +20 -1
- package/dist/operations/adapters.d.ts +1 -0
- package/dist/operations/adapters.js +3 -1
- package/dist/protocols/a2a-grpc.js +10 -3
- package/dist/protocols/a2a-signing.d.ts +65 -0
- package/dist/protocols/a2a-signing.js +133 -0
- package/dist/protocols/a2a.d.ts +14 -0
- package/dist/protocols/a2a.js +60 -5
- package/dist/providers/cohere.js +16 -2
- package/dist/providers/google.js +7 -1
- package/dist/providers/ollama.d.ts +1 -1
- package/dist/providers/ollama.js +35 -27
- package/dist/providers/openai.js +19 -2
- package/dist/rag/pinecone.d.ts +63 -2
- package/dist/rag/pinecone.js +80 -0
- package/dist/realtime/conversation.js +18 -0
- package/dist/realtime/gemini-live.transport.d.ts +28 -3
- package/dist/realtime/gemini-live.transport.js +170 -32
- package/dist/realtime/session.d.ts +2 -0
- package/dist/realtime/session.js +19 -4
- package/dist/realtime/types.d.ts +35 -3
- package/dist/security/output-guard.js +4 -1
- package/dist/server/auth.d.ts +263 -2
- package/dist/server/auth.js +432 -5
- package/dist/server/deployments.d.ts +3 -2
- package/dist/server/deployments.js +6 -1
- package/dist/telephony/index.d.ts +1 -1
- package/dist/telephony/manager.d.ts +8 -1
- package/dist/telephony/manager.js +13 -0
- package/dist/telephony/providers/telnyx.d.ts +18 -5
- package/dist/telephony/providers/telnyx.js +44 -7
- package/dist/telephony/providers/twilio.d.ts +3 -1
- package/dist/telephony/providers/twilio.js +16 -0
- package/dist/tracing/stores.d.ts +2 -0
- package/dist/tracing/stores.js +1 -0
- package/dist/tracing/tracer.d.ts +5 -2
- package/dist/tracing/tracer.js +6 -3
- package/dist/types/embeddings.d.ts +47 -5
- package/dist/types/messages.d.ts +7 -0
- package/dist/types/principal.d.ts +1 -1
- package/dist/types/response.d.ts +5 -0
- package/dist/types/telephony.d.ts +38 -0
- package/dist/types/tracing.d.ts +6 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/voice/providers/google.d.ts +44 -0
- package/dist/voice/providers/google.js +215 -0
- package/dist-cjs/adapter-kit/gather.d.ts +76 -0
- package/dist-cjs/adapter-kit/gather.js +152 -0
- package/dist-cjs/adapter-kit/index.d.ts +1 -0
- package/dist-cjs/adapter-kit/index.js +4 -1
- package/dist-cjs/adapter-kit/record.d.ts +3 -1
- package/dist-cjs/adapter-kit/record.js +20 -5
- package/dist-cjs/agent/openapi.d.ts +94 -8
- package/dist-cjs/agent/openapi.js +530 -32
- package/dist-cjs/cli/adapters.d.ts +4 -3
- package/dist-cjs/cli/adapters.js +70 -1
- package/dist-cjs/cli/doctor.js +3 -0
- package/dist-cjs/core/config-builder.d.ts +1 -0
- package/dist-cjs/core/config-builder.js +5 -1
- package/dist-cjs/core/defaults.d.ts +34 -0
- package/dist-cjs/core/defaults.js +44 -0
- package/dist-cjs/core/secure-stream.js +4 -1
- package/dist-cjs/decisions/completion.d.ts +10 -7
- package/dist-cjs/decisions/completion.js +81 -7
- package/dist-cjs/decisions/core.d.ts +88 -43
- package/dist-cjs/decisions/core.js +134 -46
- package/dist-cjs/decisions/errors.d.ts +25 -0
- package/dist-cjs/decisions/errors.js +16 -0
- package/dist-cjs/decisions/index.d.ts +6 -5
- package/dist-cjs/decisions/index.js +5 -1
- package/dist-cjs/decisions/integrations.d.ts +14 -5
- package/dist-cjs/decisions/integrations.js +16 -1
- package/dist-cjs/decisions/mock.d.ts +5 -3
- package/dist-cjs/decisions/mock.js +4 -1
- package/dist-cjs/decisions/pictures.d.ts +43 -0
- package/dist-cjs/decisions/pictures.js +109 -0
- package/dist-cjs/decisions/typesafe.d.ts +33 -5
- package/dist-cjs/decisions/typesafe.js +126 -13
- package/dist-cjs/doctor/index.d.ts +6 -9
- package/dist-cjs/doctor/index.js +41 -0
- package/dist-cjs/embeddings/adapters.d.ts +1 -1
- package/dist-cjs/embeddings/adapters.js +3 -3
- package/dist-cjs/embeddings/batch-google.d.ts +27 -8
- package/dist-cjs/embeddings/batch-google.js +112 -36
- package/dist-cjs/embeddings/batch-mock.js +1 -1
- package/dist-cjs/embeddings/batch.js +9 -3
- package/dist-cjs/embeddings/engine.js +13 -12
- package/dist-cjs/embeddings/index.d.ts +1 -1
- package/dist-cjs/embeddings/inputs.d.ts +26 -3
- package/dist-cjs/embeddings/inputs.js +89 -5
- package/dist-cjs/embeddings/mock.js +1 -1
- package/dist-cjs/embeddings/models.js +39 -4
- package/dist-cjs/loaders/csv.d.ts +23 -1
- package/dist-cjs/loaders/csv.js +126 -71
- package/dist-cjs/loaders/index.d.ts +7 -2
- package/dist-cjs/loaders/input.d.ts +6 -0
- package/dist-cjs/loaders/input.js +43 -1
- package/dist-cjs/loaders/json.d.ts +14 -3
- package/dist-cjs/loaders/json.js +170 -19
- package/dist-cjs/loaders/web.d.ts +55 -0
- package/dist-cjs/loaders/web.js +21 -1
- package/dist-cjs/mcp/client.d.ts +54 -5
- package/dist-cjs/mcp/client.js +171 -20
- package/dist-cjs/mcp/http.d.ts +45 -0
- package/dist-cjs/mcp/http.js +236 -0
- package/dist-cjs/mcp/index.d.ts +4 -2
- package/dist-cjs/mcp/index.js +5 -1
- package/dist-cjs/mcp/protocol.d.ts +107 -0
- package/dist-cjs/mcp/sampling.d.ts +41 -0
- package/dist-cjs/mcp/sampling.js +55 -0
- package/dist-cjs/mcp/server.d.ts +96 -14
- package/dist-cjs/mcp/server.js +273 -58
- package/dist-cjs/models/data.js +20 -1
- package/dist-cjs/operations/adapters.d.ts +1 -0
- package/dist-cjs/operations/adapters.js +3 -1
- package/dist-cjs/protocols/a2a-grpc.js +10 -3
- package/dist-cjs/protocols/a2a-signing.d.ts +65 -0
- package/dist-cjs/protocols/a2a-signing.js +140 -0
- package/dist-cjs/protocols/a2a.d.ts +14 -0
- package/dist-cjs/protocols/a2a.js +65 -6
- package/dist-cjs/providers/cohere.js +16 -2
- package/dist-cjs/providers/google.js +7 -1
- package/dist-cjs/providers/ollama.d.ts +1 -1
- package/dist-cjs/providers/ollama.js +35 -27
- package/dist-cjs/providers/openai.js +19 -2
- package/dist-cjs/rag/pinecone.d.ts +63 -2
- package/dist-cjs/rag/pinecone.js +81 -0
- package/dist-cjs/realtime/conversation.js +18 -0
- package/dist-cjs/realtime/gemini-live.transport.d.ts +28 -3
- package/dist-cjs/realtime/gemini-live.transport.js +170 -32
- package/dist-cjs/realtime/session.d.ts +2 -0
- package/dist-cjs/realtime/session.js +19 -4
- package/dist-cjs/realtime/types.d.ts +35 -3
- package/dist-cjs/security/output-guard.js +4 -1
- package/dist-cjs/server/auth.d.ts +263 -2
- package/dist-cjs/server/auth.js +437 -5
- package/dist-cjs/server/deployments.d.ts +3 -2
- package/dist-cjs/server/deployments.js +6 -1
- package/dist-cjs/telephony/index.d.ts +1 -1
- package/dist-cjs/telephony/manager.d.ts +8 -1
- package/dist-cjs/telephony/manager.js +13 -0
- package/dist-cjs/telephony/providers/telnyx.d.ts +18 -5
- package/dist-cjs/telephony/providers/telnyx.js +44 -7
- package/dist-cjs/telephony/providers/twilio.d.ts +3 -1
- package/dist-cjs/telephony/providers/twilio.js +16 -0
- package/dist-cjs/tracing/stores.d.ts +2 -0
- package/dist-cjs/tracing/stores.js +1 -0
- package/dist-cjs/tracing/tracer.d.ts +5 -2
- package/dist-cjs/tracing/tracer.js +6 -3
- package/dist-cjs/types/embeddings.d.ts +47 -5
- package/dist-cjs/types/messages.d.ts +7 -0
- package/dist-cjs/types/principal.d.ts +1 -1
- package/dist-cjs/types/response.d.ts +5 -0
- package/dist-cjs/types/telephony.d.ts +38 -0
- package/dist-cjs/types/tracing.d.ts +6 -0
- package/dist-cjs/version.d.ts +1 -1
- package/dist-cjs/version.js +1 -1
- package/dist-cjs/voice/providers/google.d.ts +44 -0
- package/dist-cjs/voice/providers/google.js +252 -0
- package/llms.txt +1 -1
- package/package.json +17 -1
package/dist-cjs/rag/pinecone.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.PineconeVectorStore = exports.PineconeError = void 0;
|
|
4
|
+
exports.ensurePineconeIndex = ensurePineconeIndex;
|
|
4
5
|
const retrieval_js_1 = require("../hallucination/retrieval.js");
|
|
5
6
|
const vector_helpers_js_1 = require("./vector-helpers.js");
|
|
6
7
|
class PineconeError extends Error {
|
|
@@ -25,6 +26,26 @@ class PineconeVectorStore {
|
|
|
25
26
|
this.base = this.base.replace(/\/+$/, '');
|
|
26
27
|
this.embed = options.embed ?? ((texts) => (0, retrieval_js_1.createHashEmbeddings)(texts, options.dimensions));
|
|
27
28
|
}
|
|
29
|
+
static async create(options) {
|
|
30
|
+
const { name, cloud, region, deletionProtection, tags, readyTimeoutMs, pollMs, controlUrl, ...store } = options;
|
|
31
|
+
const index = await ensurePineconeIndex({
|
|
32
|
+
apiKey: store.apiKey,
|
|
33
|
+
name,
|
|
34
|
+
dimensions: store.dimensions,
|
|
35
|
+
...(cloud !== undefined ? { cloud } : {}),
|
|
36
|
+
...(region !== undefined ? { region } : {}),
|
|
37
|
+
...(deletionProtection !== undefined ? { deletionProtection } : {}),
|
|
38
|
+
...(tags !== undefined ? { tags } : {}),
|
|
39
|
+
...(readyTimeoutMs !== undefined ? { readyTimeoutMs } : {}),
|
|
40
|
+
...(pollMs !== undefined ? { pollMs } : {}),
|
|
41
|
+
...(controlUrl !== undefined ? { controlUrl } : {}),
|
|
42
|
+
...(store.apiVersion !== undefined ? { apiVersion: store.apiVersion } : {}),
|
|
43
|
+
...(store.fetch !== undefined ? { fetch: store.fetch } : {}),
|
|
44
|
+
...(store.headers !== undefined ? { headers: store.headers } : {}),
|
|
45
|
+
...(store.timeoutMs !== undefined ? { timeoutMs: store.timeoutMs } : {}),
|
|
46
|
+
});
|
|
47
|
+
return new PineconeVectorStore({ ...store, host: index.host });
|
|
48
|
+
}
|
|
28
49
|
async add(documents) {
|
|
29
50
|
if (documents.length === 0)
|
|
30
51
|
return;
|
|
@@ -98,3 +119,63 @@ class PineconeVectorStore {
|
|
|
98
119
|
}
|
|
99
120
|
}
|
|
100
121
|
exports.PineconeVectorStore = PineconeVectorStore;
|
|
122
|
+
async function ensurePineconeIndex(options) {
|
|
123
|
+
if (!/^[a-z0-9](?:[a-z0-9-]{0,43}[a-z0-9])?$/.test(options.name)) {
|
|
124
|
+
throw new TypeError(`"${options.name}" is not an index name: 1 to 45 lowercase letters, digits, and hyphens, beginning and ending with a letter or digit`);
|
|
125
|
+
}
|
|
126
|
+
(0, vector_helpers_js_1.assertDimensions)(options.dimensions);
|
|
127
|
+
const control = (options.controlUrl ?? 'https://api.pinecone.io').replace(/\/+$/, '');
|
|
128
|
+
const path = `/indexes/${options.name}`;
|
|
129
|
+
const call = (method, url, body, missing = false) => (0, vector_helpers_js_1.requestJson)({
|
|
130
|
+
fetch: options.fetch ?? globalThis.fetch,
|
|
131
|
+
url: `${control}${url}`,
|
|
132
|
+
method,
|
|
133
|
+
headers: {
|
|
134
|
+
'Api-Key': options.apiKey,
|
|
135
|
+
'X-Pinecone-API-Version': options.apiVersion ?? '2025-04',
|
|
136
|
+
...options.headers,
|
|
137
|
+
},
|
|
138
|
+
...(body === undefined ? {} : { body }),
|
|
139
|
+
timeoutMs: options.timeoutMs ?? 30_000,
|
|
140
|
+
...(missing ? { notFound: undefined } : {}),
|
|
141
|
+
error: (message, status, text) => new PineconeError(`Pinecone ${message}`, status, text),
|
|
142
|
+
});
|
|
143
|
+
let created = false;
|
|
144
|
+
let index = await call('GET', path, undefined, true);
|
|
145
|
+
if (!index) {
|
|
146
|
+
try {
|
|
147
|
+
index = await call('POST', '/indexes', {
|
|
148
|
+
name: options.name,
|
|
149
|
+
dimension: options.dimensions,
|
|
150
|
+
metric: 'cosine',
|
|
151
|
+
spec: { serverless: { cloud: options.cloud ?? 'aws', region: options.region ?? 'us-east-1' } },
|
|
152
|
+
deletion_protection: options.deletionProtection ? 'enabled' : 'disabled',
|
|
153
|
+
...(options.tags ? { tags: options.tags } : {}),
|
|
154
|
+
});
|
|
155
|
+
created = true;
|
|
156
|
+
}
|
|
157
|
+
catch (error) {
|
|
158
|
+
if (!(error instanceof PineconeError) || error.status !== 409)
|
|
159
|
+
throw error;
|
|
160
|
+
index = await call('GET', path);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
if (index?.dimension !== options.dimensions || index.metric !== 'cosine') {
|
|
164
|
+
throw new PineconeError(`The index "${options.name}" is ${index?.dimension} wide with the ${index?.metric} metric; the store needs ${options.dimensions} and cosine`, 409);
|
|
165
|
+
}
|
|
166
|
+
const deadline = Date.now() + (options.readyTimeoutMs ?? 300_000);
|
|
167
|
+
while (!index?.status?.ready) {
|
|
168
|
+
const state = index?.status?.state;
|
|
169
|
+
if (state === 'InitializationFailed' || state === 'Terminating') {
|
|
170
|
+
throw new PineconeError(`The index "${options.name}" is ${state}`, 409);
|
|
171
|
+
}
|
|
172
|
+
if (Date.now() >= deadline) {
|
|
173
|
+
throw new PineconeError(`The index "${options.name}" was not ready within ${options.readyTimeoutMs ?? 300_000} ms`, 504);
|
|
174
|
+
}
|
|
175
|
+
await new Promise((resolve) => setTimeout(resolve, options.pollMs ?? 2_000));
|
|
176
|
+
index = await call('GET', path);
|
|
177
|
+
}
|
|
178
|
+
if (!index.host)
|
|
179
|
+
throw new PineconeError(`The index "${options.name}" names no host`, 502);
|
|
180
|
+
return { name: options.name, host: `https://${index.host.replace(/^https?:\/\//, '')}`, created };
|
|
181
|
+
}
|
|
@@ -117,6 +117,18 @@ function reduceRealtimeConversation(conversation, event, options = {}) {
|
|
|
117
117
|
item.text += redact(event.delta, event.role === 'assistant' ? 'assistant' : 'user', options);
|
|
118
118
|
break;
|
|
119
119
|
}
|
|
120
|
+
case 'message.image': {
|
|
121
|
+
next.items.push({
|
|
122
|
+
id: idFactory('item'),
|
|
123
|
+
type: 'image',
|
|
124
|
+
role: 'user',
|
|
125
|
+
mimeType: event.mimeType,
|
|
126
|
+
digest: event.digest,
|
|
127
|
+
...(event.url ? { url: event.url } : {}),
|
|
128
|
+
createdAt: timestamp,
|
|
129
|
+
});
|
|
130
|
+
break;
|
|
131
|
+
}
|
|
120
132
|
case 'message.text.completed': {
|
|
121
133
|
const item = ensureTextMessage(next, event.role, event.responseId, event.itemId, timestamp, idFactory);
|
|
122
134
|
item.text =
|
|
@@ -269,6 +281,9 @@ function replayRealtimeConversation(conversation, limit) {
|
|
|
269
281
|
created = message('assistant', item.transcript);
|
|
270
282
|
else if (item.type === 'text_message' && item.text)
|
|
271
283
|
created = message(item.role, item.text);
|
|
284
|
+
else if (item.type === 'image' && item.url) {
|
|
285
|
+
created = { type: 'message', role: 'user', content: [{ type: 'input_image', image_url: item.url }] };
|
|
286
|
+
}
|
|
272
287
|
else if (item.type === 'tool_call') {
|
|
273
288
|
created = {
|
|
274
289
|
type: 'function_call',
|
|
@@ -333,6 +348,8 @@ function exportText(conversation) {
|
|
|
333
348
|
lines.push(`Assistant: ${item.transcript}`);
|
|
334
349
|
if (item.type === 'text_message' && item.text)
|
|
335
350
|
lines.push(`${capitalize(item.role)}: ${item.text}`);
|
|
351
|
+
if (item.type === 'image')
|
|
352
|
+
lines.push(`[User showed an image, ${item.mimeType}]`);
|
|
336
353
|
if (item.type === 'tool_call')
|
|
337
354
|
lines.push(`Tool call (${item.name}): ${JSON.stringify(item.arguments)}`);
|
|
338
355
|
if (item.type === 'tool_result') {
|
|
@@ -350,6 +367,7 @@ function exportAnalytics(conversation) {
|
|
|
350
367
|
user_speech: 0,
|
|
351
368
|
assistant_speech: 0,
|
|
352
369
|
text_message: 0,
|
|
370
|
+
image: 0,
|
|
353
371
|
tool_call: 0,
|
|
354
372
|
tool_result: 0,
|
|
355
373
|
interruption: 0,
|
|
@@ -53,6 +53,17 @@ export declare class GeminiLiveTransport implements RealtimeTransport {
|
|
|
53
53
|
private inputRate;
|
|
54
54
|
/** The latest handle Gemini gave for resuming this session. */
|
|
55
55
|
private resumptionHandle?;
|
|
56
|
+
/** Whether the latest handle can resume the session now, between turns. */
|
|
57
|
+
private resumable;
|
|
58
|
+
/** The configuration the socket was set up with, which an update changes. */
|
|
59
|
+
private config?;
|
|
60
|
+
/** A configuration an update asked for, waiting for a point Gemini can resume from. */
|
|
61
|
+
private pendingConfig?;
|
|
62
|
+
/** The configuration the latest move set up, which an update made while it moves builds on. */
|
|
63
|
+
private target?;
|
|
64
|
+
/** Set while the socket is replaced for an update: what is sent meanwhile waits, in order. */
|
|
65
|
+
private moving;
|
|
66
|
+
private readonly outbox;
|
|
56
67
|
/** Tool calls Gemini is waiting on, by id, with their names, which a response must carry back. */
|
|
57
68
|
private readonly pendingCalls;
|
|
58
69
|
/** Tool calls Gemini cancelled, whose results it no longer wants. */
|
|
@@ -79,17 +90,31 @@ export declare class GeminiLiveTransport implements RealtimeTransport {
|
|
|
79
90
|
on<Event extends keyof RealtimeTransportEvents>(event: Event, listener: (payload: RealtimeTransportEvents[Event]) => void): () => void;
|
|
80
91
|
/** Opens the socket, sends the setup the session's configuration describes, and resolves once Gemini has it. */
|
|
81
92
|
connect(config: RealtimeSessionConfig): Promise<void>;
|
|
93
|
+
/** Resolves the credential, opens a socket, and sets it up with the configuration given. */
|
|
94
|
+
private open;
|
|
82
95
|
/** Sends microphone audio as `realtimeInput`, at the sample rate the session's input format names. */
|
|
83
96
|
sendAudio(chunk: ArrayBuffer): void;
|
|
84
97
|
/** Streams one frame of video as `realtimeInput`. Gemini takes about one a second. */
|
|
85
98
|
sendVideo(frame: ArrayBuffer, mimeType: string): void;
|
|
86
99
|
/**
|
|
87
100
|
* Translates one of the session's events into Gemini's: conversation items into turns, a tool's
|
|
88
|
-
* result into a `toolResponse`,
|
|
89
|
-
* Events Gemini has no counterpart for, such as a
|
|
90
|
-
* dropped.
|
|
101
|
+
* result into a `toolResponse`, a request for a response into the end of the user's turn, and a
|
|
102
|
+
* session update into a new setup, resumed. Events Gemini has no counterpart for, such as a
|
|
103
|
+
* truncation, are dropped.
|
|
91
104
|
*/
|
|
92
105
|
sendEvent(event: RealtimeClientEvent): void;
|
|
106
|
+
/**
|
|
107
|
+
* Gemini is configured once, at setup, so an update that changes the setup is made by setting up
|
|
108
|
+
* again: on a new socket, with the new setup and the latest resumption handle, once Gemini says the
|
|
109
|
+
* session can be resumed, between turns. The conversation carries on, and what is sent meanwhile
|
|
110
|
+
* waits. An update that changes nothing, as the session's own after it connects, is passed over.
|
|
111
|
+
*/
|
|
112
|
+
private update;
|
|
113
|
+
/** The setup a configuration describes, to tell two apart. */
|
|
114
|
+
private setupKey;
|
|
115
|
+
private maybeMove;
|
|
116
|
+
/** Sets the session up again with the pending configuration, resuming where Gemini left it. */
|
|
117
|
+
private move;
|
|
93
118
|
/** Gemini stops its own response when the person speaks; there is nothing to send. */
|
|
94
119
|
interrupt(): void;
|
|
95
120
|
/** Closes the socket. Resolves once closed or after `disconnectTimeoutMs`. */
|
|
@@ -20,6 +20,12 @@ class GeminiLiveTransport {
|
|
|
20
20
|
connectAbort;
|
|
21
21
|
inputRate = 16_000;
|
|
22
22
|
resumptionHandle;
|
|
23
|
+
resumable = false;
|
|
24
|
+
config;
|
|
25
|
+
pendingConfig;
|
|
26
|
+
target;
|
|
27
|
+
moving = false;
|
|
28
|
+
outbox = [];
|
|
23
29
|
pendingCalls = new Map();
|
|
24
30
|
cancelledCalls = new Set();
|
|
25
31
|
replayedCalls = new Map();
|
|
@@ -52,6 +58,10 @@ class GeminiLiveTransport {
|
|
|
52
58
|
this.disconnectEmitted = false;
|
|
53
59
|
this.resetTurn();
|
|
54
60
|
this.inputRate = config.audio?.input?.format?.rate ?? 16_000;
|
|
61
|
+
this.config = config;
|
|
62
|
+
this.pendingConfig = undefined;
|
|
63
|
+
this.target = undefined;
|
|
64
|
+
this.resumable = false;
|
|
55
65
|
const generation = ++this.generation;
|
|
56
66
|
const externalSignal = config.connection?.signal || config.signal;
|
|
57
67
|
const controller = new transport_utils_js_1.PortableAbortController();
|
|
@@ -63,6 +73,31 @@ class GeminiLiveTransport {
|
|
|
63
73
|
void this.disconnect();
|
|
64
74
|
}));
|
|
65
75
|
try {
|
|
76
|
+
await this.open(config, signal, generation);
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
const current = generation === this.generation;
|
|
80
|
+
if (current) {
|
|
81
|
+
this.state = 'failed';
|
|
82
|
+
this.cleanupListeners();
|
|
83
|
+
this.closeSocket(4000, 'connection failed');
|
|
84
|
+
this.socket = undefined;
|
|
85
|
+
this.connectAbort = undefined;
|
|
86
|
+
}
|
|
87
|
+
const normalized = (0, errors_js_1.toRealtimeError)(error, {
|
|
88
|
+
message: error instanceof errors_js_1.RealtimeError ? error.message : 'Gemini Live connection failed',
|
|
89
|
+
code: error instanceof errors_js_1.RealtimeError ? error.code : 'websocket_connection_failed',
|
|
90
|
+
category: error instanceof errors_js_1.RealtimeError ? error.category : 'network',
|
|
91
|
+
provider: 'google',
|
|
92
|
+
retryable: error instanceof errors_js_1.RealtimeError ? error.retryable : true,
|
|
93
|
+
});
|
|
94
|
+
if (current)
|
|
95
|
+
this.emitter.emit('error', normalized);
|
|
96
|
+
throw normalized;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
async open(config, signal, generation) {
|
|
100
|
+
{
|
|
66
101
|
if (signal?.aborted)
|
|
67
102
|
throw this.abortError(signal.reason);
|
|
68
103
|
const token = await (0, transport_utils_js_1.waitForTransportOperation)(this.resolveToken(config, signal), {
|
|
@@ -78,7 +113,7 @@ class GeminiLiveTransport {
|
|
|
78
113
|
retryable: true,
|
|
79
114
|
}),
|
|
80
115
|
});
|
|
81
|
-
if (generation !== this.generation || this.state !== 'connecting')
|
|
116
|
+
if (generation !== this.generation || (this.state !== 'connecting' && !this.moving))
|
|
82
117
|
throw this.abortError();
|
|
83
118
|
const ephemeral = token.kind === 'token';
|
|
84
119
|
const browser = typeof globalThis.document !== 'undefined';
|
|
@@ -99,26 +134,6 @@ class GeminiLiveTransport {
|
|
|
99
134
|
socket.binaryType = 'arraybuffer';
|
|
100
135
|
await this.awaitSetup(socket, config, signal, generation);
|
|
101
136
|
}
|
|
102
|
-
catch (error) {
|
|
103
|
-
const current = generation === this.generation;
|
|
104
|
-
if (current) {
|
|
105
|
-
this.state = 'failed';
|
|
106
|
-
this.cleanupListeners();
|
|
107
|
-
this.closeSocket(4000, 'connection failed');
|
|
108
|
-
this.socket = undefined;
|
|
109
|
-
this.connectAbort = undefined;
|
|
110
|
-
}
|
|
111
|
-
const normalized = (0, errors_js_1.toRealtimeError)(error, {
|
|
112
|
-
message: error instanceof errors_js_1.RealtimeError ? error.message : 'Gemini Live connection failed',
|
|
113
|
-
code: error instanceof errors_js_1.RealtimeError ? error.code : 'websocket_connection_failed',
|
|
114
|
-
category: error instanceof errors_js_1.RealtimeError ? error.category : 'network',
|
|
115
|
-
provider: 'google',
|
|
116
|
-
retryable: error instanceof errors_js_1.RealtimeError ? error.retryable : true,
|
|
117
|
-
});
|
|
118
|
-
if (current)
|
|
119
|
-
this.emitter.emit('error', normalized);
|
|
120
|
-
throw normalized;
|
|
121
|
-
}
|
|
122
137
|
}
|
|
123
138
|
sendAudio(chunk) {
|
|
124
139
|
const limit = this.options.maxAudioChunkBytes ?? DEFAULT_MAX_AUDIO_BYTES;
|
|
@@ -152,6 +167,9 @@ class GeminiLiveTransport {
|
|
|
152
167
|
case 'conversation.item.create':
|
|
153
168
|
this.sendItem((0, transport_utils_js_1.isRecord)(event.item) ? event.item : {});
|
|
154
169
|
return;
|
|
170
|
+
case 'session.update':
|
|
171
|
+
this.update((0, transport_utils_js_1.isRecord)(event.session) ? event.session : {});
|
|
172
|
+
return;
|
|
155
173
|
case 'response.create':
|
|
156
174
|
if (this.toolResponded) {
|
|
157
175
|
this.toolResponded = false;
|
|
@@ -163,6 +181,61 @@ class GeminiLiveTransport {
|
|
|
163
181
|
return;
|
|
164
182
|
}
|
|
165
183
|
}
|
|
184
|
+
update(session) {
|
|
185
|
+
const settled = this.target ?? this.config;
|
|
186
|
+
const base = this.pendingConfig ?? settled;
|
|
187
|
+
if (!base || !settled || this.options.sessionResumption === false)
|
|
188
|
+
return;
|
|
189
|
+
const next = configWith(base, session);
|
|
190
|
+
this.pendingConfig = this.setupKey(next) === this.setupKey(settled) ? undefined : next;
|
|
191
|
+
this.maybeMove();
|
|
192
|
+
}
|
|
193
|
+
setupKey(config) {
|
|
194
|
+
return JSON.stringify(this.setupFor(config));
|
|
195
|
+
}
|
|
196
|
+
maybeMove() {
|
|
197
|
+
if (!this.pendingConfig || this.moving || this.state !== 'connected' || !this.resumable)
|
|
198
|
+
return;
|
|
199
|
+
void this.move();
|
|
200
|
+
}
|
|
201
|
+
async move() {
|
|
202
|
+
const next = this.pendingConfig;
|
|
203
|
+
this.pendingConfig = undefined;
|
|
204
|
+
this.target = next;
|
|
205
|
+
this.moving = true;
|
|
206
|
+
this.resumable = false;
|
|
207
|
+
const generation = ++this.generation;
|
|
208
|
+
this.closeSocket(1000, 'session update');
|
|
209
|
+
const signal = (this.connectAbort ?? new transport_utils_js_1.PortableAbortController()).signal;
|
|
210
|
+
try {
|
|
211
|
+
await this.open(next, signal, generation);
|
|
212
|
+
if (generation !== this.generation)
|
|
213
|
+
return;
|
|
214
|
+
this.config = next;
|
|
215
|
+
this.moving = false;
|
|
216
|
+
this.inputRate = next.audio?.input?.format?.rate ?? this.inputRate;
|
|
217
|
+
for (const message of this.outbox.splice(0))
|
|
218
|
+
this.socket?.send(message);
|
|
219
|
+
this.emit({ type: 'session.updated', session: { id: next.id, model: next.model } }, { setupComplete: true });
|
|
220
|
+
this.maybeMove();
|
|
221
|
+
}
|
|
222
|
+
catch (error) {
|
|
223
|
+
if (generation !== this.generation)
|
|
224
|
+
return;
|
|
225
|
+
this.moving = false;
|
|
226
|
+
this.outbox.length = 0;
|
|
227
|
+
const normalized = new errors_js_1.RealtimeError({
|
|
228
|
+
message: `Gemini Live could not be set up again for a session update: ${error instanceof Error ? error.message : String(error)}`,
|
|
229
|
+
code: 'session_update_failed',
|
|
230
|
+
category: 'network',
|
|
231
|
+
provider: 'google',
|
|
232
|
+
retryable: true,
|
|
233
|
+
cause: error,
|
|
234
|
+
});
|
|
235
|
+
this.emitter.emit('error', normalized);
|
|
236
|
+
this.finishDisconnect(undefined, false, undefined, normalized.message);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
166
239
|
interrupt() { }
|
|
167
240
|
async disconnect() {
|
|
168
241
|
if (this.state === 'idle' || this.state === 'disconnected') {
|
|
@@ -326,7 +399,11 @@ class GeminiLiveTransport {
|
|
|
326
399
|
const resumption = (0, transport_utils_js_1.isRecord)(message.sessionResumptionUpdate) ? message.sessionResumptionUpdate : undefined;
|
|
327
400
|
if (resumption?.resumable !== false && typeof resumption?.newHandle === 'string') {
|
|
328
401
|
this.resumptionHandle = resumption.newHandle;
|
|
402
|
+
this.resumable = true;
|
|
403
|
+
this.maybeMove();
|
|
329
404
|
}
|
|
405
|
+
else if (resumption)
|
|
406
|
+
this.resumable = false;
|
|
330
407
|
if ((0, transport_utils_js_1.isRecord)(message.goAway))
|
|
331
408
|
this.emit({ type: 'session.go_away', time_left: message.goAway.timeLeft }, message);
|
|
332
409
|
if ((0, transport_utils_js_1.isRecord)(message.toolCall))
|
|
@@ -515,6 +592,10 @@ class GeminiLiveTransport {
|
|
|
515
592
|
this.emitter.emit('data', { event, raw });
|
|
516
593
|
}
|
|
517
594
|
send(message) {
|
|
595
|
+
if (this.moving) {
|
|
596
|
+
this.outbox.push(JSON.stringify(message));
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
518
599
|
const socket = this.requireOpenSocket();
|
|
519
600
|
try {
|
|
520
601
|
socket.send(JSON.stringify(message));
|
|
@@ -534,8 +615,10 @@ class GeminiLiveTransport {
|
|
|
534
615
|
const timers = this.timers();
|
|
535
616
|
return new Promise((resolve, reject) => {
|
|
536
617
|
let settled = false;
|
|
618
|
+
let setUp = false;
|
|
537
619
|
let timeout;
|
|
538
620
|
const current = () => generation === this.generation;
|
|
621
|
+
const settingUp = () => !setUp && (this.state === 'connecting' || this.moving);
|
|
539
622
|
const settle = (error) => {
|
|
540
623
|
if (settled)
|
|
541
624
|
return;
|
|
@@ -563,15 +646,18 @@ class GeminiLiveTransport {
|
|
|
563
646
|
const message = this.parse(raw);
|
|
564
647
|
if (!message)
|
|
565
648
|
return;
|
|
566
|
-
if (
|
|
567
|
-
|
|
568
|
-
this.
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
649
|
+
if (settingUp() && 'setupComplete' in message) {
|
|
650
|
+
setUp = true;
|
|
651
|
+
if (this.state === 'connecting') {
|
|
652
|
+
this.state = 'connected';
|
|
653
|
+
this.emitter.emit('connected', {
|
|
654
|
+
sessionId: config.id,
|
|
655
|
+
transport: this.kind,
|
|
656
|
+
timestamp: this.now(),
|
|
657
|
+
raw: message,
|
|
658
|
+
});
|
|
659
|
+
this.emit({ type: 'session.created', session: { id: config.id, model: config.model } }, message);
|
|
660
|
+
}
|
|
575
661
|
settle();
|
|
576
662
|
return;
|
|
577
663
|
}
|
|
@@ -588,7 +674,7 @@ class GeminiLiveTransport {
|
|
|
588
674
|
retryable: true,
|
|
589
675
|
raw: (0, transport_utils_js_1.isRecord)(raw) ? { message: raw.message } : undefined,
|
|
590
676
|
});
|
|
591
|
-
if (
|
|
677
|
+
if (settingUp())
|
|
592
678
|
settle(error);
|
|
593
679
|
else
|
|
594
680
|
this.emitter.emit('error', error);
|
|
@@ -599,7 +685,7 @@ class GeminiLiveTransport {
|
|
|
599
685
|
const close = (0, transport_utils_js_1.isRecord)(raw) ? raw : {};
|
|
600
686
|
const code = typeof close.code === 'number' ? close.code : undefined;
|
|
601
687
|
const reason = typeof close.reason === 'string' ? close.reason : undefined;
|
|
602
|
-
if (
|
|
688
|
+
if (settingUp()) {
|
|
603
689
|
settle(new errors_js_1.RealtimeError({
|
|
604
690
|
message: `Gemini Live closed before it was set up${reason ? `: ${reason}` : ''}`,
|
|
605
691
|
code: 'websocket_closed_during_connect',
|
|
@@ -774,3 +860,55 @@ function dataUrl(url) {
|
|
|
774
860
|
const match = /^data:([^;,]+);base64,(.*)$/s.exec(url);
|
|
775
861
|
return match ? { mimeType: match[1], data: match[2] } : undefined;
|
|
776
862
|
}
|
|
863
|
+
function configWith(base, session) {
|
|
864
|
+
const next = { ...base };
|
|
865
|
+
const providerSession = { ...base.providerSession };
|
|
866
|
+
let input = { ...base.audio?.input };
|
|
867
|
+
let output = { ...base.audio?.output };
|
|
868
|
+
for (const [key, value] of Object.entries(session)) {
|
|
869
|
+
if (key === 'instructions') {
|
|
870
|
+
if (typeof value === 'string')
|
|
871
|
+
next.instructions = value;
|
|
872
|
+
else
|
|
873
|
+
delete next.instructions;
|
|
874
|
+
}
|
|
875
|
+
else if (key === 'model' && typeof value === 'string')
|
|
876
|
+
next.model = value;
|
|
877
|
+
else if (key === 'output_modalities' && Array.isArray(value)) {
|
|
878
|
+
next.modalities = value.filter((item) => item === 'audio' || item === 'text');
|
|
879
|
+
}
|
|
880
|
+
else if (key === 'voice' && typeof value === 'string')
|
|
881
|
+
output = { ...output, voice: value };
|
|
882
|
+
else if (key === 'tools') {
|
|
883
|
+
next.tools = (Array.isArray(value) ? value : []).filter(transport_utils_js_1.isRecord).map((tool) => ({
|
|
884
|
+
name: String(tool.name ?? ''),
|
|
885
|
+
description: String(tool.description ?? ''),
|
|
886
|
+
...((0, transport_utils_js_1.isRecord)(tool.parameters) ? { parameters: tool.parameters } : {}),
|
|
887
|
+
}));
|
|
888
|
+
}
|
|
889
|
+
else if (key === 'audio' && (0, transport_utils_js_1.isRecord)(value)) {
|
|
890
|
+
const audioOutput = (0, transport_utils_js_1.isRecord)(value.output) ? value.output : {};
|
|
891
|
+
const audioInput = (0, transport_utils_js_1.isRecord)(value.input) ? value.input : {};
|
|
892
|
+
const transcription = (0, transport_utils_js_1.isRecord)(audioInput.transcription) ? audioInput.transcription : {};
|
|
893
|
+
if (typeof audioOutput.voice === 'string')
|
|
894
|
+
output = { ...output, voice: audioOutput.voice };
|
|
895
|
+
if (typeof transcription.language === 'string')
|
|
896
|
+
input = { ...input, language: transcription.language };
|
|
897
|
+
if ('turn_detection' in audioInput) {
|
|
898
|
+
input = {
|
|
899
|
+
...input,
|
|
900
|
+
turnDetection: audioInput.turn_detection === null
|
|
901
|
+
? null
|
|
902
|
+
: (base.audio?.input?.turnDetection ?? { type: 'server_vad' }),
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
else if (key !== 'type' && key !== 'tool_choice')
|
|
907
|
+
providerSession[key] = value;
|
|
908
|
+
}
|
|
909
|
+
return {
|
|
910
|
+
...next,
|
|
911
|
+
audio: { ...base.audio, input, output },
|
|
912
|
+
...(Object.keys(providerSession).length ? { providerSession } : {}),
|
|
913
|
+
};
|
|
914
|
+
}
|
|
@@ -108,6 +108,8 @@ export declare class RealtimeSession {
|
|
|
108
108
|
* two go in one message, and the model answers unless `createResponse` is false.
|
|
109
109
|
*/
|
|
110
110
|
sendImage(image: RealtimeImageInput, options?: RealtimeImageOptions): void;
|
|
111
|
+
/** Sends an image, and records it unless it is a frame of video. */
|
|
112
|
+
private sendPicture;
|
|
111
113
|
/**
|
|
112
114
|
* Streams one frame of video, from a camera or a shared screen: as realtime video input on a
|
|
113
115
|
* provider that takes it, such as Gemini Live, and otherwise as an image in the conversation. A
|
|
@@ -7,6 +7,8 @@ const errors_js_1 = require("./errors.js");
|
|
|
7
7
|
const events_js_1 = require("./events.js");
|
|
8
8
|
const id_js_1 = require("./id.js");
|
|
9
9
|
const openai_events_js_1 = require("./openai-events.js");
|
|
10
|
+
const base64_js_1 = require("../utils/base64.js");
|
|
11
|
+
const sha256_js_1 = require("../utils/sha256.js");
|
|
10
12
|
const tools_js_1 = require("./tools.js");
|
|
11
13
|
const transport_utils_js_1 = require("./transport-utils.js");
|
|
12
14
|
class RealtimeSession {
|
|
@@ -278,13 +280,25 @@ class RealtimeSession {
|
|
|
278
280
|
this.createResponse();
|
|
279
281
|
}
|
|
280
282
|
sendImage(image, options = {}) {
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
283
|
+
this.sendPicture(image, options, false);
|
|
284
|
+
}
|
|
285
|
+
sendPicture(image, options, frame) {
|
|
286
|
+
const url = imageUrl(image, options.mimeType || 'image/jpeg');
|
|
287
|
+
const content = [{ type: 'input_image', image_url: url }];
|
|
284
288
|
const text = options.text?.trim() ? options.text : undefined;
|
|
285
289
|
if (text)
|
|
286
290
|
content.push({ type: 'input_text', text });
|
|
287
291
|
this.sendProviderEvent({ type: 'conversation.item.create', item: { type: 'message', role: 'user', content } });
|
|
292
|
+
if (!frame) {
|
|
293
|
+
const comma = url.indexOf(',');
|
|
294
|
+
this.applyEvent({
|
|
295
|
+
type: 'message.image',
|
|
296
|
+
mimeType: url.slice(5, url.indexOf(';')),
|
|
297
|
+
digest: (0, sha256_js_1.sha256Hex)((0, base64_js_1.decodeBase64)(url.slice(comma + 1))),
|
|
298
|
+
...(this.config.conversation?.recordImages === 'digest' ? {} : { url }),
|
|
299
|
+
timestamp: this.clock.now(),
|
|
300
|
+
});
|
|
301
|
+
}
|
|
288
302
|
if (options.createResponse ?? Boolean(text))
|
|
289
303
|
this.createResponse();
|
|
290
304
|
}
|
|
@@ -299,7 +313,7 @@ class RealtimeSession {
|
|
|
299
313
|
if (transport.sendVideo)
|
|
300
314
|
transport.sendVideo(frame instanceof Uint8Array ? frame.slice().buffer : frame, mimeType);
|
|
301
315
|
else
|
|
302
|
-
this.
|
|
316
|
+
this.sendPicture(frame, { mimeType, createResponse: false }, true);
|
|
303
317
|
return true;
|
|
304
318
|
}
|
|
305
319
|
commitAudio(createResponse = true) {
|
|
@@ -1236,6 +1250,7 @@ function audioDurationMs(audio, format) {
|
|
|
1236
1250
|
return bytesPerSample ? (audio.byteLength / bytesPerSample / rate) * 1_000 : undefined;
|
|
1237
1251
|
}
|
|
1238
1252
|
const COMMITTED = new Set([
|
|
1253
|
+
'message.image',
|
|
1239
1254
|
'assistant.response.completed',
|
|
1240
1255
|
'assistant.response.cancelled',
|
|
1241
1256
|
'user.transcript.completed',
|
|
@@ -285,6 +285,22 @@ export interface InterruptionItem extends ConversationItemBase {
|
|
|
285
285
|
/** `barge_in` when the user started speaking, `manual` when the application interrupted. */
|
|
286
286
|
reason: 'barge_in' | 'manual';
|
|
287
287
|
}
|
|
288
|
+
/** An image the user showed: a photo, a screenshot, a page. A frame of video is not kept. */
|
|
289
|
+
export interface ImageItem extends ConversationItemBase {
|
|
290
|
+
/** Discriminates this item in `ConversationItem`. */
|
|
291
|
+
type: 'image';
|
|
292
|
+
/** Who showed it. */
|
|
293
|
+
role: 'user';
|
|
294
|
+
/** Its MIME type, such as `image/jpeg`. */
|
|
295
|
+
mimeType: string;
|
|
296
|
+
/** The SHA-256 of its bytes, as hex, which tells two images apart without keeping either. */
|
|
297
|
+
digest: string;
|
|
298
|
+
/**
|
|
299
|
+
* The image as it was sent, a data URL, which a resumed conversation gives back. Absent when the
|
|
300
|
+
* record keeps images by digest alone.
|
|
301
|
+
*/
|
|
302
|
+
url?: string;
|
|
303
|
+
}
|
|
288
304
|
/** An error recorded in the conversation, so an export shows where a session went wrong. */
|
|
289
305
|
export interface RealtimeConversationErrorItem extends ConversationItemBase {
|
|
290
306
|
/** Discriminates this item in `ConversationItem`. */
|
|
@@ -297,10 +313,10 @@ export interface RealtimeConversationErrorItem extends ConversationItemBase {
|
|
|
297
313
|
retryable: boolean;
|
|
298
314
|
}
|
|
299
315
|
/**
|
|
300
|
-
* One entry in a conversation record: speech, text, a tool call or result, an interruption,
|
|
301
|
-
* error.
|
|
316
|
+
* One entry in a conversation record: speech, text, an image, a tool call or result, an interruption,
|
|
317
|
+
* or an error.
|
|
302
318
|
*/
|
|
303
|
-
export type ConversationItem = UserSpeechItem | AssistantSpeechItem | TextMessageItem | ToolCallItem | ToolResultItem | InterruptionItem | RealtimeConversationErrorItem;
|
|
319
|
+
export type ConversationItem = UserSpeechItem | AssistantSpeechItem | TextMessageItem | ImageItem | ToolCallItem | ToolResultItem | InterruptionItem | RealtimeConversationErrorItem;
|
|
304
320
|
/**
|
|
305
321
|
* A realtime conversation as a record: what was said, what tools ran, and how it performed. What
|
|
306
322
|
* exports, analytics, and evaluation read.
|
|
@@ -443,6 +459,12 @@ export type RealtimeEvent = {
|
|
|
443
459
|
responseId?: string;
|
|
444
460
|
itemId?: string;
|
|
445
461
|
timestamp: number;
|
|
462
|
+
} | {
|
|
463
|
+
type: 'message.image';
|
|
464
|
+
mimeType: string;
|
|
465
|
+
digest: string;
|
|
466
|
+
url?: string;
|
|
467
|
+
timestamp: number;
|
|
446
468
|
} | {
|
|
447
469
|
type: 'assistant.response.created';
|
|
448
470
|
responseId: string;
|
|
@@ -535,6 +557,10 @@ export interface RealtimeSessionEvents {
|
|
|
535
557
|
'message.text.completed': Extract<RealtimeEvent, {
|
|
536
558
|
type: 'message.text.completed';
|
|
537
559
|
}>;
|
|
560
|
+
/** The user showed an image, which the record keeps. */
|
|
561
|
+
'message.image': Extract<RealtimeEvent, {
|
|
562
|
+
type: 'message.image';
|
|
563
|
+
}>;
|
|
538
564
|
/** The model started a response. */
|
|
539
565
|
'assistant.response.created': Extract<RealtimeEvent, {
|
|
540
566
|
type: 'assistant.response.created';
|
|
@@ -878,6 +904,12 @@ export interface RealtimeConversationOptions {
|
|
|
878
904
|
retainRawAudio?: boolean;
|
|
879
905
|
/** Rewrites transcript text before it is recorded. */
|
|
880
906
|
redactTranscript?: (text: string, role: 'user' | 'assistant') => string;
|
|
907
|
+
/**
|
|
908
|
+
* What the record keeps of an image the user sends: `content`, the default, the image itself, so a
|
|
909
|
+
* resumed conversation gives it back; or `digest`, its SHA-256 and type alone, for a record that
|
|
910
|
+
* must hold no pictures.
|
|
911
|
+
*/
|
|
912
|
+
recordImages?: 'content' | 'digest';
|
|
881
913
|
/**
|
|
882
914
|
* The most recent items a resumed session gives the provider back, so a long conversation does
|
|
883
915
|
* not overflow the model's context. Defaults to all of them.
|
|
@@ -182,10 +182,13 @@ class OutputGuard {
|
|
|
182
182
|
const safeFindings = blocked
|
|
183
183
|
? findings.map((finding) => (finding.value === undefined ? finding : { ...finding, value: '[REDACTED]' }))
|
|
184
184
|
: findings;
|
|
185
|
+
const { logprobs, ...unchanged } = response;
|
|
186
|
+
const keepLogprobs = logprobs !== undefined && contentTarget.value === response.content;
|
|
185
187
|
return {
|
|
186
188
|
ok: !blocked,
|
|
187
189
|
value: {
|
|
188
|
-
...
|
|
190
|
+
...unchanged,
|
|
191
|
+
...(keepLogprobs ? { logprobs } : {}),
|
|
189
192
|
content: contentTarget.value,
|
|
190
193
|
...(response.toolCalls
|
|
191
194
|
? {
|