nexus-ai-pro 2.20.2 → 2.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. package/API_STABILITY.md +93 -0
  2. package/CHANGELOG.md +208 -1
  3. package/README.md +11 -6
  4. package/SECURITY.md +16 -2
  5. package/dist/adapter-kit/gather.d.ts +76 -0
  6. package/dist/adapter-kit/gather.js +147 -0
  7. package/dist/adapter-kit/index.d.ts +1 -0
  8. package/dist/adapter-kit/index.js +1 -0
  9. package/dist/adapter-kit/record.d.ts +3 -1
  10. package/dist/adapter-kit/record.js +20 -5
  11. package/dist/agent/openapi.d.ts +94 -8
  12. package/dist/agent/openapi.js +528 -32
  13. package/dist/cli/adapters.d.ts +4 -3
  14. package/dist/cli/adapters.js +71 -2
  15. package/dist/cli/doctor.js +3 -0
  16. package/dist/cli.js +3 -1
  17. package/dist/core/config-builder.d.ts +1 -0
  18. package/dist/core/config-builder.js +1 -0
  19. package/dist/core/defaults.d.ts +34 -0
  20. package/dist/core/defaults.js +39 -0
  21. package/dist/core/secure-stream.js +4 -1
  22. package/dist/decisions/completion.d.ts +10 -7
  23. package/dist/decisions/completion.js +82 -8
  24. package/dist/decisions/core.d.ts +88 -43
  25. package/dist/decisions/core.js +115 -28
  26. package/dist/decisions/errors.d.ts +25 -0
  27. package/dist/decisions/errors.js +12 -0
  28. package/dist/decisions/index.d.ts +6 -5
  29. package/dist/decisions/index.js +2 -1
  30. package/dist/decisions/integrations.d.ts +14 -5
  31. package/dist/decisions/integrations.js +16 -1
  32. package/dist/decisions/mock.d.ts +5 -3
  33. package/dist/decisions/mock.js +4 -1
  34. package/dist/decisions/pictures.d.ts +43 -0
  35. package/dist/decisions/pictures.js +103 -0
  36. package/dist/decisions/typesafe.d.ts +33 -5
  37. package/dist/decisions/typesafe.js +121 -8
  38. package/dist/doctor/index.d.ts +6 -9
  39. package/dist/doctor/index.js +41 -0
  40. package/dist/embeddings/adapters.d.ts +1 -1
  41. package/dist/embeddings/adapters.js +3 -3
  42. package/dist/embeddings/batch-google.d.ts +27 -8
  43. package/dist/embeddings/batch-google.js +112 -36
  44. package/dist/embeddings/batch-mock.js +1 -1
  45. package/dist/embeddings/batch.js +10 -4
  46. package/dist/embeddings/engine.js +14 -13
  47. package/dist/embeddings/index.d.ts +1 -1
  48. package/dist/embeddings/inputs.d.ts +26 -3
  49. package/dist/embeddings/inputs.js +84 -5
  50. package/dist/embeddings/mock.js +1 -1
  51. package/dist/embeddings/models.js +39 -4
  52. package/dist/loaders/csv.d.ts +23 -1
  53. package/dist/loaders/csv.js +125 -72
  54. package/dist/loaders/index.d.ts +7 -2
  55. package/dist/loaders/input.d.ts +6 -0
  56. package/dist/loaders/input.js +42 -1
  57. package/dist/loaders/json.d.ts +14 -3
  58. package/dist/loaders/json.js +171 -20
  59. package/dist/loaders/web.d.ts +55 -0
  60. package/dist/loaders/web.js +20 -1
  61. package/dist/mcp/client.d.ts +54 -5
  62. package/dist/mcp/client.js +171 -20
  63. package/dist/mcp/http.d.ts +45 -0
  64. package/dist/mcp/http.js +233 -0
  65. package/dist/mcp/index.d.ts +4 -2
  66. package/dist/mcp/index.js +3 -1
  67. package/dist/mcp/protocol.d.ts +107 -0
  68. package/dist/mcp/sampling.d.ts +41 -0
  69. package/dist/mcp/sampling.js +52 -0
  70. package/dist/mcp/server.d.ts +96 -14
  71. package/dist/mcp/server.js +274 -59
  72. package/dist/models/data.js +20 -1
  73. package/dist/operations/adapters.d.ts +1 -0
  74. package/dist/operations/adapters.js +3 -1
  75. package/dist/protocols/a2a-grpc.js +10 -3
  76. package/dist/protocols/a2a-signing.d.ts +65 -0
  77. package/dist/protocols/a2a-signing.js +133 -0
  78. package/dist/protocols/a2a.d.ts +14 -0
  79. package/dist/protocols/a2a.js +60 -5
  80. package/dist/providers/cohere.js +16 -2
  81. package/dist/providers/google.js +7 -1
  82. package/dist/providers/ollama.d.ts +1 -1
  83. package/dist/providers/ollama.js +35 -27
  84. package/dist/providers/openai.js +19 -2
  85. package/dist/rag/pinecone.d.ts +63 -2
  86. package/dist/rag/pinecone.js +80 -0
  87. package/dist/realtime/conversation.js +18 -0
  88. package/dist/realtime/gemini-live.transport.d.ts +28 -3
  89. package/dist/realtime/gemini-live.transport.js +170 -32
  90. package/dist/realtime/session.d.ts +2 -0
  91. package/dist/realtime/session.js +19 -4
  92. package/dist/realtime/types.d.ts +35 -3
  93. package/dist/security/output-guard.js +4 -1
  94. package/dist/server/auth.d.ts +263 -2
  95. package/dist/server/auth.js +432 -5
  96. package/dist/server/deployments.d.ts +3 -2
  97. package/dist/server/deployments.js +6 -1
  98. package/dist/telephony/index.d.ts +1 -1
  99. package/dist/telephony/manager.d.ts +8 -1
  100. package/dist/telephony/manager.js +13 -0
  101. package/dist/telephony/providers/telnyx.d.ts +18 -5
  102. package/dist/telephony/providers/telnyx.js +44 -7
  103. package/dist/telephony/providers/twilio.d.ts +3 -1
  104. package/dist/telephony/providers/twilio.js +16 -0
  105. package/dist/tracing/stores.d.ts +2 -0
  106. package/dist/tracing/stores.js +1 -0
  107. package/dist/tracing/tracer.d.ts +5 -2
  108. package/dist/tracing/tracer.js +6 -3
  109. package/dist/types/embeddings.d.ts +47 -5
  110. package/dist/types/messages.d.ts +7 -0
  111. package/dist/types/principal.d.ts +1 -1
  112. package/dist/types/response.d.ts +5 -0
  113. package/dist/types/telephony.d.ts +38 -0
  114. package/dist/types/tracing.d.ts +6 -0
  115. package/dist/version.d.ts +1 -1
  116. package/dist/version.js +1 -1
  117. package/dist/voice/providers/google.d.ts +44 -0
  118. package/dist/voice/providers/google.js +215 -0
  119. package/dist-cjs/adapter-kit/gather.d.ts +76 -0
  120. package/dist-cjs/adapter-kit/gather.js +152 -0
  121. package/dist-cjs/adapter-kit/index.d.ts +1 -0
  122. package/dist-cjs/adapter-kit/index.js +4 -1
  123. package/dist-cjs/adapter-kit/record.d.ts +3 -1
  124. package/dist-cjs/adapter-kit/record.js +20 -5
  125. package/dist-cjs/agent/openapi.d.ts +94 -8
  126. package/dist-cjs/agent/openapi.js +530 -32
  127. package/dist-cjs/cli/adapters.d.ts +4 -3
  128. package/dist-cjs/cli/adapters.js +70 -1
  129. package/dist-cjs/cli/doctor.js +3 -0
  130. package/dist-cjs/core/config-builder.d.ts +1 -0
  131. package/dist-cjs/core/config-builder.js +5 -1
  132. package/dist-cjs/core/defaults.d.ts +34 -0
  133. package/dist-cjs/core/defaults.js +44 -0
  134. package/dist-cjs/core/secure-stream.js +4 -1
  135. package/dist-cjs/decisions/completion.d.ts +10 -7
  136. package/dist-cjs/decisions/completion.js +81 -7
  137. package/dist-cjs/decisions/core.d.ts +88 -43
  138. package/dist-cjs/decisions/core.js +134 -46
  139. package/dist-cjs/decisions/errors.d.ts +25 -0
  140. package/dist-cjs/decisions/errors.js +16 -0
  141. package/dist-cjs/decisions/index.d.ts +6 -5
  142. package/dist-cjs/decisions/index.js +5 -1
  143. package/dist-cjs/decisions/integrations.d.ts +14 -5
  144. package/dist-cjs/decisions/integrations.js +16 -1
  145. package/dist-cjs/decisions/mock.d.ts +5 -3
  146. package/dist-cjs/decisions/mock.js +4 -1
  147. package/dist-cjs/decisions/pictures.d.ts +43 -0
  148. package/dist-cjs/decisions/pictures.js +109 -0
  149. package/dist-cjs/decisions/typesafe.d.ts +33 -5
  150. package/dist-cjs/decisions/typesafe.js +126 -13
  151. package/dist-cjs/doctor/index.d.ts +6 -9
  152. package/dist-cjs/doctor/index.js +41 -0
  153. package/dist-cjs/embeddings/adapters.d.ts +1 -1
  154. package/dist-cjs/embeddings/adapters.js +3 -3
  155. package/dist-cjs/embeddings/batch-google.d.ts +27 -8
  156. package/dist-cjs/embeddings/batch-google.js +112 -36
  157. package/dist-cjs/embeddings/batch-mock.js +1 -1
  158. package/dist-cjs/embeddings/batch.js +9 -3
  159. package/dist-cjs/embeddings/engine.js +13 -12
  160. package/dist-cjs/embeddings/index.d.ts +1 -1
  161. package/dist-cjs/embeddings/inputs.d.ts +26 -3
  162. package/dist-cjs/embeddings/inputs.js +89 -5
  163. package/dist-cjs/embeddings/mock.js +1 -1
  164. package/dist-cjs/embeddings/models.js +39 -4
  165. package/dist-cjs/loaders/csv.d.ts +23 -1
  166. package/dist-cjs/loaders/csv.js +126 -71
  167. package/dist-cjs/loaders/index.d.ts +7 -2
  168. package/dist-cjs/loaders/input.d.ts +6 -0
  169. package/dist-cjs/loaders/input.js +43 -1
  170. package/dist-cjs/loaders/json.d.ts +14 -3
  171. package/dist-cjs/loaders/json.js +170 -19
  172. package/dist-cjs/loaders/web.d.ts +55 -0
  173. package/dist-cjs/loaders/web.js +21 -1
  174. package/dist-cjs/mcp/client.d.ts +54 -5
  175. package/dist-cjs/mcp/client.js +171 -20
  176. package/dist-cjs/mcp/http.d.ts +45 -0
  177. package/dist-cjs/mcp/http.js +236 -0
  178. package/dist-cjs/mcp/index.d.ts +4 -2
  179. package/dist-cjs/mcp/index.js +5 -1
  180. package/dist-cjs/mcp/protocol.d.ts +107 -0
  181. package/dist-cjs/mcp/sampling.d.ts +41 -0
  182. package/dist-cjs/mcp/sampling.js +55 -0
  183. package/dist-cjs/mcp/server.d.ts +96 -14
  184. package/dist-cjs/mcp/server.js +273 -58
  185. package/dist-cjs/models/data.js +20 -1
  186. package/dist-cjs/operations/adapters.d.ts +1 -0
  187. package/dist-cjs/operations/adapters.js +3 -1
  188. package/dist-cjs/protocols/a2a-grpc.js +10 -3
  189. package/dist-cjs/protocols/a2a-signing.d.ts +65 -0
  190. package/dist-cjs/protocols/a2a-signing.js +140 -0
  191. package/dist-cjs/protocols/a2a.d.ts +14 -0
  192. package/dist-cjs/protocols/a2a.js +65 -6
  193. package/dist-cjs/providers/cohere.js +16 -2
  194. package/dist-cjs/providers/google.js +7 -1
  195. package/dist-cjs/providers/ollama.d.ts +1 -1
  196. package/dist-cjs/providers/ollama.js +35 -27
  197. package/dist-cjs/providers/openai.js +19 -2
  198. package/dist-cjs/rag/pinecone.d.ts +63 -2
  199. package/dist-cjs/rag/pinecone.js +81 -0
  200. package/dist-cjs/realtime/conversation.js +18 -0
  201. package/dist-cjs/realtime/gemini-live.transport.d.ts +28 -3
  202. package/dist-cjs/realtime/gemini-live.transport.js +170 -32
  203. package/dist-cjs/realtime/session.d.ts +2 -0
  204. package/dist-cjs/realtime/session.js +19 -4
  205. package/dist-cjs/realtime/types.d.ts +35 -3
  206. package/dist-cjs/security/output-guard.js +4 -1
  207. package/dist-cjs/server/auth.d.ts +263 -2
  208. package/dist-cjs/server/auth.js +437 -5
  209. package/dist-cjs/server/deployments.d.ts +3 -2
  210. package/dist-cjs/server/deployments.js +6 -1
  211. package/dist-cjs/telephony/index.d.ts +1 -1
  212. package/dist-cjs/telephony/manager.d.ts +8 -1
  213. package/dist-cjs/telephony/manager.js +13 -0
  214. package/dist-cjs/telephony/providers/telnyx.d.ts +18 -5
  215. package/dist-cjs/telephony/providers/telnyx.js +44 -7
  216. package/dist-cjs/telephony/providers/twilio.d.ts +3 -1
  217. package/dist-cjs/telephony/providers/twilio.js +16 -0
  218. package/dist-cjs/tracing/stores.d.ts +2 -0
  219. package/dist-cjs/tracing/stores.js +1 -0
  220. package/dist-cjs/tracing/tracer.d.ts +5 -2
  221. package/dist-cjs/tracing/tracer.js +6 -3
  222. package/dist-cjs/types/embeddings.d.ts +47 -5
  223. package/dist-cjs/types/messages.d.ts +7 -0
  224. package/dist-cjs/types/principal.d.ts +1 -1
  225. package/dist-cjs/types/response.d.ts +5 -0
  226. package/dist-cjs/types/telephony.d.ts +38 -0
  227. package/dist-cjs/types/tracing.d.ts +6 -0
  228. package/dist-cjs/version.d.ts +1 -1
  229. package/dist-cjs/version.js +1 -1
  230. package/dist-cjs/voice/providers/google.d.ts +44 -0
  231. package/dist-cjs/voice/providers/google.js +252 -0
  232. package/llms.txt +1 -1
  233. package/package.json +17 -1
@@ -1,6 +1,7 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.PineconeVectorStore = exports.PineconeError = void 0;
4
+ exports.ensurePineconeIndex = ensurePineconeIndex;
4
5
  const retrieval_js_1 = require("../hallucination/retrieval.js");
5
6
  const vector_helpers_js_1 = require("./vector-helpers.js");
6
7
  class PineconeError extends Error {
@@ -25,6 +26,26 @@ class PineconeVectorStore {
25
26
  this.base = this.base.replace(/\/+$/, '');
26
27
  this.embed = options.embed ?? ((texts) => (0, retrieval_js_1.createHashEmbeddings)(texts, options.dimensions));
27
28
  }
29
+ static async create(options) {
30
+ const { name, cloud, region, deletionProtection, tags, readyTimeoutMs, pollMs, controlUrl, ...store } = options;
31
+ const index = await ensurePineconeIndex({
32
+ apiKey: store.apiKey,
33
+ name,
34
+ dimensions: store.dimensions,
35
+ ...(cloud !== undefined ? { cloud } : {}),
36
+ ...(region !== undefined ? { region } : {}),
37
+ ...(deletionProtection !== undefined ? { deletionProtection } : {}),
38
+ ...(tags !== undefined ? { tags } : {}),
39
+ ...(readyTimeoutMs !== undefined ? { readyTimeoutMs } : {}),
40
+ ...(pollMs !== undefined ? { pollMs } : {}),
41
+ ...(controlUrl !== undefined ? { controlUrl } : {}),
42
+ ...(store.apiVersion !== undefined ? { apiVersion: store.apiVersion } : {}),
43
+ ...(store.fetch !== undefined ? { fetch: store.fetch } : {}),
44
+ ...(store.headers !== undefined ? { headers: store.headers } : {}),
45
+ ...(store.timeoutMs !== undefined ? { timeoutMs: store.timeoutMs } : {}),
46
+ });
47
+ return new PineconeVectorStore({ ...store, host: index.host });
48
+ }
28
49
  async add(documents) {
29
50
  if (documents.length === 0)
30
51
  return;
@@ -98,3 +119,63 @@ class PineconeVectorStore {
98
119
  }
99
120
  }
100
121
  exports.PineconeVectorStore = PineconeVectorStore;
122
+ async function ensurePineconeIndex(options) {
123
+ if (!/^[a-z0-9](?:[a-z0-9-]{0,43}[a-z0-9])?$/.test(options.name)) {
124
+ throw new TypeError(`"${options.name}" is not an index name: 1 to 45 lowercase letters, digits, and hyphens, beginning and ending with a letter or digit`);
125
+ }
126
+ (0, vector_helpers_js_1.assertDimensions)(options.dimensions);
127
+ const control = (options.controlUrl ?? 'https://api.pinecone.io').replace(/\/+$/, '');
128
+ const path = `/indexes/${options.name}`;
129
+ const call = (method, url, body, missing = false) => (0, vector_helpers_js_1.requestJson)({
130
+ fetch: options.fetch ?? globalThis.fetch,
131
+ url: `${control}${url}`,
132
+ method,
133
+ headers: {
134
+ 'Api-Key': options.apiKey,
135
+ 'X-Pinecone-API-Version': options.apiVersion ?? '2025-04',
136
+ ...options.headers,
137
+ },
138
+ ...(body === undefined ? {} : { body }),
139
+ timeoutMs: options.timeoutMs ?? 30_000,
140
+ ...(missing ? { notFound: undefined } : {}),
141
+ error: (message, status, text) => new PineconeError(`Pinecone ${message}`, status, text),
142
+ });
143
+ let created = false;
144
+ let index = await call('GET', path, undefined, true);
145
+ if (!index) {
146
+ try {
147
+ index = await call('POST', '/indexes', {
148
+ name: options.name,
149
+ dimension: options.dimensions,
150
+ metric: 'cosine',
151
+ spec: { serverless: { cloud: options.cloud ?? 'aws', region: options.region ?? 'us-east-1' } },
152
+ deletion_protection: options.deletionProtection ? 'enabled' : 'disabled',
153
+ ...(options.tags ? { tags: options.tags } : {}),
154
+ });
155
+ created = true;
156
+ }
157
+ catch (error) {
158
+ if (!(error instanceof PineconeError) || error.status !== 409)
159
+ throw error;
160
+ index = await call('GET', path);
161
+ }
162
+ }
163
+ if (index?.dimension !== options.dimensions || index.metric !== 'cosine') {
164
+ throw new PineconeError(`The index "${options.name}" is ${index?.dimension} wide with the ${index?.metric} metric; the store needs ${options.dimensions} and cosine`, 409);
165
+ }
166
+ const deadline = Date.now() + (options.readyTimeoutMs ?? 300_000);
167
+ while (!index?.status?.ready) {
168
+ const state = index?.status?.state;
169
+ if (state === 'InitializationFailed' || state === 'Terminating') {
170
+ throw new PineconeError(`The index "${options.name}" is ${state}`, 409);
171
+ }
172
+ if (Date.now() >= deadline) {
173
+ throw new PineconeError(`The index "${options.name}" was not ready within ${options.readyTimeoutMs ?? 300_000} ms`, 504);
174
+ }
175
+ await new Promise((resolve) => setTimeout(resolve, options.pollMs ?? 2_000));
176
+ index = await call('GET', path);
177
+ }
178
+ if (!index.host)
179
+ throw new PineconeError(`The index "${options.name}" names no host`, 502);
180
+ return { name: options.name, host: `https://${index.host.replace(/^https?:\/\//, '')}`, created };
181
+ }
@@ -117,6 +117,18 @@ function reduceRealtimeConversation(conversation, event, options = {}) {
117
117
  item.text += redact(event.delta, event.role === 'assistant' ? 'assistant' : 'user', options);
118
118
  break;
119
119
  }
120
+ case 'message.image': {
121
+ next.items.push({
122
+ id: idFactory('item'),
123
+ type: 'image',
124
+ role: 'user',
125
+ mimeType: event.mimeType,
126
+ digest: event.digest,
127
+ ...(event.url ? { url: event.url } : {}),
128
+ createdAt: timestamp,
129
+ });
130
+ break;
131
+ }
120
132
  case 'message.text.completed': {
121
133
  const item = ensureTextMessage(next, event.role, event.responseId, event.itemId, timestamp, idFactory);
122
134
  item.text =
@@ -269,6 +281,9 @@ function replayRealtimeConversation(conversation, limit) {
269
281
  created = message('assistant', item.transcript);
270
282
  else if (item.type === 'text_message' && item.text)
271
283
  created = message(item.role, item.text);
284
+ else if (item.type === 'image' && item.url) {
285
+ created = { type: 'message', role: 'user', content: [{ type: 'input_image', image_url: item.url }] };
286
+ }
272
287
  else if (item.type === 'tool_call') {
273
288
  created = {
274
289
  type: 'function_call',
@@ -333,6 +348,8 @@ function exportText(conversation) {
333
348
  lines.push(`Assistant: ${item.transcript}`);
334
349
  if (item.type === 'text_message' && item.text)
335
350
  lines.push(`${capitalize(item.role)}: ${item.text}`);
351
+ if (item.type === 'image')
352
+ lines.push(`[User showed an image, ${item.mimeType}]`);
336
353
  if (item.type === 'tool_call')
337
354
  lines.push(`Tool call (${item.name}): ${JSON.stringify(item.arguments)}`);
338
355
  if (item.type === 'tool_result') {
@@ -350,6 +367,7 @@ function exportAnalytics(conversation) {
350
367
  user_speech: 0,
351
368
  assistant_speech: 0,
352
369
  text_message: 0,
370
+ image: 0,
353
371
  tool_call: 0,
354
372
  tool_result: 0,
355
373
  interruption: 0,
@@ -53,6 +53,17 @@ export declare class GeminiLiveTransport implements RealtimeTransport {
53
53
  private inputRate;
54
54
  /** The latest handle Gemini gave for resuming this session. */
55
55
  private resumptionHandle?;
56
+ /** Whether the latest handle can resume the session now, between turns. */
57
+ private resumable;
58
+ /** The configuration the socket was set up with, which an update changes. */
59
+ private config?;
60
+ /** A configuration an update asked for, waiting for a point Gemini can resume from. */
61
+ private pendingConfig?;
62
+ /** The configuration the latest move set up, which an update made while it moves builds on. */
63
+ private target?;
64
+ /** Set while the socket is replaced for an update: what is sent meanwhile waits, in order. */
65
+ private moving;
66
+ private readonly outbox;
56
67
  /** Tool calls Gemini is waiting on, by id, with their names, which a response must carry back. */
57
68
  private readonly pendingCalls;
58
69
  /** Tool calls Gemini cancelled, whose results it no longer wants. */
@@ -79,17 +90,31 @@ export declare class GeminiLiveTransport implements RealtimeTransport {
79
90
  on<Event extends keyof RealtimeTransportEvents>(event: Event, listener: (payload: RealtimeTransportEvents[Event]) => void): () => void;
80
91
  /** Opens the socket, sends the setup the session's configuration describes, and resolves once Gemini has it. */
81
92
  connect(config: RealtimeSessionConfig): Promise<void>;
93
+ /** Resolves the credential, opens a socket, and sets it up with the configuration given. */
94
+ private open;
82
95
  /** Sends microphone audio as `realtimeInput`, at the sample rate the session's input format names. */
83
96
  sendAudio(chunk: ArrayBuffer): void;
84
97
  /** Streams one frame of video as `realtimeInput`. Gemini takes about one a second. */
85
98
  sendVideo(frame: ArrayBuffer, mimeType: string): void;
86
99
  /**
87
100
  * Translates one of the session's events into Gemini's: conversation items into turns, a tool's
88
- * result into a `toolResponse`, and a request for a response into the end of the user's turn.
89
- * Events Gemini has no counterpart for, such as a session update after setup or a truncation, are
90
- * dropped.
101
+ * result into a `toolResponse`, a request for a response into the end of the user's turn, and a
102
+ * session update into a new setup, resumed. Events Gemini has no counterpart for, such as a
103
+ * truncation, are dropped.
91
104
  */
92
105
  sendEvent(event: RealtimeClientEvent): void;
106
+ /**
107
+ * Gemini is configured once, at setup, so an update that changes the setup is made by setting up
108
+ * again: on a new socket, with the new setup and the latest resumption handle, once Gemini says the
109
+ * session can be resumed, between turns. The conversation carries on, and what is sent meanwhile
110
+ * waits. An update that changes nothing, as the session's own after it connects, is passed over.
111
+ */
112
+ private update;
113
+ /** The setup a configuration describes, to tell two apart. */
114
+ private setupKey;
115
+ private maybeMove;
116
+ /** Sets the session up again with the pending configuration, resuming where Gemini left it. */
117
+ private move;
93
118
  /** Gemini stops its own response when the person speaks; there is nothing to send. */
94
119
  interrupt(): void;
95
120
  /** Closes the socket. Resolves once closed or after `disconnectTimeoutMs`. */
@@ -20,6 +20,12 @@ class GeminiLiveTransport {
20
20
  connectAbort;
21
21
  inputRate = 16_000;
22
22
  resumptionHandle;
23
+ resumable = false;
24
+ config;
25
+ pendingConfig;
26
+ target;
27
+ moving = false;
28
+ outbox = [];
23
29
  pendingCalls = new Map();
24
30
  cancelledCalls = new Set();
25
31
  replayedCalls = new Map();
@@ -52,6 +58,10 @@ class GeminiLiveTransport {
52
58
  this.disconnectEmitted = false;
53
59
  this.resetTurn();
54
60
  this.inputRate = config.audio?.input?.format?.rate ?? 16_000;
61
+ this.config = config;
62
+ this.pendingConfig = undefined;
63
+ this.target = undefined;
64
+ this.resumable = false;
55
65
  const generation = ++this.generation;
56
66
  const externalSignal = config.connection?.signal || config.signal;
57
67
  const controller = new transport_utils_js_1.PortableAbortController();
@@ -63,6 +73,31 @@ class GeminiLiveTransport {
63
73
  void this.disconnect();
64
74
  }));
65
75
  try {
76
+ await this.open(config, signal, generation);
77
+ }
78
+ catch (error) {
79
+ const current = generation === this.generation;
80
+ if (current) {
81
+ this.state = 'failed';
82
+ this.cleanupListeners();
83
+ this.closeSocket(4000, 'connection failed');
84
+ this.socket = undefined;
85
+ this.connectAbort = undefined;
86
+ }
87
+ const normalized = (0, errors_js_1.toRealtimeError)(error, {
88
+ message: error instanceof errors_js_1.RealtimeError ? error.message : 'Gemini Live connection failed',
89
+ code: error instanceof errors_js_1.RealtimeError ? error.code : 'websocket_connection_failed',
90
+ category: error instanceof errors_js_1.RealtimeError ? error.category : 'network',
91
+ provider: 'google',
92
+ retryable: error instanceof errors_js_1.RealtimeError ? error.retryable : true,
93
+ });
94
+ if (current)
95
+ this.emitter.emit('error', normalized);
96
+ throw normalized;
97
+ }
98
+ }
99
+ async open(config, signal, generation) {
100
+ {
66
101
  if (signal?.aborted)
67
102
  throw this.abortError(signal.reason);
68
103
  const token = await (0, transport_utils_js_1.waitForTransportOperation)(this.resolveToken(config, signal), {
@@ -78,7 +113,7 @@ class GeminiLiveTransport {
78
113
  retryable: true,
79
114
  }),
80
115
  });
81
- if (generation !== this.generation || this.state !== 'connecting')
116
+ if (generation !== this.generation || (this.state !== 'connecting' && !this.moving))
82
117
  throw this.abortError();
83
118
  const ephemeral = token.kind === 'token';
84
119
  const browser = typeof globalThis.document !== 'undefined';
@@ -99,26 +134,6 @@ class GeminiLiveTransport {
99
134
  socket.binaryType = 'arraybuffer';
100
135
  await this.awaitSetup(socket, config, signal, generation);
101
136
  }
102
- catch (error) {
103
- const current = generation === this.generation;
104
- if (current) {
105
- this.state = 'failed';
106
- this.cleanupListeners();
107
- this.closeSocket(4000, 'connection failed');
108
- this.socket = undefined;
109
- this.connectAbort = undefined;
110
- }
111
- const normalized = (0, errors_js_1.toRealtimeError)(error, {
112
- message: error instanceof errors_js_1.RealtimeError ? error.message : 'Gemini Live connection failed',
113
- code: error instanceof errors_js_1.RealtimeError ? error.code : 'websocket_connection_failed',
114
- category: error instanceof errors_js_1.RealtimeError ? error.category : 'network',
115
- provider: 'google',
116
- retryable: error instanceof errors_js_1.RealtimeError ? error.retryable : true,
117
- });
118
- if (current)
119
- this.emitter.emit('error', normalized);
120
- throw normalized;
121
- }
122
137
  }
123
138
  sendAudio(chunk) {
124
139
  const limit = this.options.maxAudioChunkBytes ?? DEFAULT_MAX_AUDIO_BYTES;
@@ -152,6 +167,9 @@ class GeminiLiveTransport {
152
167
  case 'conversation.item.create':
153
168
  this.sendItem((0, transport_utils_js_1.isRecord)(event.item) ? event.item : {});
154
169
  return;
170
+ case 'session.update':
171
+ this.update((0, transport_utils_js_1.isRecord)(event.session) ? event.session : {});
172
+ return;
155
173
  case 'response.create':
156
174
  if (this.toolResponded) {
157
175
  this.toolResponded = false;
@@ -163,6 +181,61 @@ class GeminiLiveTransport {
163
181
  return;
164
182
  }
165
183
  }
184
+ update(session) {
185
+ const settled = this.target ?? this.config;
186
+ const base = this.pendingConfig ?? settled;
187
+ if (!base || !settled || this.options.sessionResumption === false)
188
+ return;
189
+ const next = configWith(base, session);
190
+ this.pendingConfig = this.setupKey(next) === this.setupKey(settled) ? undefined : next;
191
+ this.maybeMove();
192
+ }
193
+ setupKey(config) {
194
+ return JSON.stringify(this.setupFor(config));
195
+ }
196
+ maybeMove() {
197
+ if (!this.pendingConfig || this.moving || this.state !== 'connected' || !this.resumable)
198
+ return;
199
+ void this.move();
200
+ }
201
+ async move() {
202
+ const next = this.pendingConfig;
203
+ this.pendingConfig = undefined;
204
+ this.target = next;
205
+ this.moving = true;
206
+ this.resumable = false;
207
+ const generation = ++this.generation;
208
+ this.closeSocket(1000, 'session update');
209
+ const signal = (this.connectAbort ?? new transport_utils_js_1.PortableAbortController()).signal;
210
+ try {
211
+ await this.open(next, signal, generation);
212
+ if (generation !== this.generation)
213
+ return;
214
+ this.config = next;
215
+ this.moving = false;
216
+ this.inputRate = next.audio?.input?.format?.rate ?? this.inputRate;
217
+ for (const message of this.outbox.splice(0))
218
+ this.socket?.send(message);
219
+ this.emit({ type: 'session.updated', session: { id: next.id, model: next.model } }, { setupComplete: true });
220
+ this.maybeMove();
221
+ }
222
+ catch (error) {
223
+ if (generation !== this.generation)
224
+ return;
225
+ this.moving = false;
226
+ this.outbox.length = 0;
227
+ const normalized = new errors_js_1.RealtimeError({
228
+ message: `Gemini Live could not be set up again for a session update: ${error instanceof Error ? error.message : String(error)}`,
229
+ code: 'session_update_failed',
230
+ category: 'network',
231
+ provider: 'google',
232
+ retryable: true,
233
+ cause: error,
234
+ });
235
+ this.emitter.emit('error', normalized);
236
+ this.finishDisconnect(undefined, false, undefined, normalized.message);
237
+ }
238
+ }
166
239
  interrupt() { }
167
240
  async disconnect() {
168
241
  if (this.state === 'idle' || this.state === 'disconnected') {
@@ -326,7 +399,11 @@ class GeminiLiveTransport {
326
399
  const resumption = (0, transport_utils_js_1.isRecord)(message.sessionResumptionUpdate) ? message.sessionResumptionUpdate : undefined;
327
400
  if (resumption?.resumable !== false && typeof resumption?.newHandle === 'string') {
328
401
  this.resumptionHandle = resumption.newHandle;
402
+ this.resumable = true;
403
+ this.maybeMove();
329
404
  }
405
+ else if (resumption)
406
+ this.resumable = false;
330
407
  if ((0, transport_utils_js_1.isRecord)(message.goAway))
331
408
  this.emit({ type: 'session.go_away', time_left: message.goAway.timeLeft }, message);
332
409
  if ((0, transport_utils_js_1.isRecord)(message.toolCall))
@@ -515,6 +592,10 @@ class GeminiLiveTransport {
515
592
  this.emitter.emit('data', { event, raw });
516
593
  }
517
594
  send(message) {
595
+ if (this.moving) {
596
+ this.outbox.push(JSON.stringify(message));
597
+ return;
598
+ }
518
599
  const socket = this.requireOpenSocket();
519
600
  try {
520
601
  socket.send(JSON.stringify(message));
@@ -534,8 +615,10 @@ class GeminiLiveTransport {
534
615
  const timers = this.timers();
535
616
  return new Promise((resolve, reject) => {
536
617
  let settled = false;
618
+ let setUp = false;
537
619
  let timeout;
538
620
  const current = () => generation === this.generation;
621
+ const settingUp = () => !setUp && (this.state === 'connecting' || this.moving);
539
622
  const settle = (error) => {
540
623
  if (settled)
541
624
  return;
@@ -563,15 +646,18 @@ class GeminiLiveTransport {
563
646
  const message = this.parse(raw);
564
647
  if (!message)
565
648
  return;
566
- if (this.state === 'connecting' && 'setupComplete' in message) {
567
- this.state = 'connected';
568
- this.emitter.emit('connected', {
569
- sessionId: config.id,
570
- transport: this.kind,
571
- timestamp: this.now(),
572
- raw: message,
573
- });
574
- this.emit({ type: 'session.created', session: { id: config.id, model: config.model } }, message);
649
+ if (settingUp() && 'setupComplete' in message) {
650
+ setUp = true;
651
+ if (this.state === 'connecting') {
652
+ this.state = 'connected';
653
+ this.emitter.emit('connected', {
654
+ sessionId: config.id,
655
+ transport: this.kind,
656
+ timestamp: this.now(),
657
+ raw: message,
658
+ });
659
+ this.emit({ type: 'session.created', session: { id: config.id, model: config.model } }, message);
660
+ }
575
661
  settle();
576
662
  return;
577
663
  }
@@ -588,7 +674,7 @@ class GeminiLiveTransport {
588
674
  retryable: true,
589
675
  raw: (0, transport_utils_js_1.isRecord)(raw) ? { message: raw.message } : undefined,
590
676
  });
591
- if (this.state === 'connecting')
677
+ if (settingUp())
592
678
  settle(error);
593
679
  else
594
680
  this.emitter.emit('error', error);
@@ -599,7 +685,7 @@ class GeminiLiveTransport {
599
685
  const close = (0, transport_utils_js_1.isRecord)(raw) ? raw : {};
600
686
  const code = typeof close.code === 'number' ? close.code : undefined;
601
687
  const reason = typeof close.reason === 'string' ? close.reason : undefined;
602
- if (this.state === 'connecting') {
688
+ if (settingUp()) {
603
689
  settle(new errors_js_1.RealtimeError({
604
690
  message: `Gemini Live closed before it was set up${reason ? `: ${reason}` : ''}`,
605
691
  code: 'websocket_closed_during_connect',
@@ -774,3 +860,55 @@ function dataUrl(url) {
774
860
  const match = /^data:([^;,]+);base64,(.*)$/s.exec(url);
775
861
  return match ? { mimeType: match[1], data: match[2] } : undefined;
776
862
  }
863
+ function configWith(base, session) {
864
+ const next = { ...base };
865
+ const providerSession = { ...base.providerSession };
866
+ let input = { ...base.audio?.input };
867
+ let output = { ...base.audio?.output };
868
+ for (const [key, value] of Object.entries(session)) {
869
+ if (key === 'instructions') {
870
+ if (typeof value === 'string')
871
+ next.instructions = value;
872
+ else
873
+ delete next.instructions;
874
+ }
875
+ else if (key === 'model' && typeof value === 'string')
876
+ next.model = value;
877
+ else if (key === 'output_modalities' && Array.isArray(value)) {
878
+ next.modalities = value.filter((item) => item === 'audio' || item === 'text');
879
+ }
880
+ else if (key === 'voice' && typeof value === 'string')
881
+ output = { ...output, voice: value };
882
+ else if (key === 'tools') {
883
+ next.tools = (Array.isArray(value) ? value : []).filter(transport_utils_js_1.isRecord).map((tool) => ({
884
+ name: String(tool.name ?? ''),
885
+ description: String(tool.description ?? ''),
886
+ ...((0, transport_utils_js_1.isRecord)(tool.parameters) ? { parameters: tool.parameters } : {}),
887
+ }));
888
+ }
889
+ else if (key === 'audio' && (0, transport_utils_js_1.isRecord)(value)) {
890
+ const audioOutput = (0, transport_utils_js_1.isRecord)(value.output) ? value.output : {};
891
+ const audioInput = (0, transport_utils_js_1.isRecord)(value.input) ? value.input : {};
892
+ const transcription = (0, transport_utils_js_1.isRecord)(audioInput.transcription) ? audioInput.transcription : {};
893
+ if (typeof audioOutput.voice === 'string')
894
+ output = { ...output, voice: audioOutput.voice };
895
+ if (typeof transcription.language === 'string')
896
+ input = { ...input, language: transcription.language };
897
+ if ('turn_detection' in audioInput) {
898
+ input = {
899
+ ...input,
900
+ turnDetection: audioInput.turn_detection === null
901
+ ? null
902
+ : (base.audio?.input?.turnDetection ?? { type: 'server_vad' }),
903
+ };
904
+ }
905
+ }
906
+ else if (key !== 'type' && key !== 'tool_choice')
907
+ providerSession[key] = value;
908
+ }
909
+ return {
910
+ ...next,
911
+ audio: { ...base.audio, input, output },
912
+ ...(Object.keys(providerSession).length ? { providerSession } : {}),
913
+ };
914
+ }
@@ -108,6 +108,8 @@ export declare class RealtimeSession {
108
108
  * two go in one message, and the model answers unless `createResponse` is false.
109
109
  */
110
110
  sendImage(image: RealtimeImageInput, options?: RealtimeImageOptions): void;
111
+ /** Sends an image, and records it unless it is a frame of video. */
112
+ private sendPicture;
111
113
  /**
112
114
  * Streams one frame of video, from a camera or a shared screen: as realtime video input on a
113
115
  * provider that takes it, such as Gemini Live, and otherwise as an image in the conversation. A
@@ -7,6 +7,8 @@ const errors_js_1 = require("./errors.js");
7
7
  const events_js_1 = require("./events.js");
8
8
  const id_js_1 = require("./id.js");
9
9
  const openai_events_js_1 = require("./openai-events.js");
10
+ const base64_js_1 = require("../utils/base64.js");
11
+ const sha256_js_1 = require("../utils/sha256.js");
10
12
  const tools_js_1 = require("./tools.js");
11
13
  const transport_utils_js_1 = require("./transport-utils.js");
12
14
  class RealtimeSession {
@@ -278,13 +280,25 @@ class RealtimeSession {
278
280
  this.createResponse();
279
281
  }
280
282
  sendImage(image, options = {}) {
281
- const content = [
282
- { type: 'input_image', image_url: imageUrl(image, options.mimeType || 'image/jpeg') },
283
- ];
283
+ this.sendPicture(image, options, false);
284
+ }
285
+ sendPicture(image, options, frame) {
286
+ const url = imageUrl(image, options.mimeType || 'image/jpeg');
287
+ const content = [{ type: 'input_image', image_url: url }];
284
288
  const text = options.text?.trim() ? options.text : undefined;
285
289
  if (text)
286
290
  content.push({ type: 'input_text', text });
287
291
  this.sendProviderEvent({ type: 'conversation.item.create', item: { type: 'message', role: 'user', content } });
292
+ if (!frame) {
293
+ const comma = url.indexOf(',');
294
+ this.applyEvent({
295
+ type: 'message.image',
296
+ mimeType: url.slice(5, url.indexOf(';')),
297
+ digest: (0, sha256_js_1.sha256Hex)((0, base64_js_1.decodeBase64)(url.slice(comma + 1))),
298
+ ...(this.config.conversation?.recordImages === 'digest' ? {} : { url }),
299
+ timestamp: this.clock.now(),
300
+ });
301
+ }
288
302
  if (options.createResponse ?? Boolean(text))
289
303
  this.createResponse();
290
304
  }
@@ -299,7 +313,7 @@ class RealtimeSession {
299
313
  if (transport.sendVideo)
300
314
  transport.sendVideo(frame instanceof Uint8Array ? frame.slice().buffer : frame, mimeType);
301
315
  else
302
- this.sendImage(frame, { mimeType, createResponse: false });
316
+ this.sendPicture(frame, { mimeType, createResponse: false }, true);
303
317
  return true;
304
318
  }
305
319
  commitAudio(createResponse = true) {
@@ -1236,6 +1250,7 @@ function audioDurationMs(audio, format) {
1236
1250
  return bytesPerSample ? (audio.byteLength / bytesPerSample / rate) * 1_000 : undefined;
1237
1251
  }
1238
1252
  const COMMITTED = new Set([
1253
+ 'message.image',
1239
1254
  'assistant.response.completed',
1240
1255
  'assistant.response.cancelled',
1241
1256
  'user.transcript.completed',
@@ -285,6 +285,22 @@ export interface InterruptionItem extends ConversationItemBase {
285
285
  /** `barge_in` when the user started speaking, `manual` when the application interrupted. */
286
286
  reason: 'barge_in' | 'manual';
287
287
  }
288
+ /** An image the user showed: a photo, a screenshot, a page. A frame of video is not kept. */
289
+ export interface ImageItem extends ConversationItemBase {
290
+ /** Discriminates this item in `ConversationItem`. */
291
+ type: 'image';
292
+ /** Who showed it. */
293
+ role: 'user';
294
+ /** Its MIME type, such as `image/jpeg`. */
295
+ mimeType: string;
296
+ /** The SHA-256 of its bytes, as hex, which tells two images apart without keeping either. */
297
+ digest: string;
298
+ /**
299
+ * The image as it was sent, a data URL, which a resumed conversation gives back. Absent when the
300
+ * record keeps images by digest alone.
301
+ */
302
+ url?: string;
303
+ }
288
304
  /** An error recorded in the conversation, so an export shows where a session went wrong. */
289
305
  export interface RealtimeConversationErrorItem extends ConversationItemBase {
290
306
  /** Discriminates this item in `ConversationItem`. */
@@ -297,10 +313,10 @@ export interface RealtimeConversationErrorItem extends ConversationItemBase {
297
313
  retryable: boolean;
298
314
  }
299
315
  /**
300
- * One entry in a conversation record: speech, text, a tool call or result, an interruption, or an
301
- * error.
316
+ * One entry in a conversation record: speech, text, an image, a tool call or result, an interruption,
317
+ * or an error.
302
318
  */
303
- export type ConversationItem = UserSpeechItem | AssistantSpeechItem | TextMessageItem | ToolCallItem | ToolResultItem | InterruptionItem | RealtimeConversationErrorItem;
319
+ export type ConversationItem = UserSpeechItem | AssistantSpeechItem | TextMessageItem | ImageItem | ToolCallItem | ToolResultItem | InterruptionItem | RealtimeConversationErrorItem;
304
320
  /**
305
321
  * A realtime conversation as a record: what was said, what tools ran, and how it performed. What
306
322
  * exports, analytics, and evaluation read.
@@ -443,6 +459,12 @@ export type RealtimeEvent = {
443
459
  responseId?: string;
444
460
  itemId?: string;
445
461
  timestamp: number;
462
+ } | {
463
+ type: 'message.image';
464
+ mimeType: string;
465
+ digest: string;
466
+ url?: string;
467
+ timestamp: number;
446
468
  } | {
447
469
  type: 'assistant.response.created';
448
470
  responseId: string;
@@ -535,6 +557,10 @@ export interface RealtimeSessionEvents {
535
557
  'message.text.completed': Extract<RealtimeEvent, {
536
558
  type: 'message.text.completed';
537
559
  }>;
560
+ /** The user showed an image, which the record keeps. */
561
+ 'message.image': Extract<RealtimeEvent, {
562
+ type: 'message.image';
563
+ }>;
538
564
  /** The model started a response. */
539
565
  'assistant.response.created': Extract<RealtimeEvent, {
540
566
  type: 'assistant.response.created';
@@ -878,6 +904,12 @@ export interface RealtimeConversationOptions {
878
904
  retainRawAudio?: boolean;
879
905
  /** Rewrites transcript text before it is recorded. */
880
906
  redactTranscript?: (text: string, role: 'user' | 'assistant') => string;
907
+ /**
908
+ * What the record keeps of an image the user sends: `content`, the default, the image itself, so a
909
+ * resumed conversation gives it back; or `digest`, its SHA-256 and type alone, for a record that
910
+ * must hold no pictures.
911
+ */
912
+ recordImages?: 'content' | 'digest';
881
913
  /**
882
914
  * The most recent items a resumed session gives the provider back, so a long conversation does
883
915
  * not overflow the model's context. Defaults to all of them.
@@ -182,10 +182,13 @@ class OutputGuard {
182
182
  const safeFindings = blocked
183
183
  ? findings.map((finding) => (finding.value === undefined ? finding : { ...finding, value: '[REDACTED]' }))
184
184
  : findings;
185
+ const { logprobs, ...unchanged } = response;
186
+ const keepLogprobs = logprobs !== undefined && contentTarget.value === response.content;
185
187
  return {
186
188
  ok: !blocked,
187
189
  value: {
188
- ...response,
190
+ ...unchanged,
191
+ ...(keepLogprobs ? { logprobs } : {}),
189
192
  content: contentTarget.value,
190
193
  ...(response.toolCalls
191
194
  ? {