@zhivex-ai/gemini 0.5.0 → 0.7.0-next.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,5 +1,5 @@
1
1
  import { toJSONSchema } from "zod";
2
- import { CallbackRealtimeSession, ConfigurationError, ProviderHTTPError, UnsupportedFeatureError, createMcpToolSet, createProviderAdapter, encodeAudioFrame, isCallableToolDefinition, isHostedToolDefinition, normalizeFinishReason, openWebSocketConnection, streamSSE, toToolSet, toolResultPayload, withRetry, withTimeoutSignal } from "@zhivex-ai/core";
2
+ import { CallbackRealtimeSession, ConfigurationError, ProviderHTTPError, UnsupportedFeatureError, createMcpToolSet, createProviderAdapter, encodeAudioFrame, encodeMediaFrame, isCallableToolDefinition, isHostedToolDefinition, normalizeFinishReason, openWebSocketConnection, streamSSE, toToolSet, toolResultPayload, withRetry, withTimeoutSignal } from "@zhivex-ai/core";
3
3
  const capabilities = {
4
4
  streaming: true,
5
5
  tools: true,
@@ -8,10 +8,17 @@ const capabilities = {
8
8
  toolChoice: true,
9
9
  parallelToolCalls: false,
10
10
  vision: true,
11
- files: false,
11
+ files: true,
12
12
  audioInput: false,
13
13
  audioOutput: false,
14
14
  embeddings: true,
15
+ fileSearch: true,
16
+ urlContext: true,
17
+ contextCaching: true,
18
+ batch: true,
19
+ interactions: true,
20
+ rawPrediction: true,
21
+ computerUse: true,
15
22
  reasoning: true,
16
23
  webSearch: true,
17
24
  agentCapabilities: {
@@ -19,9 +26,9 @@ const capabilities = {
19
26
  toolChoiceNone: true,
20
27
  approvalRequests: false,
21
28
  hostedWebSearch: true,
22
- hostedFileSearch: false,
29
+ hostedFileSearch: true,
23
30
  remoteMcp: false,
24
- computerUse: false,
31
+ computerUse: true,
25
32
  codeExecution: true,
26
33
  toolsets: false
27
34
  }
@@ -60,6 +67,85 @@ const groundedCapabilities = {
60
67
  ...capabilities,
61
68
  webSearch: true
62
69
  };
70
+ const imageGenerationCapabilities = {
71
+ ...capabilities,
72
+ streaming: false,
73
+ tools: false,
74
+ structuredOutput: false,
75
+ jsonMode: false,
76
+ toolChoice: false,
77
+ parallelToolCalls: false,
78
+ embeddings: false,
79
+ imageGeneration: true,
80
+ videoGeneration: false,
81
+ musicGeneration: false,
82
+ reasoning: false,
83
+ webSearch: false,
84
+ agentCapabilities: {
85
+ supportTier: "tier-c",
86
+ toolChoiceNone: false,
87
+ approvalRequests: false,
88
+ hostedWebSearch: false,
89
+ hostedFileSearch: false,
90
+ remoteMcp: false,
91
+ computerUse: false,
92
+ codeExecution: false,
93
+ toolsets: false
94
+ }
95
+ };
96
+ const videoGenerationCapabilities = {
97
+ ...capabilities,
98
+ streaming: false,
99
+ tools: false,
100
+ structuredOutput: false,
101
+ jsonMode: false,
102
+ toolChoice: false,
103
+ parallelToolCalls: false,
104
+ vision: false,
105
+ embeddings: false,
106
+ imageGeneration: false,
107
+ videoGeneration: true,
108
+ musicGeneration: false,
109
+ reasoning: false,
110
+ webSearch: false,
111
+ agentCapabilities: {
112
+ supportTier: "tier-c",
113
+ toolChoiceNone: false,
114
+ approvalRequests: false,
115
+ hostedWebSearch: false,
116
+ hostedFileSearch: false,
117
+ remoteMcp: false,
118
+ computerUse: false,
119
+ codeExecution: false,
120
+ toolsets: false
121
+ }
122
+ };
123
+ const musicGenerationCapabilities = {
124
+ ...capabilities,
125
+ streaming: false,
126
+ tools: false,
127
+ structuredOutput: false,
128
+ jsonMode: false,
129
+ toolChoice: false,
130
+ parallelToolCalls: false,
131
+ embeddings: false,
132
+ imageGeneration: false,
133
+ videoGeneration: false,
134
+ musicGeneration: true,
135
+ reasoning: false,
136
+ webSearch: false,
137
+ agentCapabilities: {
138
+ supportTier: "tier-c",
139
+ toolChoiceNone: false,
140
+ approvalRequests: false,
141
+ hostedWebSearch: false,
142
+ hostedFileSearch: false,
143
+ remoteMcp: false,
144
+ computerUse: false,
145
+ codeExecution: false,
146
+ toolsets: false
147
+ }
148
+ };
63
149
  const realtimeCapabilities = {
64
150
  ...capabilities,
65
151
  streaming: false,
@@ -69,6 +155,7 @@ const realtimeCapabilities = {
69
155
  sessions: true,
70
156
  audioInput: true,
71
157
  audioOutput: true,
158
+ imageInput: true,
72
159
  tools: true,
73
160
  browserTokens: true
74
161
  }
@@ -89,6 +176,207 @@ const toBase64 = (data) => {
89
176
  const bytes = data instanceof Uint8Array ? data : new Uint8Array(data);
90
177
  return Buffer.from(bytes).toString("base64");
91
178
  };
179
+ const sleep = (ms, signal) => new Promise((resolve, reject) => {
180
+ if (ms <= 0) {
181
+ resolve();
182
+ return;
183
+ }
184
+ if (signal?.aborted) {
185
+ reject(new Error("Operation aborted."));
186
+ return;
187
+ }
188
+ const timeout = setTimeout(() => {
189
+ signal?.removeEventListener("abort", onAbort);
190
+ resolve();
191
+ }, ms);
192
+ const onAbort = () => {
193
+ clearTimeout(timeout);
194
+ reject(new Error("Operation aborted."));
195
+ };
196
+ signal?.addEventListener("abort", onAbort, { once: true });
197
+ });
198
+ const splitGenerationConfig = (providerOptions) => {
199
+ const { generationConfig, ...rest } = providerOptions ?? {};
200
+ return {
201
+ generationConfig: generationConfig && typeof generationConfig === "object" ? generationConfig : {},
202
+ providerOptions: rest
203
+ };
204
+ };
205
+ const mediaInputToPart = (media) => media.uri
206
+ ? {
207
+ fileData: {
208
+ mimeType: media.mediaType,
209
+ fileUri: media.uri
210
+ }
211
+ }
212
+ : {
213
+ inlineData: {
214
+ mimeType: media.mediaType,
215
+ data: media.data ? toBase64(media.data) : ""
216
+ }
217
+ };
218
+ const collectInlineMedia = (json, fallbackMediaType) => {
219
+ const text = [];
220
+ const media = [];
221
+ const candidates = Array.isArray(json.candidates) ? json.candidates : [];
222
+ for (const candidate of candidates) {
223
+ const parts = Array.isArray(candidate?.content?.parts) ? candidate.content.parts : [];
224
+ for (const part of parts) {
225
+ if (typeof part.text === "string" && part.text) {
226
+ text.push(part.text);
227
+ }
228
+ const inlineData = part.inlineData ?? part.inline_data;
229
+ if (inlineData?.data) {
230
+ media.push({
231
+ data: Uint8Array.from(Buffer.from(inlineData.data, "base64")),
232
+ mediaType: inlineData.mimeType ?? inlineData.mime_type ?? fallbackMediaType,
233
+ text: typeof part.text === "string" ? part.text : undefined
234
+ });
235
+ }
236
+ }
237
+ }
238
+ return {
239
+ media,
240
+ text: text.length ? text.join("\n") : undefined
241
+ };
242
+ };
243
+ const mediaInputToVeoImage = (media) => media.uri
244
+ ? {
245
+ gcsUri: media.uri,
246
+ mimeType: media.mediaType
247
+ }
248
+ : {
249
+ bytesBase64Encoded: media.data ? toBase64(media.data) : "",
250
+ mimeType: media.mediaType
251
+ };
252
+ const collectVideos = (json) => {
253
+ const samples = json.response?.generateVideoResponse?.generatedSamples ??
254
+ json.response?.generatedVideos ??
255
+ json.response?.generated_videos ??
256
+ [];
257
+ return (Array.isArray(samples) ? samples : [])
258
+ .map((sample) => sample.video ?? sample)
259
+ .map((video) => ({
260
+ data: video.videoBytes
261
+ ? Uint8Array.from(Buffer.from(video.videoBytes, "base64"))
262
+ : video.bytesBase64Encoded
263
+ ? Uint8Array.from(Buffer.from(video.bytesBase64Encoded, "base64"))
264
+ : undefined,
265
+ uri: video.uri ?? video.gcsUri,
266
+ mediaType: video.mimeType ?? "video/mp4",
267
+ providerMetadata: video
268
+ }))
269
+ .filter((video) => video.data || video.uri);
270
+ };
271
+ const dataToBytes = async (data) => {
272
+ if (typeof data === "string") {
273
+ return new TextEncoder().encode(data);
274
+ }
275
+ if (data instanceof Uint8Array) {
276
+ return data;
277
+ }
278
+ if (data instanceof ArrayBuffer) {
279
+ return new Uint8Array(data);
280
+ }
281
+ return new Uint8Array(await data.arrayBuffer());
282
+ };
283
+ const bytesToArrayBuffer = (bytes) => bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
284
+ const normalizeUploadedFile = (json) => {
285
+ const file = json.file ?? json;
286
+ return {
287
+ name: file.name ?? "",
288
+ uri: file.uri ?? file.fileUri ?? file.file_uri,
289
+ mimeType: file.mimeType ?? file.mime_type,
290
+ sizeBytes: file.sizeBytes ?? file.size_bytes,
291
+ state: file.state,
292
+ displayName: file.displayName ?? file.display_name,
293
+ rawResponse: json,
294
+ providerMetadata: file
295
+ };
296
+ };
297
+ const normalizeFileSearchStore = (json) => ({
298
+ name: json.name ?? "",
299
+ displayName: json.displayName ?? json.display_name,
300
+ createTime: json.createTime ?? json.create_time,
301
+ updateTime: json.updateTime ?? json.update_time,
302
+ rawResponse: json,
303
+ providerMetadata: json
304
+ });
305
+ const normalizeCachedContent = (json) => ({
306
+ name: json.name ?? "",
307
+ model: json.model,
308
+ displayName: json.displayName ?? json.display_name,
309
+ createTime: json.createTime ?? json.create_time,
310
+ updateTime: json.updateTime ?? json.update_time,
311
+ expireTime: json.expireTime ?? json.expire_time,
312
+ usageMetadata: json.usageMetadata ?? json.usage_metadata,
313
+ rawResponse: json,
314
+ providerMetadata: json
315
+ });
316
+ const normalizeBatchJob = (json) => ({
317
+ name: json.name ?? "",
318
+ model: json.model,
319
+ state: json.state ?? json.metadata?.state,
320
+ done: json.done,
321
+ createTime: json.createTime ?? json.create_time ?? json.metadata?.createTime,
322
+ updateTime: json.updateTime ?? json.update_time ?? json.metadata?.updateTime,
323
+ rawResponse: json,
324
+ providerMetadata: json
325
+ });
326
+ const normalizeInteraction = (json) => ({
327
+ id: json.id ?? json.name ?? "",
328
+ name: json.name,
329
+ model: json.model,
330
+ status: json.status,
331
+ outputs: json.outputs,
332
+ rawResponse: json,
333
+ providerMetadata: json
334
+ });
335
+ const normalizeOperation = (json) => ({
336
+ name: json.name ?? "",
337
+ done: json.done,
338
+ response: json.response,
339
+ error: json.error,
340
+ metadata: json.metadata,
341
+ rawResponse: json
342
+ });
343
+ const normalizePredictionResult = (json) => ({
344
+ predictions: json.predictions,
345
+ operationName: json.name,
346
+ operation: json.name || json.done !== undefined ? normalizeOperation(json) : undefined,
347
+ rawResponse: json,
348
+ providerMetadata: json
349
+ });
350
+ const appendQuery = (url, query) => {
351
+ const parsed = new URL(url);
352
+ for (const [key, value] of Object.entries(query)) {
353
+ if (value !== undefined) {
354
+ parsed.searchParams.set(key, String(value));
355
+ }
356
+ }
357
+ return parsed.toString();
358
+ };
359
+ const geminiUploadBaseURL = (baseURL) => baseURL.replace(/\/v1beta\/?$/, "/upload/v1beta");
360
+ const pollGeminiOperation = async (operation, baseURL, apiKey, fetcher, options) => {
361
+ if (options.pollIntervalMs === undefined && options.timeoutMs === undefined) {
362
+ return operation;
363
+ }
364
+ const startedAt = Date.now();
365
+ const timeoutMs = options.timeoutMs ?? 600_000;
366
+ let current = operation;
367
+ while (!current.done) {
368
+ if (Date.now() - startedAt >= timeoutMs) {
369
+ throw new Error(`Gemini operation "${operation.name}" timed out after ${timeoutMs}ms.`);
370
+ }
371
+ await sleep(options.pollIntervalMs ?? 5_000, options.abortSignal);
372
+ const response = await withRetry(() => fetcher(`${baseURL}/${current.name}?key=${apiKey}`, {
373
+ method: "GET",
374
+ signal: options.abortSignal
375
+ }), options);
376
+ current = normalizeOperation(await parseJson(response));
377
+ }
378
+ return current;
379
+ };
92
380
  const systemInstruction = (messages) => {
93
381
  const text = messages
94
382
  .filter((message) => message.role === "system")
@@ -109,6 +397,13 @@ const mapPart = (part) => {
109
397
  data: part.image
110
398
  }
111
399
  };
400
+ case "file":
401
+ return {
402
+ fileData: {
403
+ mimeType: part.mediaType,
404
+ fileUri: part.data
405
+ }
406
+ };
112
407
  case "tool-call":
113
408
  return {
114
409
  functionCall: {
@@ -483,6 +778,9 @@ const parseAssistantMessage = (candidate) => ({
483
778
  }
484
779
  };
485
780
  }
781
+ if (part.inlineData?.data && String(part.inlineData.mimeType ?? "").startsWith("image/")) {
782
+ return { type: "image", image: part.inlineData.data, mediaType: part.inlineData.mimeType };
783
+ }
486
784
  return { type: "text", text: JSON.stringify(part) };
487
785
  }) ?? []
488
786
  });
@@ -494,162 +792,746 @@ const extractGroundingSources = (candidate) => (candidate?.groundingMetadata?.gr
494
792
  providerMetadata: chunk
495
793
  }))
496
794
  .filter((source) => typeof source.url === "string");
497
- class GeminiLanguageModel {
498
- modelId;
795
+ class GeminiFilesClient {
499
796
  apiKey;
500
797
  baseURL;
501
798
  fetcher;
502
- provider = "gemini";
503
- capabilities = capabilities;
504
- constructor(modelId, apiKey, baseURL, fetcher) {
505
- this.modelId = modelId;
799
+ constructor(apiKey, baseURL, fetcher) {
506
800
  this.apiKey = apiKey;
507
801
  this.baseURL = baseURL;
508
802
  this.fetcher = fetcher;
509
803
  }
510
- url(action) {
511
- const separator = action.includes("?") ? "&" : "?";
512
- return `${this.baseURL}/models/${this.modelId}:${action}${separator}key=${this.apiKey}`;
804
+ url(path, query = {}) {
805
+ return appendQuery(`${this.baseURL}/${path}`, { key: this.apiKey, ...query });
513
806
  }
514
- async generate(input) {
807
+ uploadUrl(path) {
808
+ return appendQuery(`${geminiUploadBaseURL(this.baseURL)}/${path}`, { key: this.apiKey });
809
+ }
810
+ async upload(input) {
811
+ const bytes = await dataToBytes(input.data);
515
812
  const { signal, cleanup } = withTimeoutSignal(input);
516
813
  try {
517
- const response = await withRetry(() => this.fetcher(this.url("generateContent"), {
814
+ const startResponse = await withRetry(() => this.fetcher(this.uploadUrl("files"), {
518
815
  method: "POST",
519
- headers: { "content-type": "application/json" },
816
+ headers: {
817
+ "content-type": "application/json",
818
+ "x-goog-upload-protocol": "resumable",
819
+ "x-goog-upload-command": "start",
820
+ "x-goog-upload-header-content-length": String(bytes.byteLength),
821
+ "x-goog-upload-header-content-type": input.mediaType
822
+ },
520
823
  signal,
521
824
  body: JSON.stringify({
522
- contents: mapMessages(input.messages),
523
- systemInstruction: systemInstruction(input.messages),
524
- tools: mapTools(input.tools),
525
- ...input.providerOptions,
526
- toolConfig: mapToolConfig(input.toolChoice, input.tools),
527
- generationConfig: generationConfig(this.modelId, input)
825
+ file: {
826
+ ...(input.displayName ? { displayName: input.displayName } : {}),
827
+ ...(input.name ? { name: input.name } : {}),
828
+ ...(input.providerOptions ?? {})
829
+ }
528
830
  })
529
831
  }), input);
530
- const json = await parseJson(response);
531
- const candidate = json.candidates?.[0];
532
- const assistantMessage = parseAssistantMessage(candidate);
533
- return {
534
- messages: [assistantMessage],
535
- text: assistantMessage.parts
536
- .filter((part) => part.type === "text")
537
- .map((part) => part.text)
538
- .join(""),
539
- finishReason: normalizeFinishReason(candidate?.finishReason),
540
- providerFinishReason: candidate?.finishReason,
541
- rawResponse: json
542
- };
832
+ if (!startResponse.ok) {
833
+ await parseJson(startResponse);
834
+ }
835
+ const resumableUrl = startResponse.headers.get("x-goog-upload-url");
836
+ if (!resumableUrl) {
837
+ throw new ProviderHTTPError('Gemini file upload did not return "x-goog-upload-url".', 500);
838
+ }
839
+ const uploadResponse = await withRetry(() => this.fetcher(resumableUrl, {
840
+ method: "POST",
841
+ headers: {
842
+ "content-type": input.mediaType,
843
+ "x-goog-upload-command": "upload, finalize",
844
+ "x-goog-upload-offset": "0"
845
+ },
846
+ signal,
847
+ body: new Blob([bytesToArrayBuffer(bytes)], { type: input.mediaType })
848
+ }), input);
849
+ return normalizeUploadedFile(await parseJson(uploadResponse));
543
850
  }
544
851
  finally {
545
852
  cleanup();
546
853
  }
547
854
  }
548
- async stream(input) {
855
+ async get(input) {
549
856
  const { signal, cleanup } = withTimeoutSignal(input);
550
- const response = await withRetry(() => this.fetcher(this.url("streamGenerateContent?alt=sse"), {
551
- method: "POST",
552
- headers: { "content-type": "application/json" },
553
- signal,
554
- body: JSON.stringify({
555
- contents: mapMessages(input.messages),
556
- systemInstruction: systemInstruction(input.messages),
557
- tools: mapTools(input.tools),
558
- ...input.providerOptions,
559
- toolConfig: mapToolConfig(input.toolChoice, input.tools),
560
- generationConfig: generationConfig(this.modelId, input)
561
- })
562
- }), input);
563
- return (async function* () {
564
- try {
565
- for await (const event of streamSSE(response)) {
566
- const json = JSON.parse(event.data);
567
- const candidate = json.candidates?.[0];
568
- const parts = candidate?.content?.parts ?? [];
569
- for (const [index, part] of parts.entries()) {
570
- if (part.text) {
571
- yield { type: "text-delta", textDelta: part.text };
572
- }
573
- if (part.functionCall) {
574
- yield {
575
- type: "tool-call",
576
- toolCall: {
577
- id: part.functionCall.id ?? `${part.functionCall.name}-${index}`,
578
- name: part.functionCall.name,
579
- input: part.functionCall.args ?? {}
580
- }
581
- };
582
- }
583
- }
584
- if (candidate?.finishReason) {
585
- yield {
586
- type: "finish",
587
- finishReason: normalizeFinishReason(candidate.finishReason),
588
- providerFinishReason: candidate.finishReason
589
- };
590
- }
591
- }
592
- }
593
- finally {
594
- cleanup();
595
- }
596
- })();
597
- }
598
- }
599
- class GeminiEmbeddingModel {
600
- modelId;
601
- apiKey;
602
- baseURL;
603
- fetcher;
604
- provider = "gemini";
605
- capabilities = capabilities;
606
- constructor(modelId, apiKey, baseURL, fetcher) {
607
- this.modelId = modelId;
608
- this.apiKey = apiKey;
609
- this.baseURL = baseURL;
610
- this.fetcher = fetcher;
857
+ try {
858
+ const response = await withRetry(() => this.fetcher(this.url(input.name), { method: "GET", signal }), input);
859
+ return normalizeUploadedFile(await parseJson(response));
860
+ }
861
+ finally {
862
+ cleanup();
863
+ }
611
864
  }
612
- async embed(input) {
865
+ async list(input = {}) {
613
866
  const { signal, cleanup } = withTimeoutSignal(input);
614
867
  try {
615
- const embeddings = await Promise.all(input.values.map(async (value) => {
616
- const response = await withRetry(() => this.fetcher(`${this.baseURL}/models/${this.modelId}:embedContent?key=${this.apiKey}`, {
617
- method: "POST",
618
- headers: { "content-type": "application/json" },
619
- signal,
620
- body: JSON.stringify({
621
- content: { parts: [{ text: value }] }
622
- })
623
- }), input);
624
- const json = await parseJson(response);
625
- return json.embedding.values;
626
- }));
868
+ const response = await withRetry(() => this.fetcher(this.url("files", { pageSize: input.pageSize, pageToken: input.pageToken }), { method: "GET", signal }), input);
869
+ const json = await parseJson(response);
627
870
  return {
628
- embeddings
871
+ files: (json.files ?? []).map(normalizeUploadedFile),
872
+ nextPageToken: json.nextPageToken,
873
+ rawResponse: json
629
874
  };
630
875
  }
631
876
  finally {
632
877
  cleanup();
633
878
  }
634
879
  }
880
+ async delete(input) {
881
+ const { signal, cleanup } = withTimeoutSignal(input);
882
+ try {
883
+ const response = await withRetry(() => this.fetcher(this.url(input.name), { method: "DELETE", signal }), input);
884
+ const json = await parseJson(response);
885
+ return { name: input.name, rawResponse: json };
886
+ }
887
+ finally {
888
+ cleanup();
889
+ }
890
+ }
635
891
  }
636
- class GeminiTranscriptionModel {
637
- modelId;
892
+ class GeminiFileSearchStoresClient {
638
893
  apiKey;
639
894
  baseURL;
640
895
  fetcher;
641
- provider = "gemini";
642
- capabilities = transcriptionCapabilities;
643
- constructor(modelId, apiKey, baseURL, fetcher) {
644
- this.modelId = modelId;
896
+ constructor(apiKey, baseURL, fetcher) {
645
897
  this.apiKey = apiKey;
646
898
  this.baseURL = baseURL;
647
899
  this.fetcher = fetcher;
648
900
  }
649
- async transcribe(input) {
901
+ url(path, query = {}) {
902
+ return appendQuery(`${this.baseURL}/${path}`, { key: this.apiKey, ...query });
903
+ }
904
+ uploadUrl(path) {
905
+ return appendQuery(`${geminiUploadBaseURL(this.baseURL)}/${path}`, { key: this.apiKey });
906
+ }
907
+ async create(input = {}) {
650
908
  const { signal, cleanup } = withTimeoutSignal(input);
651
909
  try {
652
- const response = await withRetry(() => this.fetcher(`${this.baseURL}/models/${this.modelId}:generateContent?key=${this.apiKey}`, {
910
+ const response = await withRetry(() => this.fetcher(this.url("fileSearchStores"), {
911
+ method: "POST",
912
+ headers: { "content-type": "application/json" },
913
+ signal,
914
+ body: JSON.stringify({
915
+ ...(input.displayName ? { displayName: input.displayName } : {}),
916
+ ...(input.providerOptions ?? {})
917
+ })
918
+ }), input);
919
+ return normalizeFileSearchStore(await parseJson(response));
920
+ }
921
+ finally {
922
+ cleanup();
923
+ }
924
+ }
925
+ async upload(input) {
926
+ const bytes = await dataToBytes(input.data);
927
+ const { signal, cleanup } = withTimeoutSignal(input);
928
+ try {
929
+ const startResponse = await withRetry(() => this.fetcher(this.uploadUrl(`${input.storeName}:uploadToFileSearchStore`), {
930
+ method: "POST",
931
+ headers: {
932
+ "content-type": "application/json",
933
+ "x-goog-upload-protocol": "resumable",
934
+ "x-goog-upload-command": "start",
935
+ "x-goog-upload-header-content-length": String(bytes.byteLength),
936
+ "x-goog-upload-header-content-type": input.mediaType
937
+ },
938
+ signal,
939
+ body: JSON.stringify({
940
+ file: {
941
+ ...(input.displayName ? { displayName: input.displayName } : {}),
942
+ ...(input.filename ? { name: input.filename } : {})
943
+ },
944
+ ...(input.providerOptions ?? {})
945
+ })
946
+ }), input);
947
+ if (!startResponse.ok) {
948
+ await parseJson(startResponse);
949
+ }
950
+ const resumableUrl = startResponse.headers.get("x-goog-upload-url");
951
+ if (!resumableUrl) {
952
+ throw new ProviderHTTPError('Gemini file search upload did not return "x-goog-upload-url".', 500);
953
+ }
954
+ const uploadResponse = await withRetry(() => this.fetcher(resumableUrl, {
955
+ method: "POST",
956
+ headers: {
957
+ "content-type": input.mediaType,
958
+ "x-goog-upload-command": "upload, finalize",
959
+ "x-goog-upload-offset": "0"
960
+ },
961
+ signal,
962
+ body: new Blob([bytesToArrayBuffer(bytes)], { type: input.mediaType })
963
+ }), input);
964
+ return pollGeminiOperation(normalizeOperation(await parseJson(uploadResponse)), this.baseURL, this.apiKey, this.fetcher, {
965
+ ...input,
966
+ abortSignal: signal
967
+ });
968
+ }
969
+ finally {
970
+ cleanup();
971
+ }
972
+ }
973
+ async importFile(input) {
974
+ const { signal, cleanup } = withTimeoutSignal(input);
975
+ try {
976
+ const response = await withRetry(() => this.fetcher(this.url(`${input.storeName}:importFile`), {
977
+ method: "POST",
978
+ headers: { "content-type": "application/json" },
979
+ signal,
980
+ body: JSON.stringify({
981
+ fileName: input.fileName,
982
+ ...(input.providerOptions ?? {})
983
+ })
984
+ }), input);
985
+ return pollGeminiOperation(normalizeOperation(await parseJson(response)), this.baseURL, this.apiKey, this.fetcher, {
986
+ ...input,
987
+ abortSignal: signal
988
+ });
989
+ }
990
+ finally {
991
+ cleanup();
992
+ }
993
+ }
994
+ async get(input) {
995
+ const { signal, cleanup } = withTimeoutSignal(input);
996
+ try {
997
+ const response = await withRetry(() => this.fetcher(this.url(input.name), { method: "GET", signal }), input);
998
+ return normalizeFileSearchStore(await parseJson(response));
999
+ }
1000
+ finally {
1001
+ cleanup();
1002
+ }
1003
+ }
1004
+ async list(input = {}) {
1005
+ const { signal, cleanup } = withTimeoutSignal(input);
1006
+ try {
1007
+ const response = await withRetry(() => this.fetcher(this.url("fileSearchStores", { pageSize: input.pageSize, pageToken: input.pageToken }), { method: "GET", signal }), input);
1008
+ const json = await parseJson(response);
1009
+ return {
1010
+ stores: (json.fileSearchStores ?? json.file_search_stores ?? []).map(normalizeFileSearchStore),
1011
+ nextPageToken: json.nextPageToken,
1012
+ rawResponse: json
1013
+ };
1014
+ }
1015
+ finally {
1016
+ cleanup();
1017
+ }
1018
+ }
1019
+ async delete(input) {
1020
+ const { signal, cleanup } = withTimeoutSignal(input);
1021
+ try {
1022
+ const response = await withRetry(() => this.fetcher(this.url(input.name), { method: "DELETE", signal }), input);
1023
+ const json = await parseJson(response);
1024
+ return { name: input.name, rawResponse: json };
1025
+ }
1026
+ finally {
1027
+ cleanup();
1028
+ }
1029
+ }
1030
+ }
1031
+ class GeminiContextCachesClient {
1032
+ apiKey;
1033
+ baseURL;
1034
+ fetcher;
1035
+ constructor(apiKey, baseURL, fetcher) {
1036
+ this.apiKey = apiKey;
1037
+ this.baseURL = baseURL;
1038
+ this.fetcher = fetcher;
1039
+ }
1040
+ url(path, query = {}) {
1041
+ return appendQuery(`${this.baseURL}/${path}`, { key: this.apiKey, ...query });
1042
+ }
1043
+ async create(input) {
1044
+ const { signal, cleanup } = withTimeoutSignal(input);
1045
+ try {
1046
+ const response = await withRetry(() => this.fetcher(this.url("cachedContents"), {
1047
+ method: "POST",
1048
+ headers: { "content-type": "application/json" },
1049
+ signal,
1050
+ body: JSON.stringify({
1051
+ model: input.modelId.startsWith("models/") ? input.modelId : `models/${input.modelId}`,
1052
+ contents: mapMessages(input.contents),
1053
+ ...(input.system ? { systemInstruction: { parts: [{ text: input.system }] } } : { systemInstruction: systemInstruction(input.contents) }),
1054
+ ...(input.tools ? { tools: mapTools(toToolSet(input.tools)) } : {}),
1055
+ ...(input.displayName ? { displayName: input.displayName } : {}),
1056
+ ...(input.ttl ? { ttl: input.ttl } : {}),
1057
+ ...(input.expireTime ? { expireTime: input.expireTime } : {}),
1058
+ ...(input.providerOptions ?? {})
1059
+ })
1060
+ }), input);
1061
+ return normalizeCachedContent(await parseJson(response));
1062
+ }
1063
+ finally {
1064
+ cleanup();
1065
+ }
1066
+ }
1067
+ async get(input) {
1068
+ const { signal, cleanup } = withTimeoutSignal(input);
1069
+ try {
1070
+ const response = await withRetry(() => this.fetcher(this.url(input.name), { method: "GET", signal }), input);
1071
+ return normalizeCachedContent(await parseJson(response));
1072
+ }
1073
+ finally {
1074
+ cleanup();
1075
+ }
1076
+ }
1077
+ async list(input = {}) {
1078
+ const { signal, cleanup } = withTimeoutSignal(input);
1079
+ try {
1080
+ const response = await withRetry(() => this.fetcher(this.url("cachedContents", { pageSize: input.pageSize, pageToken: input.pageToken }), { method: "GET", signal }), input);
1081
+ const json = await parseJson(response);
1082
+ return {
1083
+ caches: (json.cachedContents ?? json.cached_contents ?? []).map(normalizeCachedContent),
1084
+ nextPageToken: json.nextPageToken,
1085
+ rawResponse: json
1086
+ };
1087
+ }
1088
+ finally {
1089
+ cleanup();
1090
+ }
1091
+ }
1092
+ async delete(input) {
1093
+ const { signal, cleanup } = withTimeoutSignal(input);
1094
+ try {
1095
+ const response = await withRetry(() => this.fetcher(this.url(input.name), { method: "DELETE", signal }), input);
1096
+ const json = await parseJson(response);
1097
+ return { name: input.name, rawResponse: json };
1098
+ }
1099
+ finally {
1100
+ cleanup();
1101
+ }
1102
+ }
1103
+ }
1104
+ class GeminiBatchesClient {
1105
+ apiKey;
1106
+ baseURL;
1107
+ fetcher;
1108
+ constructor(apiKey, baseURL, fetcher) {
1109
+ this.apiKey = apiKey;
1110
+ this.baseURL = baseURL;
1111
+ this.fetcher = fetcher;
1112
+ }
1113
+ url(path, query = {}) {
1114
+ return appendQuery(`${this.baseURL}/${path}`, { key: this.apiKey, ...query });
1115
+ }
1116
+ async create(input) {
1117
+ const { signal, cleanup } = withTimeoutSignal(input);
1118
+ try {
1119
+ const response = await withRetry(() => this.fetcher(this.url(`models/${input.modelId}:batchGenerateContent`), {
1120
+ method: "POST",
1121
+ headers: { "content-type": "application/json" },
1122
+ signal,
1123
+ body: JSON.stringify({
1124
+ batch: {
1125
+ ...(input.displayName ? { displayName: input.displayName } : {}),
1126
+ inputConfig: input.fileName
1127
+ ? { fileName: input.fileName }
1128
+ : {
1129
+ requests: {
1130
+ requests: input.requests ?? []
1131
+ }
1132
+ },
1133
+ ...(input.providerOptions ?? {})
1134
+ }
1135
+ })
1136
+ }), input);
1137
+ return normalizeBatchJob(await parseJson(response));
1138
+ }
1139
+ finally {
1140
+ cleanup();
1141
+ }
1142
+ }
1143
+ async get(input) {
1144
+ const { signal, cleanup } = withTimeoutSignal(input);
1145
+ try {
1146
+ const response = await withRetry(() => this.fetcher(this.url(input.name), { method: "GET", signal }), input);
1147
+ return normalizeBatchJob(await parseJson(response));
1148
+ }
1149
+ finally {
1150
+ cleanup();
1151
+ }
1152
+ }
1153
+ async list(input = {}) {
1154
+ const { signal, cleanup } = withTimeoutSignal(input);
1155
+ try {
1156
+ const response = await withRetry(() => this.fetcher(this.url("batches", { pageSize: input.pageSize, pageToken: input.pageToken }), { method: "GET", signal }), input);
1157
+ const json = await parseJson(response);
1158
+ return {
1159
+ batches: (json.batches ?? []).map(normalizeBatchJob),
1160
+ nextPageToken: json.nextPageToken,
1161
+ rawResponse: json
1162
+ };
1163
+ }
1164
+ finally {
1165
+ cleanup();
1166
+ }
1167
+ }
1168
+ async cancel(input) {
1169
+ const { signal, cleanup } = withTimeoutSignal(input);
1170
+ try {
1171
+ const response = await withRetry(() => this.fetcher(this.url(`${input.name}:cancel`), {
1172
+ method: "POST",
1173
+ headers: { "content-type": "application/json" },
1174
+ signal,
1175
+ body: JSON.stringify(input.providerOptions ?? {})
1176
+ }), input);
1177
+ return normalizeBatchJob(await parseJson(response));
1178
+ }
1179
+ finally {
1180
+ cleanup();
1181
+ }
1182
+ }
1183
+ async delete(input) {
1184
+ const { signal, cleanup } = withTimeoutSignal(input);
1185
+ try {
1186
+ const response = await withRetry(() => this.fetcher(this.url(`${input.name}:delete`), { method: "POST", signal }), input);
1187
+ const json = await parseJson(response);
1188
+ return { name: input.name, rawResponse: json };
1189
+ }
1190
+ finally {
1191
+ cleanup();
1192
+ }
1193
+ }
1194
+ }
1195
+ class GeminiInteractionsClient {
1196
+ apiKey;
1197
+ baseURL;
1198
+ fetcher;
1199
+ constructor(apiKey, baseURL, fetcher) {
1200
+ this.apiKey = apiKey;
1201
+ this.baseURL = baseURL;
1202
+ this.fetcher = fetcher;
1203
+ }
1204
+ url(path, query = {}) {
1205
+ return appendQuery(`${this.baseURL}/${path}`, { key: this.apiKey, ...query });
1206
+ }
1207
+ body(input, stream = false) {
1208
+ return {
1209
+ ...(input.modelId ? { model: input.modelId } : {}),
1210
+ ...(input.agent ? { agent: input.agent } : {}),
1211
+ input: input.input,
1212
+ ...(input.previousInteractionId ? { previous_interaction_id: input.previousInteractionId } : {}),
1213
+ ...(input.tools ? { tools: mapTools(toToolSet(input.tools)) } : {}),
1214
+ ...(input.background !== undefined ? { background: input.background } : {}),
1215
+ ...(input.store !== undefined ? { store: input.store } : {}),
1216
+ ...(stream ? { stream: true } : {}),
1217
+ ...(input.providerOptions ?? {})
1218
+ };
1219
+ }
1220
+ async create(input) {
1221
+ const { signal, cleanup } = withTimeoutSignal(input);
1222
+ try {
1223
+ const response = await withRetry(() => this.fetcher(this.url("interactions"), {
1224
+ method: "POST",
1225
+ headers: { "content-type": "application/json" },
1226
+ signal,
1227
+ body: JSON.stringify(this.body(input))
1228
+ }), input);
1229
+ return normalizeInteraction(await parseJson(response));
1230
+ }
1231
+ finally {
1232
+ cleanup();
1233
+ }
1234
+ }
1235
+ async get(input) {
1236
+ const { signal, cleanup } = withTimeoutSignal(input);
1237
+ try {
1238
+ const response = await withRetry(() => this.fetcher(this.url(`interactions/${input.id}`), { method: "GET", signal }), input);
1239
+ return normalizeInteraction(await parseJson(response));
1240
+ }
1241
+ finally {
1242
+ cleanup();
1243
+ }
1244
+ }
1245
+ async stream(input) {
1246
+ const { signal, cleanup } = withTimeoutSignal(input);
1247
+ const response = await withRetry(() => this.fetcher(this.url("interactions", { alt: "sse" }), {
1248
+ method: "POST",
1249
+ headers: { "content-type": "application/json" },
1250
+ signal,
1251
+ body: JSON.stringify(this.body(input, true))
1252
+ }), input);
1253
+ return (async function* () {
1254
+ try {
1255
+ for await (const event of streamSSE(response)) {
1256
+ const json = JSON.parse(event.data);
1257
+ const text = json.text ??
1258
+ json.delta?.text ??
1259
+ json.output?.text ??
1260
+ (Array.isArray(json.outputs) ? json.outputs.find((output) => typeof output.text === "string")?.text : undefined);
1261
+ if (typeof text === "string" && text) {
1262
+ yield { type: "text-delta", textDelta: text };
1263
+ }
1264
+ else {
1265
+ yield { type: "provider-data", provider: "gemini", data: json };
1266
+ }
1267
+ if (json.status === "completed" || json.done) {
1268
+ yield { type: "finish", finishReason: "stop" };
1269
+ }
1270
+ }
1271
+ }
1272
+ finally {
1273
+ cleanup();
1274
+ }
1275
+ })();
1276
+ }
1277
+ }
1278
+ class GeminiPredictionModel {
1279
+ modelId;
1280
+ apiKey;
1281
+ baseURL;
1282
+ fetcher;
1283
+ provider = "gemini";
1284
+ capabilities = {
1285
+ ...capabilities,
1286
+ rawPrediction: true
1287
+ };
1288
+ constructor(modelId, apiKey, baseURL, fetcher) {
1289
+ this.modelId = modelId;
1290
+ this.apiKey = apiKey;
1291
+ this.baseURL = baseURL;
1292
+ this.fetcher = fetcher;
1293
+ }
1294
+ url(action) {
1295
+ return `${this.baseURL}/models/${this.modelId}:${action}?key=${this.apiKey}`;
1296
+ }
1297
+ body(input) {
1298
+ return input.body ?? {
1299
+ ...(input.instances ? { instances: input.instances } : {}),
1300
+ ...(input.parameters ? { parameters: input.parameters } : {}),
1301
+ ...(input.providerOptions ?? {})
1302
+ };
1303
+ }
1304
+ async predictRaw(input) {
1305
+ const action = typeof input.providerOptions?.action === "string" ? input.providerOptions.action : "predict";
1306
+ const { signal, cleanup } = withTimeoutSignal(input);
1307
+ try {
1308
+ const response = await withRetry(() => this.fetcher(this.url(action), {
1309
+ method: "POST",
1310
+ headers: { "content-type": "application/json" },
1311
+ signal,
1312
+ body: JSON.stringify(this.body(input))
1313
+ }), input);
1314
+ return normalizePredictionResult(await parseJson(response));
1315
+ }
1316
+ finally {
1317
+ cleanup();
1318
+ }
1319
+ }
1320
+ async rawPredict(input) {
1321
+ const { signal, cleanup } = withTimeoutSignal(input);
1322
+ try {
1323
+ const response = await withRetry(() => this.fetcher(this.url("rawPredict"), {
1324
+ method: "POST",
1325
+ headers: { "content-type": "application/json" },
1326
+ signal,
1327
+ body: JSON.stringify(this.body(input))
1328
+ }), input);
1329
+ return normalizePredictionResult(await parseJson(response));
1330
+ }
1331
+ finally {
1332
+ cleanup();
1333
+ }
1334
+ }
1335
+ async invoke(input) {
1336
+ const { signal, cleanup } = withTimeoutSignal(input);
1337
+ try {
1338
+ const response = await withRetry(() => this.fetcher(this.url("invoke"), {
1339
+ method: "POST",
1340
+ headers: { "content-type": "application/json" },
1341
+ signal,
1342
+ body: JSON.stringify(this.body(input))
1343
+ }), input);
1344
+ return normalizePredictionResult(await parseJson(response));
1345
+ }
1346
+ finally {
1347
+ cleanup();
1348
+ }
1349
+ }
1350
+ async predictLongRunning(input) {
1351
+ const { signal, cleanup } = withTimeoutSignal(input);
1352
+ try {
1353
+ const response = await withRetry(() => this.fetcher(this.url("predictLongRunning"), {
1354
+ method: "POST",
1355
+ headers: { "content-type": "application/json" },
1356
+ signal,
1357
+ body: JSON.stringify(this.body(input))
1358
+ }), input);
1359
+ return normalizeOperation(await parseJson(response));
1360
+ }
1361
+ finally {
1362
+ cleanup();
1363
+ }
1364
+ }
1365
+ async fetchPredictionOperation(input) {
1366
+ const { signal, cleanup } = withTimeoutSignal(input);
1367
+ try {
1368
+ const response = await withRetry(() => this.fetcher(`${this.baseURL}/${input.name}?key=${this.apiKey}`, {
1369
+ method: "GET",
1370
+ signal
1371
+ }), input);
1372
+ return normalizeOperation(await parseJson(response));
1373
+ }
1374
+ finally {
1375
+ cleanup();
1376
+ }
1377
+ }
1378
+ }
1379
+ class GeminiLanguageModel {
1380
+ modelId;
1381
+ apiKey;
1382
+ baseURL;
1383
+ fetcher;
1384
+ provider = "gemini";
1385
+ capabilities = capabilities;
1386
+ constructor(modelId, apiKey, baseURL, fetcher) {
1387
+ this.modelId = modelId;
1388
+ this.apiKey = apiKey;
1389
+ this.baseURL = baseURL;
1390
+ this.fetcher = fetcher;
1391
+ }
1392
+ url(action) {
1393
+ const separator = action.includes("?") ? "&" : "?";
1394
+ return `${this.baseURL}/models/${this.modelId}:${action}${separator}key=${this.apiKey}`;
1395
+ }
1396
+ async generate(input) {
1397
+ const { signal, cleanup } = withTimeoutSignal(input);
1398
+ try {
1399
+ const response = await withRetry(() => this.fetcher(this.url("generateContent"), {
1400
+ method: "POST",
1401
+ headers: { "content-type": "application/json" },
1402
+ signal,
1403
+ body: JSON.stringify({
1404
+ contents: mapMessages(input.messages),
1405
+ systemInstruction: systemInstruction(input.messages),
1406
+ tools: mapTools(input.tools),
1407
+ ...input.providerOptions,
1408
+ toolConfig: mapToolConfig(input.toolChoice, input.tools),
1409
+ generationConfig: generationConfig(this.modelId, input)
1410
+ })
1411
+ }), input);
1412
+ const json = await parseJson(response);
1413
+ const candidate = json.candidates?.[0];
1414
+ const assistantMessage = parseAssistantMessage(candidate);
1415
+ return {
1416
+ messages: [assistantMessage],
1417
+ text: assistantMessage.parts
1418
+ .filter((part) => part.type === "text")
1419
+ .map((part) => part.text)
1420
+ .join(""),
1421
+ finishReason: normalizeFinishReason(candidate?.finishReason),
1422
+ providerFinishReason: candidate?.finishReason,
1423
+ rawResponse: json
1424
+ };
1425
+ }
1426
+ finally {
1427
+ cleanup();
1428
+ }
1429
+ }
1430
+ async stream(input) {
1431
+ const { signal, cleanup } = withTimeoutSignal(input);
1432
+ const response = await withRetry(() => this.fetcher(this.url("streamGenerateContent?alt=sse"), {
1433
+ method: "POST",
1434
+ headers: { "content-type": "application/json" },
1435
+ signal,
1436
+ body: JSON.stringify({
1437
+ contents: mapMessages(input.messages),
1438
+ systemInstruction: systemInstruction(input.messages),
1439
+ tools: mapTools(input.tools),
1440
+ ...input.providerOptions,
1441
+ toolConfig: mapToolConfig(input.toolChoice, input.tools),
1442
+ generationConfig: generationConfig(this.modelId, input)
1443
+ })
1444
+ }), input);
1445
+ return (async function* () {
1446
+ try {
1447
+ for await (const event of streamSSE(response)) {
1448
+ const json = JSON.parse(event.data);
1449
+ const candidate = json.candidates?.[0];
1450
+ const parts = candidate?.content?.parts ?? [];
1451
+ for (const [index, part] of parts.entries()) {
1452
+ if (part.text) {
1453
+ yield { type: "text-delta", textDelta: part.text };
1454
+ }
1455
+ if (part.functionCall) {
1456
+ yield {
1457
+ type: "tool-call",
1458
+ toolCall: {
1459
+ id: part.functionCall.id ?? `${part.functionCall.name}-${index}`,
1460
+ name: part.functionCall.name,
1461
+ input: part.functionCall.args ?? {}
1462
+ }
1463
+ };
1464
+ }
1465
+ }
1466
+ if (candidate?.finishReason) {
1467
+ yield {
1468
+ type: "finish",
1469
+ finishReason: normalizeFinishReason(candidate.finishReason),
1470
+ providerFinishReason: candidate.finishReason
1471
+ };
1472
+ }
1473
+ }
1474
+ }
1475
+ finally {
1476
+ cleanup();
1477
+ }
1478
+ })();
1479
+ }
1480
+ }
1481
+ class GeminiEmbeddingModel {
1482
+ modelId;
1483
+ apiKey;
1484
+ baseURL;
1485
+ fetcher;
1486
+ provider = "gemini";
1487
+ capabilities = capabilities;
1488
+ constructor(modelId, apiKey, baseURL, fetcher) {
1489
+ this.modelId = modelId;
1490
+ this.apiKey = apiKey;
1491
+ this.baseURL = baseURL;
1492
+ this.fetcher = fetcher;
1493
+ }
1494
+ async embed(input) {
1495
+ const { signal, cleanup } = withTimeoutSignal(input);
1496
+ try {
1497
+ const embeddings = await Promise.all(input.values.map(async (value) => {
1498
+ const response = await withRetry(() => this.fetcher(`${this.baseURL}/models/${this.modelId}:embedContent?key=${this.apiKey}`, {
1499
+ method: "POST",
1500
+ headers: { "content-type": "application/json" },
1501
+ signal,
1502
+ body: JSON.stringify({
1503
+ content: { parts: [{ text: value }] }
1504
+ })
1505
+ }), input);
1506
+ const json = await parseJson(response);
1507
+ return json.embedding.values;
1508
+ }));
1509
+ return {
1510
+ embeddings
1511
+ };
1512
+ }
1513
+ finally {
1514
+ cleanup();
1515
+ }
1516
+ }
1517
+ }
1518
+ class GeminiTranscriptionModel {
1519
+ modelId;
1520
+ apiKey;
1521
+ baseURL;
1522
+ fetcher;
1523
+ provider = "gemini";
1524
+ capabilities = transcriptionCapabilities;
1525
+ constructor(modelId, apiKey, baseURL, fetcher) {
1526
+ this.modelId = modelId;
1527
+ this.apiKey = apiKey;
1528
+ this.baseURL = baseURL;
1529
+ this.fetcher = fetcher;
1530
+ }
1531
+ async transcribe(input) {
1532
+ const { signal, cleanup } = withTimeoutSignal(input);
1533
+ try {
1534
+ const response = await withRetry(() => this.fetcher(`${this.baseURL}/models/${this.modelId}:generateContent?key=${this.apiKey}`, {
653
1535
  method: "POST",
654
1536
  headers: { "content-type": "application/json" },
655
1537
  signal,
@@ -735,6 +1617,183 @@ class GeminiSpeechModel {
735
1617
  }
736
1618
  }
737
1619
  }
1620
+ class GeminiImageGenerationModel {
1621
+ modelId;
1622
+ apiKey;
1623
+ baseURL;
1624
+ fetcher;
1625
+ provider = "gemini";
1626
+ capabilities = imageGenerationCapabilities;
1627
+ constructor(modelId, apiKey, baseURL, fetcher) {
1628
+ this.modelId = modelId;
1629
+ this.apiKey = apiKey;
1630
+ this.baseURL = baseURL;
1631
+ this.fetcher = fetcher;
1632
+ }
1633
+ async generateImage(input) {
1634
+ const { signal, cleanup } = withTimeoutSignal(input);
1635
+ const { generationConfig, providerOptions } = splitGenerationConfig(input.providerOptions);
1636
+ try {
1637
+ const response = await withRetry(() => this.fetcher(`${this.baseURL}/models/${this.modelId}:generateContent?key=${this.apiKey}`, {
1638
+ method: "POST",
1639
+ headers: { "content-type": "application/json" },
1640
+ signal,
1641
+ body: JSON.stringify({
1642
+ contents: [
1643
+ {
1644
+ role: "user",
1645
+ parts: [
1646
+ { text: input.negativePrompt ? `${input.prompt}\n\nNegative prompt: ${input.negativePrompt}` : input.prompt },
1647
+ ...(input.images ?? []).map(mediaInputToPart)
1648
+ ]
1649
+ }
1650
+ ],
1651
+ ...providerOptions,
1652
+ generationConfig: {
1653
+ responseModalities: ["TEXT", "IMAGE"],
1654
+ ...(input.count ? { candidateCount: input.count } : {}),
1655
+ ...(input.outputMimeType ? { responseMimeType: input.outputMimeType } : {}),
1656
+ ...(input.aspectRatio || input.size
1657
+ ? {
1658
+ imageConfig: {
1659
+ ...(input.aspectRatio ? { aspectRatio: input.aspectRatio } : {}),
1660
+ ...(input.size ? { imageSize: input.size } : {})
1661
+ }
1662
+ }
1663
+ : {}),
1664
+ ...generationConfig
1665
+ }
1666
+ })
1667
+ }), input);
1668
+ const json = await parseJson(response);
1669
+ const { media, text } = collectInlineMedia(json, input.outputMimeType ?? "image/png");
1670
+ return {
1671
+ images: media,
1672
+ text,
1673
+ rawResponse: json
1674
+ };
1675
+ }
1676
+ finally {
1677
+ cleanup();
1678
+ }
1679
+ }
1680
+ }
1681
+ class GeminiMusicGenerationModel {
1682
+ modelId;
1683
+ apiKey;
1684
+ baseURL;
1685
+ fetcher;
1686
+ provider = "gemini";
1687
+ capabilities = musicGenerationCapabilities;
1688
+ constructor(modelId, apiKey, baseURL, fetcher) {
1689
+ this.modelId = modelId;
1690
+ this.apiKey = apiKey;
1691
+ this.baseURL = baseURL;
1692
+ this.fetcher = fetcher;
1693
+ }
1694
+ async generateMusic(input) {
1695
+ const { signal, cleanup } = withTimeoutSignal(input);
1696
+ const { generationConfig, providerOptions } = splitGenerationConfig(input.providerOptions);
1697
+ try {
1698
+ const response = await withRetry(() => this.fetcher(`${this.baseURL}/models/${this.modelId}:generateContent?key=${this.apiKey}`, {
1699
+ method: "POST",
1700
+ headers: { "content-type": "application/json" },
1701
+ signal,
1702
+ body: JSON.stringify({
1703
+ contents: [
1704
+ {
1705
+ role: "user",
1706
+ parts: [
1707
+ { text: input.negativePrompt ? `${input.prompt}\n\nNegative prompt: ${input.negativePrompt}` : input.prompt },
1708
+ ...(input.images ?? []).map(mediaInputToPart)
1709
+ ]
1710
+ }
1711
+ ],
1712
+ ...providerOptions,
1713
+ generationConfig: {
1714
+ responseModalities: ["AUDIO", "TEXT"],
1715
+ ...(input.outputMimeType ? { responseMimeType: input.outputMimeType } : {}),
1716
+ ...generationConfig
1717
+ }
1718
+ })
1719
+ }), input);
1720
+ const json = await parseJson(response);
1721
+ const { media, text } = collectInlineMedia(json, input.outputMimeType ?? "audio/mpeg");
1722
+ return {
1723
+ audio: media,
1724
+ text,
1725
+ rawResponse: json
1726
+ };
1727
+ }
1728
+ finally {
1729
+ cleanup();
1730
+ }
1731
+ }
1732
+ }
1733
+ class GeminiVideoGenerationModel {
1734
+ modelId;
1735
+ apiKey;
1736
+ baseURL;
1737
+ fetcher;
1738
+ provider = "gemini";
1739
+ capabilities = videoGenerationCapabilities;
1740
+ constructor(modelId, apiKey, baseURL, fetcher) {
1741
+ this.modelId = modelId;
1742
+ this.apiKey = apiKey;
1743
+ this.baseURL = baseURL;
1744
+ this.fetcher = fetcher;
1745
+ }
1746
+ async generateVideo(input) {
1747
+ const timeoutMs = input.timeoutMs ?? 600_000;
1748
+ const { signal, cleanup } = withTimeoutSignal({ ...input, timeoutMs });
1749
+ const startedAt = Date.now();
1750
+ try {
1751
+ const response = await withRetry(() => this.fetcher(`${this.baseURL}/models/${this.modelId}:predictLongRunning?key=${this.apiKey}`, {
1752
+ method: "POST",
1753
+ headers: { "content-type": "application/json" },
1754
+ signal,
1755
+ body: JSON.stringify({
1756
+ instances: [
1757
+ {
1758
+ prompt: input.prompt,
1759
+ ...(input.image ? { image: mediaInputToVeoImage(input.image) } : {})
1760
+ }
1761
+ ],
1762
+ parameters: {
1763
+ ...(input.count ? { sampleCount: input.count } : {}),
1764
+ ...(input.aspectRatio ? { aspectRatio: input.aspectRatio } : {}),
1765
+ ...(input.negativePrompt ? { negativePrompt: input.negativePrompt } : {}),
1766
+ ...(input.durationSeconds ? { durationSeconds: input.durationSeconds } : {}),
1767
+ ...(input.outputStorageUri ? { storageUri: input.outputStorageUri } : {}),
1768
+ ...input.providerOptions
1769
+ }
1770
+ })
1771
+ }), input);
1772
+ let operation = await parseJson(response);
1773
+ const operationName = operation.name;
1774
+ const pollIntervalMs = input.pollIntervalMs ?? 10_000;
1775
+ while (!operation.done) {
1776
+ if (Date.now() - startedAt >= timeoutMs) {
1777
+ throw new Error(`Gemini video generation timed out after ${timeoutMs}ms.`);
1778
+ }
1779
+ await sleep(pollIntervalMs, signal);
1780
+ const pollResponse = await withRetry(() => this.fetcher(`${this.baseURL}/${operationName}?key=${this.apiKey}`, {
1781
+ method: "GET",
1782
+ signal
1783
+ }), input);
1784
+ operation = await parseJson(pollResponse);
1785
+ }
1786
+ return {
1787
+ videos: collectVideos(operation),
1788
+ operationName,
1789
+ rawResponse: operation
1790
+ };
1791
+ }
1792
+ finally {
1793
+ cleanup();
1794
+ }
1795
+ }
1796
+ }
738
1797
  class GeminiGroundedLanguageModel {
739
1798
  modelId;
740
1799
  apiKey;
@@ -827,6 +1886,16 @@ class GeminiRealtimeModel {
827
1886
  }
828
1887
  }
829
1888
  ],
1889
+ buildMediaPayloads: (frame) => [
1890
+ {
1891
+ realtimeInput: {
1892
+ media: {
1893
+ mimeType: frame.mediaType,
1894
+ data: encodeMediaFrame(frame)
1895
+ }
1896
+ }
1897
+ }
1898
+ ],
830
1899
  buildTextPayloads: (text) => [
831
1900
  {
832
1901
  clientContent: {
@@ -914,8 +1983,17 @@ export const createGemini = (options = {}) => {
914
1983
  embeddingModel: (modelId) => new GeminiEmbeddingModel(modelId, apiKey, baseURL, fetcher),
915
1984
  transcriptionModel: (modelId) => new GeminiTranscriptionModel(modelId, apiKey, baseURL, fetcher),
916
1985
  speechModel: (modelId) => new GeminiSpeechModel(modelId, apiKey, baseURL, fetcher),
1986
+ imageGenerationModel: (modelId) => new GeminiImageGenerationModel(modelId, apiKey, baseURL, fetcher),
1987
+ videoGenerationModel: (modelId) => new GeminiVideoGenerationModel(modelId, apiKey, baseURL, fetcher),
1988
+ musicGenerationModel: (modelId) => new GeminiMusicGenerationModel(modelId, apiKey, baseURL, fetcher),
917
1989
  realtimeModel: (modelId) => new GeminiRealtimeModel(modelId, apiKey, baseURL, fetcher, options.realtimeConnectionFactory, options.realtimeURL, options.browserTokenURL),
918
1990
  groundedLanguageModel: (modelId) => new GeminiGroundedLanguageModel(modelId, apiKey, baseURL, fetcher),
1991
+ files: new GeminiFilesClient(apiKey, baseURL, fetcher),
1992
+ fileSearchStores: new GeminiFileSearchStoresClient(apiKey, baseURL, fetcher),
1993
+ caches: new GeminiContextCachesClient(apiKey, baseURL, fetcher),
1994
+ batches: new GeminiBatchesClient(apiKey, baseURL, fetcher),
1995
+ interactions: new GeminiInteractionsClient(apiKey, baseURL, fetcher),
1996
+ predictionModel: (modelId) => new GeminiPredictionModel(modelId, apiKey, baseURL, fetcher),
919
1997
  rawFetch: fetcher
920
1998
  });
921
1999
  };