@langchain/google-common 0.2.9 → 0.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chat_models.cjs +4 -12
- package/dist/chat_models.d.ts +3 -3
- package/dist/chat_models.js +4 -12
- package/dist/types.cjs +1 -0
- package/dist/types.d.ts +33 -5
- package/dist/types.js +1 -0
- package/dist/utils/gemini.cjs +64 -5
- package/dist/utils/gemini.js +64 -5
- package/dist/utils/zod_to_gemini_parameters.cjs +2 -2
- package/dist/utils/zod_to_gemini_parameters.d.ts +3 -3
- package/dist/utils/zod_to_gemini_parameters.js +3 -3
- package/package.json +5 -6
package/dist/chat_models.cjs
CHANGED
|
@@ -8,6 +8,7 @@ const messages_1 = require("@langchain/core/messages");
|
|
|
8
8
|
const runnables_1 = require("@langchain/core/runnables");
|
|
9
9
|
const openai_tools_1 = require("@langchain/core/output_parsers/openai_tools");
|
|
10
10
|
const stream_1 = require("@langchain/core/utils/stream");
|
|
11
|
+
const types_1 = require("@langchain/core/utils/types");
|
|
11
12
|
const common_js_1 = require("./utils/common.cjs");
|
|
12
13
|
const connection_js_1 = require("./connection.cjs");
|
|
13
14
|
const gemini_js_1 = require("./utils/gemini.cjs");
|
|
@@ -288,7 +289,7 @@ class ChatGoogleBase extends chat_models_1.BaseChatModel {
|
|
|
288
289
|
return this.connection.platform;
|
|
289
290
|
}
|
|
290
291
|
bindTools(tools, kwargs) {
|
|
291
|
-
return this.
|
|
292
|
+
return this.withConfig({ tools: (0, common_js_1.convertToGeminiTools)(tools), ...kwargs });
|
|
292
293
|
}
|
|
293
294
|
// Replace
|
|
294
295
|
_llmType() {
|
|
@@ -380,7 +381,7 @@ class ChatGoogleBase extends chat_models_1.BaseChatModel {
|
|
|
380
381
|
let functionName = name ?? "extract";
|
|
381
382
|
let outputParser;
|
|
382
383
|
let tools;
|
|
383
|
-
if (
|
|
384
|
+
if ((0, types_1.isInteropZodSchema)(schema)) {
|
|
384
385
|
const jsonSchema = (0, zod_to_gemini_parameters_js_1.schemaToGeminiParameters)(schema);
|
|
385
386
|
tools = [
|
|
386
387
|
{
|
|
@@ -424,10 +425,7 @@ class ChatGoogleBase extends chat_models_1.BaseChatModel {
|
|
|
424
425
|
keyName: functionName,
|
|
425
426
|
});
|
|
426
427
|
}
|
|
427
|
-
const llm = this.
|
|
428
|
-
tools,
|
|
429
|
-
tool_choice: functionName,
|
|
430
|
-
});
|
|
428
|
+
const llm = this.bindTools(tools).withConfig({ tool_choice: functionName });
|
|
431
429
|
if (!includeRaw) {
|
|
432
430
|
return llm.pipe(outputParser).withConfig({
|
|
433
431
|
runName: "ChatGoogleStructuredOutput",
|
|
@@ -454,9 +452,3 @@ class ChatGoogleBase extends chat_models_1.BaseChatModel {
|
|
|
454
452
|
}
|
|
455
453
|
}
|
|
456
454
|
exports.ChatGoogleBase = ChatGoogleBase;
|
|
457
|
-
function isZodSchema(
|
|
458
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
459
|
-
input) {
|
|
460
|
-
// Check for a characteristic method of Zod schemas
|
|
461
|
-
return typeof input?.parse === "function";
|
|
462
|
-
}
|
package/dist/chat_models.d.ts
CHANGED
|
@@ -4,9 +4,9 @@ import { BaseChatModel, LangSmithParams, type BaseChatModelParams } from "@langc
|
|
|
4
4
|
import { ChatGenerationChunk, ChatResult } from "@langchain/core/outputs";
|
|
5
5
|
import { AIMessageChunk } from "@langchain/core/messages";
|
|
6
6
|
import { BaseLanguageModelInput, StructuredOutputMethodOptions } from "@langchain/core/language_models/base";
|
|
7
|
-
import type { z } from "zod";
|
|
8
7
|
import { Runnable } from "@langchain/core/runnables";
|
|
9
8
|
import { AsyncCaller } from "@langchain/core/utils/async_caller";
|
|
9
|
+
import { InteropZodType } from "@langchain/core/utils/types";
|
|
10
10
|
import { GoogleAIBaseLLMInput, GoogleAIModelParams, GoogleAISafetySetting, GoogleConnectionParams, GooglePlatformType, GoogleAIBaseLanguageModelCallOptions, GoogleAIAPI, GoogleAIAPIParams, GoogleSearchToolSetting } from "./types.js";
|
|
11
11
|
import { AbstractGoogleLLMConnection } from "./connection.js";
|
|
12
12
|
import { GoogleAbstractedClient } from "./auth.js";
|
|
@@ -74,8 +74,8 @@ export declare abstract class ChatGoogleBase<AuthOptions> extends BaseChatModel<
|
|
|
74
74
|
_streamResponseChunks(_messages: BaseMessage[], options: this["ParsedCallOptions"], runManager?: CallbackManagerForLLMRun): AsyncGenerator<ChatGenerationChunk>;
|
|
75
75
|
/** @ignore */
|
|
76
76
|
_combineLLMOutput(): never[];
|
|
77
|
-
withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema:
|
|
78
|
-
withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema:
|
|
77
|
+
withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema: InteropZodType<RunOutput> | Record<string, any>, config?: StructuredOutputMethodOptions<false>): Runnable<BaseLanguageModelInput, RunOutput>;
|
|
78
|
+
withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema: InteropZodType<RunOutput> | Record<string, any>, config?: StructuredOutputMethodOptions<true>): Runnable<BaseLanguageModelInput, {
|
|
79
79
|
raw: BaseMessage;
|
|
80
80
|
parsed: RunOutput;
|
|
81
81
|
}>;
|
package/dist/chat_models.js
CHANGED
|
@@ -5,6 +5,7 @@ import { AIMessageChunk } from "@langchain/core/messages";
|
|
|
5
5
|
import { RunnablePassthrough, RunnableSequence, } from "@langchain/core/runnables";
|
|
6
6
|
import { JsonOutputKeyToolsParser } from "@langchain/core/output_parsers/openai_tools";
|
|
7
7
|
import { concat } from "@langchain/core/utils/stream";
|
|
8
|
+
import { isInteropZodSchema, } from "@langchain/core/utils/types";
|
|
8
9
|
import { convertToGeminiTools, copyAIModelParams, copyAndValidateModelParamsInto, } from "./utils/common.js";
|
|
9
10
|
import { AbstractGoogleLLMConnection } from "./connection.js";
|
|
10
11
|
import { DefaultGeminiSafetyHandler, getGeminiAPI } from "./utils/gemini.js";
|
|
@@ -284,7 +285,7 @@ export class ChatGoogleBase extends BaseChatModel {
|
|
|
284
285
|
return this.connection.platform;
|
|
285
286
|
}
|
|
286
287
|
bindTools(tools, kwargs) {
|
|
287
|
-
return this.
|
|
288
|
+
return this.withConfig({ tools: convertToGeminiTools(tools), ...kwargs });
|
|
288
289
|
}
|
|
289
290
|
// Replace
|
|
290
291
|
_llmType() {
|
|
@@ -376,7 +377,7 @@ export class ChatGoogleBase extends BaseChatModel {
|
|
|
376
377
|
let functionName = name ?? "extract";
|
|
377
378
|
let outputParser;
|
|
378
379
|
let tools;
|
|
379
|
-
if (
|
|
380
|
+
if (isInteropZodSchema(schema)) {
|
|
380
381
|
const jsonSchema = schemaToGeminiParameters(schema);
|
|
381
382
|
tools = [
|
|
382
383
|
{
|
|
@@ -420,10 +421,7 @@ export class ChatGoogleBase extends BaseChatModel {
|
|
|
420
421
|
keyName: functionName,
|
|
421
422
|
});
|
|
422
423
|
}
|
|
423
|
-
const llm = this.
|
|
424
|
-
tools,
|
|
425
|
-
tool_choice: functionName,
|
|
426
|
-
});
|
|
424
|
+
const llm = this.bindTools(tools).withConfig({ tool_choice: functionName });
|
|
427
425
|
if (!includeRaw) {
|
|
428
426
|
return llm.pipe(outputParser).withConfig({
|
|
429
427
|
runName: "ChatGoogleStructuredOutput",
|
|
@@ -449,9 +447,3 @@ export class ChatGoogleBase extends BaseChatModel {
|
|
|
449
447
|
});
|
|
450
448
|
}
|
|
451
449
|
}
|
|
452
|
-
function isZodSchema(
|
|
453
|
-
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
454
|
-
input) {
|
|
455
|
-
// Check for a characteristic method of Zod schemas
|
|
456
|
-
return typeof input?.parse === "function";
|
|
457
|
-
}
|
package/dist/types.cjs
CHANGED
package/dist/types.d.ts
CHANGED
|
@@ -266,28 +266,39 @@ export interface GoogleResponse {
|
|
|
266
266
|
export interface GoogleRawResponse extends GoogleResponse {
|
|
267
267
|
data: Blob;
|
|
268
268
|
}
|
|
269
|
-
export interface
|
|
269
|
+
export interface GeminiPartBase {
|
|
270
|
+
thought?: boolean;
|
|
271
|
+
}
|
|
272
|
+
export interface GeminiVideoMetadata {
|
|
273
|
+
fps?: number;
|
|
274
|
+
startOffset?: string;
|
|
275
|
+
endOffset?: string;
|
|
276
|
+
}
|
|
277
|
+
export interface GeminiPartBaseFile extends GeminiPartBase {
|
|
278
|
+
videoMetadata?: GeminiVideoMetadata;
|
|
279
|
+
}
|
|
280
|
+
export interface GeminiPartText extends GeminiPartBase {
|
|
270
281
|
text: string;
|
|
271
282
|
}
|
|
272
|
-
export interface GeminiPartInlineData {
|
|
283
|
+
export interface GeminiPartInlineData extends GeminiPartBaseFile {
|
|
273
284
|
inlineData: {
|
|
274
285
|
mimeType: string;
|
|
275
286
|
data: string;
|
|
276
287
|
};
|
|
277
288
|
}
|
|
278
|
-
export interface GeminiPartFileData {
|
|
289
|
+
export interface GeminiPartFileData extends GeminiPartBaseFile {
|
|
279
290
|
fileData: {
|
|
280
291
|
mimeType: string;
|
|
281
292
|
fileUri: string;
|
|
282
293
|
};
|
|
283
294
|
}
|
|
284
|
-
export interface GeminiPartFunctionCall {
|
|
295
|
+
export interface GeminiPartFunctionCall extends GeminiPartBase {
|
|
285
296
|
functionCall: {
|
|
286
297
|
name: string;
|
|
287
298
|
args?: object;
|
|
288
299
|
};
|
|
289
300
|
}
|
|
290
|
-
export interface GeminiPartFunctionResponse {
|
|
301
|
+
export interface GeminiPartFunctionResponse extends GeminiPartBase {
|
|
291
302
|
functionResponse: {
|
|
292
303
|
name: string;
|
|
293
304
|
response: object;
|
|
@@ -356,6 +367,18 @@ export interface GeminiSegment {
|
|
|
356
367
|
export interface GeminiRetrievalMetadata {
|
|
357
368
|
googleSearchDynamicRetrievalScore: number;
|
|
358
369
|
}
|
|
370
|
+
export type GeminiUrlRetrievalStatus = "URL_RETRIEVAL_STATUS_SUCCESS" | "URL_RETRIEVAL_STATUS_ERROR";
|
|
371
|
+
export interface GeminiUrlRetrievalContext {
|
|
372
|
+
retrievedUrl: string;
|
|
373
|
+
urlRetrievalStatus: GeminiUrlRetrievalStatus;
|
|
374
|
+
}
|
|
375
|
+
export interface GeminiUrlRetrievalMetadata {
|
|
376
|
+
urlRetrievalContexts: GeminiUrlRetrievalContext[];
|
|
377
|
+
}
|
|
378
|
+
export type GeminiUrlMetadata = GeminiUrlRetrievalContext;
|
|
379
|
+
export interface GeminiUrlContextMetadata {
|
|
380
|
+
urlMetadata: GeminiUrlMetadata[];
|
|
381
|
+
}
|
|
359
382
|
export interface GeminiLogprobsResult {
|
|
360
383
|
topCandidates: GeminiLogprobsTopCandidate[];
|
|
361
384
|
chosenCandidates: GeminiLogprobsResultCandidate[];
|
|
@@ -377,6 +400,7 @@ export interface GeminiTool {
|
|
|
377
400
|
functionDeclarations?: GeminiFunctionDeclaration[];
|
|
378
401
|
googleSearchRetrieval?: GoogleSearchRetrieval;
|
|
379
402
|
googleSearch?: GoogleSearch;
|
|
403
|
+
urlContext?: UrlContext;
|
|
380
404
|
retrieval?: VertexAIRetrieval;
|
|
381
405
|
}
|
|
382
406
|
export type GoogleSearchToolSetting = boolean | "googleSearchRetrieval" | "googleSearch" | string;
|
|
@@ -390,6 +414,8 @@ export interface GoogleSearchRetrieval {
|
|
|
390
414
|
}
|
|
391
415
|
export interface GoogleSearch {
|
|
392
416
|
}
|
|
417
|
+
export interface UrlContext {
|
|
418
|
+
}
|
|
393
419
|
export interface VertexAIRetrieval {
|
|
394
420
|
vertexAiSearch: {
|
|
395
421
|
datastore: string;
|
|
@@ -453,6 +479,8 @@ export interface GeminiResponseCandidate {
|
|
|
453
479
|
safetyRatings: GeminiSafetyRating[];
|
|
454
480
|
citationMetadata?: GeminiCitationMetadata;
|
|
455
481
|
groundingMetadata?: GeminiGroundingMetadata;
|
|
482
|
+
urlRetrievalMetadata?: GeminiUrlRetrievalMetadata;
|
|
483
|
+
urlContextMetadata?: GeminiUrlContextMetadata;
|
|
456
484
|
avgLogprobs?: number;
|
|
457
485
|
logprobsResult: GeminiLogprobsResult;
|
|
458
486
|
finishMessage?: string;
|
package/dist/types.js
CHANGED
package/dist/utils/gemini.cjs
CHANGED
|
@@ -136,7 +136,7 @@ function getGeminiAPI(config) {
|
|
|
136
136
|
return null;
|
|
137
137
|
}
|
|
138
138
|
}
|
|
139
|
-
function
|
|
139
|
+
function messageContentImageUrlData(content) {
|
|
140
140
|
const url = typeof content.image_url === "string"
|
|
141
141
|
? content.image_url
|
|
142
142
|
: content.image_url.url;
|
|
@@ -159,6 +159,11 @@ function getGeminiAPI(config) {
|
|
|
159
159
|
};
|
|
160
160
|
}
|
|
161
161
|
}
|
|
162
|
+
function messageContentImageUrl(content) {
|
|
163
|
+
const ret = messageContentImageUrlData(content);
|
|
164
|
+
supplementVideoMetadata(content, ret);
|
|
165
|
+
return ret;
|
|
166
|
+
}
|
|
162
167
|
async function blobToFileData(blob) {
|
|
163
168
|
return {
|
|
164
169
|
fileData: {
|
|
@@ -170,7 +175,7 @@ function getGeminiAPI(config) {
|
|
|
170
175
|
async function fileUriContentToBlob(uri) {
|
|
171
176
|
return config?.mediaManager?.getMediaBlob(uri);
|
|
172
177
|
}
|
|
173
|
-
async function
|
|
178
|
+
async function messageContentMediaData(
|
|
174
179
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
175
180
|
content) {
|
|
176
181
|
if ("mimeType" in content && "data" in content) {
|
|
@@ -198,6 +203,34 @@ function getGeminiAPI(config) {
|
|
|
198
203
|
}
|
|
199
204
|
throw new Error(`Invalid media content: ${JSON.stringify(content, null, 1)}`);
|
|
200
205
|
}
|
|
206
|
+
function supplementVideoMetadata(
|
|
207
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
208
|
+
content, ret) {
|
|
209
|
+
// Add videoMetadata if defined
|
|
210
|
+
if ("videoMetadata" in content && typeof ret === "object") {
|
|
211
|
+
// eslint-disable-next-line no-param-reassign
|
|
212
|
+
ret.videoMetadata = content.videoMetadata;
|
|
213
|
+
}
|
|
214
|
+
return ret;
|
|
215
|
+
}
|
|
216
|
+
async function messageContentMedia(
|
|
217
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
218
|
+
content) {
|
|
219
|
+
const ret = await messageContentMediaData(content);
|
|
220
|
+
supplementVideoMetadata(content, ret);
|
|
221
|
+
return ret;
|
|
222
|
+
}
|
|
223
|
+
function messageContentReasoning(content) {
|
|
224
|
+
if (content?.reasoning && content?.reasoning.length > 0) {
|
|
225
|
+
return {
|
|
226
|
+
text: content.reasoning,
|
|
227
|
+
thought: true,
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
else {
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
201
234
|
const standardContentBlockConverter = {
|
|
202
235
|
providerName: "Google Gemini",
|
|
203
236
|
fromStandardTextBlock(block) {
|
|
@@ -316,8 +349,10 @@ function getGeminiAPI(config) {
|
|
|
316
349
|
break;
|
|
317
350
|
case "media":
|
|
318
351
|
return await messageContentMedia(content);
|
|
352
|
+
case "reasoning":
|
|
353
|
+
return messageContentReasoning(content);
|
|
319
354
|
default:
|
|
320
|
-
throw new Error(`Unsupported type "${content.type}" received while converting message to message parts: ${content}`);
|
|
355
|
+
throw new Error(`Unsupported type "${content.type}" received while converting message to message parts: ${JSON.stringify(content)}`);
|
|
321
356
|
}
|
|
322
357
|
throw new Error(`Cannot coerce "${content.type}" message part into a string.`);
|
|
323
358
|
}
|
|
@@ -471,6 +506,12 @@ function getGeminiAPI(config) {
|
|
|
471
506
|
return [];
|
|
472
507
|
}
|
|
473
508
|
}
|
|
509
|
+
function thoughtPartToMessageContent(part) {
|
|
510
|
+
return {
|
|
511
|
+
type: "reasoning",
|
|
512
|
+
reasoning: part.text,
|
|
513
|
+
};
|
|
514
|
+
}
|
|
474
515
|
function textPartToMessageContent(part) {
|
|
475
516
|
return {
|
|
476
517
|
type: "text",
|
|
@@ -495,6 +536,9 @@ function getGeminiAPI(config) {
|
|
|
495
536
|
if (part === undefined || part === null) {
|
|
496
537
|
return null;
|
|
497
538
|
}
|
|
539
|
+
else if (part.thought) {
|
|
540
|
+
return thoughtPartToMessageContent(part);
|
|
541
|
+
}
|
|
498
542
|
else if ("text" in part) {
|
|
499
543
|
return textPartToMessageContent(part);
|
|
500
544
|
}
|
|
@@ -639,6 +683,19 @@ function getGeminiAPI(config) {
|
|
|
639
683
|
content,
|
|
640
684
|
};
|
|
641
685
|
}
|
|
686
|
+
function candidateToUrlContextMetadata(candidate) {
|
|
687
|
+
const retrieval = candidate?.urlRetrievalMetadata?.urlRetrievalContexts ?? [];
|
|
688
|
+
const context = candidate?.urlContextMetadata?.urlMetadata ?? [];
|
|
689
|
+
const all = [...retrieval, ...context];
|
|
690
|
+
if (all.length === 0) {
|
|
691
|
+
return undefined;
|
|
692
|
+
}
|
|
693
|
+
else {
|
|
694
|
+
return {
|
|
695
|
+
urlMetadata: all,
|
|
696
|
+
};
|
|
697
|
+
}
|
|
698
|
+
}
|
|
642
699
|
function addModalityCounts(modalityTokenCounts, details) {
|
|
643
700
|
modalityTokenCounts?.forEach((modalityTokenCount) => {
|
|
644
701
|
const { modality, tokenCount } = modalityTokenCount;
|
|
@@ -704,6 +761,7 @@ function getGeminiAPI(config) {
|
|
|
704
761
|
grounding_metadata: data.candidates[0]?.groundingMetadata,
|
|
705
762
|
finish_reason,
|
|
706
763
|
finish_message: data.candidates[0]?.finishMessage,
|
|
764
|
+
url_context_metadata: candidateToUrlContextMetadata(data.candidates[0]),
|
|
707
765
|
avgLogprobs: data.candidates[0]?.avgLogprobs,
|
|
708
766
|
logprobs: candidateToLogprobs(data.candidates[0]),
|
|
709
767
|
};
|
|
@@ -831,6 +889,7 @@ function getGeminiAPI(config) {
|
|
|
831
889
|
if (typeof item.message.content === "string") {
|
|
832
890
|
// If this is a string, turn it into a text type
|
|
833
891
|
ret.push({
|
|
892
|
+
type: "text",
|
|
834
893
|
text: item.message.content,
|
|
835
894
|
});
|
|
836
895
|
}
|
|
@@ -1094,10 +1153,10 @@ function getGeminiAPI(config) {
|
|
|
1094
1153
|
// Add thinking configuration if explicitly set
|
|
1095
1154
|
// Note that you cannot have thinkingBudget set to 0 and includeThoughts true
|
|
1096
1155
|
if (typeof parameters.maxReasoningTokens !== "undefined") {
|
|
1156
|
+
const includeThoughts = parameters.maxReasoningTokens > 0;
|
|
1097
1157
|
ret.thinkingConfig = {
|
|
1098
1158
|
thinkingBudget: parameters.maxReasoningTokens,
|
|
1099
|
-
|
|
1100
|
-
includeThoughts: false,
|
|
1159
|
+
includeThoughts,
|
|
1101
1160
|
};
|
|
1102
1161
|
}
|
|
1103
1162
|
// Remove any undefined properties, so we don't send them
|
package/dist/utils/gemini.js
CHANGED
|
@@ -131,7 +131,7 @@ export function getGeminiAPI(config) {
|
|
|
131
131
|
return null;
|
|
132
132
|
}
|
|
133
133
|
}
|
|
134
|
-
function
|
|
134
|
+
function messageContentImageUrlData(content) {
|
|
135
135
|
const url = typeof content.image_url === "string"
|
|
136
136
|
? content.image_url
|
|
137
137
|
: content.image_url.url;
|
|
@@ -154,6 +154,11 @@ export function getGeminiAPI(config) {
|
|
|
154
154
|
};
|
|
155
155
|
}
|
|
156
156
|
}
|
|
157
|
+
function messageContentImageUrl(content) {
|
|
158
|
+
const ret = messageContentImageUrlData(content);
|
|
159
|
+
supplementVideoMetadata(content, ret);
|
|
160
|
+
return ret;
|
|
161
|
+
}
|
|
157
162
|
async function blobToFileData(blob) {
|
|
158
163
|
return {
|
|
159
164
|
fileData: {
|
|
@@ -165,7 +170,7 @@ export function getGeminiAPI(config) {
|
|
|
165
170
|
async function fileUriContentToBlob(uri) {
|
|
166
171
|
return config?.mediaManager?.getMediaBlob(uri);
|
|
167
172
|
}
|
|
168
|
-
async function
|
|
173
|
+
async function messageContentMediaData(
|
|
169
174
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
170
175
|
content) {
|
|
171
176
|
if ("mimeType" in content && "data" in content) {
|
|
@@ -193,6 +198,34 @@ export function getGeminiAPI(config) {
|
|
|
193
198
|
}
|
|
194
199
|
throw new Error(`Invalid media content: ${JSON.stringify(content, null, 1)}`);
|
|
195
200
|
}
|
|
201
|
+
function supplementVideoMetadata(
|
|
202
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
203
|
+
content, ret) {
|
|
204
|
+
// Add videoMetadata if defined
|
|
205
|
+
if ("videoMetadata" in content && typeof ret === "object") {
|
|
206
|
+
// eslint-disable-next-line no-param-reassign
|
|
207
|
+
ret.videoMetadata = content.videoMetadata;
|
|
208
|
+
}
|
|
209
|
+
return ret;
|
|
210
|
+
}
|
|
211
|
+
async function messageContentMedia(
|
|
212
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
213
|
+
content) {
|
|
214
|
+
const ret = await messageContentMediaData(content);
|
|
215
|
+
supplementVideoMetadata(content, ret);
|
|
216
|
+
return ret;
|
|
217
|
+
}
|
|
218
|
+
function messageContentReasoning(content) {
|
|
219
|
+
if (content?.reasoning && content?.reasoning.length > 0) {
|
|
220
|
+
return {
|
|
221
|
+
text: content.reasoning,
|
|
222
|
+
thought: true,
|
|
223
|
+
};
|
|
224
|
+
}
|
|
225
|
+
else {
|
|
226
|
+
return null;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
196
229
|
const standardContentBlockConverter = {
|
|
197
230
|
providerName: "Google Gemini",
|
|
198
231
|
fromStandardTextBlock(block) {
|
|
@@ -311,8 +344,10 @@ export function getGeminiAPI(config) {
|
|
|
311
344
|
break;
|
|
312
345
|
case "media":
|
|
313
346
|
return await messageContentMedia(content);
|
|
347
|
+
case "reasoning":
|
|
348
|
+
return messageContentReasoning(content);
|
|
314
349
|
default:
|
|
315
|
-
throw new Error(`Unsupported type "${content.type}" received while converting message to message parts: ${content}`);
|
|
350
|
+
throw new Error(`Unsupported type "${content.type}" received while converting message to message parts: ${JSON.stringify(content)}`);
|
|
316
351
|
}
|
|
317
352
|
throw new Error(`Cannot coerce "${content.type}" message part into a string.`);
|
|
318
353
|
}
|
|
@@ -466,6 +501,12 @@ export function getGeminiAPI(config) {
|
|
|
466
501
|
return [];
|
|
467
502
|
}
|
|
468
503
|
}
|
|
504
|
+
function thoughtPartToMessageContent(part) {
|
|
505
|
+
return {
|
|
506
|
+
type: "reasoning",
|
|
507
|
+
reasoning: part.text,
|
|
508
|
+
};
|
|
509
|
+
}
|
|
469
510
|
function textPartToMessageContent(part) {
|
|
470
511
|
return {
|
|
471
512
|
type: "text",
|
|
@@ -490,6 +531,9 @@ export function getGeminiAPI(config) {
|
|
|
490
531
|
if (part === undefined || part === null) {
|
|
491
532
|
return null;
|
|
492
533
|
}
|
|
534
|
+
else if (part.thought) {
|
|
535
|
+
return thoughtPartToMessageContent(part);
|
|
536
|
+
}
|
|
493
537
|
else if ("text" in part) {
|
|
494
538
|
return textPartToMessageContent(part);
|
|
495
539
|
}
|
|
@@ -634,6 +678,19 @@ export function getGeminiAPI(config) {
|
|
|
634
678
|
content,
|
|
635
679
|
};
|
|
636
680
|
}
|
|
681
|
+
function candidateToUrlContextMetadata(candidate) {
|
|
682
|
+
const retrieval = candidate?.urlRetrievalMetadata?.urlRetrievalContexts ?? [];
|
|
683
|
+
const context = candidate?.urlContextMetadata?.urlMetadata ?? [];
|
|
684
|
+
const all = [...retrieval, ...context];
|
|
685
|
+
if (all.length === 0) {
|
|
686
|
+
return undefined;
|
|
687
|
+
}
|
|
688
|
+
else {
|
|
689
|
+
return {
|
|
690
|
+
urlMetadata: all,
|
|
691
|
+
};
|
|
692
|
+
}
|
|
693
|
+
}
|
|
637
694
|
function addModalityCounts(modalityTokenCounts, details) {
|
|
638
695
|
modalityTokenCounts?.forEach((modalityTokenCount) => {
|
|
639
696
|
const { modality, tokenCount } = modalityTokenCount;
|
|
@@ -699,6 +756,7 @@ export function getGeminiAPI(config) {
|
|
|
699
756
|
grounding_metadata: data.candidates[0]?.groundingMetadata,
|
|
700
757
|
finish_reason,
|
|
701
758
|
finish_message: data.candidates[0]?.finishMessage,
|
|
759
|
+
url_context_metadata: candidateToUrlContextMetadata(data.candidates[0]),
|
|
702
760
|
avgLogprobs: data.candidates[0]?.avgLogprobs,
|
|
703
761
|
logprobs: candidateToLogprobs(data.candidates[0]),
|
|
704
762
|
};
|
|
@@ -826,6 +884,7 @@ export function getGeminiAPI(config) {
|
|
|
826
884
|
if (typeof item.message.content === "string") {
|
|
827
885
|
// If this is a string, turn it into a text type
|
|
828
886
|
ret.push({
|
|
887
|
+
type: "text",
|
|
829
888
|
text: item.message.content,
|
|
830
889
|
});
|
|
831
890
|
}
|
|
@@ -1089,10 +1148,10 @@ export function getGeminiAPI(config) {
|
|
|
1089
1148
|
// Add thinking configuration if explicitly set
|
|
1090
1149
|
// Note that you cannot have thinkingBudget set to 0 and includeThoughts true
|
|
1091
1150
|
if (typeof parameters.maxReasoningTokens !== "undefined") {
|
|
1151
|
+
const includeThoughts = parameters.maxReasoningTokens > 0;
|
|
1092
1152
|
ret.thinkingConfig = {
|
|
1093
1153
|
thinkingBudget: parameters.maxReasoningTokens,
|
|
1094
|
-
|
|
1095
|
-
includeThoughts: false,
|
|
1154
|
+
includeThoughts,
|
|
1096
1155
|
};
|
|
1097
1156
|
}
|
|
1098
1157
|
// Remove any undefined properties, so we don't send them
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
4
4
|
exports.jsonSchemaToGeminiParameters = exports.schemaToGeminiParameters = exports.removeAdditionalProperties = void 0;
|
|
5
5
|
const types_1 = require("@langchain/core/utils/types");
|
|
6
|
-
const
|
|
6
|
+
const json_schema_1 = require("@langchain/core/utils/json_schema");
|
|
7
7
|
function removeAdditionalProperties(
|
|
8
8
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
9
9
|
obj) {
|
|
@@ -57,7 +57,7 @@ function schemaToGeminiParameters(schema) {
|
|
|
57
57
|
// attributes, so we need to explicitly remove them.
|
|
58
58
|
// Zod sometimes also makes an array of type (because of .nullish()),
|
|
59
59
|
// which needs cleaning up.
|
|
60
|
-
const jsonSchema = removeAdditionalProperties((0, types_1.
|
|
60
|
+
const jsonSchema = removeAdditionalProperties((0, types_1.isInteropZodSchema)(schema) ? (0, json_schema_1.toJsonSchema)(schema) : schema);
|
|
61
61
|
const { $schema, ...rest } = jsonSchema;
|
|
62
62
|
return rest;
|
|
63
63
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { type JsonSchema7Type } from "
|
|
1
|
+
import { InteropZodType } from "@langchain/core/utils/types";
|
|
2
|
+
import { type JsonSchema7Type } from "@langchain/core/utils/json_schema";
|
|
3
3
|
import { GeminiFunctionSchema, GeminiJsonSchema } from "../types.js";
|
|
4
4
|
export declare function removeAdditionalProperties(obj: Record<string, any>): GeminiJsonSchema;
|
|
5
|
-
export declare function schemaToGeminiParameters<RunOutput extends Record<string, any> = Record<string, any>>(schema:
|
|
5
|
+
export declare function schemaToGeminiParameters<RunOutput extends Record<string, any> = Record<string, any>>(schema: InteropZodType<RunOutput> | JsonSchema7Type): GeminiFunctionSchema;
|
|
6
6
|
export declare function jsonSchemaToGeminiParameters(schema: Record<string, any>): GeminiFunctionSchema;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* eslint-disable @typescript-eslint/no-unused-vars */
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
2
|
+
import { isInteropZodSchema, } from "@langchain/core/utils/types";
|
|
3
|
+
import { toJsonSchema, } from "@langchain/core/utils/json_schema";
|
|
4
4
|
export function removeAdditionalProperties(
|
|
5
5
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
6
6
|
obj) {
|
|
@@ -53,7 +53,7 @@ export function schemaToGeminiParameters(schema) {
|
|
|
53
53
|
// attributes, so we need to explicitly remove them.
|
|
54
54
|
// Zod sometimes also makes an array of type (because of .nullish()),
|
|
55
55
|
// which needs cleaning up.
|
|
56
|
-
const jsonSchema = removeAdditionalProperties(
|
|
56
|
+
const jsonSchema = removeAdditionalProperties(isInteropZodSchema(schema) ? toJsonSchema(schema) : schema);
|
|
57
57
|
const { $schema, ...rest } = jsonSchema;
|
|
58
58
|
return rest;
|
|
59
59
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@langchain/google-common",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.11",
|
|
4
4
|
"description": "Core types and classes for Google services.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"engines": {
|
|
@@ -32,15 +32,14 @@
|
|
|
32
32
|
"author": "LangChain",
|
|
33
33
|
"license": "MIT",
|
|
34
34
|
"dependencies": {
|
|
35
|
-
"uuid": "^10.0.0"
|
|
36
|
-
"zod-to-json-schema": "^3.22.4"
|
|
35
|
+
"uuid": "^10.0.0"
|
|
37
36
|
},
|
|
38
37
|
"peerDependencies": {
|
|
39
|
-
"@langchain/core": ">=0.3.
|
|
38
|
+
"@langchain/core": ">=0.3.58 <0.4.0"
|
|
40
39
|
},
|
|
41
40
|
"devDependencies": {
|
|
42
41
|
"@jest/globals": "^29.5.0",
|
|
43
|
-
"@langchain/core": "
|
|
42
|
+
"@langchain/core": "workspace:*",
|
|
44
43
|
"@langchain/scripts": ">=0.1.0 <0.2.0",
|
|
45
44
|
"@swc/core": "^1.3.90",
|
|
46
45
|
"@swc/jest": "^0.2.29",
|
|
@@ -138,4 +137,4 @@
|
|
|
138
137
|
"experimental/utils/media_core.d.ts",
|
|
139
138
|
"experimental/utils/media_core.d.cts"
|
|
140
139
|
]
|
|
141
|
-
}
|
|
140
|
+
}
|