@langchain/google-common 0.2.9 → 0.2.10

package/dist/chat_models.cjs

@@ -288,7 +288,7 @@ class ChatGoogleBase extends chat_models_1.BaseChatModel {
         return this.connection.platform;
     }
     bindTools(tools, kwargs) {
-        return this.bind({ tools: (0, common_js_1.convertToGeminiTools)(tools), ...kwargs });
+        return this.withConfig({ tools: (0, common_js_1.convertToGeminiTools)(tools), ...kwargs });
     }
     // Replace
     _llmType() {
@@ -424,10 +424,7 @@ class ChatGoogleBase extends chat_models_1.BaseChatModel {
                 keyName: functionName,
             });
         }
-        const llm = this.bind({
-            tools,
-            tool_choice: functionName,
-        });
+        const llm = this.bindTools(tools).withConfig({ tool_choice: functionName });
         if (!includeRaw) {
             return llm.pipe(outputParser).withConfig({
                 runName: "ChatGoogleStructuredOutput",

package/dist/chat_models.js

@@ -284,7 +284,7 @@ export class ChatGoogleBase extends BaseChatModel {
         return this.connection.platform;
     }
     bindTools(tools, kwargs) {
-        return this.bind({ tools: convertToGeminiTools(tools), ...kwargs });
+        return this.withConfig({ tools: convertToGeminiTools(tools), ...kwargs });
     }
     // Replace
     _llmType() {
@@ -420,10 +420,7 @@ export class ChatGoogleBase extends BaseChatModel {
                 keyName: functionName,
             });
         }
-        const llm = this.bind({
-            tools,
-            tool_choice: functionName,
-        });
+        const llm = this.bindTools(tools).withConfig({ tool_choice: functionName });
         if (!includeRaw) {
             return llm.pipe(outputParser).withConfig({
                 runName: "ChatGoogleStructuredOutput",

package/dist/types.cjs

@@ -61,5 +61,6 @@ exports.GeminiSearchToolAttributes = [
 exports.GeminiToolAttributes = [
     "functionDeclaration",
     "retrieval",
+    "urlContext",
     ...exports.GeminiSearchToolAttributes,
 ];

package/dist/types.d.ts

@@ -266,28 +266,39 @@ export interface GoogleResponse {
 export interface GoogleRawResponse extends GoogleResponse {
     data: Blob;
 }
-export interface GeminiPartText {
+export interface GeminiPartBase {
+    thought?: boolean;
+}
+export interface GeminiVideoMetadata {
+    fps?: number;
+    startOffset?: string;
+    endOffset?: string;
+}
+export interface GeminiPartBaseFile extends GeminiPartBase {
+    videoMetadata?: GeminiVideoMetadata;
+}
+export interface GeminiPartText extends GeminiPartBase {
     text: string;
 }
-export interface GeminiPartInlineData {
+export interface GeminiPartInlineData extends GeminiPartBaseFile {
     inlineData: {
         mimeType: string;
         data: string;
     };
 }
-export interface GeminiPartFileData {
+export interface GeminiPartFileData extends GeminiPartBaseFile {
     fileData: {
         mimeType: string;
         fileUri: string;
     };
 }
-export interface GeminiPartFunctionCall {
+export interface GeminiPartFunctionCall extends GeminiPartBase {
     functionCall: {
         name: string;
         args?: object;
     };
 }
-export interface GeminiPartFunctionResponse {
+export interface GeminiPartFunctionResponse extends GeminiPartBase {
     functionResponse: {
         name: string;
         response: object;
@@ -356,6 +367,18 @@ export interface GeminiSegment {
 export interface GeminiRetrievalMetadata {
     googleSearchDynamicRetrievalScore: number;
 }
+export type GeminiUrlRetrievalStatus = "URL_RETRIEVAL_STATUS_SUCCESS" | "URL_RETRIEVAL_STATUS_ERROR";
+export interface GeminiUrlRetrievalContext {
+    retrievedUrl: string;
+    urlRetrievalStatus: GeminiUrlRetrievalStatus;
+}
+export interface GeminiUrlRetrievalMetadata {
+    urlRetrievalContexts: GeminiUrlRetrievalContext[];
+}
+export type GeminiUrlMetadata = GeminiUrlRetrievalContext;
+export interface GeminiUrlContextMetadata {
+    urlMetadata: GeminiUrlMetadata[];
+}
 export interface GeminiLogprobsResult {
     topCandidates: GeminiLogprobsTopCandidate[];
     chosenCandidates: GeminiLogprobsResultCandidate[];
@@ -377,6 +400,7 @@ export interface GeminiTool {
     functionDeclarations?: GeminiFunctionDeclaration[];
     googleSearchRetrieval?: GoogleSearchRetrieval;
     googleSearch?: GoogleSearch;
+    urlContext?: UrlContext;
     retrieval?: VertexAIRetrieval;
 }
 export type GoogleSearchToolSetting = boolean | "googleSearchRetrieval" | "googleSearch" | string;
@@ -390,6 +414,8 @@ export interface GoogleSearchRetrieval {
 }
 export interface GoogleSearch {
 }
+export interface UrlContext {
+}
 export interface VertexAIRetrieval {
     vertexAiSearch: {
         datastore: string;
@@ -453,6 +479,8 @@ export interface GeminiResponseCandidate {
     safetyRatings: GeminiSafetyRating[];
     citationMetadata?: GeminiCitationMetadata;
     groundingMetadata?: GeminiGroundingMetadata;
+    urlRetrievalMetadata?: GeminiUrlRetrievalMetadata;
+    urlContextMetadata?: GeminiUrlContextMetadata;
     avgLogprobs?: number;
     logprobsResult: GeminiLogprobsResult;
     finishMessage?: string;

package/dist/types.js

@@ -44,5 +44,6 @@ export const GeminiSearchToolAttributes = [
 export const GeminiToolAttributes = [
     "functionDeclaration",
     "retrieval",
+    "urlContext",
     ...GeminiSearchToolAttributes,
 ];

package/dist/utils/gemini.cjs

@@ -136,7 +136,7 @@ function getGeminiAPI(config) {
             return null;
         }
     }
-    function messageContentImageUrl(content) {
+    function messageContentImageUrlData(content) {
         const url = typeof content.image_url === "string"
             ? content.image_url
             : content.image_url.url;
@@ -159,6 +159,11 @@ function getGeminiAPI(config) {
             };
         }
     }
+    function messageContentImageUrl(content) {
+        const ret = messageContentImageUrlData(content);
+        supplementVideoMetadata(content, ret);
+        return ret;
+    }
     async function blobToFileData(blob) {
         return {
             fileData: {
@@ -170,7 +175,7 @@ function getGeminiAPI(config) {
     async function fileUriContentToBlob(uri) {
         return config?.mediaManager?.getMediaBlob(uri);
     }
-    async function messageContentMedia(
+    async function messageContentMediaData(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     content) {
         if ("mimeType" in content && "data" in content) {
@@ -198,6 +203,34 @@ function getGeminiAPI(config) {
         }
         throw new Error(`Invalid media content: ${JSON.stringify(content, null, 1)}`);
     }
+    function supplementVideoMetadata(
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    content, ret) {
+        // Add videoMetadata if defined
+        if ("videoMetadata" in content && typeof ret === "object") {
+            // eslint-disable-next-line no-param-reassign
+            ret.videoMetadata = content.videoMetadata;
+        }
+        return ret;
+    }
+    async function messageContentMedia(
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    content) {
+        const ret = await messageContentMediaData(content);
+        supplementVideoMetadata(content, ret);
+        return ret;
+    }
+    function messageContentReasoning(content) {
+        if (content?.reasoning && content?.reasoning.length > 0) {
+            return {
+                text: content.reasoning,
+                thought: true,
+            };
+        }
+        else {
+            return null;
+        }
+    }
     const standardContentBlockConverter = {
         providerName: "Google Gemini",
         fromStandardTextBlock(block) {
@@ -316,8 +349,10 @@ function getGeminiAPI(config) {
                 break;
             case "media":
                 return await messageContentMedia(content);
+            case "reasoning":
+                return messageContentReasoning(content);
             default:
-                throw new Error(`Unsupported type "${content.type}" received while converting message to message parts: ${content}`);
+                throw new Error(`Unsupported type "${content.type}" received while converting message to message parts: ${JSON.stringify(content)}`);
         }
         throw new Error(`Cannot coerce "${content.type}" message part into a string.`);
     }
@@ -471,6 +506,12 @@ function getGeminiAPI(config) {
                 return [];
         }
     }
+    function thoughtPartToMessageContent(part) {
+        return {
+            type: "reasoning",
+            reasoning: part.text,
+        };
+    }
     function textPartToMessageContent(part) {
         return {
             type: "text",
@@ -495,6 +536,9 @@ function getGeminiAPI(config) {
             if (part === undefined || part === null) {
                 return null;
             }
+            else if (part.thought) {
+                return thoughtPartToMessageContent(part);
+            }
             else if ("text" in part) {
                 return textPartToMessageContent(part);
             }
@@ -639,6 +683,19 @@ function getGeminiAPI(config) {
             content,
         };
     }
+    function candidateToUrlContextMetadata(candidate) {
+        const retrieval = candidate?.urlRetrievalMetadata?.urlRetrievalContexts ?? [];
+        const context = candidate?.urlContextMetadata?.urlMetadata ?? [];
+        const all = [...retrieval, ...context];
+        if (all.length === 0) {
+            return undefined;
+        }
+        else {
+            return {
+                urlMetadata: all,
+            };
+        }
+    }
     function addModalityCounts(modalityTokenCounts, details) {
         modalityTokenCounts?.forEach((modalityTokenCount) => {
             const { modality, tokenCount } = modalityTokenCount;
@@ -704,6 +761,7 @@ function getGeminiAPI(config) {
             grounding_metadata: data.candidates[0]?.groundingMetadata,
             finish_reason,
             finish_message: data.candidates[0]?.finishMessage,
+            url_context_metadata: candidateToUrlContextMetadata(data.candidates[0]),
             avgLogprobs: data.candidates[0]?.avgLogprobs,
             logprobs: candidateToLogprobs(data.candidates[0]),
         };
@@ -831,6 +889,7 @@ function getGeminiAPI(config) {
                 if (typeof item.message.content === "string") {
                     // If this is a string, turn it into a text type
                     ret.push({
+                        type: "text",
                         text: item.message.content,
                     });
                 }
@@ -1094,10 +1153,10 @@ function getGeminiAPI(config) {
         // Add thinking configuration if explicitly set
         // Note that you cannot have thinkingBudget set to 0 and includeThoughts true
         if (typeof parameters.maxReasoningTokens !== "undefined") {
+            const includeThoughts = parameters.maxReasoningTokens > 0;
             ret.thinkingConfig = {
                 thinkingBudget: parameters.maxReasoningTokens,
-                // TODO: Expose this configuration to the user once google fully supports it
-                includeThoughts: false,
+                includeThoughts,
             };
         }
         // Remove any undefined properties, so we don't send them

package/dist/utils/gemini.js

@@ -131,7 +131,7 @@ export function getGeminiAPI(config) {
             return null;
         }
     }
-    function messageContentImageUrl(content) {
+    function messageContentImageUrlData(content) {
         const url = typeof content.image_url === "string"
             ? content.image_url
             : content.image_url.url;
@@ -154,6 +154,11 @@ export function getGeminiAPI(config) {
             };
         }
     }
+    function messageContentImageUrl(content) {
+        const ret = messageContentImageUrlData(content);
+        supplementVideoMetadata(content, ret);
+        return ret;
+    }
     async function blobToFileData(blob) {
         return {
             fileData: {
@@ -165,7 +170,7 @@ export function getGeminiAPI(config) {
     async function fileUriContentToBlob(uri) {
         return config?.mediaManager?.getMediaBlob(uri);
     }
-    async function messageContentMedia(
+    async function messageContentMediaData(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     content) {
         if ("mimeType" in content && "data" in content) {
@@ -193,6 +198,34 @@ export function getGeminiAPI(config) {
         }
         throw new Error(`Invalid media content: ${JSON.stringify(content, null, 1)}`);
     }
+    function supplementVideoMetadata(
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    content, ret) {
+        // Add videoMetadata if defined
+        if ("videoMetadata" in content && typeof ret === "object") {
+            // eslint-disable-next-line no-param-reassign
+            ret.videoMetadata = content.videoMetadata;
+        }
+        return ret;
+    }
+    async function messageContentMedia(
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    content) {
+        const ret = await messageContentMediaData(content);
+        supplementVideoMetadata(content, ret);
+        return ret;
+    }
+    function messageContentReasoning(content) {
+        if (content?.reasoning && content?.reasoning.length > 0) {
+            return {
+                text: content.reasoning,
+                thought: true,
+            };
+        }
+        else {
+            return null;
+        }
+    }
     const standardContentBlockConverter = {
         providerName: "Google Gemini",
         fromStandardTextBlock(block) {
@@ -311,8 +344,10 @@ export function getGeminiAPI(config) {
                 break;
             case "media":
                 return await messageContentMedia(content);
+            case "reasoning":
+                return messageContentReasoning(content);
             default:
-                throw new Error(`Unsupported type "${content.type}" received while converting message to message parts: ${content}`);
+                throw new Error(`Unsupported type "${content.type}" received while converting message to message parts: ${JSON.stringify(content)}`);
         }
         throw new Error(`Cannot coerce "${content.type}" message part into a string.`);
     }
@@ -466,6 +501,12 @@ export function getGeminiAPI(config) {
                 return [];
         }
     }
+    function thoughtPartToMessageContent(part) {
+        return {
+            type: "reasoning",
+            reasoning: part.text,
+        };
+    }
     function textPartToMessageContent(part) {
         return {
             type: "text",
@@ -490,6 +531,9 @@ export function getGeminiAPI(config) {
             if (part === undefined || part === null) {
                 return null;
             }
+            else if (part.thought) {
+                return thoughtPartToMessageContent(part);
+            }
             else if ("text" in part) {
                 return textPartToMessageContent(part);
             }
@@ -634,6 +678,19 @@ export function getGeminiAPI(config) {
             content,
         };
     }
+    function candidateToUrlContextMetadata(candidate) {
+        const retrieval = candidate?.urlRetrievalMetadata?.urlRetrievalContexts ?? [];
+        const context = candidate?.urlContextMetadata?.urlMetadata ?? [];
+        const all = [...retrieval, ...context];
+        if (all.length === 0) {
+            return undefined;
+        }
+        else {
+            return {
+                urlMetadata: all,
+            };
+        }
+    }
     function addModalityCounts(modalityTokenCounts, details) {
         modalityTokenCounts?.forEach((modalityTokenCount) => {
             const { modality, tokenCount } = modalityTokenCount;
@@ -699,6 +756,7 @@ export function getGeminiAPI(config) {
             grounding_metadata: data.candidates[0]?.groundingMetadata,
             finish_reason,
             finish_message: data.candidates[0]?.finishMessage,
+            url_context_metadata: candidateToUrlContextMetadata(data.candidates[0]),
             avgLogprobs: data.candidates[0]?.avgLogprobs,
             logprobs: candidateToLogprobs(data.candidates[0]),
         };
@@ -826,6 +884,7 @@ export function getGeminiAPI(config) {
                 if (typeof item.message.content === "string") {
                     // If this is a string, turn it into a text type
                     ret.push({
+                        type: "text",
                         text: item.message.content,
                     });
                 }
@@ -1089,10 +1148,10 @@ export function getGeminiAPI(config) {
         // Add thinking configuration if explicitly set
         // Note that you cannot have thinkingBudget set to 0 and includeThoughts true
         if (typeof parameters.maxReasoningTokens !== "undefined") {
+            const includeThoughts = parameters.maxReasoningTokens > 0;
             ret.thinkingConfig = {
                 thinkingBudget: parameters.maxReasoningTokens,
-                // TODO: Expose this configuration to the user once google fully supports it
-                includeThoughts: false,
+                includeThoughts,
             };
         }
         // Remove any undefined properties, so we don't send them

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/google-common",
-  "version": "0.2.9",
+  "version": "0.2.10",
   "description": "Core types and classes for Google services.",
   "type": "module",
   "engines": {
@@ -40,7 +40,7 @@
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
-    "@langchain/core": "0.3.55",
+    "@langchain/core": "0.3.57",
     "@langchain/scripts": ">=0.1.0 <0.2.0",
     "@swc/core": "^1.3.90",
     "@swc/jest": "^0.2.29",