npm - @nocobase/plugin-ai - Versions diffs - 1.9.0-beta.9 → 2.0.0-alpha.10 - Mend

@nocobase/plugin-ai 1.9.0-beta.9 → 2.0.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (2799) hide show

package/dist/node_modules/@langchain/core/dist/language_models/base.js CHANGED Viewed

@@ -185,13 +185,37 @@ export class BaseLanguageModel extends BaseLangChain {
         }
         this.caller = new AsyncCaller(params ?? {});
     }
+    /**
+     * Get the number of tokens in the content.
+     * @param content The content to get the number of tokens for.
+     * @returns The number of tokens in the content.
+     */
     async getNumTokens(content) {
-        // TODO: Figure out correct value.
-        if (typeof content !== "string") {
-            return 0;
+        // Extract text content from MessageContent
+        let textContent;
+        if (typeof content === "string") {
+            textContent = content;
+        }
+        else {
+            /**
+             * Content is an array of MessageContentComplex
+             *
+             * ToDo(@christian-bromann): This is a temporary fix to get the number of tokens for the content.
+             * We need to find a better way to do this.
+             * @see https://github.com/langchain-ai/langchainjs/pull/8341#pullrequestreview-2933713116
+             */
+            textContent = content
+                .map((item) => {
+                if (typeof item === "string")
+                    return item;
+                if (item.type === "text" && "text" in item)
+                    return item.text;
+                return "";
+            })
+                .join("");
         }
         // fallback to approximate calculation if tiktoken is not available
-        let numTokens = Math.ceil(content.length / 4);
+        let numTokens = Math.ceil(textContent.length / 4);
         if (!this._encoding) {
             try {
                 this._encoding = await encodingForModel("modelName" in this
@@ -204,7 +228,7 @@ export class BaseLanguageModel extends BaseLangChain {
         }
         if (this._encoding) {
             try {
-                numTokens = this._encoding.encode(content).length;
+                numTokens = this._encoding.encode(textContent).length;
             }
             catch (error) {
                 console.warn("Failed to calculate number of tokens, falling back to approximate count", error);

package/dist/node_modules/@langchain/core/dist/language_models/chat_models.cjs CHANGED Viewed

@@ -1,7 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SimpleChatModel = exports.BaseChatModel = exports.createChatMessageChunkEncoderStream = void 0;
-const zod_to_json_schema_1 = require("zod-to-json-schema");
+exports.SimpleChatModel = exports.BaseChatModel = void 0;
+exports.createChatMessageChunkEncoderStream = createChatMessageChunkEncoderStream;
 const index_js_1 = require("../messages/index.cjs");
 const outputs_js_1 = require("../outputs.cjs");
 const base_js_1 = require("./base.cjs");
@@ -9,8 +9,9 @@ const manager_js_1 = require("../callbacks/manager.cjs");
 const base_js_2 = require("../runnables/base.cjs");
 const stream_js_1 = require("../utils/stream.cjs");
 const passthrough_js_1 = require("../runnables/passthrough.cjs");
-const is_zod_schema_js_1 = require("../utils/types/is_zod_schema.cjs");
+const zod_js_1 = require("../utils/types/zod.cjs");
 const base_js_3 = require("../callbacks/base.cjs");
+const json_schema_js_1 = require("../utils/json_schema.cjs");
 /**
  * Creates a transform stream for encoding chat message chunks.
  * @deprecated Use {@link BytesOutputParser} instead
@@ -26,7 +27,33 @@ function createChatMessageChunkEncoderStream() {
         },
     });
 }
-exports.createChatMessageChunkEncoderStream = createChatMessageChunkEncoderStream;
+function _formatForTracing(messages) {
+    const messagesToTrace = [];
+    for (const message of messages) {
+        let messageToTrace = message;
+        if (Array.isArray(message.content)) {
+            for (let idx = 0; idx < message.content.length; idx++) {
+                const block = message.content[idx];
+                if ((0, index_js_1.isURLContentBlock)(block) || (0, index_js_1.isBase64ContentBlock)(block)) {
+                    if (messageToTrace === message) {
+                        // Also shallow-copy content
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        messageToTrace = new message.constructor({
+                            ...messageToTrace,
+                            content: [
+                                ...message.content.slice(0, idx),
+                                (0, index_js_1.convertToOpenAIImageBlock)(block),
+                                ...message.content.slice(idx + 1),
+                            ],
+                        });
+                    }
+                }
+            }
+        }
+        messagesToTrace.push(messageToTrace);
+    }
+    return messagesToTrace;
+}
 /**
  * Base class for chat models. It extends the BaseLanguageModel class and
  * provides methods for generating chat based on input messages.
@@ -92,7 +119,7 @@ class BaseChatModel extends base_js_1.BaseLanguageModel {
                 invocation_params: this?.invocationParams(callOptions),
                 batch_size: 1,
             };
-            const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), [messages], runnableConfig.runId, undefined, extra, undefined, undefined, runnableConfig.runName);
+            const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), [_formatForTracing(messages)], runnableConfig.runId, undefined, extra, undefined, undefined, runnableConfig.runName);
             let generationChunk;
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             let llmOutput;
@@ -167,7 +194,7 @@ class BaseChatModel extends base_js_1.BaseLanguageModel {
                 invocation_params: this?.invocationParams(parsedOptions),
                 batch_size: 1,
             };
-            runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages, handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
+            runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages.map(_formatForTracing), handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
         }
         const generations = [];
         const llmOutputs = [];
@@ -288,7 +315,7 @@ class BaseChatModel extends base_js_1.BaseLanguageModel {
             invocation_params: this?.invocationParams(parsedOptions),
             batch_size: 1,
         };
-        const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages, handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
+        const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages.map(_formatForTracing), handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
         // generate results
         const missingPromptIndices = [];
         const results = await Promise.allSettled(baseMessages.map(async (baseMessage, index) => {
@@ -507,7 +534,7 @@ class BaseChatModel extends base_js_1.BaseLanguageModel {
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const schema = outputSchema;
         const name = config?.name;
-        const description = schema.description ?? "A function available to call.";
+        const description = (0, zod_js_1.getSchemaDescription)(schema) ?? "A function available to call.";
         const method = config?.method;
         const includeRaw = config?.includeRaw;
         if (method === "jsonMode") {
@@ -515,14 +542,14 @@ class BaseChatModel extends base_js_1.BaseLanguageModel {
         }
         let functionName = name ?? "extract";
         let tools;
-        if ((0, is_zod_schema_js_1.isZodSchema)(schema)) {
+        if ((0, zod_js_1.isInteropZodSchema)(schema)) {
             tools = [
                 {
                     type: "function",
                     function: {
                         name: functionName,
                         description,
-                        parameters: (0, zod_to_json_schema_1.zodToJsonSchema)(schema),
+                        parameters: (0, json_schema_js_1.toJsonSchema)(schema),
                     },
                 },
             ];

package/dist/node_modules/@langchain/core/dist/language_models/chat_models.d.ts CHANGED Viewed

@@ -1,4 +1,5 @@
-import { z } from "zod";
+import type { ZodType as ZodTypeV3 } from "zod/v3";
+import type { $ZodType as ZodTypeV4 } from "zod/v4/core";
 import { type BaseMessage, BaseMessageChunk, type BaseMessageLike, AIMessageChunk } from "../messages/index.js";
 import type { BasePromptValueInterface } from "../prompt_values.js";
 import { LLMResult, ChatGenerationChunk, type ChatResult, type Generation } from "../outputs.js";
@@ -183,8 +184,13 @@ export declare abstract class BaseChatModel<CallOptions extends BaseChatModelCal
      * @returns A Promise that resolves to a string.
      */
     predict(text: string, options?: string[] | CallOptions, callbacks?: Callbacks): Promise<string>;
-    withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema: z.ZodType<RunOutput> | Record<string, any>, config?: StructuredOutputMethodOptions<false>): Runnable<BaseLanguageModelInput, RunOutput>;
-    withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema: z.ZodType<RunOutput> | Record<string, any>, config?: StructuredOutputMethodOptions<true>): Runnable<BaseLanguageModelInput, {
+    withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema: ZodTypeV4<RunOutput> | Record<string, any>, config?: StructuredOutputMethodOptions<false>): Runnable<BaseLanguageModelInput, RunOutput>;
+    withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema: ZodTypeV4<RunOutput> | Record<string, any>, config?: StructuredOutputMethodOptions<true>): Runnable<BaseLanguageModelInput, {
+        raw: BaseMessage;
+        parsed: RunOutput;
+    }>;
+    withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema: ZodTypeV3<RunOutput> | Record<string, any>, config?: StructuredOutputMethodOptions<false>): Runnable<BaseLanguageModelInput, RunOutput>;
+    withStructuredOutput<RunOutput extends Record<string, any> = Record<string, any>>(outputSchema: ZodTypeV3<RunOutput> | Record<string, any>, config?: StructuredOutputMethodOptions<true>): Runnable<BaseLanguageModelInput, {
         raw: BaseMessage;
         parsed: RunOutput;
     }>;

package/dist/node_modules/@langchain/core/dist/language_models/chat_models.js CHANGED Viewed

@@ -1,13 +1,13 @@
-import { zodToJsonSchema } from "zod-to-json-schema";
-import { AIMessage, HumanMessage, coerceMessageLikeToMessage, isAIMessageChunk, isBaseMessage, isAIMessage, } from "../messages/index.js";
+import { AIMessage, HumanMessage, coerceMessageLikeToMessage, isAIMessageChunk, isBaseMessage, isAIMessage, convertToOpenAIImageBlock, isURLContentBlock, isBase64ContentBlock, } from "../messages/index.js";
 import { RUN_KEY, } from "../outputs.js";
 import { BaseLanguageModel, } from "./base.js";
 import { CallbackManager, } from "../callbacks/manager.js";
 import { RunnableLambda, RunnableSequence, } from "../runnables/base.js";
 import { concat } from "../utils/stream.js";
 import { RunnablePassthrough } from "../runnables/passthrough.js";
-import { isZodSchema } from "../utils/types/is_zod_schema.js";
+import { getSchemaDescription, isInteropZodSchema, } from "../utils/types/zod.js";
 import { callbackHandlerPrefersStreaming } from "../callbacks/base.js";
+import { toJsonSchema } from "../utils/json_schema.js";
 /**
  * Creates a transform stream for encoding chat message chunks.
  * @deprecated Use {@link BytesOutputParser} instead
@@ -23,6 +23,33 @@ export function createChatMessageChunkEncoderStream() {
         },
     });
 }
+function _formatForTracing(messages) {
+    const messagesToTrace = [];
+    for (const message of messages) {
+        let messageToTrace = message;
+        if (Array.isArray(message.content)) {
+            for (let idx = 0; idx < message.content.length; idx++) {
+                const block = message.content[idx];
+                if (isURLContentBlock(block) || isBase64ContentBlock(block)) {
+                    if (messageToTrace === message) {
+                        // Also shallow-copy content
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        messageToTrace = new message.constructor({
+                            ...messageToTrace,
+                            content: [
+                                ...message.content.slice(0, idx),
+                                convertToOpenAIImageBlock(block),
+                                ...message.content.slice(idx + 1),
+                            ],
+                        });
+                    }
+                }
+            }
+        }
+        messagesToTrace.push(messageToTrace);
+    }
+    return messagesToTrace;
+}
 /**
  * Base class for chat models. It extends the BaseLanguageModel class and
  * provides methods for generating chat based on input messages.
@@ -88,7 +115,7 @@ export class BaseChatModel extends BaseLanguageModel {
                 invocation_params: this?.invocationParams(callOptions),
                 batch_size: 1,
             };
-            const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), [messages], runnableConfig.runId, undefined, extra, undefined, undefined, runnableConfig.runName);
+            const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), [_formatForTracing(messages)], runnableConfig.runId, undefined, extra, undefined, undefined, runnableConfig.runName);
             let generationChunk;
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             let llmOutput;
@@ -163,7 +190,7 @@ export class BaseChatModel extends BaseLanguageModel {
                 invocation_params: this?.invocationParams(parsedOptions),
                 batch_size: 1,
             };
-            runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages, handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
+            runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages.map(_formatForTracing), handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
         }
         const generations = [];
         const llmOutputs = [];
@@ -284,7 +311,7 @@ export class BaseChatModel extends BaseLanguageModel {
             invocation_params: this?.invocationParams(parsedOptions),
             batch_size: 1,
         };
-        const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages, handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
+        const runManagers = await callbackManager_?.handleChatModelStart(this.toJSON(), baseMessages.map(_formatForTracing), handledOptions.runId, undefined, extra, undefined, undefined, handledOptions.runName);
         // generate results
         const missingPromptIndices = [];
         const results = await Promise.allSettled(baseMessages.map(async (baseMessage, index) => {
@@ -503,7 +530,7 @@ export class BaseChatModel extends BaseLanguageModel {
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const schema = outputSchema;
         const name = config?.name;
-        const description = schema.description ?? "A function available to call.";
+        const description = getSchemaDescription(schema) ?? "A function available to call.";
         const method = config?.method;
         const includeRaw = config?.includeRaw;
         if (method === "jsonMode") {
@@ -511,14 +538,14 @@ export class BaseChatModel extends BaseLanguageModel {
         }
         let functionName = name ?? "extract";
         let tools;
-        if (isZodSchema(schema)) {
+        if (isInteropZodSchema(schema)) {
             tools = [
                 {
                     type: "function",
                     function: {
                         name: functionName,
                         description,
-                        parameters: zodToJsonSchema(schema),
+                        parameters: toJsonSchema(schema),
                     },
                 },
             ];

package/dist/node_modules/@langchain/core/dist/load/import_map.cjs CHANGED Viewed

@@ -16,13 +16,23 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.vectorstores = exports.utils__types = exports.utils__tiktoken = exports.utils__testing = exports.utils__stream = exports.utils__math = exports.utils__json_schema = exports.utils__json_patch = exports.utils__hash = exports.utils__function_calling = exports.utils__env = exports.utils__chunk_array = exports.utils__async_caller = exports.tracers__tracer_langchain_v1 = exports.tracers__tracer_langchain = exports.tracers__run_collector = exports.tracers__log_stream = exports.tracers__initialize = exports.tracers__console = exports.tracers__base = exports.tools = exports.stores = exports.retrievers = exports.runnables = exports.prompt_values = exports.prompts = exports.outputs = exports.output_parsers = exports.messages = exports.memory = exports.load__serializable = exports.language_models__llms = exports.language_models__chat_models = exports.language_models__base = exports.example_selectors = exports.embeddings = exports.documents = exports.chat_history = exports.callbacks__promises = exports.callbacks__manager = exports.callbacks__base = exports.caches = exports.agents = void 0;
 exports.agents = __importStar(require("../agents.cjs"));

package/dist/node_modules/@langchain/core/dist/load/index.cjs CHANGED Viewed

@@ -15,15 +15,25 @@ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (
 }) : function(o, v) {
     o["default"] = v;
 });
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.load = void 0;
+exports.load = load;
 const serializable_js_1 = require("./serializable.cjs");
 const import_constants_js_1 = require("./import_constants.cjs");
 const coreImportMap = __importStar(require("./import_map.cjs"));
@@ -186,4 +196,3 @@ async function load(text, mappings) {
     const json = JSON.parse(text);
     return reviver.call({ ...mappings }, json);
 }
-exports.load = load;

package/dist/node_modules/@langchain/core/dist/load/map_keys.cjs CHANGED Viewed

@@ -3,17 +3,17 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.mapKeys = exports.keyFromJson = exports.keyToJson = void 0;
+exports.keyToJson = keyToJson;
+exports.keyFromJson = keyFromJson;
+exports.mapKeys = mapKeys;
 const decamelize_1 = __importDefault(require("decamelize"));
 const camelcase_1 = __importDefault(require("camelcase"));
 function keyToJson(key, map) {
     return map?.[key] || (0, decamelize_1.default)(key);
 }
-exports.keyToJson = keyToJson;
 function keyFromJson(key, map) {
     return map?.[key] || (0, camelcase_1.default)(key);
 }
-exports.keyFromJson = keyFromJson;
 function mapKeys(fields, mapper, map) {
     const mapped = {};
     for (const key in fields) {
@@ -23,4 +23,3 @@ function mapKeys(fields, mapper, map) {
     }
     return mapped;
 }
-exports.mapKeys = mapKeys;

package/dist/node_modules/@langchain/core/dist/load/serializable.cjs CHANGED Viewed

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Serializable = exports.get_lc_unique_name = void 0;
+exports.Serializable = void 0;
+exports.get_lc_unique_name = get_lc_unique_name;
 const map_keys_js_1 = require("./map_keys.cjs");
 function shallowCopy(obj) {
     return Array.isArray(obj) ? [...obj] : { ...obj };
@@ -48,7 +49,6 @@ serializableClass) {
         return serializableClass.name;
     }
 }
-exports.get_lc_unique_name = get_lc_unique_name;
 class Serializable {
     /**
      * The name of the serializable. Override to provide an alias or

package/dist/node_modules/@langchain/core/dist/memory.cjs CHANGED Viewed

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getPromptInputKey = exports.getOutputValue = exports.getInputValue = exports.BaseMemory = void 0;
+exports.getOutputValue = exports.getInputValue = exports.BaseMemory = void 0;
+exports.getPromptInputKey = getPromptInputKey;
 /**
  * Abstract base class for memory in LangChain's Chains. Memory refers to
  * the state in Chains. It can be used to store information about past
@@ -60,4 +61,3 @@ function getPromptInputKey(inputs, memoryVariables) {
     }
     return promptInputKeys[0];
 }
-exports.getPromptInputKey = getPromptInputKey;

package/dist/node_modules/@langchain/core/dist/messages/ai.cjs CHANGED Viewed

@@ -1,6 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.AIMessageChunk = exports.isAIMessageChunk = exports.isAIMessage = exports.AIMessage = void 0;
+exports.AIMessageChunk = exports.AIMessage = void 0;
+exports.isAIMessage = isAIMessage;
+exports.isAIMessageChunk = isAIMessageChunk;
 const json_js_1 = require("../utils/json.cjs");
 const base_js_1 = require("./base.cjs");
 const tool_js_1 = require("./tool.cjs");
@@ -111,11 +113,9 @@ exports.AIMessage = AIMessage;
 function isAIMessage(x) {
     return x._getType() === "ai";
 }
-exports.isAIMessage = isAIMessage;
 function isAIMessageChunk(x) {
     return x._getType() === "ai";
 }
-exports.isAIMessageChunk = isAIMessageChunk;
 /**
  * Represents a chunk of an AI message, which can be concatenated with
  * other AI message chunks.

package/dist/node_modules/@langchain/core/dist/messages/ai.d.ts CHANGED Viewed

@@ -5,16 +5,36 @@ export type AIMessageFields = BaseMessageFields & {
     invalid_tool_calls?: InvalidToolCall[];
     usage_metadata?: UsageMetadata;
 };
+export type ModalitiesTokenDetails = {
+    /**
+     * Text tokens.
+     * Does not need to be reported, but some models will do so.
+     */
+    text?: number;
+    /**
+     * Image (non-video) tokens.
+     */
+    image?: number;
+    /**
+     * Audio tokens.
+     */
+    audio?: number;
+    /**
+     * Video tokens.
+     */
+    video?: number;
+    /**
+     * Document tokens.
+     * e.g. PDF
+     */
+    document?: number;
+};
 /**
  * Breakdown of input token counts.
  *
  * Does not *need* to sum to full input token count. Does *not* need to have all keys.
  */
-export type InputTokenDetails = {
-    /**
-     * Audio input tokens.
-     */
-    audio?: number;
+export type InputTokenDetails = ModalitiesTokenDetails & {
     /**
      * Input tokens that were cached and there was a cache hit.
      *
@@ -34,11 +54,7 @@ export type InputTokenDetails = {
  *
  * Does *not* need to sum to full output token count. Does *not* need to have all keys.
  */
-export type OutputTokenDetails = {
-    /**
-     * Audio output tokens
-     */
-    audio?: number;
+export type OutputTokenDetails = ModalitiesTokenDetails & {
     /**
      * Reasoning output tokens.
      *

package/dist/node_modules/@langchain/core/dist/messages/base.cjs CHANGED Viewed

@@ -1,7 +1,17 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isBaseMessageChunk = exports.isBaseMessage = exports._isMessageFieldWithRole = exports.BaseMessageChunk = exports._mergeObj = exports._mergeLists = exports._mergeDicts = exports.isOpenAIToolCallArray = exports.BaseMessage = exports._mergeStatus = exports.mergeContent = void 0;
+exports.BaseMessageChunk = exports.BaseMessage = void 0;
+exports.mergeContent = mergeContent;
+exports._mergeStatus = _mergeStatus;
+exports.isOpenAIToolCallArray = isOpenAIToolCallArray;
+exports._mergeDicts = _mergeDicts;
+exports._mergeLists = _mergeLists;
+exports._mergeObj = _mergeObj;
+exports._isMessageFieldWithRole = _isMessageFieldWithRole;
+exports.isBaseMessage = isBaseMessage;
+exports.isBaseMessageChunk = isBaseMessageChunk;
 const serializable_js_1 = require("../load/serializable.cjs");
+const content_blocks_js_1 = require("./content_blocks.cjs");
 function mergeContent(firstContent, secondContent) {
     // If first content is a string
     if (typeof firstContent === "string") {
@@ -11,6 +21,17 @@ function mergeContent(firstContent, secondContent) {
         if (typeof secondContent === "string") {
             return firstContent + secondContent;
         }
+        else if (Array.isArray(secondContent) &&
+            secondContent.some((c) => (0, content_blocks_js_1.isDataContentBlock)(c))) {
+            return [
+                {
+                    type: "text",
+                    source_type: "text",
+                    text: firstContent,
+                },
+                ...secondContent,
+            ];
+        }
         else {
             return [{ type: "text", text: firstContent }, ...secondContent];
         }
@@ -26,11 +47,22 @@ function mergeContent(firstContent, secondContent) {
         if (secondContent === "") {
             return firstContent;
         }
-        // Otherwise, add the second content as a new element of the list
-        return [...firstContent, { type: "text", text: secondContent }];
+        else if (Array.isArray(firstContent) &&
+            firstContent.some((c) => (0, content_blocks_js_1.isDataContentBlock)(c))) {
+            return [
+                ...firstContent,
+                {
+                    type: "file",
+                    source_type: "text",
+                    text: secondContent,
+                },
+            ];
+        }
+        else {
+            return [...firstContent, { type: "text", text: secondContent }];
+        }
     }
 }
-exports.mergeContent = mergeContent;
 /**
  * 'Merge' two statuses. If either value passed is 'error', it will return 'error'. Else
  * it will return 'success'.
@@ -45,7 +77,6 @@ function _mergeStatus(left, right) {
     }
     return "success";
 }
-exports._mergeStatus = _mergeStatus;
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function stringifyWithDepthLimit(obj, depthLimit) {
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -84,11 +115,23 @@ class BaseMessage extends serializable_js_1.Serializable {
         };
     }
     /**
-     * @deprecated
-     * Use {@link BaseMessage.content} instead.
+     * Get text content of the message.
      */
     get text() {
-        return typeof this.content === "string" ? this.content : "";
+        if (typeof this.content === "string") {
+            return this.content;
+        }
+        if (!Array.isArray(this.content))
+            return "";
+        return this.content
+            .map((c) => {
+            if (typeof c === "string")
+                return c;
+            if (c.type === "text")
+                return c.text;
+            return "";
+        })
+            .join("");
     }
     /** The type of the message. */
     getType() {
@@ -218,7 +261,6 @@ function isOpenAIToolCallArray(value) {
     return (Array.isArray(value) &&
         value.every((v) => typeof v.index === "number"));
 }
-exports.isOpenAIToolCallArray = isOpenAIToolCallArray;
 function _mergeDicts(
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 left,
@@ -260,7 +302,6 @@ right
     }
     return merged;
 }
-exports._mergeDicts = _mergeDicts;
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function _mergeLists(left, right) {
     if (left === undefined && right === undefined) {
@@ -296,7 +337,6 @@ function _mergeLists(left, right) {
         return merged;
     }
 }
-exports._mergeLists = _mergeLists;
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 function _mergeObj(left, right) {
     if (!left && !right) {
@@ -324,7 +364,6 @@ function _mergeObj(left, right) {
         throw new Error(`Can not merge objects of different types.\nLeft ${left}\nRight ${right}`);
     }
 }
-exports._mergeObj = _mergeObj;
 /**
  * Represents a chunk of a message, which can be concatenated with other
  * message chunks. It includes a method `_merge_kwargs_dict()` for merging
@@ -338,13 +377,10 @@ exports.BaseMessageChunk = BaseMessageChunk;
 function _isMessageFieldWithRole(x) {
     return typeof x.role === "string";
 }
-exports._isMessageFieldWithRole = _isMessageFieldWithRole;
 function isBaseMessage(messageLike) {
     return typeof messageLike?._getType === "function";
 }
-exports.isBaseMessage = isBaseMessage;
 function isBaseMessageChunk(messageLike) {
     return (isBaseMessage(messageLike) &&
         typeof messageLike.concat === "function");
 }
-exports.isBaseMessageChunk = isBaseMessageChunk;

package/dist/node_modules/@langchain/core/dist/messages/base.d.ts CHANGED Viewed

@@ -97,8 +97,7 @@ export declare abstract class BaseMessage extends Serializable implements BaseMe
     lc_serializable: boolean;
     get lc_aliases(): Record<string, string>;
     /**
-     * @deprecated
-     * Use {@link BaseMessage.content} instead.
+     * Get text content of the message.
      */
     get text(): string;
     /** The content of the message. */