langchain 0.0.183 → 0.0.185

package/dist/experimental/openai_assistant/index.js

@@ -0,0 +1,217 @@
+import { OpenAI as OpenAIClient } from "openai";
+import { Runnable } from "../../schema/runnable/base.js";
+import { sleep } from "../../util/time.js";
+import { StructuredTool } from "../../tools/base.js";
+import { formatToOpenAIAssistantTool } from "../../tools/convert_to_openai.js";
+export class OpenAIAssistantRunnable extends Runnable {
+    constructor(fields) {
+        super(fields);
+        Object.defineProperty(this, "lc_namespace", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ["langchain", "experimental", "openai_assistant"]
+        });
+        Object.defineProperty(this, "client", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "assistantId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pollIntervalMs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 1000
+        });
+        Object.defineProperty(this, "asAgent", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.client = fields.client ?? new OpenAIClient(fields?.clientOptions);
+        this.assistantId = fields.assistantId;
+        this.asAgent = fields.asAgent ?? this.asAgent;
+    }
+    static async createAssistant({ model, name, instructions, tools, client, clientOptions, asAgent, pollIntervalMs, }) {
+        const formattedTools = tools?.map((tool) => {
+            // eslint-disable-next-line no-instanceof/no-instanceof
+            if (tool instanceof StructuredTool) {
+                return formatToOpenAIAssistantTool(tool);
+            }
+            return tool;
+        }) ?? [];
+        const oaiClient = client ?? new OpenAIClient(clientOptions);
+        const assistant = await oaiClient.beta.assistants.create({
+            name,
+            instructions,
+            tools: formattedTools,
+            model,
+        });
+        return new this({
+            client: oaiClient,
+            assistantId: assistant.id,
+            asAgent,
+            pollIntervalMs,
+        });
+    }
+    async invoke(input, _options) {
+        let run;
+        if (this.asAgent && input.steps && input.steps.length > 0) {
+            const parsedStepsInput = await this._parseStepsInput(input);
+            run = await this.client.beta.threads.runs.submitToolOutputs(parsedStepsInput.threadId, parsedStepsInput.runId, {
+                tool_outputs: parsedStepsInput.toolOutputs,
+            });
+        }
+        else if (!("threadId" in input)) {
+            const thread = {
+                messages: [
+                    {
+                        role: "user",
+                        content: input.content,
+                        file_ids: input.fileIds,
+                        metadata: input.messagesMetadata,
+                    },
+                ],
+                metadata: input.threadMetadata,
+            };
+            run = await this._createThreadAndRun({
+                ...input,
+                thread,
+            });
+        }
+        else if (!("runId" in input)) {
+            await this.client.beta.threads.messages.create(input.threadId, {
+                content: input.content,
+                role: "user",
+                file_ids: input.file_ids,
+                metadata: input.messagesMetadata,
+            });
+            run = await this._createRun(input);
+        }
+        else {
+            // Submitting tool outputs to an existing run, outside the AgentExecutor
+            // framework.
+            run = await this.client.beta.threads.runs.submitToolOutputs(input.runId, input.threadId, {
+                tool_outputs: input.toolOutputs,
+            });
+        }
+        return this._getResponse(run.id, run.thread_id);
+    }
+    async _parseStepsInput(input) {
+        const { action: { runId, threadId }, } = input.steps[input.steps.length - 1];
+        const run = await this._waitForRun(runId, threadId);
+        const toolCalls = run.required_action?.submit_tool_outputs.tool_calls;
+        if (!toolCalls) {
+            return input;
+        }
+        const toolOutputs = toolCalls.flatMap((toolCall) => {
+            const matchedAction = input.steps.find((step) => step.action.toolCallId === toolCall.id);
+            return matchedAction
+                ? [
+                    {
+                        output: matchedAction.observation,
+                        tool_call_id: matchedAction.action.toolCallId,
+                    },
+                ]
+                : [];
+        });
+        return { toolOutputs, runId, threadId };
+    }
+    async _createRun({ instructions, model, tools, metadata, threadId, }) {
+        const run = this.client.beta.threads.runs.create(threadId, {
+            assistant_id: this.assistantId,
+            instructions,
+            model,
+            tools,
+            metadata,
+        });
+        return run;
+    }
+    async _createThreadAndRun(input) {
+        const params = [
+            "instructions",
+            "model",
+            "tools",
+            "run_metadata",
+        ]
+            .filter((key) => key in input)
+            .reduce((obj, key) => {
+            const newObj = obj;
+            newObj[key] = input[key];
+            return newObj;
+        }, {});
+        const run = this.client.beta.threads.createAndRun({
+            ...params,
+            thread: input.thread,
+            assistant_id: this.assistantId,
+        });
+        return run;
+    }
+    async _waitForRun(runId, threadId) {
+        let inProgress = true;
+        let run = {};
+        while (inProgress) {
+            run = await this.client.beta.threads.runs.retrieve(threadId, runId);
+            inProgress = ["in_progress", "queued"].includes(run.status);
+            if (inProgress) {
+                await sleep(this.pollIntervalMs);
+            }
+        }
+        return run;
+    }
+    async _getResponse(runId, threadId) {
+        const run = await this._waitForRun(runId, threadId);
+        if (run.status === "completed") {
+            const messages = await this.client.beta.threads.messages.list(threadId, {
+                order: "asc",
+            });
+            const newMessages = messages.data.filter((msg) => msg.run_id === runId);
+            if (!this.asAgent) {
+                return newMessages;
+            }
+            const answer = newMessages.flatMap((msg) => msg.content);
+            if (answer.every((item) => item.type === "text")) {
+                const answerString = answer
+                    .map((item) => item.type === "text" && item.text.value)
+                    .join("\n");
+                return {
+                    returnValues: {
+                        output: answerString,
+                    },
+                    log: "",
+                    runId,
+                    threadId,
+                };
+            }
+        }
+        else if (run.status === "requires_action") {
+            if (!this.asAgent) {
+                return run.required_action?.submit_tool_outputs.tool_calls ?? [];
+            }
+            const actions = [];
+            run.required_action?.submit_tool_outputs.tool_calls.forEach((item) => {
+                const functionCall = item.function;
+                const args = JSON.parse(functionCall.arguments);
+                actions.push({
+                    tool: functionCall.name,
+                    toolInput: args,
+                    toolCallId: item.id,
+                    log: "",
+                    runId,
+                    threadId,
+                });
+            });
+            return actions;
+        }
+        const runInfo = JSON.stringify(run, null, 2);
+        throw new Error(`Unexpected run status ${run.status}.\nFull run info:\n\n${runInfo}`);
+    }
+}

package/dist/experimental/openai_assistant/schema.cjs

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/experimental/openai_assistant/schema.d.ts

@@ -0,0 +1,12 @@
+import type { OpenAI as OpenAIClient } from "openai";
+import type { AgentFinish, AgentAction } from "../../schema/index.js";
+export type OpenAIAssistantFinish = AgentFinish & {
+    runId: string;
+    threadId: string;
+};
+export type OpenAIAssistantAction = AgentAction & {
+    toolCallId: string;
+    runId: string;
+    threadId: string;
+};
+export type OpenAIToolType = Array<OpenAIClient.Beta.AssistantCreateParams.AssistantToolsCode | OpenAIClient.Beta.AssistantCreateParams.AssistantToolsRetrieval | OpenAIClient.Beta.AssistantCreateParams.AssistantToolsFunction>;

package/dist/experimental/openai_assistant/schema.js

@@ -0,0 +1 @@
+export {};

package/dist/experimental/plan_and_execute/agent_executor.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.PlanAndExecuteAgentExecutor = void 0;
+exports.PlanAndExecuteAgentExecutor = exports.isDynamicStructuredTool = void 0;
 const base_js_1 = require("../../chains/base.cjs");
 const base_js_2 = require("./base.cjs");
 const executor_js_1 = require("../../agents/executor.cjs");
@@ -8,6 +8,21 @@ const prompt_js_1 = require("./prompt.cjs");
 const llm_chain_js_1 = require("../../chains/llm_chain.cjs");
 const outputParser_js_1 = require("./outputParser.cjs");
 const index_js_1 = require("../../agents/chat/index.cjs");
+const index_js_2 = require("../../agents/index.cjs");
+/**
+ * A utility function to distiguish a dynamicstructuredtool over other tools.
+ * @param tool the tool to test
+ * @returns bool
+ */
+function isDynamicStructuredTool(tool) {
+    // We check for the existence of the static lc_name method in the object's constructor
+    return (
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    typeof tool.constructor.lc_name === "function" &&
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        tool.constructor.lc_name() === "DynamicStructuredTool");
+}
+exports.isDynamicStructuredTool = isDynamicStructuredTool;
 /**
  * Class representing a plan-and-execute agent executor. This agent
  * decides on the full sequence of actions upfront, then executes them all
@@ -86,7 +101,18 @@ class PlanAndExecuteAgentExecutor extends base_js_1.BaseChain {
      * @returns A new ChainStepExecutor instance.
      */
     static getDefaultStepExecutor({ llm, tools, humanMessageTemplate = prompt_js_1.DEFAULT_STEP_EXECUTOR_HUMAN_CHAT_MESSAGE_TEMPLATE, }) {
-        const agent = index_js_1.ChatAgent.fromLLMAndTools(llm, tools, {
+        let agent;
+        if (isDynamicStructuredTool(tools[0])) {
+            agent = index_js_2.StructuredChatAgent.fromLLMAndTools(llm, tools, {
+                humanMessageTemplate,
+                inputVariables: ["previous_steps", "current_step", "agent_scratchpad"],
+            });
+            return new base_js_2.ChainStepExecutor(executor_js_1.AgentExecutor.fromAgentAndTools({
+                agent,
+                tools,
+            }));
+        }
+        agent = index_js_1.ChatAgent.fromLLMAndTools(llm, tools, {
             humanMessageTemplate,
         });
         return new base_js_2.ChainStepExecutor(executor_js_1.AgentExecutor.fromAgentAndTools({

package/dist/experimental/plan_and_execute/agent_executor.d.ts

@@ -4,7 +4,14 @@ import { ChainValues } from "../../schema/index.js";
 import { BaseLanguageModel } from "../../base_language/index.js";
 import { CallbackManagerForChainRun } from "../../callbacks/manager.js";
 import { Tool } from "../../tools/base.js";
+import { DynamicStructuredTool } from "../../tools/dynamic.js";
 import { SerializedLLMChain } from "../../chains/serde.js";
+/**
+ * A utility function to distiguish a dynamicstructuredtool over other tools.
+ * @param tool the tool to test
+ * @returns bool
+ */
+export declare function isDynamicStructuredTool(tool: Tool | DynamicStructuredTool): tool is DynamicStructuredTool;
 /**
  * Interface for the input to the PlanAndExecuteAgentExecutor class. It
  * extends ChainInputs and includes additional properties for the planner,
@@ -42,7 +49,7 @@ export declare class PlanAndExecuteAgentExecutor extends BaseChain {
      */
     static getDefaultPlanner({ llm, tools, }: {
         llm: BaseLanguageModel;
-        tools: Tool[];
+        tools: Tool[] | DynamicStructuredTool[];
     }): Promise<LLMPlanner>;
     /**
      * Static method that returns a default step executor for the agent. It
@@ -55,7 +62,7 @@ export declare class PlanAndExecuteAgentExecutor extends BaseChain {
      */
     static getDefaultStepExecutor({ llm, tools, humanMessageTemplate, }: {
         llm: BaseLanguageModel;
-        tools: Tool[];
+        tools: Tool[] | DynamicStructuredTool[];
         humanMessageTemplate?: string;
     }): ChainStepExecutor;
     /**
@@ -70,7 +77,7 @@ export declare class PlanAndExecuteAgentExecutor extends BaseChain {
      */
     static fromLLMAndTools({ llm, tools, humanMessageTemplate, }: {
         llm: BaseLanguageModel;
-        tools: Tool[];
+        tools: Tool[] | DynamicStructuredTool[];
         humanMessageTemplate?: string;
     } & Omit<PlanAndExecuteAgentExecutorInput, "planner" | "stepExecutor">): Promise<PlanAndExecuteAgentExecutor>;
     /** @ignore */

package/dist/experimental/plan_and_execute/agent_executor.js

@@ -5,6 +5,20 @@ import { DEFAULT_STEP_EXECUTOR_HUMAN_CHAT_MESSAGE_TEMPLATE, getPlannerChatPrompt
 import { LLMChain } from "../../chains/llm_chain.js";
 import { PlanOutputParser } from "./outputParser.js";
 import { ChatAgent } from "../../agents/chat/index.js";
+import { StructuredChatAgent } from "../../agents/index.js";
+/**
+ * A utility function to distiguish a dynamicstructuredtool over other tools.
+ * @param tool the tool to test
+ * @returns bool
+ */
+export function isDynamicStructuredTool(tool) {
+    // We check for the existence of the static lc_name method in the object's constructor
+    return (
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    typeof tool.constructor.lc_name === "function" &&
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        tool.constructor.lc_name() === "DynamicStructuredTool");
+}
 /**
  * Class representing a plan-and-execute agent executor. This agent
  * decides on the full sequence of actions upfront, then executes them all
@@ -83,7 +97,18 @@ export class PlanAndExecuteAgentExecutor extends BaseChain {
      * @returns A new ChainStepExecutor instance.
      */
     static getDefaultStepExecutor({ llm, tools, humanMessageTemplate = DEFAULT_STEP_EXECUTOR_HUMAN_CHAT_MESSAGE_TEMPLATE, }) {
-        const agent = ChatAgent.fromLLMAndTools(llm, tools, {
+        let agent;
+        if (isDynamicStructuredTool(tools[0])) {
+            agent = StructuredChatAgent.fromLLMAndTools(llm, tools, {
+                humanMessageTemplate,
+                inputVariables: ["previous_steps", "current_step", "agent_scratchpad"],
+            });
+            return new ChainStepExecutor(AgentExecutor.fromAgentAndTools({
+                agent,
+                tools,
+            }));
+        }
+        agent = ChatAgent.fromLLMAndTools(llm, tools, {
             humanMessageTemplate,
         });
         return new ChainStepExecutor(AgentExecutor.fromAgentAndTools({

package/dist/experimental/plan_and_execute/prompt.d.ts

@@ -1,5 +1,6 @@
 import { ChatPromptTemplate } from "../../prompts/chat.js";
 import { Tool } from "../../tools/base.js";
+import { DynamicStructuredTool } from "../../tools/dynamic.js";
 export declare const PLANNER_SYSTEM_PROMPT_MESSAGE_TEMPLATE: string;
 export declare const DEFAULT_STEP_EXECUTOR_HUMAN_CHAT_MESSAGE_TEMPLATE = "Previous steps: {previous_steps}\n\nCurrent objective: {current_step}\n\n{agent_scratchpad}\n\nYou may extract and combine relevant data from your previous steps when responding to me.";
 /**
@@ -9,4 +10,4 @@ export declare const DEFAULT_STEP_EXECUTOR_HUMAN_CHAT_MESSAGE_TEMPLATE = "Previo
  * @param tools the tools available to the `planner`
  * @returns
  */
-export declare const getPlannerChatPrompt: (tools: Tool[]) => Promise<ChatPromptTemplate<import("../../schema/index.js").InputValues<string>, any>>;
+export declare const getPlannerChatPrompt: (tools: Tool[] | DynamicStructuredTool[]) => Promise<ChatPromptTemplate<import("../../schema/index.js").InputValues<string>, any>>;

package/dist/load/import_map.cjs

@@ -24,12 +24,13 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.chat_models__cloudflare_workersai = exports.chat_models__anthropic = exports.chat_models__openai = exports.chat_models__base = exports.document_transformers__openai_functions = exports.document_loaders__web__sort_xyz_blockchain = exports.document_loaders__web__serpapi = exports.document_loaders__web__searchapi = exports.document_loaders__base = exports.document = exports.memory = exports.text_splitter = exports.vectorstores__xata = exports.vectorstores__vectara = exports.vectorstores__prisma = exports.vectorstores__memory = exports.vectorstores__base = exports.prompts = exports.llms__fake = exports.llms__yandex = exports.llms__fireworks = exports.llms__ollama = exports.llms__cloudflare_workersai = exports.llms__aleph_alpha = exports.llms__ai21 = exports.llms__openai = exports.llms__base = exports.embeddings__voyage = exports.embeddings__minimax = exports.embeddings__openai = exports.embeddings__ollama = exports.embeddings__fake = exports.embeddings__cache_backed = exports.embeddings__base = exports.chains__openai_functions = exports.chains__combine_documents__reduce = exports.chains = exports.tools__render = exports.tools = exports.base_language = exports.agents__openai__output_parser = exports.agents__xml__output_parser = exports.agents__react__output_parser = exports.agents__format_scratchpad__log_to_message = exports.agents__format_scratchpad__xml = exports.agents__format_scratchpad__log = exports.agents__format_scratchpad = exports.agents__toolkits = exports.agents = exports.load__serializable = void 0;
-exports.runnables__remote = exports.evaluation = exports.experimental__chains__violation_of_expectations = exports.experimental__chat_models__bittensor = exports.experimental__plan_and_execute = exports.experimental__generative_agents = exports.experimental__babyagi = exports.experimental__autogpt = exports.util__time = exports.util__math = exports.util__document = exports.storage__in_memory = exports.storage__encoder_backed = exports.stores__message__in_memory = exports.stores__file__in_memory = exports.stores__doc__in_memory = exports.cache = exports.retrievers__vespa = exports.retrievers__score_threshold = exports.retrievers__hyde = exports.retrievers__document_compressors__embeddings_filter = exports.retrievers__document_compressors__chain_extract = exports.retrievers__time_weighted = exports.retrievers__tavily_search_api = exports.retrievers__parent_document = exports.retrievers__multi_vector = exports.retrievers__multi_query = exports.retrievers__document_compressors = exports.retrievers__contextual_compression = exports.retrievers__databerry = exports.retrievers__chaindesk = exports.retrievers__remote = exports.output_parsers = exports.callbacks = exports.schema__storage = exports.schema__runnable = exports.schema__retriever = exports.schema__query_constructor = exports.schema__prompt_template = exports.schema__output_parser = exports.schema__document = exports.schema = exports.chat_models__fake = exports.chat_models__yandex = exports.chat_models__minimax = exports.chat_models__ollama = exports.chat_models__baiduwenxin = exports.chat_models__fireworks = void 0;
+exports.chat_models__anthropic = exports.chat_models__openai = exports.chat_models__base = exports.document_transformers__openai_functions = exports.document_loaders__web__sort_xyz_blockchain = exports.document_loaders__web__serpapi = exports.document_loaders__web__searchapi = exports.document_loaders__base = exports.document = exports.memory = exports.text_splitter = exports.vectorstores__xata = exports.vectorstores__vectara = exports.vectorstores__prisma = exports.vectorstores__memory = exports.vectorstores__base = exports.prompts = exports.llms__fake = exports.llms__yandex = exports.llms__fireworks = exports.llms__ollama = exports.llms__cloudflare_workersai = exports.llms__aleph_alpha = exports.llms__ai21 = exports.llms__openai = exports.llms__base = exports.embeddings__voyage = exports.embeddings__minimax = exports.embeddings__openai = exports.embeddings__ollama = exports.embeddings__fake = exports.embeddings__cache_backed = exports.embeddings__base = exports.chains__openai_functions = exports.chains__combine_documents__reduce = exports.chains = exports.tools__render = exports.tools = exports.base_language = exports.agents__openai__output_parser = exports.agents__xml__output_parser = exports.agents__react__output_parser = exports.agents__format_scratchpad__log_to_message = exports.agents__format_scratchpad__xml = exports.agents__format_scratchpad__log = exports.agents__format_scratchpad__openai_tools = exports.agents__format_scratchpad = exports.agents__toolkits = exports.agents = exports.load__serializable = void 0;
+exports.runnables__remote = exports.evaluation = exports.experimental__chains__violation_of_expectations = exports.experimental__chat_models__bittensor = exports.experimental__plan_and_execute = exports.experimental__generative_agents = exports.experimental__babyagi = exports.experimental__openai_assistant = exports.experimental__autogpt = exports.util__time = exports.util__math = exports.util__document = exports.storage__in_memory = exports.storage__encoder_backed = exports.stores__message__in_memory = exports.stores__file__in_memory = exports.stores__doc__in_memory = exports.cache = exports.retrievers__vespa = exports.retrievers__score_threshold = exports.retrievers__hyde = exports.retrievers__document_compressors__embeddings_filter = exports.retrievers__document_compressors__chain_extract = exports.retrievers__time_weighted = exports.retrievers__tavily_search_api = exports.retrievers__parent_document = exports.retrievers__multi_vector = exports.retrievers__multi_query = exports.retrievers__document_compressors = exports.retrievers__contextual_compression = exports.retrievers__databerry = exports.retrievers__chaindesk = exports.retrievers__remote = exports.output_parsers = exports.callbacks = exports.schema__storage = exports.schema__runnable = exports.schema__retriever = exports.schema__query_constructor = exports.schema__prompt_template = exports.schema__output_parser = exports.schema__document = exports.schema = exports.chat_models__fake = exports.chat_models__yandex = exports.chat_models__minimax = exports.chat_models__ollama = exports.chat_models__baiduwenxin = exports.chat_models__fireworks = exports.chat_models__cloudflare_workersai = void 0;
 exports.load__serializable = __importStar(require("../load/serializable.cjs"));
 exports.agents = __importStar(require("../agents/index.cjs"));
 exports.agents__toolkits = __importStar(require("../agents/toolkits/index.cjs"));
 exports.agents__format_scratchpad = __importStar(require("../agents/format_scratchpad/openai_functions.cjs"));
+exports.agents__format_scratchpad__openai_tools = __importStar(require("../agents/format_scratchpad/openai_tools.cjs"));
 exports.agents__format_scratchpad__log = __importStar(require("../agents/format_scratchpad/log.cjs"));
 exports.agents__format_scratchpad__xml = __importStar(require("../agents/format_scratchpad/xml.cjs"));
 exports.agents__format_scratchpad__log_to_message = __importStar(require("../agents/format_scratchpad/log_to_message.cjs"));
@@ -117,6 +118,7 @@ exports.util__document = __importStar(require("../util/document.cjs"));
 exports.util__math = __importStar(require("../util/math.cjs"));
 exports.util__time = __importStar(require("../util/time.cjs"));
 exports.experimental__autogpt = __importStar(require("../experimental/autogpt/index.cjs"));
+exports.experimental__openai_assistant = __importStar(require("../experimental/openai_assistant/index.cjs"));
 exports.experimental__babyagi = __importStar(require("../experimental/babyagi/index.cjs"));
 exports.experimental__generative_agents = __importStar(require("../experimental/generative_agents/index.cjs"));
 exports.experimental__plan_and_execute = __importStar(require("../experimental/plan_and_execute/index.cjs"));

package/dist/load/import_map.d.ts

@@ -2,6 +2,7 @@ export * as load__serializable from "../load/serializable.js";
 export * as agents from "../agents/index.js";
 export * as agents__toolkits from "../agents/toolkits/index.js";
 export * as agents__format_scratchpad from "../agents/format_scratchpad/openai_functions.js";
+export * as agents__format_scratchpad__openai_tools from "../agents/format_scratchpad/openai_tools.js";
 export * as agents__format_scratchpad__log from "../agents/format_scratchpad/log.js";
 export * as agents__format_scratchpad__xml from "../agents/format_scratchpad/xml.js";
 export * as agents__format_scratchpad__log_to_message from "../agents/format_scratchpad/log_to_message.js";
@@ -89,6 +90,7 @@ export * as util__document from "../util/document.js";
 export * as util__math from "../util/math.js";
 export * as util__time from "../util/time.js";
 export * as experimental__autogpt from "../experimental/autogpt/index.js";
+export * as experimental__openai_assistant from "../experimental/openai_assistant/index.js";
 export * as experimental__babyagi from "../experimental/babyagi/index.js";
 export * as experimental__generative_agents from "../experimental/generative_agents/index.js";
 export * as experimental__plan_and_execute from "../experimental/plan_and_execute/index.js";

package/dist/load/import_map.js

@@ -3,6 +3,7 @@ export * as load__serializable from "../load/serializable.js";
 export * as agents from "../agents/index.js";
 export * as agents__toolkits from "../agents/toolkits/index.js";
 export * as agents__format_scratchpad from "../agents/format_scratchpad/openai_functions.js";
+export * as agents__format_scratchpad__openai_tools from "../agents/format_scratchpad/openai_tools.js";
 export * as agents__format_scratchpad__log from "../agents/format_scratchpad/log.js";
 export * as agents__format_scratchpad__xml from "../agents/format_scratchpad/xml.js";
 export * as agents__format_scratchpad__log_to_message from "../agents/format_scratchpad/log_to_message.js";
@@ -90,6 +91,7 @@ export * as util__document from "../util/document.js";
 export * as util__math from "../util/math.js";
 export * as util__time from "../util/time.js";
 export * as experimental__autogpt from "../experimental/autogpt/index.js";
+export * as experimental__openai_assistant from "../experimental/openai_assistant/index.js";
 export * as experimental__babyagi from "../experimental/babyagi/index.js";
 export * as experimental__generative_agents from "../experimental/generative_agents/index.js";
 export * as experimental__plan_and_execute from "../experimental/plan_and_execute/index.js";

package/dist/prompts/chat.cjs

@@ -337,13 +337,34 @@ class ChatPromptTemplate extends BaseChatPromptTemplate {
     _getPromptType() {
         return "chat";
     }
+    async _parseImagePrompts(message, inputValues) {
+        if (typeof message.content === "string") {
+            return message;
+        }
+        const formattedMessageContent = await Promise.all(message.content.map(async (item) => {
+            if (item.type !== "image_url" ||
+                typeof item.image_url === "string" ||
+                !item.image_url?.url) {
+                return item;
+            }
+            const imageUrl = item.image_url.url;
+            const promptTemplatePlaceholder = prompt_js_1.PromptTemplate.fromTemplate(imageUrl);
+            const formattedUrl = await promptTemplatePlaceholder.format(inputValues);
+            // eslint-disable-next-line no-param-reassign
+            item.image_url.url = formattedUrl;
+            return item;
+        }));
+        // eslint-disable-next-line no-param-reassign
+        message.content = formattedMessageContent;
+        return message;
+    }
     async formatMessages(values) {
         const allValues = await this.mergePartialAndUserVariables(values);
         let resultMessages = [];
         for (const promptMessage of this.promptMessages) {
             // eslint-disable-next-line no-instanceof/no-instanceof
             if (promptMessage instanceof index_js_1.BaseMessage) {
-                resultMessages.push(promptMessage);
+                resultMessages.push(await this._parseImagePrompts(promptMessage, allValues));
             }
             else {
                 const inputValues = promptMessage.inputVariables.reduce((acc, inputVariable) => {

package/dist/prompts/chat.d.ts

@@ -169,6 +169,7 @@ export declare class ChatPromptTemplate<RunInput extends InputValues = any, Part
     validateTemplate: boolean;
     constructor(input: ChatPromptTemplateInput<RunInput, PartialVariableName>);
     _getPromptType(): "chat";
+    private _parseImagePrompts;
     formatMessages(values: TypedPromptInputValues<RunInput>): Promise<BaseMessage[]>;
     partial<NewPartialVariableName extends string>(values: PartialValues<NewPartialVariableName>): Promise<ChatPromptTemplate<InputValues<Exclude<Extract<keyof RunInput, string>, NewPartialVariableName>>, any>>;
     /**

package/dist/prompts/chat.js

@@ -325,13 +325,34 @@ export class ChatPromptTemplate extends BaseChatPromptTemplate {
     _getPromptType() {
         return "chat";
     }
+    async _parseImagePrompts(message, inputValues) {
+        if (typeof message.content === "string") {
+            return message;
+        }
+        const formattedMessageContent = await Promise.all(message.content.map(async (item) => {
+            if (item.type !== "image_url" ||
+                typeof item.image_url === "string" ||
+                !item.image_url?.url) {
+                return item;
+            }
+            const imageUrl = item.image_url.url;
+            const promptTemplatePlaceholder = PromptTemplate.fromTemplate(imageUrl);
+            const formattedUrl = await promptTemplatePlaceholder.format(inputValues);
+            // eslint-disable-next-line no-param-reassign
+            item.image_url.url = formattedUrl;
+            return item;
+        }));
+        // eslint-disable-next-line no-param-reassign
+        message.content = formattedMessageContent;
+        return message;
+    }
     async formatMessages(values) {
         const allValues = await this.mergePartialAndUserVariables(values);
         let resultMessages = [];
         for (const promptMessage of this.promptMessages) {
             // eslint-disable-next-line no-instanceof/no-instanceof
             if (promptMessage instanceof BaseMessage) {
-                resultMessages.push(promptMessage);
+                resultMessages.push(await this._parseImagePrompts(promptMessage, allValues));
             }
             else {
                 const inputValues = promptMessage.inputVariables.reduce((acc, inputVariable) => {

package/dist/schema/index.d.ts

@@ -70,6 +70,7 @@ export type MessageContent = string | {
     text?: string;
     image_url?: string | {
         url: string;
+        detail?: "low" | "high";
     };
 }[];
 export interface BaseMessageFields {

package/dist/tools/convert_to_openai.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatToOpenAITool = exports.formatToOpenAIFunction = void 0;
+exports.formatToOpenAIAssistantTool = exports.formatToOpenAITool = exports.formatToOpenAIFunction = void 0;
 const zod_to_json_schema_1 = require("zod-to-json-schema");
 /**
  * Formats a `StructuredTool` instance into a format that is compatible
@@ -17,13 +17,25 @@ function formatToOpenAIFunction(tool) {
 }
 exports.formatToOpenAIFunction = formatToOpenAIFunction;
 function formatToOpenAITool(tool) {
+    const schema = (0, zod_to_json_schema_1.zodToJsonSchema)(tool.schema);
     return {
         type: "function",
         function: {
             name: tool.name,
             description: tool.description,
-            parameters: (0, zod_to_json_schema_1.zodToJsonSchema)(tool.schema),
+            parameters: schema,
         },
     };
 }
 exports.formatToOpenAITool = formatToOpenAITool;
+function formatToOpenAIAssistantTool(tool) {
+    return {
+        type: "function",
+        function: {
+            name: tool.name,
+            description: tool.description,
+            parameters: (0, zod_to_json_schema_1.zodToJsonSchema)(tool.schema),
+        },
+    };
+}
+exports.formatToOpenAIAssistantTool = formatToOpenAIAssistantTool;

package/dist/tools/convert_to_openai.d.ts

@@ -8,3 +8,4 @@ import { StructuredTool } from "./base.js";
  */
 export declare function formatToOpenAIFunction(tool: StructuredTool): OpenAIClient.Chat.ChatCompletionCreateParams.Function;
 export declare function formatToOpenAITool(tool: StructuredTool): OpenAIClient.Chat.ChatCompletionTool;
+export declare function formatToOpenAIAssistantTool(tool: StructuredTool): OpenAIClient.Beta.AssistantCreateParams.AssistantToolsFunction;

package/dist/tools/convert_to_openai.js

@@ -13,6 +13,17 @@ export function formatToOpenAIFunction(tool) {
     };
 }
 export function formatToOpenAITool(tool) {
+    const schema = zodToJsonSchema(tool.schema);
+    return {
+        type: "function",
+        function: {
+            name: tool.name,
+            description: tool.description,
+            parameters: schema,
+        },
+    };
+}
+export function formatToOpenAIAssistantTool(tool) {
     return {
         type: "function",
         function: {

package/dist/tools/index.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.formatToOpenAIFunction = exports.SearchApi = exports.SearxngSearch = exports.DataForSeoAPISearch = exports.WolframAlphaTool = exports.WikipediaQueryRun = exports.BraveSearch = exports.WriteFileTool = exports.ReadFileTool = exports.AIPluginTool = exports.GoogleCustomSearch = exports.Serper = exports.ZapierNLAWrapper = exports.ZapierNLARunAction = exports.VectorStoreQATool = exports.RequestsPostTool = exports.RequestsGetTool = exports.JsonGetValueTool = exports.JsonListKeysTool = exports.JsonSpec = exports.ChainTool = exports.IFTTTWebhook = exports.DynamicStructuredTool = exports.DynamicTool = exports.StructuredTool = exports.Tool = exports.BingSerpAPI = exports.DadJokeAPI = exports.SerpAPI = void 0;
+exports.formatToOpenAITool = exports.formatToOpenAIFunction = exports.SearchApi = exports.SearxngSearch = exports.DataForSeoAPISearch = exports.WolframAlphaTool = exports.WikipediaQueryRun = exports.BraveSearch = exports.WriteFileTool = exports.ReadFileTool = exports.AIPluginTool = exports.GoogleCustomSearch = exports.Serper = exports.ZapierNLAWrapper = exports.ZapierNLARunAction = exports.VectorStoreQATool = exports.RequestsPostTool = exports.RequestsGetTool = exports.JsonGetValueTool = exports.JsonListKeysTool = exports.JsonSpec = exports.ChainTool = exports.IFTTTWebhook = exports.DynamicStructuredTool = exports.DynamicTool = exports.StructuredTool = exports.Tool = exports.BingSerpAPI = exports.DadJokeAPI = exports.SerpAPI = void 0;
 var serpapi_js_1 = require("./serpapi.cjs");
 Object.defineProperty(exports, "SerpAPI", { enumerable: true, get: function () { return serpapi_js_1.SerpAPI; } });
 var dadjokeapi_js_1 = require("./dadjokeapi.cjs");
@@ -52,3 +52,4 @@ var searchapi_js_1 = require("./searchapi.cjs");
 Object.defineProperty(exports, "SearchApi", { enumerable: true, get: function () { return searchapi_js_1.SearchApi; } });
 var convert_to_openai_js_1 = require("./convert_to_openai.cjs");
 Object.defineProperty(exports, "formatToOpenAIFunction", { enumerable: true, get: function () { return convert_to_openai_js_1.formatToOpenAIFunction; } });
+Object.defineProperty(exports, "formatToOpenAITool", { enumerable: true, get: function () { return convert_to_openai_js_1.formatToOpenAITool; } });

package/dist/tools/index.d.ts

@@ -19,4 +19,4 @@ export { WolframAlphaTool } from "./wolframalpha.js";
 export { DataForSeoAPISearch, type DataForSeoApiConfig, } from "./dataforseo_api_search.js";
 export { SearxngSearch } from "./searxng_search.js";
 export { SearchApi, type SearchApiParameters } from "./searchapi.js";
-export { formatToOpenAIFunction } from "./convert_to_openai.js";
+export { formatToOpenAIFunction, formatToOpenAITool, } from "./convert_to_openai.js";

package/dist/tools/index.js

@@ -19,4 +19,4 @@ export { WolframAlphaTool } from "./wolframalpha.js";
 export { DataForSeoAPISearch, } from "./dataforseo_api_search.js";
 export { SearxngSearch } from "./searxng_search.js";
 export { SearchApi } from "./searchapi.js";
-export { formatToOpenAIFunction } from "./convert_to_openai.js";
+export { formatToOpenAIFunction, formatToOpenAITool, } from "./convert_to_openai.js";

package/dist/vectorstores/momento_vector_index.cjs

@@ -224,7 +224,7 @@ class MomentoVectorIndex extends base_js_1.VectorStore {
                     pageContent: hit.metadata[this.textField]?.toString() ?? "",
                     metadata: Object.fromEntries(Object.entries(hit.metadata).filter(([key]) => key !== this.textField)),
                 }),
-                hit.distance,
+                hit.score,
             ]);
         }
         else if (response instanceof sdk_core_1.VectorSearch.Error) {