langchain 0.0.197 → 0.0.199

package/dist/graphs/neo4j_graph.cjs

@@ -59,6 +59,16 @@ class Neo4jGraph {
             writable: true,
             value: ""
         });
+        Object.defineProperty(this, "structuredSchema", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {
+                nodeProps: {},
+                relProps: {},
+                relationships: [],
+            }
+        });
         try {
             this.driver = neo4j_driver_1.default.driver(url, neo4j_driver_1.default.auth.basic(username, password));
             this.database = database;
@@ -96,6 +106,9 @@ class Neo4jGraph {
     getSchema() {
         return this.schema;
     }
+    getStructuredSchema() {
+        return this.structuredSchema;
+    }
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     async query(query, params = {}) {
         try {
@@ -138,21 +151,43 @@ class Neo4jGraph {
       YIELD label, other, elementType, type, property
       WHERE type = "RELATIONSHIP" AND elementType = "node"
       UNWIND other AS other_node
-      RETURN "(:" + label + ")-[:" + property + "]->(:" + toString(other_node) + ")" AS output
-    `;
-        const nodeProperties = await this.query(nodePropertiesQuery);
-        const relationshipsProperties = await this.query(relPropertiesQuery);
-        const relationships = await this.query(relQuery);
-        this.schema = `
-      Node properties are the following:
-      ${JSON.stringify(nodeProperties?.map((el) => el.output))}
-
-      Relationship properties are the following:
-      ${JSON.stringify(relationshipsProperties?.map((el) => el.output))}
-
-      The relationships are the following:
-      ${JSON.stringify(relationships?.map((el) => el.output))}
+      RETURN {start: label, type: property, end: toString(other_node)} AS output
     `;
+        // Assuming query method is defined and returns a Promise
+        const nodeProperties = (await this.query(nodePropertiesQuery))?.map((el) => el.output);
+        const relationshipsProperties = (await this.query(relPropertiesQuery))?.map((el) => el.output);
+        const relationships = (await this.query(relQuery))?.map((el) => el.output);
+        // Structured schema similar to Python's dictionary comprehension
+        this.structuredSchema = {
+            nodeProps: Object.fromEntries(nodeProperties?.map((el) => [el.labels, el.properties]) || []),
+            relProps: Object.fromEntries(relationshipsProperties?.map((el) => [el.type, el.properties]) || []),
+            relationships: relationships || [],
+        };
+        // Format node properties
+        const formattedNodeProps = nodeProperties?.map((el) => {
+            const propsStr = el.properties
+                .map((prop) => `${prop.property}: ${prop.type}`)
+                .join(", ");
+            return `${el.labels} {${propsStr}}`;
+        });
+        // Format relationship properties
+        const formattedRelProps = relationshipsProperties?.map((el) => {
+            const propsStr = el.properties
+                .map((prop) => `${prop.property}: ${prop.type}`)
+                .join(", ");
+            return `${el.type} {${propsStr}}`;
+        });
+        // Format relationships
+        const formattedRels = relationships?.map((el) => `(:${el.start})-[:${el.type}]->(:${el.end})`);
+        // Combine all formatted elements into a single string
+        this.schema = [
+            "Node properties are the following:",
+            formattedNodeProps?.join(", "),
+            "Relationship properties are the following:",
+            formattedRelProps?.join(", "),
+            "The relationships are the following:",
+            formattedRels?.join(", "),
+        ].join("\n");
     }
     async close() {
         await this.driver.close();

package/dist/graphs/neo4j_graph.d.ts

@@ -4,6 +4,34 @@ interface Neo4jGraphConfig {
     password: string;
     database?: string;
 }
+interface StructuredSchema {
+    nodeProps: {
+        [key: NodeType["labels"]]: NodeType["properties"];
+    };
+    relProps: {
+        [key: RelType["type"]]: RelType["properties"];
+    };
+    relationships: PathType[];
+}
+type NodeType = {
+    labels: string;
+    properties: {
+        property: string;
+        type: string;
+    }[];
+};
+type RelType = {
+    type: string;
+    properties: {
+        property: string;
+        type: string;
+    }[];
+};
+type PathType = {
+    start: string;
+    type: string;
+    end: string;
+};
 /**
  * @security *Security note*: Make sure that the database connection uses credentials
  * that are narrowly-scoped to only include necessary permissions.
@@ -22,9 +50,11 @@ export declare class Neo4jGraph {
     private driver;
     private database;
     private schema;
+    private structuredSchema;
     constructor({ url, username, password, database, }: Neo4jGraphConfig);
     static initialize(config: Neo4jGraphConfig): Promise<Neo4jGraph>;
     getSchema(): string;
+    getStructuredSchema(): StructuredSchema;
     query(query: string, params?: any): Promise<any[] | undefined>;
     verifyConnectivity(): Promise<void>;
     refreshSchema(): Promise<void>;

package/dist/graphs/neo4j_graph.js

@@ -33,6 +33,16 @@ export class Neo4jGraph {
             writable: true,
             value: ""
         });
+        Object.defineProperty(this, "structuredSchema", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: {
+                nodeProps: {},
+                relProps: {},
+                relationships: [],
+            }
+        });
         try {
             this.driver = neo4j.driver(url, neo4j.auth.basic(username, password));
             this.database = database;
@@ -70,6 +80,9 @@ export class Neo4jGraph {
     getSchema() {
         return this.schema;
     }
+    getStructuredSchema() {
+        return this.structuredSchema;
+    }
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     async query(query, params = {}) {
         try {
@@ -112,21 +125,43 @@ export class Neo4jGraph {
       YIELD label, other, elementType, type, property
       WHERE type = "RELATIONSHIP" AND elementType = "node"
       UNWIND other AS other_node
-      RETURN "(:" + label + ")-[:" + property + "]->(:" + toString(other_node) + ")" AS output
-    `;
-        const nodeProperties = await this.query(nodePropertiesQuery);
-        const relationshipsProperties = await this.query(relPropertiesQuery);
-        const relationships = await this.query(relQuery);
-        this.schema = `
-      Node properties are the following:
-      ${JSON.stringify(nodeProperties?.map((el) => el.output))}
-
-      Relationship properties are the following:
-      ${JSON.stringify(relationshipsProperties?.map((el) => el.output))}
-
-      The relationships are the following:
-      ${JSON.stringify(relationships?.map((el) => el.output))}
+      RETURN {start: label, type: property, end: toString(other_node)} AS output
     `;
+        // Assuming query method is defined and returns a Promise
+        const nodeProperties = (await this.query(nodePropertiesQuery))?.map((el) => el.output);
+        const relationshipsProperties = (await this.query(relPropertiesQuery))?.map((el) => el.output);
+        const relationships = (await this.query(relQuery))?.map((el) => el.output);
+        // Structured schema similar to Python's dictionary comprehension
+        this.structuredSchema = {
+            nodeProps: Object.fromEntries(nodeProperties?.map((el) => [el.labels, el.properties]) || []),
+            relProps: Object.fromEntries(relationshipsProperties?.map((el) => [el.type, el.properties]) || []),
+            relationships: relationships || [],
+        };
+        // Format node properties
+        const formattedNodeProps = nodeProperties?.map((el) => {
+            const propsStr = el.properties
+                .map((prop) => `${prop.property}: ${prop.type}`)
+                .join(", ");
+            return `${el.labels} {${propsStr}}`;
+        });
+        // Format relationship properties
+        const formattedRelProps = relationshipsProperties?.map((el) => {
+            const propsStr = el.properties
+                .map((prop) => `${prop.property}: ${prop.type}`)
+                .join(", ");
+            return `${el.type} {${propsStr}}`;
+        });
+        // Format relationships
+        const formattedRels = relationships?.map((el) => `(:${el.start})-[:${el.type}]->(:${el.end})`);
+        // Combine all formatted elements into a single string
+        this.schema = [
+            "Node properties are the following:",
+            formattedNodeProps?.join(", "),
+            "Relationship properties are the following:",
+            formattedRelProps?.join(", "),
+            "The relationships are the following:",
+            formattedRels?.join(", "),
+        ].join("\n");
     }
     async close() {
         await this.driver.close();

package/dist/llms/gradient_ai.cjs

@@ -0,0 +1,98 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GradientLLM = void 0;
+const nodejs_sdk_1 = require("@gradientai/nodejs-sdk");
+const base_js_1 = require("./base.cjs");
+const env_js_1 = require("../util/env.cjs");
+/**
+ * The GradientLLM class is used to interact with Gradient AI inference Endpoint models.
+ * This requires your Gradient AI Access Token which is autoloaded if not specified.
+ */
+class GradientLLM extends base_js_1.LLM {
+    static lc_name() {
+        return "GradientLLM";
+    }
+    get lc_secrets() {
+        return {
+            gradientAccessKey: "GRADIENT_ACCESS_TOKEN",
+            workspaceId: "GRADIENT_WORKSPACE_ID",
+        };
+    }
+    constructor(fields) {
+        super(fields);
+        Object.defineProperty(this, "modelSlug", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "llama2-7b-chat"
+        });
+        Object.defineProperty(this, "gradientAccessKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workspaceId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "inferenceParameters", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // Gradient AI does not export the BaseModel type. Once it does, we can use it here.
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        Object.defineProperty(this, "baseModel", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.modelSlug = fields?.modelSlug ?? this.modelSlug;
+        this.gradientAccessKey =
+            fields?.gradientAccessKey ??
+                (0, env_js_1.getEnvironmentVariable)("GRADIENT_ACCESS_TOKEN");
+        this.workspaceId =
+            fields?.workspaceId ?? (0, env_js_1.getEnvironmentVariable)("GRADIENT_WORKSPACE_ID");
+        this.inferenceParameters = fields.inferenceParameters;
+        if (!this.gradientAccessKey) {
+            throw new Error("Missing Gradient AI Access Token");
+        }
+        if (!this.workspaceId) {
+            throw new Error("Missing Gradient AI Workspace ID");
+        }
+    }
+    _llmType() {
+        return "gradient_ai";
+    }
+    /**
+     * Calls the Gradient AI endpoint and retrieves the result.
+     * @param {string} prompt The input prompt.
+     * @returns {Promise<string>} A promise that resolves to the generated string.
+     */
+    /** @ignore */
+    async _call(prompt, _options) {
+        await this.setBaseModel();
+        const response = (await this.caller.call(async () => this.baseModel.complete({
+            query: prompt,
+            ...this.inferenceParameters,
+        })));
+        return response.generatedOutput;
+    }
+    async setBaseModel() {
+        if (this.baseModel)
+            return;
+        const gradient = new nodejs_sdk_1.Gradient({
+            accessToken: this.gradientAccessKey,
+            workspaceId: this.workspaceId,
+        });
+        this.baseModel = await gradient.getBaseModel({
+            baseModelSlug: this.modelSlug,
+        });
+    }
+}
+exports.GradientLLM = GradientLLM;

package/dist/llms/gradient_ai.d.ts

@@ -0,0 +1,50 @@
+import { BaseLLMCallOptions, BaseLLMParams, LLM } from "./base.js";
+/**
+ * The GradientLLMParams interface defines the input parameters for
+ * the GradientLLM class.
+ */
+export interface GradientLLMParams extends BaseLLMParams {
+    /**
+     * Gradient AI Access Token.
+     * Provide Access Token if you do not wish to automatically pull from env.
+     */
+    gradientAccessKey?: string;
+    /**
+     * Gradient Workspace Id.
+     * Provide workspace id if you do not wish to automatically pull from env.
+     */
+    workspaceId?: string;
+    /**
+     * Parameters accepted by the Gradient npm package.
+     */
+    inferenceParameters?: Record<string, unknown>;
+    /**
+     * Gradient AI Model Slug.
+     */
+    modelSlug?: string;
+}
+/**
+ * The GradientLLM class is used to interact with Gradient AI inference Endpoint models.
+ * This requires your Gradient AI Access Token which is autoloaded if not specified.
+ */
+export declare class GradientLLM extends LLM<BaseLLMCallOptions> {
+    static lc_name(): string;
+    get lc_secrets(): {
+        [key: string]: string;
+    } | undefined;
+    modelSlug: string;
+    gradientAccessKey?: string;
+    workspaceId?: string;
+    inferenceParameters?: Record<string, unknown>;
+    baseModel: any;
+    constructor(fields: GradientLLMParams);
+    _llmType(): string;
+    /**
+     * Calls the Gradient AI endpoint and retrieves the result.
+     * @param {string} prompt The input prompt.
+     * @returns {Promise<string>} A promise that resolves to the generated string.
+     */
+    /** @ignore */
+    _call(prompt: string, _options: this["ParsedCallOptions"]): Promise<string>;
+    setBaseModel(): Promise<void>;
+}

package/dist/llms/gradient_ai.js

@@ -0,0 +1,94 @@
+import { Gradient } from "@gradientai/nodejs-sdk";
+import { LLM } from "./base.js";
+import { getEnvironmentVariable } from "../util/env.js";
+/**
+ * The GradientLLM class is used to interact with Gradient AI inference Endpoint models.
+ * This requires your Gradient AI Access Token which is autoloaded if not specified.
+ */
+export class GradientLLM extends LLM {
+    static lc_name() {
+        return "GradientLLM";
+    }
+    get lc_secrets() {
+        return {
+            gradientAccessKey: "GRADIENT_ACCESS_TOKEN",
+            workspaceId: "GRADIENT_WORKSPACE_ID",
+        };
+    }
+    constructor(fields) {
+        super(fields);
+        Object.defineProperty(this, "modelSlug", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "llama2-7b-chat"
+        });
+        Object.defineProperty(this, "gradientAccessKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workspaceId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "inferenceParameters", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // Gradient AI does not export the BaseModel type. Once it does, we can use it here.
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        Object.defineProperty(this, "baseModel", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.modelSlug = fields?.modelSlug ?? this.modelSlug;
+        this.gradientAccessKey =
+            fields?.gradientAccessKey ??
+                getEnvironmentVariable("GRADIENT_ACCESS_TOKEN");
+        this.workspaceId =
+            fields?.workspaceId ?? getEnvironmentVariable("GRADIENT_WORKSPACE_ID");
+        this.inferenceParameters = fields.inferenceParameters;
+        if (!this.gradientAccessKey) {
+            throw new Error("Missing Gradient AI Access Token");
+        }
+        if (!this.workspaceId) {
+            throw new Error("Missing Gradient AI Workspace ID");
+        }
+    }
+    _llmType() {
+        return "gradient_ai";
+    }
+    /**
+     * Calls the Gradient AI endpoint and retrieves the result.
+     * @param {string} prompt The input prompt.
+     * @returns {Promise<string>} A promise that resolves to the generated string.
+     */
+    /** @ignore */
+    async _call(prompt, _options) {
+        await this.setBaseModel();
+        const response = (await this.caller.call(async () => this.baseModel.complete({
+            query: prompt,
+            ...this.inferenceParameters,
+        })));
+        return response.generatedOutput;
+    }
+    async setBaseModel() {
+        if (this.baseModel)
+            return;
+        const gradient = new Gradient({
+            accessToken: this.gradientAccessKey,
+            workspaceId: this.workspaceId,
+        });
+        this.baseModel = await gradient.getBaseModel({
+            baseModelSlug: this.modelSlug,
+        });
+    }
+}

package/dist/llms/hf.cjs

@@ -76,6 +76,12 @@ class HuggingFaceInference extends base_js_1.LLM {
             writable: true,
             value: undefined
         });
+        Object.defineProperty(this, "includeCredentials", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
         this.model = fields?.model ?? this.model;
         this.temperature = fields?.temperature ?? this.temperature;
         this.maxTokens = fields?.maxTokens ?? this.maxTokens;
@@ -85,6 +91,7 @@ class HuggingFaceInference extends base_js_1.LLM {
         this.apiKey =
             fields?.apiKey ?? (0, env_js_1.getEnvironmentVariable)("HUGGINGFACEHUB_API_KEY");
         this.endpointUrl = fields?.endpointUrl;
+        this.includeCredentials = fields?.includeCredentials;
         if (!this.apiKey) {
             throw new Error("Please set an API key for HuggingFace Hub in the environment variable HUGGINGFACEHUB_API_KEY or in the apiKey field of the HuggingFaceInference constructor.");
         }
@@ -96,8 +103,12 @@ class HuggingFaceInference extends base_js_1.LLM {
     async _call(prompt, options) {
         const { HfInference } = await HuggingFaceInference.imports();
         const hf = this.endpointUrl
-            ? new HfInference(this.apiKey).endpoint(this.endpointUrl)
-            : new HfInference(this.apiKey);
+            ? new HfInference(this.apiKey, {
+                includeCredentials: this.includeCredentials,
+            }).endpoint(this.endpointUrl)
+            : new HfInference(this.apiKey, {
+                includeCredentials: this.includeCredentials,
+            });
         const res = await this.caller.callWithOptions({ signal: options.signal }, hf.textGeneration.bind(hf), {
             model: this.model,
             parameters: {

package/dist/llms/hf.d.ts

@@ -22,6 +22,10 @@ export interface HFInput {
     frequencyPenalty?: number;
     /** API key to use. */
     apiKey?: string;
+    /**
+     * Credentials to use for the request. If this is a string, it will be passed straight on. If it's a boolean, true will be "include" and false will not send credentials at all.
+     */
+    includeCredentials?: string | boolean;
 }
 /**
  * Class implementing the Large Language Model (LLM) interface using the
@@ -52,6 +56,7 @@ export declare class HuggingFaceInference extends LLM implements HFInput {
     frequencyPenalty: number | undefined;
     apiKey: string | undefined;
     endpointUrl: string | undefined;
+    includeCredentials: string | boolean | undefined;
     constructor(fields?: Partial<HFInput> & BaseLLMParams);
     _llmType(): string;
     /** @ignore */

package/dist/llms/hf.js

@@ -73,6 +73,12 @@ export class HuggingFaceInference extends LLM {
             writable: true,
             value: undefined
         });
+        Object.defineProperty(this, "includeCredentials", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: undefined
+        });
         this.model = fields?.model ?? this.model;
         this.temperature = fields?.temperature ?? this.temperature;
         this.maxTokens = fields?.maxTokens ?? this.maxTokens;
@@ -82,6 +88,7 @@ export class HuggingFaceInference extends LLM {
         this.apiKey =
             fields?.apiKey ?? getEnvironmentVariable("HUGGINGFACEHUB_API_KEY");
         this.endpointUrl = fields?.endpointUrl;
+        this.includeCredentials = fields?.includeCredentials;
         if (!this.apiKey) {
             throw new Error("Please set an API key for HuggingFace Hub in the environment variable HUGGINGFACEHUB_API_KEY or in the apiKey field of the HuggingFaceInference constructor.");
         }
@@ -93,8 +100,12 @@ export class HuggingFaceInference extends LLM {
     async _call(prompt, options) {
         const { HfInference } = await HuggingFaceInference.imports();
         const hf = this.endpointUrl
-            ? new HfInference(this.apiKey).endpoint(this.endpointUrl)
-            : new HfInference(this.apiKey);
+            ? new HfInference(this.apiKey, {
+                includeCredentials: this.includeCredentials,
+            }).endpoint(this.endpointUrl)
+            : new HfInference(this.apiKey, {
+                includeCredentials: this.includeCredentials,
+            });
         const res = await this.caller.callWithOptions({ signal: options.signal }, hf.textGeneration.bind(hf), {
             model: this.model,
             parameters: {

package/dist/llms/llama_cpp.cjs

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.LlamaCpp = void 0;
 const llama_cpp_js_1 = require("../util/llama_cpp.cjs");
 const base_js_1 = require("./base.cjs");
+const index_js_1 = require("../schema/index.cjs");
 /**
  *  To use this model you need to have the `node-llama-cpp` module installed.
  *  This can be installed using `npm install -S node-llama-cpp` and the minimum
@@ -76,9 +77,7 @@ class LlamaCpp extends base_js_1.LLM {
         return "llama2_cpp";
     }
     /** @ignore */
-    async _call(prompt, 
-    // @ts-expect-error - TS6133: 'options' is declared but its value is never read.
-    options) {
+    async _call(prompt, _options) {
         try {
             const promptOptions = {
                 maxTokens: this?.maxTokens,
@@ -94,5 +93,20 @@ class LlamaCpp extends base_js_1.LLM {
             throw new Error("Error getting prompt completion.");
         }
     }
+    async *_streamResponseChunks(prompt, _options, runManager) {
+        const promptOptions = {
+            temperature: this?.temperature,
+            topK: this?.topK,
+            topP: this?.topP,
+        };
+        const stream = await this.caller.call(async () => this._context.evaluate(this._context.encode(prompt), promptOptions));
+        for await (const chunk of stream) {
+            yield new index_js_1.GenerationChunk({
+                text: this._context.decode([chunk]),
+                generationInfo: {},
+            });
+            await runManager?.handleLLMNewToken(this._context.decode([chunk]) ?? "");
+        }
+    }
 }
 exports.LlamaCpp = LlamaCpp;

package/dist/llms/llama_cpp.d.ts

@@ -1,6 +1,8 @@
 import { LlamaModel, LlamaContext, LlamaChatSession } from "node-llama-cpp";
 import { LlamaBaseCppInputs } from "../util/llama_cpp.js";
 import { LLM, BaseLLMCallOptions, BaseLLMParams } from "./base.js";
+import { CallbackManagerForLLMRun } from "../callbacks/manager.js";
+import { GenerationChunk } from "../schema/index.js";
 /**
  * Note that the modelPath is the only required parameter. For testing you
  * can set this in the environment variable `LLAMA_PATH`.
@@ -34,5 +36,6 @@ export declare class LlamaCpp extends LLM<LlamaCppCallOptions> {
     constructor(inputs: LlamaCppInputs);
     _llmType(): string;
     /** @ignore */
-    _call(prompt: string, options?: this["ParsedCallOptions"]): Promise<string>;
+    _call(prompt: string, _options?: this["ParsedCallOptions"]): Promise<string>;
+    _streamResponseChunks(prompt: string, _options: this["ParsedCallOptions"], runManager?: CallbackManagerForLLMRun): AsyncGenerator<GenerationChunk>;
 }

package/dist/llms/llama_cpp.js

@@ -1,5 +1,6 @@
 import { createLlamaModel, createLlamaContext, createLlamaSession, } from "../util/llama_cpp.js";
 import { LLM } from "./base.js";
+import { GenerationChunk } from "../schema/index.js";
 /**
  *  To use this model you need to have the `node-llama-cpp` module installed.
  *  This can be installed using `npm install -S node-llama-cpp` and the minimum
@@ -73,9 +74,7 @@ export class LlamaCpp extends LLM {
         return "llama2_cpp";
     }
     /** @ignore */
-    async _call(prompt, 
-    // @ts-expect-error - TS6133: 'options' is declared but its value is never read.
-    options) {
+    async _call(prompt, _options) {
         try {
             const promptOptions = {
                 maxTokens: this?.maxTokens,
@@ -91,4 +90,19 @@ export class LlamaCpp extends LLM {
             throw new Error("Error getting prompt completion.");
         }
     }
+    async *_streamResponseChunks(prompt, _options, runManager) {
+        const promptOptions = {
+            temperature: this?.temperature,
+            topK: this?.topK,
+            topP: this?.topP,
+        };
+        const stream = await this.caller.call(async () => this._context.evaluate(this._context.encode(prompt), promptOptions));
+        for await (const chunk of stream) {
+            yield new GenerationChunk({
+                text: this._context.decode([chunk]),
+                generationInfo: {},
+            });
+            await runManager?.handleLLMNewToken(this._context.decode([chunk]) ?? "");
+        }
+    }
 }