langchain 0.0.133 → 0.0.134

package/dist/callbacks/index.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.consumeCallback = exports.awaitAllCallbacks = exports.traceAsGroup = exports.TraceGroup = exports.CallbackManagerForToolRun = exports.CallbackManagerForLLMRun = exports.CallbackManagerForChainRun = exports.CallbackManager = exports.getTracingV2CallbackHandler = exports.getTracingCallbackHandler = exports.LangChainTracerV1 = exports.LangChainTracer = exports.ConsoleCallbackHandler = exports.BaseTracer = exports.BaseCallbackHandler = void 0;
+exports.consumeCallback = exports.awaitAllCallbacks = exports.traceAsGroup = exports.TraceGroup = exports.CallbackManagerForToolRun = exports.CallbackManagerForLLMRun = exports.CallbackManagerForChainRun = exports.CallbackManagerForRetrieverRun = exports.CallbackManager = exports.getTracingV2CallbackHandler = exports.getTracingCallbackHandler = exports.LangChainTracerV1 = exports.LangChainTracer = exports.ConsoleCallbackHandler = exports.BaseTracer = exports.BaseCallbackHandler = void 0;
 var base_js_1 = require("./base.cjs");
 Object.defineProperty(exports, "BaseCallbackHandler", { enumerable: true, get: function () { return base_js_1.BaseCallbackHandler; } });
 var tracer_js_1 = require("./handlers/tracer.cjs");
@@ -16,6 +16,7 @@ Object.defineProperty(exports, "getTracingCallbackHandler", { enumerable: true,
 Object.defineProperty(exports, "getTracingV2CallbackHandler", { enumerable: true, get: function () { return initialize_js_1.getTracingV2CallbackHandler; } });
 var manager_js_1 = require("./manager.cjs");
 Object.defineProperty(exports, "CallbackManager", { enumerable: true, get: function () { return manager_js_1.CallbackManager; } });
+Object.defineProperty(exports, "CallbackManagerForRetrieverRun", { enumerable: true, get: function () { return manager_js_1.CallbackManagerForRetrieverRun; } });
 Object.defineProperty(exports, "CallbackManagerForChainRun", { enumerable: true, get: function () { return manager_js_1.CallbackManagerForChainRun; } });
 Object.defineProperty(exports, "CallbackManagerForLLMRun", { enumerable: true, get: function () { return manager_js_1.CallbackManagerForLLMRun; } });
 Object.defineProperty(exports, "CallbackManagerForToolRun", { enumerable: true, get: function () { return manager_js_1.CallbackManagerForToolRun; } });

package/dist/callbacks/index.d.ts

@@ -4,5 +4,5 @@ export { ConsoleCallbackHandler } from "./handlers/console.js";
 export { LangChainTracer } from "./handlers/tracer_langchain.js";
 export { LangChainTracerV1 } from "./handlers/tracer_langchain_v1.js";
 export { getTracingCallbackHandler, getTracingV2CallbackHandler, } from "./handlers/initialize.js";
-export { CallbackManager, CallbackManagerForChainRun, CallbackManagerForLLMRun, CallbackManagerForToolRun, CallbackManagerOptions, Callbacks, TraceGroup, traceAsGroup, } from "./manager.js";
+export { CallbackManager, CallbackManagerForRetrieverRun, CallbackManagerForChainRun, CallbackManagerForLLMRun, CallbackManagerForToolRun, CallbackManagerOptions, Callbacks, TraceGroup, traceAsGroup, } from "./manager.js";
 export { awaitAllCallbacks, consumeCallback } from "./promises.js";

package/dist/callbacks/index.js

@@ -4,5 +4,5 @@ export { ConsoleCallbackHandler } from "./handlers/console.js";
 export { LangChainTracer } from "./handlers/tracer_langchain.js";
 export { LangChainTracerV1 } from "./handlers/tracer_langchain_v1.js";
 export { getTracingCallbackHandler, getTracingV2CallbackHandler, } from "./handlers/initialize.js";
-export { CallbackManager, CallbackManagerForChainRun, CallbackManagerForLLMRun, CallbackManagerForToolRun, TraceGroup, traceAsGroup, } from "./manager.js";
+export { CallbackManager, CallbackManagerForRetrieverRun, CallbackManagerForChainRun, CallbackManagerForLLMRun, CallbackManagerForToolRun, TraceGroup, traceAsGroup, } from "./manager.js";
 export { awaitAllCallbacks, consumeCallback } from "./promises.js";

package/dist/chains/sql_db/sql_db_chain.d.ts

@@ -29,9 +29,9 @@ export declare class SqlDatabaseChain extends BaseChain {
     database: SqlDatabase;
     prompt: PromptTemplate<{
         input: any;
+        top_k: any;
         dialect: any;
         table_info: any;
-        top_k: any;
     }, any>;
     topK: number;
     inputKey: string;

package/dist/chains/sql_db/sql_db_prompt.d.ts

@@ -1,37 +1,37 @@
 import { PromptTemplate } from "../../prompts/prompt.js";
 export declare const DEFAULT_SQL_DATABASE_PROMPT: PromptTemplate<{
     input: any;
+    top_k: any;
     dialect: any;
     table_info: any;
-    top_k: any;
 }, any>;
 export declare const SQL_POSTGRES_PROMPT: PromptTemplate<{
     input: any;
+    top_k: any;
     dialect: any;
     table_info: any;
-    top_k: any;
 }, any>;
 export declare const SQL_SQLITE_PROMPT: PromptTemplate<{
     input: any;
+    top_k: any;
     dialect: any;
     table_info: any;
-    top_k: any;
 }, any>;
 export declare const SQL_MYSQL_PROMPT: PromptTemplate<{
     input: any;
+    top_k: any;
     dialect: any;
     table_info: any;
-    top_k: any;
 }, any>;
 export declare const SQL_MSSQL_PROMPT: PromptTemplate<{
     input: any;
+    top_k: any;
     dialect: any;
     table_info: any;
-    top_k: any;
 }, any>;
 export declare const SQL_SAP_HANA_PROMPT: PromptTemplate<{
     input: any;
+    top_k: any;
     dialect: any;
     table_info: any;
-    top_k: any;
 }, any>;

package/dist/document_loaders/web/recursive_url.cjs

@@ -0,0 +1,177 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecursiveUrlLoader = void 0;
+const jsdom_1 = require("jsdom");
+const async_caller_js_1 = require("../../util/async_caller.cjs");
+const base_js_1 = require("../base.cjs");
+class RecursiveUrlLoader extends base_js_1.BaseDocumentLoader {
+    constructor(url, options) {
+        super();
+        Object.defineProperty(this, "caller", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "url", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "excludeDirs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "extractor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxDepth", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "timeout", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "preventOutside", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.caller = new async_caller_js_1.AsyncCaller({
+            maxConcurrency: 64,
+            maxRetries: 0,
+            ...options.callerOptions,
+        });
+        this.url = url;
+        this.excludeDirs = options.excludeDirs ?? [];
+        this.extractor = options.extractor ?? ((s) => s);
+        this.maxDepth = options.maxDepth ?? 2;
+        this.timeout = options.timeout ?? 10000;
+        this.preventOutside = options.preventOutside ?? true;
+    }
+    async fetchWithTimeout(resource, options) {
+        const { timeout, ...rest } = options;
+        return this.caller.call(() => fetch(resource, { ...rest, signal: AbortSignal.timeout(timeout) }));
+    }
+    getChildLinks(html, baseUrl) {
+        const allLinks = Array.from(new jsdom_1.JSDOM(html).window.document.querySelectorAll("a")).map((a) => a.href);
+        const absolutePaths = [];
+        // eslint-disable-next-line no-script-url
+        const invalidPrefixes = ["javascript:", "mailto:", "#"];
+        const invalidSuffixes = [
+            ".css",
+            ".js",
+            ".ico",
+            ".png",
+            ".jpg",
+            ".jpeg",
+            ".gif",
+            ".svg",
+        ];
+        for (const link of allLinks) {
+            if (invalidPrefixes.some((prefix) => link.startsWith(prefix)) ||
+                invalidSuffixes.some((suffix) => link.endsWith(suffix)))
+                continue;
+            if (link.startsWith("http")) {
+                const isAllowed = !this.preventOutside || link.startsWith(baseUrl);
+                if (isAllowed)
+                    absolutePaths.push(link);
+            }
+            else if (link.startsWith("//")) {
+                const base = new URL(baseUrl);
+                absolutePaths.push(base.protocol + link);
+            }
+            else {
+                const newLink = new URL(link, baseUrl).href;
+                absolutePaths.push(newLink);
+            }
+        }
+        return Array.from(new Set(absolutePaths));
+    }
+    extractMetadata(rawHtml, url) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const metadata = { source: url };
+        const { document } = new jsdom_1.JSDOM(rawHtml).window;
+        const title = document.getElementsByTagName("title")[0];
+        if (title) {
+            metadata.title = title.textContent;
+        }
+        const description = document.querySelector("meta[name=description]");
+        if (description) {
+            metadata.description = description.getAttribute("content");
+        }
+        const html = document.getElementsByTagName("html")[0];
+        if (html) {
+            metadata.language = html.getAttribute("lang");
+        }
+        return metadata;
+    }
+    async getUrlAsDoc(url) {
+        let res;
+        try {
+            res = await this.fetchWithTimeout(url, { timeout: this.timeout });
+            res = await res.text();
+        }
+        catch (e) {
+            return null;
+        }
+        return {
+            pageContent: this.extractor(res),
+            metadata: this.extractMetadata(res, url),
+        };
+    }
+    async getChildUrlsRecursive(inputUrl, visited = new Set(), depth = 0) {
+        if (depth > this.maxDepth)
+            return [];
+        let url = inputUrl;
+        if (!inputUrl.endsWith("/"))
+            url += "/";
+        const isExcluded = this.excludeDirs.some((exDir) => url.startsWith(exDir));
+        if (isExcluded)
+            return [];
+        let res;
+        try {
+            res = await this.fetchWithTimeout(url, { timeout: this.timeout });
+            res = await res.text();
+        }
+        catch (e) {
+            return [];
+        }
+        const childUrls = this.getChildLinks(res, url);
+        const results = await Promise.all(childUrls.map((childUrl) => (async () => {
+            if (visited.has(childUrl))
+                return null;
+            visited.add(childUrl);
+            const childDoc = await this.getUrlAsDoc(childUrl);
+            if (!childDoc)
+                return null;
+            if (childUrl.endsWith("/")) {
+                const childUrlResponses = await this.getChildUrlsRecursive(childUrl, visited, depth + 1);
+                return [childDoc, ...childUrlResponses];
+            }
+            return [childDoc];
+        })()));
+        return results.flat().filter((docs) => docs !== null);
+    }
+    async load() {
+        const rootDoc = await this.getUrlAsDoc(this.url);
+        if (!rootDoc)
+            return [];
+        const docs = [rootDoc];
+        docs.push(...(await this.getChildUrlsRecursive(this.url, new Set([this.url]))));
+        return docs;
+    }
+}
+exports.RecursiveUrlLoader = RecursiveUrlLoader;

package/dist/document_loaders/web/recursive_url.d.ts

@@ -0,0 +1,27 @@
+import { Document } from "../../document.js";
+import { AsyncCaller } from "../../util/async_caller.js";
+import { BaseDocumentLoader, DocumentLoader } from "../base.js";
+export interface RecursiveUrlLoaderOptions {
+    excludeDirs?: string[];
+    extractor?: (text: string) => string;
+    maxDepth?: number;
+    timeout?: number;
+    preventOutside?: boolean;
+    callerOptions?: ConstructorParameters<typeof AsyncCaller>[0];
+}
+export declare class RecursiveUrlLoader extends BaseDocumentLoader implements DocumentLoader {
+    private caller;
+    private url;
+    private excludeDirs;
+    private extractor;
+    private maxDepth;
+    private timeout;
+    private preventOutside;
+    constructor(url: string, options: RecursiveUrlLoaderOptions);
+    private fetchWithTimeout;
+    private getChildLinks;
+    private extractMetadata;
+    private getUrlAsDoc;
+    private getChildUrlsRecursive;
+    load(): Promise<Document[]>;
+}

package/dist/document_loaders/web/recursive_url.js

@@ -0,0 +1,173 @@
+import { JSDOM } from "jsdom";
+import { AsyncCaller } from "../../util/async_caller.js";
+import { BaseDocumentLoader } from "../base.js";
+export class RecursiveUrlLoader extends BaseDocumentLoader {
+    constructor(url, options) {
+        super();
+        Object.defineProperty(this, "caller", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "url", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "excludeDirs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "extractor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxDepth", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "timeout", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "preventOutside", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.caller = new AsyncCaller({
+            maxConcurrency: 64,
+            maxRetries: 0,
+            ...options.callerOptions,
+        });
+        this.url = url;
+        this.excludeDirs = options.excludeDirs ?? [];
+        this.extractor = options.extractor ?? ((s) => s);
+        this.maxDepth = options.maxDepth ?? 2;
+        this.timeout = options.timeout ?? 10000;
+        this.preventOutside = options.preventOutside ?? true;
+    }
+    async fetchWithTimeout(resource, options) {
+        const { timeout, ...rest } = options;
+        return this.caller.call(() => fetch(resource, { ...rest, signal: AbortSignal.timeout(timeout) }));
+    }
+    getChildLinks(html, baseUrl) {
+        const allLinks = Array.from(new JSDOM(html).window.document.querySelectorAll("a")).map((a) => a.href);
+        const absolutePaths = [];
+        // eslint-disable-next-line no-script-url
+        const invalidPrefixes = ["javascript:", "mailto:", "#"];
+        const invalidSuffixes = [
+            ".css",
+            ".js",
+            ".ico",
+            ".png",
+            ".jpg",
+            ".jpeg",
+            ".gif",
+            ".svg",
+        ];
+        for (const link of allLinks) {
+            if (invalidPrefixes.some((prefix) => link.startsWith(prefix)) ||
+                invalidSuffixes.some((suffix) => link.endsWith(suffix)))
+                continue;
+            if (link.startsWith("http")) {
+                const isAllowed = !this.preventOutside || link.startsWith(baseUrl);
+                if (isAllowed)
+                    absolutePaths.push(link);
+            }
+            else if (link.startsWith("//")) {
+                const base = new URL(baseUrl);
+                absolutePaths.push(base.protocol + link);
+            }
+            else {
+                const newLink = new URL(link, baseUrl).href;
+                absolutePaths.push(newLink);
+            }
+        }
+        return Array.from(new Set(absolutePaths));
+    }
+    extractMetadata(rawHtml, url) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const metadata = { source: url };
+        const { document } = new JSDOM(rawHtml).window;
+        const title = document.getElementsByTagName("title")[0];
+        if (title) {
+            metadata.title = title.textContent;
+        }
+        const description = document.querySelector("meta[name=description]");
+        if (description) {
+            metadata.description = description.getAttribute("content");
+        }
+        const html = document.getElementsByTagName("html")[0];
+        if (html) {
+            metadata.language = html.getAttribute("lang");
+        }
+        return metadata;
+    }
+    async getUrlAsDoc(url) {
+        let res;
+        try {
+            res = await this.fetchWithTimeout(url, { timeout: this.timeout });
+            res = await res.text();
+        }
+        catch (e) {
+            return null;
+        }
+        return {
+            pageContent: this.extractor(res),
+            metadata: this.extractMetadata(res, url),
+        };
+    }
+    async getChildUrlsRecursive(inputUrl, visited = new Set(), depth = 0) {
+        if (depth > this.maxDepth)
+            return [];
+        let url = inputUrl;
+        if (!inputUrl.endsWith("/"))
+            url += "/";
+        const isExcluded = this.excludeDirs.some((exDir) => url.startsWith(exDir));
+        if (isExcluded)
+            return [];
+        let res;
+        try {
+            res = await this.fetchWithTimeout(url, { timeout: this.timeout });
+            res = await res.text();
+        }
+        catch (e) {
+            return [];
+        }
+        const childUrls = this.getChildLinks(res, url);
+        const results = await Promise.all(childUrls.map((childUrl) => (async () => {
+            if (visited.has(childUrl))
+                return null;
+            visited.add(childUrl);
+            const childDoc = await this.getUrlAsDoc(childUrl);
+            if (!childDoc)
+                return null;
+            if (childUrl.endsWith("/")) {
+                const childUrlResponses = await this.getChildUrlsRecursive(childUrl, visited, depth + 1);
+                return [childDoc, ...childUrlResponses];
+            }
+            return [childDoc];
+        })()));
+        return results.flat().filter((docs) => docs !== null);
+    }
+    async load() {
+        const rootDoc = await this.getUrlAsDoc(this.url);
+        if (!rootDoc)
+            return [];
+        const docs = [rootDoc];
+        docs.push(...(await this.getChildUrlsRecursive(this.url, new Set([this.url]))));
+        return docs;
+    }
+}

package/dist/hub.cjs

@@ -0,0 +1,16 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.pull = exports.push = void 0;
+const langchainhub_1 = require("langchainhub");
+const index_js_1 = require("./load/index.cjs");
+async function push(repoFullName, runnable, options) {
+    const client = new langchainhub_1.Client(options);
+    return client.push(repoFullName, JSON.stringify(runnable), options);
+}
+exports.push = push;
+async function pull(ownerRepoCommit, options) {
+    const client = new langchainhub_1.Client(options);
+    const result = await client.pull(ownerRepoCommit);
+    return (0, index_js_1.load)(result);
+}
+exports.pull = pull;

package/dist/hub.d.ts

@@ -0,0 +1,4 @@
+import { ClientConfiguration, HubPushOptions } from "langchainhub";
+import { Runnable } from "./schema/runnable.js";
+export declare function push(repoFullName: string, runnable: Runnable, options?: HubPushOptions & ClientConfiguration): Promise<any>;
+export declare function pull<T extends Runnable>(ownerRepoCommit: string, options?: ClientConfiguration): Promise<T>;

package/dist/hub.js

@@ -0,0 +1,11 @@
+import { Client } from "langchainhub";
+import { load } from "./load/index.js";
+export async function push(repoFullName, runnable, options) {
+    const client = new Client(options);
+    return client.push(repoFullName, JSON.stringify(runnable), options);
+}
+export async function pull(ownerRepoCommit, options) {
+    const client = new Client(options);
+    const result = await client.pull(ownerRepoCommit);
+    return load(result);
+}

package/dist/llms/bedrock.cjs

@@ -4,9 +4,12 @@ exports.Bedrock = void 0;
 const signature_v4_1 = require("@aws-sdk/signature-v4");
 const credential_provider_node_1 = require("@aws-sdk/credential-provider-node");
 const protocol_http_1 = require("@aws-sdk/protocol-http");
+const eventstream_codec_1 = require("@smithy/eventstream-codec");
+const util_utf8_1 = require("@smithy/util-utf8");
 const sha256_js_1 = require("@aws-crypto/sha256-js");
 const env_js_1 = require("../util/env.cjs");
 const base_js_1 = require("./base.cjs");
+const index_js_1 = require("../schema/index.cjs");
 /**
  * A helper class used within the `Bedrock` class. It is responsible for
  * preparing the input and output for the Bedrock service. It formats the
@@ -17,20 +20,24 @@ class BedrockLLMInputOutputAdapter {
     /** Adapter class to prepare the inputs from Langchain to a format
     that LLM model expects. Also, provides a helper function to extract
     the generated text from the model response. */
-    static prepareInput(provider, prompt) {
+    static prepareInput(provider, prompt, maxTokens = 50, temperature = 0) {
         const inputBody = {};
-        if (provider === "anthropic" || provider === "ai21") {
+        if (provider === "anthropic") {
             inputBody.prompt = prompt;
+            inputBody.max_tokens_to_sample = maxTokens;
+            inputBody.temperature = temperature;
         }
-        else if (provider === "amazon") {
-            inputBody.inputText = prompt;
-            inputBody.textGenerationConfig = {};
+        else if (provider === "ai21") {
+            inputBody.prompt = prompt;
+            inputBody.maxTokens = maxTokens;
+            inputBody.temperature = temperature;
         }
-        else {
+        else if (provider === "amazon") {
             inputBody.inputText = prompt;
-        }
-        if (provider === "anthropic" && !("max_tokens_to_sample" in inputBody)) {
-            inputBody.max_tokens_to_sample = 50;
+            inputBody.textGenerationConfig = {
+                maxTokenCount: maxTokens,
+                temperature,
+            };
         }
         return inputBody;
     }
@@ -46,9 +53,9 @@ class BedrockLLMInputOutputAdapter {
             return responseBody.completion;
         }
         else if (provider === "ai21") {
-            return responseBody.completions[0].data.text;
+            return responseBody.data.text;
         }
-        return responseBody.results[0].outputText;
+        return responseBody.outputText;
     }
 }
 /**
@@ -105,6 +112,12 @@ class Bedrock extends base_js_1.LLM {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "codec", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new eventstream_codec_1.EventStreamCodec(util_utf8_1.toUtf8, util_utf8_1.fromUtf8)
+        });
         this.model = fields?.model ?? this.model;
         const allowedModels = ["ai21", "anthropic", "amazon"];
         if (!allowedModels.includes(this.model.split(".")[0])) {
@@ -130,11 +143,18 @@ class Bedrock extends base_js_1.LLM {
       Example:
         response = model.call("Tell me a joke.")
     */
-    async _call(prompt) {
+    async _call(prompt, options, runManager) {
+        const chunks = [];
+        for await (const chunk of this._streamResponseChunks(prompt, options, runManager)) {
+            chunks.push(chunk);
+        }
+        return chunks.map((chunk) => chunk.text).join("");
+    }
+    async *_streamResponseChunks(prompt, options, runManager) {
         const provider = this.model.split(".")[0];
         const service = "bedrock";
-        const inputBody = BedrockLLMInputOutputAdapter.prepareInput(provider, prompt);
-        const url = new URL(`https://${service}.${this.region}.amazonaws.com/model/${this.model}/invoke`);
+        const inputBody = BedrockLLMInputOutputAdapter.prepareInput(provider, prompt, this.maxTokens, this.temperature);
+        const url = new URL(`https://${service}.${this.region}.amazonaws.com/model/${this.model}/invoke-with-response-stream`);
         const request = new protocol_http_1.HttpRequest({
             hostname: url.hostname,
             path: url.pathname,
@@ -157,17 +177,41 @@ class Bedrock extends base_js_1.LLM {
         });
         const signedRequest = await signer.sign(request);
         // Send request to AWS using the low-level fetch API
-        const response = await this.fetchFn(url, {
+        const response = await this.caller.callWithOptions({ signal: options.signal }, async () => this.fetchFn(url, {
             headers: signedRequest.headers,
             body: signedRequest.body,
             method: signedRequest.method,
-        });
+        }));
         if (response.status < 200 || response.status >= 300) {
             throw Error(`Failed to access underlying url '${url}': got ${response.status} ${response.statusText}: ${await response.text()}`);
         }
-        const responseJson = await response.json();
-        const text = BedrockLLMInputOutputAdapter.prepareOutput(provider, responseJson);
-        return text;
+        const reader = response.body?.getReader();
+        for await (const chunk of this._readChunks(reader)) {
+            const event = this.codec.decode(chunk);
+            if (event.headers[":event-type"].value !== "chunk" ||
+                event.headers[":content-type"].value !== "application/json") {
+                throw Error(`Failed to get event chunk: got ${chunk}`);
+            }
+            const body = JSON.parse(Buffer.from(JSON.parse(new TextDecoder("utf-8").decode(event.body)).bytes, "base64").toString());
+            const text = BedrockLLMInputOutputAdapter.prepareOutput(provider, body);
+            yield new index_js_1.GenerationChunk({
+                text,
+                generationInfo: {},
+            });
+            await runManager?.handleLLMNewToken(text);
+        }
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    _readChunks(reader) {
+        return {
+            async *[Symbol.asyncIterator]() {
+                let readResult = await reader.read();
+                while (!readResult.done) {
+                    yield readResult.value;
+                    readResult = await reader.read();
+                }
+            },
+        };
     }
 }
 exports.Bedrock = Bedrock;

package/dist/llms/bedrock.d.ts

@@ -1,5 +1,8 @@
+import { EventStreamCodec } from "@smithy/eventstream-codec";
 import type { AwsCredentialIdentity, Provider } from "@aws-sdk/types";
 import { LLM, BaseLLMParams } from "./base.js";
+import { CallbackManagerForLLMRun } from "../callbacks/manager.js";
+import { GenerationChunk } from "../schema/index.js";
 type CredentialType = AwsCredentialIdentity | Provider<AwsCredentialIdentity>;
 /** Bedrock models.
     To authenticate, the AWS client uses the following methods to automatically load credentials:
@@ -43,6 +46,7 @@ export declare class Bedrock extends LLM implements BedrockInput {
     temperature?: number | undefined;
     maxTokens?: number | undefined;
     fetchFn: typeof fetch;
+    codec: EventStreamCodec;
     get lc_secrets(): {
         [key: string]: string;
     } | undefined;
@@ -58,6 +62,10 @@ export declare class Bedrock extends LLM implements BedrockInput {
       Example:
         response = model.call("Tell me a joke.")
     */
-    _call(prompt: string): Promise<string>;
+    _call(prompt: string, options: this["ParsedCallOptions"], runManager?: CallbackManagerForLLMRun): Promise<string>;
+    _streamResponseChunks(prompt: string, options: this["ParsedCallOptions"], runManager?: CallbackManagerForLLMRun): AsyncGenerator<GenerationChunk>;
+    _readChunks(reader: any): {
+        [Symbol.asyncIterator](): AsyncGenerator<any, void, unknown>;
+    };
 }
 export {};

package/dist/llms/bedrock.js

@@ -1,9 +1,12 @@
 import { SignatureV4 } from "@aws-sdk/signature-v4";
 import { defaultProvider } from "@aws-sdk/credential-provider-node";
 import { HttpRequest } from "@aws-sdk/protocol-http";
+import { EventStreamCodec } from "@smithy/eventstream-codec";
+import { fromUtf8, toUtf8 } from "@smithy/util-utf8";
 import { Sha256 } from "@aws-crypto/sha256-js";
 import { getEnvironmentVariable } from "../util/env.js";
 import { LLM } from "./base.js";
+import { GenerationChunk } from "../schema/index.js";
 /**
  * A helper class used within the `Bedrock` class. It is responsible for
  * preparing the input and output for the Bedrock service. It formats the
@@ -14,20 +17,24 @@ class BedrockLLMInputOutputAdapter {
     /** Adapter class to prepare the inputs from Langchain to a format
     that LLM model expects. Also, provides a helper function to extract
     the generated text from the model response. */
-    static prepareInput(provider, prompt) {
+    static prepareInput(provider, prompt, maxTokens = 50, temperature = 0) {
         const inputBody = {};
-        if (provider === "anthropic" || provider === "ai21") {
+        if (provider === "anthropic") {
             inputBody.prompt = prompt;
+            inputBody.max_tokens_to_sample = maxTokens;
+            inputBody.temperature = temperature;
         }
-        else if (provider === "amazon") {
-            inputBody.inputText = prompt;
-            inputBody.textGenerationConfig = {};
+        else if (provider === "ai21") {
+            inputBody.prompt = prompt;
+            inputBody.maxTokens = maxTokens;
+            inputBody.temperature = temperature;
         }
-        else {
+        else if (provider === "amazon") {
             inputBody.inputText = prompt;
-        }
-        if (provider === "anthropic" && !("max_tokens_to_sample" in inputBody)) {
-            inputBody.max_tokens_to_sample = 50;
+            inputBody.textGenerationConfig = {
+                maxTokenCount: maxTokens,
+                temperature,
+            };
         }
         return inputBody;
     }
@@ -43,9 +50,9 @@ class BedrockLLMInputOutputAdapter {
             return responseBody.completion;
         }
         else if (provider === "ai21") {
-            return responseBody.completions[0].data.text;
+            return responseBody.data.text;
         }
-        return responseBody.results[0].outputText;
+        return responseBody.outputText;
     }
 }
 /**
@@ -102,6 +109,12 @@ export class Bedrock extends LLM {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "codec", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: new EventStreamCodec(toUtf8, fromUtf8)
+        });
         this.model = fields?.model ?? this.model;
         const allowedModels = ["ai21", "anthropic", "amazon"];
         if (!allowedModels.includes(this.model.split(".")[0])) {
@@ -127,11 +140,18 @@ export class Bedrock extends LLM {
       Example:
         response = model.call("Tell me a joke.")
     */
-    async _call(prompt) {
+    async _call(prompt, options, runManager) {
+        const chunks = [];
+        for await (const chunk of this._streamResponseChunks(prompt, options, runManager)) {
+            chunks.push(chunk);
+        }
+        return chunks.map((chunk) => chunk.text).join("");
+    }
+    async *_streamResponseChunks(prompt, options, runManager) {
         const provider = this.model.split(".")[0];
         const service = "bedrock";
-        const inputBody = BedrockLLMInputOutputAdapter.prepareInput(provider, prompt);
-        const url = new URL(`https://${service}.${this.region}.amazonaws.com/model/${this.model}/invoke`);
+        const inputBody = BedrockLLMInputOutputAdapter.prepareInput(provider, prompt, this.maxTokens, this.temperature);
+        const url = new URL(`https://${service}.${this.region}.amazonaws.com/model/${this.model}/invoke-with-response-stream`);
         const request = new HttpRequest({
             hostname: url.hostname,
             path: url.pathname,
@@ -154,16 +174,40 @@ export class Bedrock extends LLM {
         });
         const signedRequest = await signer.sign(request);
         // Send request to AWS using the low-level fetch API
-        const response = await this.fetchFn(url, {
+        const response = await this.caller.callWithOptions({ signal: options.signal }, async () => this.fetchFn(url, {
             headers: signedRequest.headers,
             body: signedRequest.body,
             method: signedRequest.method,
-        });
+        }));
         if (response.status < 200 || response.status >= 300) {
             throw Error(`Failed to access underlying url '${url}': got ${response.status} ${response.statusText}: ${await response.text()}`);
         }
-        const responseJson = await response.json();
-        const text = BedrockLLMInputOutputAdapter.prepareOutput(provider, responseJson);
-        return text;
+        const reader = response.body?.getReader();
+        for await (const chunk of this._readChunks(reader)) {
+            const event = this.codec.decode(chunk);
+            if (event.headers[":event-type"].value !== "chunk" ||
+                event.headers[":content-type"].value !== "application/json") {
+                throw Error(`Failed to get event chunk: got ${chunk}`);
+            }
+            const body = JSON.parse(Buffer.from(JSON.parse(new TextDecoder("utf-8").decode(event.body)).bytes, "base64").toString());
+            const text = BedrockLLMInputOutputAdapter.prepareOutput(provider, body);
+            yield new GenerationChunk({
+                text,
+                generationInfo: {},
+            });
+            await runManager?.handleLLMNewToken(text);
+        }
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    _readChunks(reader) {
+        return {
+            async *[Symbol.asyncIterator]() {
+                let readResult = await reader.read();
+                while (!readResult.done) {
+                    yield readResult.value;
+                    readResult = await reader.read();
+                }
+            },
+        };
     }
 }

package/dist/load/import_constants.cjs

@@ -69,6 +69,7 @@ exports.optionalImportEntrypoints = [
     "langchain/document_loaders/web/github",
     "langchain/document_loaders/web/notiondb",
     "langchain/document_loaders/web/notionapi",
+    "langchain/document_loaders/web/recursive_url",
     "langchain/document_loaders/web/s3",
     "langchain/document_loaders/web/sonix_audio",
     "langchain/document_loaders/web/confluence",
@@ -115,5 +116,6 @@ exports.optionalImportEntrypoints = [
     "langchain/stores/message/planetscale",
     "langchain/stores/message/xata",
     "langchain/storage/ioredis",
+    "langchain/hub",
     "langchain/experimental/multimodal_embeddings/googlevertexai",
 ];

package/dist/load/import_constants.js

@@ -66,6 +66,7 @@ export const optionalImportEntrypoints = [
     "langchain/document_loaders/web/github",
     "langchain/document_loaders/web/notiondb",
     "langchain/document_loaders/web/notionapi",
+    "langchain/document_loaders/web/recursive_url",
     "langchain/document_loaders/web/s3",
     "langchain/document_loaders/web/sonix_audio",
     "langchain/document_loaders/web/confluence",
@@ -112,5 +113,6 @@ export const optionalImportEntrypoints = [
     "langchain/stores/message/planetscale",
     "langchain/stores/message/xata",
     "langchain/storage/ioredis",
+    "langchain/hub",
     "langchain/experimental/multimodal_embeddings/googlevertexai",
 ];

package/dist/tools/sql.cjs

@@ -126,14 +126,20 @@ class ListTablesSqlTool extends base_js_1.Tool {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: `Input is an empty string, output is a comma separated list of tables in the database.`
+            value: `Input is an empty string, output is a comma-separated list of tables in the database.`
         });
         this.db = db;
     }
-    /** @ignore */
     async _call(_) {
         try {
-            const tables = this.db.allTables.map((table) => table.tableName);
+            let selectedTables = this.db.allTables;
+            if (this.db.includesTables.length > 0) {
+                selectedTables = selectedTables.filter((currentTable) => this.db.includesTables.includes(currentTable.tableName));
+            }
+            if (this.db.ignoreTables.length > 0) {
+                selectedTables = selectedTables.filter((currentTable) => !this.db.ignoreTables.includes(currentTable.tableName));
+            }
+            const tables = selectedTables.map((table) => table.tableName);
             return tables.join(", ");
         }
         catch (error) {

package/dist/tools/sql.d.ts

@@ -51,7 +51,6 @@ export declare class ListTablesSqlTool extends Tool implements SqlTool {
     name: string;
     db: SqlDatabase;
     constructor(db: SqlDatabase);
-    /** @ignore */
     _call(_: string): Promise<string>;
     description: string;
 }

package/dist/tools/sql.js

@@ -121,14 +121,20 @@ export class ListTablesSqlTool extends Tool {
             enumerable: true,
             configurable: true,
             writable: true,
-            value: `Input is an empty string, output is a comma separated list of tables in the database.`
+            value: `Input is an empty string, output is a comma-separated list of tables in the database.`
         });
         this.db = db;
     }
-    /** @ignore */
     async _call(_) {
         try {
-            const tables = this.db.allTables.map((table) => table.tableName);
+            let selectedTables = this.db.allTables;
+            if (this.db.includesTables.length > 0) {
+                selectedTables = selectedTables.filter((currentTable) => this.db.includesTables.includes(currentTable.tableName));
+            }
+            if (this.db.ignoreTables.length > 0) {
+                selectedTables = selectedTables.filter((currentTable) => !this.db.ignoreTables.includes(currentTable.tableName));
+            }
+            const tables = selectedTables.map((table) => table.tableName);
             return tables.join(", ");
         }
         catch (error) {

package/dist/vectorstores/googlevertexai.cjs

@@ -373,8 +373,9 @@ class MatchingEngine extends base_js_1.VectorStore {
      * @param documentMetadata - The metadata from a document
      * @returns a Restriction[] (or an array of a subclass, from the FilterType)
      */
+    metadataToRestrictions(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    metadataToRestrictions(documentMetadata) {
+    documentMetadata) {
         const metadata = this.cleanMetadata(documentMetadata);
         const restrictions = [];
         for (const key of Object.keys(metadata)) {

package/dist/vectorstores/googlevertexai.js

@@ -343,8 +343,9 @@ export class MatchingEngine extends VectorStore {
      * @param documentMetadata - The metadata from a document
      * @returns a Restriction[] (or an array of a subclass, from the FilterType)
      */
+    metadataToRestrictions(
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    metadataToRestrictions(documentMetadata) {
+    documentMetadata) {
         const metadata = this.cleanMetadata(documentMetadata);
         const restrictions = [];
         for (const key of Object.keys(metadata)) {

package/dist/vectorstores/myscale.cjs

@@ -98,7 +98,7 @@ class MyScaleStore extends base_js_1.VectorStore {
         };
         this.database = args.database || "default";
         this.table = args.table || "vector_table";
-        this.metric = args.metric || "cosine";
+        this.metric = args.metric || "Cosine";
         this.client = (0, client_1.createClient)({
             host: `${args.protocol ?? "https://"}${args.host}:${args.port}`,
             username: args.username,
@@ -263,7 +263,7 @@ class MyScaleStore extends base_js_1.VectorStore {
      * @returns The SQL query string.
      */
     buildSearchQuery(query, k, filter) {
-        const order = this.metric === "ip" ? "DESC" : "ASC";
+        const order = this.metric === "IP" ? "DESC" : "ASC";
         const whereStr = filter ? `PREWHERE ${filter.whereStr}` : "";
         return `
       SELECT ${this.columnMap.text} AS text, ${this.columnMap.metadata} AS metadata, dist

package/dist/vectorstores/myscale.d.ts

@@ -31,7 +31,7 @@ export interface ColumnMap {
 /**
  * Type of metric used in the MyScale database.
  */
-export type metric = "ip" | "cosine" | "l2";
+export type metric = "L2" | "Cosine" | "IP";
 /**
  * Type for filtering search results in the MyScale database.
  */

package/dist/vectorstores/myscale.js

@@ -72,7 +72,7 @@ export class MyScaleStore extends VectorStore {
         };
         this.database = args.database || "default";
         this.table = args.table || "vector_table";
-        this.metric = args.metric || "cosine";
+        this.metric = args.metric || "Cosine";
         this.client = createClient({
             host: `${args.protocol ?? "https://"}${args.host}:${args.port}`,
             username: args.username,
@@ -237,7 +237,7 @@ export class MyScaleStore extends VectorStore {
      * @returns The SQL query string.
      */
     buildSearchQuery(query, k, filter) {
-        const order = this.metric === "ip" ? "DESC" : "ASC";
+        const order = this.metric === "IP" ? "DESC" : "ASC";
         const whereStr = filter ? `PREWHERE ${filter.whereStr}` : "";
         return `
       SELECT ${this.columnMap.text} AS text, ${this.columnMap.metadata} AS metadata, dist

package/document_loaders/web/recursive_url.cjs

@@ -0,0 +1 @@
+module.exports = require('../../dist/document_loaders/web/recursive_url.cjs');

package/document_loaders/web/recursive_url.d.ts

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/web/recursive_url.js'

package/document_loaders/web/recursive_url.js

@@ -0,0 +1 @@
+export * from '../../dist/document_loaders/web/recursive_url.js'

package/hub.cjs

@@ -0,0 +1 @@
+module.exports = require('./dist/hub.cjs');

package/hub.d.ts

@@ -0,0 +1 @@
+export * from './dist/hub.js'

package/hub.js

@@ -0,0 +1 @@
+export * from './dist/hub.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "langchain",
-  "version": "0.0.133",
+  "version": "0.0.134",
   "description": "Typescript bindings for langchain",
   "type": "module",
   "engines": {
@@ -301,6 +301,9 @@
     "document_loaders/web/notionapi.cjs",
     "document_loaders/web/notionapi.js",
     "document_loaders/web/notionapi.d.ts",
+    "document_loaders/web/recursive_url.cjs",
+    "document_loaders/web/recursive_url.js",
+    "document_loaders/web/recursive_url.d.ts",
     "document_loaders/web/s3.cjs",
     "document_loaders/web/s3.js",
     "document_loaders/web/s3.d.ts",
@@ -535,6 +538,9 @@
     "storage/ioredis.cjs",
     "storage/ioredis.js",
     "storage/ioredis.d.ts",
+    "hub.cjs",
+    "hub.js",
+    "hub.d.ts",
     "util/math.cjs",
     "util/math.js",
     "util/math.d.ts",
@@ -614,6 +620,8 @@
     "@planetscale/database": "^1.8.0",
     "@qdrant/js-client-rest": "^1.2.0",
     "@raycast/api": "^1.55.2",
+    "@smithy/eventstream-codec": "^2.0.5",
+    "@smithy/util-utf8": "^2.0.0",
     "@supabase/postgrest-js": "^1.1.1",
     "@supabase/supabase-js": "^2.10.0",
     "@tensorflow-models/universal-sentence-encoder": "^1.3.3",
@@ -664,6 +672,7 @@
     "ioredis": "^5.3.2",
     "jest": "^29.5.0",
     "jsdom": "^22.1.0",
+    "langchainhub": "~0.0.3",
     "mammoth": "^1.5.1",
     "ml-matrix": "^6.10.4",
     "mongodb": "^5.2.0",
@@ -722,6 +731,8 @@
     "@planetscale/database": "^1.8.0",
     "@qdrant/js-client-rest": "^1.2.0",
     "@raycast/api": "^1.55.2",
+    "@smithy/eventstream-codec": "^2.0.5",
+    "@smithy/util-utf8": "^2.0.0",
     "@supabase/postgrest-js": "^1.1.1",
     "@supabase/supabase-js": "^2.10.0",
     "@tensorflow-models/universal-sentence-encoder": "*",
@@ -746,6 +757,7 @@
     "ignore": "^5.2.0",
     "ioredis": "^5.3.2",
     "jsdom": "*",
+    "langchainhub": "~0.0.3",
     "mammoth": "*",
     "mongodb": "^5.2.0",
     "mysql2": "^3.3.3",
@@ -848,6 +860,12 @@
     "@raycast/api": {
       "optional": true
     },
+    "@smithy/eventstream-codec": {
+      "optional": true
+    },
+    "@smithy/util-utf8": {
+      "optional": true
+    },
     "@supabase/postgrest-js": {
       "optional": true
     },
@@ -920,6 +938,9 @@
     "jsdom": {
       "optional": true
     },
+    "langchainhub": {
+      "optional": true
+    },
     "mammoth": {
       "optional": true
     },
@@ -1526,6 +1547,11 @@
       "import": "./document_loaders/web/notionapi.js",
       "require": "./document_loaders/web/notionapi.cjs"
     },
+    "./document_loaders/web/recursive_url": {
+      "types": "./document_loaders/web/recursive_url.d.ts",
+      "import": "./document_loaders/web/recursive_url.js",
+      "require": "./document_loaders/web/recursive_url.cjs"
+    },
     "./document_loaders/web/s3": {
       "types": "./document_loaders/web/s3.d.ts",
       "import": "./document_loaders/web/s3.js",
@@ -1920,6 +1946,11 @@
       "import": "./storage/ioredis.js",
       "require": "./storage/ioredis.cjs"
     },
+    "./hub": {
+      "types": "./hub.d.ts",
+      "import": "./hub.js",
+      "require": "./hub.cjs"
+    },
     "./util/math": {
       "types": "./util/math.d.ts",
       "import": "./util/math.js",