langchain 0.0.133 → 0.0.135

package/dist/agents/chat_convo/outputParser.cjs

@@ -36,16 +36,19 @@ class ChatConversationalAgentOutputParser extends types_js_1.AgentActionOutputPa
      */
     async parse(text) {
         let jsonOutput = text.trim();
-        if (jsonOutput.includes("```json")) {
-            jsonOutput = jsonOutput.split("```json")[1].trimStart();
-        }
-        else if (jsonOutput.includes("```")) {
-            const firstIndex = jsonOutput.indexOf("```");
-            jsonOutput = jsonOutput.slice(firstIndex + 3).trimStart();
-        }
-        const lastIndex = jsonOutput.lastIndexOf("```");
-        if (lastIndex !== -1) {
-            jsonOutput = jsonOutput.slice(0, lastIndex).trimEnd();
+        if (jsonOutput.includes("```json") || jsonOutput.includes("```")) {
+            const testString = jsonOutput.includes("```json") ? "```json" : "```";
+            const firstIndex = jsonOutput.indexOf(testString);
+            const actionInputIndex = jsonOutput.indexOf("action_input");
+            if (actionInputIndex > firstIndex) {
+                jsonOutput = jsonOutput
+                    .slice(firstIndex + testString.length)
+                    .trimStart();
+                const lastIndex = jsonOutput.lastIndexOf("```");
+                if (lastIndex !== -1) {
+                    jsonOutput = jsonOutput.slice(0, lastIndex).trimEnd();
+                }
+            }
         }
         try {
             const response = JSON.parse(jsonOutput);

package/dist/agents/chat_convo/outputParser.js

@@ -33,16 +33,19 @@ export class ChatConversationalAgentOutputParser extends AgentActionOutputParser
      */
     async parse(text) {
         let jsonOutput = text.trim();
-        if (jsonOutput.includes("```json")) {
-            jsonOutput = jsonOutput.split("```json")[1].trimStart();
-        }
-        else if (jsonOutput.includes("```")) {
-            const firstIndex = jsonOutput.indexOf("```");
-            jsonOutput = jsonOutput.slice(firstIndex + 3).trimStart();
-        }
-        const lastIndex = jsonOutput.lastIndexOf("```");
-        if (lastIndex !== -1) {
-            jsonOutput = jsonOutput.slice(0, lastIndex).trimEnd();
+        if (jsonOutput.includes("```json") || jsonOutput.includes("```")) {
+            const testString = jsonOutput.includes("```json") ? "```json" : "```";
+            const firstIndex = jsonOutput.indexOf(testString);
+            const actionInputIndex = jsonOutput.indexOf("action_input");
+            if (actionInputIndex > firstIndex) {
+                jsonOutput = jsonOutput
+                    .slice(firstIndex + testString.length)
+                    .trimStart();
+                const lastIndex = jsonOutput.lastIndexOf("```");
+                if (lastIndex !== -1) {
+                    jsonOutput = jsonOutput.slice(0, lastIndex).trimEnd();
+                }
+            }
         }
         try {
             const response = JSON.parse(jsonOutput);

package/dist/callbacks/base.d.ts

@@ -1,4 +1,4 @@
-import { AgentAction, AgentFinish, BaseMessage, ChainValues, LLMResult } from "../schema/index.js";
+import { AgentAction, AgentFinish, BaseMessage, ChainValues, ChatGenerationChunk, GenerationChunk, LLMResult } from "../schema/index.js";
 import { Serializable, Serialized, SerializedNotImplemented } from "../load/serializable.js";
 import { SerializedFields } from "../load/map_keys.js";
 import { Document } from "../document.js";
@@ -22,6 +22,9 @@ export interface NewTokenIndices {
     prompt: number;
     completion: number;
 }
+export type HandleLLMNewTokenCallbackFields = {
+    chunk?: GenerationChunk | ChatGenerationChunk;
+};
 /**
  * Abstract class that provides a set of optional methods that can be
  * overridden in derived classes to handle various events during the
@@ -43,7 +46,7 @@ declare abstract class BaseCallbackHandlerMethodsClass {
      * idx.completion is the index of the completion that produced the token
      *   (if multiple completions per prompt are requested)
      */
-    idx: NewTokenIndices, runId: string, parentRunId?: string, tags?: string[]): Promise<void> | void;
+    idx: NewTokenIndices, runId: string, parentRunId?: string, tags?: string[], fields?: HandleLLMNewTokenCallbackFields): Promise<void> | void;
     /**
      * Called if an LLM/ChatModel run encounters an error
      */
@@ -182,7 +185,7 @@ export declare abstract class BaseCallbackHandler extends BaseCallbackHandlerMet
         /**
          * Called when an LLM/ChatModel in `streaming` mode produces a new token
          */
-        handleLLMNewToken?(token: string, idx: NewTokenIndices, runId: string, parentRunId?: string | undefined, tags?: string[] | undefined): void | Promise<void>;
+        handleLLMNewToken?(token: string, idx: NewTokenIndices, runId: string, parentRunId?: string | undefined, tags?: string[] | undefined, fields?: HandleLLMNewTokenCallbackFields | undefined): void | Promise<void>;
         /**
          * Called if an LLM/ChatModel run encounters an error
          */

package/dist/callbacks/handlers/tracer.cjs

@@ -332,7 +332,7 @@ class BaseTracer extends base_js_1.BaseCallbackHandler {
         });
         await this.onText?.(run);
     }
-    async handleLLMNewToken(token, idx, runId) {
+    async handleLLMNewToken(token, idx, runId, _parentRunId, _tags, fields) {
         const run = this.runMap.get(runId);
         if (!run || run?.run_type !== "llm") {
             return;
@@ -340,7 +340,7 @@ class BaseTracer extends base_js_1.BaseCallbackHandler {
         run.events.push({
             name: "new_token",
             time: Date.now(),
-            kwargs: { token, idx },
+            kwargs: { token, idx, chunk: fields?.chunk },
         });
         await this.onLLMNewToken?.(run);
     }

package/dist/callbacks/handlers/tracer.d.ts

@@ -1,7 +1,7 @@
 import { KVMap, BaseRun } from "langsmith/schemas";
 import { AgentAction, AgentFinish, BaseMessage, ChainValues, LLMResult } from "../../schema/index.js";
 import { Serialized } from "../../load/serializable.js";
-import { BaseCallbackHandler, BaseCallbackHandlerInput, NewTokenIndices } from "../base.js";
+import { BaseCallbackHandler, BaseCallbackHandlerInput, HandleLLMNewTokenCallbackFields, NewTokenIndices } from "../base.js";
 import { Document } from "../../document.js";
 export type RunType = string;
 export interface Run extends BaseRun {
@@ -44,7 +44,7 @@ export declare abstract class BaseTracer extends BaseCallbackHandler {
     handleRetrieverEnd(documents: Document<Record<string, unknown>>[], runId: string): Promise<void>;
     handleRetrieverError(error: Error, runId: string): Promise<void>;
     handleText(text: string, runId: string): Promise<void>;
-    handleLLMNewToken(token: string, idx: NewTokenIndices, runId: string): Promise<void>;
+    handleLLMNewToken(token: string, idx: NewTokenIndices, runId: string, _parentRunId?: string, _tags?: string[], fields?: HandleLLMNewTokenCallbackFields): Promise<void>;
     onLLMStart?(run: Run): void | Promise<void>;
     onLLMEnd?(run: Run): void | Promise<void>;
     onLLMError?(run: Run): void | Promise<void>;

package/dist/callbacks/handlers/tracer.js

@@ -329,7 +329,7 @@ export class BaseTracer extends BaseCallbackHandler {
         });
         await this.onText?.(run);
     }
-    async handleLLMNewToken(token, idx, runId) {
+    async handleLLMNewToken(token, idx, runId, _parentRunId, _tags, fields) {
         const run = this.runMap.get(runId);
         if (!run || run?.run_type !== "llm") {
             return;
@@ -337,7 +337,7 @@ export class BaseTracer extends BaseCallbackHandler {
         run.events.push({
             name: "new_token",
             time: Date.now(),
-            kwargs: { token, idx },
+            kwargs: { token, idx, chunk: fields?.chunk },
         });
         await this.onLLMNewToken?.(run);
     }

package/dist/callbacks/index.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.consumeCallback = exports.awaitAllCallbacks = exports.traceAsGroup = exports.TraceGroup = exports.CallbackManagerForToolRun = exports.CallbackManagerForLLMRun = exports.CallbackManagerForChainRun = exports.CallbackManager = exports.getTracingV2CallbackHandler = exports.getTracingCallbackHandler = exports.LangChainTracerV1 = exports.LangChainTracer = exports.ConsoleCallbackHandler = exports.BaseTracer = exports.BaseCallbackHandler = void 0;
+exports.consumeCallback = exports.awaitAllCallbacks = exports.traceAsGroup = exports.TraceGroup = exports.CallbackManagerForToolRun = exports.CallbackManagerForLLMRun = exports.CallbackManagerForChainRun = exports.CallbackManagerForRetrieverRun = exports.CallbackManager = exports.getTracingV2CallbackHandler = exports.getTracingCallbackHandler = exports.LangChainTracerV1 = exports.LangChainTracer = exports.ConsoleCallbackHandler = exports.BaseTracer = exports.BaseCallbackHandler = void 0;
 var base_js_1 = require("./base.cjs");
 Object.defineProperty(exports, "BaseCallbackHandler", { enumerable: true, get: function () { return base_js_1.BaseCallbackHandler; } });
 var tracer_js_1 = require("./handlers/tracer.cjs");
@@ -16,6 +16,7 @@ Object.defineProperty(exports, "getTracingCallbackHandler", { enumerable: true,
 Object.defineProperty(exports, "getTracingV2CallbackHandler", { enumerable: true, get: function () { return initialize_js_1.getTracingV2CallbackHandler; } });
 var manager_js_1 = require("./manager.cjs");
 Object.defineProperty(exports, "CallbackManager", { enumerable: true, get: function () { return manager_js_1.CallbackManager; } });
+Object.defineProperty(exports, "CallbackManagerForRetrieverRun", { enumerable: true, get: function () { return manager_js_1.CallbackManagerForRetrieverRun; } });
 Object.defineProperty(exports, "CallbackManagerForChainRun", { enumerable: true, get: function () { return manager_js_1.CallbackManagerForChainRun; } });
 Object.defineProperty(exports, "CallbackManagerForLLMRun", { enumerable: true, get: function () { return manager_js_1.CallbackManagerForLLMRun; } });
 Object.defineProperty(exports, "CallbackManagerForToolRun", { enumerable: true, get: function () { return manager_js_1.CallbackManagerForToolRun; } });

package/dist/callbacks/index.d.ts

@@ -4,5 +4,5 @@ export { ConsoleCallbackHandler } from "./handlers/console.js";
 export { LangChainTracer } from "./handlers/tracer_langchain.js";
 export { LangChainTracerV1 } from "./handlers/tracer_langchain_v1.js";
 export { getTracingCallbackHandler, getTracingV2CallbackHandler, } from "./handlers/initialize.js";
-export { CallbackManager, CallbackManagerForChainRun, CallbackManagerForLLMRun, CallbackManagerForToolRun, CallbackManagerOptions, Callbacks, TraceGroup, traceAsGroup, } from "./manager.js";
+export { CallbackManager, CallbackManagerForRetrieverRun, CallbackManagerForChainRun, CallbackManagerForLLMRun, CallbackManagerForToolRun, CallbackManagerOptions, Callbacks, TraceGroup, traceAsGroup, } from "./manager.js";
 export { awaitAllCallbacks, consumeCallback } from "./promises.js";

package/dist/callbacks/index.js

@@ -4,5 +4,5 @@ export { ConsoleCallbackHandler } from "./handlers/console.js";
 export { LangChainTracer } from "./handlers/tracer_langchain.js";
 export { LangChainTracerV1 } from "./handlers/tracer_langchain_v1.js";
 export { getTracingCallbackHandler, getTracingV2CallbackHandler, } from "./handlers/initialize.js";
-export { CallbackManager, CallbackManagerForChainRun, CallbackManagerForLLMRun, CallbackManagerForToolRun, TraceGroup, traceAsGroup, } from "./manager.js";
+export { CallbackManager, CallbackManagerForRetrieverRun, CallbackManagerForChainRun, CallbackManagerForLLMRun, CallbackManagerForToolRun, TraceGroup, traceAsGroup, } from "./manager.js";
 export { awaitAllCallbacks, consumeCallback } from "./promises.js";

package/dist/callbacks/manager.cjs

@@ -137,11 +137,11 @@ class CallbackManagerForRetrieverRun extends BaseRunManager {
 }
 exports.CallbackManagerForRetrieverRun = CallbackManagerForRetrieverRun;
 class CallbackManagerForLLMRun extends BaseRunManager {
-    async handleLLMNewToken(token, idx = { prompt: 0, completion: 0 }) {
+    async handleLLMNewToken(token, idx, _runId, _parentRunId, _tags, fields) {
         await Promise.all(this.handlers.map((handler) => (0, promises_js_1.consumeCallback)(async () => {
             if (!handler.ignoreLLM) {
                 try {
-                    await handler.handleLLMNewToken?.(token, idx, this.runId, this._parentRunId, this.tags);
+                    await handler.handleLLMNewToken?.(token, idx ?? { prompt: 0, completion: 0 }, this.runId, this._parentRunId, this.tags, fields);
                 }
                 catch (err) {
                     console.error(`Error in handler ${handler.constructor.name}, handleLLMNewToken: ${err}`);

package/dist/callbacks/manager.d.ts

@@ -1,5 +1,5 @@
 import { AgentAction, AgentFinish, BaseMessage, ChainValues, LLMResult } from "../schema/index.js";
-import { BaseCallbackHandler, CallbackHandlerMethods, NewTokenIndices } from "./base.js";
+import { BaseCallbackHandler, CallbackHandlerMethods, HandleLLMNewTokenCallbackFields, NewTokenIndices } from "./base.js";
 import { LangChainTracerFields } from "./handlers/tracer_langchain.js";
 import { Serialized } from "../load/serializable.js";
 import { Document } from "../document.js";
@@ -62,7 +62,7 @@ export declare class CallbackManagerForRetrieverRun extends BaseRunManager imple
     handleRetrieverError(err: Error | unknown): Promise<void>;
 }
 export declare class CallbackManagerForLLMRun extends BaseRunManager implements BaseCallbackManagerMethods {
-    handleLLMNewToken(token: string, idx?: NewTokenIndices): Promise<void>;
+    handleLLMNewToken(token: string, idx?: NewTokenIndices, _runId?: string, _parentRunId?: string, _tags?: string[], fields?: HandleLLMNewTokenCallbackFields): Promise<void>;
     handleLLMError(err: Error | unknown): Promise<void>;
     handleLLMEnd(output: LLMResult): Promise<void>;
 }

package/dist/callbacks/manager.js

@@ -131,11 +131,11 @@ export class CallbackManagerForRetrieverRun extends BaseRunManager {
     }
 }
 export class CallbackManagerForLLMRun extends BaseRunManager {
-    async handleLLMNewToken(token, idx = { prompt: 0, completion: 0 }) {
+    async handleLLMNewToken(token, idx, _runId, _parentRunId, _tags, fields) {
         await Promise.all(this.handlers.map((handler) => consumeCallback(async () => {
             if (!handler.ignoreLLM) {
                 try {
-                    await handler.handleLLMNewToken?.(token, idx, this.runId, this._parentRunId, this.tags);
+                    await handler.handleLLMNewToken?.(token, idx ?? { prompt: 0, completion: 0 }, this.runId, this._parentRunId, this.tags, fields);
                 }
                 catch (err) {
                     console.error(`Error in handler ${handler.constructor.name}, handleLLMNewToken: ${err}`);

package/dist/chains/sql_db/sql_db_chain.d.ts

@@ -29,9 +29,9 @@ export declare class SqlDatabaseChain extends BaseChain {
     database: SqlDatabase;
     prompt: PromptTemplate<{
         input: any;
+        top_k: any;
         dialect: any;
         table_info: any;
-        top_k: any;
     }, any>;
     topK: number;
     inputKey: string;

package/dist/chains/sql_db/sql_db_prompt.d.ts

@@ -1,37 +1,37 @@
 import { PromptTemplate } from "../../prompts/prompt.js";
 export declare const DEFAULT_SQL_DATABASE_PROMPT: PromptTemplate<{
     input: any;
+    top_k: any;
     dialect: any;
     table_info: any;
-    top_k: any;
 }, any>;
 export declare const SQL_POSTGRES_PROMPT: PromptTemplate<{
     input: any;
+    top_k: any;
     dialect: any;
     table_info: any;
-    top_k: any;
 }, any>;
 export declare const SQL_SQLITE_PROMPT: PromptTemplate<{
     input: any;
+    top_k: any;
     dialect: any;
     table_info: any;
-    top_k: any;
 }, any>;
 export declare const SQL_MYSQL_PROMPT: PromptTemplate<{
     input: any;
+    top_k: any;
     dialect: any;
     table_info: any;
-    top_k: any;
 }, any>;
 export declare const SQL_MSSQL_PROMPT: PromptTemplate<{
     input: any;
+    top_k: any;
     dialect: any;
     table_info: any;
-    top_k: any;
 }, any>;
 export declare const SQL_SAP_HANA_PROMPT: PromptTemplate<{
     input: any;
+    top_k: any;
     dialect: any;
     table_info: any;
-    top_k: any;
 }, any>;

package/dist/chat_models/openai.cjs

@@ -402,7 +402,10 @@ class ChatOpenAI extends base_js_1.BaseChatModel {
             });
             yield generationChunk;
             // eslint-disable-next-line no-void
-            void runManager?.handleLLMNewToken(generationChunk.text ?? "");
+            void runManager?.handleLLMNewToken(generationChunk.text ?? "", {
+                prompt: 0,
+                completion: choice.index,
+            }, undefined, undefined, undefined, { chunk: generationChunk });
         }
     }
     startStream(request, options) {
@@ -548,13 +551,15 @@ class ChatOpenAI extends base_js_1.BaseChatModel {
                                         choice.message.function_call.arguments +=
                                             part.delta?.function_call?.arguments ?? "";
                                     }
-                                    // eslint-disable-next-line no-void
+                                    const chunk = _convertDeltaToMessageChunk(part.delta, "assistant");
+                                    const generationChunk = new index_js_1.ChatGenerationChunk({
+                                        message: chunk,
+                                        text: chunk.content,
+                                    });
                                     void runManager?.handleLLMNewToken(part.delta?.content ?? "", {
                                         prompt: options.promptIndex ?? 0,
                                         completion: part.index,
-                                    });
-                                    // TODO we don't currently have a callback method for
-                                    // sending the function call arguments
+                                    }, undefined, undefined, undefined, { chunk: generationChunk });
                                 }
                             }
                             // when all messages are finished, resolve

package/dist/chat_models/openai.js

@@ -396,7 +396,10 @@ export class ChatOpenAI extends BaseChatModel {
             });
             yield generationChunk;
             // eslint-disable-next-line no-void
-            void runManager?.handleLLMNewToken(generationChunk.text ?? "");
+            void runManager?.handleLLMNewToken(generationChunk.text ?? "", {
+                prompt: 0,
+                completion: choice.index,
+            }, undefined, undefined, undefined, { chunk: generationChunk });
         }
     }
     startStream(request, options) {
@@ -542,13 +545,15 @@ export class ChatOpenAI extends BaseChatModel {
                                         choice.message.function_call.arguments +=
                                             part.delta?.function_call?.arguments ?? "";
                                     }
-                                    // eslint-disable-next-line no-void
+                                    const chunk = _convertDeltaToMessageChunk(part.delta, "assistant");
+                                    const generationChunk = new ChatGenerationChunk({
+                                        message: chunk,
+                                        text: chunk.content,
+                                    });
                                     void runManager?.handleLLMNewToken(part.delta?.content ?? "", {
                                         prompt: options.promptIndex ?? 0,
                                         completion: part.index,
-                                    });
-                                    // TODO we don't currently have a callback method for
-                                    // sending the function call arguments
+                                    }, undefined, undefined, undefined, { chunk: generationChunk });
                                 }
                             }
                             // when all messages are finished, resolve

package/dist/document_loaders/web/recursive_url.cjs

@@ -0,0 +1,177 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecursiveUrlLoader = void 0;
+const jsdom_1 = require("jsdom");
+const async_caller_js_1 = require("../../util/async_caller.cjs");
+const base_js_1 = require("../base.cjs");
+class RecursiveUrlLoader extends base_js_1.BaseDocumentLoader {
+    constructor(url, options) {
+        super();
+        Object.defineProperty(this, "caller", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "url", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "excludeDirs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "extractor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxDepth", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "timeout", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "preventOutside", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.caller = new async_caller_js_1.AsyncCaller({
+            maxConcurrency: 64,
+            maxRetries: 0,
+            ...options.callerOptions,
+        });
+        this.url = url;
+        this.excludeDirs = options.excludeDirs ?? [];
+        this.extractor = options.extractor ?? ((s) => s);
+        this.maxDepth = options.maxDepth ?? 2;
+        this.timeout = options.timeout ?? 10000;
+        this.preventOutside = options.preventOutside ?? true;
+    }
+    async fetchWithTimeout(resource, options) {
+        const { timeout, ...rest } = options;
+        return this.caller.call(() => fetch(resource, { ...rest, signal: AbortSignal.timeout(timeout) }));
+    }
+    getChildLinks(html, baseUrl) {
+        const allLinks = Array.from(new jsdom_1.JSDOM(html).window.document.querySelectorAll("a")).map((a) => a.href);
+        const absolutePaths = [];
+        // eslint-disable-next-line no-script-url
+        const invalidPrefixes = ["javascript:", "mailto:", "#"];
+        const invalidSuffixes = [
+            ".css",
+            ".js",
+            ".ico",
+            ".png",
+            ".jpg",
+            ".jpeg",
+            ".gif",
+            ".svg",
+        ];
+        for (const link of allLinks) {
+            if (invalidPrefixes.some((prefix) => link.startsWith(prefix)) ||
+                invalidSuffixes.some((suffix) => link.endsWith(suffix)))
+                continue;
+            if (link.startsWith("http")) {
+                const isAllowed = !this.preventOutside || link.startsWith(baseUrl);
+                if (isAllowed)
+                    absolutePaths.push(link);
+            }
+            else if (link.startsWith("//")) {
+                const base = new URL(baseUrl);
+                absolutePaths.push(base.protocol + link);
+            }
+            else {
+                const newLink = new URL(link, baseUrl).href;
+                absolutePaths.push(newLink);
+            }
+        }
+        return Array.from(new Set(absolutePaths));
+    }
+    extractMetadata(rawHtml, url) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const metadata = { source: url };
+        const { document } = new jsdom_1.JSDOM(rawHtml).window;
+        const title = document.getElementsByTagName("title")[0];
+        if (title) {
+            metadata.title = title.textContent;
+        }
+        const description = document.querySelector("meta[name=description]");
+        if (description) {
+            metadata.description = description.getAttribute("content");
+        }
+        const html = document.getElementsByTagName("html")[0];
+        if (html) {
+            metadata.language = html.getAttribute("lang");
+        }
+        return metadata;
+    }
+    async getUrlAsDoc(url) {
+        let res;
+        try {
+            res = await this.fetchWithTimeout(url, { timeout: this.timeout });
+            res = await res.text();
+        }
+        catch (e) {
+            return null;
+        }
+        return {
+            pageContent: this.extractor(res),
+            metadata: this.extractMetadata(res, url),
+        };
+    }
+    async getChildUrlsRecursive(inputUrl, visited = new Set(), depth = 0) {
+        if (depth > this.maxDepth)
+            return [];
+        let url = inputUrl;
+        if (!inputUrl.endsWith("/"))
+            url += "/";
+        const isExcluded = this.excludeDirs.some((exDir) => url.startsWith(exDir));
+        if (isExcluded)
+            return [];
+        let res;
+        try {
+            res = await this.fetchWithTimeout(url, { timeout: this.timeout });
+            res = await res.text();
+        }
+        catch (e) {
+            return [];
+        }
+        const childUrls = this.getChildLinks(res, url);
+        const results = await Promise.all(childUrls.map((childUrl) => (async () => {
+            if (visited.has(childUrl))
+                return null;
+            visited.add(childUrl);
+            const childDoc = await this.getUrlAsDoc(childUrl);
+            if (!childDoc)
+                return null;
+            if (childUrl.endsWith("/")) {
+                const childUrlResponses = await this.getChildUrlsRecursive(childUrl, visited, depth + 1);
+                return [childDoc, ...childUrlResponses];
+            }
+            return [childDoc];
+        })()));
+        return results.flat().filter((docs) => docs !== null);
+    }
+    async load() {
+        const rootDoc = await this.getUrlAsDoc(this.url);
+        if (!rootDoc)
+            return [];
+        const docs = [rootDoc];
+        docs.push(...(await this.getChildUrlsRecursive(this.url, new Set([this.url]))));
+        return docs;
+    }
+}
+exports.RecursiveUrlLoader = RecursiveUrlLoader;

package/dist/document_loaders/web/recursive_url.d.ts

@@ -0,0 +1,27 @@
+import { Document } from "../../document.js";
+import { AsyncCaller } from "../../util/async_caller.js";
+import { BaseDocumentLoader, DocumentLoader } from "../base.js";
+export interface RecursiveUrlLoaderOptions {
+    excludeDirs?: string[];
+    extractor?: (text: string) => string;
+    maxDepth?: number;
+    timeout?: number;
+    preventOutside?: boolean;
+    callerOptions?: ConstructorParameters<typeof AsyncCaller>[0];
+}
+export declare class RecursiveUrlLoader extends BaseDocumentLoader implements DocumentLoader {
+    private caller;
+    private url;
+    private excludeDirs;
+    private extractor;
+    private maxDepth;
+    private timeout;
+    private preventOutside;
+    constructor(url: string, options: RecursiveUrlLoaderOptions);
+    private fetchWithTimeout;
+    private getChildLinks;
+    private extractMetadata;
+    private getUrlAsDoc;
+    private getChildUrlsRecursive;
+    load(): Promise<Document[]>;
+}