langchain 0.0.132 → 0.0.134

package/dist/document_loaders/web/notionapi.js

@@ -1,8 +1,22 @@
-import { Client, isFullBlock, isFullPage, iteratePaginatedAPI, } from "@notionhq/client";
+import { Client, isFullBlock, isFullPage, iteratePaginatedAPI, APIErrorCode, isNotionClientError, isFullDatabase, } from "@notionhq/client";
 import { NotionToMarkdown } from "notion-to-md";
 import { getBlockChildren } from "notion-to-md/build/utils/notion.js";
-import { BaseDocumentLoader } from "../base.js";
 import { Document } from "../../document.js";
+import { BaseDocumentLoader } from "../base.js";
+import { AsyncCaller } from "../../util/async_caller.js";
+const isPageResponse = (res) => !isNotionClientError(res) && res.object === "page";
+const isDatabaseResponse = (res) => !isNotionClientError(res) && res.object === "database";
+const isErrorResponse = (res) => isNotionClientError(res);
+const isPage = (res) => isPageResponse(res) && isFullPage(res);
+const isDatabase = (res) => isDatabaseResponse(res) && isFullDatabase(res);
+const getTitle = (obj) => {
+    if (isPage(obj) && obj.properties.title.type === "title") {
+        return obj.properties.title.title[0]?.plain_text;
+    }
+    if (isDatabase(obj))
+        return obj.title[0]?.plain_text;
+    return null;
+};
 /**
  * A class that extends the BaseDocumentLoader class. It represents a
  * document loader for loading documents from Notion using the Notion API.
@@ -10,6 +24,12 @@ import { Document } from "../../document.js";
 export class NotionAPILoader extends BaseDocumentLoader {
     constructor(options) {
         super();
+        Object.defineProperty(this, "caller", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "notionClient", {
             enumerable: true,
             configurable: true,
@@ -28,19 +48,66 @@ export class NotionAPILoader extends BaseDocumentLoader {
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "type", {
+        Object.defineProperty(this, "pageQueue", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pageCompleted", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pageQueueTotal", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "documents", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
-        this.notionClient = new Client(options.clientOptions);
+        Object.defineProperty(this, "rootTitle", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "onDocumentLoaded", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.caller = new AsyncCaller({
+            maxConcurrency: 64,
+            ...options.callerOptions,
+        });
+        this.notionClient = new Client({
+            logger: () => { },
+            ...options.clientOptions,
+        });
         this.n2mClient = new NotionToMarkdown({
             notionClient: this.notionClient,
             config: { parseChildPages: false, convertImagesToBase64: false },
         });
         this.id = options.id;
-        this.type = options.type;
+        this.pageQueue = [];
+        this.pageCompleted = [];
+        this.pageQueueTotal = 0;
+        this.documents = [];
+        this.rootTitle = "";
+        this.onDocumentLoaded = options.onDocumentLoaded ?? ((_ti, _cu) => { });
+    }
+    addToQueue(...items) {
+        const deDuped = items.filter((item) => !this.pageCompleted.concat(this.pageQueue).includes(item));
+        this.pageQueue.push(...deDuped);
+        this.pageQueueTotal += deDuped.length;
     }
     /**
      * Parses the properties of a Notion page and returns them as key-value
@@ -123,123 +190,135 @@ export class NotionAPILoader extends BaseDocumentLoader {
      * @returns A Promise that resolves to an MdBlock object.
      */
     async loadBlock(block) {
-        return {
+        const mdBlock = {
             type: block.type,
             blockId: block.id,
-            parent: await this.n2mClient.blockToMarkdown(block),
+            parent: await this.caller.call(() => this.n2mClient.blockToMarkdown(block)),
             children: [],
         };
+        if (block.has_children) {
+            const block_id = block.type === "synced_block" &&
+                block.synced_block?.synced_from?.block_id
+                ? block.synced_block.synced_from.block_id
+                : block.id;
+            const childBlocks = await this.loadBlocks(await this.caller.call(() => getBlockChildren(this.notionClient, block_id, null)));
+            mdBlock.children = childBlocks;
+        }
+        return mdBlock;
     }
     /**
-     * Loads Notion blocks and their child documents recursively.
+     * Loads Notion blocks and their children recursively.
      * @param blocksResponse The response from the Notion API containing the blocks to load.
-     * @returns A Promise that resolves to an object containing the loaded MdBlocks and child Documents.
+     * @returns A Promise that resolves to an array containing the loaded MdBlocks.
      */
-    async loadBlocksAndDocs(blocksResponse) {
+    async loadBlocks(blocksResponse) {
         const blocks = blocksResponse.filter(isFullBlock);
-        const [childPageDocuments, childDatabaseDocuments, blocksDocsArray] = await Promise.all([
-            Promise.all(blocks
-                .filter((block) => block.type.includes("child_page"))
-                .map((block) => this.loadPage(block.id))),
-            Promise.all(blocks
-                .filter((block) => block.type.includes("child_database"))
-                .map((block) => this.loadDatabase(block.id))),
-            Promise.all(blocks
-                .filter((block) => !["child_page", "child_database"].includes(block.type))
-                .map(async (block) => {
-                const mdBlock = await this.loadBlock(block);
-                let childDocuments = [];
-                if (block.has_children) {
-                    const block_id = block.type === "synced_block" &&
-                        block.synced_block?.synced_from?.block_id
-                        ? block.synced_block.synced_from.block_id
-                        : block.id;
-                    const childBlocksDocs = await this.loadBlocksAndDocs(await getBlockChildren(this.notionClient, block_id, null));
-                    mdBlock.children = childBlocksDocs.mdBlocks;
-                    childDocuments = childBlocksDocs.childDocuments;
-                }
-                return {
-                    mdBlocks: [mdBlock],
-                    childDocuments,
-                };
-            })),
+        // Add child pages to queue
+        const childPages = blocks
+            .filter((block) => block.type.includes("child_page"))
+            .map((block) => block.id);
+        if (childPages.length > 0)
+            this.addToQueue(...childPages);
+        // Add child database pages to queue
+        const childDatabases = blocks
+            .filter((block) => block.type.includes("child_database"))
+            .map((block) => this.caller.call(() => this.loadDatabase(block.id)));
+        // Load this block and child blocks
+        const loadingMdBlocks = blocks
+            .filter((block) => !["child_page", "child_database"].includes(block.type))
+            .map((block) => this.loadBlock(block));
+        const [mdBlocks] = await Promise.all([
+            Promise.all(loadingMdBlocks),
+            Promise.all(childDatabases),
         ]);
-        const allMdBlocks = blocksDocsArray
-            .flat()
-            .map((blockDoc) => blockDoc.mdBlocks);
-        const childDocuments = blocksDocsArray
-            .flat()
-            .map((blockDoc) => blockDoc.childDocuments);
-        return {
-            mdBlocks: [...allMdBlocks.flat()],
-            childDocuments: [
-                ...childPageDocuments.flat(),
-                ...childDatabaseDocuments.flat(),
-                ...childDocuments.flat(),
-            ],
-        };
+        return mdBlocks;
     }
     /**
-     * Loads a Notion page and its child documents.
+     * Loads a Notion page and its child documents, then adds it to the completed documents array.
      * @param page The Notion page or page ID to load.
-     * @returns A Promise that resolves to an array of Documents.
      */
     async loadPage(page) {
-        // Check page is a page ID or a GetPageResponse
+        // Check page is a page ID or a PageObjectResponse
         const [pageData, pageId] = typeof page === "string"
-            ? [this.notionClient.pages.retrieve({ page_id: page }), page]
+            ? [
+                this.caller.call(() => this.notionClient.pages.retrieve({ page_id: page })),
+                page,
+            ]
             : [page, page.id];
         const [pageDetails, pageBlocks] = await Promise.all([
             pageData,
-            getBlockChildren(this.notionClient, pageId, null),
+            this.caller.call(() => getBlockChildren(this.notionClient, pageId, null)),
         ]);
         if (!isFullPage(pageDetails))
-            return [];
-        const { mdBlocks, childDocuments } = await this.loadBlocksAndDocs(pageBlocks);
+            return;
+        const mdBlocks = await this.loadBlocks(pageBlocks);
         const mdStringObject = this.n2mClient.toMarkdownString(mdBlocks);
         const pageDocument = new Document({
             pageContent: mdStringObject.parent,
             metadata: this.parsePageDetails(pageDetails),
         });
-        return [pageDocument, ...childDocuments];
+        this.documents.push(pageDocument);
+        this.pageCompleted.push(pageId);
+        this.onDocumentLoaded(this.documents.length, this.pageQueueTotal, pageDocument.metadata.properties.title, this.rootTitle);
     }
     /**
-     * Loads a Notion database and its documents.
+     * Loads a Notion database and adds it's pages to the queue.
      * @param id The ID of the Notion database to load.
-     * @returns A Promise that resolves to an array of Documents.
      */
     async loadDatabase(id) {
-        const documents = [];
         try {
             for await (const page of iteratePaginatedAPI(this.notionClient.databases.query, {
                 database_id: id,
+                page_size: 50,
             })) {
-                if (!isFullPage(page))
-                    continue;
-                documents.push(...(await this.loadPage(page)));
+                this.addToQueue(page.id);
             }
         }
         catch (e) {
             console.log(e);
             // TODO: Catch and report api request errors
         }
-        return documents;
     }
     /**
      * Loads the documents from Notion based on the specified options.
      * @returns A Promise that resolves to an array of Documents.
      */
     async load() {
-        const documents = [];
-        switch (this.type) {
-            case "page":
-                documents.push(...(await this.loadPage(this.id)));
-                break;
-            case "database":
-                documents.push(...(await this.loadDatabase(this.id)));
-                break;
-            default:
+        const resPagePromise = this.notionClient.pages
+            .retrieve({ page_id: this.id })
+            .then((res) => {
+            this.addToQueue(this.id);
+            return res;
+        })
+            .catch((error) => error);
+        const resDatabasePromise = this.notionClient.databases
+            .retrieve({ database_id: this.id })
+            .then(async (res) => {
+            await this.loadDatabase(this.id);
+            return res;
+        })
+            .catch((error) => error);
+        const [resPage, resDatabase] = await Promise.all([
+            resPagePromise,
+            resDatabasePromise,
+        ]);
+        // Check if both resPage and resDatabase resulted in error responses
+        const errors = [resPage, resDatabase].filter(isErrorResponse);
+        if (errors.length === 2) {
+            if (errors.every((e) => e.code === APIErrorCode.ObjectNotFound)) {
+                throw new AggregateError([
+                    Error(`Could not find object with ID: ${this.id}. Make sure the relevant pages and databases are shared with your integration.`),
+                    ...errors,
+                ]);
+            }
+            throw new AggregateError(errors);
+        }
+        this.rootTitle = getTitle(resPage) || getTitle(resDatabase) || this.id;
+        let pageId = this.pageQueue.shift();
+        while (pageId) {
+            await this.loadPage(pageId);
+            pageId = this.pageQueue.shift();
         }
-        return documents;
+        return this.documents;
     }
 }

package/dist/document_loaders/web/recursive_url.cjs

@@ -0,0 +1,177 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RecursiveUrlLoader = void 0;
+const jsdom_1 = require("jsdom");
+const async_caller_js_1 = require("../../util/async_caller.cjs");
+const base_js_1 = require("../base.cjs");
+class RecursiveUrlLoader extends base_js_1.BaseDocumentLoader {
+    constructor(url, options) {
+        super();
+        Object.defineProperty(this, "caller", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "url", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "excludeDirs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "extractor", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "maxDepth", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "timeout", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "preventOutside", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.caller = new async_caller_js_1.AsyncCaller({
+            maxConcurrency: 64,
+            maxRetries: 0,
+            ...options.callerOptions,
+        });
+        this.url = url;
+        this.excludeDirs = options.excludeDirs ?? [];
+        this.extractor = options.extractor ?? ((s) => s);
+        this.maxDepth = options.maxDepth ?? 2;
+        this.timeout = options.timeout ?? 10000;
+        this.preventOutside = options.preventOutside ?? true;
+    }
+    async fetchWithTimeout(resource, options) {
+        const { timeout, ...rest } = options;
+        return this.caller.call(() => fetch(resource, { ...rest, signal: AbortSignal.timeout(timeout) }));
+    }
+    getChildLinks(html, baseUrl) {
+        const allLinks = Array.from(new jsdom_1.JSDOM(html).window.document.querySelectorAll("a")).map((a) => a.href);
+        const absolutePaths = [];
+        // eslint-disable-next-line no-script-url
+        const invalidPrefixes = ["javascript:", "mailto:", "#"];
+        const invalidSuffixes = [
+            ".css",
+            ".js",
+            ".ico",
+            ".png",
+            ".jpg",
+            ".jpeg",
+            ".gif",
+            ".svg",
+        ];
+        for (const link of allLinks) {
+            if (invalidPrefixes.some((prefix) => link.startsWith(prefix)) ||
+                invalidSuffixes.some((suffix) => link.endsWith(suffix)))
+                continue;
+            if (link.startsWith("http")) {
+                const isAllowed = !this.preventOutside || link.startsWith(baseUrl);
+                if (isAllowed)
+                    absolutePaths.push(link);
+            }
+            else if (link.startsWith("//")) {
+                const base = new URL(baseUrl);
+                absolutePaths.push(base.protocol + link);
+            }
+            else {
+                const newLink = new URL(link, baseUrl).href;
+                absolutePaths.push(newLink);
+            }
+        }
+        return Array.from(new Set(absolutePaths));
+    }
+    extractMetadata(rawHtml, url) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const metadata = { source: url };
+        const { document } = new jsdom_1.JSDOM(rawHtml).window;
+        const title = document.getElementsByTagName("title")[0];
+        if (title) {
+            metadata.title = title.textContent;
+        }
+        const description = document.querySelector("meta[name=description]");
+        if (description) {
+            metadata.description = description.getAttribute("content");
+        }
+        const html = document.getElementsByTagName("html")[0];
+        if (html) {
+            metadata.language = html.getAttribute("lang");
+        }
+        return metadata;
+    }
+    async getUrlAsDoc(url) {
+        let res;
+        try {
+            res = await this.fetchWithTimeout(url, { timeout: this.timeout });
+            res = await res.text();
+        }
+        catch (e) {
+            return null;
+        }
+        return {
+            pageContent: this.extractor(res),
+            metadata: this.extractMetadata(res, url),
+        };
+    }
+    async getChildUrlsRecursive(inputUrl, visited = new Set(), depth = 0) {
+        if (depth > this.maxDepth)
+            return [];
+        let url = inputUrl;
+        if (!inputUrl.endsWith("/"))
+            url += "/";
+        const isExcluded = this.excludeDirs.some((exDir) => url.startsWith(exDir));
+        if (isExcluded)
+            return [];
+        let res;
+        try {
+            res = await this.fetchWithTimeout(url, { timeout: this.timeout });
+            res = await res.text();
+        }
+        catch (e) {
+            return [];
+        }
+        const childUrls = this.getChildLinks(res, url);
+        const results = await Promise.all(childUrls.map((childUrl) => (async () => {
+            if (visited.has(childUrl))
+                return null;
+            visited.add(childUrl);
+            const childDoc = await this.getUrlAsDoc(childUrl);
+            if (!childDoc)
+                return null;
+            if (childUrl.endsWith("/")) {
+                const childUrlResponses = await this.getChildUrlsRecursive(childUrl, visited, depth + 1);
+                return [childDoc, ...childUrlResponses];
+            }
+            return [childDoc];
+        })()));
+        return results.flat().filter((docs) => docs !== null);
+    }
+    async load() {
+        const rootDoc = await this.getUrlAsDoc(this.url);
+        if (!rootDoc)
+            return [];
+        const docs = [rootDoc];
+        docs.push(...(await this.getChildUrlsRecursive(this.url, new Set([this.url]))));
+        return docs;
+    }
+}
+exports.RecursiveUrlLoader = RecursiveUrlLoader;

package/dist/document_loaders/web/recursive_url.d.ts

@@ -0,0 +1,27 @@
+import { Document } from "../../document.js";
+import { AsyncCaller } from "../../util/async_caller.js";
+import { BaseDocumentLoader, DocumentLoader } from "../base.js";
+export interface RecursiveUrlLoaderOptions {
+    excludeDirs?: string[];
+    extractor?: (text: string) => string;
+    maxDepth?: number;
+    timeout?: number;
+    preventOutside?: boolean;
+    callerOptions?: ConstructorParameters<typeof AsyncCaller>[0];
+}
+export declare class RecursiveUrlLoader extends BaseDocumentLoader implements DocumentLoader {
+    private caller;
+    private url;
+    private excludeDirs;
+    private extractor;
+    private maxDepth;
+    private timeout;
+    private preventOutside;
+    constructor(url: string, options: RecursiveUrlLoaderOptions);
+    private fetchWithTimeout;
+    private getChildLinks;
+    private extractMetadata;
+    private getUrlAsDoc;
+    private getChildUrlsRecursive;
+    load(): Promise<Document[]>;
+}