langchain 0.0.132 → 0.0.134

package/dist/document_loaders/web/notionapi.cjs

@@ -4,8 +4,22 @@ exports.NotionAPILoader = void 0;
 const client_1 = require("@notionhq/client");
 const notion_to_md_1 = require("notion-to-md");
 const notion_js_1 = require("notion-to-md/build/utils/notion.js");
-const base_js_1 = require("../base.cjs");
 const document_js_1 = require("../../document.cjs");
+const base_js_1 = require("../base.cjs");
+const async_caller_js_1 = require("../../util/async_caller.cjs");
+const isPageResponse = (res) => !(0, client_1.isNotionClientError)(res) && res.object === "page";
+const isDatabaseResponse = (res) => !(0, client_1.isNotionClientError)(res) && res.object === "database";
+const isErrorResponse = (res) => (0, client_1.isNotionClientError)(res);
+const isPage = (res) => isPageResponse(res) && (0, client_1.isFullPage)(res);
+const isDatabase = (res) => isDatabaseResponse(res) && (0, client_1.isFullDatabase)(res);
+const getTitle = (obj) => {
+    if (isPage(obj) && obj.properties.title.type === "title") {
+        return obj.properties.title.title[0]?.plain_text;
+    }
+    if (isDatabase(obj))
+        return obj.title[0]?.plain_text;
+    return null;
+};
 /**
  * A class that extends the BaseDocumentLoader class. It represents a
  * document loader for loading documents from Notion using the Notion API.
@@ -13,6 +27,12 @@ const document_js_1 = require("../../document.cjs");
 class NotionAPILoader extends base_js_1.BaseDocumentLoader {
     constructor(options) {
         super();
+        Object.defineProperty(this, "caller", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "notionClient", {
             enumerable: true,
             configurable: true,
@@ -31,19 +51,66 @@ class NotionAPILoader extends base_js_1.BaseDocumentLoader {
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "type", {
+        Object.defineProperty(this, "pageQueue", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pageCompleted", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "pageQueueTotal", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "documents", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
-        this.notionClient = new client_1.Client(options.clientOptions);
+        Object.defineProperty(this, "rootTitle", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "onDocumentLoaded", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.caller = new async_caller_js_1.AsyncCaller({
+            maxConcurrency: 64,
+            ...options.callerOptions,
+        });
+        this.notionClient = new client_1.Client({
+            logger: () => { },
+            ...options.clientOptions,
+        });
         this.n2mClient = new notion_to_md_1.NotionToMarkdown({
             notionClient: this.notionClient,
             config: { parseChildPages: false, convertImagesToBase64: false },
         });
         this.id = options.id;
-        this.type = options.type;
+        this.pageQueue = [];
+        this.pageCompleted = [];
+        this.pageQueueTotal = 0;
+        this.documents = [];
+        this.rootTitle = "";
+        this.onDocumentLoaded = options.onDocumentLoaded ?? ((_ti, _cu) => { });
+    }
+    addToQueue(...items) {
+        const deDuped = items.filter((item) => !this.pageCompleted.concat(this.pageQueue).includes(item));
+        this.pageQueue.push(...deDuped);
+        this.pageQueueTotal += deDuped.length;
     }
     /**
      * Parses the properties of a Notion page and returns them as key-value
@@ -126,124 +193,136 @@ class NotionAPILoader extends base_js_1.BaseDocumentLoader {
      * @returns A Promise that resolves to an MdBlock object.
      */
     async loadBlock(block) {
-        return {
+        const mdBlock = {
             type: block.type,
             blockId: block.id,
-            parent: await this.n2mClient.blockToMarkdown(block),
+            parent: await this.caller.call(() => this.n2mClient.blockToMarkdown(block)),
             children: [],
         };
+        if (block.has_children) {
+            const block_id = block.type === "synced_block" &&
+                block.synced_block?.synced_from?.block_id
+                ? block.synced_block.synced_from.block_id
+                : block.id;
+            const childBlocks = await this.loadBlocks(await this.caller.call(() => (0, notion_js_1.getBlockChildren)(this.notionClient, block_id, null)));
+            mdBlock.children = childBlocks;
+        }
+        return mdBlock;
     }
     /**
-     * Loads Notion blocks and their child documents recursively.
+     * Loads Notion blocks and their children recursively.
      * @param blocksResponse The response from the Notion API containing the blocks to load.
-     * @returns A Promise that resolves to an object containing the loaded MdBlocks and child Documents.
+     * @returns A Promise that resolves to an array containing the loaded MdBlocks.
      */
-    async loadBlocksAndDocs(blocksResponse) {
+    async loadBlocks(blocksResponse) {
         const blocks = blocksResponse.filter(client_1.isFullBlock);
-        const [childPageDocuments, childDatabaseDocuments, blocksDocsArray] = await Promise.all([
-            Promise.all(blocks
-                .filter((block) => block.type.includes("child_page"))
-                .map((block) => this.loadPage(block.id))),
-            Promise.all(blocks
-                .filter((block) => block.type.includes("child_database"))
-                .map((block) => this.loadDatabase(block.id))),
-            Promise.all(blocks
-                .filter((block) => !["child_page", "child_database"].includes(block.type))
-                .map(async (block) => {
-                const mdBlock = await this.loadBlock(block);
-                let childDocuments = [];
-                if (block.has_children) {
-                    const block_id = block.type === "synced_block" &&
-                        block.synced_block?.synced_from?.block_id
-                        ? block.synced_block.synced_from.block_id
-                        : block.id;
-                    const childBlocksDocs = await this.loadBlocksAndDocs(await (0, notion_js_1.getBlockChildren)(this.notionClient, block_id, null));
-                    mdBlock.children = childBlocksDocs.mdBlocks;
-                    childDocuments = childBlocksDocs.childDocuments;
-                }
-                return {
-                    mdBlocks: [mdBlock],
-                    childDocuments,
-                };
-            })),
+        // Add child pages to queue
+        const childPages = blocks
+            .filter((block) => block.type.includes("child_page"))
+            .map((block) => block.id);
+        if (childPages.length > 0)
+            this.addToQueue(...childPages);
+        // Add child database pages to queue
+        const childDatabases = blocks
+            .filter((block) => block.type.includes("child_database"))
+            .map((block) => this.caller.call(() => this.loadDatabase(block.id)));
+        // Load this block and child blocks
+        const loadingMdBlocks = blocks
+            .filter((block) => !["child_page", "child_database"].includes(block.type))
+            .map((block) => this.loadBlock(block));
+        const [mdBlocks] = await Promise.all([
+            Promise.all(loadingMdBlocks),
+            Promise.all(childDatabases),
         ]);
-        const allMdBlocks = blocksDocsArray
-            .flat()
-            .map((blockDoc) => blockDoc.mdBlocks);
-        const childDocuments = blocksDocsArray
-            .flat()
-            .map((blockDoc) => blockDoc.childDocuments);
-        return {
-            mdBlocks: [...allMdBlocks.flat()],
-            childDocuments: [
-                ...childPageDocuments.flat(),
-                ...childDatabaseDocuments.flat(),
-                ...childDocuments.flat(),
-            ],
-        };
+        return mdBlocks;
     }
     /**
-     * Loads a Notion page and its child documents.
+     * Loads a Notion page and its child documents, then adds it to the completed documents array.
      * @param page The Notion page or page ID to load.
-     * @returns A Promise that resolves to an array of Documents.
      */
     async loadPage(page) {
-        // Check page is a page ID or a GetPageResponse
+        // Check page is a page ID or a PageObjectResponse
         const [pageData, pageId] = typeof page === "string"
-            ? [this.notionClient.pages.retrieve({ page_id: page }), page]
+            ? [
+                this.caller.call(() => this.notionClient.pages.retrieve({ page_id: page })),
+                page,
+            ]
             : [page, page.id];
         const [pageDetails, pageBlocks] = await Promise.all([
             pageData,
-            (0, notion_js_1.getBlockChildren)(this.notionClient, pageId, null),
+            this.caller.call(() => (0, notion_js_1.getBlockChildren)(this.notionClient, pageId, null)),
         ]);
         if (!(0, client_1.isFullPage)(pageDetails))
-            return [];
-        const { mdBlocks, childDocuments } = await this.loadBlocksAndDocs(pageBlocks);
+            return;
+        const mdBlocks = await this.loadBlocks(pageBlocks);
         const mdStringObject = this.n2mClient.toMarkdownString(mdBlocks);
         const pageDocument = new document_js_1.Document({
             pageContent: mdStringObject.parent,
             metadata: this.parsePageDetails(pageDetails),
         });
-        return [pageDocument, ...childDocuments];
+        this.documents.push(pageDocument);
+        this.pageCompleted.push(pageId);
+        this.onDocumentLoaded(this.documents.length, this.pageQueueTotal, pageDocument.metadata.properties.title, this.rootTitle);
     }
     /**
-     * Loads a Notion database and its documents.
+     * Loads a Notion database and adds it's pages to the queue.
      * @param id The ID of the Notion database to load.
-     * @returns A Promise that resolves to an array of Documents.
      */
     async loadDatabase(id) {
-        const documents = [];
         try {
             for await (const page of (0, client_1.iteratePaginatedAPI)(this.notionClient.databases.query, {
                 database_id: id,
+                page_size: 50,
             })) {
-                if (!(0, client_1.isFullPage)(page))
-                    continue;
-                documents.push(...(await this.loadPage(page)));
+                this.addToQueue(page.id);
             }
         }
         catch (e) {
             console.log(e);
             // TODO: Catch and report api request errors
         }
-        return documents;
     }
     /**
      * Loads the documents from Notion based on the specified options.
      * @returns A Promise that resolves to an array of Documents.
      */
     async load() {
-        const documents = [];
-        switch (this.type) {
-            case "page":
-                documents.push(...(await this.loadPage(this.id)));
-                break;
-            case "database":
-                documents.push(...(await this.loadDatabase(this.id)));
-                break;
-            default:
+        const resPagePromise = this.notionClient.pages
+            .retrieve({ page_id: this.id })
+            .then((res) => {
+            this.addToQueue(this.id);
+            return res;
+        })
+            .catch((error) => error);
+        const resDatabasePromise = this.notionClient.databases
+            .retrieve({ database_id: this.id })
+            .then(async (res) => {
+            await this.loadDatabase(this.id);
+            return res;
+        })
+            .catch((error) => error);
+        const [resPage, resDatabase] = await Promise.all([
+            resPagePromise,
+            resDatabasePromise,
+        ]);
+        // Check if both resPage and resDatabase resulted in error responses
+        const errors = [resPage, resDatabase].filter(isErrorResponse);
+        if (errors.length === 2) {
+            if (errors.every((e) => e.code === client_1.APIErrorCode.ObjectNotFound)) {
+                throw new AggregateError([
+                    Error(`Could not find object with ID: ${this.id}. Make sure the relevant pages and databases are shared with your integration.`),
+                    ...errors,
+                ]);
+            }
+            throw new AggregateError(errors);
+        }
+        this.rootTitle = getTitle(resPage) || getTitle(resDatabase) || this.id;
+        let pageId = this.pageQueue.shift();
+        while (pageId) {
+            await this.loadPage(pageId);
+            pageId = this.pageQueue.shift();
         }
-        return documents;
+        return this.documents;
     }
 }
 exports.NotionAPILoader = NotionAPILoader;

package/dist/document_loaders/web/notionapi.d.ts

@@ -1,26 +1,37 @@
 import { Client } from "@notionhq/client";
-import { BaseDocumentLoader } from "../base.js";
 import { Document } from "../../document.js";
+import { BaseDocumentLoader } from "../base.js";
+import { AsyncCaller } from "../../util/async_caller.js";
 /**
  * Represents the type of Notion API to load documents from. The options
  * are "database" or "page".
  */
 export type NotionAPIType = "database" | "page";
+export type OnDocumentLoadedCallback = (current: number, total: number, currentTitle?: string, rootTitle?: string) => void;
 export type NotionAPILoaderOptions = {
     clientOptions: ConstructorParameters<typeof Client>[0];
     id: string;
-    type: NotionAPIType;
+    type?: NotionAPIType;
+    callerOptions?: ConstructorParameters<typeof AsyncCaller>[0];
+    onDocumentLoaded?: OnDocumentLoadedCallback;
 };
 /**
  * A class that extends the BaseDocumentLoader class. It represents a
  * document loader for loading documents from Notion using the Notion API.
  */
 export declare class NotionAPILoader extends BaseDocumentLoader {
+    private caller;
     private notionClient;
     private n2mClient;
     private id;
-    private type;
+    private pageQueue;
+    private pageCompleted;
+    pageQueueTotal: number;
+    private documents;
+    private rootTitle;
+    private onDocumentLoaded;
     constructor(options: NotionAPILoaderOptions);
+    private addToQueue;
     /**
      * Parses the properties of a Notion page and returns them as key-value
      * pairs.
@@ -41,21 +52,19 @@ export declare class NotionAPILoader extends BaseDocumentLoader {
      */
     private loadBlock;
     /**
-     * Loads Notion blocks and their child documents recursively.
+     * Loads Notion blocks and their children recursively.
      * @param blocksResponse The response from the Notion API containing the blocks to load.
-     * @returns A Promise that resolves to an object containing the loaded MdBlocks and child Documents.
+     * @returns A Promise that resolves to an array containing the loaded MdBlocks.
      */
-    private loadBlocksAndDocs;
+    private loadBlocks;
     /**
-     * Loads a Notion page and its child documents.
+     * Loads a Notion page and its child documents, then adds it to the completed documents array.
      * @param page The Notion page or page ID to load.
-     * @returns A Promise that resolves to an array of Documents.
      */
     private loadPage;
     /**
-     * Loads a Notion database and its documents.
+     * Loads a Notion database and adds it's pages to the queue.
      * @param id The ID of the Notion database to load.
-     * @returns A Promise that resolves to an array of Documents.
      */
     private loadDatabase;
     /**