langchain 0.2.17 → 0.3.0-rc.0

package/dist/document_loaders/web/notiondb.cjs

@@ -1,199 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.NotionDBLoader = void 0;
-const env_1 = require("@langchain/core/utils/env");
-const base_js_1 = require("../base.cjs");
-const NOTION_BASE_URL = "https://api.notion.com/v1";
-/**
- * @deprecated use the `NotionAPILoader` class instead.
- * @example
- * ```typescript
- * const loader = new NotionDBLoader({
- *   pageSizeLimit: 10,
- *   databaseId: "{databaseId}",
- *   notionIntegrationToken: "{notionIntegrationToken}",
- * });
- * const docs = await loader.load();
- * ```
- */
-class NotionDBLoader extends base_js_1.BaseDocumentLoader {
-    constructor({ databaseId, notionApiVersion = "2022-06-28", notionIntegrationToken = (0, env_1.getEnvironmentVariable)("NOTION_INTEGRATION_TOKEN"), pageSizeLimit = 50, }) {
-        super();
-        Object.defineProperty(this, "integrationToken", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "databaseId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "notionApiVersion", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "pageSizeLimit", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "headers", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: {}
-        });
-        if (!notionIntegrationToken) {
-            throw new Error("You must provide a Notion integration token.");
-        }
-        this.integrationToken = notionIntegrationToken;
-        this.pageSizeLimit = pageSizeLimit;
-        this.notionApiVersion = notionApiVersion;
-        this.databaseId = databaseId;
-        this.headers = {
-            Authorization: `Bearer ${this.integrationToken}`,
-            "Content-Type": "application/json",
-            "Notion-Version": notionApiVersion,
-        };
-    }
-    /**
-     * Loads the documents from Notion based on the specified options.
-     * @returns An array of Document objects.
-     */
-    async load() {
-        const pageIds = await this.retrievePageIds();
-        const documents = [];
-        for (const pageId of pageIds) {
-            documents.push(await this.loadPage(pageId));
-        }
-        return documents;
-    }
-    /**
-     * Retrieves the IDs of the pages in the Notion database.
-     * @returns An array of page IDs.
-     */
-    async retrievePageIds() {
-        const url = `${NOTION_BASE_URL}/databases/${this.databaseId}/query`;
-        const pageIds = [];
-        const query = {
-            page_size: this.pageSizeLimit,
-        };
-        let hasMore = true;
-        while (hasMore) {
-            const response = await fetch(url, {
-                method: "POST",
-                body: JSON.stringify(query),
-                headers: this.headers,
-            });
-            const data = await response.json();
-            if (!response.ok) {
-                throw new Error(`Failed to load data from Notion. Please check your integration token and database id.`);
-            }
-            const { results, has_more, next_cursor } = data;
-            pageIds.push(...(results?.map((page) => page.id) ?? []));
-            hasMore = has_more;
-            query.start_cursor = next_cursor;
-        }
-        return pageIds;
-    }
-    /**
-     * Loads a Notion page and returns it as a Document object.
-     * @param pageId The ID of the Notion page to load.
-     * @returns A Document object representing the loaded Notion page.
-     */
-    async loadPage(pageId) {
-        const url = `${NOTION_BASE_URL}/pages/${pageId}`;
-        const response = await fetch(url, { method: "GET", headers: this.headers });
-        const data = await response.json();
-        if (!response.ok) {
-            throw new Error(`Unable to fetch page: ${response.status} ${JSON.stringify(data)}`);
-        }
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const metadata = {};
-        const { properties } = data;
-        for (const key of Object.keys(properties)) {
-            const item = properties[key];
-            const itemType = item.type;
-            let value;
-            switch (itemType) {
-                case "rich_text":
-                    value =
-                        item?.rich_text && item?.rich_text.length > 0
-                            ? item?.rich_text[0].plain_text
-                            : null;
-                    break;
-                case "title":
-                    value =
-                        item?.title && item?.title.length > 0
-                            ? item?.title[0].plain_text
-                            : null;
-                    break;
-                case "multi_select":
-                    if (item?.multi_select && item?.multi_select.length > 0) {
-                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                        value = item?.multi_select.map((el) => el.name);
-                    }
-                    break;
-                case "url":
-                    value = item?.url ? item.url : null;
-                    break;
-                default:
-                    break;
-            }
-            if (value) {
-                metadata[key.toLowerCase()] = value;
-            }
-        }
-        metadata.id = pageId;
-        return {
-            pageContent: await this.loadBlocks(pageId),
-            metadata,
-        };
-    }
-    /**
-     * Loads the blocks of a Notion page and returns them as a string.
-     * @param blockId The ID of the block to load.
-     * @param numberOfTabs The number of tabs to use for indentation.
-     * @returns A string representing the loaded blocks.
-     */
-    async loadBlocks(blockId, numberOfTabs = 0) {
-        const resultLinesArr = [];
-        let currentBlockId = blockId;
-        while (currentBlockId) {
-            const response = await fetch(`${NOTION_BASE_URL}/blocks/${currentBlockId}/children`, {
-                method: "GET",
-                headers: this.headers,
-            });
-            const data = await response.json();
-            if (!response.ok) {
-                throw new Error(`Unable to fetch block: ${response.status} ${JSON.stringify(data)}`);
-            }
-            for (const result of data.results) {
-                const resultObj = result[result.type];
-                if (!resultObj.rich_text) {
-                    continue;
-                }
-                const curResultTextArr = [];
-                for (const richText of resultObj.rich_text) {
-                    if (richText.text) {
-                        curResultTextArr.push("\t".repeat(numberOfTabs) + richText.text.content);
-                    }
-                }
-                if (result.has_children) {
-                    const childrenText = await this.loadBlocks(result.id, numberOfTabs + 1);
-                    curResultTextArr.push(childrenText);
-                }
-                resultLinesArr.push(curResultTextArr.join("\n"));
-            }
-            currentBlockId = data.next_cursor;
-        }
-        return resultLinesArr.join("\n");
-    }
-}
-exports.NotionDBLoader = NotionDBLoader;

package/dist/document_loaders/web/notiondb.d.ts

@@ -1,56 +0,0 @@
-import { Document } from "@langchain/core/documents";
-import { BaseDocumentLoader } from "../base.js";
-/**
- * Interface representing the parameters for the NotionDBLoader class. It
- * includes the database ID, Notion integration token, Notion API version,
- * and page size limit.
- */
-export interface NotionDBLoaderParams {
-    databaseId: string;
-    notionIntegrationToken?: string;
-    notionApiVersion?: string;
-    pageSizeLimit?: number;
-}
-/**
- * @deprecated use the `NotionAPILoader` class instead.
- * @example
- * ```typescript
- * const loader = new NotionDBLoader({
- *   pageSizeLimit: 10,
- *   databaseId: "{databaseId}",
- *   notionIntegrationToken: "{notionIntegrationToken}",
- * });
- * const docs = await loader.load();
- * ```
- */
-export declare class NotionDBLoader extends BaseDocumentLoader implements NotionDBLoaderParams {
-    integrationToken: string;
-    databaseId: string;
-    notionApiVersion: string;
-    pageSizeLimit: number;
-    private headers;
-    constructor({ databaseId, notionApiVersion, notionIntegrationToken, pageSizeLimit, }: NotionDBLoaderParams);
-    /**
-     * Loads the documents from Notion based on the specified options.
-     * @returns An array of Document objects.
-     */
-    load(): Promise<Document[]>;
-    /**
-     * Retrieves the IDs of the pages in the Notion database.
-     * @returns An array of page IDs.
-     */
-    private retrievePageIds;
-    /**
-     * Loads a Notion page and returns it as a Document object.
-     * @param pageId The ID of the Notion page to load.
-     * @returns A Document object representing the loaded Notion page.
-     */
-    private loadPage;
-    /**
-     * Loads the blocks of a Notion page and returns them as a string.
-     * @param blockId The ID of the block to load.
-     * @param numberOfTabs The number of tabs to use for indentation.
-     * @returns A string representing the loaded blocks.
-     */
-    private loadBlocks;
-}

package/dist/document_loaders/web/notiondb.js

@@ -1,195 +0,0 @@
-import { getEnvironmentVariable } from "@langchain/core/utils/env";
-import { BaseDocumentLoader } from "../base.js";
-const NOTION_BASE_URL = "https://api.notion.com/v1";
-/**
- * @deprecated use the `NotionAPILoader` class instead.
- * @example
- * ```typescript
- * const loader = new NotionDBLoader({
- *   pageSizeLimit: 10,
- *   databaseId: "{databaseId}",
- *   notionIntegrationToken: "{notionIntegrationToken}",
- * });
- * const docs = await loader.load();
- * ```
- */
-export class NotionDBLoader extends BaseDocumentLoader {
-    constructor({ databaseId, notionApiVersion = "2022-06-28", notionIntegrationToken = getEnvironmentVariable("NOTION_INTEGRATION_TOKEN"), pageSizeLimit = 50, }) {
-        super();
-        Object.defineProperty(this, "integrationToken", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "databaseId", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "notionApiVersion", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "pageSizeLimit", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "headers", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: {}
-        });
-        if (!notionIntegrationToken) {
-            throw new Error("You must provide a Notion integration token.");
-        }
-        this.integrationToken = notionIntegrationToken;
-        this.pageSizeLimit = pageSizeLimit;
-        this.notionApiVersion = notionApiVersion;
-        this.databaseId = databaseId;
-        this.headers = {
-            Authorization: `Bearer ${this.integrationToken}`,
-            "Content-Type": "application/json",
-            "Notion-Version": notionApiVersion,
-        };
-    }
-    /**
-     * Loads the documents from Notion based on the specified options.
-     * @returns An array of Document objects.
-     */
-    async load() {
-        const pageIds = await this.retrievePageIds();
-        const documents = [];
-        for (const pageId of pageIds) {
-            documents.push(await this.loadPage(pageId));
-        }
-        return documents;
-    }
-    /**
-     * Retrieves the IDs of the pages in the Notion database.
-     * @returns An array of page IDs.
-     */
-    async retrievePageIds() {
-        const url = `${NOTION_BASE_URL}/databases/${this.databaseId}/query`;
-        const pageIds = [];
-        const query = {
-            page_size: this.pageSizeLimit,
-        };
-        let hasMore = true;
-        while (hasMore) {
-            const response = await fetch(url, {
-                method: "POST",
-                body: JSON.stringify(query),
-                headers: this.headers,
-            });
-            const data = await response.json();
-            if (!response.ok) {
-                throw new Error(`Failed to load data from Notion. Please check your integration token and database id.`);
-            }
-            const { results, has_more, next_cursor } = data;
-            pageIds.push(...(results?.map((page) => page.id) ?? []));
-            hasMore = has_more;
-            query.start_cursor = next_cursor;
-        }
-        return pageIds;
-    }
-    /**
-     * Loads a Notion page and returns it as a Document object.
-     * @param pageId The ID of the Notion page to load.
-     * @returns A Document object representing the loaded Notion page.
-     */
-    async loadPage(pageId) {
-        const url = `${NOTION_BASE_URL}/pages/${pageId}`;
-        const response = await fetch(url, { method: "GET", headers: this.headers });
-        const data = await response.json();
-        if (!response.ok) {
-            throw new Error(`Unable to fetch page: ${response.status} ${JSON.stringify(data)}`);
-        }
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const metadata = {};
-        const { properties } = data;
-        for (const key of Object.keys(properties)) {
-            const item = properties[key];
-            const itemType = item.type;
-            let value;
-            switch (itemType) {
-                case "rich_text":
-                    value =
-                        item?.rich_text && item?.rich_text.length > 0
-                            ? item?.rich_text[0].plain_text
-                            : null;
-                    break;
-                case "title":
-                    value =
-                        item?.title && item?.title.length > 0
-                            ? item?.title[0].plain_text
-                            : null;
-                    break;
-                case "multi_select":
-                    if (item?.multi_select && item?.multi_select.length > 0) {
-                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-                        value = item?.multi_select.map((el) => el.name);
-                    }
-                    break;
-                case "url":
-                    value = item?.url ? item.url : null;
-                    break;
-                default:
-                    break;
-            }
-            if (value) {
-                metadata[key.toLowerCase()] = value;
-            }
-        }
-        metadata.id = pageId;
-        return {
-            pageContent: await this.loadBlocks(pageId),
-            metadata,
-        };
-    }
-    /**
-     * Loads the blocks of a Notion page and returns them as a string.
-     * @param blockId The ID of the block to load.
-     * @param numberOfTabs The number of tabs to use for indentation.
-     * @returns A string representing the loaded blocks.
-     */
-    async loadBlocks(blockId, numberOfTabs = 0) {
-        const resultLinesArr = [];
-        let currentBlockId = blockId;
-        while (currentBlockId) {
-            const response = await fetch(`${NOTION_BASE_URL}/blocks/${currentBlockId}/children`, {
-                method: "GET",
-                headers: this.headers,
-            });
-            const data = await response.json();
-            if (!response.ok) {
-                throw new Error(`Unable to fetch block: ${response.status} ${JSON.stringify(data)}`);
-            }
-            for (const result of data.results) {
-                const resultObj = result[result.type];
-                if (!resultObj.rich_text) {
-                    continue;
-                }
-                const curResultTextArr = [];
-                for (const richText of resultObj.rich_text) {
-                    if (richText.text) {
-                        curResultTextArr.push("\t".repeat(numberOfTabs) + richText.text.content);
-                    }
-                }
-                if (result.has_children) {
-                    const childrenText = await this.loadBlocks(result.id, numberOfTabs + 1);
-                    curResultTextArr.push(childrenText);
-                }
-                resultLinesArr.push(curResultTextArr.join("\n"));
-            }
-            currentBlockId = data.next_cursor;
-        }
-        return resultLinesArr.join("\n");
-    }
-}

package/dist/document_loaders/web/pdf.cjs

@@ -1,140 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.WebPDFLoader = void 0;
-const documents_1 = require("@langchain/core/documents");
-const base_js_1 = require("../base.cjs");
-const document_js_1 = require("../../util/document.cjs");
-const entrypoint_deprecation_js_1 = require("../../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion020MigrationWarning)({
-    oldEntrypointName: "document_loaders/web/pdf",
-    newPackageName: "@langchain/community",
-});
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/pdf" instead. This entrypoint will be removed in 0.3.0.
- *
- * A document loader for loading data from PDFs.
- * @example
- * ```typescript
- * const loader = new WebPDFLoader(new Blob());
- * const docs = await loader.load();
- * console.log({ docs });
- * ```
- */
-class WebPDFLoader extends base_js_1.BaseDocumentLoader {
-    constructor(blob, { splitPages = true, pdfjs = PDFLoaderImports, parsedItemSeparator = "", } = {}) {
-        super();
-        Object.defineProperty(this, "blob", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "splitPages", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: true
-        });
-        Object.defineProperty(this, "pdfjs", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "parsedItemSeparator", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.blob = blob;
-        this.splitPages = splitPages ?? this.splitPages;
-        this.pdfjs = pdfjs;
-        this.parsedItemSeparator = parsedItemSeparator;
-    }
-    /**
-     * Loads the contents of the PDF as documents.
-     * @returns An array of Documents representing the retrieved data.
-     */
-    async load() {
-        const { getDocument, version } = await this.pdfjs();
-        const parsedPdf = await getDocument({
-            data: new Uint8Array(await this.blob.arrayBuffer()),
-            useWorkerFetch: false,
-            isEvalSupported: false,
-            useSystemFonts: true,
-        }).promise;
-        const meta = await parsedPdf.getMetadata().catch(() => null);
-        const documents = [];
-        for (let i = 1; i <= parsedPdf.numPages; i += 1) {
-            const page = await parsedPdf.getPage(i);
-            const content = await page.getTextContent();
-            if (content.items.length === 0) {
-                continue;
-            }
-            // Eliminate excessive newlines
-            // Source: https://github.com/albertcui/pdf-parse/blob/7086fc1cc9058545cdf41dd0646d6ae5832c7107/lib/pdf-parse.js#L16
-            let lastY;
-            const textItems = [];
-            for (const item of content.items) {
-                if ("str" in item) {
-                    if (lastY === item.transform[5] || !lastY) {
-                        textItems.push(item.str);
-                    }
-                    else {
-                        textItems.push(`\n${item.str}`);
-                    }
-                    // eslint-disable-next-line prefer-destructuring
-                    lastY = item.transform[5];
-                }
-            }
-            const text = textItems.join(this.parsedItemSeparator);
-            documents.push(new documents_1.Document({
-                pageContent: text,
-                metadata: {
-                    pdf: {
-                        version,
-                        info: meta?.info,
-                        metadata: meta?.metadata,
-                        totalPages: parsedPdf.numPages,
-                    },
-                    loc: {
-                        pageNumber: i,
-                    },
-                },
-            }));
-        }
-        if (this.splitPages) {
-            return documents;
-        }
-        if (documents.length === 0) {
-            return [];
-        }
-        return [
-            new documents_1.Document({
-                pageContent: (0, document_js_1.formatDocumentsAsString)(documents),
-                metadata: {
-                    pdf: {
-                        version,
-                        info: meta?.info,
-                        metadata: meta?.metadata,
-                        totalPages: parsedPdf.numPages,
-                    },
-                },
-            }),
-        ];
-        return documents;
-    }
-}
-exports.WebPDFLoader = WebPDFLoader;
-async function PDFLoaderImports() {
-    try {
-        const { default: mod } = await import("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js");
-        const { getDocument, version } = mod;
-        return { getDocument, version };
-    }
-    catch (e) {
-        console.error(e);
-        throw new Error("Failed to load pdf-parse. Please install it with eg. `npm install pdf-parse`.");
-    }
-}

package/dist/document_loaders/web/pdf.d.ts

@@ -1,35 +0,0 @@
-/// <reference path="../../../src/types/pdf-parse.d.ts" />
-import { Document } from "@langchain/core/documents";
-import { BaseDocumentLoader } from "../base.js";
-/**
- * @deprecated - Import from "@langchain/community/document_loaders/web/pdf" instead. This entrypoint will be removed in 0.3.0.
- *
- * A document loader for loading data from PDFs.
- * @example
- * ```typescript
- * const loader = new WebPDFLoader(new Blob());
- * const docs = await loader.load();
- * console.log({ docs });
- * ```
- */
-export declare class WebPDFLoader extends BaseDocumentLoader {
-    protected blob: Blob;
-    protected splitPages: boolean;
-    private pdfjs;
-    protected parsedItemSeparator: string;
-    constructor(blob: Blob, { splitPages, pdfjs, parsedItemSeparator, }?: {
-        splitPages?: boolean | undefined;
-        pdfjs?: typeof PDFLoaderImports | undefined;
-        parsedItemSeparator?: string | undefined;
-    });
-    /**
-     * Loads the contents of the PDF as documents.
-     * @returns An array of Documents representing the retrieved data.
-     */
-    load(): Promise<Document[]>;
-}
-declare function PDFLoaderImports(): Promise<{
-    getDocument: typeof import("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js").getDocument;
-    version: string;
-}>;
-export {};