langchain 0.2.9 → 0.2.11

package/dist/document_loaders/tests/multi_file.test.js

@@ -1,49 +0,0 @@
-import * as url from "node:url";
-import * as path from "node:path";
-import { test, expect } from "@jest/globals";
-import { MultiFileLoader } from "../fs/multi_file.js";
-import { CSVLoader } from "../fs/csv.js";
-import { PDFLoader } from "../fs/pdf.js";
-import { TextLoader } from "../fs/text.js";
-import { JSONLoader } from "../fs/json.js";
-import { UnknownHandling } from "../fs/directory.js";
-test("Test MultiFileLoader", async () => {
-    const baseDirectory = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data");
-    const filePaths = [
-        path.resolve(baseDirectory, "1706.03762.pdf"),
-        path.resolve(baseDirectory, "Jacob_Lee_Resume_2023.pdf"),
-        path.resolve(baseDirectory, "Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.csv"),
-        path.resolve(baseDirectory, "Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.json"),
-        path.resolve(baseDirectory, "complex.json"),
-        path.resolve(baseDirectory, "example.txt"),
-        path.resolve(baseDirectory, "example_separator.csv"),
-    ];
-    const loader = new MultiFileLoader(filePaths, {
-        ".csv": (p) => {
-            if (p.includes("separator.csv")) {
-                return new CSVLoader(p, { column: "html", separator: "｜" });
-            }
-            return new CSVLoader(p, "html");
-        },
-        ".pdf": (p) => new PDFLoader(p),
-        ".txt": (p) => new TextLoader(p),
-        ".json": (p) => new JSONLoader(p),
-    }, UnknownHandling.Ignore);
-    const docs = await loader.load();
-    expect(docs.length).toBe(123);
-    const expectedSources = [
-        // PDF
-        ...Array.from({ length: 15 }, (_) => path.resolve(baseDirectory, "1706.03762.pdf")),
-        path.resolve(baseDirectory, "Jacob_Lee_Resume_2023.pdf"),
-        // CSV
-        ...Array.from({ length: 32 }, (_) => path.resolve(baseDirectory, "Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.csv")),
-        // JSON
-        ...Array.from({ length: 32 }, (_) => path.resolve(baseDirectory, "Star_Wars_The_Clone_Wars_S06E07_Crisis_at_the_Heart.json")),
-        ...Array.from({ length: 10 }, (_) => path.resolve(baseDirectory, "complex.json")),
-        // TXT
-        path.resolve(baseDirectory, "example.txt"),
-        // CSV
-        ...Array.from({ length: 32 }, (_) => path.resolve(baseDirectory, "example_separator.csv")),
-    ];
-    expect(docs.map((d) => d.metadata.source).sort()).toEqual(expectedSources);
-});

package/dist/document_loaders/tests/notion.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/document_loaders/tests/notion.test.js

@@ -1,11 +0,0 @@
-import { test, expect } from "@jest/globals";
-import * as url from "node:url";
-import * as path from "node:path";
-import { NotionLoader } from "../fs/notion.js";
-test("Test Notion Loader", async () => {
-    const directoryPath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/notion");
-    const loader = new NotionLoader(directoryPath);
-    const docs = await loader.load();
-    expect(docs.length).toBe(1);
-    expect(docs[0].pageContent).toContain("Testing the notion markdownloader");
-});

package/dist/document_loaders/tests/notionapi.int.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/document_loaders/tests/notionapi.int.test.js

@@ -1,80 +0,0 @@
-/* eslint-disable no-process-env */
-/* eslint-disable @typescript-eslint/no-non-null-assertion */
-import { test } from "@jest/globals";
-import { NotionAPILoader } from "../web/notionapi.js";
-test.skip("Test Notion API Loader Page", async () => {
-    const loader = new NotionAPILoader({
-        clientOptions: {
-            auth: process.env.NOTION_INTEGRATION_TOKEN,
-        },
-        id: process.env.NOTION_PAGE_ID ?? "",
-        onDocumentLoaded: (current, total, currentTitle, rootTitle) => {
-            console.log(`Loaded ${currentTitle} in ${rootTitle}:  (${current}/${total})`);
-        },
-    });
-    const docs = await loader.load();
-    const titles = docs.map((doc) => doc.metadata.properties._title);
-    console.log("Titles:", titles);
-    console.log(`Loaded ${docs.length} pages`);
-});
-test.skip("Test Notion API Loader Database", async () => {
-    const loader = new NotionAPILoader({
-        clientOptions: {
-            auth: process.env.NOTION_INTEGRATION_TOKEN,
-        },
-        id: process.env.NOTION_DATABASE_ID ?? "",
-        onDocumentLoaded: (current, total, currentTitle, rootTitle) => {
-            console.log(`Loaded ${currentTitle} in ${rootTitle}:  (${current}/${total})`);
-        },
-    });
-    const docs = await loader.load();
-    const titles = docs.map((doc) => doc.metadata.properties._title);
-    console.log("Titles:", titles);
-    console.log(`Loaded ${docs.length} pages from the database`);
-});
-test.skip("Test Notion API Loader onDocumentLoad", async () => {
-    const onDocumentLoadedCheck = [];
-    const loader = new NotionAPILoader({
-        clientOptions: {
-            auth: process.env.NOTION_INTEGRATION_TOKEN,
-        },
-        id: process.env.NOTION_DATABASE_ID ?? "",
-        onDocumentLoaded: (current, total, currentTitle, rootTitle) => {
-            onDocumentLoadedCheck.push(`Loaded ${currentTitle} from ${rootTitle}: (${current}/${total})`);
-        },
-        propertiesAsHeader: true,
-    });
-    await loader.load();
-    expect(onDocumentLoadedCheck.length).toBe(3);
-    console.log(onDocumentLoadedCheck);
-});
-test.skip("Test docs with empty database page content", async () => {
-    const onDocumentLoadedCheck = [];
-    const loader = new NotionAPILoader({
-        clientOptions: {
-            auth: process.env.NOTION_INTEGRATION_TOKEN,
-        },
-        id: process.env.NOTION_DATABASE_ID ?? "",
-        onDocumentLoaded: (current, total, currentTitle, rootTitle) => {
-            onDocumentLoadedCheck.push(`Loaded ${currentTitle} from ${rootTitle}: (${current}/${total})`);
-        },
-    });
-    const docs = await loader.load();
-    expect(docs.length).toBe(0);
-});
-test.skip("Test docs with empty database page content and propertiesAsHeader enabled", async () => {
-    const onDocumentLoadedCheck = [];
-    const loader = new NotionAPILoader({
-        clientOptions: {
-            auth: process.env.NOTION_INTEGRATION_TOKEN,
-        },
-        id: process.env.NOTION_DATABASE_ID ?? "",
-        onDocumentLoaded: (current, total, currentTitle, rootTitle) => {
-            onDocumentLoadedCheck.push(`Loaded ${currentTitle} from ${rootTitle}: (${current}/${total})`);
-        },
-        propertiesAsHeader: true,
-    });
-    const docs = await loader.load();
-    expect(docs.length).toBe(3);
-    console.log(docs);
-});

package/dist/document_loaders/tests/notionapi.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/document_loaders/tests/notionapi.test.js

@@ -1,84 +0,0 @@
-/* eslint-disable no-process-env */
-/* eslint-disable @typescript-eslint/no-non-null-assertion */
-import * as fs from "node:fs";
-import * as url from "node:url";
-import * as path from "node:path";
-import { test } from "@jest/globals";
-import { NotionAPILoader, } from "../web/notionapi.js";
-test("Properties Parser", async () => {
-    const loader = new NotionAPILoader({
-        clientOptions: {
-            auth: process.env.NOTION_INTEGRATION_TOKEN,
-        },
-        id: process.env.NOTION_PAGE_ID ?? "",
-        onDocumentLoaded: (current, total, currentTitle) => {
-            console.log(`Loaded Page: ${currentTitle} (${current}/${total})`);
-        },
-    });
-    const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/notion_api/notion_page_response.json");
-    const pageDetails = JSON.parse(fs.readFileSync(filePath).toString());
-    // Accessing private class method
-    // eslint-disable-next-line dot-notation
-    const contents = loader["parsePageProperties"](pageDetails);
-    expect(contents.File).toBe('["MetaLumna Logo Square.png"]');
-    expect(contents.Number).toBe("1234");
-    expect(contents.Rollup).toBe("Unsupported type: rollup");
-    expect(contents.Status).toBe("In progress");
-    expect(contents["Multi-select"]).toBe('["Red", "Green", "Blue"]');
-    expect(contents.Select).toBe("Magenta");
-    expect(contents["Last edited time"]).toBe("2023-09-27T09:11:00.000Z");
-    expect(contents.Text).toBe("This is just some **text** with some formatting and new lines.\n" +
-        "\n" +
-        "_italic_\n" +
-        "**bold**\n" +
-        "`inline code`\n" +
-        "~~strike through~~");
-    expect(contents["Last edited by"]).toBe('["user", "c9b34ba3-5b62-4aa9-aae2-ed6024ffb0fd"]');
-    expect(contents["Related Example Database"]).toBe('["7d9d1b96-34fa-4da3-b70d-9f71a75e1291", "7a1c25f2-d8a7-46d9-a2da-c9e39e018a56", "67662914-f22b-47fd-98aa-19988c40c77d"]');
-    expect(contents["Created by"]).toBe('["user", "c9b34ba3-5b62-4aa9-aae2-ed6024ffb0fd"]');
-    expect(contents.Checkbox).toBe("true");
-    expect(contents.Formula).toBe("Unsupported type: formula");
-    expect(contents.Phone).toBe("555-1234");
-    expect(contents.Email).toBe("skarard@gmail.com");
-    expect(contents.URL).toBe("https://example.com");
-    expect(contents["Created time"]).toBe("2023-06-14T10:29:00.000Z");
-    expect(contents.ID).toBe("2");
-    expect(contents.Date).toBe("2023-08-09T00:00:00.000+01:00 - 2023-08-09T00:00:00.000+01:00");
-    expect(contents.Person).toBe('[["user", "c9b34ba3-5b62-4aa9-aae2-ed6024ffb0fd"], ["user", "9d6b0c60-efdd-48d1-b63e-027d9b7d66a0"]]');
-    expect(contents.Name).toBe("An example ~~page~~ in a database");
-    expect(contents._title).toBe("An example ~~page~~ in a database");
-});
-test("Get Title (page)", async () => {
-    const loader = new NotionAPILoader({
-        clientOptions: {
-            auth: process.env.NOTION_INTEGRATION_TOKEN,
-        },
-        id: process.env.NOTION_PAGE_ID ?? "",
-        onDocumentLoaded: (current, total, currentTitle) => {
-            console.log(`Loaded Page: ${currentTitle} (${current}/${total})`);
-        },
-    });
-    const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/notion_api/notion_page_response.json");
-    const pageDetails = JSON.parse(fs.readFileSync(filePath).toString());
-    // Accessing private class method
-    // eslint-disable-next-line dot-notation
-    const title = loader["getTitle"](pageDetails);
-    expect(title).toBe("An example ~~page~~ in a database");
-});
-test("Get Title (database)", async () => {
-    const loader = new NotionAPILoader({
-        clientOptions: {
-            auth: process.env.NOTION_INTEGRATION_TOKEN,
-        },
-        id: process.env.NOTION_PAGE_ID ?? "",
-        onDocumentLoaded: (current, total, currentTitle) => {
-            console.log(`Loaded Page: ${currentTitle} (${current}/${total})`);
-        },
-    });
-    const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/notion_api/notion_database_response.json");
-    const dbDetails = JSON.parse(fs.readFileSync(filePath).toString());
-    // Accessing private class method
-    // eslint-disable-next-line dot-notation
-    const title = loader["getTitle"](dbDetails);
-    expect(title).toBe("Example ~~_**Database**_~~");
-});

package/dist/document_loaders/tests/notiondb.int.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/document_loaders/tests/notiondb.int.test.js

@@ -1,13 +0,0 @@
-/* eslint-disable no-process-env */
-/* eslint-disable @typescript-eslint/no-non-null-assertion */
-import { test } from "@jest/globals";
-import { NotionDBLoader } from "../web/notiondb.js";
-test.skip("Test NotionDBLoader", async () => {
-    const loader = new NotionDBLoader({
-        pageSizeLimit: 10,
-        notionApiVersion: "2022-06-28",
-        databaseId: process.env.NOTION_DATABASE_ID,
-    });
-    const documents = await loader.load();
-    console.log({ documents });
-});

package/dist/document_loaders/tests/obsidian.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/document_loaders/tests/obsidian.test.js

@@ -1,119 +0,0 @@
-import { test, expect } from "@jest/globals";
-import * as url from "node:url";
-import * as path from "node:path";
-import { ObsidianLoader } from "../fs/obsidian.js";
-const STANDARD_METADATA_FIELDS = [
-    "created",
-    "path",
-    "source",
-    "lastAccessed",
-    "lastModified",
-];
-const FRONTMATTER_FIELDS = [
-    "aBool",
-    "aFloat",
-    "anInt",
-    "anArray",
-    "aString",
-    "aDict",
-    "tags",
-];
-const DATAVIEW_FIELDS = ["dataview1", "dataview2", "dataview3"];
-const directoryPath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/obsidian");
-let docs;
-beforeAll(async () => {
-    const loader = new ObsidianLoader(directoryPath);
-    docs = await loader.load();
-});
-test("Test page content is loaded", async () => {
-    expect(docs.length).toBe(5);
-    docs.forEach((doc) => {
-        expect(doc.pageContent).toBeTruthy();
-    });
-});
-test("Test no additional metadata is collected if collectMetadata is false", async () => {
-    const noMetadataLoader = new ObsidianLoader(directoryPath, {
-        collectMetadata: false,
-    });
-    const noMetadataDocs = await noMetadataLoader.load();
-    expect(noMetadataDocs.length).toBe(5);
-    expect(noMetadataDocs.every((doc) => Object.keys(doc.metadata).length ===
-        Object.keys(STANDARD_METADATA_FIELDS).length &&
-        Object.keys(doc.metadata).every((key) => STANDARD_METADATA_FIELDS.includes(key)))).toBe(true);
-});
-test("Test docs without frontmatter still have basic metadata", async () => {
-    const doc = docs.find((doc) => doc.metadata.source === "no_metadata.md");
-    if (!doc) {
-        fail("'no_metadata.md' not found.");
-    }
-    expect(Object.keys(doc.metadata).every((key) => STANDARD_METADATA_FIELDS.includes(key))).toBe(true);
-});
-test("Test standard frontmatter fields are loaded", async () => {
-    const doc = docs.find((doc) => doc.metadata.source === "frontmatter.md");
-    if (!doc) {
-        fail("'frontmatter.md' not found.");
-    }
-    expect(Object.keys(doc.metadata)).toEqual(expect.arrayContaining(STANDARD_METADATA_FIELDS.concat("tags")));
-    const tagsSet = new Set(doc.metadata.tags?.split(","));
-    expect(tagsSet.has("journal/entry")).toBe(true);
-    expect(tagsSet.has("obsidian")).toBe(true);
-});
-test("Test a doc with non-yaml frontmatter still have basic metadata", async () => {
-    const doc = docs.find((doc) => doc.metadata.source === "bad_frontmatter.md");
-    if (!doc) {
-        fail("'bad_frontmatter.md' not found.");
-    }
-    expect(Object.keys(doc.metadata).every((key) => STANDARD_METADATA_FIELDS.includes(key))).toBe(true);
-});
-test("Test a doc with frontmatter and tags/dataview tags are all added to metadata", () => {
-    const doc = docs.find((doc) => doc.metadata.source === "tags_and_frontmatter.md");
-    if (!doc) {
-        fail("'tags_and_frontmatter.md' not found.");
-    }
-    const expectedFields = [
-        ...STANDARD_METADATA_FIELDS,
-        ...FRONTMATTER_FIELDS,
-        ...DATAVIEW_FIELDS,
-    ];
-    expect(Object.keys(doc.metadata)).toEqual(expect.arrayContaining(expectedFields));
-});
-test("Test float metadata is loaded correctly", () => {
-    const doc = docs.find((doc) => doc.metadata.source === "tags_and_frontmatter.md");
-    if (!doc) {
-        fail("Document 'tags_and_frontmatter.md' not found.");
-        return;
-    }
-    expect(doc.metadata.aFloat).toBe(13.12345);
-});
-test("Test int metadata is loaded correctly", () => {
-    const doc = docs.find((doc) => doc.metadata.source === "tags_and_frontmatter.md");
-    if (!doc) {
-        fail("Document 'tags_and_frontmatter.md' not found.");
-        return;
-    }
-    expect(doc.metadata.anInt).toBe(15);
-});
-test("Test string metadata is loaded correctly", () => {
-    const doc = docs.find((doc) => doc.metadata.source === "tags_and_frontmatter.md");
-    if (!doc) {
-        fail("Document 'tags_and_frontmatter.md' not found.");
-        return;
-    }
-    expect(doc.metadata.aString).toBe("string value");
-});
-test("Test array metadata is loaded as a string", () => {
-    const doc = docs.find((doc) => doc.metadata.source === "tags_and_frontmatter.md");
-    if (!doc) {
-        fail("Document 'tags_and_frontmatter.md' not found.");
-        return;
-    }
-    expect(doc.metadata.anArray).toBe('["one","two","three"]');
-});
-test("Test dict metadata is stored as a string", () => {
-    const doc = docs.find((doc) => doc.metadata.source === "tags_and_frontmatter.md");
-    if (!doc) {
-        fail("Document 'tags_and_frontmatter.md' not found.");
-        return;
-    }
-    expect(doc.metadata.aDict).toBe('{"dictId1":"58417","dictId2":1500}');
-});

package/dist/document_loaders/tests/pdf-blob.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/document_loaders/tests/pdf-blob.test.js

@@ -1,44 +0,0 @@
-import { test, expect } from "@jest/globals";
-import * as url from "node:url";
-import * as path from "node:path";
-import * as fs from "node:fs/promises";
-import { PDFLoader } from "../fs/pdf.js";
-test("Test PDF loader from blob", async () => {
-    const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/1706.03762.pdf");
-    const loader = new PDFLoader(new Blob([await fs.readFile(filePath)], {
-        type: "application/pdf",
-    }));
-    const docs = await loader.load();
-    expect(docs.length).toBe(15);
-    expect(docs[0].pageContent).toContain("Attention Is All You Need");
-    expect(docs[0].metadata).toMatchInlineSnapshot(`
-    {
-      "blobType": "application/pdf",
-      "loc": {
-        "pageNumber": 1,
-      },
-      "pdf": {
-        "info": {
-          "Author": "",
-          "CreationDate": "D:20171207010315Z",
-          "Creator": "LaTeX with hyperref package",
-          "IsAcroFormPresent": false,
-          "IsXFAPresent": false,
-          "Keywords": "",
-          "ModDate": "D:20171207010315Z",
-          "PDFFormatVersion": "1.5",
-          "Producer": "pdfTeX-1.40.17",
-          "Subject": "",
-          "Title": "",
-          "Trapped": {
-            "name": "False",
-          },
-        },
-        "metadata": null,
-        "totalPages": 15,
-        "version": "1.10.100",
-      },
-      "source": "blob",
-    }
-  `);
-});

package/dist/document_loaders/tests/pdf.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/document_loaders/tests/pdf.test.js

@@ -1,25 +0,0 @@
-import { test, expect } from "@jest/globals";
-import * as url from "node:url";
-import * as path from "node:path";
-import { PDFLoader } from "../fs/pdf.js";
-test("Test PDF loader from file", async () => {
-    const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/1706.03762.pdf");
-    const loader = new PDFLoader(filePath);
-    const docs = await loader.load();
-    expect(docs.length).toBe(15);
-    expect(docs[0].pageContent).toContain("Attention Is All You Need");
-});
-test("Test PDF loader from file to single document", async () => {
-    const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/1706.03762.pdf");
-    const loader = new PDFLoader(filePath, { splitPages: false });
-    const docs = await loader.load();
-    expect(docs.length).toBe(1);
-    expect(docs[0].pageContent).toContain("Attention Is All You Need");
-});
-test("Test PDF loader should not create documents with excessive newlines", async () => {
-    const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/Jacob_Lee_Resume_2023.pdf");
-    const loader = new PDFLoader(filePath, { splitPages: false });
-    const docs = await loader.load();
-    expect(docs.length).toBe(1);
-    expect(docs[0].pageContent.split("\n").length).toBeLessThan(100);
-});

package/dist/document_loaders/tests/playwright_web.int.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/document_loaders/tests/playwright_web.int.test.js

@@ -1,27 +0,0 @@
-import { expect, test } from "@jest/globals";
-import { PlaywrightWebBaseLoader } from "../web/playwright.js";
-test("Test playwright web scraper loader", async () => {
-    const loader = new PlaywrightWebBaseLoader("https://www.google.com/");
-    const result = await loader.load();
-    expect(result).toBeDefined();
-    expect(result.length).toBe(1);
-}, 20000);
-test("Test playwright web scraper loader with evaluate options", async () => {
-    let nrTimesCalled = 0;
-    const loader = new PlaywrightWebBaseLoader("https://www.google.com/", {
-        launchOptions: {
-            headless: true,
-        },
-        gotoOptions: {
-            waitUntil: "domcontentloaded",
-        },
-        async evaluate(page) {
-            nrTimesCalled += 1;
-            return page.content();
-        },
-    });
-    const result = await loader.load();
-    expect(nrTimesCalled).toBe(1);
-    expect(result).toBeDefined();
-    expect(result.length).toBe(1);
-}, 20000);

package/dist/document_loaders/tests/pptx.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/document_loaders/tests/pptx.test.js

@@ -1,17 +0,0 @@
-import { expect, test } from "@jest/globals";
-import * as url from "node:url";
-import * as path from "node:path";
-import { PPTXLoader } from "../fs/pptx.js";
-test.skip("Test PowerPoint loader from file", async () => {
-    const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/theikuntest.pptx");
-    const loader = new PPTXLoader(filePath);
-    const docs = await loader.load();
-    expect(docs.length).toBe(1); // not much text in the example
-    expect(docs[0].pageContent).toContain("UTSC IS THE BEST");
-});
-test.skip("Test PowerPoint loader from empty powerpoint file", async () => {
-    const filePath = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "./example_data/emptyfile.pptx");
-    const loader = new PPTXLoader(filePath);
-    const docs = await loader.load();
-    expect(docs.length).toBe(0); // not much text in the example
-});

package/dist/document_loaders/tests/puppeteer.int.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/document_loaders/tests/puppeteer.int.test.js

@@ -1,47 +0,0 @@
-import { expect, test } from "@jest/globals";
-import { PuppeteerWebBaseLoader } from "../web/puppeteer.js";
-test.skip("Test puppeteer web scraper loader", async () => {
-    const loader = new PuppeteerWebBaseLoader("https://www.google.com/");
-    const result = await loader.load();
-    expect(result).toBeDefined();
-    expect(result.length).toBe(1);
-}, 20000);
-test.skip("Test puppeteer web scraper loader with evaluate options", async () => {
-    let nrTimesCalled = 0;
-    const loader = new PuppeteerWebBaseLoader("https://www.google.com/", {
-        launchOptions: {
-            headless: true,
-            ignoreDefaultArgs: ["--disable-extensions"],
-        },
-        gotoOptions: {
-            waitUntil: "domcontentloaded",
-        },
-        async evaluate(page) {
-            nrTimesCalled += 1;
-            return page.evaluate(() => document.body.innerHTML);
-        },
-    });
-    const result = await loader.load();
-    expect(nrTimesCalled).toBe(1);
-    expect(result).toBeDefined();
-    expect(result.length).toBe(1);
-}, 20000);
-test.skip("Test puppeteer web scraper can screenshot page", async () => {
-    const loader = new PuppeteerWebBaseLoader("https://langchain.com/", {
-        launchOptions: {
-            headless: true,
-            ignoreDefaultArgs: ["--disable-extensions"],
-        },
-        gotoOptions: {
-            waitUntil: "domcontentloaded",
-        },
-    });
-    const screenshotDocument = await loader.screenshot();
-    expect(screenshotDocument.metadata.source).toBe("https://langchain.com/");
-    // verify screenshotDocument.pageContent is a base64 encoded string
-    expect(screenshotDocument.pageContent).toMatch(/^([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{4}|[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)$/);
-    // Uncomment if you want to write the screenshot to a file
-    // await fs.writeFile("langchain.png", screenshotDocument.pageContent, {
-    //   encoding: "base64",
-    // });
-}, 20000);

package/dist/document_loaders/tests/recursive_url.int.test.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/document_loaders/tests/recursive_url.int.test.js

@@ -1,64 +0,0 @@
-/* eslint-disable no-process-env */
-/* eslint-disable @typescript-eslint/no-non-null-assertion */
-import { test } from "@jest/globals";
-import { compile } from "html-to-text";
-import { RecursiveUrlLoader } from "../web/recursive_url.js";
-describe("RecursiveUrlLoader", () => {
-    test("loading valid url", async () => {
-        const url = "https://js.langchain.com/v0.2/docs/introduction";
-        const compiledConvert = compile({ wordwrap: 130 }); // returns (input: string) => string;
-        const loader = new RecursiveUrlLoader(url, {
-            extractor: compiledConvert,
-            maxDepth: 1,
-            excludeDirs: ["https://js.langchain.com/docs/api/"],
-        });
-        const docs = await loader.load();
-        expect(docs.length).toBeGreaterThan(1);
-        expect(docs[0].pageContent).toContain("LangChain");
-    });
-    test("loading single page", async () => {
-        const url = "https://js.langchain.com/v0.2/docs/introduction";
-        const compiledConvert = compile({ wordwrap: 130 }); // returns (input: string) => string;
-        const loader = new RecursiveUrlLoader(url, {
-            extractor: compiledConvert,
-            maxDepth: 0,
-            excludeDirs: ["https://js.langchain.com/docs/api/"],
-        });
-        const docs = await loader.load();
-        expect(docs.length).toBe(1);
-        expect(docs[0].pageContent).toContain("LangChain");
-    });
-    test("loading invalid url", async () => {
-        const url = "https://this.url.is.invalid/this/is/a/test";
-        const loader = new RecursiveUrlLoader(url, {
-            maxDepth: 1,
-            preventOutside: true,
-        });
-        const docs = await loader.load();
-        expect(docs.length).toBe(0);
-    });
-    test("excludeDirs works for top-level calls", async () => {
-        const url = "https://js.langchain.com/docs/";
-        const compiledConvert = compile({ wordwrap: 130 });
-        const excludeDir = "https://js.langchain.com/docs/api/";
-        const loader = new RecursiveUrlLoader(url, {
-            extractor: compiledConvert,
-            maxDepth: 1,
-            excludeDirs: [excludeDir],
-        });
-        const docs = await loader.load();
-        expect(docs.some((doc) => doc.metadata.source.startsWith(excludeDir))).toBe(false);
-    });
-    test("excludeDirs works for recursive calls", async () => {
-        const url = "https://js.langchain.com/docs/";
-        const compiledConvert = compile({ wordwrap: 130 });
-        const excludeDir = "https://js.langchain.com/docs/api/";
-        const loader = new RecursiveUrlLoader(url, {
-            extractor: compiledConvert,
-            maxDepth: 2,
-            excludeDirs: [excludeDir],
-        });
-        const docs = await loader.load();
-        expect(docs.some((doc) => doc.metadata.source.startsWith(excludeDir))).toBe(false);
-    });
-});

package/dist/document_loaders/tests/s3.int.test.d.ts

@@ -1 +0,0 @@
-export {};