langchain 0.0.131 → 0.0.132

package/dist/document_loaders/fs/unstructured.d.ts

@@ -5,6 +5,10 @@ import type { readFile as ReadFileT } from "node:fs/promises";
 import { DirectoryLoader, UnknownHandling } from "./directory.js";
 import { Document } from "../../document.js";
 import { BaseDocumentLoader } from "../base.js";
+/**
+ * Represents an element returned by the Unstructured API. It has
+ * properties for the element type, text content, and metadata.
+ */
 type Element = {
     type: string;
     text: string;
@@ -12,7 +16,15 @@ type Element = {
         [key: string]: unknown;
     };
 };
+/**
+ * Represents the available strategies for the UnstructuredLoader. It can
+ * be one of "hi_res", "fast", "ocr_only", or "auto".
+ */
 export type UnstructuredLoaderStrategy = "hi_res" | "fast" | "ocr_only" | "auto";
+/**
+ * Represents a string value with autocomplete suggestions. It is used for
+ * the `strategy` property in the UnstructuredLoaderOptions.
+ */
 type StringWithAutocomplete<T> = T | (string & Record<never, never>);
 export type UnstructuredLoaderOptions = {
     apiKey?: string;
@@ -28,6 +40,14 @@ type UnstructuredDirectoryLoaderOptions = UnstructuredLoaderOptions & {
     recursive?: boolean;
     unknown?: UnknownHandling;
 };
+/**
+ * A document loader that uses the Unstructured API to load unstructured
+ * documents. It supports both the new syntax with options object and the
+ * legacy syntax for backward compatibility. The load() method sends a
+ * partitioning request to the Unstructured API and retrieves the
+ * partitioned elements. It creates a Document instance for each element
+ * and returns an array of Document instances.
+ */
 export declare class UnstructuredLoader extends BaseDocumentLoader {
     filePath: string;
     private apiUrl;
@@ -46,6 +66,12 @@ export declare class UnstructuredLoader extends BaseDocumentLoader {
         basename: typeof BasenameT;
     }>;
 }
+/**
+ * A document loader that loads unstructured documents from a directory
+ * using the UnstructuredLoader. It creates a UnstructuredLoader instance
+ * for each supported file type and passes it to the DirectoryLoader
+ * constructor.
+ */
 export declare class UnstructuredDirectoryLoader extends DirectoryLoader {
     constructor(directoryPathOrLegacyApiUrl: string, optionsOrLegacyDirectoryPath: UnstructuredDirectoryLoaderOptions | string, legacyOptionRecursive?: boolean, legacyOptionUnknown?: UnknownHandling);
 }

package/dist/document_loaders/fs/unstructured.js

@@ -23,6 +23,14 @@ const UNSTRUCTURED_API_FILETYPES = [
     ".odt",
     ".epub",
 ];
+/**
+ * A document loader that uses the Unstructured API to load unstructured
+ * documents. It supports both the new syntax with options object and the
+ * legacy syntax for backward compatibility. The load() method sends a
+ * partitioning request to the Unstructured API and retrieves the
+ * partitioned elements. It creates a Document instance for each element
+ * and returns an array of Document instances.
+ */
 export class UnstructuredLoader extends BaseDocumentLoader {
     constructor(filePathOrLegacyApiUrl, optionsOrLegacyFilePath = {}) {
         super();
@@ -171,6 +179,12 @@ export class UnstructuredLoader extends BaseDocumentLoader {
         }
     }
 }
+/**
+ * A document loader that loads unstructured documents from a directory
+ * using the UnstructuredLoader. It creates a UnstructuredLoader instance
+ * for each supported file type and passes it to the DirectoryLoader
+ * constructor.
+ */
 export class UnstructuredDirectoryLoader extends DirectoryLoader {
     constructor(directoryPathOrLegacyApiUrl, optionsOrLegacyDirectoryPath, legacyOptionRecursive = true, legacyOptionUnknown = UnknownHandling.Warn) {
         let directoryPath;

package/dist/document_loaders/web/apify_dataset.cjs

@@ -4,6 +4,11 @@ exports.ApifyDatasetLoader = void 0;
 const apify_client_1 = require("apify-client");
 const base_js_1 = require("../base.cjs");
 const env_js_1 = require("../../util/env.cjs");
+/**
+ * A class that extends the BaseDocumentLoader and implements the
+ * DocumentLoader interface. It represents a document loader that loads
+ * documents from an Apify dataset.
+ */
 class ApifyDatasetLoader extends base_js_1.BaseDocumentLoader {
     constructor(datasetId, config) {
         super();
@@ -36,6 +41,12 @@ class ApifyDatasetLoader extends base_js_1.BaseDocumentLoader {
     static _getApifyApiToken(config) {
         return config?.token ?? (0, env_js_1.getEnvironmentVariable)("APIFY_API_TOKEN");
     }
+    /**
+     * Retrieves the dataset items from the Apify platform and applies the
+     * datasetMappingFunction to each item to create an array of Document
+     * instances.
+     * @returns An array of Document instances.
+     */
     async load() {
         const datasetItems = (await this.apifyClient.dataset(this.datasetId).listItems({ clean: true })).items;
         return datasetItems.map(this.datasetMappingFunction);

package/dist/document_loaders/web/apify_dataset.d.ts

@@ -1,7 +1,16 @@
 import { ApifyClient, ApifyClientOptions, ActorCallOptions, TaskCallOptions } from "apify-client";
 import { Document } from "../../document.js";
 import { BaseDocumentLoader, DocumentLoader } from "../base.js";
+/**
+ * A type that represents a function that takes a single object (an Apify
+ * dataset item) and converts it to an instance of the Document class.
+ */
 export type ApifyDatasetMappingFunction = (item: Record<string | number, unknown>) => Document;
+/**
+ * A class that extends the BaseDocumentLoader and implements the
+ * DocumentLoader interface. It represents a document loader that loads
+ * documents from an Apify dataset.
+ */
 export declare class ApifyDatasetLoader extends BaseDocumentLoader implements DocumentLoader {
     protected apifyClient: ApifyClient;
     protected datasetId: string;
@@ -11,6 +20,12 @@ export declare class ApifyDatasetLoader extends BaseDocumentLoader implements Do
         clientOptions?: ApifyClientOptions;
     });
     private static _getApifyApiToken;
+    /**
+     * Retrieves the dataset items from the Apify platform and applies the
+     * datasetMappingFunction to each item to create an array of Document
+     * instances.
+     * @returns An array of Document instances.
+     */
     load(): Promise<Document[]>;
     /**
      * Create an ApifyDatasetLoader by calling an Actor on the Apify platform and waiting for its results to be ready.

package/dist/document_loaders/web/apify_dataset.js

@@ -1,6 +1,11 @@
 import { ApifyClient, } from "apify-client";
 import { BaseDocumentLoader } from "../base.js";
 import { getEnvironmentVariable } from "../../util/env.js";
+/**
+ * A class that extends the BaseDocumentLoader and implements the
+ * DocumentLoader interface. It represents a document loader that loads
+ * documents from an Apify dataset.
+ */
 export class ApifyDatasetLoader extends BaseDocumentLoader {
     constructor(datasetId, config) {
         super();
@@ -33,6 +38,12 @@ export class ApifyDatasetLoader extends BaseDocumentLoader {
     static _getApifyApiToken(config) {
         return config?.token ?? getEnvironmentVariable("APIFY_API_TOKEN");
     }
+    /**
+     * Retrieves the dataset items from the Apify platform and applies the
+     * datasetMappingFunction to each item to create an array of Document
+     * instances.
+     * @returns An array of Document instances.
+     */
     async load() {
         const datasetItems = (await this.apifyClient.dataset(this.datasetId).listItems({ clean: true })).items;
         return datasetItems.map(this.datasetMappingFunction);

package/dist/document_loaders/web/azure_blob_storage_container.cjs

@@ -4,6 +4,10 @@ exports.AzureBlobStorageContainerLoader = void 0;
 const storage_blob_1 = require("@azure/storage-blob");
 const azure_blob_storage_file_js_1 = require("./azure_blob_storage_file.cjs");
 const base_js_1 = require("../base.cjs");
+/**
+ * Class representing a document loader that loads documents from an Azure
+ * Blob Storage container. It extends the BaseDocumentLoader class.
+ */
 class AzureBlobStorageContainerLoader extends base_js_1.BaseDocumentLoader {
     constructor({ azureConfig, unstructuredConfig, }) {
         super();
@@ -29,6 +33,15 @@ class AzureBlobStorageContainerLoader extends base_js_1.BaseDocumentLoader {
         this.container = azureConfig.container;
         this.unstructuredConfig = unstructuredConfig;
     }
+    /**
+     * Method to load documents from an Azure Blob Storage container. It
+     * creates a BlobServiceClient using the connection string, gets the
+     * container client using the container name, and iterates over the blobs
+     * in the container. For each blob, it creates an instance of
+     * AzureBlobStorageFileLoader and loads the documents using the loader.
+     * The loaded documents are concatenated to the docs array and returned.
+     * @returns An array of loaded documents.
+     */
     async load() {
         const blobServiceClient = storage_blob_1.BlobServiceClient.fromConnectionString(this.connectionString);
         const containerClient = blobServiceClient.getContainerClient(this.container);

package/dist/document_loaders/web/azure_blob_storage_container.d.ts

@@ -1,19 +1,45 @@
 import { BaseDocumentLoader } from "../base.js";
 import { Document } from "../../document.js";
 import { UnstructuredLoaderOptions } from "../fs/unstructured.js";
+/**
+ * Interface representing the configuration for accessing an Azure Blob
+ * Storage container. It includes properties for the connection string and
+ * container name.
+ */
 interface AzureBlobStorageContainerConfig {
     connectionString: string;
     container: string;
 }
+/**
+ * Interface representing the configuration for the
+ * AzureBlobStorageContainerLoader. It includes properties for the
+ * azureConfig and unstructuredConfig. The azureConfig property contains
+ * the Azure Blob Storage container configuration, and the
+ * unstructuredConfig property contains the options for the
+ * UnstructuredLoader.
+ */
 interface AzureBlobStorageContainerLoaderConfig {
     azureConfig: AzureBlobStorageContainerConfig;
     unstructuredConfig?: UnstructuredLoaderOptions;
 }
+/**
+ * Class representing a document loader that loads documents from an Azure
+ * Blob Storage container. It extends the BaseDocumentLoader class.
+ */
 export declare class AzureBlobStorageContainerLoader extends BaseDocumentLoader {
     private readonly connectionString;
     private readonly container;
     private readonly unstructuredConfig?;
     constructor({ azureConfig, unstructuredConfig, }: AzureBlobStorageContainerLoaderConfig);
+    /**
+     * Method to load documents from an Azure Blob Storage container. It
+     * creates a BlobServiceClient using the connection string, gets the
+     * container client using the container name, and iterates over the blobs
+     * in the container. For each blob, it creates an instance of
+     * AzureBlobStorageFileLoader and loads the documents using the loader.
+     * The loaded documents are concatenated to the docs array and returned.
+     * @returns An array of loaded documents.
+     */
     load(): Promise<Document<Record<string, any>>[]>;
 }
 export {};

package/dist/document_loaders/web/azure_blob_storage_container.js

@@ -1,6 +1,10 @@
 import { BlobServiceClient } from "@azure/storage-blob";
 import { AzureBlobStorageFileLoader } from "./azure_blob_storage_file.js";
 import { BaseDocumentLoader } from "../base.js";
+/**
+ * Class representing a document loader that loads documents from an Azure
+ * Blob Storage container. It extends the BaseDocumentLoader class.
+ */
 export class AzureBlobStorageContainerLoader extends BaseDocumentLoader {
     constructor({ azureConfig, unstructuredConfig, }) {
         super();
@@ -26,6 +30,15 @@ export class AzureBlobStorageContainerLoader extends BaseDocumentLoader {
         this.container = azureConfig.container;
         this.unstructuredConfig = unstructuredConfig;
     }
+    /**
+     * Method to load documents from an Azure Blob Storage container. It
+     * creates a BlobServiceClient using the connection string, gets the
+     * container client using the container name, and iterates over the blobs
+     * in the container. For each blob, it creates an instance of
+     * AzureBlobStorageFileLoader and loads the documents using the loader.
+     * The loaded documents are concatenated to the docs array and returned.
+     * @returns An array of loaded documents.
+     */
     async load() {
         const blobServiceClient = BlobServiceClient.fromConnectionString(this.connectionString);
         const containerClient = blobServiceClient.getContainerClient(this.container);

package/dist/document_loaders/web/azure_blob_storage_file.cjs

@@ -30,6 +30,11 @@ const os = __importStar(require("node:os"));
 const storage_blob_1 = require("@azure/storage-blob");
 const base_js_1 = require("../base.cjs");
 const unstructured_js_1 = require("../fs/unstructured.cjs");
+/**
+ * Class representing a document loader that loads a specific file from
+ * Azure Blob Storage. It extends the BaseDocumentLoader class and
+ * implements the DocumentLoader interface.
+ */
 class AzureBlobStorageFileLoader extends base_js_1.BaseDocumentLoader {
     constructor({ azureConfig, unstructuredConfig, }) {
         super();
@@ -62,6 +67,13 @@ class AzureBlobStorageFileLoader extends base_js_1.BaseDocumentLoader {
         this.blobName = azureConfig.blobName;
         this.unstructuredConfig = unstructuredConfig;
     }
+    /**
+     * Method to load a specific file from Azure Blob Storage. It creates a
+     * temporary directory, constructs the file path, downloads the file, and
+     * loads the documents using the UnstructuredLoader. The loaded documents
+     * are returned, and the temporary directory is deleted.
+     * @returns An array of documents loaded from the file in Azure Blob Storage.
+     */
     async load() {
         const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "azureblobfileloader-"));
         const filePath = path.join(tempDir, this.blobName);

package/dist/document_loaders/web/azure_blob_storage_file.d.ts

@@ -1,20 +1,41 @@
 import { BaseDocumentLoader } from "../base.js";
 import { UnstructuredLoaderOptions } from "../fs/unstructured.js";
+/**
+ * Interface representing the configuration for accessing a specific file
+ * in Azure Blob Storage.
+ */
 interface AzureBlobStorageFileConfig {
     connectionString: string;
     container: string;
     blobName: string;
 }
+/**
+ * Interface representing the configuration for the
+ * AzureBlobStorageFileLoader. It contains the Azure Blob Storage file
+ * configuration and the options for the UnstructuredLoader.
+ */
 interface AzureBlobStorageFileLoaderConfig {
     azureConfig: AzureBlobStorageFileConfig;
     unstructuredConfig?: UnstructuredLoaderOptions;
 }
+/**
+ * Class representing a document loader that loads a specific file from
+ * Azure Blob Storage. It extends the BaseDocumentLoader class and
+ * implements the DocumentLoader interface.
+ */
 export declare class AzureBlobStorageFileLoader extends BaseDocumentLoader {
     private readonly connectionString;
     private readonly container;
     private readonly blobName;
     private readonly unstructuredConfig?;
     constructor({ azureConfig, unstructuredConfig, }: AzureBlobStorageFileLoaderConfig);
+    /**
+     * Method to load a specific file from Azure Blob Storage. It creates a
+     * temporary directory, constructs the file path, downloads the file, and
+     * loads the documents using the UnstructuredLoader. The loaded documents
+     * are returned, and the temporary directory is deleted.
+     * @returns An array of documents loaded from the file in Azure Blob Storage.
+     */
     load(): Promise<import("../../document.js").Document<Record<string, any>>[]>;
 }
 export {};

package/dist/document_loaders/web/azure_blob_storage_file.js

@@ -4,6 +4,11 @@ import * as os from "node:os";
 import { BlobServiceClient } from "@azure/storage-blob";
 import { BaseDocumentLoader } from "../base.js";
 import { UnstructuredLoader, } from "../fs/unstructured.js";
+/**
+ * Class representing a document loader that loads a specific file from
+ * Azure Blob Storage. It extends the BaseDocumentLoader class and
+ * implements the DocumentLoader interface.
+ */
 export class AzureBlobStorageFileLoader extends BaseDocumentLoader {
     constructor({ azureConfig, unstructuredConfig, }) {
         super();
@@ -36,6 +41,13 @@ export class AzureBlobStorageFileLoader extends BaseDocumentLoader {
         this.blobName = azureConfig.blobName;
         this.unstructuredConfig = unstructuredConfig;
     }
+    /**
+     * Method to load a specific file from Azure Blob Storage. It creates a
+     * temporary directory, constructs the file path, downloads the file, and
+     * loads the documents using the UnstructuredLoader. The loaded documents
+     * are returned, and the temporary directory is deleted.
+     * @returns An array of documents loaded from the file in Azure Blob Storage.
+     */
     async load() {
         const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "azureblobfileloader-"));
         const filePath = path.join(tempDir, this.blobName);

package/dist/document_loaders/web/cheerio.cjs

@@ -4,6 +4,11 @@ exports.CheerioWebBaseLoader = void 0;
 const document_js_1 = require("../../document.cjs");
 const base_js_1 = require("../base.cjs");
 const async_caller_js_1 = require("../../util/async_caller.cjs");
+/**
+ * A class that extends the BaseDocumentLoader and implements the
+ * DocumentLoader interface. It represents a document loader for loading
+ * web-based documents using Cheerio.
+ */
 class CheerioWebBaseLoader extends base_js_1.BaseDocumentLoader {
     constructor(webPath, fields) {
         super();
@@ -52,15 +57,31 @@ class CheerioWebBaseLoader extends base_js_1.BaseDocumentLoader {
             (await response.text());
         return load(html);
     }
+    /**
+     * Fetches the web document from the webPath and loads it using Cheerio.
+     * It returns a CheerioAPI instance.
+     * @returns A Promise that resolves to a CheerioAPI instance.
+     */
     async scrape() {
         return CheerioWebBaseLoader._scrape(this.webPath, this.caller, this.timeout, this.textDecoder);
     }
+    /**
+     * Extracts the text content from the loaded document using the selector
+     * and creates a Document instance with the extracted text and metadata.
+     * It returns an array of Document instances.
+     * @returns A Promise that resolves to an array of Document instances.
+     */
     async load() {
         const $ = await this.scrape();
         const text = $(this.selector).text();
         const metadata = { source: this.webPath };
         return [new document_js_1.Document({ pageContent: text, metadata })];
     }
+    /**
+     * A static method that dynamically imports the Cheerio library and
+     * returns the load function. If the import fails, it throws an error.
+     * @returns A Promise that resolves to an object containing the load function from the Cheerio library.
+     */
     static async imports() {
         try {
             const { load } = await import("cheerio");

package/dist/document_loaders/web/cheerio.d.ts

@@ -3,6 +3,11 @@ import { Document } from "../../document.js";
 import { BaseDocumentLoader } from "../base.js";
 import type { DocumentLoader } from "../base.js";
 import { AsyncCaller, AsyncCallerParams } from "../../util/async_caller.js";
+/**
+ * Represents the parameters for configuring the CheerioWebBaseLoader. It
+ * extends the AsyncCallerParams interface and adds additional parameters
+ * specific to web-based loaders.
+ */
 export interface WebBaseLoaderParams extends AsyncCallerParams {
     /**
      * The timeout in milliseconds for the fetch request. Defaults to 10s.
@@ -18,6 +23,11 @@ export interface WebBaseLoaderParams extends AsyncCallerParams {
      */
     textDecoder?: TextDecoder;
 }
+/**
+ * A class that extends the BaseDocumentLoader and implements the
+ * DocumentLoader interface. It represents a document loader for loading
+ * web-based documents using Cheerio.
+ */
 export declare class CheerioWebBaseLoader extends BaseDocumentLoader implements DocumentLoader {
     webPath: string;
     timeout: number;
@@ -26,8 +36,24 @@ export declare class CheerioWebBaseLoader extends BaseDocumentLoader implements
     textDecoder?: TextDecoder;
     constructor(webPath: string, fields?: WebBaseLoaderParams);
     static _scrape(url: string, caller: AsyncCaller, timeout: number | undefined, textDecoder?: TextDecoder): Promise<CheerioAPI>;
+    /**
+     * Fetches the web document from the webPath and loads it using Cheerio.
+     * It returns a CheerioAPI instance.
+     * @returns A Promise that resolves to a CheerioAPI instance.
+     */
     scrape(): Promise<CheerioAPI>;
+    /**
+     * Extracts the text content from the loaded document using the selector
+     * and creates a Document instance with the extracted text and metadata.
+     * It returns an array of Document instances.
+     * @returns A Promise that resolves to an array of Document instances.
+     */
     load(): Promise<Document[]>;
+    /**
+     * A static method that dynamically imports the Cheerio library and
+     * returns the load function. If the import fails, it throws an error.
+     * @returns A Promise that resolves to an object containing the load function from the Cheerio library.
+     */
     static imports(): Promise<{
         load: typeof LoadT;
     }>;

package/dist/document_loaders/web/cheerio.js

@@ -1,6 +1,11 @@
 import { Document } from "../../document.js";
 import { BaseDocumentLoader } from "../base.js";
 import { AsyncCaller } from "../../util/async_caller.js";
+/**
+ * A class that extends the BaseDocumentLoader and implements the
+ * DocumentLoader interface. It represents a document loader for loading
+ * web-based documents using Cheerio.
+ */
 export class CheerioWebBaseLoader extends BaseDocumentLoader {
     constructor(webPath, fields) {
         super();
@@ -49,15 +54,31 @@ export class CheerioWebBaseLoader extends BaseDocumentLoader {
             (await response.text());
         return load(html);
     }
+    /**
+     * Fetches the web document from the webPath and loads it using Cheerio.
+     * It returns a CheerioAPI instance.
+     * @returns A Promise that resolves to a CheerioAPI instance.
+     */
     async scrape() {
         return CheerioWebBaseLoader._scrape(this.webPath, this.caller, this.timeout, this.textDecoder);
     }
+    /**
+     * Extracts the text content from the loaded document using the selector
+     * and creates a Document instance with the extracted text and metadata.
+     * It returns an array of Document instances.
+     * @returns A Promise that resolves to an array of Document instances.
+     */
     async load() {
         const $ = await this.scrape();
         const text = $(this.selector).text();
         const metadata = { source: this.webPath };
         return [new Document({ pageContent: text, metadata })];
     }
+    /**
+     * A static method that dynamically imports the Cheerio library and
+     * returns the load function. If the import fails, it throws an error.
+     * @returns A Promise that resolves to an object containing the load function from the Cheerio library.
+     */
     static async imports() {
         try {
             const { load } = await import("cheerio");

package/dist/document_loaders/web/college_confidential.cjs

@@ -3,10 +3,21 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.CollegeConfidentialLoader = void 0;
 const document_js_1 = require("../../document.cjs");
 const cheerio_js_1 = require("./cheerio.cjs");
+/**
+ * A document loader specifically designed for loading documents from the
+ * College Confidential website. It extends the CheerioWebBaseLoader.
+ */
 class CollegeConfidentialLoader extends cheerio_js_1.CheerioWebBaseLoader {
     constructor(webPath) {
         super(webPath);
     }
+    /**
+     * Overrides the base load() method to extract the text content from the
+     * loaded document using a specific selector for the College Confidential
+     * website. It creates a Document instance with the extracted text and
+     * metadata, and returns an array containing the Document instance.
+     * @returns An array containing a Document instance with the extracted text and metadata from the loaded College Confidential web document.
+     */
     async load() {
         const $ = await this.scrape();
         const text = $("main[class='skin-handler']").text();

package/dist/document_loaders/web/college_confidential.d.ts

@@ -1,6 +1,17 @@
 import { Document } from "../../document.js";
 import { CheerioWebBaseLoader } from "./cheerio.js";
+/**
+ * A document loader specifically designed for loading documents from the
+ * College Confidential website. It extends the CheerioWebBaseLoader.
+ */
 export declare class CollegeConfidentialLoader extends CheerioWebBaseLoader {
     constructor(webPath: string);
+    /**
+     * Overrides the base load() method to extract the text content from the
+     * loaded document using a specific selector for the College Confidential
+     * website. It creates a Document instance with the extracted text and
+     * metadata, and returns an array containing the Document instance.
+     * @returns An array containing a Document instance with the extracted text and metadata from the loaded College Confidential web document.
+     */
     load(): Promise<Document[]>;
 }

package/dist/document_loaders/web/college_confidential.js

@@ -1,9 +1,20 @@
 import { Document } from "../../document.js";
 import { CheerioWebBaseLoader } from "./cheerio.js";
+/**
+ * A document loader specifically designed for loading documents from the
+ * College Confidential website. It extends the CheerioWebBaseLoader.
+ */
 export class CollegeConfidentialLoader extends CheerioWebBaseLoader {
     constructor(webPath) {
         super(webPath);
     }
+    /**
+     * Overrides the base load() method to extract the text content from the
+     * loaded document using a specific selector for the College Confidential
+     * website. It creates a Document instance with the extracted text and
+     * metadata, and returns an array containing the Document instance.
+     * @returns An array containing a Document instance with the extracted text and metadata from the loaded College Confidential web document.
+     */
     async load() {
         const $ = await this.scrape();
         const text = $("main[class='skin-handler']").text();

package/dist/document_loaders/web/confluence.cjs

@@ -4,6 +4,9 @@ exports.ConfluencePagesLoader = void 0;
 const html_to_text_1 = require("html-to-text");
 const document_js_1 = require("../../document.cjs");
 const base_js_1 = require("../base.cjs");
+/**
+ * Class representing a document loader for loading pages from Confluence.
+ */
 class ConfluencePagesLoader extends base_js_1.BaseDocumentLoader {
     constructor({ baseUrl, spaceKey, username, accessToken, limit = 25, }) {
         super();
@@ -43,6 +46,11 @@ class ConfluencePagesLoader extends base_js_1.BaseDocumentLoader {
         this.accessToken = accessToken;
         this.limit = limit;
     }
+    /**
+     * Fetches all the pages in the specified space and converts each page to
+     * a Document instance.
+     * @returns Promise resolving to an array of Document instances.
+     */
     async load() {
         try {
             const pages = await this.fetchAllPagesInSpace();
@@ -53,6 +61,11 @@ class ConfluencePagesLoader extends base_js_1.BaseDocumentLoader {
             return [];
         }
     }
+    /**
+     * Fetches data from the Confluence API using the provided URL.
+     * @param url The URL to fetch data from.
+     * @returns Promise resolving to the JSON response from the API.
+     */
     async fetchConfluenceData(url) {
         try {
             const authToken = Buffer.from(`${this.username}:${this.accessToken}`).toString("base64");
@@ -72,6 +85,11 @@ class ConfluencePagesLoader extends base_js_1.BaseDocumentLoader {
             throw new Error(`Failed to fetch ${url} from Confluence: ${error}`);
         }
     }
+    /**
+     * Recursively fetches all the pages in the specified space.
+     * @param start The start parameter to paginate through the results.
+     * @returns Promise resolving to an array of ConfluencePage objects.
+     */
     async fetchAllPagesInSpace(start = 0) {
         const url = `${this.baseUrl}/rest/api/content?spaceKey=${this.spaceKey}&limit=${this.limit}&start=${start}&expand=body.storage`;
         const data = await this.fetchConfluenceData(url);
@@ -82,6 +100,11 @@ class ConfluencePagesLoader extends base_js_1.BaseDocumentLoader {
         const nextPageResults = await this.fetchAllPagesInSpace(nextPageStart);
         return data.results.concat(nextPageResults);
     }
+    /**
+     * Creates a Document instance from a ConfluencePage object.
+     * @param page The ConfluencePage object to convert.
+     * @returns A Document instance.
+     */
     createDocumentFromPage(page) {
         // Convert the HTML content to plain text
         const plainTextContent = (0, html_to_text_1.htmlToText)(page.body.storage.value, {