langchain 0.0.198 → 0.0.200

package/dist/document_loaders/fs/chatgpt.cjs

@@ -0,0 +1,85 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ChatGPTLoader = void 0;
+const text_js_1 = require("./text.cjs");
+const document_js_1 = require("../../document.cjs");
+function concatenateRows(message, title) {
+    /**
+     * Combine message information in a readable format ready to be used.
+     * @param {ChatGPTMessage} message - Message to be concatenated
+     * @param {string} title - Title of the conversation
+     *
+     * @returns {string} Concatenated message
+     */
+    if (!message) {
+        return "";
+    }
+    const sender = message.author ? message.author.role : "unknown";
+    const text = message.content.parts[0];
+    const date = new Date(message.create_time * 1000)
+        .toISOString()
+        .slice(0, 19)
+        .replace("T", " ");
+    return `${title} - ${sender} on ${date}: ${text}\n\n`;
+}
+class ChatGPTLoader extends text_js_1.TextLoader {
+    constructor(filePathOrBlob, numLogs = 0) {
+        super(filePathOrBlob);
+        Object.defineProperty(this, "numLogs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.numLogs = numLogs;
+    }
+    async parse(raw) {
+        let data;
+        try {
+            data = JSON.parse(raw);
+        }
+        catch (e) {
+            console.error(e);
+            throw new Error("Failed to parse JSON");
+        }
+        const truncatedData = this.numLogs > 0 ? data.slice(0, this.numLogs) : data;
+        return truncatedData.map((d) => Object.values(d.mapping)
+            .filter((msg, idx) => !(idx === 0 && msg.message.author.role === "system"))
+            .map((msg) => concatenateRows(msg.message, d.title))
+            .join(""));
+    }
+    async load() {
+        let text;
+        let metadata;
+        if (typeof this.filePathOrBlob === "string") {
+            const { readFile } = await text_js_1.TextLoader.imports();
+            try {
+                text = await readFile(this.filePathOrBlob, "utf8");
+            }
+            catch (e) {
+                console.error(e);
+                throw new Error("Failed to read file");
+            }
+            metadata = { source: this.filePathOrBlob };
+        }
+        else {
+            try {
+                text = await this.filePathOrBlob.text();
+            }
+            catch (e) {
+                console.error(e);
+                throw new Error("Failed to read blob");
+            }
+            metadata = { source: "blob", blobType: this.filePathOrBlob.type };
+        }
+        const parsed = await this.parse(text);
+        return parsed.map((pageContent, i) => new document_js_1.Document({
+            pageContent,
+            metadata: {
+                ...metadata,
+                logIndex: i + 1,
+            },
+        }));
+    }
+}
+exports.ChatGPTLoader = ChatGPTLoader;

package/dist/document_loaders/fs/chatgpt.d.ts

@@ -0,0 +1,8 @@
+import { TextLoader } from "./text.js";
+import { Document } from "../../document.js";
+export declare class ChatGPTLoader extends TextLoader {
+    numLogs: number;
+    constructor(filePathOrBlob: string | Blob, numLogs?: number);
+    protected parse(raw: string): Promise<string[]>;
+    load(): Promise<Document[]>;
+}

package/dist/document_loaders/fs/chatgpt.js

@@ -0,0 +1,81 @@
+import { TextLoader } from "./text.js";
+import { Document } from "../../document.js";
+function concatenateRows(message, title) {
+    /**
+     * Combine message information in a readable format ready to be used.
+     * @param {ChatGPTMessage} message - Message to be concatenated
+     * @param {string} title - Title of the conversation
+     *
+     * @returns {string} Concatenated message
+     */
+    if (!message) {
+        return "";
+    }
+    const sender = message.author ? message.author.role : "unknown";
+    const text = message.content.parts[0];
+    const date = new Date(message.create_time * 1000)
+        .toISOString()
+        .slice(0, 19)
+        .replace("T", " ");
+    return `${title} - ${sender} on ${date}: ${text}\n\n`;
+}
+export class ChatGPTLoader extends TextLoader {
+    constructor(filePathOrBlob, numLogs = 0) {
+        super(filePathOrBlob);
+        Object.defineProperty(this, "numLogs", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.numLogs = numLogs;
+    }
+    async parse(raw) {
+        let data;
+        try {
+            data = JSON.parse(raw);
+        }
+        catch (e) {
+            console.error(e);
+            throw new Error("Failed to parse JSON");
+        }
+        const truncatedData = this.numLogs > 0 ? data.slice(0, this.numLogs) : data;
+        return truncatedData.map((d) => Object.values(d.mapping)
+            .filter((msg, idx) => !(idx === 0 && msg.message.author.role === "system"))
+            .map((msg) => concatenateRows(msg.message, d.title))
+            .join(""));
+    }
+    async load() {
+        let text;
+        let metadata;
+        if (typeof this.filePathOrBlob === "string") {
+            const { readFile } = await TextLoader.imports();
+            try {
+                text = await readFile(this.filePathOrBlob, "utf8");
+            }
+            catch (e) {
+                console.error(e);
+                throw new Error("Failed to read file");
+            }
+            metadata = { source: this.filePathOrBlob };
+        }
+        else {
+            try {
+                text = await this.filePathOrBlob.text();
+            }
+            catch (e) {
+                console.error(e);
+                throw new Error("Failed to read blob");
+            }
+            metadata = { source: "blob", blobType: this.filePathOrBlob.type };
+        }
+        const parsed = await this.parse(text);
+        return parsed.map((pageContent, i) => new Document({
+            pageContent,
+            metadata: {
+                ...metadata,
+                logIndex: i + 1,
+            },
+        }));
+    }
+}

package/dist/document_loaders/web/confluence.cjs

@@ -19,7 +19,7 @@ const base_js_1 = require("../base.cjs");
  * ```
  */
 class ConfluencePagesLoader extends base_js_1.BaseDocumentLoader {
-    constructor({ baseUrl, spaceKey, username, accessToken, limit = 25, }) {
+    constructor({ baseUrl, spaceKey, username, accessToken, limit = 25, personalAccessToken, }) {
         super();
         Object.defineProperty(this, "baseUrl", {
             enumerable: true,
@@ -51,11 +51,32 @@ class ConfluencePagesLoader extends base_js_1.BaseDocumentLoader {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "personalAccessToken", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.baseUrl = baseUrl;
         this.spaceKey = spaceKey;
         this.username = username;
         this.accessToken = accessToken;
         this.limit = limit;
+        this.personalAccessToken = personalAccessToken;
+    }
+    /**
+     * Returns the authorization header for the request.
+     * @returns The authorization header as a string, or undefined if no credentials were provided.
+     */
+    get authorizationHeader() {
+        if (this.personalAccessToken) {
+            return `Bearer ${this.personalAccessToken}`;
+        }
+        else if (this.username && this.accessToken) {
+            const authToken = Buffer.from(`${this.username}:${this.accessToken}`).toString("base64");
+            return `Basic ${authToken}`;
+        }
+        return undefined;
     }
     /**
      * Fetches all the pages in the specified space and converts each page to
@@ -79,13 +100,16 @@ class ConfluencePagesLoader extends base_js_1.BaseDocumentLoader {
      */
     async fetchConfluenceData(url) {
         try {
-            const authToken = Buffer.from(`${this.username}:${this.accessToken}`).toString("base64");
+            const initialHeaders = {
+                "Content-Type": "application/json",
+                Accept: "application/json",
+            };
+            const authHeader = this.authorizationHeader;
+            if (authHeader) {
+                initialHeaders.Authorization = authHeader;
+            }
             const response = await fetch(url, {
-                headers: {
-                    Authorization: `Basic ${authToken}`,
-                    "Content-Type": "application/json",
-                    Accept: "application/json",
-                },
+                headers: initialHeaders,
             });
             if (!response.ok) {
                 throw new Error(`Failed to fetch ${url} from Confluence: ${response.status}`);

package/dist/document_loaders/web/confluence.d.ts

@@ -7,8 +7,9 @@ import { BaseDocumentLoader } from "../base.js";
 export interface ConfluencePagesLoaderParams {
     baseUrl: string;
     spaceKey: string;
-    username: string;
-    accessToken: string;
+    username?: string;
+    accessToken?: string;
+    personalAccessToken?: string;
     limit?: number;
 }
 /**
@@ -47,10 +48,16 @@ export interface ConfluenceAPIResponse {
 export declare class ConfluencePagesLoader extends BaseDocumentLoader {
     readonly baseUrl: string;
     readonly spaceKey: string;
-    readonly username: string;
-    readonly accessToken: string;
+    readonly username?: string;
+    readonly accessToken?: string;
     readonly limit: number;
-    constructor({ baseUrl, spaceKey, username, accessToken, limit, }: ConfluencePagesLoaderParams);
+    readonly personalAccessToken?: string;
+    constructor({ baseUrl, spaceKey, username, accessToken, limit, personalAccessToken, }: ConfluencePagesLoaderParams);
+    /**
+     * Returns the authorization header for the request.
+     * @returns The authorization header as a string, or undefined if no credentials were provided.
+     */
+    private get authorizationHeader();
     /**
      * Fetches all the pages in the specified space and converts each page to
      * a Document instance.

package/dist/document_loaders/web/confluence.js

@@ -16,7 +16,7 @@ import { BaseDocumentLoader } from "../base.js";
  * ```
  */
 export class ConfluencePagesLoader extends BaseDocumentLoader {
-    constructor({ baseUrl, spaceKey, username, accessToken, limit = 25, }) {
+    constructor({ baseUrl, spaceKey, username, accessToken, limit = 25, personalAccessToken, }) {
         super();
         Object.defineProperty(this, "baseUrl", {
             enumerable: true,
@@ -48,11 +48,32 @@ export class ConfluencePagesLoader extends BaseDocumentLoader {
             writable: true,
             value: void 0
         });
+        Object.defineProperty(this, "personalAccessToken", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         this.baseUrl = baseUrl;
         this.spaceKey = spaceKey;
         this.username = username;
         this.accessToken = accessToken;
         this.limit = limit;
+        this.personalAccessToken = personalAccessToken;
+    }
+    /**
+     * Returns the authorization header for the request.
+     * @returns The authorization header as a string, or undefined if no credentials were provided.
+     */
+    get authorizationHeader() {
+        if (this.personalAccessToken) {
+            return `Bearer ${this.personalAccessToken}`;
+        }
+        else if (this.username && this.accessToken) {
+            const authToken = Buffer.from(`${this.username}:${this.accessToken}`).toString("base64");
+            return `Basic ${authToken}`;
+        }
+        return undefined;
     }
     /**
      * Fetches all the pages in the specified space and converts each page to
@@ -76,13 +97,16 @@ export class ConfluencePagesLoader extends BaseDocumentLoader {
      */
     async fetchConfluenceData(url) {
         try {
-            const authToken = Buffer.from(`${this.username}:${this.accessToken}`).toString("base64");
+            const initialHeaders = {
+                "Content-Type": "application/json",
+                Accept: "application/json",
+            };
+            const authHeader = this.authorizationHeader;
+            if (authHeader) {
+                initialHeaders.Authorization = authHeader;
+            }
             const response = await fetch(url, {
-                headers: {
-                    Authorization: `Basic ${authToken}`,
-                    "Content-Type": "application/json",
-                    Accept: "application/json",
-                },
+                headers: initialHeaders,
             });
             if (!response.ok) {
                 throw new Error(`Failed to fetch ${url} from Confluence: ${response.status}`);

package/dist/embeddings/gradient_ai.cjs

@@ -0,0 +1,102 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GradientEmbeddings = void 0;
+const nodejs_sdk_1 = require("@gradientai/nodejs-sdk");
+const env_js_1 = require("../util/env.cjs");
+const chunk_js_1 = require("../util/chunk.cjs");
+const base_js_1 = require("./base.cjs");
+/**
+ * Class for generating embeddings using the Gradient AI's API. Extends the
+ * Embeddings class and implements GradientEmbeddingsParams and
+ */
+class GradientEmbeddings extends base_js_1.Embeddings {
+    constructor(fields) {
+        super(fields);
+        Object.defineProperty(this, "gradientAccessKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workspaceId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "batchSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 128
+        });
+        Object.defineProperty(this, "model", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.gradientAccessKey =
+            fields?.gradientAccessKey ??
+                (0, env_js_1.getEnvironmentVariable)("GRADIENT_ACCESS_TOKEN");
+        this.workspaceId =
+            fields?.workspaceId ?? (0, env_js_1.getEnvironmentVariable)("GRADIENT_WORKSPACE_ID");
+        if (!this.gradientAccessKey) {
+            throw new Error("Missing Gradient AI Access Token");
+        }
+        if (!this.workspaceId) {
+            throw new Error("Missing Gradient AI Workspace ID");
+        }
+    }
+    /**
+     * Method to generate embeddings for an array of documents. Splits the
+     * documents into batches and makes requests to the Gradient API to generate
+     * embeddings.
+     * @param texts Array of documents to generate embeddings for.
+     * @returns Promise that resolves to a 2D array of embeddings for each document.
+     */
+    async embedDocuments(texts) {
+        await this.setModel();
+        const mappedTexts = texts.map((text) => ({ input: text }));
+        const batches = (0, chunk_js_1.chunkArray)(mappedTexts, this.batchSize);
+        const batchRequests = batches.map((batch) => this.caller.call(async () => this.model.generateEmbeddings({
+            inputs: batch,
+        })));
+        const batchResponses = await Promise.all(batchRequests);
+        const embeddings = [];
+        for (let i = 0; i < batchResponses.length; i += 1) {
+            const batch = batches[i];
+            const { embeddings: batchResponse } = batchResponses[i];
+            for (let j = 0; j < batch.length; j += 1) {
+                embeddings.push(batchResponse[j].embedding);
+            }
+        }
+        return embeddings;
+    }
+    /**
+     * Method to generate an embedding for a single document. Calls the
+     * embedDocuments method with the document as the input.
+     * @param text Document to generate an embedding for.
+     * @returns Promise that resolves to an embedding for the document.
+     */
+    async embedQuery(text) {
+        const data = await this.embedDocuments([text]);
+        return data[0];
+    }
+    /**
+     * Method to set the model to use for generating embeddings.
+     * @sets the class' `model` value to that of the retrieved Embeddings Model.
+     */
+    async setModel() {
+        if (this.model)
+            return;
+        const gradient = new nodejs_sdk_1.Gradient({
+            accessToken: this.gradientAccessKey,
+            workspaceId: this.workspaceId,
+        });
+        this.model = await gradient.getEmbeddingsModel({
+            slug: "bge-large",
+        });
+    }
+}
+exports.GradientEmbeddings = GradientEmbeddings;

package/dist/embeddings/gradient_ai.d.ts

@@ -0,0 +1,48 @@
+import { Embeddings, EmbeddingsParams } from "./base.js";
+/**
+ * Interface for GradientEmbeddings parameters. Extends EmbeddingsParams and
+ * defines additional parameters specific to the GradientEmbeddings class.
+ */
+export interface GradientEmbeddingsParams extends EmbeddingsParams {
+    /**
+     * Gradient AI Access Token.
+     * Provide Access Token if you do not wish to automatically pull from env.
+     */
+    gradientAccessKey?: string;
+    /**
+     * Gradient Workspace Id.
+     * Provide workspace id if you do not wish to automatically pull from env.
+     */
+    workspaceId?: string;
+}
+/**
+ * Class for generating embeddings using the Gradient AI's API. Extends the
+ * Embeddings class and implements GradientEmbeddingsParams and
+ */
+export declare class GradientEmbeddings extends Embeddings implements GradientEmbeddingsParams {
+    gradientAccessKey?: string;
+    workspaceId?: string;
+    batchSize: number;
+    model: any;
+    constructor(fields: GradientEmbeddingsParams);
+    /**
+     * Method to generate embeddings for an array of documents. Splits the
+     * documents into batches and makes requests to the Gradient API to generate
+     * embeddings.
+     * @param texts Array of documents to generate embeddings for.
+     * @returns Promise that resolves to a 2D array of embeddings for each document.
+     */
+    embedDocuments(texts: string[]): Promise<number[][]>;
+    /**
+     * Method to generate an embedding for a single document. Calls the
+     * embedDocuments method with the document as the input.
+     * @param text Document to generate an embedding for.
+     * @returns Promise that resolves to an embedding for the document.
+     */
+    embedQuery(text: string): Promise<number[]>;
+    /**
+     * Method to set the model to use for generating embeddings.
+     * @sets the class' `model` value to that of the retrieved Embeddings Model.
+     */
+    setModel(): Promise<void>;
+}

package/dist/embeddings/gradient_ai.js

@@ -0,0 +1,98 @@
+import { Gradient } from "@gradientai/nodejs-sdk";
+import { getEnvironmentVariable } from "../util/env.js";
+import { chunkArray } from "../util/chunk.js";
+import { Embeddings } from "./base.js";
+/**
+ * Class for generating embeddings using the Gradient AI's API. Extends the
+ * Embeddings class and implements GradientEmbeddingsParams and
+ */
+export class GradientEmbeddings extends Embeddings {
+    constructor(fields) {
+        super(fields);
+        Object.defineProperty(this, "gradientAccessKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workspaceId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "batchSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 128
+        });
+        Object.defineProperty(this, "model", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.gradientAccessKey =
+            fields?.gradientAccessKey ??
+                getEnvironmentVariable("GRADIENT_ACCESS_TOKEN");
+        this.workspaceId =
+            fields?.workspaceId ?? getEnvironmentVariable("GRADIENT_WORKSPACE_ID");
+        if (!this.gradientAccessKey) {
+            throw new Error("Missing Gradient AI Access Token");
+        }
+        if (!this.workspaceId) {
+            throw new Error("Missing Gradient AI Workspace ID");
+        }
+    }
+    /**
+     * Method to generate embeddings for an array of documents. Splits the
+     * documents into batches and makes requests to the Gradient API to generate
+     * embeddings.
+     * @param texts Array of documents to generate embeddings for.
+     * @returns Promise that resolves to a 2D array of embeddings for each document.
+     */
+    async embedDocuments(texts) {
+        await this.setModel();
+        const mappedTexts = texts.map((text) => ({ input: text }));
+        const batches = chunkArray(mappedTexts, this.batchSize);
+        const batchRequests = batches.map((batch) => this.caller.call(async () => this.model.generateEmbeddings({
+            inputs: batch,
+        })));
+        const batchResponses = await Promise.all(batchRequests);
+        const embeddings = [];
+        for (let i = 0; i < batchResponses.length; i += 1) {
+            const batch = batches[i];
+            const { embeddings: batchResponse } = batchResponses[i];
+            for (let j = 0; j < batch.length; j += 1) {
+                embeddings.push(batchResponse[j].embedding);
+            }
+        }
+        return embeddings;
+    }
+    /**
+     * Method to generate an embedding for a single document. Calls the
+     * embedDocuments method with the document as the input.
+     * @param text Document to generate an embedding for.
+     * @returns Promise that resolves to an embedding for the document.
+     */
+    async embedQuery(text) {
+        const data = await this.embedDocuments([text]);
+        return data[0];
+    }
+    /**
+     * Method to set the model to use for generating embeddings.
+     * @sets the class' `model` value to that of the retrieved Embeddings Model.
+     */
+    async setModel() {
+        if (this.model)
+            return;
+        const gradient = new Gradient({
+            accessToken: this.gradientAccessKey,
+            workspaceId: this.workspaceId,
+        });
+        this.model = await gradient.getEmbeddingsModel({
+            slug: "bge-large",
+        });
+    }
+}