langchain 0.0.199 → 0.0.201

package/dist/document_loaders/fs/obsidian.cjs

@@ -0,0 +1,240 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ObsidianLoader = void 0;
+const js_yaml_1 = __importDefault(require("js-yaml"));
+const env_js_1 = require("../../util/env.cjs");
+const directory_js_1 = require("./directory.cjs");
+const base_js_1 = require("../base.cjs");
+const document_js_1 = require("../../document.cjs");
+/**
+ * Represents a loader for Obsidian markdown files. This loader extends the BaseDocumentLoader
+ * and provides functionality to parse and extract metadata, tags, and dataview fields from
+ * Obsidian markdown files.
+ */
+class ObsidianFileLoader extends base_js_1.BaseDocumentLoader {
+    /**
+     * Initializes a new instance of the ObsidianFileLoader class.
+     * @param filePath The path to the Obsidian markdown file.
+     * @param encoding The character encoding to use when reading the file. Defaults to 'utf-8'.
+     * @param collectMetadata Determines whether metadata should be collected from the file. Defaults to true.
+     */
+    constructor(filePath, { encoding = "utf-8", collectMetadata = true, } = {}) {
+        super();
+        Object.defineProperty(this, "filePath", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "encoding", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "collectMetadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.filePath = filePath;
+        this.encoding = encoding;
+        this.collectMetadata = collectMetadata;
+    }
+    /**
+     * Parses the YAML front matter from the given content string.
+     * @param content The string content of the markdown file.
+     * @returns An object representing the parsed front matter.
+     */
+    parseFrontMatter(content) {
+        if (!this.collectMetadata) {
+            return {};
+        }
+        const match = content.match(ObsidianFileLoader.FRONT_MATTER_REGEX);
+        if (!match) {
+            return {};
+        }
+        try {
+            const frontMatter = js_yaml_1.default.load(match[1]);
+            if (frontMatter.tags && typeof frontMatter.tags === "string") {
+                frontMatter.tags = frontMatter.tags.split(", ");
+            }
+            return frontMatter;
+        }
+        catch (e) {
+            console.warn("Encountered non-yaml frontmatter");
+            return {};
+        }
+    }
+    /**
+     * Removes YAML front matter from the given content string.
+     * @param content The string content of the markdown file.
+     * @returns The content string with the front matter removed.
+     */
+    removeFrontMatter(content) {
+        if (!this.collectMetadata) {
+            return content;
+        }
+        return content.replace(ObsidianFileLoader.FRONT_MATTER_REGEX, "");
+    }
+    /**
+     * Parses Obsidian-style tags from the given content string.
+     * @param content The string content of the markdown file.
+     * @returns A set of parsed tags.
+     */
+    parseObsidianTags(content) {
+        if (!this.collectMetadata) {
+            return new Set();
+        }
+        const matches = content.matchAll(ObsidianFileLoader.TAG_REGEX);
+        const tags = new Set();
+        for (const match of matches) {
+            tags.add(match[1]);
+        }
+        return tags;
+    }
+    /**
+     * Parses dataview fields from the given content string.
+     * @param content The string content of the markdown file.
+     * @returns A record object containing key-value pairs of dataview fields.
+     */
+    parseObsidianDataviewFields(content) {
+        if (!this.collectMetadata) {
+            return {};
+        }
+        const fields = {};
+        const lineMatches = content.matchAll(ObsidianFileLoader.DATAVIEW_LINE_REGEX);
+        for (const [, key, value] of lineMatches) {
+            fields[key] = value;
+        }
+        const bracketMatches = content.matchAll(ObsidianFileLoader.DATAVIEW_INLINE_BRACKET_REGEX);
+        for (const [, key, value] of bracketMatches) {
+            fields[key] = value;
+        }
+        const parenMatches = content.matchAll(ObsidianFileLoader.DATAVIEW_INLINE_PAREN_REGEX);
+        for (const [, key, value] of parenMatches) {
+            fields[key] = value;
+        }
+        return fields;
+    }
+    /**
+     * Converts metadata to a format compatible with Langchain.
+     * @param metadata The metadata object to convert.
+     * @returns A record object containing key-value pairs of Langchain-compatible metadata.
+     */
+    toLangchainCompatibleMetadata(metadata) {
+        const result = {};
+        for (const [key, value] of Object.entries(metadata)) {
+            if (typeof value === "string" || typeof value === "number") {
+                result[key] = value;
+            }
+            else {
+                result[key] = JSON.stringify(value);
+            }
+        }
+        return result;
+    }
+    /**
+     * It loads the Obsidian file, parses it, and returns a `Document` instance.
+     * @returns An array of `Document` instances to comply with the BaseDocumentLoader interface.
+     */
+    async load() {
+        const documents = [];
+        const { basename, readFile, stat } = await ObsidianFileLoader.imports();
+        const fileName = basename(this.filePath);
+        const stats = await stat(this.filePath);
+        let content = await readFile(this.filePath, this.encoding);
+        const frontMatter = this.parseFrontMatter(content);
+        const tags = this.parseObsidianTags(content);
+        const dataviewFields = this.parseObsidianDataviewFields(content);
+        content = this.removeFrontMatter(content);
+        const metadata = {
+            source: fileName,
+            path: this.filePath,
+            created: stats.birthtimeMs,
+            lastModified: stats.mtimeMs,
+            lastAccessed: stats.atimeMs,
+            ...this.toLangchainCompatibleMetadata(frontMatter),
+            ...dataviewFields,
+        };
+        if (tags.size || frontMatter.tags) {
+            metadata.tags = Array.from(new Set([...tags, ...(frontMatter.tags ?? [])])).join(",");
+        }
+        documents.push(new document_js_1.Document({
+            pageContent: content,
+            metadata,
+        }));
+        return documents;
+    }
+    /**
+     * Imports the necessary functions from the `node:path` and
+     * `node:fs/promises` modules. It is used to dynamically import the
+     * functions when needed. If the import fails, it throws an error
+     * indicating that the modules failed to load.
+     * @returns A promise that resolves to an object containing the imported functions.
+     */
+    static async imports() {
+        try {
+            const { basename } = await import("node:path");
+            const { readFile, stat } = await import("node:fs/promises");
+            return { basename, readFile, stat };
+        }
+        catch (e) {
+            console.error(e);
+            throw new Error(`Failed to load fs/promises. ObsidianFileLoader available only on environment 'node'. It appears you are running environment '${(0, env_js_1.getEnv)()}'. See https://<link to docs> for alternatives.`);
+        }
+    }
+}
+Object.defineProperty(ObsidianFileLoader, "FRONT_MATTER_REGEX", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: /^---\n(.*?)\n---\n/s
+});
+Object.defineProperty(ObsidianFileLoader, "TAG_REGEX", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: /(?:\s|^)#([a-zA-Z_][\w/-]*)/g
+});
+Object.defineProperty(ObsidianFileLoader, "DATAVIEW_LINE_REGEX", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: /^\s*(\w+)::\s*(.*)$/gm
+});
+Object.defineProperty(ObsidianFileLoader, "DATAVIEW_INLINE_BRACKET_REGEX", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: /\[(\w+)::\s*(.*)\]/gm
+});
+Object.defineProperty(ObsidianFileLoader, "DATAVIEW_INLINE_PAREN_REGEX", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: /\((\w+)::\s*(.*)\)/gm
+});
+/**
+ * Represents a loader for directories containing Obsidian markdown files. This loader extends
+ * the DirectoryLoader and provides functionality to load and parse '.md' files with YAML frontmatter,
+ * Obsidian tags, and Dataview fields.
+ */
+class ObsidianLoader extends directory_js_1.DirectoryLoader {
+    /**
+     * Initializes a new instance of the ObsidianLoader class.
+     * @param directoryPath The path to the directory containing Obsidian markdown files.
+     * @param encoding The character encoding to use when reading files. Defaults to 'utf-8'.
+     * @param collectMetadata Determines whether metadata should be collected from the files. Defaults to true.
+     */
+    constructor(directoryPath, options) {
+        super(directoryPath, {
+            ".md": (filePath) => new ObsidianFileLoader(filePath, options),
+        }, true, directory_js_1.UnknownHandling.Ignore);
+    }
+}
+exports.ObsidianLoader = ObsidianLoader;

package/dist/document_loaders/fs/obsidian.d.ts

@@ -0,0 +1,26 @@
+/// <reference types="node" resolution-mode="require"/>
+import { DirectoryLoader } from "./directory.js";
+export type FrontMatter = {
+    title?: string;
+    description?: string;
+    tags?: string[] | string;
+    [key: string]: unknown;
+};
+export interface ObsidianFileLoaderOptions {
+    encoding?: BufferEncoding;
+    collectMetadata?: boolean;
+}
+/**
+ * Represents a loader for directories containing Obsidian markdown files. This loader extends
+ * the DirectoryLoader and provides functionality to load and parse '.md' files with YAML frontmatter,
+ * Obsidian tags, and Dataview fields.
+ */
+export declare class ObsidianLoader extends DirectoryLoader {
+    /**
+     * Initializes a new instance of the ObsidianLoader class.
+     * @param directoryPath The path to the directory containing Obsidian markdown files.
+     * @param encoding The character encoding to use when reading files. Defaults to 'utf-8'.
+     * @param collectMetadata Determines whether metadata should be collected from the files. Defaults to true.
+     */
+    constructor(directoryPath: string, options?: ObsidianFileLoaderOptions);
+}

package/dist/document_loaders/fs/obsidian.js

@@ -0,0 +1,233 @@
+import yaml from "js-yaml";
+import { getEnv } from "../../util/env.js";
+import { DirectoryLoader, UnknownHandling } from "./directory.js";
+import { BaseDocumentLoader } from "../base.js";
+import { Document } from "../../document.js";
+/**
+ * Represents a loader for Obsidian markdown files. This loader extends the BaseDocumentLoader
+ * and provides functionality to parse and extract metadata, tags, and dataview fields from
+ * Obsidian markdown files.
+ */
+class ObsidianFileLoader extends BaseDocumentLoader {
+    /**
+     * Initializes a new instance of the ObsidianFileLoader class.
+     * @param filePath The path to the Obsidian markdown file.
+     * @param encoding The character encoding to use when reading the file. Defaults to 'utf-8'.
+     * @param collectMetadata Determines whether metadata should be collected from the file. Defaults to true.
+     */
+    constructor(filePath, { encoding = "utf-8", collectMetadata = true, } = {}) {
+        super();
+        Object.defineProperty(this, "filePath", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "encoding", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "collectMetadata", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.filePath = filePath;
+        this.encoding = encoding;
+        this.collectMetadata = collectMetadata;
+    }
+    /**
+     * Parses the YAML front matter from the given content string.
+     * @param content The string content of the markdown file.
+     * @returns An object representing the parsed front matter.
+     */
+    parseFrontMatter(content) {
+        if (!this.collectMetadata) {
+            return {};
+        }
+        const match = content.match(ObsidianFileLoader.FRONT_MATTER_REGEX);
+        if (!match) {
+            return {};
+        }
+        try {
+            const frontMatter = yaml.load(match[1]);
+            if (frontMatter.tags && typeof frontMatter.tags === "string") {
+                frontMatter.tags = frontMatter.tags.split(", ");
+            }
+            return frontMatter;
+        }
+        catch (e) {
+            console.warn("Encountered non-yaml frontmatter");
+            return {};
+        }
+    }
+    /**
+     * Removes YAML front matter from the given content string.
+     * @param content The string content of the markdown file.
+     * @returns The content string with the front matter removed.
+     */
+    removeFrontMatter(content) {
+        if (!this.collectMetadata) {
+            return content;
+        }
+        return content.replace(ObsidianFileLoader.FRONT_MATTER_REGEX, "");
+    }
+    /**
+     * Parses Obsidian-style tags from the given content string.
+     * @param content The string content of the markdown file.
+     * @returns A set of parsed tags.
+     */
+    parseObsidianTags(content) {
+        if (!this.collectMetadata) {
+            return new Set();
+        }
+        const matches = content.matchAll(ObsidianFileLoader.TAG_REGEX);
+        const tags = new Set();
+        for (const match of matches) {
+            tags.add(match[1]);
+        }
+        return tags;
+    }
+    /**
+     * Parses dataview fields from the given content string.
+     * @param content The string content of the markdown file.
+     * @returns A record object containing key-value pairs of dataview fields.
+     */
+    parseObsidianDataviewFields(content) {
+        if (!this.collectMetadata) {
+            return {};
+        }
+        const fields = {};
+        const lineMatches = content.matchAll(ObsidianFileLoader.DATAVIEW_LINE_REGEX);
+        for (const [, key, value] of lineMatches) {
+            fields[key] = value;
+        }
+        const bracketMatches = content.matchAll(ObsidianFileLoader.DATAVIEW_INLINE_BRACKET_REGEX);
+        for (const [, key, value] of bracketMatches) {
+            fields[key] = value;
+        }
+        const parenMatches = content.matchAll(ObsidianFileLoader.DATAVIEW_INLINE_PAREN_REGEX);
+        for (const [, key, value] of parenMatches) {
+            fields[key] = value;
+        }
+        return fields;
+    }
+    /**
+     * Converts metadata to a format compatible with Langchain.
+     * @param metadata The metadata object to convert.
+     * @returns A record object containing key-value pairs of Langchain-compatible metadata.
+     */
+    toLangchainCompatibleMetadata(metadata) {
+        const result = {};
+        for (const [key, value] of Object.entries(metadata)) {
+            if (typeof value === "string" || typeof value === "number") {
+                result[key] = value;
+            }
+            else {
+                result[key] = JSON.stringify(value);
+            }
+        }
+        return result;
+    }
+    /**
+     * It loads the Obsidian file, parses it, and returns a `Document` instance.
+     * @returns An array of `Document` instances to comply with the BaseDocumentLoader interface.
+     */
+    async load() {
+        const documents = [];
+        const { basename, readFile, stat } = await ObsidianFileLoader.imports();
+        const fileName = basename(this.filePath);
+        const stats = await stat(this.filePath);
+        let content = await readFile(this.filePath, this.encoding);
+        const frontMatter = this.parseFrontMatter(content);
+        const tags = this.parseObsidianTags(content);
+        const dataviewFields = this.parseObsidianDataviewFields(content);
+        content = this.removeFrontMatter(content);
+        const metadata = {
+            source: fileName,
+            path: this.filePath,
+            created: stats.birthtimeMs,
+            lastModified: stats.mtimeMs,
+            lastAccessed: stats.atimeMs,
+            ...this.toLangchainCompatibleMetadata(frontMatter),
+            ...dataviewFields,
+        };
+        if (tags.size || frontMatter.tags) {
+            metadata.tags = Array.from(new Set([...tags, ...(frontMatter.tags ?? [])])).join(",");
+        }
+        documents.push(new Document({
+            pageContent: content,
+            metadata,
+        }));
+        return documents;
+    }
+    /**
+     * Imports the necessary functions from the `node:path` and
+     * `node:fs/promises` modules. It is used to dynamically import the
+     * functions when needed. If the import fails, it throws an error
+     * indicating that the modules failed to load.
+     * @returns A promise that resolves to an object containing the imported functions.
+     */
+    static async imports() {
+        try {
+            const { basename } = await import("node:path");
+            const { readFile, stat } = await import("node:fs/promises");
+            return { basename, readFile, stat };
+        }
+        catch (e) {
+            console.error(e);
+            throw new Error(`Failed to load fs/promises. ObsidianFileLoader available only on environment 'node'. It appears you are running environment '${getEnv()}'. See https://<link to docs> for alternatives.`);
+        }
+    }
+}
+Object.defineProperty(ObsidianFileLoader, "FRONT_MATTER_REGEX", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: /^---\n(.*?)\n---\n/s
+});
+Object.defineProperty(ObsidianFileLoader, "TAG_REGEX", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: /(?:\s|^)#([a-zA-Z_][\w/-]*)/g
+});
+Object.defineProperty(ObsidianFileLoader, "DATAVIEW_LINE_REGEX", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: /^\s*(\w+)::\s*(.*)$/gm
+});
+Object.defineProperty(ObsidianFileLoader, "DATAVIEW_INLINE_BRACKET_REGEX", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: /\[(\w+)::\s*(.*)\]/gm
+});
+Object.defineProperty(ObsidianFileLoader, "DATAVIEW_INLINE_PAREN_REGEX", {
+    enumerable: true,
+    configurable: true,
+    writable: true,
+    value: /\((\w+)::\s*(.*)\)/gm
+});
+/**
+ * Represents a loader for directories containing Obsidian markdown files. This loader extends
+ * the DirectoryLoader and provides functionality to load and parse '.md' files with YAML frontmatter,
+ * Obsidian tags, and Dataview fields.
+ */
+export class ObsidianLoader extends DirectoryLoader {
+    /**
+     * Initializes a new instance of the ObsidianLoader class.
+     * @param directoryPath The path to the directory containing Obsidian markdown files.
+     * @param encoding The character encoding to use when reading files. Defaults to 'utf-8'.
+     * @param collectMetadata Determines whether metadata should be collected from the files. Defaults to true.
+     */
+    constructor(directoryPath, options) {
+        super(directoryPath, {
+            ".md": (filePath) => new ObsidianFileLoader(filePath, options),
+        }, true, UnknownHandling.Ignore);
+    }
+}

package/dist/embeddings/gradient_ai.cjs

@@ -0,0 +1,103 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GradientEmbeddings = void 0;
+const nodejs_sdk_1 = require("@gradientai/nodejs-sdk");
+const env_js_1 = require("../util/env.cjs");
+const chunk_js_1 = require("../util/chunk.cjs");
+const base_js_1 = require("./base.cjs");
+/**
+ * Class for generating embeddings using the Gradient AI's API. Extends the
+ * Embeddings class and implements GradientEmbeddingsParams and
+ */
+class GradientEmbeddings extends base_js_1.Embeddings {
+    constructor(fields) {
+        super(fields);
+        Object.defineProperty(this, "gradientAccessKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "workspaceId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "batchSize", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: 128
+        });
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        Object.defineProperty(this, "model", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.gradientAccessKey =
+            fields?.gradientAccessKey ??
+                (0, env_js_1.getEnvironmentVariable)("GRADIENT_ACCESS_TOKEN");
+        this.workspaceId =
+            fields?.workspaceId ?? (0, env_js_1.getEnvironmentVariable)("GRADIENT_WORKSPACE_ID");
+        if (!this.gradientAccessKey) {
+            throw new Error("Missing Gradient AI Access Token");
+        }
+        if (!this.workspaceId) {
+            throw new Error("Missing Gradient AI Workspace ID");
+        }
+    }
+    /**
+     * Method to generate embeddings for an array of documents. Splits the
+     * documents into batches and makes requests to the Gradient API to generate
+     * embeddings.
+     * @param texts Array of documents to generate embeddings for.
+     * @returns Promise that resolves to a 2D array of embeddings for each document.
+     */
+    async embedDocuments(texts) {
+        await this.setModel();
+        const mappedTexts = texts.map((text) => ({ input: text }));
+        const batches = (0, chunk_js_1.chunkArray)(mappedTexts, this.batchSize);
+        const batchRequests = batches.map((batch) => this.caller.call(async () => this.model.generateEmbeddings({
+            inputs: batch,
+        })));
+        const batchResponses = await Promise.all(batchRequests);
+        const embeddings = [];
+        for (let i = 0; i < batchResponses.length; i += 1) {
+            const batch = batches[i];
+            const { embeddings: batchResponse } = batchResponses[i];
+            for (let j = 0; j < batch.length; j += 1) {
+                embeddings.push(batchResponse[j].embedding);
+            }
+        }
+        return embeddings;
+    }
+    /**
+     * Method to generate an embedding for a single document. Calls the
+     * embedDocuments method with the document as the input.
+     * @param text Document to generate an embedding for.
+     * @returns Promise that resolves to an embedding for the document.
+     */
+    async embedQuery(text) {
+        const data = await this.embedDocuments([text]);
+        return data[0];
+    }
+    /**
+     * Method to set the model to use for generating embeddings.
+     * @sets the class' `model` value to that of the retrieved Embeddings Model.
+     */
+    async setModel() {
+        if (this.model)
+            return;
+        const gradient = new nodejs_sdk_1.Gradient({
+            accessToken: this.gradientAccessKey,
+            workspaceId: this.workspaceId,
+        });
+        this.model = await gradient.getEmbeddingsModel({
+            slug: "bge-large",
+        });
+    }
+}
+exports.GradientEmbeddings = GradientEmbeddings;

package/dist/embeddings/gradient_ai.d.ts

@@ -0,0 +1,48 @@
+import { Embeddings, EmbeddingsParams } from "./base.js";
+/**
+ * Interface for GradientEmbeddings parameters. Extends EmbeddingsParams and
+ * defines additional parameters specific to the GradientEmbeddings class.
+ */
+export interface GradientEmbeddingsParams extends EmbeddingsParams {
+    /**
+     * Gradient AI Access Token.
+     * Provide Access Token if you do not wish to automatically pull from env.
+     */
+    gradientAccessKey?: string;
+    /**
+     * Gradient Workspace Id.
+     * Provide workspace id if you do not wish to automatically pull from env.
+     */
+    workspaceId?: string;
+}
+/**
+ * Class for generating embeddings using the Gradient AI's API. Extends the
+ * Embeddings class and implements GradientEmbeddingsParams and
+ */
+export declare class GradientEmbeddings extends Embeddings implements GradientEmbeddingsParams {
+    gradientAccessKey?: string;
+    workspaceId?: string;
+    batchSize: number;
+    model: any;
+    constructor(fields: GradientEmbeddingsParams);
+    /**
+     * Method to generate embeddings for an array of documents. Splits the
+     * documents into batches and makes requests to the Gradient API to generate
+     * embeddings.
+     * @param texts Array of documents to generate embeddings for.
+     * @returns Promise that resolves to a 2D array of embeddings for each document.
+     */
+    embedDocuments(texts: string[]): Promise<number[][]>;
+    /**
+     * Method to generate an embedding for a single document. Calls the
+     * embedDocuments method with the document as the input.
+     * @param text Document to generate an embedding for.
+     * @returns Promise that resolves to an embedding for the document.
+     */
+    embedQuery(text: string): Promise<number[]>;
+    /**
+     * Method to set the model to use for generating embeddings.
+     * @sets the class' `model` value to that of the retrieved Embeddings Model.
+     */
+    setModel(): Promise<void>;
+}