langchain 0.1.36 → 0.2.0-rc.1

package/dist/experimental/multimodal_embeddings/googlevertexai.cjs

@@ -1,122 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.GoogleVertexAIMultimodalEmbeddings = void 0;
-const google_auth_library_1 = require("google-auth-library");
-const embeddings_1 = require("@langchain/core/embeddings");
-const googlevertexai_connection_js_1 = require("../../util/googlevertexai-connection.cjs");
-/**
- * Class for generating embeddings for text and images using Google's
- * Vertex AI. It extends the Embeddings base class and implements the
- * GoogleVertexAIMultimodalEmbeddingsParams interface.
- */
-class GoogleVertexAIMultimodalEmbeddings extends embeddings_1.Embeddings {
-    constructor(fields) {
-        super(fields ?? {});
-        Object.defineProperty(this, "model", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: "multimodalembedding@001"
-        });
-        Object.defineProperty(this, "connection", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.model = fields?.model ?? this.model;
-        this.connection = new googlevertexai_connection_js_1.GoogleVertexAILLMConnection({ ...fields, ...this }, this.caller, new google_auth_library_1.GoogleAuth({
-            scopes: "https://www.googleapis.com/auth/cloud-platform",
-            ...fields?.authOptions,
-        }));
-    }
-    /**
-     * Converts media (text or image) to an instance that can be used for
-     * generating embeddings.
-     * @param media The media (text or image) to be converted.
-     * @returns An instance of media that can be used for generating embeddings.
-     */
-    mediaToInstance(media) {
-        const ret = {};
-        if (media?.text) {
-            ret.text = media.text;
-        }
-        if (media.image) {
-            ret.image = {
-                bytesBase64Encoded: media.image.toString("base64"),
-            };
-        }
-        return ret;
-    }
-    /**
-     * Converts the response from Google Vertex AI to embeddings.
-     * @param response The response from Google Vertex AI.
-     * @returns An array of media embeddings.
-     */
-    responseToEmbeddings(response) {
-        return (response?.data).predictions.map((r) => ({
-            text: r.textEmbedding,
-            image: r.imageEmbedding,
-        }));
-    }
-    /**
-     * Generates embeddings for multiple media instances.
-     * @param media An array of media instances.
-     * @returns A promise that resolves to an array of media embeddings.
-     */
-    async embedMedia(media) {
-        // Only one media embedding request is allowed
-        return Promise.all(media.map((m) => this.embedMediaQuery(m)));
-    }
-    /**
-     * Generates embeddings for a single media instance.
-     * @param media A single media instance.
-     * @returns A promise that resolves to a media embedding.
-     */
-    async embedMediaQuery(media) {
-        const instance = this.mediaToInstance(media);
-        const instances = [instance];
-        const parameters = {};
-        const options = {};
-        const responses = await this.connection.request(instances, parameters, options);
-        const result = this.responseToEmbeddings(responses);
-        return result[0];
-    }
-    /**
-     * Generates embeddings for multiple images.
-     * @param images An array of images.
-     * @returns A promise that resolves to an array of image embeddings.
-     */
-    async embedImage(images) {
-        return this.embedMedia(images.map((image) => ({ image }))).then((embeddings) => embeddings.map((e) => e.image ?? []));
-    }
-    /**
-     * Generates embeddings for a single image.
-     * @param image A single image.
-     * @returns A promise that resolves to an image embedding.
-     */
-    async embedImageQuery(image) {
-        return this.embedMediaQuery({
-            image,
-        }).then((embeddings) => embeddings.image ?? []);
-    }
-    /**
-     * Generates embeddings for multiple text documents.
-     * @param documents An array of text documents.
-     * @returns A promise that resolves to an array of text document embeddings.
-     */
-    async embedDocuments(documents) {
-        return this.embedMedia(documents.map((text) => ({ text }))).then((embeddings) => embeddings.map((e) => e.text ?? []));
-    }
-    /**
-     * Generates embeddings for a single text document.
-     * @param document A single text document.
-     * @returns A promise that resolves to a text document embedding.
-     */
-    async embedQuery(document) {
-        return this.embedMediaQuery({
-            text: document,
-        }).then((embeddings) => embeddings.text ?? []);
-    }
-}
-exports.GoogleVertexAIMultimodalEmbeddings = GoogleVertexAIMultimodalEmbeddings;

package/dist/experimental/multimodal_embeddings/googlevertexai.d.ts

@@ -1,103 +0,0 @@
-/// <reference types="node" resolution-mode="require"/>
-import { GoogleAuthOptions } from "google-auth-library";
-import { Embeddings, EmbeddingsParams } from "@langchain/core/embeddings";
-import { GoogleVertexAIBaseLLMInput, GoogleVertexAIBasePrediction } from "../../types/googlevertexai-types.js";
-import { GoogleVertexAILLMResponse } from "../../util/googlevertexai-connection.js";
-/**
- * Parameters for the GoogleVertexAIMultimodalEmbeddings class, extending
- * both EmbeddingsParams and GoogleVertexAIConnectionParams.
- */
-export interface GoogleVertexAIMultimodalEmbeddingsParams extends EmbeddingsParams, GoogleVertexAIBaseLLMInput<GoogleAuthOptions> {
-}
-/**
- * An instance of media (text or image) that can be used for generating
- * embeddings.
- */
-interface GoogleVertexAIMultimodalEmbeddingsInstance {
-    text?: string;
-    image?: {
-        bytesBase64Encoded: string;
-    };
-}
-/**
- * The results of generating embeddings, extending
- * GoogleVertexAIBasePrediction. It includes text and image embeddings.
- */
-interface GoogleVertexAIMultimodalEmbeddingsResults extends GoogleVertexAIBasePrediction {
-    textEmbedding?: number[];
-    imageEmbedding?: number[];
-}
-/**
- * The media should have a text property, an image property, or both.
- */
-export type GoogleVertexAIMedia = {
-    text: string;
-    image?: Buffer;
-} | {
-    text?: string;
-    image: Buffer;
-};
-export type MediaEmbeddings = {
-    text?: number[];
-    image?: number[];
-};
-/**
- * Class for generating embeddings for text and images using Google's
- * Vertex AI. It extends the Embeddings base class and implements the
- * GoogleVertexAIMultimodalEmbeddingsParams interface.
- */
-export declare class GoogleVertexAIMultimodalEmbeddings extends Embeddings implements GoogleVertexAIMultimodalEmbeddingsParams {
-    model: string;
-    private connection;
-    constructor(fields?: GoogleVertexAIMultimodalEmbeddingsParams);
-    /**
-     * Converts media (text or image) to an instance that can be used for
-     * generating embeddings.
-     * @param media The media (text or image) to be converted.
-     * @returns An instance of media that can be used for generating embeddings.
-     */
-    mediaToInstance(media: GoogleVertexAIMedia): GoogleVertexAIMultimodalEmbeddingsInstance;
-    /**
-     * Converts the response from Google Vertex AI to embeddings.
-     * @param response The response from Google Vertex AI.
-     * @returns An array of media embeddings.
-     */
-    responseToEmbeddings(response: GoogleVertexAILLMResponse<GoogleVertexAIMultimodalEmbeddingsResults>): MediaEmbeddings[];
-    /**
-     * Generates embeddings for multiple media instances.
-     * @param media An array of media instances.
-     * @returns A promise that resolves to an array of media embeddings.
-     */
-    embedMedia(media: GoogleVertexAIMedia[]): Promise<MediaEmbeddings[]>;
-    /**
-     * Generates embeddings for a single media instance.
-     * @param media A single media instance.
-     * @returns A promise that resolves to a media embedding.
-     */
-    embedMediaQuery(media: GoogleVertexAIMedia): Promise<MediaEmbeddings>;
-    /**
-     * Generates embeddings for multiple images.
-     * @param images An array of images.
-     * @returns A promise that resolves to an array of image embeddings.
-     */
-    embedImage(images: Buffer[]): Promise<number[][]>;
-    /**
-     * Generates embeddings for a single image.
-     * @param image A single image.
-     * @returns A promise that resolves to an image embedding.
-     */
-    embedImageQuery(image: Buffer): Promise<number[]>;
-    /**
-     * Generates embeddings for multiple text documents.
-     * @param documents An array of text documents.
-     * @returns A promise that resolves to an array of text document embeddings.
-     */
-    embedDocuments(documents: string[]): Promise<number[][]>;
-    /**
-     * Generates embeddings for a single text document.
-     * @param document A single text document.
-     * @returns A promise that resolves to a text document embedding.
-     */
-    embedQuery(document: string): Promise<number[]>;
-}
-export {};

package/dist/experimental/multimodal_embeddings/googlevertexai.js

@@ -1,118 +0,0 @@
-import { GoogleAuth } from "google-auth-library";
-import { Embeddings } from "@langchain/core/embeddings";
-import { GoogleVertexAILLMConnection, } from "../../util/googlevertexai-connection.js";
-/**
- * Class for generating embeddings for text and images using Google's
- * Vertex AI. It extends the Embeddings base class and implements the
- * GoogleVertexAIMultimodalEmbeddingsParams interface.
- */
-export class GoogleVertexAIMultimodalEmbeddings extends Embeddings {
-    constructor(fields) {
-        super(fields ?? {});
-        Object.defineProperty(this, "model", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: "multimodalembedding@001"
-        });
-        Object.defineProperty(this, "connection", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.model = fields?.model ?? this.model;
-        this.connection = new GoogleVertexAILLMConnection({ ...fields, ...this }, this.caller, new GoogleAuth({
-            scopes: "https://www.googleapis.com/auth/cloud-platform",
-            ...fields?.authOptions,
-        }));
-    }
-    /**
-     * Converts media (text or image) to an instance that can be used for
-     * generating embeddings.
-     * @param media The media (text or image) to be converted.
-     * @returns An instance of media that can be used for generating embeddings.
-     */
-    mediaToInstance(media) {
-        const ret = {};
-        if (media?.text) {
-            ret.text = media.text;
-        }
-        if (media.image) {
-            ret.image = {
-                bytesBase64Encoded: media.image.toString("base64"),
-            };
-        }
-        return ret;
-    }
-    /**
-     * Converts the response from Google Vertex AI to embeddings.
-     * @param response The response from Google Vertex AI.
-     * @returns An array of media embeddings.
-     */
-    responseToEmbeddings(response) {
-        return (response?.data).predictions.map((r) => ({
-            text: r.textEmbedding,
-            image: r.imageEmbedding,
-        }));
-    }
-    /**
-     * Generates embeddings for multiple media instances.
-     * @param media An array of media instances.
-     * @returns A promise that resolves to an array of media embeddings.
-     */
-    async embedMedia(media) {
-        // Only one media embedding request is allowed
-        return Promise.all(media.map((m) => this.embedMediaQuery(m)));
-    }
-    /**
-     * Generates embeddings for a single media instance.
-     * @param media A single media instance.
-     * @returns A promise that resolves to a media embedding.
-     */
-    async embedMediaQuery(media) {
-        const instance = this.mediaToInstance(media);
-        const instances = [instance];
-        const parameters = {};
-        const options = {};
-        const responses = await this.connection.request(instances, parameters, options);
-        const result = this.responseToEmbeddings(responses);
-        return result[0];
-    }
-    /**
-     * Generates embeddings for multiple images.
-     * @param images An array of images.
-     * @returns A promise that resolves to an array of image embeddings.
-     */
-    async embedImage(images) {
-        return this.embedMedia(images.map((image) => ({ image }))).then((embeddings) => embeddings.map((e) => e.image ?? []));
-    }
-    /**
-     * Generates embeddings for a single image.
-     * @param image A single image.
-     * @returns A promise that resolves to an image embedding.
-     */
-    async embedImageQuery(image) {
-        return this.embedMediaQuery({
-            image,
-        }).then((embeddings) => embeddings.image ?? []);
-    }
-    /**
-     * Generates embeddings for multiple text documents.
-     * @param documents An array of text documents.
-     * @returns A promise that resolves to an array of text document embeddings.
-     */
-    async embedDocuments(documents) {
-        return this.embedMedia(documents.map((text) => ({ text }))).then((embeddings) => embeddings.map((e) => e.text ?? []));
-    }
-    /**
-     * Generates embeddings for a single text document.
-     * @param document A single text document.
-     * @returns A promise that resolves to a text document embedding.
-     */
-    async embedQuery(document) {
-        return this.embedMediaQuery({
-            text: document,
-        }).then((embeddings) => embeddings.text ?? []);
-    }
-}

package/dist/graphs/neo4j_graph.cjs

@@ -1,21 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const entrypoint_deprecation_js_1 = require("../util/entrypoint_deprecation.cjs");
-/* #__PURE__ */ (0, entrypoint_deprecation_js_1.logVersion010MigrationWarning)({
-    oldEntrypointName: "graphs/neo4j_graph",
-});
-__exportStar(require("@langchain/community/graphs/neo4j_graph"), exports);

package/dist/graphs/neo4j_graph.d.ts

@@ -1 +0,0 @@
-export * from "@langchain/community/graphs/neo4j_graph";

package/dist/graphs/neo4j_graph.js

@@ -1,5 +0,0 @@
-import { logVersion010MigrationWarning } from "../util/entrypoint_deprecation.js";
-/* #__PURE__ */ logVersion010MigrationWarning({
-    oldEntrypointName: "graphs/neo4j_graph",
-});
-export * from "@langchain/community/graphs/neo4j_graph";

package/dist/indexes/indexing.cjs

@@ -1,270 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.index = exports._isBaseDocumentLoader = exports._getSourceIdAssigner = exports._deduplicateInOrder = exports._batch = exports._HashedDocument = void 0;
-const uuid_1 = require("uuid");
-const base_1 = require("@langchain/community/indexes/base");
-const hash_1 = require("@langchain/core/utils/hash");
-const document_js_1 = require("../document.cjs");
-/**
- * HashedDocument is a Document with hashes calculated.
- * Hashes are calculated based on page content and metadata.
- * It is used for indexing.
- */
-class _HashedDocument {
-    constructor(fields) {
-        Object.defineProperty(this, "uid", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "hash_", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "contentHash", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "metadataHash", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "pageContent", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        Object.defineProperty(this, "metadata", {
-            enumerable: true,
-            configurable: true,
-            writable: true,
-            value: void 0
-        });
-        this.uid = fields.uid;
-        this.pageContent = fields.pageContent;
-        this.metadata = fields.metadata;
-    }
-    calculateHashes() {
-        const forbiddenKeys = ["hash_", "content_hash", "metadata_hash"];
-        for (const key of forbiddenKeys) {
-            if (key in this.metadata) {
-                throw new Error(`Metadata cannot contain key ${key} as it is reserved for internal use. Restricted keys: [${forbiddenKeys.join(", ")}]`);
-            }
-        }
-        const contentHash = this._hashStringToUUID(this.pageContent);
-        try {
-            const metadataHash = this._hashNestedDictToUUID(this.metadata);
-            this.contentHash = contentHash;
-            this.metadataHash = metadataHash;
-        }
-        catch (e) {
-            throw new Error(`Failed to hash metadata: ${e}. Please use a dict that can be serialized using json.`);
-        }
-        this.hash_ = this._hashStringToUUID(this.contentHash + this.metadataHash);
-        if (!this.uid) {
-            this.uid = this.hash_;
-        }
-    }
-    toDocument() {
-        return new document_js_1.Document({
-            pageContent: this.pageContent,
-            metadata: this.metadata,
-        });
-    }
-    static fromDocument(document, uid) {
-        const doc = new this({
-            pageContent: document.pageContent,
-            metadata: document.metadata,
-            uid: uid || document.uid,
-        });
-        doc.calculateHashes();
-        return doc;
-    }
-    _hashStringToUUID(inputString) {
-        const hash_value = (0, hash_1.insecureHash)(inputString);
-        return (0, uuid_1.v5)(hash_value, base_1.UUIDV5_NAMESPACE);
-    }
-    _hashNestedDictToUUID(data) {
-        const serialized_data = JSON.stringify(data, Object.keys(data).sort());
-        const hash_value = (0, hash_1.insecureHash)(serialized_data);
-        return (0, uuid_1.v5)(hash_value, base_1.UUIDV5_NAMESPACE);
-    }
-}
-exports._HashedDocument = _HashedDocument;
-function _batch(size, iterable) {
-    const batches = [];
-    let currentBatch = [];
-    iterable.forEach((item) => {
-        currentBatch.push(item);
-        if (currentBatch.length >= size) {
-            batches.push(currentBatch);
-            currentBatch = [];
-        }
-    });
-    if (currentBatch.length > 0) {
-        batches.push(currentBatch);
-    }
-    return batches;
-}
-exports._batch = _batch;
-function _deduplicateInOrder(hashedDocuments) {
-    const seen = new Set();
-    const deduplicated = [];
-    for (const hashedDoc of hashedDocuments) {
-        if (!hashedDoc.hash_) {
-            throw new Error("Hashed document does not have a hash");
-        }
-        if (!seen.has(hashedDoc.hash_)) {
-            seen.add(hashedDoc.hash_);
-            deduplicated.push(hashedDoc);
-        }
-    }
-    return deduplicated;
-}
-exports._deduplicateInOrder = _deduplicateInOrder;
-function _getSourceIdAssigner(sourceIdKey) {
-    if (sourceIdKey === null) {
-        return (_doc) => null;
-    }
-    else if (typeof sourceIdKey === "string") {
-        return (doc) => doc.metadata[sourceIdKey];
-    }
-    else if (typeof sourceIdKey === "function") {
-        return sourceIdKey;
-    }
-    else {
-        throw new Error(`sourceIdKey should be null, a string or a function, got ${typeof sourceIdKey}`);
-    }
-}
-exports._getSourceIdAssigner = _getSourceIdAssigner;
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-const _isBaseDocumentLoader = (arg) => {
-    if ("load" in arg &&
-        typeof arg.load === "function" &&
-        "loadAndSplit" in arg &&
-        typeof arg.loadAndSplit === "function") {
-        return true;
-    }
-    return false;
-};
-exports._isBaseDocumentLoader = _isBaseDocumentLoader;
-/**
- * Index data from the doc source into the vector store.
- *
- * Indexing functionality uses a manager to keep track of which documents
- * are in the vector store.
- *
- * This allows us to keep track of which documents were updated, and which
- * documents were deleted, which documents should be skipped.
- *
- * For the time being, documents are indexed using their hashes, and users
- *  are not able to specify the uid of the document.
- *
- * @param {IndexArgs} args
- * @param {BaseDocumentLoader | DocumentInterface[]} args.docsSource The source of documents to index. Can be a DocumentLoader or a list of Documents.
- * @param {RecordManagerInterface} args.recordManager The record manager to use for keeping track of indexed documents.
- * @param {VectorStore} args.vectorStore The vector store to use for storing the documents.
- * @param {IndexOptions | undefined} args.options Options for indexing.
- * @returns {Promise<IndexingResult>}
- */
-async function index(args) {
-    const { docsSource, recordManager, vectorStore, options } = args;
-    const { batchSize = 100, cleanup, sourceIdKey, cleanupBatchSize = 1000, forceUpdate = false, } = options ?? {};
-    if (cleanup === "incremental" && !sourceIdKey) {
-        throw new Error("sourceIdKey is required when cleanup mode is incremental. Please provide through 'options.sourceIdKey'.");
-    }
-    const docs = (0, exports._isBaseDocumentLoader)(docsSource)
-        ? await docsSource.load()
-        : docsSource;
-    const sourceIdAssigner = _getSourceIdAssigner(sourceIdKey ?? null);
-    const indexStartDt = await recordManager.getTime();
-    let numAdded = 0;
-    let numDeleted = 0;
-    let numUpdated = 0;
-    let numSkipped = 0;
-    const batches = _batch(batchSize ?? 100, docs);
-    for (const batch of batches) {
-        const hashedDocs = _deduplicateInOrder(batch.map((doc) => _HashedDocument.fromDocument(doc)));
-        const sourceIds = hashedDocs.map((doc) => sourceIdAssigner(doc));
-        if (cleanup === "incremental") {
-            hashedDocs.forEach((_hashedDoc, index) => {
-                const source = sourceIds[index];
-                if (source === null) {
-                    throw new Error("sourceIdKey must be provided when cleanup is incremental");
-                }
-            });
-        }
-        const batchExists = await recordManager.exists(hashedDocs.map((doc) => doc.uid));
-        const uids = [];
-        const docsToIndex = [];
-        const docsToUpdate = [];
-        const seenDocs = new Set();
-        hashedDocs.forEach((hashedDoc, i) => {
-            const docExists = batchExists[i];
-            if (docExists) {
-                if (forceUpdate) {
-                    seenDocs.add(hashedDoc.uid);
-                }
-                else {
-                    docsToUpdate.push(hashedDoc.uid);
-                    return;
-                }
-            }
-            uids.push(hashedDoc.uid);
-            docsToIndex.push(hashedDoc.toDocument());
-        });
-        if (docsToUpdate.length > 0) {
-            await recordManager.update(docsToUpdate, { timeAtLeast: indexStartDt });
-            numSkipped += docsToUpdate.length;
-        }
-        if (docsToIndex.length > 0) {
-            await vectorStore.addDocuments(docsToIndex, { ids: uids });
-            numAdded += docsToIndex.length - seenDocs.size;
-            numUpdated += seenDocs.size;
-        }
-        await recordManager.update(hashedDocs.map((doc) => doc.uid), { timeAtLeast: indexStartDt, groupIds: sourceIds });
-        if (cleanup === "incremental") {
-            sourceIds.forEach((sourceId) => {
-                if (!sourceId)
-                    throw new Error("Source id cannot be null");
-            });
-            const uidsToDelete = await recordManager.listKeys({
-                before: indexStartDt,
-                groupIds: sourceIds,
-            });
-            await vectorStore.delete({ ids: uidsToDelete });
-            await recordManager.deleteKeys(uidsToDelete);
-            numDeleted += uidsToDelete.length;
-        }
-    }
-    if (cleanup === "full") {
-        let uidsToDelete = await recordManager.listKeys({
-            before: indexStartDt,
-            limit: cleanupBatchSize,
-        });
-        while (uidsToDelete.length > 0) {
-            await vectorStore.delete({ ids: uidsToDelete });
-            await recordManager.deleteKeys(uidsToDelete);
-            numDeleted += uidsToDelete.length;
-            uidsToDelete = await recordManager.listKeys({
-                before: indexStartDt,
-                limit: cleanupBatchSize,
-            });
-        }
-    }
-    return {
-        numAdded,
-        numDeleted,
-        numUpdated,
-        numSkipped,
-    };
-}
-exports.index = index;