langchain 0.0.132 → 0.0.133

package/dist/vectorstores/googlevertexai.cjs

@@ -0,0 +1,550 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MatchingEngine = exports.IdDocument = void 0;
+const uuid = __importStar(require("uuid"));
+const flat_1 = __importDefault(require("flat"));
+const base_js_1 = require("./base.cjs");
+const document_js_1 = require("../document.cjs");
+const googlevertexai_connection_js_1 = require("../util/googlevertexai-connection.cjs");
+const async_caller_js_1 = require("../util/async_caller.cjs");
+/**
+ * A Document that optionally includes the ID of the document.
+ */
+class IdDocument extends document_js_1.Document {
+    constructor(fields) {
+        super(fields);
+        Object.defineProperty(this, "id", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.id = fields.id;
+    }
+}
+exports.IdDocument = IdDocument;
+class IndexEndpointConnection extends googlevertexai_connection_js_1.GoogleVertexAIConnection {
+    constructor(fields, caller) {
+        super(fields, caller);
+        Object.defineProperty(this, "indexEndpoint", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.indexEndpoint = fields.indexEndpoint;
+    }
+    async buildUrl() {
+        const projectId = await this.auth.getProjectId();
+        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/indexEndpoints/${this.indexEndpoint}`;
+        return url;
+    }
+    buildMethod() {
+        return "GET";
+    }
+    async request(options) {
+        return this._request(undefined, options);
+    }
+}
+class RemoveDatapointConnection extends googlevertexai_connection_js_1.GoogleVertexAIConnection {
+    constructor(fields, caller) {
+        super(fields, caller);
+        Object.defineProperty(this, "index", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.index = fields.index;
+    }
+    async buildUrl() {
+        const projectId = await this.auth.getProjectId();
+        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/indexes/${this.index}:removeDatapoints`;
+        return url;
+    }
+    buildMethod() {
+        return "POST";
+    }
+    async request(datapointIds, options) {
+        const data = {
+            datapointIds,
+        };
+        return this._request(data, options);
+    }
+}
+class UpsertDatapointConnection extends googlevertexai_connection_js_1.GoogleVertexAIConnection {
+    constructor(fields, caller) {
+        super(fields, caller);
+        Object.defineProperty(this, "index", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.index = fields.index;
+    }
+    async buildUrl() {
+        const projectId = await this.auth.getProjectId();
+        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/indexes/${this.index}:upsertDatapoints`;
+        return url;
+    }
+    buildMethod() {
+        return "POST";
+    }
+    async request(datapoints, options) {
+        const data = {
+            datapoints,
+        };
+        return this._request(data, options);
+    }
+}
+class FindNeighborsConnection extends googlevertexai_connection_js_1.GoogleVertexAIConnection {
+    constructor(params, caller) {
+        super(params, caller);
+        Object.defineProperty(this, "indexEndpoint", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "deployedIndexId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.indexEndpoint = params.indexEndpoint;
+        this.deployedIndexId = params.deployedIndexId;
+    }
+    async buildUrl() {
+        const projectId = await this.auth.getProjectId();
+        const url = `https://${this.endpoint}/${this.apiVersion}/projects/${projectId}/locations/${this.location}/indexEndpoints/${this.indexEndpoint}:findNeighbors`;
+        return url;
+    }
+    buildMethod() {
+        return "POST";
+    }
+    async request(request, options) {
+        return this._request(request, options);
+    }
+}
+/**
+ * A class that represents a connection to a Google Vertex AI Matching Engine
+ * instance.
+ */
+class MatchingEngine extends base_js_1.VectorStore {
+    constructor(embeddings, args) {
+        super(embeddings, args);
+        /**
+         * Docstore that retains the document, stored by ID
+         */
+        Object.defineProperty(this, "docstore", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * The host to connect to for queries and upserts.
+         */
+        Object.defineProperty(this, "apiEndpoint", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "apiVersion", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "v1"
+        });
+        Object.defineProperty(this, "endpoint", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "us-central1-aiplatform.googleapis.com"
+        });
+        Object.defineProperty(this, "location", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "us-central1"
+        });
+        /**
+         * The id for the index endpoint
+         */
+        Object.defineProperty(this, "indexEndpoint", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * The id for the index
+         */
+        Object.defineProperty(this, "index", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        /**
+         * The id for the "deployed index", which is an identifier in the
+         * index endpoint that references the index (but is not the index id)
+         */
+        Object.defineProperty(this, "deployedIndexId", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "callerParams", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "callerOptions", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "caller", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "indexEndpointClient", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "removeDatapointClient", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "upsertDatapointClient", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.embeddings = embeddings;
+        this.docstore = args.docstore;
+        this.apiEndpoint = args.apiEndpoint ?? this.apiEndpoint;
+        this.deployedIndexId = args.deployedIndexId ?? this.deployedIndexId;
+        this.apiVersion = args.apiVersion ?? this.apiVersion;
+        this.endpoint = args.endpoint ?? this.endpoint;
+        this.location = args.location ?? this.location;
+        this.indexEndpoint = args.indexEndpoint ?? this.indexEndpoint;
+        this.index = args.index ?? this.index;
+        this.callerParams = args.callerParams ?? this.callerParams;
+        this.callerOptions = args.callerOptions ?? this.callerOptions;
+        this.caller = new async_caller_js_1.AsyncCaller(this.callerParams || {});
+        const indexClientParams = {
+            endpoint: this.endpoint,
+            location: this.location,
+            apiVersion: this.apiVersion,
+            indexEndpoint: this.indexEndpoint,
+        };
+        this.indexEndpointClient = new IndexEndpointConnection(indexClientParams, this.caller);
+        const removeClientParams = {
+            endpoint: this.endpoint,
+            location: this.location,
+            apiVersion: this.apiVersion,
+            index: this.index,
+        };
+        this.removeDatapointClient = new RemoveDatapointConnection(removeClientParams, this.caller);
+        const upsertClientParams = {
+            endpoint: this.endpoint,
+            location: this.location,
+            apiVersion: this.apiVersion,
+            index: this.index,
+        };
+        this.upsertDatapointClient = new UpsertDatapointConnection(upsertClientParams, this.caller);
+    }
+    _vectorstoreType() {
+        return "googlevertexai";
+    }
+    async addDocuments(documents) {
+        const texts = documents.map((doc) => doc.pageContent);
+        const vectors = await this.embeddings.embedDocuments(texts);
+        return this.addVectors(vectors, documents);
+    }
+    async addVectors(vectors, documents) {
+        if (vectors.length !== documents.length) {
+            throw new Error(`Vectors and metadata must have the same length`);
+        }
+        const datapoints = vectors.map((vector, idx) => this.buildDatapoint(vector, documents[idx]));
+        const options = {};
+        const response = await this.upsertDatapointClient.request(datapoints, options);
+        if (Object.keys(response?.data ?? {}).length === 0) {
+            // Nothing in the response in the body means we saved it ok
+            const idDoc = documents;
+            const docsToStore = {};
+            idDoc.forEach((doc) => {
+                if (doc.id) {
+                    docsToStore[doc.id] = doc;
+                }
+            });
+            await this.docstore.add(docsToStore);
+        }
+    }
+    // TODO: Refactor this into a utility type and use with pinecone as well?
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    cleanMetadata(documentMetadata) {
+        function getStringArrays(prefix, 
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        m) {
+            let ret = {};
+            Object.keys(m).forEach((key) => {
+                const newPrefix = prefix.length > 0 ? `${prefix}.${key}` : key;
+                const val = m[key];
+                if (!val) {
+                    // Ignore it
+                }
+                else if (Array.isArray(val)) {
+                    // Make sure everything in the array is a string
+                    ret[newPrefix] = val.map((v) => `${v}`);
+                }
+                else if (typeof val === "object") {
+                    const subArrays = getStringArrays(newPrefix, val);
+                    ret = { ...ret, ...subArrays };
+                }
+            });
+            return ret;
+        }
+        const stringArrays = getStringArrays("", documentMetadata);
+        const flatMetadata = (0, flat_1.default)(documentMetadata);
+        Object.keys(flatMetadata).forEach((key) => {
+            Object.keys(stringArrays).forEach((arrayKey) => {
+                const matchKey = `${arrayKey}.`;
+                if (key.startsWith(matchKey)) {
+                    delete flatMetadata[key];
+                }
+            });
+        });
+        const metadata = {
+            ...flatMetadata,
+            ...stringArrays,
+        };
+        return metadata;
+    }
+    /**
+     * Given the metadata from a document, convert it to an array of Restriction
+     * objects that may be passed to the Matching Engine and stored.
+     * The default implementation flattens any metadata and includes it as
+     * an "allowList". Subclasses can choose to convert some of these to
+     * "denyList" items or to add additional restrictions (for example, to format
+     * dates into a different structure or to add additional restrictions
+     * based on the date).
+     * @param documentMetadata - The metadata from a document
+     * @returns a Restriction[] (or an array of a subclass, from the FilterType)
+     */
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    metadataToRestrictions(documentMetadata) {
+        const metadata = this.cleanMetadata(documentMetadata);
+        const restrictions = [];
+        for (const key of Object.keys(metadata)) {
+            // Make sure the value is an array (or that we'll ignore it)
+            let valArray;
+            const val = metadata[key];
+            if (val === null) {
+                valArray = null;
+            }
+            else if (Array.isArray(val) && val.length > 0) {
+                valArray = val;
+            }
+            else {
+                valArray = [`${val}`];
+            }
+            // Add to the restrictions if we do have a valid value
+            if (valArray) {
+                // Determine if this key is for the allowList or denyList
+                // TODO: get which ones should be on the deny list
+                const listType = "allowList";
+                // Create the restriction
+                const restriction = {
+                    namespace: key,
+                    [listType]: valArray,
+                };
+                // Add it to the restriction list
+                restrictions.push(restriction);
+            }
+        }
+        return restrictions;
+    }
+    /**
+     * Create an index datapoint for the vector and document id.
+     * If an id does not exist, create it and set the document to its value.
+     * @param vector
+     * @param document
+     */
+    buildDatapoint(vector, document) {
+        if (!document.id) {
+            // eslint-disable-next-line no-param-reassign
+            document.id = uuid.v4();
+        }
+        const ret = {
+            datapointId: document.id,
+            featureVector: vector,
+        };
+        const restrictions = this.metadataToRestrictions(document.metadata);
+        if (restrictions?.length > 0) {
+            ret.restricts = restrictions;
+        }
+        return ret;
+    }
+    async delete(params) {
+        const options = {};
+        await this.removeDatapointClient.request(params.ids, options);
+    }
+    async similaritySearchVectorWithScore(query, k, filter) {
+        // Format the query into the request
+        const deployedIndexId = await this.getDeployedIndexId();
+        const requestQuery = {
+            neighborCount: k,
+            datapoint: {
+                datapointId: `0`,
+                featureVector: query,
+            },
+        };
+        if (filter) {
+            requestQuery.datapoint.restricts = filter;
+        }
+        const request = {
+            deployedIndexId,
+            queries: [requestQuery],
+        };
+        // Build the connection.
+        // Has to be done here, since we defer getting the endpoint until
+        // we need it.
+        const apiEndpoint = await this.getPublicAPIEndpoint();
+        const findNeighborsParams = {
+            endpoint: apiEndpoint,
+            indexEndpoint: this.indexEndpoint,
+            apiVersion: this.apiVersion,
+            location: this.location,
+            deployedIndexId,
+        };
+        const connection = new FindNeighborsConnection(findNeighborsParams, this.caller);
+        // Make the call
+        const options = {};
+        const response = await connection.request(request, options);
+        // Get the document for each datapoint id and return them
+        const nearestNeighbors = response?.data?.nearestNeighbors ?? [];
+        const nearestNeighbor = nearestNeighbors[0];
+        const neighbors = nearestNeighbor?.neighbors ?? [];
+        const ret = await Promise.all(neighbors.map(async (neighbor) => {
+            const id = neighbor?.datapoint?.datapointId;
+            const distance = neighbor?.distance;
+            let doc;
+            try {
+                doc = await this.docstore.search(id);
+            }
+            catch (xx) {
+                // Documents that are in the index are returned, even if they
+                // are not in the document store, to allow for some way to get
+                // the id so they can be deleted.
+                console.error(xx);
+                console.warn([
+                    `Document with id "${id}" is missing from the backing docstore.`,
+                    `This can occur if you clear the docstore without deleting from the corresponding Matching Engine index.`,
+                    `To resolve this, you should call .delete() with this id as part of the "ids" parameter.`,
+                ].join("\n"));
+                doc = new document_js_1.Document({ pageContent: `Missing document ${id}` });
+            }
+            doc.id ??= id;
+            return [doc, distance];
+        }));
+        return ret;
+    }
+    /**
+     * For this index endpoint, figure out what API Endpoint URL and deployed
+     * index ID should be used to do upserts and queries.
+     * Also sets the `apiEndpoint` and `deployedIndexId` property for future use.
+     * @return The URL
+     */
+    async determinePublicAPIEndpoint() {
+        const response = await this.indexEndpointClient.request(this.callerOptions);
+        // Get the endpoint
+        const publicEndpointDomainName = response?.data?.publicEndpointDomainName;
+        this.apiEndpoint = publicEndpointDomainName;
+        // Determine which of the deployed indexes match the index id
+        // and get the deployed index id. The list of deployed index ids
+        // contain the "index name" or path, but not the index id by itself,
+        // so we need to extract it from the name
+        const indexPathPattern = /projects\/.+\/locations\/.+\/indexes\/(.+)$/;
+        const deployedIndexes = response?.data?.deployedIndexes ?? [];
+        const deployedIndex = deployedIndexes.find((index) => {
+            const deployedIndexPath = index.index;
+            const match = deployedIndexPath.match(indexPathPattern);
+            if (match) {
+                const [, potentialIndexId] = match;
+                if (potentialIndexId === this.index) {
+                    return true;
+                }
+            }
+            return false;
+        });
+        if (deployedIndex) {
+            this.deployedIndexId = deployedIndex.id;
+        }
+        return {
+            apiEndpoint: this.apiEndpoint,
+            deployedIndexId: this.deployedIndexId,
+        };
+    }
+    async getPublicAPIEndpoint() {
+        return (this.apiEndpoint ?? (await this.determinePublicAPIEndpoint()).apiEndpoint);
+    }
+    async getDeployedIndexId() {
+        return (this.deployedIndexId ??
+            (await this.determinePublicAPIEndpoint()).deployedIndexId);
+    }
+    static async fromTexts(texts, metadatas, embeddings, dbConfig) {
+        const docs = texts.map((text, index) => ({
+            pageContent: text,
+            metadata: Array.isArray(metadatas) ? metadatas[index] : metadatas,
+        }));
+        return this.fromDocuments(docs, embeddings, dbConfig);
+    }
+    static async fromDocuments(docs, embeddings, dbConfig) {
+        const ret = new MatchingEngine(embeddings, dbConfig);
+        await ret.addDocuments(docs);
+        return ret;
+    }
+}
+exports.MatchingEngine = MatchingEngine;