hyperspace-sdk-ts 3.1.1 → 3.1.3

package/dist/client.js

@@ -33,13 +33,14 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.HyperspaceClient = exports.HyperbolicMath = exports.DurabilityLevel = exports.TribunalContext = exports.CognitiveMath = void 0;
+exports.HyperspaceClient = exports.HyperbolicMath = exports.DurabilityLevel = exports.TribunalContext = exports.CognitiveMathExport = void 0;
 const grpc = __importStar(require("@grpc/grpc-js"));
 const hyperspace_grpc_pb_1 = require("./proto/hyperspace_grpc_pb");
 const hyperspace_pb_1 = require("./proto/hyperspace_pb");
 Object.defineProperty(exports, "DurabilityLevel", { enumerable: true, get: function () { return hyperspace_pb_1.DurabilityLevel; } });
 const hyperspace_pb = __importStar(require("./proto/hyperspace_pb")); // New, for direct access to types
-exports.CognitiveMath = __importStar(require("./math"));
+const CognitiveMath = __importStar(require("./math"));
+exports.CognitiveMathExport = __importStar(require("./math"));
 var agents_1 = require("./agents");
 Object.defineProperty(exports, "TribunalContext", { enumerable: true, get: function () { return agents_1.TribunalContext; } });
 exports.HyperbolicMath = {
@@ -224,7 +225,82 @@ class HyperspaceClient {
         });
         return out;
     }
+    toProtoFilter(f) {
+        const pf = new hyperspace_pb.Filter();
+        if (f.match) {
+            const m = new hyperspace_pb.Match();
+            m.setKey(f.match.key);
+            m.setValue(f.match.value);
+            pf.setMatch(m);
+        }
+        else if (f.prefix) {
+            const p = new hyperspace_pb.Prefix();
+            p.setKey(f.prefix.key);
+            p.setPrefix(f.prefix.prefix);
+            pf.setPrefix(p);
+        }
+        else if (f.range) {
+            const r = new hyperspace_pb.Range();
+            r.setKey(f.range.key);
+            if (f.range.gte !== undefined) {
+                if (Number.isInteger(f.range.gte))
+                    r.setGte(f.range.gte);
+                else
+                    r.setGteF64(f.range.gte);
+            }
+            if (f.range.lte !== undefined) {
+                if (Number.isInteger(f.range.lte))
+                    r.setLte(f.range.lte);
+                else
+                    r.setLteF64(f.range.lte);
+            }
+            pf.setRange(r);
+        }
+        else if (f.inCone) {
+            const c = new hyperspace_pb.InCone();
+            c.setAxesList(f.inCone.axes);
+            c.setAperturesList(f.inCone.apertures);
+            c.setCen(f.inCone.cen[0] || 0);
+            pf.setInCone(c);
+        }
+        else if (f.inBall) {
+            const b = new hyperspace_pb.InBall();
+            b.setCenterList(f.inBall.center);
+            b.setRadius(f.inBall.radius);
+            pf.setInBall(b);
+        }
+        else if (f.inBox) {
+            const b = new hyperspace_pb.InBox();
+            b.setMinBoundsList(f.inBox.minBounds);
+            b.setMaxBoundsList(f.inBox.maxBounds);
+            pf.setInBox(b);
+        }
+        else if (f.and) {
+            const andOp = new hyperspace_pb.FilterAnd();
+            andOp.setConditionsList(f.and.map(cond => this.toProtoFilter(cond)));
+            pf.setAndOp(andOp);
+        }
+        else if (f.or) {
+            const orOp = new hyperspace_pb.FilterOr();
+            orOp.setConditionsList(f.or.map(cond => this.toProtoFilter(cond)));
+            pf.setOrOp(orOp);
+        }
+        else if (f.not) {
+            const notOp = new hyperspace_pb.FilterNot();
+            notOp.setCondition(this.toProtoFilter(f.not));
+            pf.setNotOp(notOp);
+        }
+        return pf;
+    }
     constructor(host = 'localhost:50051', apiKey, userId) {
+        this.collectionKeys = {};
+        this.encryptionContexts = {};
+        this.collectionMetrics = {};
+        this.collectionNoiseSigmas = {};
+        this.collectionSchemas = {};
+        this.host = host;
+        this.apiKey = apiKey;
+        this.userId = userId;
         const options = {
             'grpc.max_send_message_length': 64 * 1024 * 1024,
             'grpc.max_receive_message_length': 64 * 1024 * 1024,
@@ -243,13 +319,246 @@ class HyperspaceClient {
             this.metadata.add('x-hyperspace-user-id', userId);
         }
     }
+    registerCollectionKey(collectionName, key, metric = "l2", noiseSigma = 0.02, schema) {
+        this.collectionKeys[collectionName] = key;
+        this.collectionMetrics[collectionName] = metric;
+        this.collectionNoiseSigmas[collectionName] = noiseSigma;
+        if (schema) {
+            this.collectionSchemas[collectionName] = schema;
+        }
+        if (this.encryptionContexts[collectionName]) {
+            delete this.encryptionContexts[collectionName];
+        }
+    }
+    deriveKeys(password, collectionName) {
+        const crypto = require('crypto');
+        const salt = crypto.createHash('sha256').update(collectionName).digest();
+        const aesKey = crypto.pbkdf2Sync(password, salt, 100000, 32, 'sha256');
+        const hmacKey = crypto.pbkdf2Sync(password, salt, 100000, 32, 'sha256');
+        return { aesKey, hmacKey };
+    }
+    encryptPayload(plaintext, aesKey) {
+        const crypto = require('crypto');
+        const iv = crypto.randomBytes(12);
+        const pbkdf2Salt = crypto.randomBytes(16);
+        const derivedAesKey = crypto.pbkdf2Sync(aesKey, pbkdf2Salt, 100000, 32, 'sha256');
+        const cipherPayload = crypto.createCipheriv('aes-256-gcm', derivedAesKey, iv);
+        const encryptedPayload = Buffer.concat([cipherPayload.update(plaintext), cipherPayload.final()]);
+        const payloadTag = cipherPayload.getAuthTag();
+        return Buffer.concat([pbkdf2Salt, iv, encryptedPayload, payloadTag]);
+    }
+    decryptPayload(data, aesKey) {
+        const crypto = require('crypto');
+        if (data.length < 16 + 12 + 16) {
+            throw new Error("Invalid encrypted payload size");
+        }
+        const pbkdf2Salt = data.subarray(0, 16);
+        const iv = data.subarray(16, 28);
+        const ciphertext = data.subarray(28, data.length - 16);
+        const tag = data.subarray(data.length - 16);
+        const derivedAesKey = crypto.pbkdf2Sync(aesKey, pbkdf2Salt, 100000, 32, 'sha256');
+        const decipher = crypto.createDecipheriv('aes-256-gcm', derivedAesKey, iv);
+        decipher.setAuthTag(tag);
+        return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
+    }
+    hashMetadataKey(key, hmacKey) {
+        const crypto = require('crypto');
+        const hash = crypto.createHmac('sha256', hmacKey).update(key).digest('hex');
+        return "tag_" + hash.slice(0, 16);
+    }
+    hashMetadataValue(value, hmacKey) {
+        const crypto = require('crypto');
+        const hash = crypto.createHmac('sha256', hmacKey).update(value).digest('hex');
+        return "val_" + hash;
+    }
+    async _getEncryptionContext(collection, vectorDim, metric = "l2") {
+        if (!collection)
+            return null;
+        const key = this.collectionKeys[collection];
+        if (!key)
+            return null;
+        if (!this.collectionSchemas[collection]) {
+            try {
+                const stats = await this.getCollectionStats(collection);
+                if (stats && stats.schema) {
+                    this.collectionSchemas[collection] = stats.schema;
+                }
+            }
+            catch (e) { }
+        }
+        if (!this.encryptionContexts[collection]) {
+            const { aesKey, hmacKey } = this.deriveKeys(key, collection);
+            this.encryptionContexts[collection] = {
+                aesKey,
+                hmacKey,
+                projectionMatrices: {}
+            };
+        }
+        const context = this.encryptionContexts[collection];
+        if (vectorDim !== undefined) {
+            if (!context.projectionMatrices[vectorDim]) {
+                const isLorentz = ["lorentz", "poincare"].includes(metric.toLowerCase());
+                const matrixDim = metric.toLowerCase() === "poincare" ? vectorDim + 1 : vectorDim;
+                if (isLorentz) {
+                    context.projectionMatrices[vectorDim] = CognitiveMath.generateLorentzMatrix(matrixDim, context.hmacKey);
+                }
+                else {
+                    context.projectionMatrices[vectorDim] = CognitiveMath.generateOrthogonalMatrix(matrixDim, context.hmacKey);
+                }
+            }
+        }
+        return context;
+    }
+    _projectSingleBlock(subVec, metric, context, blockId) {
+        const dim = subVec.length;
+        if (dim === 0)
+            return [];
+        const cacheKey = blockId ? `${dim}_${blockId}` : `${dim}`;
+        if (!context.projectionMatrices[cacheKey]) {
+            const isLorentz = ["lorentz", "poincare"].includes(metric.toLowerCase());
+            const matrixDim = metric.toLowerCase() === "poincare" ? dim + 1 : dim;
+            const crypto = require('crypto');
+            let seed = context.hmacKey;
+            if (blockId) {
+                seed = crypto.createHash('sha256').update(seed).update(blockId).digest();
+            }
+            if (isLorentz) {
+                context.projectionMatrices[cacheKey] = CognitiveMath.generateLorentzMatrix(matrixDim, seed);
+            }
+            else {
+                context.projectionMatrices[cacheKey] = CognitiveMath.generateOrthogonalMatrix(matrixDim, seed);
+            }
+        }
+        const matrix = context.projectionMatrices[cacheKey];
+        if (metric.toLowerCase() === "poincare") {
+            const lorentzVec = CognitiveMath.poincareToLorentz(subVec);
+            const projLorentz = CognitiveMath.projectVector(lorentzVec, matrix);
+            return CognitiveMath.lorentzToPoincare(projLorentz);
+        }
+        else {
+            return CognitiveMath.projectVector(subVec, matrix);
+        }
+    }
+    _projectCollectionVector(collection, vector, context, metric = "l2") {
+        const schema = this.collectionSchemas[collection];
+        if (!schema || !schema.components || schema.components.length === 0) {
+            return this._projectSingleBlock(vector, metric, context);
+        }
+        const components = schema.components;
+        const cascade = schema.cascadePipeline || [];
+        const componentCutoffs = {};
+        for (const layer of cascade) {
+            const compName = layer.componentName;
+            const cutoff = layer.cutoffDimension;
+            if (compName && cutoff) {
+                if (!componentCutoffs[compName]) {
+                    componentCutoffs[compName] = [];
+                }
+                componentCutoffs[compName].push(cutoff);
+            }
+        }
+        for (const compName in componentCutoffs) {
+            componentCutoffs[compName] = Array.from(new Set(componentCutoffs[compName])).sort((a, b) => a - b);
+        }
+        const projectedParts = [];
+        let currentOffset = 0;
+        for (const comp of components) {
+            const compName = comp.name;
+            const compMetric = comp.metric;
+            const compDim = comp.fullDimension;
+            if (currentOffset >= vector.length) {
+                break;
+            }
+            let subVec = vector.slice(currentOffset, currentOffset + compDim);
+            if (subVec.length < compDim) {
+                subVec = subVec.concat(new Array(compDim - subVec.length).fill(0));
+            }
+            const cutoffs = componentCutoffs[compName] || [];
+            const validCutoffs = cutoffs.filter(c => c < compDim);
+            let projSub = [];
+            if (validCutoffs.length === 0) {
+                projSub = this._projectSingleBlock(subVec, compMetric, context);
+            }
+            else {
+                let blockStart = 0;
+                for (const cutoff of validCutoffs) {
+                    const blockData = subVec.slice(blockStart, cutoff);
+                    const projBlock = this._projectSingleBlock(blockData, compMetric, context, `${compName}_block_${blockStart}_${cutoff}`);
+                    projSub = projSub.concat(projBlock);
+                    blockStart = cutoff;
+                }
+                if (blockStart < compDim) {
+                    const blockData = subVec.slice(blockStart, compDim);
+                    const projBlock = this._projectSingleBlock(blockData, compMetric, context, `${compName}_block_${blockStart}_${compDim}`);
+                    projSub = projSub.concat(projBlock);
+                }
+            }
+            projectedParts.push(...projSub);
+            currentOffset += compDim;
+        }
+        if (currentOffset < vector.length) {
+            projectedParts.push(...vector.slice(currentOffset));
+        }
+        return projectedParts;
+    }
+    _encryptFilters(filters, context) {
+        if (!filters)
+            return filters;
+        return filters.map(f => {
+            const nf = { ...f };
+            if (nf.match) {
+                nf.match = {
+                    key: this.hashMetadataKey(nf.match.key, context.hmacKey),
+                    value: this.hashMetadataValue(nf.match.value, context.hmacKey)
+                };
+            }
+            if (nf.prefix) {
+                nf.prefix = {
+                    key: this.hashMetadataKey(nf.prefix.key, context.hmacKey),
+                    prefix: this.hashMetadataValue(nf.prefix.prefix, context.hmacKey)
+                };
+            }
+            if (nf.and) {
+                nf.and = this._encryptFilters(nf.and, context);
+            }
+            if (nf.or) {
+                nf.or = this._encryptFilters(nf.or, context);
+            }
+            if (nf.not) {
+                nf.not = this._encryptFilters([nf.not], context)[0];
+            }
+            return nf;
+        });
+    }
     // ... (create/delete unchanged) ...
-    createCollection(name, dimension, metric) {
+    createCollection(name, schema, encryptionKey = '', noiseSigma = 0.02) {
+        const metric = (schema.components && schema.components[0]) ? schema.components[0].metric : "l2";
+        if (encryptionKey) {
+            this.registerCollectionKey(name, encryptionKey, metric, noiseSigma, schema);
+        }
         return new Promise((resolve, reject) => {
             const req = new hyperspace_pb_1.CreateCollectionRequest();
             req.setName(name);
-            req.setDimension(dimension);
-            req.setMetric(metric);
+            const protoSchema = new hyperspace_pb.CollectionSchema();
+            const components = schema.components.map(c => {
+                const comp = new hyperspace_pb.VectorComponent();
+                comp.setName(c.name);
+                comp.setMetric(c.metric);
+                comp.setFullDimension(c.fullDimension);
+                comp.setWeight(c.weight);
+                return comp;
+            });
+            protoSchema.setComponentsList(components);
+            const pipeline = schema.cascadePipeline.map(l => {
+                const layer = new hyperspace_pb.MrlLayer();
+                layer.setComponentName(l.componentName);
+                layer.setCutoffDimension(l.cutoffDimension);
+                layer.setStoreInRam(l.storeInRam);
+                layer.setRerankTopK(l.rerankTopK);
+                return layer;
+            });
+            protoSchema.setCascadePipelineList(pipeline);
+            req.setSchema(protoSchema);
             this.client.createCollection(req, this.metadata, (err, resp) => {
                 if (err)
                     return reject(err);
@@ -268,22 +577,79 @@ class HyperspaceClient {
             });
         });
     }
+    freezeCollection(name) {
+        return new Promise((resolve, reject) => {
+            const req = new hyperspace_pb.FreezeCollectionRequest();
+            req.setName(name);
+            this.client.freezeCollection(req, this.metadata, (err, resp) => {
+                if (err)
+                    return reject(err);
+                resolve(resp.getStatus());
+            });
+        });
+    }
+    unfreezeCollection(name) {
+        return new Promise((resolve, reject) => {
+            const req = new hyperspace_pb.UnfreezeCollectionRequest();
+            req.setName(name);
+            this.client.unfreezeCollection(req, this.metadata, (err, resp) => {
+                if (err)
+                    return reject(err);
+                resolve(resp.getStatus());
+            });
+        });
+    }
     listCollections() {
         return new Promise((resolve, reject) => {
             const req = new hyperspace_pb_1.Empty();
             this.client.listCollections(req, this.metadata, (err, resp) => {
                 if (err)
                     return reject(err);
-                const list = resp.getCollectionsList().map(c => ({
-                    name: c.getName(),
-                    count: c.getCount(),
-                    dimension: c.getDimension(),
-                    metric: c.getMetric()
-                }));
+                const list = resp.getCollectionsList().map(c => {
+                    const info = {
+                        name: c.getName(),
+                        count: c.getCount()
+                    };
+                    const protoSchema = c.getSchema();
+                    if (protoSchema) {
+                        info.schema = {
+                            components: protoSchema.getComponentsList().map((comp) => ({
+                                name: comp.getName(),
+                                metric: comp.getMetric(),
+                                fullDimension: comp.getFullDimension(),
+                                weight: comp.getWeight()
+                            })),
+                            cascadePipeline: protoSchema.getCascadePipelineList().map((layer) => ({
+                                componentName: layer.getComponentName(),
+                                cutoffDimension: layer.getCutoffDimension(),
+                                storeInRam: layer.getStoreInRam(),
+                                rerankTopK: layer.getRerankTopK()
+                            }))
+                        };
+                    }
+                    return info;
+                });
                 resolve(list);
             });
         });
     }
+    getPoints(ids, collection = '') {
+        return new Promise((resolve, reject) => {
+            const req = new hyperspace_pb_1.GetPointsRequest();
+            req.setCollection(collection);
+            req.setIdsList(ids);
+            this.client.getPoints(req, this.metadata, (err, resp) => {
+                if (err)
+                    return reject(err);
+                const points = resp.getPointsList().map((p) => ({
+                    id: p.getId(),
+                    vector: p.getVectorList(),
+                    metadata: p.getMetadataMap().toObject()
+                }));
+                resolve(points);
+            });
+        });
+    }
     delete(id, collection = '') {
         return new Promise((resolve, reject) => {
             const req = new hyperspace_pb.DeleteRequest();
@@ -296,33 +662,87 @@ class HyperspaceClient {
             });
         });
     }
-    insert(id, vector, meta, collection = '', durability = hyperspace_pb_1.DurabilityLevel.DEFAULT_LEVEL, typedMetadata) {
-        return new Promise((resolve, reject) => {
-            const req = new hyperspace_pb_1.InsertRequest();
-            req.setVectorList(HyperspaceClient.toVectorList(vector));
-            req.setId(id);
-            if (meta) {
-                const map = req.getMetadataMap();
-                for (const k in meta)
-                    map.set(k, meta[k]);
+    insert(id, vector, meta, collection = '', durability = hyperspace_pb_1.DurabilityLevel.DEFAULT_LEVEL, typedMetadata, payload // Sidecar Payload Storage (v3.2)
+    ) {
+        return new Promise(async (resolve, reject) => {
+            var _a;
+            try {
+                const req = new hyperspace_pb_1.InsertRequest();
+                let vectorList = HyperspaceClient.toVectorList(vector);
+                const metric = this.collectionMetrics[collection] || "l2";
+                const context = await this._getEncryptionContext(collection, vectorList.length, metric);
+                if (context) {
+                    const sigma = (_a = this.collectionNoiseSigmas[collection]) !== null && _a !== void 0 ? _a : 0.02;
+                    if (sigma > 0.0) {
+                        vectorList = CognitiveMath.injectAnisotropicNoise(vectorList, context.hmacKey, sigma);
+                    }
+                    vectorList = this._projectCollectionVector(collection, vectorList, context, metric);
+                    let rawPayload = null;
+                    if (payload) {
+                        rawPayload = Buffer.from(payload);
+                    }
+                    if (rawPayload) {
+                        const encrypted = this.encryptPayload(rawPayload, context.aesKey);
+                        req.setPayload(new Uint8Array(encrypted));
+                    }
+                    if (meta) {
+                        const map = req.getMetadataMap();
+                        for (const k in meta) {
+                            const ek = this.hashMetadataKey(k, context.hmacKey);
+                            const ev = this.hashMetadataValue(meta[k], context.hmacKey);
+                            map.set(ek, ev);
+                        }
+                    }
+                    if (typedMetadata) {
+                        const map = req.getTypedMetadataMap();
+                        for (const k in typedMetadata) {
+                            const ek = this.hashMetadataKey(k, context.hmacKey);
+                            const ev = this.hashMetadataValue(String(typedMetadata[k]), context.hmacKey);
+                            map.set(ek, HyperspaceClient.toProtoMetadataValue(ev));
+                        }
+                    }
+                }
+                else {
+                    if (meta) {
+                        const map = req.getMetadataMap();
+                        for (const k in meta)
+                            map.set(k, meta[k]);
+                    }
+                    if (typedMetadata) {
+                        const map = req.getTypedMetadataMap();
+                        for (const k in typedMetadata)
+                            map.set(k, HyperspaceClient.toProtoMetadataValue(typedMetadata[k]));
+                    }
+                    if (payload) {
+                        req.setPayload(payload);
+                    }
+                }
+                req.setVectorList(vectorList);
+                req.setId(id);
+                req.setCollection(collection);
+                req.setOriginNodeId('');
+                req.setLogicalClock(0);
+                req.setDurability(durability);
+                this.client.insert(req, this.metadata, (err, resp) => {
+                    if (err)
+                        return reject(err);
+                    resolve(resp.getSuccess());
+                });
             }
-            if (typedMetadata) {
-                const map = req.getTypedMetadataMap();
-                for (const k in typedMetadata)
-                    map.set(k, HyperspaceClient.toProtoMetadataValue(typedMetadata[k]));
+            catch (e) {
+                reject(e);
             }
-            req.setCollection(collection);
-            req.setOriginNodeId('');
-            req.setLogicalClock(0);
-            req.setDurability(durability);
-            this.client.insert(req, this.metadata, (err, resp) => {
-                if (err)
-                    return reject(err);
-                resolve(resp.getSuccess());
-            });
         });
     }
     insertText(id, text, meta, collection = '', durability = hyperspace_pb_1.DurabilityLevel.DEFAULT_LEVEL) {
+        if (this.collectionKeys[collection]) {
+            if (!this.embedder) {
+                return Promise.reject(new Error("An embedder must be configured to use insertText on encrypted collections."));
+            }
+            return Promise.resolve(this.embedder.encode(text)).then(vector => {
+                return this.insert(id, vector, meta, collection, durability, undefined, Buffer.from(text, 'utf-8'));
+            });
+        }
         return new Promise((resolve, reject) => {
             const req = new hyperspace_pb_1.InsertTextRequest();
             req.setText(text);
@@ -383,126 +803,145 @@ class HyperspaceClient {
         });
     }
     search(vector, topK, collection = '', options) {
-        return new Promise((resolve, reject) => {
-            const req = new hyperspace_pb_1.SearchRequest();
-            req.setVectorList(HyperspaceClient.toVectorList(vector));
-            req.setTopK(topK);
-            req.setCollection(collection);
-            if (options === null || options === void 0 ? void 0 : options.filters) {
-                const protoFilters = options.filters.map(f => {
-                    const pf = new hyperspace_pb.Filter();
-                    if (f.match) {
-                        const m = new hyperspace_pb.Match();
-                        m.setKey(f.match.key);
-                        m.setValue(f.match.value);
-                        pf.setMatch(m);
+        return new Promise(async (resolve, reject) => {
+            var _a;
+            try {
+                const req = new hyperspace_pb_1.SearchRequest();
+                let vectorList = HyperspaceClient.toVectorList(vector);
+                req.setTopK(topK);
+                req.setCollection(collection);
+                const metric = this.collectionMetrics[collection] || "l2";
+                const context = await this._getEncryptionContext(collection, vectorList.length, metric);
+                let optFilters = options === null || options === void 0 ? void 0 : options.filters;
+                let optFilter = options === null || options === void 0 ? void 0 : options.filter;
+                let includePayload = options === null || options === void 0 ? void 0 : options.includePayload;
+                if (context) {
+                    const sigma = (_a = this.collectionNoiseSigmas[collection]) !== null && _a !== void 0 ? _a : 0.02;
+                    if (sigma > 0.0) {
+                        vectorList = CognitiveMath.injectAnisotropicNoise(vectorList, context.hmacKey, sigma);
                     }
-                    else if (f.range) {
-                        const r = new hyperspace_pb.Range();
-                        r.setKey(f.range.key);
-                        if (f.range.gte !== undefined) {
-                            if (Number.isInteger(f.range.gte))
-                                r.setGte(f.range.gte);
-                            else
-                                r.setGteF64(f.range.gte);
+                    vectorList = this._projectCollectionVector(collection, vectorList, context, metric);
+                    if (optFilter) {
+                        const hashedFilter = {};
+                        for (const k in optFilter) {
+                            const ek = this.hashMetadataKey(k, context.hmacKey);
+                            const ev = this.hashMetadataValue(optFilter[k], context.hmacKey);
+                            hashedFilter[ek] = ev;
                         }
-                        if (f.range.lte !== undefined) {
-                            if (Number.isInteger(f.range.lte))
-                                r.setLte(f.range.lte);
-                            else
-                                r.setLteF64(f.range.lte);
-                        }
-                        pf.setRange(r);
+                        optFilter = hashedFilter;
+                    }
+                    if (optFilters) {
+                        optFilters = this._encryptFilters(optFilters, context);
+                    }
+                    includePayload = true;
+                }
+                req.setVectorList(vectorList);
+                if (optFilter) {
+                    const map = req.getFilterMap();
+                    for (const k in optFilter) {
+                        map.set(k, optFilter[k]);
                     }
-                    return pf;
+                }
+                if ((options === null || options === void 0 ? void 0 : options.restartFactor) !== undefined) {
+                    const map = req.getFilterMap();
+                    map.set('wave_restart_factor', options.restartFactor.toString());
+                }
+                if ((options === null || options === void 0 ? void 0 : options.useWave) !== undefined) {
+                    req.setUseWave(options.useWave);
+                }
+                if (optFilters) {
+                    req.setFiltersList(optFilters.map(f => this.toProtoFilter(f)));
+                }
+                if (options === null || options === void 0 ? void 0 : options.hybridQuery)
+                    req.setHybridQuery(options.hybridQuery);
+                if ((options === null || options === void 0 ? void 0 : options.hybridAlpha) !== undefined)
+                    req.setHybridAlpha(options.hybridAlpha);
+                if ((options === null || options === void 0 ? void 0 : options.mrlDimension) !== undefined)
+                    req.setMrlDimension(options.mrlDimension);
+                if ((options === null || options === void 0 ? void 0 : options.useWasserstein) !== undefined)
+                    req.setUseWasserstein(options.useWasserstein);
+                if (includePayload !== undefined)
+                    req.setIncludePayload(includePayload);
+                if (options === null || options === void 0 ? void 0 : options.componentWeights) {
+                    const map = req.getComponentWeightsMap();
+                    for (const k in options.componentWeights) {
+                        map.set(k, options.componentWeights[k]);
+                    }
+                }
+                if (options === null || options === void 0 ? void 0 : options.bm25) {
+                    const bm25Msg = new hyperspace_pb_1.Bm25Options();
+                    if (options.bm25.method !== undefined)
+                        bm25Msg.setMethod(options.bm25.method);
+                    if (options.bm25.k1 !== undefined)
+                        bm25Msg.setK1(options.bm25.k1);
+                    if (options.bm25.b !== undefined)
+                        bm25Msg.setB(options.bm25.b);
+                    if (options.bm25.delta !== undefined)
+                        bm25Msg.setDelta(options.bm25.delta);
+                    if (options.bm25.language !== undefined)
+                        bm25Msg.setLanguage(options.bm25.language);
+                    if (options.bm25.ngrams !== undefined)
+                        bm25Msg.setNgrams(options.bm25.ngrams);
+                    if (options.bm25.fusionMethod !== undefined)
+                        bm25Msg.setFusionMethod(options.bm25.fusionMethod);
+                    req.setBm25Options(bm25Msg);
+                }
+                this.client.search(req, this.metadata, (err, resp) => {
+                    if (err)
+                        return reject(err);
+                    const results = resp.getResultsList().map(r => {
+                        const metaMap = r.getMetadataMap();
+                        const meta = {};
+                        if (metaMap.getLength() > 0) {
+                            metaMap.forEach((entry, key) => {
+                                meta[key] = entry;
+                            });
+                        }
+                        let payloadBytes = r.getPayload_asU8();
+                        if (context && payloadBytes && payloadBytes.length > 0) {
+                            try {
+                                const decrypted = this.decryptPayload(Buffer.from(payloadBytes), context.aesKey);
+                                payloadBytes = new Uint8Array(decrypted);
+                            }
+                            catch (e) {
+                                // Decryption error
+                            }
+                        }
+                        return {
+                            id: r.getId(),
+                            distance: r.getDistance(),
+                            metadata: meta,
+                            typedMetadata: HyperspaceClient.parseTypedMetadata(r.getTypedMetadataMap()),
+                            payload: payloadBytes
+                        };
+                    });
+                    resolve(results);
                 });
-                req.setFiltersList(protoFilters);
             }
-            if (options === null || options === void 0 ? void 0 : options.hybridQuery)
-                req.setHybridQuery(options.hybridQuery);
-            if ((options === null || options === void 0 ? void 0 : options.hybridAlpha) !== undefined)
-                req.setHybridAlpha(options.hybridAlpha);
-            if (options === null || options === void 0 ? void 0 : options.bm25) {
-                // @ts-ignore: Bm25Options might not be in generated proto yet
-                const bm25Msg = new hyperspace_pb.Bm25Options();
-                if (options.bm25.method !== undefined)
-                    bm25Msg.setMethod(options.bm25.method);
-                if (options.bm25.k1 !== undefined)
-                    bm25Msg.setK1(options.bm25.k1);
-                if (options.bm25.b !== undefined)
-                    bm25Msg.setB(options.bm25.b);
-                if (options.bm25.delta !== undefined)
-                    bm25Msg.setDelta(options.bm25.delta);
-                if (options.bm25.language !== undefined)
-                    bm25Msg.setLanguage(options.bm25.language);
-                if (options.bm25.ngrams !== undefined)
-                    bm25Msg.setNgrams(options.bm25.ngrams);
-                if (options.bm25.fusionMethod !== undefined)
-                    bm25Msg.setFusionMethod(options.bm25.fusionMethod);
-                // @ts-ignore
-                req.setBm25Options(bm25Msg);
+            catch (e) {
+                reject(e);
             }
-            this.client.search(req, this.metadata, (err, resp) => {
-                if (err)
-                    return reject(err);
-                const results = resp.getResultsList().map(r => {
-                    const metaMap = r.getMetadataMap();
-                    const meta = {};
-                    if (metaMap.getLength() > 0) {
-                        metaMap.forEach((entry, key) => {
-                            meta[key] = entry;
-                        });
-                    }
-                    return {
-                        id: r.getId(),
-                        distance: r.getDistance(),
-                        metadata: meta,
-                        typedMetadata: HyperspaceClient.parseTypedMetadata(r.getTypedMetadataMap())
-                    };
-                });
-                resolve(results);
-            });
         });
     }
     searchText(text, topK, collection = '', options) {
+        if (this.collectionKeys[collection]) {
+            if (!this.embedder) {
+                return Promise.reject(new Error("An embedder must be configured to use searchText on encrypted collections."));
+            }
+            return Promise.resolve(this.embedder.encode(text)).then(vector => {
+                return this.search(vector, topK, collection, options);
+            });
+        }
         return new Promise((resolve, reject) => {
             const req = new hyperspace_pb_1.SearchTextRequest();
             req.setText(text);
             req.setTopK(topK);
             req.setCollection(collection);
             if (options === null || options === void 0 ? void 0 : options.filters) {
-                const protoFilters = options.filters.map(f => {
-                    const pf = new hyperspace_pb.Filter();
-                    if (f.match) {
-                        const m = new hyperspace_pb.Match();
-                        m.setKey(f.match.key);
-                        m.setValue(f.match.value);
-                        pf.setMatch(m);
-                    }
-                    else if (f.range) {
-                        const r = new hyperspace_pb.Range();
-                        r.setKey(f.range.key);
-                        if (f.range.gte !== undefined) {
-                            if (Number.isInteger(f.range.gte))
-                                r.setGte(f.range.gte);
-                            else
-                                r.setGteF64(f.range.gte);
-                        }
-                        if (f.range.lte !== undefined) {
-                            if (Number.isInteger(f.range.lte))
-                                r.setLte(f.range.lte);
-                            else
-                                r.setLteF64(f.range.lte);
-                        }
-                        pf.setRange(r);
-                    }
-                    return pf;
-                });
-                req.setFiltersList(protoFilters);
+                req.setFiltersList(options.filters.map(f => this.toProtoFilter(f)));
             }
             if (options === null || options === void 0 ? void 0 : options.bm25) {
-                // @ts-ignore
-                const bm25Msg = new hyperspace_pb.Bm25Options();
+                const bm25Msg = new hyperspace_pb_1.Bm25Options();
                 if (options.bm25.method !== undefined)
                     bm25Msg.setMethod(options.bm25.method);
                 if (options.bm25.k1 !== undefined)
@@ -517,13 +956,14 @@ class HyperspaceClient {
                     bm25Msg.setNgrams(options.bm25.ngrams);
                 if (options.bm25.fusionMethod !== undefined)
                     bm25Msg.setFusionMethod(options.bm25.fusionMethod);
-                // @ts-ignore
                 req.setBm25Options(bm25Msg);
             }
             if ((options === null || options === void 0 ? void 0 : options.hybridAlpha) !== undefined) {
-                // @ts-ignore
                 req.setHybridAlpha(options.hybridAlpha);
             }
+            if ((options === null || options === void 0 ? void 0 : options.includePayload) !== undefined) {
+                req.setIncludePayload(options.includePayload);
+            }
             this.client.searchText(req, this.metadata, (err, resp) => {
                 if (err)
                     return reject(err);
@@ -539,7 +979,8 @@ class HyperspaceClient {
                         id: r.getId(),
                         distance: r.getDistance(),
                         metadata: meta,
-                        typedMetadata: HyperspaceClient.parseTypedMetadata(r.getTypedMetadataMap())
+                        typedMetadata: HyperspaceClient.parseTypedMetadata(r.getTypedMetadataMap()),
+                        payload: r.getPayload_asU8()
                     };
                 });
                 resolve(results);
@@ -571,7 +1012,8 @@ class HyperspaceClient {
                         id: r.getId(),
                         distance: r.getDistance(),
                         metadata: meta,
-                        typedMetadata: HyperspaceClient.parseTypedMetadata(r.getTypedMetadataMap())
+                        typedMetadata: HyperspaceClient.parseTypedMetadata(r.getTypedMetadataMap()),
+                        payload: r.getPayload_asU8()
                     };
                 }));
                 resolve(batch);
@@ -722,34 +1164,7 @@ class HyperspaceClient {
                 }
             }
             if (options === null || options === void 0 ? void 0 : options.filters) {
-                const protoFilters = options.filters.map(f => {
-                    const pf = new hyperspace_pb.Filter();
-                    if (f.match) {
-                        const m = new hyperspace_pb.Match();
-                        m.setKey(f.match.key);
-                        m.setValue(f.match.value);
-                        pf.setMatch(m);
-                    }
-                    else if (f.range) {
-                        const r = new hyperspace_pb.Range();
-                        r.setKey(f.range.key);
-                        if (f.range.gte !== undefined) {
-                            if (Number.isInteger(f.range.gte))
-                                r.setGte(f.range.gte);
-                            else
-                                r.setGteF64(f.range.gte);
-                        }
-                        if (f.range.lte !== undefined) {
-                            if (Number.isInteger(f.range.lte))
-                                r.setLte(f.range.lte);
-                            else
-                                r.setLteF64(f.range.lte);
-                        }
-                        pf.setRange(r);
-                    }
-                    return pf;
-                });
-                req.setFiltersList(protoFilters);
+                req.setFiltersList(options.filters.map(f => this.toProtoFilter(f)));
             }
             this.client.traverse(req, this.metadata, (err, resp) => {
                 if (err)
@@ -836,6 +1251,251 @@ class HyperspaceClient {
         }
         return stream;
     }
+    getCollectionStats(name) {
+        return new Promise((resolve, reject) => {
+            const req = new hyperspace_pb_1.CollectionStatsRequest();
+            req.setName(name);
+            this.client.getCollectionStats(req, this.metadata, (err, resp) => {
+                if (err)
+                    return reject(err);
+                const stats = {
+                    count: resp.getCount(),
+                    indexingQueue: resp.getIndexingQueue(),
+                    diskUsageBytes: resp.getDiskUsageBytes(),
+                    ramUsageBytes: resp.getRamUsageBytes(),
+                    activeTasks: resp.getActiveTasks()
+                };
+                const protoSchema = resp.getSchema();
+                if (protoSchema) {
+                    stats.schema = {
+                        components: protoSchema.getComponentsList().map((comp) => ({
+                            name: comp.getName(),
+                            metric: comp.getMetric(),
+                            fullDimension: comp.getFullDimension(),
+                            weight: comp.getWeight()
+                        })),
+                        cascadePipeline: protoSchema.getCascadePipelineList().map((layer) => ({
+                            componentName: layer.getComponentName(),
+                            cutoffDimension: layer.getCutoffDimension(),
+                            storeInRam: layer.getStoreInRam(),
+                            rerankTopK: layer.getRerankTopK()
+                        }))
+                    };
+                }
+                resolve(stats);
+            });
+        });
+    }
+    async exists(name) {
+        try {
+            await this.getCollectionStats(name);
+            return true;
+        }
+        catch (e) {
+            if (e.code === grpc.status.NOT_FOUND || (e.message && e.message.includes('not found'))) {
+                return false;
+            }
+            throw e;
+        }
+    }
+    async getCacheStats(name) {
+        const ip = this.host.split(':')[0];
+        const url = `http://${ip}:50050/api/collections/${name}/cache/stats`;
+        const headers = {};
+        if (this.apiKey)
+            headers['x-api-key'] = this.apiKey;
+        if (this.userId)
+            headers['x-hyperspace-user-id'] = this.userId;
+        const res = await fetch(url, { headers });
+        if (!res.ok) {
+            throw new Error(`Failed to get cache stats: ${res.statusText} (${await res.text()})`);
+        }
+        return res.json();
+    }
+    async clearCache(name) {
+        const ip = this.host.split(':')[0];
+        const url = `http://${ip}:50050/api/collections/${name}/cache/clear`;
+        const headers = { 'Content-Type': 'application/json' };
+        if (this.apiKey)
+            headers['x-api-key'] = this.apiKey;
+        if (this.userId)
+            headers['x-hyperspace-user-id'] = this.userId;
+        const res = await fetch(url, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({})
+        });
+        if (!res.ok) {
+            throw new Error(`Failed to clear cache: ${res.statusText} (${await res.text()})`);
+        }
+        const data = await res.json();
+        return data.status === 'success';
+    }
+    async updateCacheConfig(name, policy, annThreshold) {
+        const ip = this.host.split(':')[0];
+        const url = `http://${ip}:50050/api/collections/${name}/cache/config`;
+        const headers = { 'Content-Type': 'application/json' };
+        if (this.apiKey)
+            headers['x-api-key'] = this.apiKey;
+        if (this.userId)
+            headers['x-hyperspace-user-id'] = this.userId;
+        const res = await fetch(url, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({
+                policy,
+                ann_threshold: annThreshold
+            })
+        });
+        if (!res.ok) {
+            throw new Error(`Failed to update cache config: ${res.statusText} (${await res.text()})`);
+        }
+        const data = await res.json();
+        return data.status === 'success';
+    }
+    updateCollection(name, config) {
+        return new Promise((resolve, reject) => {
+            const req = new hyperspace_pb_1.ConfigUpdate();
+            req.setCollection(name);
+            if (config.efSearch !== undefined)
+                req.setEfSearch(config.efSearch);
+            if (config.efConstruction !== undefined)
+                req.setEfConstruction(config.efConstruction);
+            if (config.m !== undefined)
+                req.setM(config.m);
+            this.client.configure(req, this.metadata, (err, resp) => {
+                if (err)
+                    return reject(err);
+                resolve(resp.getStatus() === 'success' || (!!resp.getStatus() && resp.getStatus().includes('updated')));
+            });
+        });
+    }
+    createSnapshot() {
+        return new Promise((resolve, reject) => {
+            this.client.triggerSnapshot(new hyperspace_pb_1.Empty(), this.metadata, (err, resp) => {
+                if (err)
+                    return reject(err);
+                resolve(resp.getStatus() === 'success');
+            });
+        });
+    }
+    vacuum() {
+        return new Promise((resolve, reject) => {
+            this.client.triggerVacuum(new hyperspace_pb_1.Empty(), this.metadata, (err, resp) => {
+                if (err)
+                    return reject(err);
+                resolve(resp.getStatus() === 'success');
+            });
+        });
+    }
+    getMetrics(onData, onError) {
+        const req = new hyperspace_pb_1.MonitorRequest();
+        const stream = this.client.monitor(req, this.metadata);
+        stream.on('data', onData);
+        if (onError) {
+            stream.on('error', onError);
+        }
+        return stream;
+    }
+    searchMultiCollection(collections, query) {
+        return new Promise((resolve, reject) => {
+            const req = new hyperspace_pb_1.SearchMultiCollectionRequest();
+            req.setCollectionsList(collections);
+            req.setVectorList(HyperspaceClient.toVectorList(query));
+            req.setTopK(10);
+            this.client.searchMultiCollection(req, this.metadata, (err, resp) => {
+                if (err)
+                    return reject(err);
+                resolve(resp.toObject());
+            });
+        });
+    }
+    triggerReconsolidation(collection, targetVector, learningRate = 0.01) {
+        return new Promise((resolve, reject) => {
+            const req = new hyperspace_pb_1.ReconsolidationRequest();
+            req.setCollection(collection);
+            req.setTargetVectorList(targetVector);
+            req.setLearningRate(learningRate);
+            this.client.triggerReconsolidation(req, this.metadata, (err, resp) => {
+                if (err)
+                    return reject(err);
+                resolve(resp.getStatus() === 'success');
+            });
+        });
+    }
+    getSubsumptionTree(rootId, maxDepth = 3, collection = '') {
+        return new Promise((resolve, reject) => {
+            const req = new hyperspace_pb.GetSubsumptionTreeRequest();
+            req.setRootId(rootId);
+            req.setMaxDepth(maxDepth);
+            req.setCollection(collection);
+            this.client.getSubsumptionTree(req, this.metadata, (err, resp) => {
+                if (err)
+                    return reject(err);
+                const nodes = resp.getNodesList().map((n) => ({
+                    id: n.getId(),
+                    metadata: n.getMetadataMap().toObject(),
+                    edgeTypes: n.getEdgeTypesList()
+                }));
+                resolve(nodes);
+            });
+        });
+    }
+    async exploreGraph(startId, maxDepth = 2, maxNodes = 256, collection = '') {
+        const req = new hyperspace_pb_1.TraverseRequest();
+        req.setStartId(startId);
+        req.setMaxDepth(maxDepth);
+        req.setMaxNodes(maxNodes);
+        req.setCollection(collection);
+        req.setTraversalMode(1); // DIFFUSIVE
+        req.setBreadthLimit(10);
+        return new Promise((resolve, reject) => {
+            this.client.traverse(req, this.metadata, (err, resp) => {
+                if (err)
+                    return reject(err);
+                const nodes = resp.getNodesList().map((n) => ({
+                    id: n.getId(),
+                    metadata: n.getMetadataMap().toObject(),
+                    edgeTypes: n.getEdgeTypesList(),
+                    neighbors: n.getNeighborsList()
+                }));
+                resolve({
+                    nodes,
+                    centerId: startId,
+                    count: nodes.length
+                });
+            });
+        });
+    }
+    async predictMomentum(trajectoryIds, steps = 1.0, collection = '', curvature = 1.0) {
+        if (trajectoryIds.length < 2)
+            return [];
+        const pts = await this.getPoints(trajectoryIds, collection);
+        const idToVec = new Map();
+        pts.forEach(p => idToVec.set(p.id, p.vector));
+        const vectors = trajectoryIds.map(id => idToVec.get(id)).filter(v => !!v);
+        if (vectors.length < 2)
+            return [];
+        const past = vectors[vectors.length - 2];
+        const current = vectors[vectors.length - 1];
+        const { koopmanExtrapolate } = require('./math');
+        return koopmanExtrapolate(past, current, steps, curvature);
+    }
+    async getTrustScore(trajectoryIds, collection = '', curvature = 1.0) {
+        const pts = await this.getPoints(trajectoryIds, collection);
+        const idToVec = new Map();
+        pts.forEach(p => idToVec.set(p.id, p.vector));
+        const vectors = trajectoryIds.map(id => idToVec.get(id)).filter(v => !!v);
+        if (vectors.length < 3)
+            return 0.0;
+        const { lyapunovConvergence } = require('./math');
+        try {
+            return lyapunovConvergence(vectors, curvature);
+        }
+        catch (e) {
+            return 1.0;
+        }
+    }
     close() {
         this.client.close();
     }