@frogfish/k2db 2.0.7 → 3.0.2

package/db.js

@@ -0,0 +1,1761 @@
+// src/db.ts
+import { K2Error, ServiceError, wrap } from "@frogfish/k2error"; // Keep the existing error structure
+import { MongoClient, } from "mongodb";
+import { randomBytes, createHash, createCipheriv, createDecipheriv } from "crypto";
+import { Topic } from '@frogfish/ratatouille';
+import { z } from "zod";
+// const debug = debugLib("k2:db");
+const debug = Topic('k2db#random');
+function _hashSecret(secret) {
+    return createHash("sha256").update(secret).digest("hex");
+}
+// ---- Shared MongoClient pool (per cluster+auth), reused across DB names ----
+const _clientByKey = new Map();
+const _connectingByKey = new Map();
+const _refCountByKey = new Map();
+function _hostsKey(hosts) {
+    const hs = hosts ?? [];
+    return hs
+        .map((h) => `${h.host}:${h.port ?? ""}`)
+        .sort()
+        .join(",");
+}
+/**
+ * Cache key for a MongoClient pool. Intentionally excludes `conf.name` (db name),
+ * so multiple DBs share the same connection pool.
+ */
+function _clientCacheKey(conf) {
+    const user = conf.user ?? "";
+    const pass = conf.password ?? "";
+    const passKey = pass ? `sha256:${_hashSecret(pass)}` : "";
+    const authSource = user && pass ? (conf.authSource ?? "admin") : "";
+    const rs = conf.replicaset ?? "";
+    const hosts = _hostsKey(conf.hosts);
+    return `hosts=${hosts}|user=${user}|pass=${passKey}|authSource=${authSource}|rs=${rs}`;
+}
+async function _acquireClient(key, uri, options) {
+    const existing = _clientByKey.get(key);
+    if (existing)
+        return existing;
+    const inflight = _connectingByKey.get(key);
+    if (inflight)
+        return inflight;
+    const p = MongoClient.connect(uri, options)
+        .then((client) => {
+        _clientByKey.set(key, client);
+        _connectingByKey.delete(key);
+        return client;
+    })
+        .catch((err) => {
+        _connectingByKey.delete(key);
+        throw err;
+    });
+    _connectingByKey.set(key, p);
+    return p;
+}
+function _increfClient(key) {
+    _refCountByKey.set(key, (_refCountByKey.get(key) ?? 0) + 1);
+}
+async function _decrefClient(key) {
+    const next = (_refCountByKey.get(key) ?? 0) - 1;
+    if (next <= 0) {
+        _refCountByKey.delete(key);
+        const client = _clientByKey.get(key);
+        if (client) {
+            _clientByKey.delete(key);
+            try {
+                await client.close();
+            }
+            catch (err) {
+                // Best-effort shutdown: never throw from release/close paths.
+                debug(`MongoClient close failed for key=${key}: ${err instanceof Error ? err.message : String(err)}`);
+            }
+        }
+        return;
+    }
+    _refCountByKey.set(key, next);
+}
+// ---- End shared MongoClient pool helpers ----
+// Crockford Base32 alphabet (no I, L, O, U)
+const CROCKFORD32 = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
+/**
+ * Generates a UUIDv7 (time-ordered) and encodes it as Crockford Base32 with hyphens.
+ * Format: 26 base32 chars grouped as 8-4-4-4-6 (total 26)
+ */
+function uuidv7Base32Hyphenated() {
+    // 1) Build UUIDv7 bytes
+    // Layout per RFC: time_low(32) | time_mid(16) | time_hi_and_version(16) | clock_seq(16) | node(48)
+    // Encode 60-bit ms timestamp across time_* fields, version 7, RFC4122 variant in clock_seq_hi.
+    const ts = BigInt(Date.now()); // milliseconds
+    const timeLow = Number((ts >> 28n) & 0xffffffffn);
+    const timeMid = Number((ts >> 12n) & 0xffffn);
+    const timeHi = Number(ts & 0xfffn); // lower 12 bits
+    const bytes = new Uint8Array(16);
+    // time_low (big-endian)
+    bytes[0] = (timeLow >>> 24) & 0xff;
+    bytes[1] = (timeLow >>> 16) & 0xff;
+    bytes[2] = (timeLow >>> 8) & 0xff;
+    bytes[3] = timeLow & 0xff;
+    // time_mid (big-endian)
+    bytes[4] = (timeMid >>> 8) & 0xff;
+    bytes[5] = timeMid & 0xff;
+    // time_high_and_version: version 7 in high nibble + top 4 bits of timeHi
+    bytes[6] = 0x70 | ((timeHi >>> 8) & 0x0f); // 0x7- version
+    bytes[7] = timeHi & 0xff;
+    // clock_seq + node: 8 random bytes; set RFC4122 variant (10xxxxxx)
+    const rnd = randomBytes(8);
+    bytes.set(rnd, 8);
+    bytes[8] = (bytes[8] & 0x3f) | 0x80; // set variant 10xxxxxx
+    // 2) Encode as Crockford Base32 (26 chars). 128 bits -> 26*5 bits (pad 2 high bits)
+    let value = 0n;
+    for (let i = 0; i < 16; i++) {
+        value = (value << 8n) | BigInt(bytes[i]);
+    }
+    value <<= 2n; // pad to 130 bits so we can take 26 groups cleanly
+    let encoded = "";
+    for (let i = 25; i >= 0; i--) {
+        const idx = Number((value >> BigInt(i * 5)) & 0x1fn);
+        encoded += CROCKFORD32[idx];
+    }
+    // 3) Insert hyphens in groups: 8-4-4-4-6
+    return (encoded.slice(0, 8) +
+        "-" +
+        encoded.slice(8, 12) +
+        "-" +
+        encoded.slice(12, 16) +
+        "-" +
+        encoded.slice(16, 20) +
+        "-" +
+        encoded.slice(20));
+}
+/**
+ * Test helper: fully reset the shared MongoClient pool.
+ *
+ * Not for production usage; intended for test runners to clean up
+ * between suites without restarting the process.
+ */
+export async function resetSharedMongoClientsForTests() {
+    const entries = Array.from(_clientByKey.entries());
+    for (const [key, client] of entries) {
+        try {
+            await client.close();
+        }
+        catch (err) {
+            debug(`MongoClient close failed during reset for key=${key}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    _clientByKey.clear();
+    _connectingByKey.clear();
+    _refCountByKey.clear();
+}
+export class K2DB {
+    conf;
+    db;
+    connection;
+    clientKey;
+    initialized = false;
+    initPromise;
+    schemas = new Map();
+    ownershipMode;
+    aggregationMode;
+    secureFieldPrefixes;
+    secureFieldEncryptionKey;
+    secureFieldEncryptionKeyId;
+    constructor(conf) {
+        this.conf = conf;
+        this.ownershipMode = conf.ownershipMode ?? "lax";
+        this.aggregationMode = conf.aggregationMode ?? "loose";
+        this.secureFieldPrefixes = (conf.secureFieldPrefixes ?? [])
+            .map((p) => (typeof p === "string" ? p.trim() : ""))
+            .filter((p) => !!p);
+        const keyB64 = (conf.secureFieldEncryptionKey ?? "").trim();
+        const kid = (conf.secureFieldEncryptionKeyId ?? "").trim();
+        this.secureFieldEncryptionKeyId = kid || undefined;
+        if (keyB64) {
+            let keyBuf;
+            try {
+                keyBuf = Buffer.from(keyB64, "base64");
+            }
+            catch (e) {
+                throw new K2Error(ServiceError.CONFIGURATION_ERROR, "secureFieldEncryptionKey must be base64-encoded", "sys_mdb_secure_key_invalid");
+            }
+            if (keyBuf.length !== 32) {
+                throw new K2Error(ServiceError.CONFIGURATION_ERROR, "secureFieldEncryptionKey must decode to 32 bytes (AES-256)", "sys_mdb_secure_key_invalid");
+            }
+            this.secureFieldEncryptionKey = keyBuf;
+        }
+    }
+    /**
+     * Normalize a scope value for ownership enforcement.
+     */
+    normalizeScope(scope) {
+        if (scope === undefined || scope === null)
+            return undefined;
+        if (typeof scope !== "string")
+            return undefined;
+        const s = scope.trim();
+        if (!s)
+            return undefined;
+        if (s === "*")
+            return "*";
+        return s.toLowerCase();
+    }
+    /**
+     * Apply a scope constraint to criteria for ownership enforcement.
+     */
+    applyScopeToCriteria(criteria, scope) {
+        const normalizedScope = this.normalizeScope(scope);
+        // Strict mode requires an explicit scope per call.
+        if (this.ownershipMode === "strict" && !normalizedScope) {
+            throw new K2Error(ServiceError.BAD_REQUEST, "Scope is required in strict ownership mode", "sys_mdb_scope_required");
+        }
+        // Lax mode with no scope: legacy behavior (no owner constraint injected).
+        if (!normalizedScope)
+            return criteria;
+        // Explicit all-scope request: do not constrain by _owner.
+        if (normalizedScope === "*")
+            return criteria;
+        // If caller already provided _owner in criteria, ensure it matches the scope to avoid ambiguity/bypass.
+        if (criteria && typeof criteria === "object" && Object.prototype.hasOwnProperty.call(criteria, "_owner")) {
+            const existing = criteria._owner;
+            if (typeof existing === "string" && existing.trim().toLowerCase() !== normalizedScope) {
+                throw new K2Error(ServiceError.BAD_REQUEST, "Conflicting _owner in criteria and provided scope", "sys_mdb_scope_conflict");
+            }
+            // If it matches (or is non-string), prefer the explicit scope value.
+        }
+        return { ...(criteria || {}), _owner: normalizedScope };
+    }
+    /**
+     * Initializes the MongoDB connection.
+     */
+    async init() {
+        if (this.initialized)
+            return;
+        if (this.initPromise)
+            return this.initPromise;
+        this.initPromise = (async () => {
+            // Build URI and options
+            const { uri, options } = this.buildMongoUri();
+            // Mask sensitive information in logs
+            const safeConnectUrl = uri.replace(/\/\/.*?:.*?@/, "//*****:*****@");
+            debug(`Connecting to MongoDB: ${safeConnectUrl}`);
+            try {
+                // Establish (or reuse) a shared MongoClient pool per cluster+auth (NOT per db name)
+                const key = _clientCacheKey(this.conf);
+                const client = await _acquireClient(key, uri, options);
+                this.connection = client;
+                this.db = this.connection.db(this.conf.name);
+                this.clientKey = key;
+                if (!this.initialized) {
+                    _increfClient(key);
+                    this.initialized = true;
+                }
+                debug("Successfully connected to MongoDB");
+            }
+            catch (err) {
+                // Handle connection error
+                const msg = err instanceof Error
+                    ? `Failed to connect to MongoDB: ${err.message}`
+                    : `Failed to connect to MongoDB: ${String(err)}`;
+                throw wrap(err, ServiceError.SERVICE_UNAVAILABLE, "sys_mdb_init", msg);
+            }
+        })().finally(() => {
+            // Allow retry after failure; once initialized, subsequent calls return early.
+            this.initPromise = undefined;
+        });
+        return this.initPromise;
+    }
+    /**
+     * Build a robust MongoDB URI based on config (supports SRV and standard).
+     */
+    buildMongoUri() {
+        if (!this.conf.hosts || this.conf.hosts.length === 0) {
+            throw new K2Error(ServiceError.CONFIGURATION_ERROR, "No valid hosts provided in configuration", "sys_mdb_no_hosts");
+        }
+        const auth = this.conf.user && this.conf.password
+            ? `${encodeURIComponent(this.conf.user)}:${encodeURIComponent(this.conf.password)}@`
+            : "";
+        const singleNoPort = this.conf.hosts.length === 1 && !this.conf.hosts[0].port;
+        const useSrv = singleNoPort;
+        const dbName = this.conf.name;
+        let uri;
+        if (useSrv) {
+            const host = this.conf.hosts[0].host;
+            uri = `mongodb+srv://${auth}${host}/?retryWrites=true&w=majority`;
+        }
+        else {
+            const hostList = this.conf.hosts
+                .map((h) => `${h.host}:${h.port || 27017}`)
+                .join(",");
+            const params = ["retryWrites=true", "w=majority"];
+            if (this.conf.replicaset)
+                params.push(`replicaSet=${this.conf.replicaset}`);
+            uri = `mongodb://${auth}${hostList}/?${params.join("&")}`;
+        }
+        // Determine authSource based on user and password presence
+        const authSource = this.conf.user && this.conf.password
+            ? this.conf.authSource ?? "admin"
+            : undefined;
+        const options = {
+            connectTimeoutMS: 2000,
+            serverSelectionTimeoutMS: 2000,
+            ...(authSource ? { authSource } : {}),
+        };
+        return { uri, options };
+    }
+    /** Load DatabaseConfig from environment variables. */
+    static fromEnv(prefix = "K2DB_") {
+        const get = (k) => globalThis.process?.env?.[`${prefix}${k}`];
+        const name = get("NAME");
+        const hostsEnv = get("HOSTS");
+        if (!name || !hostsEnv) {
+            throw new Error("K2DB_NAME and K2DB_HOSTS are required in environment");
+        }
+        const hosts = hostsEnv.split(",").map((h) => {
+            const [host, port] = h.trim().split(":");
+            return { host, port: port ? parseInt(port, 10) : undefined };
+        });
+        const conf = {
+            name,
+            hosts,
+            user: get("USER"),
+            password: get("PASSWORD"),
+            authSource: get("AUTH_SOURCE"),
+            replicaset: get("REPLICASET"),
+        };
+        const slow = get("SLOW_MS");
+        if (slow)
+            conf.slowQueryMs = parseInt(slow, 10);
+        return conf;
+    }
+    /**
+     * Retrieves a collection from the database.
+     * @param collectionName - Name of the collection.
+     */
+    async getCollection(collectionName) {
+        try {
+            this.validateCollectionName(collectionName); // Validate the collection name
+            const collection = this.db.collection(collectionName);
+            return collection;
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_gc", `Error getting collection: ${collectionName}`);
+        }
+    }
+    /**
+     * Retrieves a single document by UUID.
+     * @param collectionName - Name of the collection.
+     * @param uuid - UUID of the document.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async get(collectionName, uuid, scope) {
+        const id = K2DB.normalizeId(uuid);
+        // Note: findOne() decrypts secure-prefixed fields for single-record reads when encryption is enabled.
+        const res = await this.findOne(collectionName, {
+            _uuid: id,
+            _deleted: { $ne: true },
+        }, undefined, scope);
+        if (!res) {
+            throw new K2Error(ServiceError.NOT_FOUND, "Document not found", "sys_mdb_get_not_found");
+        }
+        return res;
+    }
+    /**
+     * Retrieves a single document by criteria.
+     * @param collectionName - Name of the collection.
+     * @param criteria - Criteria to find the document.
+     * @param fields - Optional array of fields to include.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async findOne(collectionName, criteria, fields, scope) {
+        const collection = await this.getCollection(collectionName);
+        const projection = {};
+        // Exclude soft-deleted documents by default unless caller specifies otherwise
+        const normalizedCriteria = K2DB.normalizeCriteriaIds(criteria || {});
+        let query = {
+            ...normalizedCriteria,
+            ...(criteria && Object.prototype.hasOwnProperty.call(criteria, "_deleted")
+                ? {}
+                : { _deleted: { $ne: true } }),
+        };
+        query = this.applyScopeToCriteria(query, scope);
+        // If a projection is requested, forbid explicitly projecting secure-prefixed fields.
+        // Also ensure _uuid is fetched when secure encryption is enabled so decryption AAD is correct.
+        const requestedFields = Array.isArray(fields) ? fields.slice() : undefined;
+        if (requestedFields && requestedFields.length > 0) {
+            requestedFields.forEach((field) => {
+                if (typeof field !== "string")
+                    return;
+                const f = field.trim();
+                if (!f)
+                    return;
+                // Deny any projection that includes secure-prefixed fields (including nested dotted paths)
+                if (this.pathHasSecureSegment(f)) {
+                    throw new K2Error(ServiceError.BAD_REQUEST, "Projection cannot include secure-prefixed field(s)", "sys_mdb_projection_secure_field");
+                }
+                projection[f] = 1;
+            });
+            // Ensure we can decrypt secure fields that are present in the returned doc
+            if (this.hasSecureEncryption()) {
+                projection["_uuid"] = 1;
+            }
+        }
+        try {
+            const item = await this.runTimed("findOne", { collectionName, query, projection }, async () => await collection.findOne(query, { projection }));
+            if (item) {
+                const { _id, ...rest } = item;
+                const out = rest;
+                const aadBase = out._uuid
+                    ? `k2db|${collectionName}|${out._uuid}`
+                    : `k2db|${collectionName}`;
+                const decrypted = this.decryptSecureFieldsDeep(out, aadBase);
+                // If caller requested a projection and did NOT request _uuid, hide it again (it may have been added for decrypt AAD).
+                if (requestedFields && requestedFields.length > 0) {
+                    const wantsUuid = requestedFields.some((f) => typeof f === "string" && f.trim() === "_uuid");
+                    if (!wantsUuid) {
+                        delete decrypted._uuid;
+                    }
+                }
+                return decrypted;
+            }
+            return null;
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_fo", "Error finding document");
+        }
+    }
+    /**
+     * Finds documents based on parameters with pagination support.
+     * @param collectionName - Name of the collection.
+     * @param filter - Criteria to filter the documents.
+     * @param params - Optional search parameters (for sorting, including/excluding fields).
+     * @param skip - Number of documents to skip (for pagination).
+     * @param limit - Maximum number of documents to return.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async find(collectionName, filter, params = {}, skip = 0, limit = 100, scope) {
+        const collection = await this.getCollection(collectionName);
+        // Ensure filter is valid, defaulting to an empty object
+        let criteria = { ...(filter || {}) };
+        criteria = K2DB.normalizeCriteriaIds(criteria);
+        criteria = this.applyScopeToCriteria(criteria, scope);
+        // Handle the _deleted field if params specify not to include deleted documents
+        if (!params?.includeDeleted && !Object.prototype.hasOwnProperty.call(criteria, "_deleted")) {
+            if (params?.deleted === true) {
+                criteria._deleted = true; // Explicitly search for deleted documents
+            }
+            else {
+                criteria._deleted = { $ne: true }; // Exclude deleted by default
+            }
+        }
+        // Build projection (fields to include or exclude)
+        let projection;
+        if (typeof params.filter === "string" && params.filter === "all") {
+            projection = {}; // Include all fields
+        }
+        else if (Array.isArray(params.filter)) {
+            projection = {};
+            params.filter.forEach((field) => {
+                const f = typeof field === "string" ? field.trim() : "";
+                if (!f)
+                    return;
+                // Deny any projection that includes secure-prefixed fields (including nested dotted paths)
+                if (this.pathHasSecureSegment(f)) {
+                    throw new K2Error(ServiceError.BAD_REQUEST, "Projection cannot include secure-prefixed field(s)", "sys_mdb_projection_secure_field");
+                }
+                projection[f] = 1; // Only include the specified fields
+            });
+            projection._id = 0; // Hide _id when using include list
+        }
+        else if (Array.isArray(params.exclude)) {
+            projection = { _id: 0 }; // Start by hiding _id
+            params.exclude.forEach((field) => {
+                projection[field] = 0; // Exclude the specified fields
+            });
+        }
+        else {
+            projection = { _id: 0 }; // Default: hide _id only
+        }
+        // Build sorting options
+        let sort = undefined;
+        if (params.order) {
+            sort = {};
+            for (const [key, value] of Object.entries(params.order)) {
+                sort[key] = value === "asc" ? 1 : -1;
+            }
+        }
+        try {
+            let cursor = collection.find(criteria, { projection });
+            // Apply pagination
+            cursor = cursor.skip(skip).limit(limit);
+            if (sort) {
+                cursor = cursor.sort(sort);
+            }
+            const data = await this.runTimed("find", { collectionName, criteria, projection, sort, skip, limit }, async () => await cursor.toArray());
+            // Remove _id safely from each document
+            const result = data.map((doc) => {
+                const { _id, ...rest } = doc;
+                // For multi-record reads, never return secure-prefixed fields (even if encrypted-at-rest is enabled)
+                return this.stripSecureFieldsDeep(rest);
+            });
+            return result;
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_find_error", "Error executing find query");
+        }
+    }
+    /**
+     * Aggregates documents based on criteria with pagination support (may validate/limit stages in guarded/strict aggregationMode). Secure-prefixed fields may be stripped from results when configured.
+     * @param collectionName - Name of the collection.
+     * @param criteria - Aggregation pipeline criteria.
+     * @param skip - Number of documents to skip (for pagination).
+     * @param limit - Maximum number of documents to return.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async aggregate(collectionName, criteria, skip = 0, limit = 100, scope) {
+        if (criteria.length === 0) {
+            throw new K2Error(ServiceError.SYSTEM_ERROR, "Aggregation criteria cannot be empty", "sys_mdb_ag_empty");
+        }
+        // Prevent aggregation from reading/deriving values from secure-prefixed fields (when configured)
+        this.assertNoSecureFieldRefsInPipeline(criteria);
+        // Validate the aggregation pipeline for allowed/disallowed stages and safety caps
+        this.validateAggregationPipeline(criteria, skip, limit);
+        // Enforce soft-delete behavior: never return documents marked as deleted
+        criteria = K2DB.enforceNoDeletedInPipeline(criteria);
+        // Enforce ownership scope within the pipeline (and nested pipelines)
+        criteria = this.enforceScopeInPipeline(criteria, scope);
+        // Add pagination stages to the aggregation pipeline
+        if (skip > 0) {
+            criteria.push({ $skip: skip });
+        }
+        if (limit > 0) {
+            criteria.push({ $limit: limit });
+        }
+        debug(`Aggregating with criteria: ${JSON.stringify(criteria, null, 2)}`);
+        const collection = await this.getCollection(collectionName);
+        // Sanitize criteria
+        const sanitizedCriteria = criteria.map((stage) => {
+            if (stage.$match) {
+                return K2DB.sanitiseCriteria(stage);
+            }
+            return stage;
+        });
+        try {
+            const data = await this.runTimed("aggregate", { collectionName, pipeline: sanitizedCriteria }, async () => {
+                const mode = this.aggregationMode ?? "loose";
+                const opts = {};
+                if (mode !== "loose") {
+                    opts.maxTimeMS = 2000;
+                }
+                return await collection.aggregate(sanitizedCriteria, opts).toArray();
+            });
+            // Enforce BaseDocument type on each document and strip secure-prefixed fields (if configured)
+            return data.map((doc) => this.stripSecureFieldsDeep(doc));
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_ag", "Aggregation failed");
+        }
+    }
+    /**
+     * Validate an aggregation pipeline for safety based on aggregationMode.
+     * - loose: no validation
+     * - guarded: deny obvious footguns (writes, server-side code) and enforce basic caps
+     * - strict: allow only a small safe subset of stages and enforce basic caps
+     */
+    validateAggregationPipeline(pipeline, skip, limit) {
+        const mode = this.aggregationMode ?? "loose";
+        if (mode === "loose")
+            return;
+        // Hard caps to reduce accidental/abusive heavy queries
+        const maxStages = 50;
+        if (pipeline.length > maxStages) {
+            throw new K2Error(ServiceError.BAD_REQUEST, `Aggregation pipeline too long (max ${maxStages} stages)`, "sys_mdb_ag_pipeline_too_long");
+        }
+        // Require a positive limit in guarded/strict to avoid accidental full scans.
+        // Note: aggregate() appends $limit later; enforce here based on the provided arg.
+        if (!(typeof limit === "number") || !isFinite(limit) || limit <= 0) {
+            throw new K2Error(ServiceError.BAD_REQUEST, "Aggregation requires a positive limit in guarded/strict mode", "sys_mdb_ag_limit_required");
+        }
+        const maxLimit = 1000;
+        if (limit > maxLimit) {
+            throw new K2Error(ServiceError.BAD_REQUEST, `Aggregation limit too large (max ${maxLimit})`, "sys_mdb_ag_limit_too_large");
+        }
+        const ops = this.collectStageOps(pipeline);
+        const denyGuarded = new Set(["$out", "$merge", "$function", "$accumulator"]);
+        const allowStrict = new Set(["$match", "$project", "$sort", "$skip", "$limit"]);
+        if (mode === "guarded") {
+            for (const op of ops) {
+                if (denyGuarded.has(op)) {
+                    throw new K2Error(ServiceError.BAD_REQUEST, `Aggregation stage ${op} is not allowed in guarded mode`, "sys_mdb_ag_stage_denied");
+                }
+            }
+            return;
+        }
+        // strict
+        for (const op of ops) {
+            if (!allowStrict.has(op)) {
+                throw new K2Error(ServiceError.BAD_REQUEST, `Aggregation stage ${op} is not allowed in strict mode`, "sys_mdb_ag_stage_denied");
+            }
+        }
+    }
+    /** Collect top-level stage operators for a pipeline (e.g. "$match", "$lookup"). */
+    collectStageOps(pipeline) {
+        const ops = [];
+        for (const stage of pipeline || []) {
+            if (!stage || typeof stage !== "object")
+                continue;
+            const keys = Object.keys(stage);
+            if (keys.length === 1 && keys[0].startsWith("$")) {
+                ops.push(keys[0]);
+            }
+            else {
+                // Unknown/non-canonical stage shape; treat as invalid in strict/guarded
+                ops.push("__invalid__");
+            }
+        }
+        return ops;
+    }
+    /** True if a field key is considered secure and must not be returned. */
+    isSecureFieldKey(key) {
+        if (!this.secureFieldPrefixes.length)
+            return false;
+        for (const p of this.secureFieldPrefixes) {
+            if (key.startsWith(p))
+                return true;
+        }
+        return false;
+    }
+    /**
+     * Recursively strips secure-prefixed fields from objects/arrays (e.g. "#passport_number").
+     * This is applied on read results (e.g. aggregate) so pipelines cannot exfiltrate secure fields.
+     */
+    stripSecureFieldsDeep(val) {
+        if (!this.secureFieldPrefixes.length)
+            return val;
+        if (val === null || val === undefined)
+            return val;
+        if (Array.isArray(val)) {
+            return val.map((x) => this.stripSecureFieldsDeep(x));
+        }
+        if (typeof val !== "object") {
+            return val;
+        }
+        const out = {};
+        for (const [k, v] of Object.entries(val)) {
+            if (this.isSecureFieldKey(k))
+                continue;
+            out[k] = this.stripSecureFieldsDeep(v);
+        }
+        return out;
+    }
+    /** True if secure-field encryption is enabled (requires both key and keyId). */
+    hasSecureEncryption() {
+        return !!this.secureFieldEncryptionKey && !!this.secureFieldEncryptionKeyId;
+    }
+    /** Encrypt a JS value using AES-256-GCM and return "<kid>:<ivB64>.<tagB64>.<ctB64>". */
+    encryptSecureValueToString(value, aad) {
+        const kid = this.secureFieldEncryptionKeyId;
+        const iv = randomBytes(12);
+        const cipher = createCipheriv("aes-256-gcm", this.secureFieldEncryptionKey, iv);
+        cipher.setAAD(Buffer.from(aad, "utf8"));
+        const pt = Buffer.from(JSON.stringify(value), "utf8");
+        const ct = Buffer.concat([cipher.update(pt), cipher.final()]);
+        const tag = cipher.getAuthTag();
+        const payload = `${iv.toString("base64")}.${tag.toString("base64")}.${ct.toString("base64")}`;
+        return `${kid}:${payload}`;
+    }
+    /** Decrypt a "<kid>:<ivB64>.<tagB64>.<ctB64>" string back into a JS value. */
+    decryptSecureStringToValue(s, aad) {
+        if (!this.hasSecureEncryption())
+            return s;
+        if (typeof s !== "string")
+            return s;
+        const idx = s.indexOf(":");
+        if (idx <= 0)
+            return s;
+        const kid = s.slice(0, idx);
+        const payload = s.slice(idx + 1);
+        // Only decrypt if kid matches the configured key id
+        if (!this.secureFieldEncryptionKeyId || kid !== this.secureFieldEncryptionKeyId) {
+            return s;
+        }
+        const parts = payload.split(".");
+        if (parts.length !== 3)
+            return s;
+        try {
+            const iv = Buffer.from(parts[0], "base64");
+            const tag = Buffer.from(parts[1], "base64");
+            const ct = Buffer.from(parts[2], "base64");
+            const decipher = createDecipheriv("aes-256-gcm", this.secureFieldEncryptionKey, iv);
+            decipher.setAAD(Buffer.from(aad, "utf8"));
+            decipher.setAuthTag(tag);
+            const pt = Buffer.concat([decipher.update(ct), decipher.final()]);
+            return JSON.parse(pt.toString("utf8"));
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_secure_decrypt_failed", "Failed to decrypt secure field");
+        }
+    }
+    /**
+     * Encrypt secure-prefixed fields in an object/array tree.
+     * Only keys with secure prefixes are encrypted; other keys are recursed to allow nested secure keys.
+     */
+    encryptSecureFieldsDeep(val, aadPrefix) {
+        if (!this.secureFieldPrefixes.length)
+            return val;
+        if (!this.hasSecureEncryption())
+            return val;
+        if (val === null || val === undefined)
+            return val;
+        if (Array.isArray(val)) {
+            return val.map((x) => this.encryptSecureFieldsDeep(x, aadPrefix));
+        }
+        if (typeof val !== "object")
+            return val;
+        const out = {};
+        for (const [k, v] of Object.entries(val)) {
+            if (this.isSecureFieldKey(k)) {
+                const aad = `${aadPrefix}|${k}`;
+                out[k] = this.encryptSecureValueToString(v, aad);
+            }
+            else {
+                out[k] = this.encryptSecureFieldsDeep(v, aadPrefix);
+            }
+        }
+        return out;
+    }
+    /**
+     * Decrypt secure-prefixed fields in an object/array tree.
+     * If a secure field value is not an encrypted string, it is returned as-is.
+     */
+    decryptSecureFieldsDeep(val, aadPrefix) {
+        if (!this.secureFieldPrefixes.length)
+            return val;
+        if (!this.hasSecureEncryption())
+            return val;
+        if (val === null || val === undefined)
+            return val;
+        if (Array.isArray(val)) {
+            return val.map((x) => this.decryptSecureFieldsDeep(x, aadPrefix));
+        }
+        if (typeof val !== "object")
+            return val;
+        const out = {};
+        for (const [k, v] of Object.entries(val)) {
+            if (this.isSecureFieldKey(k)) {
+                const aad = `${aadPrefix}|${k}`;
+                out[k] = typeof v === "string" ? this.decryptSecureStringToValue(v, aad) : v;
+            }
+            else {
+                out[k] = this.decryptSecureFieldsDeep(v, aadPrefix);
+            }
+        }
+        return out;
+    }
+    /**
+     * Throws if an aggregation pipeline references any secure-prefixed field paths.
+     * This prevents deriving output from secure fields (e.g. {$group: {x: {$sum: "$#passport_number"}}}).
+     */
+    assertNoSecureFieldRefsInPipeline(pipeline) {
+        if (!this.secureFieldPrefixes.length)
+            return;
+        if (!Array.isArray(pipeline))
+            return;
+        if (this.containsSecureFieldRefDeep(pipeline)) {
+            throw new K2Error(ServiceError.BAD_REQUEST, "Aggregation pipeline references secure-prefixed field(s)", "sys_mdb_ag_secure_field_ref");
+        }
+    }
+    /** Recursively detects secure field references in an aggregation pipeline AST. */
+    containsSecureFieldRefDeep(val) {
+        if (!this.secureFieldPrefixes.length)
+            return false;
+        if (val === null || val === undefined)
+            return false;
+        if (Array.isArray(val)) {
+            return val.some((x) => this.containsSecureFieldRefDeep(x));
+        }
+        if (typeof val === "string") {
+            return this.stringHasSecureFieldPath(val);
+        }
+        if (typeof val !== "object")
+            return false;
+        for (const [k, v] of Object.entries(val)) {
+            // Object keys can be field names in some expressions; treat them as potential paths too.
+            if (this.stringHasSecureFieldPath(k))
+                return true;
+            if (this.containsSecureFieldRefDeep(v))
+                return true;
+        }
+        return false;
+    }
+    /**
+     * Returns true if a string appears to reference a secure field path.
+     * We consider:
+     * - "$path.to.field"
+     * - "$$var.path.to.field"
+     * - plain "path.to.field" (e.g. $getField: { field: "#passport_number" })
+     */
+    stringHasSecureFieldPath(s) {
+        if (!this.secureFieldPrefixes.length)
+            return false;
+        if (typeof s !== "string")
+            return false;
+        const raw = s.trim();
+        if (!raw)
+            return false;
+        // Handle "$$var.path" form (aggregation variables)
+        if (raw.startsWith("$$")) {
+            const after = raw.slice(2);
+            const dot = after.indexOf(".");
+            if (dot === -1)
+                return false; // just a variable name
+            const path = after.slice(dot + 1);
+            return this.pathHasSecureSegment(path);
+        }
+        // Handle "$path" form (field paths)
+        if (raw.startsWith("$")) {
+            const path = raw.slice(1);
+            return this.pathHasSecureSegment(path);
+        }
+        // Handle plain "path" strings (e.g. $getField field names)
+        return this.pathHasSecureSegment(raw);
+    }
+    /** True if any segment in a dotted path starts with a secure prefix. */
+    pathHasSecureSegment(path) {
+        const p = path.trim();
+        if (!p)
+            return false;
+        const segments = p.split(".");
+        for (const seg of segments) {
+            if (!seg)
+                continue;
+            if (this.isSecureFieldKey(seg))
+                return true;
+        }
+        return false;
+    }
+    /**
+     * Ensures an aggregation pipeline respects ownership scope for the root
+     * collection and any joined collections ($lookup, $unionWith, $graphLookup, $facet).
+     */
+    enforceScopeInPipeline(pipeline, scope) {
+        const normalizedScope = this.normalizeScope(scope);
+        // Strict mode requires an explicit scope per call.
+        if (this.ownershipMode === "strict" && !normalizedScope) {
+            throw new K2Error(ServiceError.BAD_REQUEST, "Scope is required in strict ownership mode", "sys_mdb_scope_required");
+        }
+        // Lax mode with no scope, or explicit all-scope: no owner constraint injected.
+        if (!normalizedScope || normalizedScope === "*") {
+            return Array.isArray(pipeline) ? pipeline.map((s) => ({ ...s })) : [];
+        }
+        const cloned = Array.isArray(pipeline) ? pipeline.map((s) => ({ ...s })) : [];
+        // Insert a $match to constrain owner near the start, but after any
+        // first-stage-only operators like $search, $geoNear, $vectorSearch.
+        const reservedFirst = ["$search", "$geoNear", "$vectorSearch"];
+        let insertIdx = 0;
+        while (insertIdx < cloned.length &&
+            typeof cloned[insertIdx] === "object" &&
+            cloned[insertIdx] !== null &&
+            Object.keys(cloned[insertIdx]).length === 1 &&
+            reservedFirst.includes(Object.keys(cloned[insertIdx])[0])) {
+            insertIdx++;
+        }
+        const ownerMatch = { $match: { _owner: normalizedScope } };
+        cloned.splice(insertIdx, 0, ownerMatch);
+        const mapStage = (stage) => {
+            if (!stage || typeof stage !== "object")
+                return stage;
+            if (stage.$lookup) {
+                const lu = { ...stage.$lookup };
+                if (Array.isArray(lu.pipeline)) {
+                    lu.pipeline = this.enforceScopeInPipeline(lu.pipeline, normalizedScope);
+                }
+                else if (lu.localField && lu.foreignField) {
+                    // Convert simple lookup to pipeline lookup so we can enforce owner scope (and deleted) in foreign coll.
+                    const localVar = "__lk";
+                    const ownerVar = "__own";
+                    lu.let = { [localVar]: `$${lu.localField}`, [ownerVar]: normalizedScope };
+                    lu.pipeline = [
+                        {
+                            $match: {
+                                $expr: {
+                                    $and: [
+                                        {
+                                            $cond: [
+                                                { $isArray: "$$" + localVar },
+                                                { $in: ["$" + lu.foreignField, "$$" + localVar] },
+                                                { $eq: ["$" + lu.foreignField, "$$" + localVar] },
+                                            ],
+                                        },
+                                        { $eq: ["$_owner", "$$" + ownerVar] },
+                                        { $ne: ["$_deleted", true] },
+                                    ],
+                                },
+                            },
+                        },
+                    ];
+                    delete lu.localField;
+                    delete lu.foreignField;
+                }
+                return { $lookup: lu };
+            }
+            if (stage.$unionWith) {
+                const uw = stage.$unionWith;
+                if (typeof uw === "string") {
+                    return {
+                        $unionWith: {
+                            coll: uw,
+                            pipeline: [
+                                { $match: { _owner: normalizedScope } },
+                                { $match: { _deleted: { $ne: true } } },
+                            ],
+                        },
+                    };
+                }
+                else if (uw && typeof uw === "object") {
+                    const uwc = { ...uw };
+                    uwc.pipeline = this.enforceScopeInPipeline(uwc.pipeline || [], normalizedScope);
+                    return { $unionWith: uwc };
+                }
+            }
+            if (stage.$graphLookup) {
+                const gl = { ...stage.$graphLookup };
+                const existing = gl.restrictSearchWithMatch || {};
+                gl.restrictSearchWithMatch = { ...existing, _owner: normalizedScope };
+                return { $graphLookup: gl };
+            }
+            if (stage.$facet) {
+                const facets = { ...stage.$facet };
+                for (const key of Object.keys(facets)) {
+                    facets[key] = this.enforceScopeInPipeline(facets[key] || [], normalizedScope);
+                }
+                return { $facet: facets };
+            }
+            return stage;
+        };
+        return cloned.map(mapStage);
+    }
+    /**
+     * Ensures an aggregation pipeline excludes soft-deleted documents for the root
+     * collection and any joined collections ($lookup, $unionWith, $graphLookup, $facet).
+     */
+    static enforceNoDeletedInPipeline(pipeline) {
+        const cloned = Array.isArray(pipeline) ? pipeline.map((s) => ({ ...s })) : [];
+        // Insert a $match to exclude deleted near the start, but after any
+        // first-stage-only operators like $search, $geoNear, $vectorSearch.
+        const reservedFirst = ["$search", "$geoNear", "$vectorSearch"];
+        let insertIdx = 0;
+        while (insertIdx < cloned.length &&
+            typeof cloned[insertIdx] === "object" &&
+            cloned[insertIdx] !== null &&
+            Object.keys(cloned[insertIdx]).length === 1 &&
+            reservedFirst.includes(Object.keys(cloned[insertIdx])[0])) {
+            insertIdx++;
+        }
+        const nonDeletedMatch = { $match: { _deleted: { $ne: true } } };
+        cloned.splice(insertIdx, 0, nonDeletedMatch);
+        // Walk stages and enforce inside nested pipelines
+        const mapStage = (stage) => {
+            if (!stage || typeof stage !== "object")
+                return stage;
+            if (stage.$lookup) {
+                const lu = { ...stage.$lookup };
+                if (Array.isArray(lu.pipeline)) {
+                    // Ensure the foreign pipeline excludes deleted
+                    lu.pipeline = K2DB.enforceNoDeletedInPipeline(lu.pipeline);
+                }
+                else if (lu.localField && lu.foreignField) {
+                    // Convert simple lookup to pipeline lookup to filter _deleted
+                    const localVar = "__lk";
+                    lu.let = { [localVar]: `$${lu.localField}` };
+                    lu.pipeline = [
+                        {
+                            $match: {
+                                $expr: {
+                                    $and: [
+                                        {
+                                            $cond: [
+                                                { $isArray: "$$" + localVar },
+                                                { $in: ["$" + lu.foreignField, "$$" + localVar] },
+                                                { $eq: ["$" + lu.foreignField, "$$" + localVar] },
+                                            ],
+                                        },
+                                        { $ne: ["$_deleted", true] },
+                                    ],
+                                },
+                            },
+                        },
+                    ];
+                    delete lu.localField;
+                    delete lu.foreignField;
+                }
+                return { $lookup: lu };
+            }
+            if (stage.$unionWith) {
+                const uw = stage.$unionWith;
+                if (typeof uw === "string") {
+                    return {
+                        $unionWith: {
+                            coll: uw,
+                            pipeline: [{ $match: { _deleted: { $ne: true } } }],
+                        },
+                    };
+                }
+                else if (uw && typeof uw === "object") {
+                    const uwc = { ...uw };
+                    uwc.pipeline = K2DB.enforceNoDeletedInPipeline(uwc.pipeline || []);
+                    return { $unionWith: uwc };
+                }
+            }
+            if (stage.$graphLookup) {
+                const gl = { ...stage.$graphLookup };
+                const existing = gl.restrictSearchWithMatch || {};
+                gl.restrictSearchWithMatch = { ...existing, _deleted: { $ne: true } };
+                return { $graphLookup: gl };
+            }
+            if (stage.$facet) {
+                const facets = { ...stage.$facet };
+                for (const key of Object.keys(facets)) {
+                    facets[key] = K2DB.enforceNoDeletedInPipeline(facets[key] || []);
+                }
+                return { $facet: facets };
+            }
+            return stage;
+        };
+        return cloned.map(mapStage);
+    }
+    /**
+     * Creates a new document in the collection.
+     * @param collectionName - Name of the collection.
+     * @param owner - Owner of the document.
+     * @param data - Data to insert.
+     */
+    async create(collectionName, owner, data) {
+        if (!collectionName || !owner || !data) {
+            throw new K2Error(ServiceError.BAD_REQUEST, "Invalid method usage, parameters not defined", "sys_mdb_crv1");
+        }
+        if (typeof owner !== "string") {
+            throw new K2Error(ServiceError.BAD_REQUEST, "Owner must be of a string type", "sys_mdb_crv2");
+        }
+        const normalizedOwner = owner.trim().toLowerCase();
+        if (!normalizedOwner) {
+            throw new K2Error(ServiceError.BAD_REQUEST, "Owner must be a non-empty string", "sys_mdb_owner_empty");
+        }
+        if (normalizedOwner === "*") {
+            throw new K2Error(ServiceError.BAD_REQUEST, "Owner cannot be '*'", "sys_mdb_owner_star");
+        }
+        const collection = await this.getCollection(collectionName);
+        const timestamp = Date.now();
+        // Generate a new UUIDv7 encoded as Crockford Base32 with hyphens
+        const newUuid = uuidv7Base32Hyphenated();
+        // Remove reserved fields from user data, then validate/transform via schema if present
+        const safeData = K2DB.stripReservedFields(data);
+        const validated = this.applySchema(collectionName, safeData, /*partial*/ false);
+        const storedData = this.encryptSecureFieldsDeep(validated, `k2db|${collectionName}|${newUuid}`);
+        // Spread validated data first, then set internal fields to prevent overwriting
+        const document = {
+            ...storedData,
+            _created: timestamp,
+            _updated: timestamp,
+            _owner: normalizedOwner,
+            _uuid: newUuid,
+        };
+        try {
+            await this.runTimed("insertOne", { collectionName }, async () => await collection.insertOne(document));
+            return { id: document._uuid };
+        }
+        catch (err) {
+            // Use appropriate error typing
+            // Check if the error is a duplicate key error
+            if (err.code === 11000 && err.keyPattern && err.keyPattern._uuid) {
+                throw new K2Error(ServiceError.ALREADY_EXISTS, `A document with _uuid ${document._uuid} already exists.`, "sys_mdb_crv3");
+            }
+            // Log the error details for debugging
+            debug(`Was trying to insert into collection ${collectionName}, data: ${JSON.stringify(document)}`);
+            debug(err);
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_sav", "Error saving object to database");
+        }
+    }
+    /**
+     * Updates multiple documents based on criteria.
+     * Can either replace the documents or patch them.
+     * @param collectionName - Name of the collection.
+     * @param criteria - Update criteria.
+     * @param values - Values to update or replace with.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async updateAll(collectionName, criteria, values, scope) {
+        this.validateCollectionName(collectionName);
+        const collection = await this.getCollection(collectionName);
+        debug(`Updating ${collectionName} with criteria: ${JSON.stringify(criteria)}`);
+        // Preserve intent to set _deleted during internal soft-delete operations.
+        // stripReservedFields removes underscore-prefixed keys (by design) to protect
+        // internal fields from user updates. However, deleteAll() legitimately passes
+        // {_deleted: true}. Capture and restore it here.
+        const deletedFlag = Object.prototype.hasOwnProperty.call(values, "_deleted")
+            ? values._deleted
+            : undefined;
+        values = K2DB.stripReservedFields(values);
+        values = this.applySchema(collectionName, values, /*partial*/ true);
+        values._updated = Date.now();
+        if (deletedFlag !== undefined) {
+            values._deleted = deletedFlag;
+        }
+        // Encrypt secure-prefixed fields at rest (no-op unless encryption is configured)
+        values = this.encryptSecureFieldsDeep(values, `k2db|${collectionName}`);
+        criteria = K2DB.normalizeCriteriaIds(criteria || {});
+        criteria = this.applyScopeToCriteria(criteria, scope);
+        criteria = {
+            ...criteria,
+            _deleted: { $ne: true },
+        };
+        try {
+            const res = await this.runTimed("updateMany", { collectionName, criteria, values }, async () => await collection.updateMany(criteria, { $set: values }));
+            return {
+                updated: res.modifiedCount,
+            };
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_update1", `Error updating ${collectionName}`);
+        }
+    }
+    /**
+     * Updates a single document by UUID.
+     * Can either replace the document or patch it.
+     * @param collectionName - Name of the collection.
+     * @param id - UUID string to identify the document.
+     * @param data - Data to update or replace with.
+     * @param replace - If true, replaces the entire document (PUT), otherwise patches (PATCH).
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async update(collectionName, id, data, replace = false, scope) {
+        id = K2DB.normalizeId(id);
+        this.validateCollectionName(collectionName);
+        const collection = await this.getCollection(collectionName);
+        data = K2DB.stripReservedFields(data);
+        data = this.applySchema(collectionName, data, /*partial*/ !replace);
+        data._updated = Date.now(); // Set the _updated timestamp
+        // PATCH path: encrypt secure-prefixed fields at rest (no-op unless encryption is configured).
+        // REPLACE path: encrypt after underscore-field merge to avoid double-encrypt.
+        if (!replace) {
+            data = this.encryptSecureFieldsDeep(data, `k2db|${collectionName}|${id}`);
+        }
+        try {
+            let res;
+            // If replacing the document, first get the original document
+            if (replace) {
+                // Get the original document to preserve fields starting with underscore
+                const originalDoc = await this.get(collectionName, id, scope);
+                // Override all fields starting with underscore from the original document
+                const fieldsToPreserve = Object.keys(originalDoc).reduce((acc, key) => {
+                    if (key.startsWith("_")) {
+                        acc[key] = originalDoc[key];
+                    }
+                    return acc;
+                }, {});
+                // Merge the preserved fields into the data
+                data = { ...data, ...fieldsToPreserve };
+                data._updated = Date.now();
+                // Encrypt secure-prefixed fields at rest (no-op unless encryption is configured)
+                data = this.encryptSecureFieldsDeep(data, `k2db|${collectionName}|${id}`);
+                // Now replace the document with the merged data
+                res = await this.runTimed("replaceOne", { collectionName, _uuid: id }, async () => await collection.replaceOne(this.applyScopeToCriteria({ _uuid: id, _deleted: { $ne: true } }, scope), data));
+            }
+            else {
+                // If patching, just update specific fields using $set
+                res = await this.runTimed("updateOne", { collectionName, _uuid: id }, async () => await collection.updateOne(this.applyScopeToCriteria({ _uuid: id, _deleted: { $ne: true } }, scope), { $set: data }));
+            }
+            // Use matchedCount to determine existence; modifiedCount indicates actual change
+            if (res.matchedCount === 0) {
+                throw new K2Error(ServiceError.NOT_FOUND, `Object in ${collectionName} with UUID ${id} not found`, "sys_mdb_update_not_found");
+            }
+            return { updated: res.modifiedCount };
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_update_error", `Error updating ${collectionName}`);
+        }
+    }
+    /**
+     * Removes (soft deletes) multiple documents based on criteria.
+     * @param collectionName - Name of the collection.
+     * @param criteria - Removal criteria.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async deleteAll(collectionName, criteria, scope) {
+        this.validateCollectionName(collectionName);
+        try {
+            const result = await this.updateAll(collectionName, criteria, {
+                _deleted: true,
+            }, scope);
+            return { deleted: result.updated };
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_deleteall_update", `Error updating ${collectionName}`);
+        }
+    }
+    /**
+     * Removes (soft deletes) a single document by UUID.
+     * @param collectionName - Name of the collection.
+     * @param id - UUID of the document.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async delete(collectionName, id, scope) {
+        id = K2DB.normalizeId(id);
+        try {
+            // Call deleteAll to soft delete the document by UUID
+            const result = await this.deleteAll(collectionName, { _uuid: id }, scope);
+            // Check the result of the deleteAll operation
+            if (result.deleted === 1) {
+                // Successfully deleted one document
+                return { deleted: 1 };
+            }
+            else if (result.deleted === 0) {
+                // No document was found to delete
+                throw new K2Error(ServiceError.NOT_FOUND, "Document not found", "sys_mdb_remove_not_found");
+            }
+            else {
+                // More than one document was deleted, which is unexpected
+                throw new K2Error(ServiceError.SYSTEM_ERROR, "Multiple documents deleted when only one was expected", "sys_mdb_remove_multiple_deleted");
+            }
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_remove_upd", "Error removing object from collection");
+        }
+    }
+    /**
+     * Permanently deletes a document that has been soft-deleted.
+     * @param collectionName - Name of the collection.
+     * @param id - UUID of the document.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async purge(collectionName, id, scope) {
+        id = K2DB.normalizeId(id);
+        const collection = await this.getCollection(collectionName);
+        try {
+            const findFilter = this.applyScopeToCriteria({ _uuid: id, _deleted: true }, scope);
+            const item = await this.runTimed("findOne", { collectionName, ...findFilter }, async () => await collection.findOne(findFilter));
+            if (!item) {
+                throw new K2Error(ServiceError.SYSTEM_ERROR, "Cannot purge item that is not deleted", "sys_mdb_gcol_pg2");
+            }
+            const delFilter = this.applyScopeToCriteria({ _uuid: id }, scope);
+            await this.runTimed("deleteOne", { collectionName, ...delFilter }, async () => await collection.deleteOne(delFilter));
+            return { id };
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_pg", `Error purging item with id: ${id}`);
+        }
+    }
+    /**
+     * Permanently deletes all documents that are soft-deleted and whose _updated
+     * timestamp is older than the provided threshold (in milliseconds ago).
+     * @param collectionName - Name of the collection.
+     * @param olderThanMs - Age threshold in milliseconds; documents with
+     *                      `_updated <= (Date.now() - olderThanMs)` will be purged.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async purgeDeletedOlderThan(collectionName, olderThanMs, scope) {
+        this.validateCollectionName(collectionName);
+        if (typeof olderThanMs !== 'number' || !isFinite(olderThanMs) || olderThanMs < 0) {
+            throw new K2Error(ServiceError.BAD_REQUEST, 'olderThanMs must be a non-negative number', 'sys_mdb_purge_older_invalid');
+        }
+        const collection = await this.getCollection(collectionName);
+        const cutoff = Date.now() - olderThanMs;
+        try {
+            const delFilter = this.applyScopeToCriteria({
+                _deleted: true,
+                _updated: { $lte: cutoff },
+            }, scope);
+            const res = await this.runTimed('deleteMany', { collectionName, olderThanMs, cutoff, ...delFilter }, async () => await collection.deleteMany(delFilter));
+            return { purged: res.deletedCount ?? 0 };
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, 'sys_mdb_purge_older', 'Error purging deleted items by age');
+        }
+    }
+    /**
+     * Restores a soft-deleted document.
+     * @param collectionName - Name of the collection.
+     * @param criteria - Criteria to identify the document.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async restore(collectionName, criteria, scope) {
+        const collection = await this.getCollection(collectionName);
+        let crit = K2DB.normalizeCriteriaIds(criteria || {});
+        crit = this.applyScopeToCriteria(crit, scope);
+        const query = { ...crit, _deleted: true };
+        try {
+            const res = await this.runTimed("updateMany", { collectionName, query }, async () => await collection.updateMany(query, {
+                // Restoring is a data change: flip _deleted and bump _updated
+                $set: { _deleted: false, _updated: Date.now() },
+            }));
+            return { status: "restored", modified: res.modifiedCount };
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_pres", "Error restoring a deleted item");
+        }
+    }
+    /**
+     * Counts documents based on criteria.
+     * @param collectionName - Name of the collection.
+     * @param criteria - Counting criteria.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async count(collectionName, criteria, scope) {
+        const collection = await this.getCollection(collectionName);
+        try {
+            const norm = K2DB.normalizeCriteriaIds(criteria || {});
+            let query = {
+                ...norm,
+                ...(criteria && Object.prototype.hasOwnProperty.call(criteria, "_deleted")
+                    ? {}
+                    : { _deleted: { $ne: true } }),
+            };
+            query = this.applyScopeToCriteria(query, scope);
+            const cnt = await this.runTimed("countDocuments", { collectionName, query }, async () => await collection.countDocuments(query));
+            return { count: cnt };
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_cn", "Error counting objects with given criteria");
+        }
+    }
+    /**
+     * Drops an entire collection (global destructive operation).
+     * @param collectionName - Name of the collection.
+     * @param scope - (optional) Must be "*" in strict ownership mode.
+     */
+    async drop(collectionName, scope) {
+        const collection = await this.getCollection(collectionName);
+        const normalizedScope = this.normalizeScope(scope);
+        // Global destructive operation: require explicit all-scope gate in strict mode.
+        if (this.ownershipMode === "strict" && normalizedScope !== "*") {
+            throw new K2Error(ServiceError.BAD_REQUEST, 'Dropping a collection requires scope="*" in strict ownership mode', "sys_mdb_drop_scope_required");
+        }
+        // In lax mode, allow legacy behavior when scope is omitted; if provided, it must be "*".
+        if (this.ownershipMode === "lax" && normalizedScope !== undefined && normalizedScope !== "*") {
+            throw new K2Error(ServiceError.BAD_REQUEST, 'Dropping a collection only supports scope="*"', "sys_mdb_drop_scope_invalid");
+        }
+        try {
+            await collection.drop();
+            return { status: "ok" };
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_drop", "Error dropping collection");
+        }
+    }
+    /**
+     * Sanitizes aggregation criteria.
+     * @param criteria - Aggregation stage criteria.
+     */
+    static sanitiseCriteria(criteria) {
+        if (criteria.$match) {
+            // Normalize any _uuid values in the match object to uppercase
+            criteria.$match = K2DB.normalizeCriteriaIds(criteria.$match);
+            for (const key of Object.keys(criteria.$match)) {
+                if (typeof criteria.$match[key] !== "string") {
+                    criteria.$match[key] = K2DB.sanitiseCriteria({
+                        [key]: criteria.$match[key],
+                    })[key];
+                }
+                else {
+                    if (key === "$exists") {
+                        criteria.$match[key] = criteria.$match[key] === "true";
+                    }
+                }
+            }
+        }
+        return criteria;
+    }
+    /** Recursively normalizes query fields: `_uuid` uppercased, `_owner` lowercased. */
+    static normalizeCriteriaIds(obj) {
+        if (!obj || typeof obj !== "object")
+            return obj;
+        if (Array.isArray(obj))
+            return obj.map((v) => K2DB.normalizeCriteriaIds(v));
+        const out = Array.isArray(obj) ? [] : { ...obj };
+        for (const [k, v] of Object.entries(obj)) {
+            if (k === "_uuid") {
+                out[k] = K2DB.normalizeUuidField(v);
+            }
+            else if (k === "_owner") {
+                out[k] = K2DB.normalizeOwnerField(v);
+            }
+            else if (v && typeof v === "object") {
+                out[k] = K2DB.normalizeCriteriaIds(v);
+            }
+            else if (Array.isArray(v)) {
+                out[k] = v.map((x) => K2DB.normalizeCriteriaIds(x));
+            }
+            else {
+                out[k] = v;
+            }
+        }
+        return out;
+    }
+    /** Uppercase helper for `_uuid` field supporting operators like $in/$nin/$eq/$ne and arrays. */
+    static normalizeUuidField(val) {
+        if (typeof val === "string")
+            return val.toUpperCase();
+        if (Array.isArray(val))
+            return val.map((x) => (typeof x === "string" ? x.toUpperCase() : x));
+        if (val && typeof val === "object") {
+            const out = { ...val };
+            for (const op of ["$in", "$nin", "$eq", "$ne", "$all"]) {
+                if (op in out)
+                    out[op] = K2DB.normalizeUuidField(out[op]);
+            }
+            return out;
+        }
+        return val;
+    }
+    /** Lowercase helper for `_owner` field supporting operators like $in/$nin/$eq/$ne and arrays. */
+    static normalizeOwnerField(val) {
+        if (typeof val === "string")
+            return val.trim().toLowerCase();
+        if (Array.isArray(val)) {
+            return val.map((x) => (typeof x === "string" ? x.trim().toLowerCase() : x));
+        }
+        if (val && typeof val === "object") {
+            const out = { ...val };
+            for (const op of ["$in", "$nin", "$eq", "$ne", "$all"]) {
+                if (op in out)
+                    out[op] = K2DB.normalizeOwnerField(out[op]);
+            }
+            return out;
+        }
+        return val;
+    }
+    /** Strip any user-provided fields that start with '_' (reserved). */
+    static stripReservedFields(obj) {
+        const out = {};
+        for (const [k, v] of Object.entries(obj || {})) {
+            if (!k.startsWith("_"))
+                out[k] = v;
+        }
+        return out;
+    }
+    /** True if string matches K2 ID format (Crockford Base32, 8-4-4-4-6, uppercase). */
+    static isK2ID(id) {
+        if (typeof id !== "string")
+            return false;
+        const s = id.trim().toUpperCase();
+        const CROCK_RE = /^[0-9A-HJKMNPQRSTVWXYZ]{8}-[0-9A-HJKMNPQRSTVWXYZ]{4}-[0-9A-HJKMNPQRSTVWXYZ]{4}-[0-9A-HJKMNPQRSTVWXYZ]{4}-[0-9A-HJKMNPQRSTVWXYZ]{6}$/;
+        return CROCK_RE.test(s);
+    }
+    /** Uppercase incoming IDs for case-insensitive lookups. */
+    static normalizeId(id) {
+        return id.toUpperCase();
+    }
+    /**
+     * Run an async DB operation with timing, slow logging, and hooks.
+     */
+    async runTimed(op, details, fn) {
+        try {
+            this.conf.hooks?.beforeQuery?.(op, details);
+        }
+        catch { }
+        const start = Date.now();
+        try {
+            const res = await fn();
+            const dur = Date.now() - start;
+            const slow = this.conf.slowQueryMs ?? 200;
+            if (dur > slow) {
+                debug(`[SLOW ${op}] ${dur}ms ${JSON.stringify(details)}`);
+            }
+            try {
+                this.conf.hooks?.afterQuery?.(op, { ...details, ok: true }, dur);
+            }
+            catch { }
+            return res;
+        }
+        catch (e) {
+            const dur = Date.now() - start;
+            try {
+                this.conf.hooks?.afterQuery?.(op, { ...details, ok: false, error: e instanceof Error ? e.message : String(e) }, dur);
+            }
+            catch { }
+            throw e;
+        }
+    }
+    /**
+     * Ensure commonly needed indexes exist.
+     */
+    async ensureIndexes(collectionName, opts = {}) {
+        const { uuidUnique = false, uuidPartialUnique = true, ownerIndex = true, deletedIndex = true, } = opts;
+        const collection = await this.getCollection(collectionName);
+        if (uuidPartialUnique) {
+            // Use a compound unique index to ensure at most one non-deleted document per _uuid
+            // without relying on partialFilterExpression (which may be limited in some environments).
+            await collection.createIndex({ _uuid: 1, _deleted: 1 }, { unique: true });
+        }
+        else if (uuidUnique) {
+            await collection.createIndex({ _uuid: 1 }, { unique: true });
+        }
+        if (ownerIndex) {
+            await collection.createIndex({ _owner: 1 });
+        }
+        if (deletedIndex) {
+            await collection.createIndex({ _deleted: 1 });
+        }
+    }
+    /**
+     * Optional: Executes a transaction with the provided operations.
+     * @param operations - A function that performs operations within a transaction session.
+     */
+    async executeTransaction(operations) {
+        const session = this.connection.startSession();
+        session.startTransaction();
+        try {
+            await operations(session);
+            await session.commitTransaction();
+        }
+        catch (error) {
+            await session.abortTransaction();
+            throw wrap(error, ServiceError.BAD_GATEWAY, "sys_mdb_txn", "Transaction failed");
+        }
+        finally {
+            session.endSession();
+        }
+    }
+    /**
+     * Optional: Creates an index on the specified collection.
+     * @param collectionName - Name of the collection.
+     * @param indexSpec - Specification of the index.
+     * @param options - Optional index options.
+     */
+    async createIndex(collectionName, indexSpec, options) {
+        const collection = await this.getCollection(collectionName);
+        try {
+            await collection.createIndex(indexSpec, options);
+            debug(`Index created on ${collectionName}: ${JSON.stringify(indexSpec)}`);
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_idx", `Error creating index on ${collectionName}`);
+        }
+    }
+    /**
+     * Releases the MongoDB connection.
+     */
+    async release() {
+        if (this.initialized && this.clientKey) {
+            const key = this.clientKey;
+            this.initialized = false;
+            this.clientKey = undefined;
+            await _decrefClient(key);
+            debug("MongoDB connection released");
+            return;
+        }
+        debug("MongoDB connection released");
+    }
+    /**
+     * Closes the MongoDB connection.
+     */
+    close() {
+        // Fire-and-forget async release (shared pool is refcounted)
+        void this.release();
+    }
+    /**
+     * Drops the entire database.
+     */
+    async dropDatabase() {
+        try {
+            await this.db.dropDatabase();
+            debug("Database dropped successfully");
+        }
+        catch (err) {
+            throw wrap(err, ServiceError.SYSTEM_ERROR, "sys_mdb_drop_db", "Error dropping database");
+        }
+    }
+    /**
+     * Validates the MongoDB collection name.
+     * @param collectionName - The name of the collection to validate.
+     * @throws {K2Error} - If the collection name is invalid.
+     */
+    validateCollectionName(collectionName) {
+        // Check for null character
+        if (collectionName.includes("\0")) {
+            throw new K2Error(ServiceError.BAD_REQUEST, "Collection name cannot contain null characters", "sys_mdb_invalid_collection_name");
+        }
+        // Check if it starts with 'system.'
+        if (collectionName.startsWith("system.")) {
+            throw new K2Error(ServiceError.BAD_REQUEST, "Collection name cannot start with 'system.'", "sys_mdb_invalid_collection_name");
+        }
+        // Check for invalid characters (e.g., '$')
+        if (collectionName.includes("$")) {
+            throw new K2Error(ServiceError.BAD_REQUEST, "Collection name cannot contain the '$' character", "sys_mdb_invalid_collection_name");
+        }
+        // Additional checks can be added here as needed
+    }
+    /**
+     * Optional: Checks the health of the database connection.
+     */
+    async isHealthy() {
+        try {
+            await this.db.command({ ping: 1 });
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    // ===== Versioning helpers and APIs =====
+    /** Name of the history collection for a given collection. */
+    historyName(collectionName) {
+        return `${collectionName}__history`;
+    }
+    // ===== Zod schema registry (opt-in) =====
+    /** Register a Zod schema for a collection. */
+    setSchema(collectionName, schema, options = {}) {
+        const mode = options.mode ?? "strip";
+        this.schemas.set(collectionName, { schema, mode });
+    }
+    /** Clear a collection's schema. */
+    clearSchema(collectionName) {
+        this.schemas.delete(collectionName);
+    }
+    /** Clear all schemas. */
+    clearSchemas() {
+        this.schemas.clear();
+    }
+    /** Apply registered schema (if any) to data. For updates, partial=true allows partial input. */
+    applySchema(collectionName, data, partial) {
+        const entry = this.schemas.get(collectionName);
+        if (!entry)
+            return data;
+        let s = entry.schema;
+        // If schema is an object, apply unknown key policy and partial when needed
+        if (s instanceof z.ZodObject) {
+            const so = s;
+            const shaped = entry.mode === "strict" ? so.strict() : entry.mode === "passthrough" ? so.passthrough() : so.strip();
+            s = partial ? shaped.partial() : shaped;
+        }
+        else {
+            // Non-object schema: partial has no effect; leave as-is
+        }
+        const parsed = s.safeParse(data);
+        if (!parsed.success) {
+            throw new K2Error(ServiceError.VALIDATION_ERROR, parsed.error.message, "sys_mdb_schema_validation", parsed.error);
+        }
+        return parsed.data;
+    }
+    /** Get the history collection. */
+    async getHistoryCollection(collectionName) {
+        return this.db.collection(this.historyName(collectionName));
+    }
+    /** Ensure indexes for history tracking. */
+    async ensureHistoryIndexes(collectionName) {
+        const hc = await this.getHistoryCollection(collectionName);
+        await hc.createIndex({ _uuid: 1, _v: 1 }, { unique: true });
+        await hc.createIndex({ _uuid: 1, _at: -1 });
+    }
+    /** Compute the next version number for a document. */
+    async nextVersion(collectionName, id) {
+        id = K2DB.normalizeId(id);
+        const hc = await this.getHistoryCollection(collectionName);
+        const last = await hc
+            .find({ _uuid: id })
+            .project({ _v: 1 })
+            .sort({ _v: -1 })
+            .limit(1)
+            .toArray();
+        return last.length ? last[0]._v + 1 : 1;
+    }
+    /** Save a snapshot of the current document into the history collection. */
+    async snapshotCurrent(collectionName, current) {
+        const hc = await this.getHistoryCollection(collectionName);
+        const version = await this.nextVersion(collectionName, current._uuid);
+        const storedSnapshot = this.encryptSecureFieldsDeep(current, `k2db|${collectionName}|${current._uuid}`);
+        await hc.insertOne({
+            _uuid: current._uuid,
+            _v: version,
+            _at: Date.now(),
+            snapshot: storedSnapshot,
+        });
+        return { version };
+    }
+    /**
+     * Update a document and keep the previous version in a history collection.
+     * If maxVersions is provided, prunes oldest snapshots beyond that number.
+     * @param collectionName - Name of the collection.
+     * @param id - UUID string to identify the document.
+     * @param data - Data to update or replace with.
+     * @param replace - If true, replaces the entire document (PUT), otherwise patches (PATCH).
+     * @param maxVersions - Maximum number of versions to keep (optional).
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async updateVersioned(collectionName, id, data, replace = false, maxVersions, scope) {
+        id = K2DB.normalizeId(id);
+        // Get current doc (excludes deleted) and snapshot it
+        const current = await this.get(collectionName, id, scope);
+        await this.ensureHistoryIndexes(collectionName);
+        const { version } = await this.snapshotCurrent(collectionName, current);
+        // Perform update
+        const res = await this.update(collectionName, id, data, replace, scope);
+        // Optionally prune old versions
+        if (typeof maxVersions === "number" && maxVersions >= 0) {
+            const hc = await this.getHistoryCollection(collectionName);
+            const count = await hc.countDocuments({ _uuid: id });
+            const overflow = count - maxVersions;
+            if (overflow > 0) {
+                const olds = await hc
+                    .find({ _uuid: id })
+                    .project({ _v: 1 })
+                    .sort({ _v: 1 })
+                    .limit(overflow)
+                    .toArray();
+                const vs = olds.map((o) => o._v);
+                if (vs.length) {
+                    await hc.deleteMany({ _uuid: id, _v: { $in: vs } });
+                }
+            }
+        }
+        return [{ updated: res.updated, versionSaved: version }];
+    }
+    /**
+     * List versions (latest first).
+     * @param collectionName - Name of the collection.
+     * @param id - UUID string to identify the document.
+     * @param skip - Number of versions to skip (for pagination).
+     * @param limit - Maximum number of versions to return.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async listVersions(collectionName, id, skip = 0, limit = 20, scope) {
+        id = K2DB.normalizeId(id);
+        // Gate history access by ownership of the current document
+        await this.get(collectionName, id, scope);
+        const hc = await this.getHistoryCollection(collectionName);
+        const rows = await hc
+            .find({ _uuid: id })
+            .project({ _uuid: 1, _v: 1, _at: 1 })
+            .sort({ _v: -1 })
+            .skip(skip)
+            .limit(limit)
+            .toArray();
+        return rows;
+    }
+    /**
+     * Revert the current document to a specific historical version (preserves metadata).
+     * @param collectionName - Name of the collection.
+     * @param id - UUID string to identify the document.
+     * @param version - Version number to revert to.
+     * @param scope - (optional) Owner selector; "*" means all owners.
+     */
+    async revertToVersion(collectionName, id, version, scope) {
+        id = K2DB.normalizeId(id);
+        // Gate history access by ownership of the current document
+        await this.get(collectionName, id, scope);
+        const hc = await this.getHistoryCollection(collectionName);
+        const row = await hc.findOne({ _uuid: id, _v: version });
+        if (!row) {
+            throw new K2Error(ServiceError.NOT_FOUND, `Version ${version} for ${id} not found`, "sys_mdb_version_not_found");
+        }
+        const snapshot = row.snapshot;
+        const snapshotPlain = this.decryptSecureFieldsDeep(snapshot, `k2db|${collectionName}|${id}`);
+        // Only apply non-underscore fields; metadata is preserved by replace=true path
+        const apply = {};
+        for (const [k, v] of Object.entries(snapshotPlain)) {
+            if (!k.startsWith("_"))
+                apply[k] = v;
+        }
+        return this.update(collectionName, id, apply, true, scope);
+    }
+}