cry-db 2.4.18 → 2.4.21

package/dist/mongo.mjs

@@ -4,7 +4,7 @@ import cloneDeep from "lodash.clonedeep";
 import { ObjectId, ReadConcern, ReadPreference, Timestamp, WriteConcern } from 'mongodb';
 import { TypedEmitter } from "tiny-typed-emitter";
 import { Db, log } from './db.mjs';
-import { SEQUENCES_COLLECTION } from './types.mjs';
+import { FIELD_SEQUENCES_COLLECTION, SEQUENCES_COLLECTION } from './types.mjs';
 import { Base } from "./base.mjs";
 const assert = (cond, msg) => {
     if (!cond) {
@@ -70,16 +70,19 @@ export class Mongo extends Db {
         super(db, url);
         this.revisions = false;
         this.softdelete = false;
+        this.softarchive = false;
         this.syncSupport = false;
         this.session = undefined;
         this.emittingPublishEvents = false;
         this.emittingPublishRevEvents = false;
         this.auditing = false;
         this.auditCollectionName = "dblog";
+        this.auditedCollectionsLower = [];
         this.auditedCollections = this.auditCollections(process.env.AUDIT_COLLECTIONS || []);
         this.emitter = new TypedEmitter();
         this.user = undefined;
         this.audit = undefined;
+        this._sequencesIndexEnsured = false;
         fjLog.debug('new Mongo:', this.url, this.db);
     }
     on(evt, listener) {
@@ -112,6 +115,17 @@ export class Mongo extends Db {
     usesSoftDelete() {
         return this.softdelete;
     }
+    /**
+     * Enable or disable archive filtering.
+     * When enabled, query operations exclude records with `_archived` set,
+     * unless `returnArchived: true` is passed in opts.
+     */
+    useArchive(enabled) {
+        return this.softarchive = !!enabled;
+    }
+    usesArchive() {
+        return this.softarchive;
+    }
     useAuditing(enabled) {
         return this.auditing = enabled;
     }
@@ -137,9 +151,11 @@ export class Mongo extends Db {
             if (this.auditedCollections.includes(coll))
                 return;
             this.auditedCollections.push(coll);
+            this.auditedCollectionsLower.push(coll);
         }
         else {
             this.auditedCollections = this.auditedCollections.filter((c) => c != coll);
+            this.auditedCollectionsLower = this.auditedCollectionsLower.filter((c) => c != coll);
         }
     }
     auditCollections(arr) {
@@ -149,6 +165,7 @@ export class Mongo extends Db {
             this.auditedCollections = arr.toString().split(',').map(a => a.trim().toLowerCase()).filter(a => !!a);
         if (arr instanceof Array)
             this.auditedCollections = arr.map(a => a.trim().toLowerCase()).filter(a => !!a);
+        this.auditedCollectionsLower = this.auditedCollections.map(a => a.toLowerCase());
         return this.auditedCollections;
     }
     auditingCollection(coll, db = this.db) {
@@ -172,11 +189,19 @@ export class Mongo extends Db {
     emitsPublishRevEvents() {
         return this.emittingPublishRevEvents;
     }
-    async distinct(collection, field) {
+    /**
+     * Returns distinct values for a field. Results are sorted.
+     * @param opts - Use `{ collation: { locale: "en", strength: 2 } }` for case-insensitive
+     */
+    async distinct(collection, field, opts = {}) {
+        assert(collection);
+        assert(field);
         fjLog.debug('distinct called', collection, field);
-        let ret = await this.executeTransactionally(collection, async (conn) => {
-            return await conn.distinct(field);
-        }, false, { operation: "distinct", collection, field });
+        const dbName = this.db;
+        let client = await this.connect();
+        let conn = client.db(dbName).collection(collection);
+        let ret = await conn.distinct(field, {}, opts);
+        ret.sort();
         fjLog.debug('distinct returns', ret);
         return ret;
     }
@@ -186,6 +211,10 @@ export class Mongo extends Db {
             if (!query._deleted)
                 query._deleted = { $exists: false };
         }
+        if (this.softarchive && !opts.returnArchived) {
+            if (!query._archived)
+                query._archived = { $exists: false };
+        }
         fjLog.debug('count called', collection, query, opts);
         let ret = await this.executeTransactionally(collection, async (conn) => {
             return await conn.countDocuments(query, opts);
@@ -200,6 +229,10 @@ export class Mongo extends Db {
             if (!query._deleted)
                 query._deleted = { $exists: false };
         }
+        if (this.softarchive && !opts.returnArchived) {
+            if (!query._archived)
+                query._archived = { $exists: false };
+        }
         fjLog.debug('find called', collection, query, opts);
         let ret = await this.executeTransactionally(collection, async (conn) => {
             let r = this._applyQueryOpts(conn.find(query, this._buildFindOptions(opts)), opts);
@@ -224,6 +257,10 @@ export class Mongo extends Db {
             if (!query._deleted)
                 query._deleted = { $exists: false };
         }
+        if (this.softarchive && !opts.returnArchived) {
+            if (!query._archived)
+                query._archived = { $exists: false };
+        }
         fjLog.debug('findNewer called', collection, timestamp, query, opts);
         let ret = await this.executeTransactionally(collection, async (conn) => {
             let r = this._applyQueryOpts(conn.find(query, this._buildFindOptions(opts)).sort({ _ts: 1 }), opts);
@@ -232,18 +269,22 @@ export class Mongo extends Db {
         fjLog.debug('findNewer returns', ret);
         return ret;
     }
-    async findNewerMany(spec = []) {
+    async findNewerMany(spec = [], defaultOpts = {}) {
         var _a;
         fjLog.debug('findNewerMany called', spec);
         const dbName = this.db; // Capture db at operation start
         let conn = await this.connect();
         const getOneColl = async (coll) => {
             let query = this._createQueryForNewer(coll.timestamp, coll.query || {});
-            const opts = coll.opts || {};
+            const opts = { ...defaultOpts, ...coll.opts };
             if (this.softdelete && !opts.returnDeleted) {
                 if (!query._deleted)
                     query._deleted = { $exists: false };
             }
+            if (this.softarchive && !opts.returnArchived) {
+                if (!query._archived)
+                    query._archived = { $exists: false };
+            }
             if (process.env.MONGO_DEBUG_FINDNEWERMANY) {
                 fjLog.debug("findNewerMany <-", coll.collection, coll.timestamp, coll.query, " -> ", JSON.stringify(query));
             }
@@ -393,6 +434,10 @@ export class Mongo extends Db {
             if (!query._deleted)
                 query._deleted = { $exists: false };
         }
+        if (this.softarchive && !opts.returnArchived) {
+            if (!query._archived)
+                query._archived = { $exists: false };
+        }
         // if (!query._blocked) query._blocked = { $exists: false };   // intentionally - blocked records are returned
         fjLog.debug('findOne called', collection, query, projection);
         let ret = await this.executeTransactionally(collection, async (conn) => await conn.findOne(query, { ...(projection ? { projection } : {}), ...this._sessionOpt() }), false, { operation: "findOne", collection, query, projection });
@@ -407,30 +452,32 @@ export class Mongo extends Db {
         const dbName = this.db; // Capture db at operation start
         let query = {
             _id: Mongo._toId(id),
-            // _deleted: { $exists: false }
         };
+        if (this.softdelete && !opts.returnDeleted) {
+            query._deleted = { $exists: false };
+        }
+        if (this.softarchive && !opts.returnArchived) {
+            query._archived = { $exists: false };
+        }
         fjLog.debug('findById called', dbName, collection, id, projection);
         fjLog.trace('findById executing with query', collection, query, projection);
         let ret = await this.executeTransactionally(collection, async (conn) => {
             let r = await conn.findOne(query, { ...(projection ? { projection } : {}), ...this._sessionOpt() });
             return r;
         }, false, { operation: "findById", collection, id, projection });
-        if ((ret === null || ret === void 0 ? void 0 : ret._deleted) && this.softdelete && !opts.returnDeleted)
-            ret = null;
         fjLog.debug('findById returns', ret);
         return this._processReturnedObject(ret);
     }
     async findByIdsInManyCollections(request, opts = {}) {
-        const result = {};
         if (!request || request.length === 0)
-            return result;
+            return {};
         const dbName = this.db;
         fjLog.debug('findByIdsInManyCollections called', dbName, request);
-        for (const req of request) {
+        // Process all collections in parallel
+        const fetchOne = async (req) => {
             const { collection, ids, projection } = req;
             if (!collection || !ids || ids.length === 0) {
-                result[collection] = [];
-                continue;
+                return { collection, entities: [] };
             }
             const objectIds = ids.map(id => Mongo._toId(id));
             const entities = await this.executeTransactionally(collection, async (conn) => {
@@ -438,10 +485,18 @@ export class Mongo extends Db {
                 if (this.softdelete && !opts.returnDeleted) {
                     query._deleted = { $exists: false };
                 }
+                if (this.softarchive && !opts.returnArchived) {
+                    query._archived = { $exists: false };
+                }
                 let r = conn.find(query, { ...(projection ? { projection } : {}), ...this._sessionOpt() });
                 return await r.toArray();
             }, false, { operation: "findByIdsInManyCollections", collection, ids, projection });
-            result[collection] = this._processReturnedObject(entities);
+            return { collection, entities: this._processReturnedObject(entities) };
+        };
+        const results = await Promise.all(request.map(fetchOne));
+        const result = {};
+        for (const { collection, entities } of results) {
+            result[collection] = entities;
         }
         fjLog.debug('findByIdsInManyCollections returns', result);
         return result;
@@ -459,6 +514,9 @@ export class Mongo extends Db {
             if (this.softdelete && !opts.returnDeleted) {
                 query._deleted = { $exists: false };
             }
+            if (this.softarchive && !opts.returnArchived) {
+                query._archived = { $exists: false };
+            }
             let r = conn.find(query, { ...(projection ? { projection } : {}), ...this._sessionOpt() });
             return await r.toArray();
         }, false, { operation: "findByIds", collection, ids, projection });
@@ -477,7 +535,9 @@ export class Mongo extends Db {
             returnDocument: "after",
             ...this._sessionOpt()
         };
-        update = await this._processUpdateObject(update);
+        if (this._hasHashedKeys(update))
+            await this._processHashedKeys(update);
+        update = this._processUpdateObject(update);
         fjLog.debug('updateOne called', collection, query, update);
         let seqKeys = this._findSequenceKeys(update.$set);
         let obj = await this.executeTransactionally(collection, async (conn, client) => {
@@ -507,7 +567,9 @@ export class Mongo extends Db {
             ...this._sessionOpt()
         };
         let _id = Mongo.toId(id || update._id) || Mongo.newid();
-        update = await this._processUpdateObject(update);
+        if (this._hasHashedKeys(update))
+            await this._processHashedKeys(update);
+        update = this._processUpdateObject(update);
         fjLog.debug('save called', collection, id, update);
         let seqKeys = this._findSequenceKeys(update.$set);
         let obj = await this.executeTransactionally(collection, async (conn, client) => {
@@ -541,7 +603,9 @@ export class Mongo extends Db {
             returnDocument: "after",
             ...this._sessionOpt()
         };
-        update = await this._processUpdateObject(update);
+        if (this._hasHashedKeys(update))
+            await this._processHashedKeys(update);
+        update = this._processUpdateObject(update);
         fjLog.debug('update called', collection, query, update);
         let seqKeys = this._findSequenceKeys(update.$set);
         let obj = await this.executeTransactionally(collection, async (conn, client) => {
@@ -578,7 +642,9 @@ export class Mongo extends Db {
             ...this._sessionOpt()
         };
         fjLog.debug('upsert called', collection, query, update);
-        update = await this._processUpdateObject(update);
+        if (this._hasHashedKeys(update))
+            await this._processHashedKeys(update);
+        update = this._processUpdateObject(update);
         let seqKeys = this._findSequenceKeys(update.$set);
         fjLog.debug('upsert processed', collection, query, update);
         if (Object.keys(query).length === 0)
@@ -612,6 +678,7 @@ export class Mongo extends Db {
         fjLog.debug('insert called', collection, insert);
         const dbName = this.db; // Capture db at operation start
         insert = this.replaceIds(insert);
+        this._stripDoubleUnderscoreKeys(insert);
         if (this.revisions) {
             insert._rev = 1;
             insert._ts = Base.timestamp();
@@ -660,13 +727,22 @@ export class Mongo extends Db {
                 };
                 const batchData = [];
                 const errors = [];
+                // Handle sequences in updates before processing
+                if (updates === null || updates === void 0 ? void 0 : updates.length) {
+                    const updatesWithSeq = updates.filter(u => this._hasSequenceFields(u.update));
+                    if (updatesWithSeq.length > 0) {
+                        await this._processSequenceFieldForMany(conn, dbName, collection, updatesWithSeq.map(u => u.update));
+                    }
+                }
                 // Process updates (with upsert) in parallel
                 if (updates === null || updates === void 0 ? void 0 : updates.length) {
                     const updatePromises = updates.map(async ({ _id, update }) => {
                         var _a;
                         try {
                             const objectId = Mongo._toId(_id);
-                            const processedUpdate = await this._processUpdateObject({ ...update });
+                            if (this._hasHashedKeys(update))
+                                await this._processHashedKeys(update);
+                            const processedUpdate = this._processUpdateObject({ ...update });
                             const opts = {
                                 upsert: true,
                                 returnDocument: "after",
@@ -821,48 +897,56 @@ export class Mongo extends Db {
         batch = this.replaceIds(batch);
         await Promise.all(batch.map(item => this._processHashedKeys(item === null || item === void 0 ? void 0 : item.update)));
         let ret = await this.executeTransactionally(collection, async (conn, client) => {
-            var _a;
-            await this._processSequenceFieldForMany(client, dbName, collection, batch.map(b => b.update));
-            let batchData = [];
-            let changes = [];
-            for await (let part of batch) {
-                let { query, update, opts } = part;
-                let options = {
+            // Only process sequences if any batch item has SEQ_NEXT or SEQ_LAST
+            const updatesWithSeq = batch.filter(b => this._hasSequenceFields(b.update));
+            if (updatesWithSeq.length > 0) {
+                await this._processSequenceFieldForMany(client, dbName, collection, updatesWithSeq.map(b => b.update));
+            }
+            // Process all updates in parallel
+            const upsertPromises = batch.map(async (part) => {
+                var _a, _b;
+                const { query, update, opts } = part;
+                const options = {
                     ...(opts || {}),
                     upsert: true,
                     returnDocument: "after",
                     includeResultMetadata: true,
                     ...this._sessionOpt()
                 };
-                update = await this._processUpdateObject(update);
-                let result = await conn.findOneAndUpdate(query, update, options);
-                if (!result)
-                    continue;
-                let ret = result.value;
-                if (ret === null || ret === void 0 ? void 0 : ret._id) {
-                    // Determine operation based on result fields
-                    let oper;
-                    if (ret._deleted !== undefined) {
-                        oper = "delete";
-                    }
-                    else if ((_a = result.lastErrorObject) === null || _a === void 0 ? void 0 : _a.updatedExisting) {
-                        oper = "update";
-                    }
-                    else {
-                        oper = "insert";
-                    }
-                    let retObj = oper === "insert" ? ret : this._removeUnchanged(ret, update, !!(opts === null || opts === void 0 ? void 0 : opts.returnFullObject));
-                    this._processReturnedObject(retObj);
-                    batchData.push({
-                        operation: oper,
-                        data: retObj
-                    });
-                    changes.push(retObj);
+                if (this._hasHashedKeys(update))
+                    await this._processHashedKeys(update);
+                const processedUpdate = this._processUpdateObject({ ...update });
+                const result = await conn.findOneAndUpdate(query, processedUpdate, options);
+                if (!((_a = result === null || result === void 0 ? void 0 : result.value) === null || _a === void 0 ? void 0 : _a._id))
+                    return null;
+                const ret = result.value;
+                // Determine operation based on result fields
+                let oper;
+                if (ret._deleted !== undefined) {
+                    oper = "delete";
+                }
+                else if ((_b = result.lastErrorObject) === null || _b === void 0 ? void 0 : _b.updatedExisting) {
+                    oper = "update";
+                }
+                else {
+                    oper = "insert";
+                }
+                const retObj = oper === "insert" ? ret : this._removeUnchanged(ret, processedUpdate, !!(opts === null || opts === void 0 ? void 0 : opts.returnFullObject));
+                this._processReturnedObject(retObj);
+                return { operation: oper, data: retObj };
+            });
+            const results = await Promise.all(upsertPromises);
+            // Filter out nulls and separate batchData and changes
+            const batchData = [];
+            const changes = [];
+            for (const result of results) {
+                if (result) {
+                    batchData.push(result);
+                    changes.push(result.data);
                 }
-                ;
             }
             if (this.emittingPublishEvents || this.auditing) {
-                await this.emit("publish", {
+                this.emit("publish", {
                     channel: `db/${dbName}/${collection}`,
                     payload: {
                         operation: "batch",
@@ -883,7 +967,7 @@ export class Mongo extends Db {
                         _ts: item.data._ts,
                     }))
                 };
-                await this.emit("publishRev", {
+                this.emit("publishRev", {
                     channel: `dbrev/${dbName}/${collection}`,
                     payload,
                 });
@@ -900,11 +984,16 @@ export class Mongo extends Db {
         fjLog.debug('insertMany called', collection, insert);
         const dbName = this.db; // Capture db at operation start
         insert = this.replaceIds(insert);
+        insert.forEach(item => this._stripDoubleUnderscoreKeys(item));
         await Promise.all(insert.map(item => this._processHashedKeys(item)));
         if (this.revisions)
             insert.forEach(ins => { ins._rev = 1; ins._ts = Base.timestamp(); });
         let ret = await this.executeTransactionally(collection, async (conn, client) => {
-            await this._processSequenceFieldForMany(client, dbName, collection, insert);
+            // Only process sequences if any insert has SEQ_NEXT or SEQ_LAST
+            const insertsWithSeq = insert.filter(item => this._hasSequenceFields(item));
+            if (insertsWithSeq.length > 0) {
+                await this._processSequenceFieldForMany(client, dbName, collection, insert);
+            }
             let obj = await conn.insertMany(insert, this._sessionOpt());
             let ret = [];
             for (let ns of Object.keys(obj.insertedIds)) {
@@ -1019,6 +1108,158 @@ export class Mongo extends Db {
         fjLog.debug('unblockOne returns', ret);
         return ret;
     }
+    /**
+     * Archives a single document by setting `_archived` to the current date.
+     * Increments `_rev` and updates `_ts`.
+     */
+    async archiveOne(collection, query) {
+        let opts = {
+            upsert: false,
+            returnDocument: "after",
+            ...this._sessionOpt()
+        };
+        const dbName = this.db;
+        query = this.replaceIds(query);
+        fjLog.debug('archiveOne called', collection, query);
+        let ret = await this.executeTransactionally(collection, async (conn) => {
+            query._archived = { $exists: false };
+            let update = {
+                $set: { _archived: new Date(), },
+                $inc: { _rev: 1, },
+                $currentDate: { _ts: { $type: 'timestamp' } }
+            };
+            let obj = await conn.findOneAndUpdate(query, update, opts);
+            if (!obj || !(obj === null || obj === void 0 ? void 0 : obj._id))
+                return {
+                    ok: false
+                };
+            await this._publishAndAudit('update', dbName, collection, obj);
+            return obj;
+        }, false, { operation: "archiveOne", collection, query });
+        fjLog.debug('archiveOne returns', ret);
+        return ret;
+    }
+    /**
+     * Unarchives a single document by removing the `_archived` field.
+     * Increments `_rev` and updates `_ts`.
+     */
+    async unarchiveOne(collection, query) {
+        let opts = {
+            upsert: false,
+            returnDocument: "after",
+            ...this._sessionOpt()
+        };
+        const dbName = this.db;
+        query = this.replaceIds(query);
+        fjLog.debug('unarchiveOne called', collection, query);
+        let ret = await this.executeTransactionally(collection, async (conn) => {
+            query._archived = { $exists: true };
+            let update = {
+                $unset: { _archived: 1 },
+                $inc: { _rev: 1, },
+                $currentDate: { _ts: { $type: 'timestamp' } }
+            };
+            let obj = await conn.findOneAndUpdate(query, update, opts);
+            if (!obj || !(obj === null || obj === void 0 ? void 0 : obj._id))
+                return {
+                    ok: false
+                };
+            await this._publishAndAudit('update', dbName, collection, obj);
+            return obj;
+        }, false, { operation: "unarchiveOne", collection, query });
+        fjLog.debug('unarchiveOne returns', ret);
+        return ret;
+    }
+    /**
+     * Archives a single document found by its `_id`.
+     */
+    async archiveOneById(collection, id) {
+        assert(collection);
+        assert(id);
+        return await this.archiveOne(collection, { _id: Mongo._toId(id) });
+    }
+    /**
+     * Archives multiple documents found by their `_id`s.
+     * Returns the array of archived documents.
+     */
+    async archiveManyByIds(collection, ids) {
+        assert(collection);
+        assert(ids);
+        if (!ids || ids.length === 0)
+            return [];
+        const opts = {
+            upsert: false,
+            returnDocument: "after",
+            ...this._sessionOpt()
+        };
+        const dbName = this.db;
+        fjLog.debug('archiveManyByIds called', collection, ids);
+        let ret = [];
+        for (const id of ids) {
+            let r = await this.executeTransactionally(collection, async (conn) => {
+                let query = { _id: Mongo._toId(id), _archived: { $exists: false } };
+                let update = {
+                    $set: { _archived: new Date() },
+                    $inc: { _rev: 1 },
+                    $currentDate: { _ts: { $type: 'timestamp' } }
+                };
+                let obj = await conn.findOneAndUpdate(query, update, opts);
+                if (obj === null || obj === void 0 ? void 0 : obj._id) {
+                    await this._publishAndAudit('update', dbName, collection, obj);
+                }
+                return obj;
+            }, false, { operation: "archiveManyByIds", collection, id });
+            if (r === null || r === void 0 ? void 0 : r._id)
+                ret.push(r);
+        }
+        fjLog.debug('archiveManyByIds returns', ret);
+        return ret;
+    }
+    /**
+     * Unarchives a single document found by its `_id`.
+     */
+    async unarchiveOneById(collection, id) {
+        assert(collection);
+        assert(id);
+        return await this.unarchiveOne(collection, { _id: Mongo._toId(id) });
+    }
+    /**
+     * Unarchives multiple documents found by their `_id`s.
+     * Returns the array of unarchived documents.
+     */
+    async unarchiveManyByIds(collection, ids) {
+        assert(collection);
+        assert(ids);
+        if (!ids || ids.length === 0)
+            return [];
+        const opts = {
+            upsert: false,
+            returnDocument: "after",
+            ...this._sessionOpt()
+        };
+        const dbName = this.db;
+        fjLog.debug('unarchiveManyByIds called', collection, ids);
+        let ret = [];
+        for (const id of ids) {
+            let r = await this.executeTransactionally(collection, async (conn) => {
+                let query = { _id: Mongo._toId(id), _archived: { $exists: true } };
+                let update = {
+                    $unset: { _archived: 1 },
+                    $inc: { _rev: 1 },
+                    $currentDate: { _ts: { $type: 'timestamp' } }
+                };
+                let obj = await conn.findOneAndUpdate(query, update, opts);
+                if (obj === null || obj === void 0 ? void 0 : obj._id) {
+                    await this._publishAndAudit('update', dbName, collection, obj);
+                }
+                return obj;
+            }, false, { operation: "unarchiveManyByIds", collection, id });
+            if (r === null || r === void 0 ? void 0 : r._id)
+                ret.push(r);
+        }
+        fjLog.debug('unarchiveManyByIds returns', ret);
+        return ret;
+    }
     async hardDeleteOne(collection, query) {
         assert(collection);
         assert(query);
@@ -1175,35 +1416,14 @@ export class Mongo extends Db {
         fjLog.debug('isUnique returns', ret);
         return ret;
     }
-    async collectFieldValues(collection, field, inArray = false, opts) {
-        assert(collection);
-        assert(field);
-        fjLog.debug('collectFieldValues called', collection, field);
-        let pipeline = [
-            { $group: { _id: '$' + field } },
-            { $sort: { _id: 1 } }
-        ];
-        if (inArray)
-            pipeline = [
-                { $unwind: '$' + field },
-                ...pipeline
-            ];
-        let res = await this.executeTransactionally(collection, async (conn) => {
-            let agg = await conn.aggregate(pipeline, opts);
-            let res = await agg.toArray();
-            return res;
-        }, false, { operation: "collectFieldValues", collection, field, inArray, pipeline, opts });
-        let ret = res === null || res === void 0 ? void 0 : res.map((v) => v._id);
-        fjLog.debug('collectFieldValues returns', ret);
-        return ret;
-    }
     async dropCollection(collection) {
         assert(collection);
         fjLog.debug('dropCollection called', this.auditCollections);
         const dbName = this.db; // Capture db at operation start
         let client = await this.connect();
         let existing = await client.db(dbName).collections();
-        if (existing.map((c) => c.collectionName).includes(collection)) {
+        const existingNames = new Set(existing.map((c) => c.collectionName));
+        if (existingNames.has(collection)) {
             await client.db(dbName).dropCollection(collection);
         }
         fjLog.debug('dropCollection returns');
@@ -1225,8 +1445,9 @@ export class Mongo extends Db {
         const dbName = this.db; // Capture db at operation start
         let client = await this.connect();
         let existing = await client.db(dbName).collections();
-        for await (let collection of collections) {
-            if (existing.map((c) => c.collectionName).includes(collection)) {
+        const existingNames = new Set(existing.map((c) => c.collectionName));
+        for (const collection of collections) {
+            if (existingNames.has(collection)) {
                 await client.db(dbName).dropCollection(collection);
             }
         }
@@ -1239,8 +1460,9 @@ export class Mongo extends Db {
         const dbName = this.db; // Capture db at operation start
         let client = await this.connect();
         let existing = await this.getCollections();
-        for await (let collection of collections) {
-            if (!existing.includes(collection)) {
+        const existingSet = new Set(existing);
+        for (const collection of collections) {
+            if (!existingSet.has(collection)) {
                 await client.db(dbName).createCollection(collection);
             }
         }
@@ -1388,6 +1610,9 @@ export class Mongo extends Db {
         }
         // 4. Clean up state
         this.session = undefined;
+        this.auditedCollections.length = 0;
+        this.auditedCollectionsLower.length = 0;
+        this._sequencesIndexEnsured = false;
     }
     async inTransaction() {
         if (!await this.isOnReplicaSet())
@@ -1575,6 +1800,139 @@ export class Mongo extends Db {
             return undefined;
         return parseInt((_a = maxfld === null || maxfld === void 0 ? void 0 : maxfld[0]) === null || _a === void 0 ? void 0 : _a[key]) || 0;
     }
+    /**
+     * Ensures the unique index on _sequences collection exists.
+     * Called lazily on first sequence operation.
+     */
+    async _ensureSequencesIndex(dbConnection) {
+        if (this._sequencesIndexEnsured)
+            return;
+        try {
+            await dbConnection
+                .collection(FIELD_SEQUENCES_COLLECTION)
+                .createIndex({ collection: 1, field: 1 }, { unique: true });
+            this._sequencesIndexEnsured = true;
+        }
+        catch (err) {
+            // Index may already exist, which is fine
+            if (err.code === 85 || err.code === 86) {
+                this._sequencesIndexEnsured = true;
+            }
+            else {
+                throw err;
+            }
+        }
+    }
+    /**
+     * Atomically gets the next sequence value for a field in a collection.
+     * On first use, auto-seeds from existing data in the collection.
+     */
+    async _getNextSequenceValue(dbConnection, collection, field) {
+        // Check current max in the actual collection
+        const existingMax = await this._findLastSequenceForKey(dbConnection.collection(collection), field);
+        // Check if we have a sequence document
+        const existingSeq = await dbConnection
+            .collection(FIELD_SEQUENCES_COLLECTION)
+            .findOne({ collection, field }, this._sessionOpt());
+        if (!existingSeq) {
+            // First time for this collection/field - ensure index exists
+            await this._ensureSequencesIndex(dbConnection);
+            const seedValue = (existingMax !== undefined && existingMax >= 1) ? existingMax : 0;
+            // Use $max to handle concurrent seeding - only sets if greater than current value
+            // This ensures the highest seed value wins in case of concurrent operations
+            await dbConnection
+                .collection(FIELD_SEQUENCES_COLLECTION)
+                .findOneAndUpdate({ collection, field }, {
+                $max: { lastSeqNum: seedValue },
+                $setOnInsert: { collection, field }
+            }, { upsert: true, returnDocument: 'after', ...this._sessionOpt() });
+            // Now increment atomically to get next value
+            const updated = await dbConnection
+                .collection(FIELD_SEQUENCES_COLLECTION)
+                .findOneAndUpdate({ collection, field }, { $inc: { lastSeqNum: 1 } }, { returnDocument: 'after', ...this._sessionOpt() });
+            return updated.lastSeqNum;
+        }
+        // Sequence exists - check if collection was cleared (reset scenario)
+        if (existingMax === undefined && existingSeq.lastSeqNum > 0) {
+            // Collection is empty but we have a non-zero sequence - reset to 0 then increment to 1
+            await dbConnection
+                .collection(FIELD_SEQUENCES_COLLECTION)
+                .updateOne({ collection, field }, { $set: { lastSeqNum: 0 } }, this._sessionOpt());
+        }
+        // Increment and return
+        const result = await dbConnection
+            .collection(FIELD_SEQUENCES_COLLECTION)
+            .findOneAndUpdate({ collection, field }, { $inc: { lastSeqNum: 1 } }, { returnDocument: 'after', ...this._sessionOpt() });
+        return result.lastSeqNum;
+    }
+    /**
+     * Gets the current (last) sequence value without incrementing.
+     * On first use, auto-seeds from existing data in the collection.
+     * If collection becomes empty, resets the sequence to 0.
+     */
+    async _getLastSequenceValue(dbConnection, collection, field) {
+        // Check current max in the actual collection
+        const existingMax = await this._findLastSequenceForKey(dbConnection.collection(collection), field);
+        // Try to get existing sequence value
+        const existing = await dbConnection
+            .collection(FIELD_SEQUENCES_COLLECTION)
+            .findOne({ collection, field }, this._sessionOpt());
+        if (existing) {
+            // If collection is empty (existingMax undefined), reset sequence to 0
+            if (existingMax === undefined && existing.lastSeqNum > 0) {
+                await dbConnection
+                    .collection(FIELD_SEQUENCES_COLLECTION)
+                    .updateOne({ collection, field }, { $set: { lastSeqNum: 0 } }, this._sessionOpt());
+                return 0;
+            }
+            return existing.lastSeqNum;
+        }
+        if (existingMax !== undefined && existingMax >= 0) {
+            // First time with existing data - ensure index exists and seed
+            await this._ensureSequencesIndex(dbConnection);
+            await dbConnection
+                .collection(FIELD_SEQUENCES_COLLECTION)
+                .updateOne({ collection, field }, { $setOnInsert: { lastSeqNum: existingMax } }, { upsert: true, ...this._sessionOpt() });
+            return existingMax;
+        }
+        return 0;
+    }
+    /**
+     * Atomically reserves a range of sequence numbers for batch operations.
+     * Returns { start, end } where start is the first reserved value and end is the last.
+     */
+    async _reserveSequenceRange(dbConnection, collection, field, count) {
+        // Check current max in the actual collection
+        const existingMax = await this._findLastSequenceForKey(dbConnection.collection(collection), field);
+        // Check if we have a sequence document
+        const existingSeq = await dbConnection
+            .collection(FIELD_SEQUENCES_COLLECTION)
+            .findOne({ collection, field }, this._sessionOpt());
+        if (!existingSeq) {
+            // First time for this collection/field - ensure index exists
+            await this._ensureSequencesIndex(dbConnection);
+            const seedValue = (existingMax !== undefined && existingMax >= 1) ? existingMax : 0;
+            // Use $max to handle concurrent seeding
+            await dbConnection
+                .collection(FIELD_SEQUENCES_COLLECTION)
+                .findOneAndUpdate({ collection, field }, {
+                $max: { lastSeqNum: seedValue },
+                $setOnInsert: { collection, field }
+            }, { upsert: true, returnDocument: 'after', ...this._sessionOpt() });
+        }
+        else if (existingMax === undefined && existingSeq.lastSeqNum > 0) {
+            // Collection is empty but we have a non-zero sequence - reset to 0
+            await dbConnection
+                .collection(FIELD_SEQUENCES_COLLECTION)
+                .updateOne({ collection, field }, { $set: { lastSeqNum: 0 } }, this._sessionOpt());
+        }
+        // Now atomically reserve the range
+        const result = await dbConnection
+            .collection(FIELD_SEQUENCES_COLLECTION)
+            .findOneAndUpdate({ collection, field }, { $inc: { lastSeqNum: count } }, { returnDocument: 'after', ...this._sessionOpt() });
+        const end = result.lastSeqNum;
+        return { start: end - count + 1, end };
+    }
     // private async _getNextCollectionUpdateSeqNo(collection: string, conn: MongoClient) {
     //     let opts : FindOneAndUpdateOptions = {
     //         upsert: true,
@@ -1594,22 +1952,24 @@ export class Mongo extends Db {
     _findSequenceKeys(object) {
         if (!object)
             return;
-        let seqKeys = Object.keys(object).filter(key => object[key] === 'SEQ_NEXT' || object[key] === 'SEQ_LAST');
-        return ((seqKeys === null || seqKeys === void 0 ? void 0 : seqKeys.length) > 0 || this.syncSupport) ? { seqKeys } : undefined;
+        if (!this._hasSequenceFields(object) && !this.syncSupport)
+            return;
+        const seqKeys = Object.keys(object).filter(key => object[key] === 'SEQ_NEXT' || object[key] === 'SEQ_LAST');
+        return { seqKeys };
     }
     async _processSequenceField(client, dbName, collection, insert, seqKeys) {
         assert(this.client);
         // if (this.syncSupport) {
         //     insert._csq = (await this._getNextCollectionUpdateSeqNo(collection, client));
         // }
-        for await (let seqKey of (seqKeys === null || seqKeys === void 0 ? void 0 : seqKeys.seqKeys) || []) {
-            let last = await this._findLastSequenceForKey(client.db(dbName).collection(collection), seqKey);
-            if (last === undefined) {
-                await this.createCollection(collection);
-                last = 0;
+        const db = client.db(dbName);
+        for (const seqKey of (seqKeys === null || seqKeys === void 0 ? void 0 : seqKeys.seqKeys) || []) {
+            if (insert[seqKey] === 'SEQ_LAST') {
+                insert[seqKey] = await this._getLastSequenceValue(db, collection, seqKey);
+            }
+            else {
+                insert[seqKey] = await this._getNextSequenceValue(db, collection, seqKey);
             }
-            let next = insert[seqKey] === 'SEQ_LAST' ? last : last + 1;
-            insert[seqKey] = next;
         }
         return insert;
     }
@@ -1618,35 +1978,45 @@ export class Mongo extends Db {
         assert(connection);
         if (!(inserts === null || inserts === void 0 ? void 0 : inserts.length))
             return;
+        // Collect all unique sequence keys across all inserts
         let seqKeysSet = new Set();
         for (let insert of inserts) {
             let spec = this._findSequenceKeys(insert);
             spec === null || spec === void 0 ? void 0 : spec.seqKeys.forEach(key => seqKeysSet.add(key));
         }
         let seqKeys = Array.from(seqKeysSet);
-        // let seq: number = 0;
-        // if (this.syncSupport) seq = await this._getNextCollectionUpdateSeqNo(collection, connection);
         if (!seqKeys.length)
             return inserts;
-        // Fetch all sequence last values in parallel
-        const coll = connection.db(dbName).collection(collection);
-        const lastValues = await Promise.all(seqKeys.map(seqKey => this._findLastSequenceForKey(coll, seqKey)));
-        // Check if collection needs to be created (any undefined value)
-        if (lastValues.some(v => v === undefined)) {
-            try {
-                await this.createCollection(collection);
+        const db = connection.db(dbName);
+        // Process each sequence key
+        for (const seqKey of seqKeys) {
+            // Count SEQ_NEXT occurrences to reserve the range
+            let seqNextCount = 0;
+            for (const insert of inserts) {
+                if (insert[seqKey] === 'SEQ_NEXT')
+                    seqNextCount++;
+            }
+            // Get current value and reserve range for SEQ_NEXT if needed
+            let currentValue;
+            if (seqNextCount > 0) {
+                const range = await this._reserveSequenceRange(db, collection, seqKey, seqNextCount);
+                // currentValue starts at one before the range start (since SEQ_LAST gets current, SEQ_NEXT gets next)
+                currentValue = range.start - 1;
             }
-            catch { /* collection may already exist */ }
-        }
-        // Build a map of seqKey -> last value
-        const seqKeyToLast = new Map(seqKeys.map((key, idx) => { var _a; return [key, (_a = lastValues[idx]) !== null && _a !== void 0 ? _a : 0]; }));
-        // Process inserts for all sequence keys
-        for (const insert of inserts) {
-            for (const seqKey of seqKeys) {
-                const last = seqKeyToLast.get(seqKey);
-                const next = insert[seqKey] === 'SEQ_LAST' ? last : last + 1;
-                insert[seqKey] = next;
-                seqKeyToLast.set(seqKey, next);
+            else {
+                // Only SEQ_LAST values, just get the current value
+                currentValue = await this._getLastSequenceValue(db, collection, seqKey);
+            }
+            // Process inserts in order - SEQ_LAST gets current, SEQ_NEXT increments and gets new value
+            for (const insert of inserts) {
+                const val = insert[seqKey];
+                if (val === 'SEQ_LAST') {
+                    insert[seqKey] = currentValue;
+                }
+                else if (val === 'SEQ_NEXT') {
+                    currentValue++;
+                    insert[seqKey] = currentValue;
+                }
             }
         }
         return inserts;
@@ -1677,11 +2047,11 @@ export class Mongo extends Db {
             a._id = wholerecord._id;
         return a;
     }
-    _shouldAuditCollection(db, col, audited = this.auditedCollections) {
+    _shouldAuditCollection(db, col) {
         if (!this.auditing)
             return false;
         const fullName = ((db ? db + "." : "") + (col || "")).toLowerCase();
-        return audited.some(m => fullName.includes(m.toLowerCase()));
+        return this.auditedCollectionsLower.some(m => fullName.includes(m));
     }
     async _publishAndAudit(operation, db, collection, dataToPublish, noEmit) {
         if (!dataToPublish._id && !["deleteMany", "updateMany"].includes(operation))
@@ -1875,8 +2245,13 @@ export class Mongo extends Db {
             r = r.collation(opts.collation);
         return r;
     }
-    async _processUpdateObject(update) {
-        await this._processHashedKeys(update);
+    _hasHashedKeys(obj) {
+        return Object.keys(obj).some(startsWithHashedPrefix);
+    }
+    _hasSequenceFields(obj) {
+        return Object.values(obj).some(v => v === 'SEQ_NEXT' || v === 'SEQ_LAST');
+    }
+    _processUpdateObject(update) {
         for (let k in update) {
             let key = k;
             const keyStr = String(key);
@@ -1919,16 +2294,27 @@ export class Mongo extends Db {
         }
         return update;
     }
-    async _processHashedKeys(update) {
-        // Pre-filter hashed keys before async processing
-        const hashedKeys = Object.keys(update).filter(startsWithHashedPrefix);
-        // Process all hashed keys in parallel
+    async _processHashedKeys(obj) {
+        const hashedKeys = Object.keys(obj).filter(startsWithHashedPrefix);
         await Promise.all(hashedKeys.map(async (key) => {
             const salt = await bcrypt.genSalt(saltRounds);
-            const hash = await bcrypt.hash(update[key], salt);
-            update[key] = { salt, hash };
+            const hash = await bcrypt.hash(obj[key], salt);
+            obj[key] = { salt, hash };
         }));
     }
+    /**
+     * Removes fields starting with `__` from an object (mutates in place).
+     * Used to sanitize input on insert/update operations, preventing clients
+     * from writing internal/reserved fields directly to the database.
+     * Fields with `__hashed__` prefix are preserved (used for password hashing).
+     */
+    _stripDoubleUnderscoreKeys(obj) {
+        for (const key in obj) {
+            if (startsWithDoubleUnderscore(key) && !startsWithHashedPrefix(key)) {
+                delete obj[key];
+            }
+        }
+    }
     _processReturnedObject(ret) {
         if (!ret)
             return ret;