mongodash 2.0.0 → 2.1.1

package/dist/lib/src/prefixFilterKeys.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.prefixFilterKeys = prefixFilterKeys;
+/**
+ * Recursively prefixes all keys in a MongoDB filter object.
+ * This is used to apply a filter meant for a document to a field
+ * within a larger document, such as 'fullDocument' in a change stream event.
+ *
+ * @param filter The filter object to process.
+ * @param prefix The prefix to add to each key (e.g., 'fullDocument').
+ * @returns A new filter object with all keys prefixed.
+ */
+function prefixFilterKeys(filter, prefix) {
+    const newFilter = {};
+    for (const key in filter) {
+        if (Object.prototype.hasOwnProperty.call(filter, key)) {
+            const value = filter[key];
+            if (key === '$expr') {
+                // Handle $expr specially: we need to prefix field paths inside it
+                newFilter[key] = prefixExpr(value, prefix);
+            }
+            else if (key.startsWith('$')) {
+                // Handle logical operators like $or, $and, $nor
+                if (Array.isArray(value) && (key === '$or' || key === '$and' || key === '$nor')) {
+                    newFilter[key] = value.map((item) => typeof item === 'object' && item !== null && !Array.isArray(item) ? prefixFilterKeys(item, prefix) : item); // eslint-disable-line @typescript-eslint/no-explicit-any
+                }
+                else {
+                    // For other operators ($in, $eq, etc.), keep them as they are.
+                    newFilter[key] = value;
+                }
+            }
+            else {
+                // For regular field names, add the prefix.
+                const newKey = `${prefix}.${key}`;
+                newFilter[newKey] = value;
+            }
+        }
+    }
+    return newFilter;
+}
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function prefixExpr(expr, prefix) {
+    if (Array.isArray(expr)) {
+        return expr.map((item) => prefixExpr(item, prefix));
+    }
+    else if (typeof expr === 'object' && expr !== null) {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const newExpr = {};
+        for (const key in expr) {
+            newExpr[key] = prefixExpr(expr[key], prefix);
+        }
+        return newExpr;
+    }
+    else if (typeof expr === 'string') {
+        // Prefix field paths (starting with $) but not system variables (starting with $$)
+        if (expr.startsWith('$') && !expr.startsWith('$$')) {
+            return `$${prefix}.${expr.substring(1)}`;
+        }
+    }
+    return expr;
+}
+//# sourceMappingURL=prefixFilterKeys.js.map

package/dist/lib/src/prefixFilterKeys.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prefixFilterKeys.js","sourceRoot":"","sources":["../../../src/prefixFilterKeys.ts"],"names":[],"mappings":";;AAWA,4CA6BC;AAtCD;;;;;;;;GAQG;AACH,SAAgB,gBAAgB,CAAqB,MAAiB,EAAE,MAAc;IAClF,MAAM,SAAS,GAAc,EAAE,CAAC;IAEhC,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC;YACpD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAsB,CAAC,CAAC;YAE7C,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;gBAClB,kEAAkE;gBAClE,SAAS,CAAC,GAAsB,CAAC,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAClE,CAAC;iBAAM,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC7B,gDAAgD;gBAChD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,KAAK,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,MAAM,CAAC,EAAE,CAAC;oBAC9E,SAAS,CAAC,GAAsB,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACnD,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAiB,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAClH,CAAC,CAAC,yDAAyD;gBACvE,CAAC;qBAAM,CAAC;oBACJ,+DAA+D;oBAC/D,SAAS,CAAC,GAAsB,CAAC,GAAG,KAAK,CAAC;gBAC9C,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,2CAA2C;gBAC3C,MAAM,MAAM,GAAG,GAAG,MAAM,IAAI,GAAG,EAAE,CAAC;gBAClC,SAAS,CAAC,MAAyB,CAAC,GAAG,KAAK,CAAC;YACjD,CAAC;QACL,CAAC;IACL,CAAC;IAED,OAAO,SAAS,CAAC;AACrB,CAAC;AAED,8DAA8D;AAC9D,SAAS,UAAU,CAAC,IAAS,EAAE,MAAc;IACzC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;SAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QACnD,8DAA8D;QAC9D,MAAM,OAAO,GAAQ,EAAE,CAAC;QACxB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,OAAO,CAAC;IACnB,CAAC;SAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAClC,mFAAmF;QACnF,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;YACjD,OAAO,IAAI,MAAM,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7C,CAAC;IACL,CAAC;IACD,OAAO,IAAI,CAAC;AAChB,CAAC"}

package/dist/lib/src/processInBatches.js

@@ -0,0 +1,39 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processInBatches = processInBatches;
+async function processInBatches(collection, queryOrPipeline, transform, executeBatch, options = {}) {
+    const { batchSize = 1000, shouldStop } = options;
+    const cursor = Array.isArray(queryOrPipeline) ? collection.aggregate(queryOrPipeline) : collection.find(queryOrPipeline);
+    let batch = [];
+    let processedDocuments = 0;
+    let operationsPerformed = 0;
+    while (await cursor.hasNext()) {
+        if (shouldStop && shouldStop()) {
+            break;
+        }
+        const doc = (await cursor.next());
+        if (!doc)
+            continue;
+        processedDocuments++;
+        const result = await transform(doc);
+        if (result !== null && result !== undefined) {
+            if (Array.isArray(result)) {
+                batch.push(...result);
+            }
+            else {
+                batch.push(result);
+            }
+        }
+        if (batch.length >= batchSize) {
+            await executeBatch(batch);
+            operationsPerformed += batch.length;
+            batch = [];
+        }
+    }
+    if (batch.length > 0) {
+        await executeBatch(batch);
+        operationsPerformed += batch.length;
+    }
+    return { processedDocuments, operationsPerformed };
+}
+//# sourceMappingURL=processInBatches.js.map

package/dist/lib/src/processInBatches.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"processInBatches.js","sourceRoot":"","sources":["../../../src/processInBatches.ts"],"names":[],"mappings":";;AAYA,4CA+CC;AA/CM,KAAK,UAAU,gBAAgB,CAClC,UAA4B,EAC5B,eAA0C,EAC1C,SAAkG,EAClG,YAA2C,EAC3C,UAAmC,EAAE;IAErC,MAAM,EAAE,SAAS,GAAG,IAAI,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAEjD,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAO,eAAe,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAO,eAAe,CAAC,CAAC;IAErI,IAAI,KAAK,GAAU,EAAE,CAAC;IACtB,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,IAAI,mBAAmB,GAAG,CAAC,CAAC;IAE5B,OAAO,MAAM,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5B,IAAI,UAAU,IAAI,UAAU,EAAE,EAAE,CAAC;YAC7B,MAAM;QACV,CAAC;QAED,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAS,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,SAAS;QAEnB,kBAAkB,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,CAAC,CAAC;QAEpC,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAC1C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxB,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACJ,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACL,CAAC;QAED,IAAI,KAAK,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;YAC1B,mBAAmB,IAAI,KAAK,CAAC,MAAM,CAAC;YACpC,KAAK,GAAG,EAAE,CAAC;QACf,CAAC;IACL,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACnB,MAAM,YAAY,CAAC,KAAK,CAAC,CAAC;QAC1B,mBAAmB,IAAI,KAAK,CAAC,MAAM,CAAC;IACxC,CAAC;IAED,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,CAAC;AACvD,CAAC"}

package/dist/lib/src/reactiveTasks/LeaderElector.js

@@ -0,0 +1,148 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LeaderElector = void 0;
+const mongodb_1 = require("mongodb");
+const ReactiveTaskTypes_1 = require("./ReactiveTaskTypes");
+const _debug = require("debug");
+const OnInfo_1 = require("../OnInfo");
+const OnError_1 = require("../OnError");
+const debug = _debug('mongodash:reactiveTasks:leader');
+/**
+ * Manages leader election among multiple scheduler instances.
+ *
+ * Responsibilities:
+ * - Attempts to acquire a distributed lock in the globals collection.
+ * - Maintains the lock by periodically renewing it (heartbeat).
+ * - Notifies callbacks when the instance becomes leader or loses leadership.
+ * - Ensures only one instance (the leader) runs the `ReactiveTaskPlanner` at a time.
+ */
+class LeaderElector {
+    constructor(globalsCollection, instanceId, options, callbacks, onInfo = OnInfo_1.defaultOnInfo, onError = OnError_1.defaultOnError) {
+        this.globalsCollection = globalsCollection;
+        this.instanceId = instanceId;
+        this.options = options;
+        this.callbacks = callbacks;
+        this.onInfo = onInfo;
+        this.onError = onError;
+        this.isRunning = false;
+        this._isLeader = false;
+        this.leaderTimer = null;
+        this.metaDocId = ReactiveTaskTypes_1.REACTIVE_TASK_META_DOC_ID;
+        this.metaDocId = options.metaDocId || this.metaDocId;
+    }
+    get isLeader() {
+        return this._isLeader;
+    }
+    async start() {
+        if (this.isRunning)
+            return;
+        this.isRunning = true;
+        await this.runLeaderElectionLoop();
+    }
+    async stop() {
+        if (!this.isRunning)
+            return;
+        this.isRunning = false;
+        if (this.leaderTimer) {
+            clearTimeout(this.leaderTimer);
+            this.leaderTimer = null;
+        }
+        if (this._isLeader) {
+            await this.releaseLock();
+            this._isLeader = false;
+        }
+    }
+    forceLoseLeader() {
+        this._isLeader = false;
+    }
+    async runLeaderElectionLoop() {
+        const loop = async () => {
+            try {
+                await this.tryAcquireLock();
+                if (this._isLeader) {
+                    await this.callbacks.onHeartbeat();
+                }
+            }
+            catch (error) {
+                this.onError(error);
+            }
+            finally {
+                if (this.isRunning) {
+                    this.leaderTimer = setTimeout(loop, this.options.lockHeartbeatMs);
+                }
+            }
+        };
+        await loop();
+    }
+    async tryAcquireLock() {
+        var _a, _b;
+        const now = new Date();
+        const expiresAt = new Date(now.getTime() + this.options.lockTtlMs);
+        try {
+            debug(`[Scheduler ${this.instanceId}] Trying to acquire lock on ${this.metaDocId} in ${this.globalsCollection.collectionName}`);
+            const updatePipeline = [
+                {
+                    $set: {
+                        lock: {
+                            $cond: {
+                                if: {
+                                    $or: [
+                                        { $lt: ['$lock.expiresAt', now] },
+                                        { $eq: ['$lock.expiresAt', null] },
+                                        { $eq: ['$lock', null] },
+                                        { $eq: ['$lock.instanceId', this.instanceId] },
+                                    ],
+                                },
+                                then: { expiresAt, instanceId: this.instanceId },
+                                else: '$lock',
+                            },
+                        },
+                    },
+                },
+            ];
+            const result = (await this.globalsCollection.findOneAndUpdate({ _id: this.metaDocId }, updatePipeline, {
+                upsert: true,
+                returnDocument: 'after',
+                includeResultMetadata: true,
+            }));
+            if (((_b = (_a = result.value) === null || _a === void 0 ? void 0 : _a.lock) === null || _b === void 0 ? void 0 : _b.instanceId) === this.instanceId) {
+                if (!this._isLeader) {
+                    this._isLeader = true;
+                    debug(`[Scheduler ${this.instanceId}] Leader lock acquired.`);
+                    await this.callbacks.onBecomeLeader();
+                }
+            }
+            else {
+                if (this._isLeader) {
+                    this._isLeader = false;
+                    this.onInfo({ message: `Leader lock lost.`, code: ReactiveTaskTypes_1.CODE_REACTIVE_TASK_LEADER_LOCK_LOST });
+                    await this.callbacks.onLoseLeader();
+                }
+            }
+        }
+        catch (error) {
+            const closedErrorAfterStop = !this.isRunning && error instanceof mongodb_1.MongoClientClosedError;
+            if (!closedErrorAfterStop) {
+                this.onError(error);
+            }
+            if (this._isLeader) {
+                this._isLeader = false;
+                await this.callbacks.onLoseLeader();
+            }
+        }
+    }
+    async releaseLock() {
+        try {
+            await this.globalsCollection.updateOne({
+                _id: this.metaDocId,
+                'lock.instanceId': this.instanceId,
+            }, { $unset: { lock: '' } });
+            debug(`[Scheduler ${this.instanceId}] Leader lock released.`);
+        }
+        catch (error) {
+            this.onError(error);
+        }
+    }
+}
+exports.LeaderElector = LeaderElector;
+//# sourceMappingURL=LeaderElector.js.map

package/dist/lib/src/reactiveTasks/LeaderElector.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"LeaderElector.js","sourceRoot":"","sources":["../../../../src/reactiveTasks/LeaderElector.ts"],"names":[],"mappings":";;;AAAA,qCAA+D;AAE/D,2DAAmH;AACnH,gCAAgC;AAChC,sCAAkD;AAClD,wCAAqD;AAErD,MAAM,KAAK,GAAG,MAAM,CAAC,gCAAgC,CAAC,CAAC;AAcvD;;;;;;;;GAQG;AACH,MAAa,aAAa;IAMtB,YACY,iBAAoC,EACpC,UAAkB,EAClB,OAA6B,EAC7B,SAAiC,EACjC,SAAiB,sBAAa,EAC9B,UAAmB,wBAAc;QALjC,sBAAiB,GAAjB,iBAAiB,CAAmB;QACpC,eAAU,GAAV,UAAU,CAAQ;QAClB,YAAO,GAAP,OAAO,CAAsB;QAC7B,cAAS,GAAT,SAAS,CAAwB;QACjC,WAAM,GAAN,MAAM,CAAwB;QAC9B,YAAO,GAAP,OAAO,CAA0B;QAXrC,cAAS,GAAG,KAAK,CAAC;QAClB,cAAS,GAAG,KAAK,CAAC;QAClB,gBAAW,GAA0B,IAAI,CAAC;QAC1C,cAAS,GAAG,6CAAyB,CAAC;QAU1C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;IACzD,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,SAAS,CAAC;IAC1B,CAAC;IAEM,KAAK,CAAC,KAAK;QACd,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;IACvC,CAAC;IAEM,KAAK,CAAC,IAAI;QACb,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QAEvB,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACnB,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC/B,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC5B,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YACzB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QAC3B,CAAC;IACL,CAAC;IAEM,eAAe;QAClB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,qBAAqB;QAC/B,MAAM,IAAI,GAAG,KAAK,IAAI,EAAE;YACpB,IAAI,CAAC;gBACD,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;gBAE5B,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACjB,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC;gBACvC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,CAAC,OAAO,CAAC,KAAc,CAAC,CAAC;YACjC,CAAC;oBAAS,CAAC;gBACP,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACjB,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;gBACtE,CAAC;YACL,CAAC;QACL,CAAC,CAAC;QAEF,MAAM,IAAI,EAAE,CAAC;IACjB,CAAC;IAEO,KAAK,CAAC,cAAc;;QACxB,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEnE,IAAI,CAAC;YACD,KAAK,CAAC,cAAc,IAAI,CAAC,UAAU,+BAA+B,IAAI,CAAC,SAAS,OAAO,IAAI,CAAC,iBAAiB,CAAC,cAAc,EAAE,CAAC,CAAC;YAEhI,MAAM,cAAc,GAAG;gBACnB;oBACI,IAAI,EAAE;wBACF,IAAI,EAAE;4BACF,KAAK,EAAE;gCACH,EAAE,EAAE;oCACA,GAAG,EAAE;wCACD,EAAE,GAAG,EAAE,CAAC,iBAAiB,EAAE,GAAG,CAAC,EAAE;wCACjC,EAAE,GAAG,EAAE,CAAC,iBAAiB,EAAE,IAAI,CAAC,EAAE;wCAClC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE;wCACxB,EAAE,GAAG,EAAE,CAAC,kBAAkB,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE;qCACjD;iCACJ;gCACD,IAAI,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE;gCAChD,IAAI,EAAE,OAAO;6BAChB;yBACJ;qBACJ;iBACJ;aACJ,CAAC;YAEF,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,EAAE,EAAE,cAAc,EAAE;gBACnG,MAAM,EAAE,IAAI;gBACZ,cAAc,EAAE,OAAO;gBACvB,qBAAqB,EAAE,IAAI;aAC9B,CAAC,CAA0C,CAAC;YAE7C,IAAI,CAAA,MAAA,MAAA,MAAM,CAAC,KAAK,0CAAE,IAAI,0CAAE,UAAU,MAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBACrD,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;oBAClB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;oBACtB,KAAK,CAAC,cAAc,IAAI,CAAC,UAAU,yBAAyB,CAAC,CAAC;oBAC9D,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,CAAC;gBAC1C,CAAC;YACL,CAAC;iBAAM,CAAC;gBACJ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACjB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;oBACvB,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,mBAAmB,EAAE,IAAI,EAAE,uDAAmC,EAAE,CAAC,CAAC;oBACzF,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;gBACxC,CAAC;YACL,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,MAAM,oBAAoB,GAAG,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,YAAY,gCAAsB,CAAC;YACxF,IAAI,CAAC,oBAAoB,EAAE,CAAC;gBACxB,IAAI,CAAC,OAAO,CAAC,KAAc,CAAC,CAAC;YACjC,CAAC;YAED,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBACvB,MAAM,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,CAAC;YACxC,CAAC;QACL,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,WAAW;QACrB,IAAI,CAAC;YACD,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAClC;gBACI,GAAG,EAAE,IAAI,CAAC,SAAS;gBACnB,iBAAiB,EAAE,IAAI,CAAC,UAAU;aACrC,EACD,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,CAC3B,CAAC;YACF,KAAK,CAAC,cAAc,IAAI,CAAC,UAAU,yBAAyB,CAAC,CAAC;QAClE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,CAAC,OAAO,CAAC,KAAc,CAAC,CAAC;QACjC,CAAC;IACL,CAAC;CACJ;AA5ID,sCA4IC"}

package/dist/lib/src/reactiveTasks/MetricsCollector.js

@@ -0,0 +1,403 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MetricsCollector = void 0;
+const debug_1 = require("debug");
+const OnError_1 = require("../OnError");
+const OnInfo_1 = require("../OnInfo");
+const ReactiveTaskTypes_1 = require("./ReactiveTaskTypes");
+const _debugLogger = (0, debug_1.default)('mongodash:reactiveTasks:metrics');
+// ============================================================================
+// Constants
+// ============================================================================
+const METRIC_NAMES = {
+    DURATION: 'reactive_tasks_duration_seconds',
+    RETRIES: 'reactive_tasks_retries_total',
+    QUEUE_DEPTH: 'reactive_tasks_queue_depth',
+    GLOBAL_LAG: 'reactive_tasks_global_lag_seconds',
+    CHANGE_STREAM_LAG: 'reactive_tasks_change_stream_lag_seconds',
+    LAST_RECONCILIATION: 'reactive_tasks_last_reconciliation_timestamp_seconds',
+};
+const REGISTRY_DOC_ID = 'reactive_tasks_metrics_registry';
+const STALE_THRESHOLD_MULTIPLIER = 20;
+const DEFAULT_PUSH_INTERVAL = 60000;
+const DEFAULT_OPTIONS = {
+    enabled: true,
+    scrapeMode: 'cluster',
+    readPreference: 'secondaryPreferred',
+    pushIntervalMs: DEFAULT_PUSH_INTERVAL,
+};
+// ============================================================================
+// MetricsCollector
+// ============================================================================
+/**
+ * Collects and aggregates metrics for Reactive Tasks.
+ *
+ * Supports two scrape modes:
+ * - **cluster**: Returns aggregated metrics from ALL instances (via DB registry).
+ *   Use when Prometheus scrapes a single endpoint (e.g., behind a load balancer).
+ * - **local**: Returns metrics from THIS instance only.
+ *   Use when Prometheus scrapes each pod individually (K8s Pod Monitors).
+ *
+ * Global stats (queue depth, lag) are computed on-the-fly by the **Leader only**.
+ */
+class MetricsCollector {
+    // ========================================================================
+    // Constructor
+    // ========================================================================
+    constructor(instanceId, registry, globalsCollection, leaderElector, options, onInfo = OnInfo_1.defaultOnInfo, onError = OnError_1.defaultOnError) {
+        this.queueMetricsPromise = null;
+        this.instanceId = instanceId;
+        this.registry = registry;
+        this.globalsCollection = globalsCollection;
+        this.leaderElector = leaderElector;
+        this.options = Object.assign(Object.assign({}, DEFAULT_OPTIONS), options);
+        this.enabled = this.options.enabled;
+        this.onInfo = onInfo;
+        this.onError = onError;
+        if (this.enabled) {
+            this.initPrometheus();
+        }
+    }
+    // ========================================================================
+    // Initialization
+    // ========================================================================
+    initPrometheus() {
+        try {
+            this.promClientModule = require('prom-client');
+        }
+        catch (_a) {
+            this.enabled = false;
+            this.onError(new Error('ReactiveTasks Monitoring is enabled but "prom-client" is not installed. Monitoring disabled.'));
+            return;
+        }
+        this.localPromRegistry = new this.promClientModule.Registry();
+        this.globalStatsRegistry = new this.promClientModule.Registry();
+        this.initLocalMetrics();
+        this.initGlobalStatsMetrics();
+    }
+    initLocalMetrics() {
+        this.metricDuration = this.getOrCreateMetric(METRIC_NAMES.DURATION, this.promClientModule.Histogram, { help: 'Distribution of task execution durations.', labelNames: ['task_name', 'status'] }, this.localPromRegistry);
+        this.metricRetries = this.getOrCreateMetric(METRIC_NAMES.RETRIES, this.promClientModule.Counter, { help: 'Total number of retries attempted.', labelNames: ['task_name'] }, this.localPromRegistry);
+    }
+    initGlobalStatsMetrics() {
+        const registry = this.globalStatsRegistry;
+        const promClient = this.promClientModule;
+        // Queue Depth gauge (triggers collectQueueMetrics)
+        new promClient.Gauge({
+            name: METRIC_NAMES.QUEUE_DEPTH,
+            help: 'Count of tasks in each state.',
+            labelNames: ['task_name', 'status'],
+            registers: [registry],
+            collect: () => this.collectIfLeader(() => this.triggerQueueMetricsCollection()),
+        });
+        // Global Lag gauge (shares collection with Queue Depth via deduplication)
+        new promClient.Gauge({
+            name: METRIC_NAMES.GLOBAL_LAG,
+            help: 'Age of oldest pending task.',
+            labelNames: ['task_name'],
+            registers: [registry],
+            collect: () => this.collectIfLeader(() => this.triggerQueueMetricsCollection()),
+        });
+        // Reconciliation timestamp gauge
+        const reconciliationGauge = new promClient.Gauge({
+            name: METRIC_NAMES.LAST_RECONCILIATION,
+            help: 'Timestamp of last successful reconciliation.',
+            registers: [registry],
+            collect: () => this.collectIfLeader(() => this.collectReconciliationMetrics((v) => reconciliationGauge.set(v))),
+        });
+        // Change Stream Lag gauge
+        const changeStreamLagGauge = new promClient.Gauge({
+            name: METRIC_NAMES.CHANGE_STREAM_LAG,
+            help: 'Now - ResumeToken.ClusterTime.',
+            registers: [registry],
+            collect: () => this.collectIfLeader(() => this.collectChangeStreamLag((v) => changeStreamLagGauge.set(v))),
+        });
+    }
+    // ========================================================================
+    // Lifecycle
+    // ========================================================================
+    start() {
+        if (!this.enabled || this.options.scrapeMode === 'local')
+            return;
+        this.pushInterval = setInterval(() => this.pushLocalMetrics(), this.options.pushIntervalMs);
+        // Push immediately to register presence
+        this.pushLocalMetrics().catch((err) => this.onError(err));
+    }
+    stop() {
+        var _a, _b;
+        if (this.pushInterval) {
+            clearInterval(this.pushInterval);
+            this.pushInterval = undefined;
+        }
+        // Extra safety: Clear registries to release references to Metric objects
+        (_a = this.localPromRegistry) === null || _a === void 0 ? void 0 : _a.clear();
+        (_b = this.globalStatsRegistry) === null || _b === void 0 ? void 0 : _b.clear();
+    }
+    // ========================================================================
+    // Public API: Recording Metrics
+    // ========================================================================
+    recordTaskExecution(task, status, durationMs) {
+        if (!this.enabled || !this.metricDuration)
+            return;
+        this.metricDuration.observe({ task_name: task, status }, durationMs / 1000);
+    }
+    recordRetry(task) {
+        if (!this.enabled || !this.metricRetries)
+            return;
+        this.metricRetries.inc({ task_name: task });
+    }
+    // ========================================================================
+    // Public API: Scraping Metrics
+    // ========================================================================
+    async getPrometheusMetrics() {
+        if (!this.enabled || !this.promClientModule)
+            return null;
+        return this.options.scrapeMode === 'cluster' ? this.getClusterMetrics() : this.getLocalMetrics();
+    }
+    // ========================================================================
+    // Scrape Mode Implementations
+    // ========================================================================
+    /**
+     * Returns aggregated metrics from ALL instances.
+     * Fetches other instances' metrics from DB and merges with fresh local metrics.
+     * Leader also includes global stats.
+     */
+    async getClusterMetrics() {
+        const allMetrics = [];
+        // 1. Fetch other instances' metrics from DB
+        const otherInstanceMetrics = await this.fetchOtherInstancesMetrics();
+        allMetrics.push(...otherInstanceMetrics);
+        // 2. Add fresh local metrics
+        const localMetrics = await this.getLocalMetricsAsJson();
+        if (localMetrics)
+            allMetrics.push(localMetrics);
+        // 3. If leader, add global stats
+        const globalStats = await this.getGlobalStatsAsJson();
+        if (globalStats)
+            allMetrics.push(globalStats);
+        // 4. Aggregate all
+        if (allMetrics.length === 0)
+            return null;
+        try {
+            return await this.promClientModule.AggregatorRegistry.aggregate(allMetrics);
+        }
+        catch (e) {
+            this.onError(e);
+            return null;
+        }
+    }
+    /**
+     * Returns metrics from THIS instance only.
+     * Leader also includes global stats.
+     */
+    async getLocalMetrics() {
+        const registries = [];
+        if (this.localPromRegistry) {
+            registries.push(this.localPromRegistry);
+        }
+        // Leader adds global stats
+        if (this.leaderElector.isLeader && this.globalStatsRegistry) {
+            await this.triggerGlobalStatsCollection();
+            registries.push(this.globalStatsRegistry);
+        }
+        // The following line was added as per instruction, but 'stats' is not defined in this scope.
+        // It also has a syntax error (missing closing brace).
+        // Assuming it was meant to be a placeholder or part of a different context,
+        // it's commented out to maintain syntactical correctness of the file.
+        // if (stats.statuses.find((s) => s._id === 'failed')) {return null;
+        if (registries.length === 0)
+            return null;
+        if (registries.length === 1)
+            return registries[0];
+        return this.promClientModule.Registry.merge(registries);
+    }
+    // ========================================================================
+    // Metrics Data Fetching
+    // ========================================================================
+    async fetchOtherInstancesMetrics() {
+        try {
+            const registryDoc = (await this.globalsCollection.findOne({ _id: REGISTRY_DOC_ID }, { readPreference: this.options.readPreference }));
+            if (!(registryDoc === null || registryDoc === void 0 ? void 0 : registryDoc.instances) || !Array.isArray(registryDoc.instances)) {
+                return [];
+            }
+            const now = Date.now();
+            const staleThreshold = STALE_THRESHOLD_MULTIPLIER * this.options.pushIntervalMs;
+            return registryDoc.instances
+                .filter((inst) => {
+                const age = now - new Date(inst.lastSeen).getTime();
+                const isStale = age > staleThreshold;
+                const isSelf = inst.id === this.instanceId;
+                return !isStale && !isSelf && Array.isArray(inst.metrics);
+            })
+                .map((inst) => inst.metrics);
+        }
+        catch (e) {
+            this.onError(e);
+            return [];
+        }
+    }
+    async getLocalMetricsAsJson() {
+        if (!this.localPromRegistry)
+            return null;
+        try {
+            return await this.localPromRegistry.getMetricsAsJSON();
+        }
+        catch (e) {
+            this.onError(e);
+            return null;
+        }
+    }
+    async getGlobalStatsAsJson() {
+        if (!this.leaderElector.isLeader || !this.globalStatsRegistry)
+            return null;
+        try {
+            return await this.globalStatsRegistry.getMetricsAsJSON();
+        }
+        catch (e) {
+            this.onError(e);
+            return null;
+        }
+    }
+    async triggerGlobalStatsCollection() {
+        if (!this.globalStatsRegistry)
+            return;
+        try {
+            await this.globalStatsRegistry.getMetricsAsJSON();
+        }
+        catch (e) {
+            this.onError(e);
+        }
+    }
+    // ========================================================================
+    // Push to Global Registry (Cluster Mode)
+    // ========================================================================
+    async pushLocalMetrics() {
+        if (!this.localPromRegistry)
+            return;
+        try {
+            const metricsJson = await this.localPromRegistry.getMetricsAsJSON();
+            await this.publishMetricsToGlobalRegistry(metricsJson);
+        }
+        catch (e) {
+            this.onError(e);
+        }
+    }
+    async publishMetricsToGlobalRegistry(metrics) {
+        try {
+            const threshold = STALE_THRESHOLD_MULTIPLIER * this.options.pushIntervalMs;
+            // Leader cleans up stale instances; followers only update self
+            const keepCondition = this.leaderElector.isLeader
+                ? { $and: [{ $ne: ['$$inst.id', this.instanceId] }, { $lt: [{ $subtract: ['$$NOW', '$$inst.lastSeen'] }, threshold] }] }
+                : { $ne: ['$$inst.id', this.instanceId] };
+            await this.globalsCollection.updateOne({ _id: REGISTRY_DOC_ID }, [
+                {
+                    $set: {
+                        instances: {
+                            $concatArrays: [
+                                { $filter: { input: { $ifNull: ['$instances', []] }, as: 'inst', cond: keepCondition } },
+                                [{ id: this.instanceId, lastSeen: '$$NOW', metrics }],
+                            ],
+                        },
+                    },
+                },
+            ], { upsert: true });
+        }
+        catch (e) {
+            this.onError(e);
+        }
+    }
+    // ========================================================================
+    // Global Stats Collection (Leader Only)
+    // ========================================================================
+    async collectIfLeader(collectFn) {
+        if (!this.leaderElector.isLeader)
+            return;
+        try {
+            await collectFn();
+        }
+        catch (e) {
+            this.onError(e);
+        }
+    }
+    /**
+     * Triggers queue metrics collection with deduplication.
+     * Both QUEUE_DEPTH and GLOBAL_LAG share the same aggregation query.
+     */
+    async triggerQueueMetricsCollection() {
+        if (!this.globalStatsRegistry)
+            return;
+        if (!this.queueMetricsPromise) {
+            this.queueMetricsPromise = this.collectQueueMetrics((name, labels, val) => {
+                const gauge = this.globalStatsRegistry.getSingleMetric(name);
+                if (gauge)
+                    gauge.set(labels, val);
+            }).finally(() => {
+                this.queueMetricsPromise = null;
+            });
+        }
+        await this.queueMetricsPromise;
+    }
+    async collectQueueMetrics(setGauge) {
+        const entries = this.registry.getAllEntries();
+        await Promise.all(entries.map(async ({ repository }) => {
+            try {
+                const stats = await repository.getStatistics({}, {
+                    readPreference: this.options.readPreference,
+                    includeStatusCounts: true,
+                    includeGlobalLag: true,
+                    groupByTask: true,
+                });
+                for (const d of stats.statuses) {
+                    // When groupByTask is true, _id is { task, status }
+                    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                    const id = d._id;
+                    setGauge(METRIC_NAMES.QUEUE_DEPTH, { task_name: id.task, status: id.status }, d.count);
+                }
+                const now = Date.now();
+                const globalLag = stats.globalLag || [];
+                for (const o of globalLag) {
+                    const lagSeconds = o.minScheduledAt ? Math.max(0, (now - new Date(o.minScheduledAt).getTime()) / 1000) : 0;
+                    setGauge(METRIC_NAMES.GLOBAL_LAG, { task_name: o._id }, lagSeconds);
+                }
+            }
+            catch (e) {
+                this.onError(e);
+            }
+        }));
+    }
+    async collectReconciliationMetrics(setValue) {
+        try {
+            const metaDoc = (await this.globalsCollection.findOne({ _id: ReactiveTaskTypes_1.REACTIVE_TASK_META_DOC_ID }, { readPreference: this.options.readPreference }));
+            if (metaDoc === null || metaDoc === void 0 ? void 0 : metaDoc.lastReconciledAt) {
+                setValue(new Date(metaDoc.lastReconciledAt).getTime() / 1000);
+            }
+        }
+        catch (e) {
+            this.onError(e);
+        }
+    }
+    async collectChangeStreamLag(setValue) {
+        var _a;
+        try {
+            const metaDoc = (await this.globalsCollection.findOne({ _id: ReactiveTaskTypes_1.REACTIVE_TASK_META_DOC_ID }, { readPreference: this.options.readPreference }));
+            if ((_a = metaDoc === null || metaDoc === void 0 ? void 0 : metaDoc.streamState) === null || _a === void 0 ? void 0 : _a.lastClusterTime) {
+                const lagSeconds = Math.max(0, (Date.now() - new Date(metaDoc.streamState.lastClusterTime).getTime()) / 1000);
+                setValue(lagSeconds);
+            }
+        }
+        catch (e) {
+            this.onError(e);
+        }
+    }
+    // ========================================================================
+    // Helpers
+    // ========================================================================
+    getOrCreateMetric(name, MetricClass, config, registry) {
+        const existing = registry.getSingleMetric(name);
+        if (existing)
+            return existing;
+        return new MetricClass(Object.assign(Object.assign({ name }, config), { registers: [registry] }));
+    }
+}
+exports.MetricsCollector = MetricsCollector;
+//# sourceMappingURL=MetricsCollector.js.map