@oneuptime/common 10.5.23 → 10.5.25

package/build/dist/Server/Utils/Monitor/MonitorResource.js

@@ -32,16 +32,6 @@ import { LIMIT_PER_PROJECT } from "../../../Types/Database/LimitMax";
 export default class MonitorResourceUtil {
     static async monitorResource(dataToProcess) {
         var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s;
-        let mutex = null;
-        try {
-            mutex = await Semaphore.lock({
-                key: dataToProcess.monitorId.toString(),
-                namespace: "MonitorResourceUtil.monitorResource",
-            });
-        }
-        catch (err) {
-            logger.error(err);
-        }
         let response = {
             monitorId: dataToProcess.monitorId,
             criteriaMetId: undefined,
@@ -124,458 +114,486 @@ export default class MonitorResourceUtil {
             logger.debug(`${dataToProcess.monitorId.toString()} Monitor is disabled because one of the scheduled maintenance event this monitor is attached to has not ended. Please end the scheduled maintenance event to start monitoring again.`);
             throw new BadDataException("Monitor is disabled because one of the scheduled maintenance event this monitor is attached to has not ended. Please end the scheduled maintenance event to start monitoring again.");
         }
-        let probeName = undefined;
-        const monitorName = monitor.name || undefined;
-        // save the last log to MonitorProbe.
-        // get last log. We do this because there are many monitoring steps and we need to store those.
-        logger.debug(`${dataToProcess.monitorId.toString()} - monitor type ${monitor.monitorType}`);
-        if (monitor.monitorType &&
-            MonitorTypeHelper.isProbableMonitor(monitor.monitorType)) {
-            dataToProcess = dataToProcess;
-            if (dataToProcess.probeId) {
-                const monitorProbe = await MonitorProbeService.findOneBy({
-                    query: {
-                        monitorId: monitor.id,
-                        probeId: dataToProcess.probeId,
-                    },
-                    select: {
-                        lastMonitoringLog: true,
-                        probe: {
-                            name: true,
-                        },
-                    },
-                    props: {
-                        isRoot: true,
-                    },
-                });
-                if (!monitorProbe) {
-                    throw new BadDataException("Probe is not assigned to this monitor");
+        /*
+         * Acquire a per-monitor lock so concurrent results for the same monitor are
+         * processed serially. This MUST come after the validation above: acquiring
+         * before the not-found/disabled checks meant those throw paths held — and
+         * leaked — the lock. redis-semaphore keeps a refresh timer alive until
+         * release() is called, so a leaked lock pinned the Redis key until pod
+         * restart and forced every other worker to spin on acquire. The try/finally
+         * below guarantees the lock is released on every exit path (return or
+         * throw); acquireTimeout/retryInterval cap the acquire spin so a contended
+         * lock fails fast instead of polling Redis for 10s.
+         */
+        let mutex = null;
+        try {
+            mutex = await Semaphore.lock({
+                key: dataToProcess.monitorId.toString(),
+                namespace: "MonitorResourceUtil.monitorResource",
+                acquireTimeout: 2000,
+                retryInterval: 100,
+                acquireAttemptsLimit: 20,
+            });
+        }
+        catch (err) {
+            logger.error(err);
+        }
+        const releaseMutex = async () => {
+            if (mutex) {
+                try {
+                    await Semaphore.release(mutex);
                 }
-                probeName = ((_a = monitorProbe.probe) === null || _a === void 0 ? void 0 : _a.name) || undefined;
-                await MonitorProbeService.updateOneBy({
-                    query: {
-                        monitorId: monitor.id,
-                        probeId: dataToProcess.probeId,
-                    },
-                    data: {
-                        lastMonitoringLog: Object.assign(Object.assign({}, (monitorProbe.lastMonitoringLog || {})), { [dataToProcess.monitorStepId.toString()]: Object.assign(Object.assign({}, JSON.parse(JSON.stringify(dataToProcess))), { monitoredAt: OneUptimeDate.getCurrentDate() }) }),
-                    },
-                    props: {
-                        isRoot: true,
-                    },
-                });
+                catch (err) {
+                    logger.error(err);
+                }
+                mutex = null;
             }
-        }
-        const serverMonitorResponse = monitor.monitorType === MonitorType.Server &&
-            dataToProcess.requestReceivedAt
-            ? dataToProcess
-            : undefined;
-        const incomingMonitorRequest = monitor.monitorType === MonitorType.IncomingRequest &&
-            dataToProcess.incomingRequestReceivedAt &&
-            !dataToProcess
-                .onlyCheckForIncomingRequestReceivedAt
-            ? dataToProcess
-            : undefined;
-        let hasPersistedMonitorData = false;
-        const persistLatestMonitorPayload = async () => {
-            if (hasPersistedMonitorData) {
-                return;
+        };
+        try {
+            let probeName = undefined;
+            const monitorName = monitor.name || undefined;
+            // save the last log to MonitorProbe.
+            // get last log. We do this because there are many monitoring steps and we need to store those.
+            logger.debug(`${dataToProcess.monitorId.toString()} - monitor type ${monitor.monitorType}`);
+            if (monitor.monitorType &&
+                MonitorTypeHelper.isProbableMonitor(monitor.monitorType)) {
+                dataToProcess = dataToProcess;
+                if (dataToProcess.probeId) {
+                    const monitorProbe = await MonitorProbeService.findOneBy({
+                        query: {
+                            monitorId: monitor.id,
+                            probeId: dataToProcess.probeId,
+                        },
+                        select: {
+                            lastMonitoringLog: true,
+                            probe: {
+                                name: true,
+                            },
+                        },
+                        props: {
+                            isRoot: true,
+                        },
+                    });
+                    if (!monitorProbe) {
+                        throw new BadDataException("Probe is not assigned to this monitor");
+                    }
+                    probeName = ((_a = monitorProbe.probe) === null || _a === void 0 ? void 0 : _a.name) || undefined;
+                    await MonitorProbeService.updateOneBy({
+                        query: {
+                            monitorId: monitor.id,
+                            probeId: dataToProcess.probeId,
+                        },
+                        data: {
+                            lastMonitoringLog: Object.assign(Object.assign({}, (monitorProbe.lastMonitoringLog || {})), { [dataToProcess.monitorStepId.toString()]: Object.assign(Object.assign({}, JSON.parse(JSON.stringify(dataToProcess))), { monitoredAt: OneUptimeDate.getCurrentDate() }) }),
+                        },
+                        props: {
+                            isRoot: true,
+                        },
+                    });
+                }
             }
-            if (serverMonitorResponse) {
-                logger.debug(`${dataToProcess.monitorId.toString()} - Server request received at ${serverMonitorResponse.requestReceivedAt}`);
-                logger.debug(dataToProcess);
-                /*
-                 * Skip persistence when this evaluation originated from the
-                 * CheckOnlineStatus cron (onlyCheckRequestReceivedAt=true). The cron
-                 * re-evaluates using the already-stale value read from the DB and has
-                 * no new heartbeat data to persist — writing it back would race with
-                 * (and overwrite) the ingest path's fresh heartbeat update, causing
-                 * the monitor to flap between Online and Offline every minute.
-                 */
-                if (!serverMonitorResponse.onlyCheckRequestReceivedAt) {
+            const serverMonitorResponse = monitor.monitorType === MonitorType.Server &&
+                dataToProcess.requestReceivedAt
+                ? dataToProcess
+                : undefined;
+            const incomingMonitorRequest = monitor.monitorType === MonitorType.IncomingRequest &&
+                dataToProcess.incomingRequestReceivedAt &&
+                !dataToProcess
+                    .onlyCheckForIncomingRequestReceivedAt
+                ? dataToProcess
+                : undefined;
+            let hasPersistedMonitorData = false;
+            const persistLatestMonitorPayload = async () => {
+                if (hasPersistedMonitorData) {
+                    return;
+                }
+                if (serverMonitorResponse) {
+                    logger.debug(`${dataToProcess.monitorId.toString()} - Server request received at ${serverMonitorResponse.requestReceivedAt}`);
+                    logger.debug(dataToProcess);
+                    /*
+                     * Skip persistence when this evaluation originated from the
+                     * CheckOnlineStatus cron (onlyCheckRequestReceivedAt=true). The cron
+                     * re-evaluates using the already-stale value read from the DB and has
+                     * no new heartbeat data to persist — writing it back would race with
+                     * (and overwrite) the ingest path's fresh heartbeat update, causing
+                     * the monitor to flap between Online and Offline every minute.
+                     */
+                    if (!serverMonitorResponse.onlyCheckRequestReceivedAt) {
+                        await MonitorService.updateOneById({
+                            id: monitor.id,
+                            data: {
+                                serverMonitorRequestReceivedAt: serverMonitorResponse.requestReceivedAt,
+                                serverMonitorResponse,
+                            },
+                            props: {
+                                isRoot: true,
+                                ignoreHooks: true,
+                            },
+                        });
+                        logger.debug(`${dataToProcess.monitorId.toString()} - Monitor Server Response Updated`);
+                    }
+                    else {
+                        logger.debug(`${dataToProcess.monitorId.toString()} - Skipping Monitor Server Response persist (cron re-evaluation).`);
+                    }
+                }
+                if (incomingMonitorRequest) {
+                    logger.debug(`${dataToProcess.monitorId.toString()} - Incoming request received at ${incomingMonitorRequest.incomingRequestReceivedAt}`);
                     await MonitorService.updateOneById({
                         id: monitor.id,
                         data: {
-                            serverMonitorRequestReceivedAt: serverMonitorResponse.requestReceivedAt,
-                            serverMonitorResponse,
+                            incomingRequestMonitorHeartbeatCheckedAt: OneUptimeDate.getCurrentDate(),
+                            incomingMonitorRequest: JSON.parse(JSON.stringify(incomingMonitorRequest)),
                         },
                         props: {
                             isRoot: true,
                             ignoreHooks: true,
                         },
                     });
-                    logger.debug(`${dataToProcess.monitorId.toString()} - Monitor Server Response Updated`);
-                }
-                else {
-                    logger.debug(`${dataToProcess.monitorId.toString()} - Skipping Monitor Server Response persist (cron re-evaluation).`);
+                    logger.debug(`${dataToProcess.monitorId.toString()} - Monitor Incoming Request Updated`);
                 }
+                hasPersistedMonitorData = true;
+            };
+            logger.debug(`${dataToProcess.monitorId.toString()} - Saving monitor metrics`);
+            try {
+                await MonitorMetricUtil.saveMonitorMetrics({
+                    monitorId: monitor.id,
+                    projectId: monitor.projectId,
+                    dataToProcess: dataToProcess,
+                    probeName: probeName || undefined,
+                    monitorName: monitorName || undefined,
+                });
             }
-            if (incomingMonitorRequest) {
-                logger.debug(`${dataToProcess.monitorId.toString()} - Incoming request received at ${incomingMonitorRequest.incomingRequestReceivedAt}`);
-                await MonitorService.updateOneById({
-                    id: monitor.id,
-                    data: {
-                        incomingRequestMonitorHeartbeatCheckedAt: OneUptimeDate.getCurrentDate(),
-                        incomingMonitorRequest: JSON.parse(JSON.stringify(incomingMonitorRequest)),
-                    },
-                    props: {
-                        isRoot: true,
-                        ignoreHooks: true,
-                    },
+            catch (err) {
+                logger.error("Unable to save metrics");
+                logger.error(err);
+            }
+            logger.debug(`${dataToProcess.monitorId.toString()} - Monitor metrics saved`);
+            const monitorSteps = monitor.monitorSteps;
+            if (!((_b = monitorSteps.data) === null || _b === void 0 ? void 0 : _b.monitorStepsInstanceArray) ||
+                ((_c = monitorSteps.data) === null || _c === void 0 ? void 0 : _c.monitorStepsInstanceArray.length) === 0) {
+                logger.debug(`${dataToProcess.monitorId.toString()} - No monitoring steps.`);
+                await persistLatestMonitorPayload();
+                MonitorLogUtil.saveMonitorLog({
+                    monitorId: monitor.id,
+                    projectId: monitor.projectId,
+                    dataToProcess: dataToProcess,
                 });
-                logger.debug(`${dataToProcess.monitorId.toString()} - Monitor Incoming Request Updated`);
+                return response;
             }
-            hasPersistedMonitorData = true;
-        };
-        logger.debug(`${dataToProcess.monitorId.toString()} - Saving monitor metrics`);
-        try {
-            await MonitorMetricUtil.saveMonitorMetrics({
-                monitorId: monitor.id,
-                projectId: monitor.projectId,
-                dataToProcess: dataToProcess,
-                probeName: probeName || undefined,
-                monitorName: monitorName || undefined,
-            });
-        }
-        catch (err) {
-            logger.error("Unable to save metrics");
-            logger.error(err);
-        }
-        logger.debug(`${dataToProcess.monitorId.toString()} - Monitor metrics saved`);
-        const monitorSteps = monitor.monitorSteps;
-        if (!((_b = monitorSteps.data) === null || _b === void 0 ? void 0 : _b.monitorStepsInstanceArray) ||
-            ((_c = monitorSteps.data) === null || _c === void 0 ? void 0 : _c.monitorStepsInstanceArray.length) === 0) {
-            logger.debug(`${dataToProcess.monitorId.toString()} - No monitoring steps.`);
-            await persistLatestMonitorPayload();
-            MonitorLogUtil.saveMonitorLog({
-                monitorId: monitor.id,
-                projectId: monitor.projectId,
-                dataToProcess: dataToProcess,
-            });
-            return response;
-        }
-        logger.debug(`${dataToProcess.monitorId.toString()} - Auto resolving criteria instances.`);
-        const criteriaInstances = monitorSteps.data.monitorStepsInstanceArray
-            .map((step) => {
-            var _a;
-            return (_a = step.data) === null || _a === void 0 ? void 0 : _a.monitorCriteria;
-        })
-            .filter((criteria) => {
-            return Boolean(criteria);
-        })
-            .map((criteria) => {
-            var _a;
-            return [...(((_a = criteria === null || criteria === void 0 ? void 0 : criteria.data) === null || _a === void 0 ? void 0 : _a.monitorCriteriaInstanceArray) || [])];
-        })
-            .flat();
-        const autoResolveCriteriaInstanceIdIncidentIdsDictionary = {};
-        const criteriaInstanceMap = {};
-        for (const criteriaInstance of criteriaInstances) {
-            criteriaInstanceMap[((_d = criteriaInstance.data) === null || _d === void 0 ? void 0 : _d.id) || ""] = criteriaInstance;
-            if (((_e = criteriaInstance.data) === null || _e === void 0 ? void 0 : _e.incidents) &&
-                ((_f = criteriaInstance.data) === null || _f === void 0 ? void 0 : _f.incidents.length) > 0) {
-                for (const incidentTemplate of criteriaInstance.data.incidents) {
-                    if (incidentTemplate.autoResolveIncident) {
-                        if (!autoResolveCriteriaInstanceIdIncidentIdsDictionary[criteriaInstance.data.id.toString()]) {
-                            autoResolveCriteriaInstanceIdIncidentIdsDictionary[criteriaInstance.data.id.toString()] = [];
+            logger.debug(`${dataToProcess.monitorId.toString()} - Auto resolving criteria instances.`);
+            const criteriaInstances = monitorSteps.data.monitorStepsInstanceArray
+                .map((step) => {
+                var _a;
+                return (_a = step.data) === null || _a === void 0 ? void 0 : _a.monitorCriteria;
+            })
+                .filter((criteria) => {
+                return Boolean(criteria);
+            })
+                .map((criteria) => {
+                var _a;
+                return [...(((_a = criteria === null || criteria === void 0 ? void 0 : criteria.data) === null || _a === void 0 ? void 0 : _a.monitorCriteriaInstanceArray) || [])];
+            })
+                .flat();
+            const autoResolveCriteriaInstanceIdIncidentIdsDictionary = {};
+            const criteriaInstanceMap = {};
+            for (const criteriaInstance of criteriaInstances) {
+                criteriaInstanceMap[((_d = criteriaInstance.data) === null || _d === void 0 ? void 0 : _d.id) || ""] = criteriaInstance;
+                if (((_e = criteriaInstance.data) === null || _e === void 0 ? void 0 : _e.incidents) &&
+                    ((_f = criteriaInstance.data) === null || _f === void 0 ? void 0 : _f.incidents.length) > 0) {
+                    for (const incidentTemplate of criteriaInstance.data.incidents) {
+                        if (incidentTemplate.autoResolveIncident) {
+                            if (!autoResolveCriteriaInstanceIdIncidentIdsDictionary[criteriaInstance.data.id.toString()]) {
+                                autoResolveCriteriaInstanceIdIncidentIdsDictionary[criteriaInstance.data.id.toString()] = [];
+                            }
+                            (_g = autoResolveCriteriaInstanceIdIncidentIdsDictionary[criteriaInstance.data.id.toString()]) === null || _g === void 0 ? void 0 : _g.push(incidentTemplate.id);
                         }
-                        (_g = autoResolveCriteriaInstanceIdIncidentIdsDictionary[criteriaInstance.data.id.toString()]) === null || _g === void 0 ? void 0 : _g.push(incidentTemplate.id);
                     }
                 }
             }
-        }
-        // alerts.
-        const autoResolveCriteriaInstanceIdAlertIdsDictionary = {};
-        const criteriaInstanceAlertMap = {};
-        for (const criteriaInstance of criteriaInstances) {
-            criteriaInstanceAlertMap[((_h = criteriaInstance.data) === null || _h === void 0 ? void 0 : _h.id) || ""] =
-                criteriaInstance;
-            if (((_j = criteriaInstance.data) === null || _j === void 0 ? void 0 : _j.alerts) &&
-                ((_k = criteriaInstance.data) === null || _k === void 0 ? void 0 : _k.alerts.length) > 0) {
-                for (const alertTemplate of criteriaInstance.data.alerts) {
-                    if (alertTemplate.autoResolveAlert) {
-                        if (!autoResolveCriteriaInstanceIdAlertIdsDictionary[criteriaInstance.data.id.toString()]) {
-                            autoResolveCriteriaInstanceIdAlertIdsDictionary[criteriaInstance.data.id.toString()] = [];
+            // alerts.
+            const autoResolveCriteriaInstanceIdAlertIdsDictionary = {};
+            const criteriaInstanceAlertMap = {};
+            for (const criteriaInstance of criteriaInstances) {
+                criteriaInstanceAlertMap[((_h = criteriaInstance.data) === null || _h === void 0 ? void 0 : _h.id) || ""] =
+                    criteriaInstance;
+                if (((_j = criteriaInstance.data) === null || _j === void 0 ? void 0 : _j.alerts) &&
+                    ((_k = criteriaInstance.data) === null || _k === void 0 ? void 0 : _k.alerts.length) > 0) {
+                    for (const alertTemplate of criteriaInstance.data.alerts) {
+                        if (alertTemplate.autoResolveAlert) {
+                            if (!autoResolveCriteriaInstanceIdAlertIdsDictionary[criteriaInstance.data.id.toString()]) {
+                                autoResolveCriteriaInstanceIdAlertIdsDictionary[criteriaInstance.data.id.toString()] = [];
+                            }
+                            (_l = autoResolveCriteriaInstanceIdAlertIdsDictionary[criteriaInstance.data.id.toString()]) === null || _l === void 0 ? void 0 : _l.push(alertTemplate.id);
                         }
-                        (_l = autoResolveCriteriaInstanceIdAlertIdsDictionary[criteriaInstance.data.id.toString()]) === null || _l === void 0 ? void 0 : _l.push(alertTemplate.id);
                     }
                 }
             }
-        }
-        const monitorStep = monitorSteps.data.monitorStepsInstanceArray[0];
-        logger.debug(`Monitor Step: ${monitorStep ? monitorStep.id : "undefined"}`);
-        if (dataToProcess.monitorStepId) {
-            monitorSteps.data.monitorStepsInstanceArray.find((monitorStep) => {
-                return (monitorStep.id.toString() ===
-                    dataToProcess.monitorStepId.toString());
-            });
-            logger.debug(`Found Monitor Step ID: ${dataToProcess.monitorStepId}`);
-        }
-        if (!monitorStep) {
-            logger.debug("No steps found, ignoring everything.");
-            await persistLatestMonitorPayload();
-            MonitorLogUtil.saveMonitorLog({
-                monitorId: monitor.id,
-                projectId: monitor.projectId,
-                dataToProcess: dataToProcess,
-            });
-            return response;
-        }
-        // now process the monitor step
-        response.ingestedMonitorStepId = monitorStep.id;
-        logger.debug(`Ingested Monitor Step ID: ${monitorStep.id}`);
-        //find next monitor step after this one.
-        const nextMonitorStepIndex = monitorSteps.data.monitorStepsInstanceArray.findIndex((step) => {
-            return step.id.toString() === monitorStep.id.toString();
-        });
-        response.nextMonitorStepId =
-            (_m = monitorSteps.data.monitorStepsInstanceArray[nextMonitorStepIndex + 1]) === null || _m === void 0 ? void 0 : _m.id;
-        logger.debug(`Next Monitor Step ID: ${response.nextMonitorStepId}`);
-        // now process probe response monitors
-        logger.debug(`${dataToProcess.monitorId.toString()} - Processing monitor step...`);
-        response = await MonitorCriteriaEvaluator.processMonitorStep({
-            dataToProcess: dataToProcess,
-            monitorStep: monitorStep,
-            monitor: monitor,
-            probeApiIngestResponse: response,
-            evaluationSummary: evaluationSummary,
-        });
-        // Check probe agreement for probe-based monitors
-        if (monitor.monitorType &&
-            MonitorTypeHelper.isProbableMonitor(monitor.monitorType)) {
-            const probeAgreementResult = await MonitorResourceUtil.checkProbeAgreement({
-                monitor: monitor,
-                monitorStep: monitorStep,
-                currentCriteriaMetId: response.criteriaMetId || null,
-                currentRootCause: response.rootCause || null,
-            });
-            // Add probe agreement event to evaluation summary
-            evaluationSummary.events.push({
-                type: "probe-agreement",
-                title: "Probe Agreement Check",
-                message: probeAgreementResult.hasAgreement
-                    ? `Probe agreement reached: ${probeAgreementResult.agreementCount}/${probeAgreementResult.requiredCount} probes agree (${probeAgreementResult.totalActiveProbes} active probes total).`
-                    : `Probe agreement not reached: ${probeAgreementResult.agreementCount}/${probeAgreementResult.requiredCount} probes agree (${probeAgreementResult.totalActiveProbes} active probes total). Skipping status change.`,
-                at: OneUptimeDate.getCurrentDate(),
-            });
-            if (!probeAgreementResult.hasAgreement) {
-                logger.debug(`${dataToProcess.monitorId.toString()} - Probe agreement not met. ${probeAgreementResult.agreementCount}/${probeAgreementResult.requiredCount} probes agree. Skipping status change.`);
-                // Release lock and return early - no status change
-                if (mutex) {
-                    try {
-                        await Semaphore.release(mutex);
-                    }
-                    catch (err) {
-                        logger.error(err);
-                    }
-                }
+            const monitorStep = monitorSteps.data.monitorStepsInstanceArray[0];
+            logger.debug(`Monitor Step: ${monitorStep ? monitorStep.id : "undefined"}`);
+            if (dataToProcess.monitorStepId) {
+                monitorSteps.data.monitorStepsInstanceArray.find((monitorStep) => {
+                    return (monitorStep.id.toString() ===
+                        dataToProcess.monitorStepId.toString());
+                });
+                logger.debug(`Found Monitor Step ID: ${dataToProcess.monitorStepId}`);
+            }
+            if (!monitorStep) {
+                logger.debug("No steps found, ignoring everything.");
                 await persistLatestMonitorPayload();
                 MonitorLogUtil.saveMonitorLog({
                     monitorId: monitor.id,
                     projectId: monitor.projectId,
                     dataToProcess: dataToProcess,
                 });
-                response.evaluationSummary = evaluationSummary;
                 return response;
             }
-            // Use the agreed criteria result
-            response.criteriaMetId = probeAgreementResult.agreedCriteriaId
-                ? probeAgreementResult.agreedCriteriaId
-                : undefined;
-            response.rootCause = probeAgreementResult.agreedRootCause;
-            // Add probe names in agreement to the root cause
-            if (response.rootCause &&
-                probeAgreementResult.agreedProbeNames.length > 0) {
-                response.rootCause += `
-**Probes in Agreement**: ${probeAgreementResult.agreedProbeNames.join(", ")}
-`;
-            }
-        }
-        if (response.criteriaMetId && response.rootCause) {
-            logger.debug(`${dataToProcess.monitorId.toString()} - Criteria met: ${response.criteriaMetId}`);
-            logger.debug(`${dataToProcess.monitorId.toString()} - Root cause: ${response.rootCause}`);
-            let telemetryQuery = undefined;
-            if (dataToProcess && dataToProcess.logQuery) {
-                telemetryQuery = {
-                    telemetryQuery: dataToProcess.logQuery,
-                    telemetryType: TelemetryType.Log,
-                    metricViewData: null,
-                };
-                logger.debug(`${dataToProcess.monitorId.toString()} - Log query found.`);
-            }
-            if (dataToProcess && dataToProcess.spanQuery) {
-                telemetryQuery = {
-                    telemetryQuery: dataToProcess.spanQuery,
-                    telemetryType: TelemetryType.Trace,
-                    metricViewData: null,
-                };
-                logger.debug(`${dataToProcess.monitorId.toString()} - Span query found.`);
-            }
-            if (dataToProcess &&
-                dataToProcess.metricViewConfig &&
-                dataToProcess.startAndEndDate) {
-                telemetryQuery = {
-                    telemetryQuery: null,
-                    telemetryType: TelemetryType.Metric,
-                    metricViewData: {
-                        startAndEndDate: dataToProcess.startAndEndDate || null,
-                        queryConfigs: dataToProcess
-                            .metricViewConfig.queryConfigs,
-                        formulaConfigs: dataToProcess
-                            .metricViewConfig.formulaConfigs,
-                    },
-                };
-                logger.debug(`${dataToProcess.monitorId.toString()} - Span query found.`);
-            }
-            if (dataToProcess &&
-                dataToProcess.exceptionQuery) {
-                const exceptionResponse = dataToProcess;
-                telemetryQuery = {
-                    telemetryQuery: exceptionResponse.exceptionQuery,
-                    telemetryType: TelemetryType.Exception,
-                    metricViewData: null,
-                };
-                logger.debug(`${dataToProcess.monitorId.toString()} - Exception query found.`);
-            }
-            const matchedCriteriaInstance = criteriaInstanceMap[response.criteriaMetId];
-            const monitorStatusTimelineChange = await MonitorStatusTimelineUtil.updateMonitorStatusTimeline({
-                monitor: monitor,
-                rootCause: response.rootCause,
+            // now process the monitor step
+            response.ingestedMonitorStepId = monitorStep.id;
+            logger.debug(`Ingested Monitor Step ID: ${monitorStep.id}`);
+            //find next monitor step after this one.
+            const nextMonitorStepIndex = monitorSteps.data.monitorStepsInstanceArray.findIndex((step) => {
+                return step.id.toString() === monitorStep.id.toString();
+            });
+            response.nextMonitorStepId =
+                (_m = monitorSteps.data.monitorStepsInstanceArray[nextMonitorStepIndex + 1]) === null || _m === void 0 ? void 0 : _m.id;
+            logger.debug(`Next Monitor Step ID: ${response.nextMonitorStepId}`);
+            // now process probe response monitors
+            logger.debug(`${dataToProcess.monitorId.toString()} - Processing monitor step...`);
+            response = await MonitorCriteriaEvaluator.processMonitorStep({
                 dataToProcess: dataToProcess,
-                criteriaInstance: matchedCriteriaInstance,
-                props: {
-                    telemetryQuery: telemetryQuery,
-                },
+                monitorStep: monitorStep,
+                monitor: monitor,
+                probeApiIngestResponse: response,
+                evaluationSummary: evaluationSummary,
             });
-            if (monitorStatusTimelineChange) {
-                const changedStatusName = await getMonitorStatusName(((_o = matchedCriteriaInstance.data) === null || _o === void 0 ? void 0 : _o.monitorStatusId) ||
-                    monitorStatusTimelineChange.monitorStatusId);
+            // Check probe agreement for probe-based monitors
+            if (monitor.monitorType &&
+                MonitorTypeHelper.isProbableMonitor(monitor.monitorType)) {
+                const probeAgreementResult = await MonitorResourceUtil.checkProbeAgreement({
+                    monitor: monitor,
+                    monitorStep: monitorStep,
+                    currentCriteriaMetId: response.criteriaMetId || null,
+                    currentRootCause: response.rootCause || null,
+                });
+                // Add probe agreement event to evaluation summary
                 evaluationSummary.events.push({
-                    type: "monitor-status-changed",
-                    title: "Monitor status updated",
-                    message: changedStatusName
-                        ? `Monitor status changed to "${changedStatusName}" because criteria "${((_p = matchedCriteriaInstance.data) === null || _p === void 0 ? void 0 : _p.name) || "Unnamed criteria"}" was met.`
-                        : `Monitor status changed because criteria "${((_q = matchedCriteriaInstance.data) === null || _q === void 0 ? void 0 : _q.name) || "Unnamed criteria"}" was met.`,
-                    relatedCriteriaId: (_r = matchedCriteriaInstance.data) === null || _r === void 0 ? void 0 : _r.id,
+                    type: "probe-agreement",
+                    title: "Probe Agreement Check",
+                    message: probeAgreementResult.hasAgreement
+                        ? `Probe agreement reached: ${probeAgreementResult.agreementCount}/${probeAgreementResult.requiredCount} probes agree (${probeAgreementResult.totalActiveProbes} active probes total).`
+                        : `Probe agreement not reached: ${probeAgreementResult.agreementCount}/${probeAgreementResult.requiredCount} probes agree (${probeAgreementResult.totalActiveProbes} active probes total). Skipping status change.`,
                     at: OneUptimeDate.getCurrentDate(),
                 });
+                if (!probeAgreementResult.hasAgreement) {
+                    logger.debug(`${dataToProcess.monitorId.toString()} - Probe agreement not met. ${probeAgreementResult.agreementCount}/${probeAgreementResult.requiredCount} probes agree. Skipping status change.`);
+                    // Release lock and return early - no status change
+                    await releaseMutex();
+                    await persistLatestMonitorPayload();
+                    MonitorLogUtil.saveMonitorLog({
+                        monitorId: monitor.id,
+                        projectId: monitor.projectId,
+                        dataToProcess: dataToProcess,
+                    });
+                    response.evaluationSummary = evaluationSummary;
+                    return response;
+                }
+                // Use the agreed criteria result
+                response.criteriaMetId = probeAgreementResult.agreedCriteriaId
+                    ? probeAgreementResult.agreedCriteriaId
+                    : undefined;
+                response.rootCause = probeAgreementResult.agreedRootCause;
+                // Add probe names in agreement to the root cause
+                if (response.rootCause &&
+                    probeAgreementResult.agreedProbeNames.length > 0) {
+                    response.rootCause += `
+**Probes in Agreement**: ${probeAgreementResult.agreedProbeNames.join(", ")}
+`;
+                }
             }
-            await MonitorIncident.criteriaMetCreateIncidentsAndUpdateMonitorStatus({
-                monitor: monitor,
-                rootCause: response.rootCause,
-                dataToProcess: dataToProcess,
-                autoResolveCriteriaInstanceIdIncidentIdsDictionary,
-                criteriaInstance: matchedCriteriaInstance,
-                evaluationSummary: evaluationSummary,
-                props: {
-                    telemetryQuery: telemetryQuery,
-                },
-                matchesPerSeries: response.perSeriesMatches,
-            });
-            await MonitorAlert.criteriaMetCreateAlertsAndUpdateMonitorStatus({
-                monitor: monitor,
-                rootCause: response.rootCause,
-                dataToProcess: dataToProcess,
-                autoResolveCriteriaInstanceIdAlertIdsDictionary,
-                criteriaInstance: criteriaInstanceAlertMap[response.criteriaMetId],
-                evaluationSummary: evaluationSummary,
-                props: {
-                    telemetryQuery: telemetryQuery,
-                },
-                matchesPerSeries: response.perSeriesMatches,
-            });
-        }
-        else if (!response.criteriaMetId &&
-            monitorSteps.data.defaultMonitorStatusId &&
-            ((_s = monitor.currentMonitorStatusId) === null || _s === void 0 ? void 0 : _s.toString()) !==
-                monitorSteps.data.defaultMonitorStatusId.toString()) {
-            logger.debug(`${dataToProcess.monitorId.toString()} - No criteria met. Change to default status.`);
-            await MonitorIncident.checkOpenIncidentsAndCloseIfResolved({
-                monitorId: monitor.id,
-                autoResolveCriteriaInstanceIdIncidentIdsDictionary,
-                rootCause: "No monitoring criteria met. Change to default status.",
-                criteriaInstance: null, // no criteria met!
-                dataToProcess: dataToProcess,
-                evaluationSummary: evaluationSummary,
-            });
-            await MonitorAlert.checkOpenAlertsAndCloseIfResolved({
-                monitorId: monitor.id,
-                autoResolveCriteriaInstanceIdAlertIdsDictionary,
-                rootCause: "No monitoring criteria met. Change to default status.",
-                criteriaInstance: null, // no criteria met!
-                dataToProcess: dataToProcess,
-                evaluationSummary: evaluationSummary,
-            });
-            // get last monitor status timeline.
-            const lastMonitorStatusTimeline = await MonitorStatusTimelineService.findOneBy({
-                query: {
-                    monitorId: monitor.id,
-                    projectId: monitor.projectId,
-                },
-                select: {
-                    _id: true,
-                    monitorStatusId: true,
-                },
-                sort: {
-                    startsAt: SortOrder.Descending,
-                },
-                props: {
-                    isRoot: true,
-                },
-            });
-            if (lastMonitorStatusTimeline &&
-                lastMonitorStatusTimeline.monitorStatusId &&
-                lastMonitorStatusTimeline.monitorStatusId.toString() ===
-                    monitorSteps.data.defaultMonitorStatusId.toString()) {
-                /*
-                 * status is same as last status. do not create new status timeline.
-                 * do nothing! status is same as last status.
-                 */
+            if (response.criteriaMetId && response.rootCause) {
+                logger.debug(`${dataToProcess.monitorId.toString()} - Criteria met: ${response.criteriaMetId}`);
+                logger.debug(`${dataToProcess.monitorId.toString()} - Root cause: ${response.rootCause}`);
+                let telemetryQuery = undefined;
+                if (dataToProcess && dataToProcess.logQuery) {
+                    telemetryQuery = {
+                        telemetryQuery: dataToProcess.logQuery,
+                        telemetryType: TelemetryType.Log,
+                        metricViewData: null,
+                    };
+                    logger.debug(`${dataToProcess.monitorId.toString()} - Log query found.`);
+                }
+                if (dataToProcess &&
+                    dataToProcess.spanQuery) {
+                    telemetryQuery = {
+                        telemetryQuery: dataToProcess.spanQuery,
+                        telemetryType: TelemetryType.Trace,
+                        metricViewData: null,
+                    };
+                    logger.debug(`${dataToProcess.monitorId.toString()} - Span query found.`);
+                }
+                if (dataToProcess &&
+                    dataToProcess.metricViewConfig &&
+                    dataToProcess.startAndEndDate) {
+                    telemetryQuery = {
+                        telemetryQuery: null,
+                        telemetryType: TelemetryType.Metric,
+                        metricViewData: {
+                            startAndEndDate: dataToProcess.startAndEndDate ||
+                                null,
+                            queryConfigs: dataToProcess
+                                .metricViewConfig.queryConfigs,
+                            formulaConfigs: dataToProcess
+                                .metricViewConfig.formulaConfigs,
+                        },
+                    };
+                    logger.debug(`${dataToProcess.monitorId.toString()} - Span query found.`);
+                }
+                if (dataToProcess &&
+                    dataToProcess.exceptionQuery) {
+                    const exceptionResponse = dataToProcess;
+                    telemetryQuery = {
+                        telemetryQuery: exceptionResponse.exceptionQuery,
+                        telemetryType: TelemetryType.Exception,
+                        metricViewData: null,
+                    };
+                    logger.debug(`${dataToProcess.monitorId.toString()} - Exception query found.`);
+                }
+                const matchedCriteriaInstance = criteriaInstanceMap[response.criteriaMetId];
+                const monitorStatusTimelineChange = await MonitorStatusTimelineUtil.updateMonitorStatusTimeline({
+                    monitor: monitor,
+                    rootCause: response.rootCause,
+                    dataToProcess: dataToProcess,
+                    criteriaInstance: matchedCriteriaInstance,
+                    props: {
+                        telemetryQuery: telemetryQuery,
+                    },
+                });
+                if (monitorStatusTimelineChange) {
+                    const changedStatusName = await getMonitorStatusName(((_o = matchedCriteriaInstance.data) === null || _o === void 0 ? void 0 : _o.monitorStatusId) ||
+                        monitorStatusTimelineChange.monitorStatusId);
+                    evaluationSummary.events.push({
+                        type: "monitor-status-changed",
+                        title: "Monitor status updated",
+                        message: changedStatusName
+                            ? `Monitor status changed to "${changedStatusName}" because criteria "${((_p = matchedCriteriaInstance.data) === null || _p === void 0 ? void 0 : _p.name) || "Unnamed criteria"}" was met.`
+                            : `Monitor status changed because criteria "${((_q = matchedCriteriaInstance.data) === null || _q === void 0 ? void 0 : _q.name) || "Unnamed criteria"}" was met.`,
+                        relatedCriteriaId: (_r = matchedCriteriaInstance.data) === null || _r === void 0 ? void 0 : _r.id,
+                        at: OneUptimeDate.getCurrentDate(),
+                    });
+                }
+                await MonitorIncident.criteriaMetCreateIncidentsAndUpdateMonitorStatus({
+                    monitor: monitor,
+                    rootCause: response.rootCause,
+                    dataToProcess: dataToProcess,
+                    autoResolveCriteriaInstanceIdIncidentIdsDictionary,
+                    criteriaInstance: matchedCriteriaInstance,
+                    evaluationSummary: evaluationSummary,
+                    props: {
+                        telemetryQuery: telemetryQuery,
+                    },
+                    matchesPerSeries: response.perSeriesMatches,
+                });
+                await MonitorAlert.criteriaMetCreateAlertsAndUpdateMonitorStatus({
+                    monitor: monitor,
+                    rootCause: response.rootCause,
+                    dataToProcess: dataToProcess,
+                    autoResolveCriteriaInstanceIdAlertIdsDictionary,
+                    criteriaInstance: criteriaInstanceAlertMap[response.criteriaMetId],
+                    evaluationSummary: evaluationSummary,
+                    props: {
+                        telemetryQuery: telemetryQuery,
+                    },
+                    matchesPerSeries: response.perSeriesMatches,
+                });
             }
-            else {
-                // if no criteria is met then update monitor to default state.
-                const monitorStatusTimeline = new MonitorStatusTimeline();
-                monitorStatusTimeline.monitorId = monitor.id;
-                monitorStatusTimeline.monitorStatusId =
-                    monitorSteps.data.defaultMonitorStatusId;
-                monitorStatusTimeline.projectId = monitor.projectId;
-                monitorStatusTimeline.isOwnerNotified = true; // no need to notify owner as this is default status.
-                monitorStatusTimeline.statusChangeLog = JSON.parse(JSON.stringify(dataToProcess));
-                monitorStatusTimeline.rootCause =
-                    "No monitoring criteria met. Change to default status. ";
-                await MonitorStatusTimelineService.create({
-                    data: monitorStatusTimeline,
+            else if (!response.criteriaMetId &&
+                monitorSteps.data.defaultMonitorStatusId &&
+                ((_s = monitor.currentMonitorStatusId) === null || _s === void 0 ? void 0 : _s.toString()) !==
+                    monitorSteps.data.defaultMonitorStatusId.toString()) {
+                logger.debug(`${dataToProcess.monitorId.toString()} - No criteria met. Change to default status.`);
+                await MonitorIncident.checkOpenIncidentsAndCloseIfResolved({
+                    monitorId: monitor.id,
+                    autoResolveCriteriaInstanceIdIncidentIdsDictionary,
+                    rootCause: "No monitoring criteria met. Change to default status.",
+                    criteriaInstance: null, // no criteria met!
+                    dataToProcess: dataToProcess,
+                    evaluationSummary: evaluationSummary,
+                });
+                await MonitorAlert.checkOpenAlertsAndCloseIfResolved({
+                    monitorId: monitor.id,
+                    autoResolveCriteriaInstanceIdAlertIdsDictionary,
+                    rootCause: "No monitoring criteria met. Change to default status.",
+                    criteriaInstance: null, // no criteria met!
+                    dataToProcess: dataToProcess,
+                    evaluationSummary: evaluationSummary,
+                });
+                // get last monitor status timeline.
+                const lastMonitorStatusTimeline = await MonitorStatusTimelineService.findOneBy({
+                    query: {
+                        monitorId: monitor.id,
+                        projectId: monitor.projectId,
+                    },
+                    select: {
+                        _id: true,
+                        monitorStatusId: true,
+                    },
+                    sort: {
+                        startsAt: SortOrder.Descending,
+                    },
                     props: {
                         isRoot: true,
                     },
                 });
-                logger.debug(`${dataToProcess.monitorId.toString()} - Monitor status updated to default.`);
-                const defaultStatusName = await getMonitorStatusName(monitorSteps.data.defaultMonitorStatusId);
-                evaluationSummary.events.push({
-                    type: "monitor-status-changed",
-                    title: "Monitor status reverted",
-                    message: defaultStatusName
-                        ? `Monitor status reverted to "${defaultStatusName}" because no monitoring criteria were met.`
-                        : "Monitor status reverted to its default state because no monitoring criteria were met.",
-                    at: OneUptimeDate.getCurrentDate(),
-                });
+                if (lastMonitorStatusTimeline &&
+                    lastMonitorStatusTimeline.monitorStatusId &&
+                    lastMonitorStatusTimeline.monitorStatusId.toString() ===
+                        monitorSteps.data.defaultMonitorStatusId.toString()) {
+                    /*
+                     * status is same as last status. do not create new status timeline.
+                     * do nothing! status is same as last status.
+                     */
+                }
+                else {
+                    // if no criteria is met then update monitor to default state.
+                    const monitorStatusTimeline = new MonitorStatusTimeline();
+                    monitorStatusTimeline.monitorId = monitor.id;
+                    monitorStatusTimeline.monitorStatusId =
+                        monitorSteps.data.defaultMonitorStatusId;
+                    monitorStatusTimeline.projectId = monitor.projectId;
+                    monitorStatusTimeline.isOwnerNotified = true; // no need to notify owner as this is default status.
+                    monitorStatusTimeline.statusChangeLog = JSON.parse(JSON.stringify(dataToProcess));
+                    monitorStatusTimeline.rootCause =
+                        "No monitoring criteria met. Change to default status. ";
+                    await MonitorStatusTimelineService.create({
+                        data: monitorStatusTimeline,
+                        props: {
+                            isRoot: true,
+                        },
+                    });
+                    logger.debug(`${dataToProcess.monitorId.toString()} - Monitor status updated to default.`);
+                    const defaultStatusName = await getMonitorStatusName(monitorSteps.data.defaultMonitorStatusId);
+                    evaluationSummary.events.push({
+                        type: "monitor-status-changed",
+                        title: "Monitor status reverted",
+                        message: defaultStatusName
+                            ? `Monitor status reverted to "${defaultStatusName}" because no monitoring criteria were met.`
+                            : "Monitor status reverted to its default state because no monitoring criteria were met.",
+                        at: OneUptimeDate.getCurrentDate(),
+                    });
+                }
             }
+            await releaseMutex();
+            await persistLatestMonitorPayload();
+            MonitorLogUtil.saveMonitorLog({
+                monitorId: monitor.id,
+                projectId: monitor.projectId,
+                dataToProcess: dataToProcess,
+            });
+            return response;
         }
-        if (mutex) {
-            try {
-                await Semaphore.release(mutex);
-            }
-            catch (err) {
-                logger.error(err);
-            }
+        finally {
+            await releaseMutex();
         }
-        await persistLatestMonitorPayload();
-        MonitorLogUtil.saveMonitorLog({
-            monitorId: monitor.id,
-            projectId: monitor.projectId,
-            dataToProcess: dataToProcess,
-        });
-        return response;
     }
     static async checkProbeAgreement(input) {
         var _a, _b, _c, _d, _e, _f, _g;