@oneuptime/common 12.0.34 → 13.0.1

package/build/dist/Server/Services/NetworkDeviceAutoImportRuleEngineService.js

@@ -18,7 +18,7 @@ import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import MonitorType from "../../Types/Monitor/MonitorType";
-import { MAX_MATCHED_IP_SAMPLE, } from "../../Types/NetworkAutomation/RuleRunResult";
+import { MAX_MATCHED_IP_SAMPLE, MAX_RUN_FAILURE_REASON_LENGTH, MAX_RUN_FAILURE_REASONS, } from "../../Types/NetworkAutomation/RuleRunResult";
 import AutoImportRuleMatcher from "../../Utils/NetworkDiscovery/AutoImportRuleMatcher";
 import { buildFallbackDeviceName, buildNetworkDeviceFromDiscoveredHost, } from "../../Utils/NetworkDiscovery/DiscoveredDeviceBuilder";
 import { normalizeDiscoveredHosts } from "../../Utils/NetworkDiscovery/DiscoveredHostUtil";
@@ -40,13 +40,18 @@ import logger from "../Utils/Logger";
  * then apply themselves through NetworkDeviceService.onCreateSuccess's
  * existing rule chain, exactly as they do for a manual import.
  *
- * Two entry points, mirroring the label-rule engine:
+ * Three entry points, the first two mirroring the label-rule engine:
  *
  *   - processCompletedScan: the automatic path, called by the
  *     NetworkDeviceDiscovery worker for each Completed scan whose
  *     autoImportProcessedAt marker is NULL.
  *   - applyRuleToCompletedScans: the manual "Run Now" (and its dry run),
  *     applying ONE rule to the project's existing completed scans.
+ *   - rearmRecentScansForRuleChange: called when a rule is written, it
+ *     imports nothing itself — it clears the marker on the project's recent
+ *     results so the automatic path above evaluates them against the rule
+ *     that just changed, instead of the operator having to press Run Now
+ *     (issue #3487).
  *
  * Everything here is idempotent by construction: a device exists per
  * (project, address), recurring scans re-report the same hosts every
@@ -101,6 +106,33 @@ export const MAX_DEVICES_PER_AUTO_IMPORT_RUN = 500;
 export const MAX_MONITORS_PER_AUTO_IMPORT_RUN = 500;
 // Scans one manual "Run Now" will read, oldest results last.
 export const MAX_SCANS_PER_AUTO_IMPORT_RULE_RUN = 100;
+/*
+ * Scans one rule write re-arms — see rearmRecentScansForRuleChange. The same
+ * ceiling as a manual run's, because it feeds the same work: a project whose
+ * subnets are swept by more fresh scans than this has its newest results
+ * re-armed and the rest left to "Run Now", rather than a rule save turning
+ * into an unbounded burst of writes.
+ */
+export const MAX_SCANS_REARMED_PER_RULE_WRITE = 100;
+/*
+ * How many monitor creates may fail IDENTICALLY, back to back with no success
+ * between them, before the run stops attempting any more.
+ *
+ * A monitor create fails for one of two kinds of reason. Per host — this
+ * device's name collides, this one raced another writer — where the next host
+ * is unaffected and trying it is right. Or systemic: the template's criteria
+ * name a monitor status that no longer exists, the plan's monitor limit is
+ * reached, the database is refusing writes. Those fail for the FIRST device
+ * and for all 500 after it, and the old engine could not tell the difference:
+ * it spent the whole per-run monitor budget re-proving one broken thing, then
+ * reported "Stopped at the run cap — run again to continue", which is an
+ * instruction to do it all again. That is OneUptime/oneuptime#3643.
+ *
+ * Ten consecutive failures carrying the SAME message, with no create
+ * succeeding in between, is not a run of bad luck. Any success resets it, so a
+ * genuinely per-host fault never trips this.
+ */
+export const MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES = 10;
 /*
  * Results older than this are stamped processed WITHOUT importing. A late
  * result can land on a reaper-Failed scan hours after its sweep ran (the
@@ -172,6 +204,11 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
                 scannedHostCount: true, respondedHostCount: true, autoImportProcessedAt: true, discoveredDevices: true }, AUTO_IMPORT_SCAN_CREDENTIAL_SELECT),
             props: {
                 isRoot: true,
+                /*
+                 * Keep the stored payload for stampScan; the engine checks live
+                 * inventory itself before evaluating these hosts.
+                 */
+                ignoreHooks: true,
             },
         });
         /*
@@ -269,6 +306,110 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         });
         return result;
     }
+    /*
+     * The third way stored results become importable, alongside a new upload
+     * and a manual "Run Now": the RULES changed.
+     *
+     * The sweep is driven entirely by the marker — it looks only at scans whose
+     * autoImportProcessedAt is NULL — so results the engine has already
+     * evaluated are, as far as the worker is concerned, finished with. That is
+     * right for results evaluated against the rules the project HAS, and wrong
+     * the moment somebody writes a new one: the rule an operator just authored
+     * has nothing left to apply to until a scan reports again. In a project
+     * whose scans are one-shot, that is never — so a brand-new rule appears to
+     * do nothing at all and the operator is left pressing Run Now by hand,
+     * which is what OneUptime issue #3487 reported as "auto import rules have
+     * no schedule".
+     *
+     * Clearing the marker on the project's recent results hands them back to
+     * the sweep, which applies the FULL rule set to them within a minute —
+     * same pass, same lock, same idempotency as any other tick. Nothing is
+     * imported here: this method writes one column and no devices, so a rule
+     * save stays a rule save and the minutes of paced import work stay in the
+     * worker where a crash is resumable.
+     *
+     * Bounded to results still inside the engine's own freshness horizon
+     * (MAX_RESULT_AGE_IN_HOURS), which preserves exactly the guarantee that
+     * stamping rule-less projects exists to give: a first rule can never
+     * mass-import an estate discovered months ago. Older results remain Run
+     * Now's business, where "import these old hosts" is a deliberate click.
+     *
+     * Returns how many scans were re-armed. It may throw — the callers are rule
+     * writes, and they catch and log rather than fail an operator's save over a
+     * re-arm that can be had again from the next scan result or from Run Now.
+     */
+    async rearmRecentScansForRuleChange(data) {
+        const scanStubs = await NetworkDeviceDiscoveryScanService.findBy({
+            query: {
+                projectId: data.projectId,
+                /*
+                 * A scan that is still sweeping counts, for the reason its partial
+                 * results are importable at all: those hosts have been found.
+                 */
+                status: QueryHelper.any(DISCOVERY_SCAN_IMPORTABLE_STATUSES),
+                /*
+                 * Only STAMPED scans need re-arming. One whose marker is already
+                 * NULL is queued for the next tick as it stands, and clearing it
+                 * again would be a write that changes nothing.
+                 */
+                autoImportProcessedAt: QueryHelper.notNull(),
+                /*
+                 * Fresh results only — the horizon processCompletedScan enforces
+                 * anyway, applied here so a rule save does not re-arm scans the
+                 * sweep would only retire again (one pointless write per scan, and
+                 * a "too old to auto-import" warning per scan in the log).
+                 * completedAt is NULL while a scan is still sweeping, and those
+                 * results are the freshest the project has.
+                 */
+                completedAt: QueryHelper.greaterThanEqualToOrNull(OneUptimeDate.getSomeHoursAgo(MAX_RESULT_AGE_IN_HOURS)),
+            },
+            select: {
+                _id: true,
+            },
+            // Newest results first, so a capped re-arm keeps the ones that matter.
+            sort: {
+                completedAt: SortOrder.Descending,
+            },
+            limit: MAX_SCANS_REARMED_PER_RULE_WRITE,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        for (const scanStub of scanStubs) {
+            /*
+             * The same hook-free single statement the ingest endpoint clears this
+             * marker with, and for the same reasons (see stampScan) — but
+             * deliberately WITHOUT refreshing updatedAt. On an In Progress scan
+             * that column is the "has this probe gone silent" clock the stale-scan
+             * reaper reads (Workers/Jobs/NetworkDeviceDiscovery/
+             * RequeueRecurringScans.ts); a rule save is not the probe saying
+             * something, and bumping it would postpone the rescue of a scan whose
+             * probe had already died.
+             *
+             * No compare-and-set, because the query above already excludes the row
+             * every racing writer is interested in: the sweep only ever reads scans
+             * whose marker is NULL, and these are the ones whose marker is set. The
+             * one interleaving left is ingest clearing the marker between the read
+             * and this write and a sweep consuming those new results — and that
+             * sweep loads its rules after the rule write that brought us here, so
+             * it applies them. Re-clearing behind it costs one more pass over
+             * results whose devices already exist.
+             */
+            await NetworkDeviceDiscoveryScanService.updateColumnsByIdWithoutHooks({
+                id: scanStub.id,
+                // Cast: the model's JSON column makes DeepPartial recursion blow up.
+                data: {
+                    autoImportProcessedAt: null,
+                },
+                skipUpdateDateColumn: true,
+            });
+        }
+        if (scanStubs.length > 0) {
+            logger.info(`Auto-import: rules changed, so ${scanStubs.length} recent discovery scan result(s) were re-armed; the next sweep will apply the project's rules to them.`, { projectId: data.projectId.toString() });
+        }
+        return scanStubs.length;
+    }
     /*
      * The manual path: apply ONE rule to the project's completed scans, newest
      * first — or, as a dry run, evaluate everything and write nothing. The
@@ -428,7 +569,8 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         if (result.hasMoreScans) {
             scanStubs.pop();
         }
-        for (const scanStub of scanStubs) {
+        for (let index = 0; index < scanStubs.length; index++) {
+            const scanStub = scanStubs[index];
             const scan = await NetworkDeviceDiscoveryScanService.findOneBy({
                 query: {
                     _id: scanStub.id,
@@ -438,7 +580,8 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
                 select: Object.assign({ _id: true, projectId: true, status: true, completedAt: true, 
                     // The version token stampScan's compare-and-set uses; see there.
                     scannedHostCount: true, respondedHostCount: true, discoveredDevices: true }, AUTO_IMPORT_SCAN_CREDENTIAL_SELECT),
-                props: { isRoot: true },
+                // Preserve the raw scan rows for the compare-and-set write-back.
+                props: { isRoot: true, ignoreHooks: true },
             });
             // Re-queued or deleted since the stub query — nothing to evaluate.
             if (!scan) {
@@ -468,8 +611,15 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
                     leaveMarkerUnstamped: true,
                 });
             }
-            // The device cap is shared across the whole run, dry or real.
+            /*
+             * The caps are shared across the whole run, dry or real. Stop OPENING
+             * further scans once they are spent — each one is a multi-megabyte
+             * jsonb read whose hosts this pass could not act on anyway — but say
+             * that scans were left unread, so the pending counters below are
+             * reported as a floor rather than as the estate's total.
+             */
             if (result.isTruncated) {
+                result.hasUnevaluatedScans = index < scanStubs.length - 1;
                 break;
             }
             /*
@@ -503,17 +653,60 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             monitorsSkippedAlreadyExisting: 0,
             monitorsSkippedUnsupportedHost: 0,
             monitorsFailed: 0,
+            deviceFailureReasons: [],
+            monitorFailureReasons: [],
+            monitorProvisioningHalted: false,
             isTruncated: false,
+            hostsPendingImport: 0,
+            monitorsPendingCreation: 0,
+            hasUnevaluatedScans: false,
             hasMoreScans: false,
             isDryRun: isDryRun,
             matchedIpAddressSample: [],
         };
     }
+    /*
+     * Keep the reason a create failed, so the run can say WHY it created
+     * nothing instead of pointing at server logs the operator of a self-hosted
+     * install may not be able to read. Deduplicated (500 identical failures are
+     * one reason) and capped, in both count and length, because this string
+     * ends up in a modal.
+     */
+    recordFailureReason(reasons, error) {
+        const message = this.describeError(error);
+        if (!message) {
+            return;
+        }
+        const trimmed = message.substring(0, MAX_RUN_FAILURE_REASON_LENGTH);
+        if (reasons.includes(trimmed)) {
+            return;
+        }
+        if (reasons.length >= MAX_RUN_FAILURE_REASONS) {
+            return;
+        }
+        reasons.push(trimmed);
+    }
+    /*
+     * The message an operator should see. Exception extends Error, so one arm
+     * covers both; anything else is stringified rather than dropped, because a
+     * thrown non-Error is exactly the case where the reason matters most.
+     */
+    describeError(error) {
+        if (error instanceof Error) {
+            return (error.message || "").replace(/\s+/g, " ").trim();
+        }
+        return String(error).replace(/\s+/g, " ").trim();
+    }
     emptyExistingMonitorProvisioningState() {
         return {
             autoProvisionedKeys: new Set(),
             manuallyMonitoredDeviceIds: new Set(),
             attemptedProvisioningKeys: new Set(),
+            unresolvableTemplateIds: new Set(),
+            consecutiveCreateFailures: 0,
+            lastCreateFailureReason: "",
+            hasProvisionedAnyMonitor: false,
+            isProvisioningHalted: false,
         };
     }
     /*
@@ -825,6 +1018,17 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             into: data.existingDevices,
         });
         const createdIpAddresses = [];
+        /*
+         * What this pass counted as still-to-do, deduplicated.
+         *
+         * A stored result can list the same address twice (duplicate rows are
+         * normal enough that normalisation preserves them and the import path
+         * treats the second as already-registered), so counting rows rather than
+         * distinct work would report a remainder larger than the work that
+         * actually remains — in a number whose whole purpose is to be trusted.
+         */
+        const pendingImportAddresses = new Set();
+        const pendingProvisioningKeys = new Set();
         for (const host of hosts) {
             /*
              * The address becomes the device's varchar(100) hostname, on which
@@ -878,21 +1082,13 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             let networkDevice = data.existingDevices.get(host.ipAddress);
             let deviceWasCreated = false;
             /*
-             * The frozen isAlreadyRegistered flag AND the live set: the flag is a
-             * point-in-time answer from the last upload, the set covers devices
-             * created since — including by this very run, which is what collapses
-             * duplicate rows and overlapping scans into one device.
+             * Current inventory is authoritative. The stored isAlreadyRegistered
+             * flag can outlive a deleted device and must not veto re-importing it.
+             * The map also includes devices created during this run, keeping
+             * duplicate rows and overlapping scans idempotent.
              */
-            if (host.isAlreadyRegistered || networkDevice) {
+            if (networkDevice) {
                 result.hostsSkippedAlreadyRegistered++;
-                /*
-                 * A stale scan can claim a deleted device is still registered. With
-                 * no live row there is neither a safe binding nor enough intent to
-                 * recreate it from old data, so preserve the established skip.
-                 */
-                if (!networkDevice) {
-                    continue;
-                }
             }
             if (result.matchedIpAddressSample.length < MAX_MATCHED_IP_SAMPLE) {
                 if (!result.matchedIpAddressSample.includes(host.ipAddress)) {
@@ -901,8 +1097,23 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             }
             if (!networkDevice) {
                 if (data.attempts.deviceCount >= MAX_DEVICES_PER_AUTO_IMPORT_RUN) {
+                    /*
+                     * Budget spent — stop WRITING, keep COUNTING. The rest of this
+                     * loop is pure matching over a jsonb array that is already in
+                     * memory against addresses primeExistingDevices already resolved,
+                     * so finishing the scan costs nothing the cap exists to prevent,
+                     * and it is the difference between "imported 500" and "imported
+                     * 500 of 909, 409 still to go". A run that stopped mid-count
+                     * reported the truncated numbers as if they were the estate,
+                     * which is what OneUptime issue #3642 experienced as the
+                     * remainder being silently dropped.
+                     */
                     result.isTruncated = true;
-                    break;
+                    if (!pendingImportAddresses.has(host.ipAddress)) {
+                        pendingImportAddresses.add(host.ipAddress);
+                        result.hostsPendingImport++;
+                    }
+                    continue;
                 }
                 if (data.isDryRun) {
                     /*
@@ -935,7 +1146,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             if (templateIds.length === 0) {
                 continue;
             }
-            const monitorsWereTruncated = await this.ensureMonitorsForDevice({
+            const monitorsLeftPending = await this.ensureMonitorsForDevice({
                 projectId: scan.projectId,
                 host: host,
                 networkDevice: networkDevice,
@@ -946,13 +1157,35 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
                 attempts: data.attempts,
                 isDryRun: data.isDryRun,
             });
-            if (monitorsWereTruncated) {
+            /*
+             * Same protocol as the device cap above: the monitor budget stops the
+             * creates, not the accounting. It also has its OWN cap, so an estate
+             * that is fully imported but unmonitored — the shape of issue #3642 —
+             * runs out of monitor budget with nothing pending on the device side,
+             * and the operator needs to be told how many devices are still
+             * waiting for a monitor.
+             */
+            for (const templateId of monitorsLeftPending) {
                 result.isTruncated = true;
-                break;
+                const pendingKey = NetworkDeviceAutoImportRuleEngineServiceClass.monitorProvisioningKey(networkDevice.id, templateId);
+                if (!pendingProvisioningKeys.has(pendingKey)) {
+                    pendingProvisioningKeys.add(pendingKey);
+                    result.monitorsPendingCreation++;
+                }
             }
         }
         return createdIpAddresses;
     }
+    /*
+     * Reconcile one device against the templates the matched rules selected.
+     *
+     * Returns the template ids the run's budget left UNATTEMPTED — empty when
+     * every requested monitor was settled, whether created, failed, or skipped
+     * because monitoring already existed. A non-empty answer is what makes the
+     * run report itself truncated; the ids (rather than a bare count) are what
+     * let the caller count one device's leftover work once, however many
+     * duplicate rows the scan lists that host under.
+     */
     async ensureMonitorsForDevice(data) {
         /*
          * A Network Device monitor is fed by the device's polls, and nothing
@@ -966,12 +1199,12 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
          */
         if (NetworkDeviceMonitoringMethodUtil.isMonitorBacked(data.networkDevice.monitoringMethod)) {
             data.result.monitorsSkippedUnsupportedHost += data.templateIds.length;
-            return false;
+            return [];
         }
         if (!data.networkDevice.id) {
             data.result.monitorsFailed += data.templateIds.length;
             logger.error(`Auto-import: cannot provision monitor(s) for ${data.host.ipAddress} because the Network Device has no id.`, { projectId: data.projectId.toString() });
-            return false;
+            return [];
         }
         const networkDeviceId = data.networkDevice.id;
         const networkDeviceIdString = networkDeviceId.toString();
@@ -994,19 +1227,59 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             pendingTemplateIds.push(templateId);
         }
         if (pendingTemplateIds.length === 0) {
-            return false;
+            return [];
         }
+        /*
+         * The project-wide snapshot already knows this device is monitored by
+         * hand, and that answer outranks any failure below: a device nobody was
+         * going to provision cannot be a monitor this run failed to create. Read
+         * from memory, so it stays ahead of the systemic verdicts.
+         */
         if (data.existingMonitors.manuallyMonitoredDeviceIds.has(networkDeviceIdString)) {
             data.result.monitorsSkippedAlreadyExisting += pendingTemplateIds.length;
-            return false;
+            return [];
+        }
+        /*
+         * Everything below here costs the run something — a monitor budget slot, a
+         * per-device search of the unindexed monitor step payload, a create round
+         * trip. None of it is worth spending on work this run has already proved
+         * cannot succeed, so the two systemic verdicts are settled next: a
+         * template this run could not load, and a create pipeline that has failed
+         * identically for every device it tried.
+         *
+         * Both still COUNT the monitors that are missing. "0 created, and here is
+         * why" is the answer; quietly reporting fewer missing monitors than there
+         * are would trade one confusing dialog for another.
+         */
+        const resolvableTemplateIds = [];
+        for (const templateId of pendingTemplateIds) {
+            if (!data.existingMonitors.isProvisioningHalted &&
+                !data.existingMonitors.unresolvableTemplateIds.has(templateId)) {
+                resolvableTemplateIds.push(templateId);
+                continue;
+            }
+            /*
+             * Counted once and then settled. Marking the (device, template) key
+             * attempted is what keeps a host carried by two overlapping scans in the
+             * same run from being counted as two missing monitors — the same
+             * bookkeeping a real attempt does, for the same reason.
+             */
+            data.existingMonitors.attemptedProvisioningKeys.add(NetworkDeviceAutoImportRuleEngineServiceClass.monitorProvisioningKey(networkDeviceId, templateId));
+            data.result.monitorsFailed++;
+            data.result.monitorProvisioningHalted = true;
+        }
+        if (resolvableTemplateIds.length === 0) {
+            return [];
         }
         /*
          * Do not run the unindexed JSON race-check after this run's monitor-work
          * budget is already exhausted. The device remains unreconciled and the
-         * truncated marker protocol brings it back on the next bounded pass.
+         * truncated marker protocol brings it back on the next bounded pass —
+         * counted, so the operator is told how many devices are still waiting
+         * rather than left to infer it from a total that stopped early.
          */
         if (data.attempts.monitorCount >= MAX_MONITORS_PER_AUTO_IMPORT_RUN) {
-            return true;
+            return resolvableTemplateIds;
         }
         if (!data.isDryRun) {
             await this.refreshExistingMonitorsForDevice({
@@ -1016,31 +1289,56 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             });
         }
         if (data.existingMonitors.manuallyMonitoredDeviceIds.has(networkDeviceIdString)) {
-            data.result.monitorsSkippedAlreadyExisting += pendingTemplateIds.length;
-            return false;
+            data.result.monitorsSkippedAlreadyExisting +=
+                resolvableTemplateIds.length;
+            return [];
         }
-        for (const templateId of pendingTemplateIds) {
+        for (let index = 0; index < resolvableTemplateIds.length; index++) {
+            const templateId = resolvableTemplateIds[index];
             const key = NetworkDeviceAutoImportRuleEngineServiceClass.monitorProvisioningKey(networkDeviceId, templateId);
+            /*
+             * A create earlier in THIS device's loop can be the one that trips the
+             * systemic verdict. The rest of its templates are then in exactly the
+             * position of every device after it.
+             */
+            if (data.existingMonitors.isProvisioningHalted) {
+                data.existingMonitors.attemptedProvisioningKeys.add(key);
+                data.result.monitorsFailed++;
+                data.result.monitorProvisioningHalted = true;
+                continue;
+            }
             /* The race-check may have found this key after the initial snapshot. */
             if (data.existingMonitors.autoProvisionedKeys.has(key)) {
                 data.result.monitorsSkippedAlreadyExisting++;
                 continue;
             }
+            /*
+             * Budget spent mid-device. Everything from here on is untouched, so
+             * that is exactly what is left pending for this device.
+             */
             if (data.attempts.monitorCount >= MAX_MONITORS_PER_AUTO_IMPORT_RUN) {
-                return true;
+                return resolvableTemplateIds.slice(index);
             }
             data.attempts.monitorCount++;
             data.existingMonitors.attemptedProvisioningKeys.add(key);
             /*
-             * Invalid or concurrently deleted templates are failed attempts too.
-             * Count them inside the same cap before looking up the cached template;
-             * otherwise one stale rule could walk and log an unbounded estate in a
-             * single API request or worker tick.
+             * A template the run could not load is a fault in the RULE, not in this
+             * host: the rule points at a template that has been deleted, moved to
+             * another project, or changed to a different monitor type since the rule
+             * was saved (the rule form validates all three, but only at save time).
+             * It will be just as unloadable for every other device, so the verdict is
+             * recorded once and every later host short-circuits above — instead of
+             * spending the run's whole monitor budget re-proving it, which is what
+             * turned this into "500 monitors failed, stopped at the run cap, run
+             * again to continue" in issue #3643.
              */
             const template = data.monitorTemplates.get(templateId);
             if (!template) {
+                data.existingMonitors.unresolvableTemplateIds.add(templateId);
                 data.result.monitorsFailed++;
-                logger.error(`Auto-import: Network Device monitor template ${templateId} is missing, deleted, or not valid for project ${data.projectId.toString()}.`, { projectId: data.projectId.toString() });
+                data.result.monitorProvisioningHalted = true;
+                this.recordFailureReason(data.result.monitorFailureReasons, new Error("This rule's Monitor Template could not be loaded. It may have been deleted, moved to another project, or changed to a monitor type other than Network Device. Edit the rule and select a Network Device Monitor Template."));
+                logger.error(`Auto-import: Network Device monitor template ${templateId} is missing, deleted, or not valid for project ${data.projectId.toString()}. No monitors will be provisioned from it for the rest of this run.`, { projectId: data.projectId.toString() });
                 continue;
             }
             if (data.isDryRun) {
@@ -1062,6 +1360,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
                 });
                 data.existingMonitors.autoProvisionedKeys.add(key);
                 data.result.monitorsCreated++;
+                this.recordMonitorCreateSuccess(data.existingMonitors);
             }
             catch (error) {
                 /*
@@ -1090,14 +1389,64 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
                     this.recordExistingMonitor(data.existingMonitors, createdMeanwhile);
                     if (data.existingMonitors.autoProvisionedKeys.has(key)) {
                         data.result.monitorsSkippedAlreadyExisting++;
+                        /*
+                         * Losing this race proves the create pipeline works — somebody
+                         * else just used it — so it is not evidence of a systemic fault.
+                         */
+                        this.recordMonitorCreateSuccess(data.existingMonitors);
                         continue;
                     }
                 }
                 data.result.monitorsFailed++;
+                this.recordFailureReason(data.result.monitorFailureReasons, error);
                 logger.error(`Auto-import: could not create monitor from template ${templateId} for Network Device ${networkDeviceId.toString()} (${data.host.ipAddress}): ${error}`, { projectId: data.projectId.toString() });
+                this.recordMonitorCreateFailure({
+                    state: data.existingMonitors,
+                    result: data.result,
+                    projectId: data.projectId,
+                    error: error,
+                });
             }
         }
-        return false;
+        return [];
+    }
+    /*
+     * A create succeeded (or lost a race to another writer, which proves the
+     * same thing): whatever the last failures were, they were not systemic, and
+     * this run has now seen the create pipeline work at least once.
+     */
+    recordMonitorCreateSuccess(state) {
+        state.consecutiveCreateFailures = 0;
+        state.lastCreateFailureReason = "";
+        state.hasProvisionedAnyMonitor = true;
+    }
+    /*
+     * Decide whether one monitor create failure is this host's problem or the
+     * whole run's. The bar is deliberately high, because giving up early on a
+     * per-host fault would silently cost an estate its monitors — the failure
+     * this fix exists to make visible. All three must hold: the run has never
+     * created a monitor, the last MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES
+     * attempts all failed, and they all failed with the SAME message.
+     */
+    recordMonitorCreateFailure(data) {
+        const reason = this.describeError(data.error);
+        if (reason && reason === data.state.lastCreateFailureReason) {
+            data.state.consecutiveCreateFailures++;
+        }
+        else {
+            data.state.lastCreateFailureReason = reason;
+            data.state.consecutiveCreateFailures = 1;
+        }
+        if (data.state.hasProvisionedAnyMonitor) {
+            return;
+        }
+        if (data.state.consecutiveCreateFailures <
+            MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES) {
+            return;
+        }
+        data.state.isProvisioningHalted = true;
+        data.result.monitorProvisioningHalted = true;
+        logger.error(`Auto-import: ${MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES} Network Device monitor creates failed identically for project ${data.projectId.toString()} with none succeeding ("${reason}"). Treating this as a systemic failure and provisioning no further monitors this run.`, { projectId: data.projectId.toString() });
     }
     /*
      * One host -> one device, with the collision protocol the estate needs:
@@ -1160,6 +1509,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             }
             catch (secondError) {
                 data.result.devicesFailed++;
+                this.recordFailureReason(data.result.deviceFailureReasons, secondError);
                 logger.error(`Auto-import: could not create a device for ${data.host.ipAddress} (scan ${(_a = data.scan.id) === null || _a === void 0 ? void 0 : _a.toString()}): ${firstError} / retry: ${secondError}`, { projectId: data.projectId.toString() });
                 return { device: null, wasCreated: false };
             }
@@ -1257,6 +1607,12 @@ __decorate([
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
 ], NetworkDeviceAutoImportRuleEngineServiceClass.prototype, "processCompletedScan", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], NetworkDeviceAutoImportRuleEngineServiceClass.prototype, "rearmRecentScansForRuleChange", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),