@oneuptime/common 12.0.34 → 13.0.1

package/Server/Services/NetworkDeviceAutoImportRuleEngineService.ts

@@ -21,6 +21,8 @@ import MonitorType from "../../Types/Monitor/MonitorType";
 import {
   AutoImportRuleRunResult,
   MAX_MATCHED_IP_SAMPLE,
+  MAX_RUN_FAILURE_REASON_LENGTH,
+  MAX_RUN_FAILURE_REASONS,
 } from "../../Types/NetworkAutomation/RuleRunResult";
 import AutoImportRuleMatcher, {
   AutoImportHostEvaluation,
@@ -50,13 +52,18 @@ import logger, { LogAttributes } from "../Utils/Logger";
  * then apply themselves through NetworkDeviceService.onCreateSuccess's
  * existing rule chain, exactly as they do for a manual import.
  *
- * Two entry points, mirroring the label-rule engine:
+ * Three entry points, the first two mirroring the label-rule engine:
  *
  *   - processCompletedScan: the automatic path, called by the
  *     NetworkDeviceDiscovery worker for each Completed scan whose
  *     autoImportProcessedAt marker is NULL.
  *   - applyRuleToCompletedScans: the manual "Run Now" (and its dry run),
  *     applying ONE rule to the project's existing completed scans.
+ *   - rearmRecentScansForRuleChange: called when a rule is written, it
+ *     imports nothing itself — it clears the marker on the project's recent
+ *     results so the automatic path above evaluates them against the rule
+ *     that just changed, instead of the operator having to press Run Now
+ *     (issue #3487).
  *
  * Everything here is idempotent by construction: a device exists per
  * (project, address), recurring scans re-report the same hosts every
@@ -118,6 +125,35 @@ export const MAX_MONITORS_PER_AUTO_IMPORT_RUN: number = 500;
 // Scans one manual "Run Now" will read, oldest results last.
 export const MAX_SCANS_PER_AUTO_IMPORT_RULE_RUN: number = 100;
 
+/*
+ * Scans one rule write re-arms — see rearmRecentScansForRuleChange. The same
+ * ceiling as a manual run's, because it feeds the same work: a project whose
+ * subnets are swept by more fresh scans than this has its newest results
+ * re-armed and the rest left to "Run Now", rather than a rule save turning
+ * into an unbounded burst of writes.
+ */
+export const MAX_SCANS_REARMED_PER_RULE_WRITE: number = 100;
+
+/*
+ * How many monitor creates may fail IDENTICALLY, back to back with no success
+ * between them, before the run stops attempting any more.
+ *
+ * A monitor create fails for one of two kinds of reason. Per host — this
+ * device's name collides, this one raced another writer — where the next host
+ * is unaffected and trying it is right. Or systemic: the template's criteria
+ * name a monitor status that no longer exists, the plan's monitor limit is
+ * reached, the database is refusing writes. Those fail for the FIRST device
+ * and for all 500 after it, and the old engine could not tell the difference:
+ * it spent the whole per-run monitor budget re-proving one broken thing, then
+ * reported "Stopped at the run cap — run again to continue", which is an
+ * instruction to do it all again. That is OneUptime/oneuptime#3643.
+ *
+ * Ten consecutive failures carrying the SAME message, with no create
+ * succeeding in between, is not a run of bad luck. Any success resets it, so a
+ * genuinely per-host fault never trips this.
+ */
+export const MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES: number = 10;
+
 /*
  * Results older than this are stamped processed WITHOUT importing. A late
  * result can land on a reaper-Failed scan hours after its sweep ran (the
@@ -161,6 +197,28 @@ export interface ExistingMonitorProvisioningState {
   autoProvisionedKeys: Set<string>;
   manuallyMonitoredDeviceIds: Set<string>;
   attemptedProvisioningKeys: Set<string>;
+  /*
+   * Monitor template ids this run has already proved it cannot resolve —
+   * deleted, moved to another project, or no longer of type Network Device.
+   * That is a property of the RULE, settled the first time a host needs the
+   * template, and re-deciding it per host costs one wasted monitor budget slot
+   * and one log line per device in the estate.
+   */
+  unresolvableTemplateIds: Set<string>;
+  /*
+   * Consecutive monitor creates that failed with the same message and no
+   * success since — see MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES.
+   */
+  consecutiveCreateFailures: number;
+  lastCreateFailureReason: string;
+  /*
+   * Whether this run has EVER provisioned a monitor (or lost a race to
+   * somebody who did). One success is proof the create pipeline works, which
+   * is what disarms the systemic-failure guard for good.
+   */
+  hasProvisionedAnyMonitor: boolean;
+  // Set once this run gives up on provisioning monitors at all.
+  isProvisioningHalted: boolean;
 }
 
 export type ExistingMonitorsByProjectId = Map<
@@ -244,6 +302,11 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         },
         props: {
           isRoot: true,
+          /*
+           * Keep the stored payload for stampScan; the engine checks live
+           * inventory itself before evaluating these hosts.
+           */
+          ignoreHooks: true,
         },
       });
 
@@ -380,6 +443,123 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
     return result;
   }
 
+  /*
+   * The third way stored results become importable, alongside a new upload
+   * and a manual "Run Now": the RULES changed.
+   *
+   * The sweep is driven entirely by the marker — it looks only at scans whose
+   * autoImportProcessedAt is NULL — so results the engine has already
+   * evaluated are, as far as the worker is concerned, finished with. That is
+   * right for results evaluated against the rules the project HAS, and wrong
+   * the moment somebody writes a new one: the rule an operator just authored
+   * has nothing left to apply to until a scan reports again. In a project
+   * whose scans are one-shot, that is never — so a brand-new rule appears to
+   * do nothing at all and the operator is left pressing Run Now by hand,
+   * which is what OneUptime issue #3487 reported as "auto import rules have
+   * no schedule".
+   *
+   * Clearing the marker on the project's recent results hands them back to
+   * the sweep, which applies the FULL rule set to them within a minute —
+   * same pass, same lock, same idempotency as any other tick. Nothing is
+   * imported here: this method writes one column and no devices, so a rule
+   * save stays a rule save and the minutes of paced import work stay in the
+   * worker where a crash is resumable.
+   *
+   * Bounded to results still inside the engine's own freshness horizon
+   * (MAX_RESULT_AGE_IN_HOURS), which preserves exactly the guarantee that
+   * stamping rule-less projects exists to give: a first rule can never
+   * mass-import an estate discovered months ago. Older results remain Run
+   * Now's business, where "import these old hosts" is a deliberate click.
+   *
+   * Returns how many scans were re-armed. It may throw — the callers are rule
+   * writes, and they catch and log rather than fail an operator's save over a
+   * re-arm that can be had again from the next scan result or from Run Now.
+   */
+  @CaptureSpan()
+  public async rearmRecentScansForRuleChange(data: {
+    projectId: ObjectID;
+  }): Promise<number> {
+    const scanStubs: Array<NetworkDeviceDiscoveryScan> =
+      await NetworkDeviceDiscoveryScanService.findBy({
+        query: {
+          projectId: data.projectId,
+          /*
+           * A scan that is still sweeping counts, for the reason its partial
+           * results are importable at all: those hosts have been found.
+           */
+          status: QueryHelper.any(DISCOVERY_SCAN_IMPORTABLE_STATUSES),
+          /*
+           * Only STAMPED scans need re-arming. One whose marker is already
+           * NULL is queued for the next tick as it stands, and clearing it
+           * again would be a write that changes nothing.
+           */
+          autoImportProcessedAt: QueryHelper.notNull(),
+          /*
+           * Fresh results only — the horizon processCompletedScan enforces
+           * anyway, applied here so a rule save does not re-arm scans the
+           * sweep would only retire again (one pointless write per scan, and
+           * a "too old to auto-import" warning per scan in the log).
+           * completedAt is NULL while a scan is still sweeping, and those
+           * results are the freshest the project has.
+           */
+          completedAt: QueryHelper.greaterThanEqualToOrNull(
+            OneUptimeDate.getSomeHoursAgo(MAX_RESULT_AGE_IN_HOURS),
+          ),
+        },
+        select: {
+          _id: true,
+        },
+        // Newest results first, so a capped re-arm keeps the ones that matter.
+        sort: {
+          completedAt: SortOrder.Descending,
+        },
+        limit: MAX_SCANS_REARMED_PER_RULE_WRITE,
+        skip: 0,
+        props: {
+          isRoot: true,
+        },
+      });
+
+    for (const scanStub of scanStubs) {
+      /*
+       * The same hook-free single statement the ingest endpoint clears this
+       * marker with, and for the same reasons (see stampScan) — but
+       * deliberately WITHOUT refreshing updatedAt. On an In Progress scan
+       * that column is the "has this probe gone silent" clock the stale-scan
+       * reaper reads (Workers/Jobs/NetworkDeviceDiscovery/
+       * RequeueRecurringScans.ts); a rule save is not the probe saying
+       * something, and bumping it would postpone the rescue of a scan whose
+       * probe had already died.
+       *
+       * No compare-and-set, because the query above already excludes the row
+       * every racing writer is interested in: the sweep only ever reads scans
+       * whose marker is NULL, and these are the ones whose marker is set. The
+       * one interleaving left is ingest clearing the marker between the read
+       * and this write and a sweep consuming those new results — and that
+       * sweep loads its rules after the rule write that brought us here, so
+       * it applies them. Re-clearing behind it costs one more pass over
+       * results whose devices already exist.
+       */
+      await NetworkDeviceDiscoveryScanService.updateColumnsByIdWithoutHooks({
+        id: scanStub.id!,
+        // Cast: the model's JSON column makes DeepPartial recursion blow up.
+        data: {
+          autoImportProcessedAt: null,
+        } as unknown as QueryDeepPartialEntity<NetworkDeviceDiscoveryScan>,
+        skipUpdateDateColumn: true,
+      });
+    }
+
+    if (scanStubs.length > 0) {
+      logger.info(
+        `Auto-import: rules changed, so ${scanStubs.length} recent discovery scan result(s) were re-armed; the next sweep will apply the project's rules to them.`,
+        { projectId: data.projectId.toString() } as LogAttributes,
+      );
+    }
+
+    return scanStubs.length;
+  }
+
   /*
    * The manual path: apply ONE rule to the project's completed scans, newest
    * first — or, as a dry run, evaluate everything and write nothing. The
@@ -586,7 +766,8 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
       scanStubs.pop();
     }
 
-    for (const scanStub of scanStubs) {
+    for (let index: number = 0; index < scanStubs.length; index++) {
+      const scanStub: NetworkDeviceDiscoveryScan = scanStubs[index]!;
       const scan: NetworkDeviceDiscoveryScan | null =
         await NetworkDeviceDiscoveryScanService.findOneBy({
           query: {
@@ -605,7 +786,8 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             discoveredDevices: true,
             ...AUTO_IMPORT_SCAN_CREDENTIAL_SELECT,
           },
-          props: { isRoot: true },
+          // Preserve the raw scan rows for the compare-and-set write-back.
+          props: { isRoot: true, ignoreHooks: true },
         });
 
       // Re-queued or deleted since the stub query — nothing to evaluate.
@@ -639,8 +821,15 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         });
       }
 
-      // The device cap is shared across the whole run, dry or real.
+      /*
+       * The caps are shared across the whole run, dry or real. Stop OPENING
+       * further scans once they are spent — each one is a multi-megabyte
+       * jsonb read whose hosts this pass could not act on anyway — but say
+       * that scans were left unread, so the pending counters below are
+       * reported as a floor rather than as the estate's total.
+       */
       if (result.isTruncated) {
+        result.hasUnevaluatedScans = index < scanStubs.length - 1;
         break;
       }
 
@@ -678,18 +867,69 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
       monitorsSkippedAlreadyExisting: 0,
       monitorsSkippedUnsupportedHost: 0,
       monitorsFailed: 0,
+      deviceFailureReasons: [],
+      monitorFailureReasons: [],
+      monitorProvisioningHalted: false,
       isTruncated: false,
+      hostsPendingImport: 0,
+      monitorsPendingCreation: 0,
+      hasUnevaluatedScans: false,
       hasMoreScans: false,
       isDryRun: isDryRun,
       matchedIpAddressSample: [],
     };
   }
 
+  /*
+   * Keep the reason a create failed, so the run can say WHY it created
+   * nothing instead of pointing at server logs the operator of a self-hosted
+   * install may not be able to read. Deduplicated (500 identical failures are
+   * one reason) and capped, in both count and length, because this string
+   * ends up in a modal.
+   */
+  private recordFailureReason(reasons: Array<string>, error: unknown): void {
+    const message: string = this.describeError(error);
+
+    if (!message) {
+      return;
+    }
+
+    const trimmed: string = message.substring(0, MAX_RUN_FAILURE_REASON_LENGTH);
+
+    if (reasons.includes(trimmed)) {
+      return;
+    }
+
+    if (reasons.length >= MAX_RUN_FAILURE_REASONS) {
+      return;
+    }
+
+    reasons.push(trimmed);
+  }
+
+  /*
+   * The message an operator should see. Exception extends Error, so one arm
+   * covers both; anything else is stringified rather than dropped, because a
+   * thrown non-Error is exactly the case where the reason matters most.
+   */
+  private describeError(error: unknown): string {
+    if (error instanceof Error) {
+      return (error.message || "").replace(/\s+/g, " ").trim();
+    }
+
+    return String(error).replace(/\s+/g, " ").trim();
+  }
+
   private emptyExistingMonitorProvisioningState(): ExistingMonitorProvisioningState {
     return {
       autoProvisionedKeys: new Set(),
       manuallyMonitoredDeviceIds: new Set(),
       attemptedProvisioningKeys: new Set(),
+      unresolvableTemplateIds: new Set(),
+      consecutiveCreateFailures: 0,
+      lastCreateFailureReason: "",
+      hasProvisionedAnyMonitor: false,
+      isProvisioningHalted: false,
     };
   }
 
@@ -1108,6 +1348,18 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
 
     const createdIpAddresses: Array<string> = [];
 
+    /*
+     * What this pass counted as still-to-do, deduplicated.
+     *
+     * A stored result can list the same address twice (duplicate rows are
+     * normal enough that normalisation preserves them and the import path
+     * treats the second as already-registered), so counting rows rather than
+     * distinct work would report a remainder larger than the work that
+     * actually remains — in a number whose whole purpose is to be trusted.
+     */
+    const pendingImportAddresses: Set<string> = new Set<string>();
+    const pendingProvisioningKeys: Set<string> = new Set<string>();
+
     for (const host of hosts) {
       /*
        * The address becomes the device's varchar(100) hostname, on which
@@ -1180,22 +1432,13 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
       let deviceWasCreated: boolean = false;
 
       /*
-       * The frozen isAlreadyRegistered flag AND the live set: the flag is a
-       * point-in-time answer from the last upload, the set covers devices
-       * created since — including by this very run, which is what collapses
-       * duplicate rows and overlapping scans into one device.
+       * Current inventory is authoritative. The stored isAlreadyRegistered
+       * flag can outlive a deleted device and must not veto re-importing it.
+       * The map also includes devices created during this run, keeping
+       * duplicate rows and overlapping scans idempotent.
        */
-      if (host.isAlreadyRegistered || networkDevice) {
+      if (networkDevice) {
         result.hostsSkippedAlreadyRegistered++;
-
-        /*
-         * A stale scan can claim a deleted device is still registered. With
-         * no live row there is neither a safe binding nor enough intent to
-         * recreate it from old data, so preserve the established skip.
-         */
-        if (!networkDevice) {
-          continue;
-        }
       }
 
       if (result.matchedIpAddressSample.length < MAX_MATCHED_IP_SAMPLE) {
@@ -1206,8 +1449,25 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
 
       if (!networkDevice) {
         if (data.attempts.deviceCount >= MAX_DEVICES_PER_AUTO_IMPORT_RUN) {
+          /*
+           * Budget spent — stop WRITING, keep COUNTING. The rest of this
+           * loop is pure matching over a jsonb array that is already in
+           * memory against addresses primeExistingDevices already resolved,
+           * so finishing the scan costs nothing the cap exists to prevent,
+           * and it is the difference between "imported 500" and "imported
+           * 500 of 909, 409 still to go". A run that stopped mid-count
+           * reported the truncated numbers as if they were the estate,
+           * which is what OneUptime issue #3642 experienced as the
+           * remainder being silently dropped.
+           */
           result.isTruncated = true;
-          break;
+
+          if (!pendingImportAddresses.has(host.ipAddress)) {
+            pendingImportAddresses.add(host.ipAddress);
+            result.hostsPendingImport++;
+          }
+
+          continue;
         }
 
         if (data.isDryRun) {
@@ -1262,8 +1522,8 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         continue;
       }
 
-      const monitorsWereTruncated: boolean = await this.ensureMonitorsForDevice(
-        {
+      const monitorsLeftPending: Array<string> =
+        await this.ensureMonitorsForDevice({
           projectId: scan.projectId,
           host: host,
           networkDevice: networkDevice,
@@ -1273,18 +1533,45 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
           result: result,
           attempts: data.attempts,
           isDryRun: data.isDryRun,
-        },
-      );
+        });
 
-      if (monitorsWereTruncated) {
+      /*
+       * Same protocol as the device cap above: the monitor budget stops the
+       * creates, not the accounting. It also has its OWN cap, so an estate
+       * that is fully imported but unmonitored — the shape of issue #3642 —
+       * runs out of monitor budget with nothing pending on the device side,
+       * and the operator needs to be told how many devices are still
+       * waiting for a monitor.
+       */
+      for (const templateId of monitorsLeftPending) {
         result.isTruncated = true;
-        break;
+
+        const pendingKey: string =
+          NetworkDeviceAutoImportRuleEngineServiceClass.monitorProvisioningKey(
+            networkDevice.id!,
+            templateId,
+          );
+
+        if (!pendingProvisioningKeys.has(pendingKey)) {
+          pendingProvisioningKeys.add(pendingKey);
+          result.monitorsPendingCreation++;
+        }
       }
     }
 
     return createdIpAddresses;
   }
 
+  /*
+   * Reconcile one device against the templates the matched rules selected.
+   *
+   * Returns the template ids the run's budget left UNATTEMPTED — empty when
+   * every requested monitor was settled, whether created, failed, or skipped
+   * because monitoring already existed. A non-empty answer is what makes the
+   * run report itself truncated; the ids (rather than a bare count) are what
+   * let the caller count one device's leftover work once, however many
+   * duplicate rows the scan lists that host under.
+   */
   private async ensureMonitorsForDevice(data: {
     projectId: ObjectID;
     host: DiscoveredNetworkDevice;
@@ -1295,7 +1582,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
     result: AutoImportRuleRunResult;
     attempts: ImportAttemptBudget;
     isDryRun: boolean;
-  }): Promise<boolean> {
+  }): Promise<Array<string>> {
     /*
      * A Network Device monitor is fed by the device's polls, and nothing
      * polls a monitor-backed device — its bound monitor's status IS its
@@ -1312,7 +1599,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
       )
     ) {
       data.result.monitorsSkippedUnsupportedHost += data.templateIds.length;
-      return false;
+      return [];
     }
 
     if (!data.networkDevice.id) {
@@ -1321,7 +1608,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         `Auto-import: cannot provision monitor(s) for ${data.host.ipAddress} because the Network Device has no id.`,
         { projectId: data.projectId.toString() } as LogAttributes,
       );
-      return false;
+      return [];
     }
 
     const networkDeviceId: ObjectID = data.networkDevice.id;
@@ -1354,25 +1641,76 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
     }
 
     if (pendingTemplateIds.length === 0) {
-      return false;
+      return [];
     }
 
+    /*
+     * The project-wide snapshot already knows this device is monitored by
+     * hand, and that answer outranks any failure below: a device nobody was
+     * going to provision cannot be a monitor this run failed to create. Read
+     * from memory, so it stays ahead of the systemic verdicts.
+     */
     if (
       data.existingMonitors.manuallyMonitoredDeviceIds.has(
         networkDeviceIdString,
       )
     ) {
       data.result.monitorsSkippedAlreadyExisting += pendingTemplateIds.length;
-      return false;
+      return [];
+    }
+
+    /*
+     * Everything below here costs the run something — a monitor budget slot, a
+     * per-device search of the unindexed monitor step payload, a create round
+     * trip. None of it is worth spending on work this run has already proved
+     * cannot succeed, so the two systemic verdicts are settled next: a
+     * template this run could not load, and a create pipeline that has failed
+     * identically for every device it tried.
+     *
+     * Both still COUNT the monitors that are missing. "0 created, and here is
+     * why" is the answer; quietly reporting fewer missing monitors than there
+     * are would trade one confusing dialog for another.
+     */
+    const resolvableTemplateIds: Array<string> = [];
+
+    for (const templateId of pendingTemplateIds) {
+      if (
+        !data.existingMonitors.isProvisioningHalted &&
+        !data.existingMonitors.unresolvableTemplateIds.has(templateId)
+      ) {
+        resolvableTemplateIds.push(templateId);
+        continue;
+      }
+
+      /*
+       * Counted once and then settled. Marking the (device, template) key
+       * attempted is what keeps a host carried by two overlapping scans in the
+       * same run from being counted as two missing monitors — the same
+       * bookkeeping a real attempt does, for the same reason.
+       */
+      data.existingMonitors.attemptedProvisioningKeys.add(
+        NetworkDeviceAutoImportRuleEngineServiceClass.monitorProvisioningKey(
+          networkDeviceId,
+          templateId,
+        ),
+      );
+      data.result.monitorsFailed++;
+      data.result.monitorProvisioningHalted = true;
+    }
+
+    if (resolvableTemplateIds.length === 0) {
+      return [];
     }
 
     /*
      * Do not run the unindexed JSON race-check after this run's monitor-work
      * budget is already exhausted. The device remains unreconciled and the
-     * truncated marker protocol brings it back on the next bounded pass.
+     * truncated marker protocol brings it back on the next bounded pass —
+     * counted, so the operator is told how many devices are still waiting
+     * rather than left to infer it from a total that stopped early.
      */
     if (data.attempts.monitorCount >= MAX_MONITORS_PER_AUTO_IMPORT_RUN) {
-      return true;
+      return resolvableTemplateIds;
     }
 
     if (!data.isDryRun) {
@@ -1388,43 +1726,74 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         networkDeviceIdString,
       )
     ) {
-      data.result.monitorsSkippedAlreadyExisting += pendingTemplateIds.length;
-      return false;
+      data.result.monitorsSkippedAlreadyExisting +=
+        resolvableTemplateIds.length;
+      return [];
     }
 
-    for (const templateId of pendingTemplateIds) {
+    for (let index: number = 0; index < resolvableTemplateIds.length; index++) {
+      const templateId: string = resolvableTemplateIds[index]!;
       const key: string =
         NetworkDeviceAutoImportRuleEngineServiceClass.monitorProvisioningKey(
           networkDeviceId,
           templateId,
         );
 
+      /*
+       * A create earlier in THIS device's loop can be the one that trips the
+       * systemic verdict. The rest of its templates are then in exactly the
+       * position of every device after it.
+       */
+      if (data.existingMonitors.isProvisioningHalted) {
+        data.existingMonitors.attemptedProvisioningKeys.add(key);
+        data.result.monitorsFailed++;
+        data.result.monitorProvisioningHalted = true;
+        continue;
+      }
+
       /* The race-check may have found this key after the initial snapshot. */
       if (data.existingMonitors.autoProvisionedKeys.has(key)) {
         data.result.monitorsSkippedAlreadyExisting++;
         continue;
       }
 
+      /*
+       * Budget spent mid-device. Everything from here on is untouched, so
+       * that is exactly what is left pending for this device.
+       */
       if (data.attempts.monitorCount >= MAX_MONITORS_PER_AUTO_IMPORT_RUN) {
-        return true;
+        return resolvableTemplateIds.slice(index);
       }
 
       data.attempts.monitorCount++;
       data.existingMonitors.attemptedProvisioningKeys.add(key);
 
       /*
-       * Invalid or concurrently deleted templates are failed attempts too.
-       * Count them inside the same cap before looking up the cached template;
-       * otherwise one stale rule could walk and log an unbounded estate in a
-       * single API request or worker tick.
+       * A template the run could not load is a fault in the RULE, not in this
+       * host: the rule points at a template that has been deleted, moved to
+       * another project, or changed to a different monitor type since the rule
+       * was saved (the rule form validates all three, but only at save time).
+       * It will be just as unloadable for every other device, so the verdict is
+       * recorded once and every later host short-circuits above — instead of
+       * spending the run's whole monitor budget re-proving it, which is what
+       * turned this into "500 monitors failed, stopped at the run cap, run
+       * again to continue" in issue #3643.
        */
       const template: MonitorTemplate | undefined =
         data.monitorTemplates.get(templateId);
 
       if (!template) {
+        data.existingMonitors.unresolvableTemplateIds.add(templateId);
         data.result.monitorsFailed++;
+        data.result.monitorProvisioningHalted = true;
+        this.recordFailureReason(
+          data.result.monitorFailureReasons,
+          new Error(
+            "This rule's Monitor Template could not be loaded. It may have been deleted, moved to another project, or changed to a monitor type other than Network Device. Edit the rule and select a Network Device Monitor Template.",
+          ),
+        );
         logger.error(
-          `Auto-import: Network Device monitor template ${templateId} is missing, deleted, or not valid for project ${data.projectId.toString()}.`,
+          `Auto-import: Network Device monitor template ${templateId} is missing, deleted, or not valid for project ${data.projectId.toString()}. No monitors will be provisioned from it for the rest of this run.`,
           { projectId: data.projectId.toString() } as LogAttributes,
         );
         continue;
@@ -1452,6 +1821,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
 
         data.existingMonitors.autoProvisionedKeys.add(key);
         data.result.monitorsCreated++;
+        this.recordMonitorCreateSuccess(data.existingMonitors);
       } catch (error) {
         /*
          * The partial unique index is the final race backstop. If another
@@ -1483,19 +1853,87 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
 
           if (data.existingMonitors.autoProvisionedKeys.has(key)) {
             data.result.monitorsSkippedAlreadyExisting++;
+            /*
+             * Losing this race proves the create pipeline works — somebody
+             * else just used it — so it is not evidence of a systemic fault.
+             */
+            this.recordMonitorCreateSuccess(data.existingMonitors);
             continue;
           }
         }
 
         data.result.monitorsFailed++;
+        this.recordFailureReason(data.result.monitorFailureReasons, error);
         logger.error(
           `Auto-import: could not create monitor from template ${templateId} for Network Device ${networkDeviceId.toString()} (${data.host.ipAddress}): ${error}`,
           { projectId: data.projectId.toString() } as LogAttributes,
         );
+        this.recordMonitorCreateFailure({
+          state: data.existingMonitors,
+          result: data.result,
+          projectId: data.projectId,
+          error: error,
+        });
       }
     }
 
-    return false;
+    return [];
+  }
+
+  /*
+   * A create succeeded (or lost a race to another writer, which proves the
+   * same thing): whatever the last failures were, they were not systemic, and
+   * this run has now seen the create pipeline work at least once.
+   */
+  private recordMonitorCreateSuccess(
+    state: ExistingMonitorProvisioningState,
+  ): void {
+    state.consecutiveCreateFailures = 0;
+    state.lastCreateFailureReason = "";
+    state.hasProvisionedAnyMonitor = true;
+  }
+
+  /*
+   * Decide whether one monitor create failure is this host's problem or the
+   * whole run's. The bar is deliberately high, because giving up early on a
+   * per-host fault would silently cost an estate its monitors — the failure
+   * this fix exists to make visible. All three must hold: the run has never
+   * created a monitor, the last MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES
+   * attempts all failed, and they all failed with the SAME message.
+   */
+  private recordMonitorCreateFailure(data: {
+    state: ExistingMonitorProvisioningState;
+    result: AutoImportRuleRunResult;
+    projectId: ObjectID;
+    error: unknown;
+  }): void {
+    const reason: string = this.describeError(data.error);
+
+    if (reason && reason === data.state.lastCreateFailureReason) {
+      data.state.consecutiveCreateFailures++;
+    } else {
+      data.state.lastCreateFailureReason = reason;
+      data.state.consecutiveCreateFailures = 1;
+    }
+
+    if (data.state.hasProvisionedAnyMonitor) {
+      return;
+    }
+
+    if (
+      data.state.consecutiveCreateFailures <
+      MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES
+    ) {
+      return;
+    }
+
+    data.state.isProvisioningHalted = true;
+    data.result.monitorProvisioningHalted = true;
+
+    logger.error(
+      `Auto-import: ${MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES} Network Device monitor creates failed identically for project ${data.projectId.toString()} with none succeeding ("${reason}"). Treating this as a systemic failure and provisioning no further monitors this run.`,
+      { projectId: data.projectId.toString() } as LogAttributes,
+    );
   }
 
   /*
@@ -1580,6 +2018,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         createdDevice = await attemptCreate(buildFallbackDeviceName(data.host));
       } catch (secondError) {
         data.result.devicesFailed++;
+        this.recordFailureReason(data.result.deviceFailureReasons, secondError);
         logger.error(
           `Auto-import: could not create a device for ${data.host.ipAddress} (scan ${data.scan.id?.toString()}): ${firstError} / retry: ${secondError}`,
           { projectId: data.projectId.toString() } as LogAttributes,