@oneuptime/common 12.0.33 → 13.0.0

package/Server/Services/NetworkDeviceAutoImportRuleEngineService.ts

@@ -21,6 +21,8 @@ import MonitorType from "../../Types/Monitor/MonitorType";
 import {
   AutoImportRuleRunResult,
   MAX_MATCHED_IP_SAMPLE,
+  MAX_RUN_FAILURE_REASON_LENGTH,
+  MAX_RUN_FAILURE_REASONS,
 } from "../../Types/NetworkAutomation/RuleRunResult";
 import AutoImportRuleMatcher, {
   AutoImportHostEvaluation,
@@ -31,6 +33,7 @@ import {
   buildNetworkDeviceFromDiscoveredHost,
 } from "../../Utils/NetworkDiscovery/DiscoveredDeviceBuilder";
 import { normalizeDiscoveredHosts } from "../../Utils/NetworkDiscovery/DiscoveredHostUtil";
+import { DISCOVERY_SCAN_IMPORTABLE_STATUSES } from "../../Utils/NetworkDiscovery/DiscoveryScanStatus";
 import { NetworkDeviceMonitoringMethodUtil } from "../../Types/NetworkDevice/NetworkDeviceMonitoringMethod";
 import NetworkDeviceMonitorTemplateUtil from "../../Utils/Monitor/NetworkDeviceMonitorTemplateUtil";
 import Semaphore, { SemaphoreMutex } from "../Infrastructure/Semaphore";
@@ -41,7 +44,8 @@ import logger, { LogAttributes } from "../Utils/Logger";
 
 /*
  * The engine behind network device auto-import rules (issue #3378): reads a
- * completed discovery scan's results, asks the project's rules which hosts
+ * discovery scan's results — finished or still arriving from a running sweep,
+ * see DISCOVERY_SCAN_IMPORTABLE_STATUSES — asks the project's rules which hosts
  * import, and creates the Network Devices the manual Review dialog would
  * have — through the same builder, so a rule-imported host and a
  * hand-imported host are the same device. Site assignment, owners and labels
@@ -116,6 +120,26 @@ export const MAX_MONITORS_PER_AUTO_IMPORT_RUN: number = 500;
 // Scans one manual "Run Now" will read, oldest results last.
 export const MAX_SCANS_PER_AUTO_IMPORT_RULE_RUN: number = 100;
 
+/*
+ * How many monitor creates may fail IDENTICALLY, back to back with no success
+ * between them, before the run stops attempting any more.
+ *
+ * A monitor create fails for one of two kinds of reason. Per host — this
+ * device's name collides, this one raced another writer — where the next host
+ * is unaffected and trying it is right. Or systemic: the template's criteria
+ * name a monitor status that no longer exists, the plan's monitor limit is
+ * reached, the database is refusing writes. Those fail for the FIRST device
+ * and for all 500 after it, and the old engine could not tell the difference:
+ * it spent the whole per-run monitor budget re-proving one broken thing, then
+ * reported "Stopped at the run cap — run again to continue", which is an
+ * instruction to do it all again. That is OneUptime/oneuptime#3643.
+ *
+ * Ten consecutive failures carrying the SAME message, with no create
+ * succeeding in between, is not a run of bad luck. Any success resets it, so a
+ * genuinely per-host fault never trips this.
+ */
+export const MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES: number = 10;
+
 /*
  * Results older than this are stamped processed WITHOUT importing. A late
  * result can land on a reaper-Failed scan hours after its sweep ran (the
@@ -159,6 +183,28 @@ export interface ExistingMonitorProvisioningState {
   autoProvisionedKeys: Set<string>;
   manuallyMonitoredDeviceIds: Set<string>;
   attemptedProvisioningKeys: Set<string>;
+  /*
+   * Monitor template ids this run has already proved it cannot resolve —
+   * deleted, moved to another project, or no longer of type Network Device.
+   * That is a property of the RULE, settled the first time a host needs the
+   * template, and re-deciding it per host costs one wasted monitor budget slot
+   * and one log line per device in the estate.
+   */
+  unresolvableTemplateIds: Set<string>;
+  /*
+   * Consecutive monitor creates that failed with the same message and no
+   * success since — see MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES.
+   */
+  consecutiveCreateFailures: number;
+  lastCreateFailureReason: string;
+  /*
+   * Whether this run has EVER provisioned a monitor (or lost a race to
+   * somebody who did). One success is proof the create pipeline works, which
+   * is what disarms the systemic-failure guard for good.
+   */
+  hasProvisionedAnyMonitor: boolean;
+  // Set once this run gives up on provisioning monitors at all.
+  isProvisioningHalted: boolean;
 }
 
 export type ExistingMonitorsByProjectId = Map<
@@ -182,24 +228,32 @@ export type ImportAttemptBudgetsByProjectId = Map<string, ImportAttemptBudget>;
 
 class NetworkDeviceAutoImportRuleEngineServiceClass {
   /**
-   * The automatic path: apply the project's enabled rules to one Completed
-   * scan, then stamp the scan's autoImportProcessedAt marker.
+   * The automatic path: apply the project's enabled rules to one scan's
+   * stored results, then stamp the scan's autoImportProcessedAt marker.
+   *
+   * The scan may still be SWEEPING — see
+   * DISCOVERY_SCAN_IMPORTABLE_STATUSES. Its partial results are as real as
+   * a finished scan's (the probe found those hosts; it simply has more of the
+   * range to cover), and waiting for the whole sweep is what left 527
+   * discovered switches unimportable for a day in OneUptime issue #3599. The
+   * name is kept because it is the automatic path's identity across the
+   * worker, its tests and its logs.
    *
    * Returns null when there was nothing to do (scan gone, superseded,
    * already processed, no import rules, or results too old) and the run's
    * counters otherwise. Throws only on unexpected failures — the caller
    * (the worker sweep) isolates those per scan.
    *
-   * The marker protocol, and why every stamp is a compare-and-set on
-   * (status, completedAt): the ingest endpoint clears the marker in the same
-   * write that stores new results, so "marker is NULL" always means "the
-   * results now on the row have not been processed". If new results land
-   * while this pass is reading the old ones, the CAS misses, the marker
-   * stays NULL, and the next tick processes the new upload — the stamp can
-   * never retire results it did not see, and the jsonb write-back can never
-   * clobber a newer host list. A truncated pass that created something
-   * stamps nothing on purpose: the NULL marker is what makes the next tick
-   * resume. (A truncated pass where every create FAILED stamps anyway — see
+   * The marker protocol, and why every stamp is a compare-and-set on the run
+   * state this pass read: the ingest endpoint clears the marker in the same
+   * write that stores new results — partial uploads included — so "marker is
+   * NULL" always means "the results now on the row have not been processed".
+   * If new results land while this pass is reading the old ones, the CAS
+   * misses, the marker stays NULL, and the next tick processes the new upload
+   * — the stamp can never retire results it did not see, and the jsonb
+   * write-back can never clobber a newer host list. A truncated pass that
+   * created something stamps nothing on purpose: the NULL marker is what
+   * makes the next tick resume. (A truncated pass where every create FAILED stamps anyway — see
    * the zero-progress note at the stamp below.)
    */
   @CaptureSpan()
@@ -213,19 +267,32 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
       await NetworkDeviceDiscoveryScanService.findOneBy({
         query: {
           _id: data.scanId,
-          status: "Completed",
+          status: QueryHelper.any(DISCOVERY_SCAN_IMPORTABLE_STATUSES),
         },
         select: {
           _id: true,
           projectId: true,
           status: true,
           completedAt: true,
+          /*
+           * The two counters a running scan's partial uploads move. They are
+           * not read for anything; they are the version token stampScan's
+           * compare-and-set uses to tell "the results I evaluated" from "the
+           * results that landed while I was evaluating".
+           */
+          scannedHostCount: true,
+          respondedHostCount: true,
           autoImportProcessedAt: true,
           discoveredDevices: true,
           ...AUTO_IMPORT_SCAN_CREDENTIAL_SELECT,
         },
         props: {
           isRoot: true,
+          /*
+           * Keep the stored payload for stampScan; the engine checks live
+           * inventory itself before evaluating these hosts.
+           */
+          ignoreHooks: true,
         },
       });
 
@@ -538,11 +605,21 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
       await NetworkDeviceDiscoveryScanService.findBy({
         query: {
           projectId: data.projectId,
-          status: "Completed",
+          /*
+           * A scan that is still sweeping counts too: its partial results are
+           * already on the row, and "Run Now found 25 hosts while 527 sat in
+           * the Devices list" was exactly the report (OneUptime issue #3599).
+           */
+          status: QueryHelper.any(DISCOVERY_SCAN_IMPORTABLE_STATUSES),
         },
         select: {
           _id: true,
         },
+        /*
+         * Newest results first. An in-progress scan has no completedAt, and
+         * Postgres orders NULLs first under DESC — which is the right place
+         * for it: its results are the freshest the project has.
+         */
         sort: {
           completedAt: SortOrder.Descending,
         },
@@ -558,23 +635,28 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
       scanStubs.pop();
     }
 
-    for (const scanStub of scanStubs) {
+    for (let index: number = 0; index < scanStubs.length; index++) {
+      const scanStub: NetworkDeviceDiscoveryScan = scanStubs[index]!;
       const scan: NetworkDeviceDiscoveryScan | null =
         await NetworkDeviceDiscoveryScanService.findOneBy({
           query: {
             _id: scanStub.id!,
             projectId: data.projectId,
-            status: "Completed",
+            status: QueryHelper.any(DISCOVERY_SCAN_IMPORTABLE_STATUSES),
           },
           select: {
             _id: true,
             projectId: true,
             status: true,
             completedAt: true,
+            // The version token stampScan's compare-and-set uses; see there.
+            scannedHostCount: true,
+            respondedHostCount: true,
             discoveredDevices: true,
             ...AUTO_IMPORT_SCAN_CREDENTIAL_SELECT,
           },
-          props: { isRoot: true },
+          // Preserve the raw scan rows for the compare-and-set write-back.
+          props: { isRoot: true, ignoreHooks: true },
         });
 
       // Re-queued or deleted since the stub query — nothing to evaluate.
@@ -608,8 +690,15 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         });
       }
 
-      // The device cap is shared across the whole run, dry or real.
+      /*
+       * The caps are shared across the whole run, dry or real. Stop OPENING
+       * further scans once they are spent — each one is a multi-megabyte
+       * jsonb read whose hosts this pass could not act on anyway — but say
+       * that scans were left unread, so the pending counters below are
+       * reported as a floor rather than as the estate's total.
+       */
       if (result.isTruncated) {
+        result.hasUnevaluatedScans = index < scanStubs.length - 1;
         break;
       }
 
@@ -647,18 +736,69 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
       monitorsSkippedAlreadyExisting: 0,
       monitorsSkippedUnsupportedHost: 0,
       monitorsFailed: 0,
+      deviceFailureReasons: [],
+      monitorFailureReasons: [],
+      monitorProvisioningHalted: false,
       isTruncated: false,
+      hostsPendingImport: 0,
+      monitorsPendingCreation: 0,
+      hasUnevaluatedScans: false,
       hasMoreScans: false,
       isDryRun: isDryRun,
       matchedIpAddressSample: [],
     };
   }
 
+  /*
+   * Keep the reason a create failed, so the run can say WHY it created
+   * nothing instead of pointing at server logs the operator of a self-hosted
+   * install may not be able to read. Deduplicated (500 identical failures are
+   * one reason) and capped, in both count and length, because this string
+   * ends up in a modal.
+   */
+  private recordFailureReason(reasons: Array<string>, error: unknown): void {
+    const message: string = this.describeError(error);
+
+    if (!message) {
+      return;
+    }
+
+    const trimmed: string = message.substring(0, MAX_RUN_FAILURE_REASON_LENGTH);
+
+    if (reasons.includes(trimmed)) {
+      return;
+    }
+
+    if (reasons.length >= MAX_RUN_FAILURE_REASONS) {
+      return;
+    }
+
+    reasons.push(trimmed);
+  }
+
+  /*
+   * The message an operator should see. Exception extends Error, so one arm
+   * covers both; anything else is stringified rather than dropped, because a
+   * thrown non-Error is exactly the case where the reason matters most.
+   */
+  private describeError(error: unknown): string {
+    if (error instanceof Error) {
+      return (error.message || "").replace(/\s+/g, " ").trim();
+    }
+
+    return String(error).replace(/\s+/g, " ").trim();
+  }
+
   private emptyExistingMonitorProvisioningState(): ExistingMonitorProvisioningState {
     return {
       autoProvisionedKeys: new Set(),
       manuallyMonitoredDeviceIds: new Set(),
       attemptedProvisioningKeys: new Set(),
+      unresolvableTemplateIds: new Set(),
+      consecutiveCreateFailures: 0,
+      lastCreateFailureReason: "",
+      hasProvisionedAnyMonitor: false,
+      isProvisioningHalted: false,
     };
   }
 
@@ -1077,6 +1217,18 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
 
     const createdIpAddresses: Array<string> = [];
 
+    /*
+     * What this pass counted as still-to-do, deduplicated.
+     *
+     * A stored result can list the same address twice (duplicate rows are
+     * normal enough that normalisation preserves them and the import path
+     * treats the second as already-registered), so counting rows rather than
+     * distinct work would report a remainder larger than the work that
+     * actually remains — in a number whose whole purpose is to be trusted.
+     */
+    const pendingImportAddresses: Set<string> = new Set<string>();
+    const pendingProvisioningKeys: Set<string> = new Set<string>();
+
     for (const host of hosts) {
       /*
        * The address becomes the device's varchar(100) hostname, on which
@@ -1149,22 +1301,13 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
       let deviceWasCreated: boolean = false;
 
       /*
-       * The frozen isAlreadyRegistered flag AND the live set: the flag is a
-       * point-in-time answer from the last upload, the set covers devices
-       * created since — including by this very run, which is what collapses
-       * duplicate rows and overlapping scans into one device.
+       * Current inventory is authoritative. The stored isAlreadyRegistered
+       * flag can outlive a deleted device and must not veto re-importing it.
+       * The map also includes devices created during this run, keeping
+       * duplicate rows and overlapping scans idempotent.
        */
-      if (host.isAlreadyRegistered || networkDevice) {
+      if (networkDevice) {
         result.hostsSkippedAlreadyRegistered++;
-
-        /*
-         * A stale scan can claim a deleted device is still registered. With
-         * no live row there is neither a safe binding nor enough intent to
-         * recreate it from old data, so preserve the established skip.
-         */
-        if (!networkDevice) {
-          continue;
-        }
       }
 
       if (result.matchedIpAddressSample.length < MAX_MATCHED_IP_SAMPLE) {
@@ -1175,8 +1318,25 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
 
       if (!networkDevice) {
         if (data.attempts.deviceCount >= MAX_DEVICES_PER_AUTO_IMPORT_RUN) {
+          /*
+           * Budget spent — stop WRITING, keep COUNTING. The rest of this
+           * loop is pure matching over a jsonb array that is already in
+           * memory against addresses primeExistingDevices already resolved,
+           * so finishing the scan costs nothing the cap exists to prevent,
+           * and it is the difference between "imported 500" and "imported
+           * 500 of 909, 409 still to go". A run that stopped mid-count
+           * reported the truncated numbers as if they were the estate,
+           * which is what OneUptime issue #3642 experienced as the
+           * remainder being silently dropped.
+           */
           result.isTruncated = true;
-          break;
+
+          if (!pendingImportAddresses.has(host.ipAddress)) {
+            pendingImportAddresses.add(host.ipAddress);
+            result.hostsPendingImport++;
+          }
+
+          continue;
         }
 
         if (data.isDryRun) {
@@ -1231,8 +1391,8 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         continue;
       }
 
-      const monitorsWereTruncated: boolean = await this.ensureMonitorsForDevice(
-        {
+      const monitorsLeftPending: Array<string> =
+        await this.ensureMonitorsForDevice({
           projectId: scan.projectId,
           host: host,
           networkDevice: networkDevice,
@@ -1242,18 +1402,45 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
           result: result,
           attempts: data.attempts,
           isDryRun: data.isDryRun,
-        },
-      );
+        });
 
-      if (monitorsWereTruncated) {
+      /*
+       * Same protocol as the device cap above: the monitor budget stops the
+       * creates, not the accounting. It also has its OWN cap, so an estate
+       * that is fully imported but unmonitored — the shape of issue #3642 —
+       * runs out of monitor budget with nothing pending on the device side,
+       * and the operator needs to be told how many devices are still
+       * waiting for a monitor.
+       */
+      for (const templateId of monitorsLeftPending) {
         result.isTruncated = true;
-        break;
+
+        const pendingKey: string =
+          NetworkDeviceAutoImportRuleEngineServiceClass.monitorProvisioningKey(
+            networkDevice.id!,
+            templateId,
+          );
+
+        if (!pendingProvisioningKeys.has(pendingKey)) {
+          pendingProvisioningKeys.add(pendingKey);
+          result.monitorsPendingCreation++;
+        }
       }
     }
 
     return createdIpAddresses;
   }
 
+  /*
+   * Reconcile one device against the templates the matched rules selected.
+   *
+   * Returns the template ids the run's budget left UNATTEMPTED — empty when
+   * every requested monitor was settled, whether created, failed, or skipped
+   * because monitoring already existed. A non-empty answer is what makes the
+   * run report itself truncated; the ids (rather than a bare count) are what
+   * let the caller count one device's leftover work once, however many
+   * duplicate rows the scan lists that host under.
+   */
   private async ensureMonitorsForDevice(data: {
     projectId: ObjectID;
     host: DiscoveredNetworkDevice;
@@ -1264,7 +1451,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
     result: AutoImportRuleRunResult;
     attempts: ImportAttemptBudget;
     isDryRun: boolean;
-  }): Promise<boolean> {
+  }): Promise<Array<string>> {
     /*
      * A Network Device monitor is fed by the device's polls, and nothing
      * polls a monitor-backed device — its bound monitor's status IS its
@@ -1281,7 +1468,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
       )
     ) {
       data.result.monitorsSkippedUnsupportedHost += data.templateIds.length;
-      return false;
+      return [];
     }
 
     if (!data.networkDevice.id) {
@@ -1290,7 +1477,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         `Auto-import: cannot provision monitor(s) for ${data.host.ipAddress} because the Network Device has no id.`,
         { projectId: data.projectId.toString() } as LogAttributes,
       );
-      return false;
+      return [];
     }
 
     const networkDeviceId: ObjectID = data.networkDevice.id;
@@ -1323,25 +1510,76 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
     }
 
     if (pendingTemplateIds.length === 0) {
-      return false;
+      return [];
     }
 
+    /*
+     * The project-wide snapshot already knows this device is monitored by
+     * hand, and that answer outranks any failure below: a device nobody was
+     * going to provision cannot be a monitor this run failed to create. Read
+     * from memory, so it stays ahead of the systemic verdicts.
+     */
     if (
       data.existingMonitors.manuallyMonitoredDeviceIds.has(
         networkDeviceIdString,
       )
     ) {
       data.result.monitorsSkippedAlreadyExisting += pendingTemplateIds.length;
-      return false;
+      return [];
+    }
+
+    /*
+     * Everything below here costs the run something — a monitor budget slot, a
+     * per-device search of the unindexed monitor step payload, a create round
+     * trip. None of it is worth spending on work this run has already proved
+     * cannot succeed, so the two systemic verdicts are settled next: a
+     * template this run could not load, and a create pipeline that has failed
+     * identically for every device it tried.
+     *
+     * Both still COUNT the monitors that are missing. "0 created, and here is
+     * why" is the answer; quietly reporting fewer missing monitors than there
+     * are would trade one confusing dialog for another.
+     */
+    const resolvableTemplateIds: Array<string> = [];
+
+    for (const templateId of pendingTemplateIds) {
+      if (
+        !data.existingMonitors.isProvisioningHalted &&
+        !data.existingMonitors.unresolvableTemplateIds.has(templateId)
+      ) {
+        resolvableTemplateIds.push(templateId);
+        continue;
+      }
+
+      /*
+       * Counted once and then settled. Marking the (device, template) key
+       * attempted is what keeps a host carried by two overlapping scans in the
+       * same run from being counted as two missing monitors — the same
+       * bookkeeping a real attempt does, for the same reason.
+       */
+      data.existingMonitors.attemptedProvisioningKeys.add(
+        NetworkDeviceAutoImportRuleEngineServiceClass.monitorProvisioningKey(
+          networkDeviceId,
+          templateId,
+        ),
+      );
+      data.result.monitorsFailed++;
+      data.result.monitorProvisioningHalted = true;
+    }
+
+    if (resolvableTemplateIds.length === 0) {
+      return [];
     }
 
     /*
      * Do not run the unindexed JSON race-check after this run's monitor-work
      * budget is already exhausted. The device remains unreconciled and the
-     * truncated marker protocol brings it back on the next bounded pass.
+     * truncated marker protocol brings it back on the next bounded pass —
+     * counted, so the operator is told how many devices are still waiting
+     * rather than left to infer it from a total that stopped early.
      */
     if (data.attempts.monitorCount >= MAX_MONITORS_PER_AUTO_IMPORT_RUN) {
-      return true;
+      return resolvableTemplateIds;
     }
 
     if (!data.isDryRun) {
@@ -1357,43 +1595,74 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         networkDeviceIdString,
       )
     ) {
-      data.result.monitorsSkippedAlreadyExisting += pendingTemplateIds.length;
-      return false;
+      data.result.monitorsSkippedAlreadyExisting +=
+        resolvableTemplateIds.length;
+      return [];
     }
 
-    for (const templateId of pendingTemplateIds) {
+    for (let index: number = 0; index < resolvableTemplateIds.length; index++) {
+      const templateId: string = resolvableTemplateIds[index]!;
       const key: string =
         NetworkDeviceAutoImportRuleEngineServiceClass.monitorProvisioningKey(
           networkDeviceId,
           templateId,
         );
 
+      /*
+       * A create earlier in THIS device's loop can be the one that trips the
+       * systemic verdict. The rest of its templates are then in exactly the
+       * position of every device after it.
+       */
+      if (data.existingMonitors.isProvisioningHalted) {
+        data.existingMonitors.attemptedProvisioningKeys.add(key);
+        data.result.monitorsFailed++;
+        data.result.monitorProvisioningHalted = true;
+        continue;
+      }
+
       /* The race-check may have found this key after the initial snapshot. */
       if (data.existingMonitors.autoProvisionedKeys.has(key)) {
         data.result.monitorsSkippedAlreadyExisting++;
         continue;
       }
 
+      /*
+       * Budget spent mid-device. Everything from here on is untouched, so
+       * that is exactly what is left pending for this device.
+       */
       if (data.attempts.monitorCount >= MAX_MONITORS_PER_AUTO_IMPORT_RUN) {
-        return true;
+        return resolvableTemplateIds.slice(index);
       }
 
       data.attempts.monitorCount++;
       data.existingMonitors.attemptedProvisioningKeys.add(key);
 
       /*
-       * Invalid or concurrently deleted templates are failed attempts too.
-       * Count them inside the same cap before looking up the cached template;
-       * otherwise one stale rule could walk and log an unbounded estate in a
-       * single API request or worker tick.
+       * A template the run could not load is a fault in the RULE, not in this
+       * host: the rule points at a template that has been deleted, moved to
+       * another project, or changed to a different monitor type since the rule
+       * was saved (the rule form validates all three, but only at save time).
+       * It will be just as unloadable for every other device, so the verdict is
+       * recorded once and every later host short-circuits above — instead of
+       * spending the run's whole monitor budget re-proving it, which is what
+       * turned this into "500 monitors failed, stopped at the run cap, run
+       * again to continue" in issue #3643.
        */
       const template: MonitorTemplate | undefined =
         data.monitorTemplates.get(templateId);
 
       if (!template) {
+        data.existingMonitors.unresolvableTemplateIds.add(templateId);
         data.result.monitorsFailed++;
+        data.result.monitorProvisioningHalted = true;
+        this.recordFailureReason(
+          data.result.monitorFailureReasons,
+          new Error(
+            "This rule's Monitor Template could not be loaded. It may have been deleted, moved to another project, or changed to a monitor type other than Network Device. Edit the rule and select a Network Device Monitor Template.",
+          ),
+        );
         logger.error(
-          `Auto-import: Network Device monitor template ${templateId} is missing, deleted, or not valid for project ${data.projectId.toString()}.`,
+          `Auto-import: Network Device monitor template ${templateId} is missing, deleted, or not valid for project ${data.projectId.toString()}. No monitors will be provisioned from it for the rest of this run.`,
           { projectId: data.projectId.toString() } as LogAttributes,
         );
         continue;
@@ -1421,6 +1690,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
 
         data.existingMonitors.autoProvisionedKeys.add(key);
         data.result.monitorsCreated++;
+        this.recordMonitorCreateSuccess(data.existingMonitors);
       } catch (error) {
         /*
          * The partial unique index is the final race backstop. If another
@@ -1452,19 +1722,87 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
 
           if (data.existingMonitors.autoProvisionedKeys.has(key)) {
             data.result.monitorsSkippedAlreadyExisting++;
+            /*
+             * Losing this race proves the create pipeline works — somebody
+             * else just used it — so it is not evidence of a systemic fault.
+             */
+            this.recordMonitorCreateSuccess(data.existingMonitors);
             continue;
           }
         }
 
         data.result.monitorsFailed++;
+        this.recordFailureReason(data.result.monitorFailureReasons, error);
         logger.error(
           `Auto-import: could not create monitor from template ${templateId} for Network Device ${networkDeviceId.toString()} (${data.host.ipAddress}): ${error}`,
           { projectId: data.projectId.toString() } as LogAttributes,
         );
+        this.recordMonitorCreateFailure({
+          state: data.existingMonitors,
+          result: data.result,
+          projectId: data.projectId,
+          error: error,
+        });
       }
     }
 
-    return false;
+    return [];
+  }
+
+  /*
+   * A create succeeded (or lost a race to another writer, which proves the
+   * same thing): whatever the last failures were, they were not systemic, and
+   * this run has now seen the create pipeline work at least once.
+   */
+  private recordMonitorCreateSuccess(
+    state: ExistingMonitorProvisioningState,
+  ): void {
+    state.consecutiveCreateFailures = 0;
+    state.lastCreateFailureReason = "";
+    state.hasProvisionedAnyMonitor = true;
+  }
+
+  /*
+   * Decide whether one monitor create failure is this host's problem or the
+   * whole run's. The bar is deliberately high, because giving up early on a
+   * per-host fault would silently cost an estate its monitors — the failure
+   * this fix exists to make visible. All three must hold: the run has never
+   * created a monitor, the last MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES
+   * attempts all failed, and they all failed with the SAME message.
+   */
+  private recordMonitorCreateFailure(data: {
+    state: ExistingMonitorProvisioningState;
+    result: AutoImportRuleRunResult;
+    projectId: ObjectID;
+    error: unknown;
+  }): void {
+    const reason: string = this.describeError(data.error);
+
+    if (reason && reason === data.state.lastCreateFailureReason) {
+      data.state.consecutiveCreateFailures++;
+    } else {
+      data.state.lastCreateFailureReason = reason;
+      data.state.consecutiveCreateFailures = 1;
+    }
+
+    if (data.state.hasProvisionedAnyMonitor) {
+      return;
+    }
+
+    if (
+      data.state.consecutiveCreateFailures <
+      MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES
+    ) {
+      return;
+    }
+
+    data.state.isProvisioningHalted = true;
+    data.result.monitorProvisioningHalted = true;
+
+    logger.error(
+      `Auto-import: ${MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES} Network Device monitor creates failed identically for project ${data.projectId.toString()} with none succeeding ("${reason}"). Treating this as a systemic failure and provisioning no further monitors this run.`,
+      { projectId: data.projectId.toString() } as LogAttributes,
+    );
   }
 
   /*
@@ -1549,6 +1887,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         createdDevice = await attemptCreate(buildFallbackDeviceName(data.host));
       } catch (secondError) {
         data.result.devicesFailed++;
+        this.recordFailureReason(data.result.deviceFailureReasons, secondError);
         logger.error(
           `Auto-import: could not create a device for ${data.host.ipAddress} (scan ${data.scan.id?.toString()}): ${firstError} / retry: ${secondError}`,
           { projectId: data.projectId.toString() } as LogAttributes,
@@ -1566,20 +1905,20 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
   /*
    * The one write this engine makes to the scan row: retire the consumed
    * hosts in the stored jsonb (so the Review dialog stops offering them) and
-   * stamp the processed marker — in a single hook-free compare-and-set on
-   * (status, completedAt), for the reasons on processCompletedScan. The
+   * stamp the processed marker — in a single hook-free compare-and-set on the
+   * run state that was read, for the reasons on processCompletedScan. The
    * claim endpoint's hook-free contract is why this must never go through
    * updateOneById: the scan service deliberately has no update-success hooks
    * to piggyback on, and this write must not fire the full pipeline on every
    * worker tick.
    *
    * The CAS defends against exactly one concurrent writer — the ingest
-   * endpoint, which rewrites completedAt with every upload. It cannot
-   * defend against another ENGINE write (which changes neither status nor
-   * completedAt), and does not need to: every path that reaches this method
-   * holds the sweep lock (the worker for its whole sweep, Run Now for a
-   * real run; dry runs never write), so the full-array replace below always
-   * has the row's only engine writer. Weaken that invariant and this
+   * endpoint, which rewrites completedAt with a final upload and the host
+   * counters with a partial one. It cannot defend against another ENGINE
+   * write (which changes none of those columns), and does not need to: every
+   * path that reaches this method holds the sweep lock (the worker for its
+   * whole sweep, Run Now for a real run; dry runs never write), so the
+   * full-array replace below always has the row's only engine writer. Weaken that invariant and this
    * becomes last-writer-wins on the isAlreadyRegistered flips.
    */
   private async stampScan(data: {
@@ -1632,8 +1971,30 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
       // Cast: the model's JSON column makes DeepPartial recursion blow up.
       data: payload as unknown as QueryDeepPartialEntity<NetworkDeviceDiscoveryScan>,
       expectedData: {
-        status: "Completed",
+        /*
+         * The scan's own status, not the literal "Completed" this used to
+         * pin. A scan that is still sweeping is importable now (issue
+         * #3599), and its status is exactly the thing that changes when its
+         * run ends — so reading it off the row keeps the guard doing what it
+         * always did rather than making an In Progress stamp always miss.
+         */
+        status: data.scan.status ?? null,
         completedAt: data.scan.completedAt || null,
+        /*
+         * The counters a running scan's partial uploads move, so a partial
+         * landing between the read and this write is caught the way a
+         * completion is: the stamp misses, the marker stays NULL, and the
+         * next tick processes the newer host list.
+         *
+         * Not a perfect version token for an in-progress scan — the
+         * ICMP-filtered fallback pass finds hosts without advancing
+         * scannedHostCount, so two partials can carry both counts unchanged.
+         * The consequence is bounded and self-healing: one batch of hosts
+         * waits for the next partial upload (which clears the marker again),
+         * and the run's final result clears it unconditionally.
+         */
+        scannedHostCount: data.scan.scannedHostCount ?? null,
+        respondedHostCount: data.scan.respondedHostCount ?? null,
       } as unknown as QueryDeepPartialEntity<NetworkDeviceDiscoveryScan>,
     });
   }