@oneuptime/common 12.0.33 → 13.0.0

package/build/dist/Server/Services/NetworkDeviceAutoImportRuleEngineService.js

@@ -18,10 +18,11 @@ import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import MonitorType from "../../Types/Monitor/MonitorType";
-import { MAX_MATCHED_IP_SAMPLE, } from "../../Types/NetworkAutomation/RuleRunResult";
+import { MAX_MATCHED_IP_SAMPLE, MAX_RUN_FAILURE_REASON_LENGTH, MAX_RUN_FAILURE_REASONS, } from "../../Types/NetworkAutomation/RuleRunResult";
 import AutoImportRuleMatcher from "../../Utils/NetworkDiscovery/AutoImportRuleMatcher";
 import { buildFallbackDeviceName, buildNetworkDeviceFromDiscoveredHost, } from "../../Utils/NetworkDiscovery/DiscoveredDeviceBuilder";
 import { normalizeDiscoveredHosts } from "../../Utils/NetworkDiscovery/DiscoveredHostUtil";
+import { DISCOVERY_SCAN_IMPORTABLE_STATUSES } from "../../Utils/NetworkDiscovery/DiscoveryScanStatus";
 import { NetworkDeviceMonitoringMethodUtil } from "../../Types/NetworkDevice/NetworkDeviceMonitoringMethod";
 import NetworkDeviceMonitorTemplateUtil from "../../Utils/Monitor/NetworkDeviceMonitorTemplateUtil";
 import Semaphore from "../Infrastructure/Semaphore";
@@ -31,7 +32,8 @@ import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import logger from "../Utils/Logger";
 /*
  * The engine behind network device auto-import rules (issue #3378): reads a
- * completed discovery scan's results, asks the project's rules which hosts
+ * discovery scan's results — finished or still arriving from a running sweep,
+ * see DISCOVERY_SCAN_IMPORTABLE_STATUSES — asks the project's rules which hosts
  * import, and creates the Network Devices the manual Review dialog would
  * have — through the same builder, so a rule-imported host and a
  * hand-imported host are the same device. Site assignment, owners and labels
@@ -99,6 +101,25 @@ export const MAX_DEVICES_PER_AUTO_IMPORT_RUN = 500;
 export const MAX_MONITORS_PER_AUTO_IMPORT_RUN = 500;
 // Scans one manual "Run Now" will read, oldest results last.
 export const MAX_SCANS_PER_AUTO_IMPORT_RULE_RUN = 100;
+/*
+ * How many monitor creates may fail IDENTICALLY, back to back with no success
+ * between them, before the run stops attempting any more.
+ *
+ * A monitor create fails for one of two kinds of reason. Per host — this
+ * device's name collides, this one raced another writer — where the next host
+ * is unaffected and trying it is right. Or systemic: the template's criteria
+ * name a monitor status that no longer exists, the plan's monitor limit is
+ * reached, the database is refusing writes. Those fail for the FIRST device
+ * and for all 500 after it, and the old engine could not tell the difference:
+ * it spent the whole per-run monitor budget re-proving one broken thing, then
+ * reported "Stopped at the run cap — run again to continue", which is an
+ * instruction to do it all again. That is OneUptime/oneuptime#3643.
+ *
+ * Ten consecutive failures carrying the SAME message, with no create
+ * succeeding in between, is not a run of bad luck. Any success resets it, so a
+ * genuinely per-host fault never trips this.
+ */
+export const MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES = 10;
 /*
  * Results older than this are stamped processed WITHOUT importing. A late
  * result can land on a reaper-Failed scan hours after its sweep ran (the
@@ -125,24 +146,32 @@ export const AUTO_IMPORT_SWEEP_LOCK_NAMESPACE = "Workers.Cron";
 export const AUTO_IMPORT_SWEEP_LOCK_TIMEOUT_MS = 11 * 60 * 1000;
 class NetworkDeviceAutoImportRuleEngineServiceClass {
     /**
-     * The automatic path: apply the project's enabled rules to one Completed
-     * scan, then stamp the scan's autoImportProcessedAt marker.
+     * The automatic path: apply the project's enabled rules to one scan's
+     * stored results, then stamp the scan's autoImportProcessedAt marker.
+     *
+     * The scan may still be SWEEPING — see
+     * DISCOVERY_SCAN_IMPORTABLE_STATUSES. Its partial results are as real as
+     * a finished scan's (the probe found those hosts; it simply has more of the
+     * range to cover), and waiting for the whole sweep is what left 527
+     * discovered switches unimportable for a day in OneUptime issue #3599. The
+     * name is kept because it is the automatic path's identity across the
+     * worker, its tests and its logs.
      *
      * Returns null when there was nothing to do (scan gone, superseded,
      * already processed, no import rules, or results too old) and the run's
      * counters otherwise. Throws only on unexpected failures — the caller
      * (the worker sweep) isolates those per scan.
      *
-     * The marker protocol, and why every stamp is a compare-and-set on
-     * (status, completedAt): the ingest endpoint clears the marker in the same
-     * write that stores new results, so "marker is NULL" always means "the
-     * results now on the row have not been processed". If new results land
-     * while this pass is reading the old ones, the CAS misses, the marker
-     * stays NULL, and the next tick processes the new upload — the stamp can
-     * never retire results it did not see, and the jsonb write-back can never
-     * clobber a newer host list. A truncated pass that created something
-     * stamps nothing on purpose: the NULL marker is what makes the next tick
-     * resume. (A truncated pass where every create FAILED stamps anyway — see
+     * The marker protocol, and why every stamp is a compare-and-set on the run
+     * state this pass read: the ingest endpoint clears the marker in the same
+     * write that stores new results — partial uploads included — so "marker is
+     * NULL" always means "the results now on the row have not been processed".
+     * If new results land while this pass is reading the old ones, the CAS
+     * misses, the marker stays NULL, and the next tick processes the new upload
+     * — the stamp can never retire results it did not see, and the jsonb
+     * write-back can never clobber a newer host list. A truncated pass that
+     * created something stamps nothing on purpose: the NULL marker is what
+     * makes the next tick resume. (A truncated pass where every create FAILED stamps anyway — see
      * the zero-progress note at the stamp below.)
      */
     async processCompletedScan(data) {
@@ -150,11 +179,23 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         const scan = await NetworkDeviceDiscoveryScanService.findOneBy({
             query: {
                 _id: data.scanId,
-                status: "Completed",
+                status: QueryHelper.any(DISCOVERY_SCAN_IMPORTABLE_STATUSES),
             },
-            select: Object.assign({ _id: true, projectId: true, status: true, completedAt: true, autoImportProcessedAt: true, discoveredDevices: true }, AUTO_IMPORT_SCAN_CREDENTIAL_SELECT),
+            select: Object.assign({ _id: true, projectId: true, status: true, completedAt: true, 
+                /*
+                 * The two counters a running scan's partial uploads move. They are
+                 * not read for anything; they are the version token stampScan's
+                 * compare-and-set uses to tell "the results I evaluated" from "the
+                 * results that landed while I was evaluating".
+                 */
+                scannedHostCount: true, respondedHostCount: true, autoImportProcessedAt: true, discoveredDevices: true }, AUTO_IMPORT_SCAN_CREDENTIAL_SELECT),
             props: {
                 isRoot: true,
+                /*
+                 * Keep the stored payload for stampScan; the engine checks live
+                 * inventory itself before evaluating these hosts.
+                 */
+                ignoreHooks: true,
             },
         });
         /*
@@ -384,11 +425,21 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         const scanStubs = await NetworkDeviceDiscoveryScanService.findBy({
             query: {
                 projectId: data.projectId,
-                status: "Completed",
+                /*
+                 * A scan that is still sweeping counts too: its partial results are
+                 * already on the row, and "Run Now found 25 hosts while 527 sat in
+                 * the Devices list" was exactly the report (OneUptime issue #3599).
+                 */
+                status: QueryHelper.any(DISCOVERY_SCAN_IMPORTABLE_STATUSES),
             },
             select: {
                 _id: true,
             },
+            /*
+             * Newest results first. An in-progress scan has no completedAt, and
+             * Postgres orders NULLs first under DESC — which is the right place
+             * for it: its results are the freshest the project has.
+             */
             sort: {
                 completedAt: SortOrder.Descending,
             },
@@ -401,15 +452,19 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
         if (result.hasMoreScans) {
             scanStubs.pop();
         }
-        for (const scanStub of scanStubs) {
+        for (let index = 0; index < scanStubs.length; index++) {
+            const scanStub = scanStubs[index];
             const scan = await NetworkDeviceDiscoveryScanService.findOneBy({
                 query: {
                     _id: scanStub.id,
                     projectId: data.projectId,
-                    status: "Completed",
+                    status: QueryHelper.any(DISCOVERY_SCAN_IMPORTABLE_STATUSES),
                 },
-                select: Object.assign({ _id: true, projectId: true, status: true, completedAt: true, discoveredDevices: true }, AUTO_IMPORT_SCAN_CREDENTIAL_SELECT),
-                props: { isRoot: true },
+                select: Object.assign({ _id: true, projectId: true, status: true, completedAt: true, 
+                    // The version token stampScan's compare-and-set uses; see there.
+                    scannedHostCount: true, respondedHostCount: true, discoveredDevices: true }, AUTO_IMPORT_SCAN_CREDENTIAL_SELECT),
+                // Preserve the raw scan rows for the compare-and-set write-back.
+                props: { isRoot: true, ignoreHooks: true },
             });
             // Re-queued or deleted since the stub query — nothing to evaluate.
             if (!scan) {
@@ -439,8 +494,15 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
                     leaveMarkerUnstamped: true,
                 });
             }
-            // The device cap is shared across the whole run, dry or real.
+            /*
+             * The caps are shared across the whole run, dry or real. Stop OPENING
+             * further scans once they are spent — each one is a multi-megabyte
+             * jsonb read whose hosts this pass could not act on anyway — but say
+             * that scans were left unread, so the pending counters below are
+             * reported as a floor rather than as the estate's total.
+             */
             if (result.isTruncated) {
+                result.hasUnevaluatedScans = index < scanStubs.length - 1;
                 break;
             }
             /*
@@ -474,17 +536,60 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             monitorsSkippedAlreadyExisting: 0,
             monitorsSkippedUnsupportedHost: 0,
             monitorsFailed: 0,
+            deviceFailureReasons: [],
+            monitorFailureReasons: [],
+            monitorProvisioningHalted: false,
             isTruncated: false,
+            hostsPendingImport: 0,
+            monitorsPendingCreation: 0,
+            hasUnevaluatedScans: false,
             hasMoreScans: false,
             isDryRun: isDryRun,
             matchedIpAddressSample: [],
         };
     }
+    /*
+     * Keep the reason a create failed, so the run can say WHY it created
+     * nothing instead of pointing at server logs the operator of a self-hosted
+     * install may not be able to read. Deduplicated (500 identical failures are
+     * one reason) and capped, in both count and length, because this string
+     * ends up in a modal.
+     */
+    recordFailureReason(reasons, error) {
+        const message = this.describeError(error);
+        if (!message) {
+            return;
+        }
+        const trimmed = message.substring(0, MAX_RUN_FAILURE_REASON_LENGTH);
+        if (reasons.includes(trimmed)) {
+            return;
+        }
+        if (reasons.length >= MAX_RUN_FAILURE_REASONS) {
+            return;
+        }
+        reasons.push(trimmed);
+    }
+    /*
+     * The message an operator should see. Exception extends Error, so one arm
+     * covers both; anything else is stringified rather than dropped, because a
+     * thrown non-Error is exactly the case where the reason matters most.
+     */
+    describeError(error) {
+        if (error instanceof Error) {
+            return (error.message || "").replace(/\s+/g, " ").trim();
+        }
+        return String(error).replace(/\s+/g, " ").trim();
+    }
     emptyExistingMonitorProvisioningState() {
         return {
             autoProvisionedKeys: new Set(),
             manuallyMonitoredDeviceIds: new Set(),
             attemptedProvisioningKeys: new Set(),
+            unresolvableTemplateIds: new Set(),
+            consecutiveCreateFailures: 0,
+            lastCreateFailureReason: "",
+            hasProvisionedAnyMonitor: false,
+            isProvisioningHalted: false,
         };
     }
     /*
@@ -796,6 +901,17 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             into: data.existingDevices,
         });
         const createdIpAddresses = [];
+        /*
+         * What this pass counted as still-to-do, deduplicated.
+         *
+         * A stored result can list the same address twice (duplicate rows are
+         * normal enough that normalisation preserves them and the import path
+         * treats the second as already-registered), so counting rows rather than
+         * distinct work would report a remainder larger than the work that
+         * actually remains — in a number whose whole purpose is to be trusted.
+         */
+        const pendingImportAddresses = new Set();
+        const pendingProvisioningKeys = new Set();
         for (const host of hosts) {
             /*
              * The address becomes the device's varchar(100) hostname, on which
@@ -849,21 +965,13 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             let networkDevice = data.existingDevices.get(host.ipAddress);
             let deviceWasCreated = false;
             /*
-             * The frozen isAlreadyRegistered flag AND the live set: the flag is a
-             * point-in-time answer from the last upload, the set covers devices
-             * created since — including by this very run, which is what collapses
-             * duplicate rows and overlapping scans into one device.
+             * Current inventory is authoritative. The stored isAlreadyRegistered
+             * flag can outlive a deleted device and must not veto re-importing it.
+             * The map also includes devices created during this run, keeping
+             * duplicate rows and overlapping scans idempotent.
              */
-            if (host.isAlreadyRegistered || networkDevice) {
+            if (networkDevice) {
                 result.hostsSkippedAlreadyRegistered++;
-                /*
-                 * A stale scan can claim a deleted device is still registered. With
-                 * no live row there is neither a safe binding nor enough intent to
-                 * recreate it from old data, so preserve the established skip.
-                 */
-                if (!networkDevice) {
-                    continue;
-                }
             }
             if (result.matchedIpAddressSample.length < MAX_MATCHED_IP_SAMPLE) {
                 if (!result.matchedIpAddressSample.includes(host.ipAddress)) {
@@ -872,8 +980,23 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             }
             if (!networkDevice) {
                 if (data.attempts.deviceCount >= MAX_DEVICES_PER_AUTO_IMPORT_RUN) {
+                    /*
+                     * Budget spent — stop WRITING, keep COUNTING. The rest of this
+                     * loop is pure matching over a jsonb array that is already in
+                     * memory against addresses primeExistingDevices already resolved,
+                     * so finishing the scan costs nothing the cap exists to prevent,
+                     * and it is the difference between "imported 500" and "imported
+                     * 500 of 909, 409 still to go". A run that stopped mid-count
+                     * reported the truncated numbers as if they were the estate,
+                     * which is what OneUptime issue #3642 experienced as the
+                     * remainder being silently dropped.
+                     */
                     result.isTruncated = true;
-                    break;
+                    if (!pendingImportAddresses.has(host.ipAddress)) {
+                        pendingImportAddresses.add(host.ipAddress);
+                        result.hostsPendingImport++;
+                    }
+                    continue;
                 }
                 if (data.isDryRun) {
                     /*
@@ -906,7 +1029,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             if (templateIds.length === 0) {
                 continue;
             }
-            const monitorsWereTruncated = await this.ensureMonitorsForDevice({
+            const monitorsLeftPending = await this.ensureMonitorsForDevice({
                 projectId: scan.projectId,
                 host: host,
                 networkDevice: networkDevice,
@@ -917,13 +1040,35 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
                 attempts: data.attempts,
                 isDryRun: data.isDryRun,
             });
-            if (monitorsWereTruncated) {
+            /*
+             * Same protocol as the device cap above: the monitor budget stops the
+             * creates, not the accounting. It also has its OWN cap, so an estate
+             * that is fully imported but unmonitored — the shape of issue #3642 —
+             * runs out of monitor budget with nothing pending on the device side,
+             * and the operator needs to be told how many devices are still
+             * waiting for a monitor.
+             */
+            for (const templateId of monitorsLeftPending) {
                 result.isTruncated = true;
-                break;
+                const pendingKey = NetworkDeviceAutoImportRuleEngineServiceClass.monitorProvisioningKey(networkDevice.id, templateId);
+                if (!pendingProvisioningKeys.has(pendingKey)) {
+                    pendingProvisioningKeys.add(pendingKey);
+                    result.monitorsPendingCreation++;
+                }
             }
         }
         return createdIpAddresses;
     }
+    /*
+     * Reconcile one device against the templates the matched rules selected.
+     *
+     * Returns the template ids the run's budget left UNATTEMPTED — empty when
+     * every requested monitor was settled, whether created, failed, or skipped
+     * because monitoring already existed. A non-empty answer is what makes the
+     * run report itself truncated; the ids (rather than a bare count) are what
+     * let the caller count one device's leftover work once, however many
+     * duplicate rows the scan lists that host under.
+     */
     async ensureMonitorsForDevice(data) {
         /*
          * A Network Device monitor is fed by the device's polls, and nothing
@@ -937,12 +1082,12 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
          */
         if (NetworkDeviceMonitoringMethodUtil.isMonitorBacked(data.networkDevice.monitoringMethod)) {
             data.result.monitorsSkippedUnsupportedHost += data.templateIds.length;
-            return false;
+            return [];
         }
         if (!data.networkDevice.id) {
             data.result.monitorsFailed += data.templateIds.length;
             logger.error(`Auto-import: cannot provision monitor(s) for ${data.host.ipAddress} because the Network Device has no id.`, { projectId: data.projectId.toString() });
-            return false;
+            return [];
         }
         const networkDeviceId = data.networkDevice.id;
         const networkDeviceIdString = networkDeviceId.toString();
@@ -965,19 +1110,59 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             pendingTemplateIds.push(templateId);
         }
         if (pendingTemplateIds.length === 0) {
-            return false;
+            return [];
         }
+        /*
+         * The project-wide snapshot already knows this device is monitored by
+         * hand, and that answer outranks any failure below: a device nobody was
+         * going to provision cannot be a monitor this run failed to create. Read
+         * from memory, so it stays ahead of the systemic verdicts.
+         */
         if (data.existingMonitors.manuallyMonitoredDeviceIds.has(networkDeviceIdString)) {
             data.result.monitorsSkippedAlreadyExisting += pendingTemplateIds.length;
-            return false;
+            return [];
+        }
+        /*
+         * Everything below here costs the run something — a monitor budget slot, a
+         * per-device search of the unindexed monitor step payload, a create round
+         * trip. None of it is worth spending on work this run has already proved
+         * cannot succeed, so the two systemic verdicts are settled next: a
+         * template this run could not load, and a create pipeline that has failed
+         * identically for every device it tried.
+         *
+         * Both still COUNT the monitors that are missing. "0 created, and here is
+         * why" is the answer; quietly reporting fewer missing monitors than there
+         * are would trade one confusing dialog for another.
+         */
+        const resolvableTemplateIds = [];
+        for (const templateId of pendingTemplateIds) {
+            if (!data.existingMonitors.isProvisioningHalted &&
+                !data.existingMonitors.unresolvableTemplateIds.has(templateId)) {
+                resolvableTemplateIds.push(templateId);
+                continue;
+            }
+            /*
+             * Counted once and then settled. Marking the (device, template) key
+             * attempted is what keeps a host carried by two overlapping scans in the
+             * same run from being counted as two missing monitors — the same
+             * bookkeeping a real attempt does, for the same reason.
+             */
+            data.existingMonitors.attemptedProvisioningKeys.add(NetworkDeviceAutoImportRuleEngineServiceClass.monitorProvisioningKey(networkDeviceId, templateId));
+            data.result.monitorsFailed++;
+            data.result.monitorProvisioningHalted = true;
+        }
+        if (resolvableTemplateIds.length === 0) {
+            return [];
         }
         /*
          * Do not run the unindexed JSON race-check after this run's monitor-work
          * budget is already exhausted. The device remains unreconciled and the
-         * truncated marker protocol brings it back on the next bounded pass.
+         * truncated marker protocol brings it back on the next bounded pass —
+         * counted, so the operator is told how many devices are still waiting
+         * rather than left to infer it from a total that stopped early.
          */
         if (data.attempts.monitorCount >= MAX_MONITORS_PER_AUTO_IMPORT_RUN) {
-            return true;
+            return resolvableTemplateIds;
         }
         if (!data.isDryRun) {
             await this.refreshExistingMonitorsForDevice({
@@ -987,31 +1172,56 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             });
         }
         if (data.existingMonitors.manuallyMonitoredDeviceIds.has(networkDeviceIdString)) {
-            data.result.monitorsSkippedAlreadyExisting += pendingTemplateIds.length;
-            return false;
+            data.result.monitorsSkippedAlreadyExisting +=
+                resolvableTemplateIds.length;
+            return [];
         }
-        for (const templateId of pendingTemplateIds) {
+        for (let index = 0; index < resolvableTemplateIds.length; index++) {
+            const templateId = resolvableTemplateIds[index];
             const key = NetworkDeviceAutoImportRuleEngineServiceClass.monitorProvisioningKey(networkDeviceId, templateId);
+            /*
+             * A create earlier in THIS device's loop can be the one that trips the
+             * systemic verdict. The rest of its templates are then in exactly the
+             * position of every device after it.
+             */
+            if (data.existingMonitors.isProvisioningHalted) {
+                data.existingMonitors.attemptedProvisioningKeys.add(key);
+                data.result.monitorsFailed++;
+                data.result.monitorProvisioningHalted = true;
+                continue;
+            }
             /* The race-check may have found this key after the initial snapshot. */
             if (data.existingMonitors.autoProvisionedKeys.has(key)) {
                 data.result.monitorsSkippedAlreadyExisting++;
                 continue;
             }
+            /*
+             * Budget spent mid-device. Everything from here on is untouched, so
+             * that is exactly what is left pending for this device.
+             */
             if (data.attempts.monitorCount >= MAX_MONITORS_PER_AUTO_IMPORT_RUN) {
-                return true;
+                return resolvableTemplateIds.slice(index);
             }
             data.attempts.monitorCount++;
             data.existingMonitors.attemptedProvisioningKeys.add(key);
             /*
-             * Invalid or concurrently deleted templates are failed attempts too.
-             * Count them inside the same cap before looking up the cached template;
-             * otherwise one stale rule could walk and log an unbounded estate in a
-             * single API request or worker tick.
+             * A template the run could not load is a fault in the RULE, not in this
+             * host: the rule points at a template that has been deleted, moved to
+             * another project, or changed to a different monitor type since the rule
+             * was saved (the rule form validates all three, but only at save time).
+             * It will be just as unloadable for every other device, so the verdict is
+             * recorded once and every later host short-circuits above — instead of
+             * spending the run's whole monitor budget re-proving it, which is what
+             * turned this into "500 monitors failed, stopped at the run cap, run
+             * again to continue" in issue #3643.
              */
             const template = data.monitorTemplates.get(templateId);
             if (!template) {
+                data.existingMonitors.unresolvableTemplateIds.add(templateId);
                 data.result.monitorsFailed++;
-                logger.error(`Auto-import: Network Device monitor template ${templateId} is missing, deleted, or not valid for project ${data.projectId.toString()}.`, { projectId: data.projectId.toString() });
+                data.result.monitorProvisioningHalted = true;
+                this.recordFailureReason(data.result.monitorFailureReasons, new Error("This rule's Monitor Template could not be loaded. It may have been deleted, moved to another project, or changed to a monitor type other than Network Device. Edit the rule and select a Network Device Monitor Template."));
+                logger.error(`Auto-import: Network Device monitor template ${templateId} is missing, deleted, or not valid for project ${data.projectId.toString()}. No monitors will be provisioned from it for the rest of this run.`, { projectId: data.projectId.toString() });
                 continue;
             }
             if (data.isDryRun) {
@@ -1033,6 +1243,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
                 });
                 data.existingMonitors.autoProvisionedKeys.add(key);
                 data.result.monitorsCreated++;
+                this.recordMonitorCreateSuccess(data.existingMonitors);
             }
             catch (error) {
                 /*
@@ -1061,14 +1272,64 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
                     this.recordExistingMonitor(data.existingMonitors, createdMeanwhile);
                     if (data.existingMonitors.autoProvisionedKeys.has(key)) {
                         data.result.monitorsSkippedAlreadyExisting++;
+                        /*
+                         * Losing this race proves the create pipeline works — somebody
+                         * else just used it — so it is not evidence of a systemic fault.
+                         */
+                        this.recordMonitorCreateSuccess(data.existingMonitors);
                         continue;
                     }
                 }
                 data.result.monitorsFailed++;
+                this.recordFailureReason(data.result.monitorFailureReasons, error);
                 logger.error(`Auto-import: could not create monitor from template ${templateId} for Network Device ${networkDeviceId.toString()} (${data.host.ipAddress}): ${error}`, { projectId: data.projectId.toString() });
+                this.recordMonitorCreateFailure({
+                    state: data.existingMonitors,
+                    result: data.result,
+                    projectId: data.projectId,
+                    error: error,
+                });
             }
         }
-        return false;
+        return [];
+    }
+    /*
+     * A create succeeded (or lost a race to another writer, which proves the
+     * same thing): whatever the last failures were, they were not systemic, and
+     * this run has now seen the create pipeline work at least once.
+     */
+    recordMonitorCreateSuccess(state) {
+        state.consecutiveCreateFailures = 0;
+        state.lastCreateFailureReason = "";
+        state.hasProvisionedAnyMonitor = true;
+    }
+    /*
+     * Decide whether one monitor create failure is this host's problem or the
+     * whole run's. The bar is deliberately high, because giving up early on a
+     * per-host fault would silently cost an estate its monitors — the failure
+     * this fix exists to make visible. All three must hold: the run has never
+     * created a monitor, the last MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES
+     * attempts all failed, and they all failed with the SAME message.
+     */
+    recordMonitorCreateFailure(data) {
+        const reason = this.describeError(data.error);
+        if (reason && reason === data.state.lastCreateFailureReason) {
+            data.state.consecutiveCreateFailures++;
+        }
+        else {
+            data.state.lastCreateFailureReason = reason;
+            data.state.consecutiveCreateFailures = 1;
+        }
+        if (data.state.hasProvisionedAnyMonitor) {
+            return;
+        }
+        if (data.state.consecutiveCreateFailures <
+            MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES) {
+            return;
+        }
+        data.state.isProvisioningHalted = true;
+        data.result.monitorProvisioningHalted = true;
+        logger.error(`Auto-import: ${MAX_CONSECUTIVE_MONITOR_CREATE_FAILURES} Network Device monitor creates failed identically for project ${data.projectId.toString()} with none succeeding ("${reason}"). Treating this as a systemic failure and provisioning no further monitors this run.`, { projectId: data.projectId.toString() });
     }
     /*
      * One host -> one device, with the collision protocol the estate needs:
@@ -1131,6 +1392,7 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             }
             catch (secondError) {
                 data.result.devicesFailed++;
+                this.recordFailureReason(data.result.deviceFailureReasons, secondError);
                 logger.error(`Auto-import: could not create a device for ${data.host.ipAddress} (scan ${(_a = data.scan.id) === null || _a === void 0 ? void 0 : _a.toString()}): ${firstError} / retry: ${secondError}`, { projectId: data.projectId.toString() });
                 return { device: null, wasCreated: false };
             }
@@ -1142,23 +1404,24 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
     /*
      * The one write this engine makes to the scan row: retire the consumed
      * hosts in the stored jsonb (so the Review dialog stops offering them) and
-     * stamp the processed marker — in a single hook-free compare-and-set on
-     * (status, completedAt), for the reasons on processCompletedScan. The
+     * stamp the processed marker — in a single hook-free compare-and-set on the
+     * run state that was read, for the reasons on processCompletedScan. The
      * claim endpoint's hook-free contract is why this must never go through
      * updateOneById: the scan service deliberately has no update-success hooks
      * to piggyback on, and this write must not fire the full pipeline on every
      * worker tick.
      *
      * The CAS defends against exactly one concurrent writer — the ingest
-     * endpoint, which rewrites completedAt with every upload. It cannot
-     * defend against another ENGINE write (which changes neither status nor
-     * completedAt), and does not need to: every path that reaches this method
-     * holds the sweep lock (the worker for its whole sweep, Run Now for a
-     * real run; dry runs never write), so the full-array replace below always
-     * has the row's only engine writer. Weaken that invariant and this
+     * endpoint, which rewrites completedAt with a final upload and the host
+     * counters with a partial one. It cannot defend against another ENGINE
+     * write (which changes none of those columns), and does not need to: every
+     * path that reaches this method holds the sweep lock (the worker for its
+     * whole sweep, Run Now for a real run; dry runs never write), so the
+     * full-array replace below always has the row's only engine writer. Weaken that invariant and this
      * becomes last-writer-wins on the isAlreadyRegistered flips.
      */
     async stampScan(data) {
+        var _a, _b, _c;
         const payload = {};
         if (!data.leaveMarkerUnstamped) {
             payload["autoImportProcessedAt"] = OneUptimeDate.getCurrentDate();
@@ -1193,8 +1456,30 @@ class NetworkDeviceAutoImportRuleEngineServiceClass {
             // Cast: the model's JSON column makes DeepPartial recursion blow up.
             data: payload,
             expectedData: {
-                status: "Completed",
+                /*
+                 * The scan's own status, not the literal "Completed" this used to
+                 * pin. A scan that is still sweeping is importable now (issue
+                 * #3599), and its status is exactly the thing that changes when its
+                 * run ends — so reading it off the row keeps the guard doing what it
+                 * always did rather than making an In Progress stamp always miss.
+                 */
+                status: (_a = data.scan.status) !== null && _a !== void 0 ? _a : null,
                 completedAt: data.scan.completedAt || null,
+                /*
+                 * The counters a running scan's partial uploads move, so a partial
+                 * landing between the read and this write is caught the way a
+                 * completion is: the stamp misses, the marker stays NULL, and the
+                 * next tick processes the newer host list.
+                 *
+                 * Not a perfect version token for an in-progress scan — the
+                 * ICMP-filtered fallback pass finds hosts without advancing
+                 * scannedHostCount, so two partials can carry both counts unchanged.
+                 * The consequence is bounded and self-healing: one batch of hosts
+                 * waits for the next partial upload (which clears the marker again),
+                 * and the run's final result clears it unconditionally.
+                 */
+                scannedHostCount: (_b = data.scan.scannedHostCount) !== null && _b !== void 0 ? _b : null,
+                respondedHostCount: (_c = data.scan.respondedHostCount) !== null && _c !== void 0 ? _c : null,
             },
         });
     }