@oneuptime/common 12.0.21 → 12.0.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/Models/DatabaseModels/Index.ts +4 -0
  2. package/Models/DatabaseModels/NetworkDevice.ts +44 -0
  3. package/Models/DatabaseModels/NetworkDeviceAutoImportRule.ts +564 -0
  4. package/Models/DatabaseModels/NetworkDeviceDiscoveryScan.ts +38 -0
  5. package/Models/DatabaseModels/UserTwoFactorBackupCode.ts +262 -0
  6. package/Server/API/UserTwoFactorBackupCodeAPI.ts +258 -0
  7. package/Server/Infrastructure/Postgres/SchemaMigrations/1789100000000-AddNetworkDeviceAutoImportRule.ts +60 -0
  8. package/Server/Infrastructure/Postgres/SchemaMigrations/1789100000000-AddUserTwoFactorBackupCode.ts +63 -0
  9. package/Server/Infrastructure/Postgres/SchemaMigrations/1789200000000-AddSysObjectIdPatternToAutoImportRule.ts +26 -0
  10. package/Server/Infrastructure/Postgres/SchemaMigrations/1789300000000-AddAutoApplyVendorHealthTemplate.ts +26 -0
  11. package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +8 -0
  12. package/Server/Services/Index.ts +4 -0
  13. package/Server/Services/NetworkDeviceAutoImportRuleEngineService.ts +917 -0
  14. package/Server/Services/NetworkDeviceAutoImportRuleService.ts +181 -0
  15. package/Server/Services/UserService.ts +42 -0
  16. package/Server/Services/UserTwoFactorBackupCodeService.ts +355 -0
  17. package/Server/Utils/Monitor/NetworkInventoryUtil.ts +39 -1
  18. package/Server/Utils/TwoFactorBackupCode.ts +266 -0
  19. package/Tests/Server/API/UserAuthenticationAPI.test.ts +16 -3
  20. package/Tests/Server/API/UserTwoFactorAuthAdminAPI.test.ts +18 -2
  21. package/Tests/Server/API/UserTwoFactorBackupCodeAPI.test.ts +1390 -0
  22. package/Tests/Server/Services/AutoImportScanCredentialSelect.test.ts +205 -0
  23. package/Tests/Server/Services/NetworkDeviceAutoImportRuleEngineService.test.ts +933 -0
  24. package/Tests/Server/Services/UserAuthenticationService.test.ts +23 -1
  25. package/Tests/Server/Services/UserTwoFactorAuthAdmin.test.ts +21 -0
  26. package/Tests/Server/Services/UserTwoFactorBackupCodeAdminSurface.test.ts +919 -0
  27. package/Tests/Server/Services/UserTwoFactorBackupCodeService.test.ts +862 -0
  28. package/Tests/Server/Utils/Monitor/NetworkInventoryUtil.test.ts +97 -1
  29. package/Tests/Server/Utils/TwoFactorBackupCode.test.ts +475 -0
  30. package/Tests/Utils/NetworkDiscovery/AutoImportRuleMatcher.test.ts +516 -0
  31. package/Tests/Utils/NetworkDiscovery/DiscoveredDeviceBuilder.test.ts +272 -0
  32. package/Tests/Utils/NetworkDiscovery/ScanTargetUtil.test.ts +164 -0
  33. package/Types/Email/EmailTemplateType.ts +2 -0
  34. package/Types/NetworkAutomation/RuleRunResult.ts +84 -0
  35. package/Types/Permission.ts +47 -0
  36. package/Types/UserAuthenticationStatus.ts +16 -0
  37. package/Utils/NetworkDiscovery/AutoImportRuleMatcher.ts +257 -0
  38. package/Utils/NetworkDiscovery/DiscoveredDeviceBuilder.ts +183 -0
  39. package/Utils/NetworkDiscovery/DiscoveredHostUtil.ts +95 -0
  40. package/Utils/NetworkDiscovery/DiscoveryImportEligibility.ts +66 -0
  41. package/Utils/NetworkDiscovery/ScanTargetUtil.ts +82 -0
  42. package/build/dist/Models/DatabaseModels/Index.js +4 -0
  43. package/build/dist/Models/DatabaseModels/Index.js.map +1 -1
  44. package/build/dist/Models/DatabaseModels/NetworkDevice.js +45 -0
  45. package/build/dist/Models/DatabaseModels/NetworkDevice.js.map +1 -1
  46. package/build/dist/Models/DatabaseModels/NetworkDeviceAutoImportRule.js +589 -0
  47. package/build/dist/Models/DatabaseModels/NetworkDeviceAutoImportRule.js.map +1 -0
  48. package/build/dist/Models/DatabaseModels/NetworkDeviceDiscoveryScan.js +28 -0
  49. package/build/dist/Models/DatabaseModels/NetworkDeviceDiscoveryScan.js.map +1 -1
  50. package/build/dist/Models/DatabaseModels/UserTwoFactorBackupCode.js +277 -0
  51. package/build/dist/Models/DatabaseModels/UserTwoFactorBackupCode.js.map +1 -0
  52. package/build/dist/Server/API/UserTwoFactorBackupCodeAPI.js +201 -0
  53. package/build/dist/Server/API/UserTwoFactorBackupCodeAPI.js.map +1 -0
  54. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1789100000000-AddNetworkDeviceAutoImportRule.js +35 -0
  55. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1789100000000-AddNetworkDeviceAutoImportRule.js.map +1 -0
  56. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1789100000000-AddUserTwoFactorBackupCode.js +46 -0
  57. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1789100000000-AddUserTwoFactorBackupCode.js.map +1 -0
  58. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1789200000000-AddSysObjectIdPatternToAutoImportRule.js +19 -0
  59. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1789200000000-AddSysObjectIdPatternToAutoImportRule.js.map +1 -0
  60. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1789300000000-AddAutoApplyVendorHealthTemplate.js +19 -0
  61. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/1789300000000-AddAutoApplyVendorHealthTemplate.js.map +1 -0
  62. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js +8 -0
  63. package/build/dist/Server/Infrastructure/Postgres/SchemaMigrations/Index.js.map +1 -1
  64. package/build/dist/Server/Services/Index.js +4 -0
  65. package/build/dist/Server/Services/Index.js.map +1 -1
  66. package/build/dist/Server/Services/NetworkDeviceAutoImportRuleEngineService.js +711 -0
  67. package/build/dist/Server/Services/NetworkDeviceAutoImportRuleEngineService.js.map +1 -0
  68. package/build/dist/Server/Services/NetworkDeviceAutoImportRuleService.js +155 -0
  69. package/build/dist/Server/Services/NetworkDeviceAutoImportRuleService.js.map +1 -0
  70. package/build/dist/Server/Services/UserService.js +39 -0
  71. package/build/dist/Server/Services/UserService.js.map +1 -1
  72. package/build/dist/Server/Services/UserTwoFactorBackupCodeService.js +327 -0
  73. package/build/dist/Server/Services/UserTwoFactorBackupCodeService.js.map +1 -0
  74. package/build/dist/Server/Utils/Monitor/NetworkInventoryUtil.js +26 -0
  75. package/build/dist/Server/Utils/Monitor/NetworkInventoryUtil.js.map +1 -1
  76. package/build/dist/Server/Utils/TwoFactorBackupCode.js +269 -0
  77. package/build/dist/Server/Utils/TwoFactorBackupCode.js.map +1 -0
  78. package/build/dist/Types/Email/EmailTemplateType.js +2 -0
  79. package/build/dist/Types/Email/EmailTemplateType.js.map +1 -1
  80. package/build/dist/Types/NetworkAutomation/RuleRunResult.js +24 -0
  81. package/build/dist/Types/NetworkAutomation/RuleRunResult.js.map +1 -1
  82. package/build/dist/Types/Permission.js +41 -0
  83. package/build/dist/Types/Permission.js.map +1 -1
  84. package/build/dist/Utils/NetworkDiscovery/AutoImportRuleMatcher.js +149 -0
  85. package/build/dist/Utils/NetworkDiscovery/AutoImportRuleMatcher.js.map +1 -0
  86. package/build/dist/Utils/NetworkDiscovery/DiscoveredDeviceBuilder.js +121 -0
  87. package/build/dist/Utils/NetworkDiscovery/DiscoveredDeviceBuilder.js.map +1 -0
  88. package/build/dist/Utils/NetworkDiscovery/DiscoveredHostUtil.js +75 -0
  89. package/build/dist/Utils/NetworkDiscovery/DiscoveredHostUtil.js.map +1 -0
  90. package/build/dist/Utils/NetworkDiscovery/DiscoveryImportEligibility.js +59 -0
  91. package/build/dist/Utils/NetworkDiscovery/DiscoveryImportEligibility.js.map +1 -0
  92. package/build/dist/Utils/NetworkDiscovery/ScanTargetUtil.js +60 -0
  93. package/build/dist/Utils/NetworkDiscovery/ScanTargetUtil.js.map +1 -1
  94. package/package.json +1 -1
@@ -0,0 +1,711 @@
1
+ var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
2
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
3
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
4
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
5
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
6
+ };
7
+ var __metadata = (this && this.__metadata) || function (k, v) {
8
+ if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
9
+ };
10
+ import NetworkDeviceAutoImportRuleService from "./NetworkDeviceAutoImportRuleService";
11
+ import NetworkDeviceDiscoveryScanService from "./NetworkDeviceDiscoveryScanService";
12
+ import NetworkDeviceService from "./NetworkDeviceService";
13
+ import BadDataException from "../../Types/Exception/BadDataException";
14
+ import LIMIT_MAX from "../../Types/Database/LimitMax";
15
+ import OneUptimeDate from "../../Types/Date";
16
+ import SortOrder from "../../Types/BaseDatabase/SortOrder";
17
+ import { MAX_MATCHED_IP_SAMPLE, } from "../../Types/NetworkAutomation/RuleRunResult";
18
+ import AutoImportRuleMatcher from "../../Utils/NetworkDiscovery/AutoImportRuleMatcher";
19
+ import { buildFallbackDeviceName, buildNetworkDeviceFromDiscoveredHost, } from "../../Utils/NetworkDiscovery/DiscoveredDeviceBuilder";
20
+ import { normalizeDiscoveredHosts } from "../../Utils/NetworkDiscovery/DiscoveredHostUtil";
21
+ import Semaphore from "../Infrastructure/Semaphore";
22
+ import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
23
+ import logger from "../Utils/Logger";
24
+ /*
25
+ * The engine behind network device auto-import rules (issue #3378): reads a
26
+ * completed discovery scan's results, asks the project's rules which hosts
27
+ * import, and creates the Network Devices the manual Review dialog would
28
+ * have — through the same builder, so a rule-imported host and a
29
+ * hand-imported host are the same device. Site assignment, owners and labels
30
+ * then apply themselves through NetworkDeviceService.onCreateSuccess's
31
+ * existing rule chain, exactly as they do for a manual import.
32
+ *
33
+ * Two entry points, mirroring the label-rule engine:
34
+ *
35
+ * - processCompletedScan: the automatic path, called by the
36
+ * NetworkDeviceDiscovery worker for each Completed scan whose
37
+ * autoImportProcessedAt marker is NULL.
38
+ * - applyRuleToCompletedScans: the manual "Run Now" (and its dry run),
39
+ * applying ONE rule to the project's existing completed scans.
40
+ *
41
+ * Everything here is idempotent by construction: a device exists per
42
+ * (project, address), recurring scans re-report the same hosts every
43
+ * interval, and both paths skip hosts whose address is already registered —
44
+ * so seeing the same results twice creates nothing twice.
45
+ */
46
+ /*
47
+ * The scan columns an import copies onto an SNMP device — the same set the
48
+ * probe claim endpoint selects, and for the same reason: an unselected
49
+ * credential column arrives undefined and would be silently NOT copied,
50
+ * leaving the device unable to poll forever. Exported so a test can pin this
51
+ * list against the builder's DiscoveredDeviceScanSource shape.
52
+ */
53
+ export const AUTO_IMPORT_SCAN_CREDENTIAL_SELECT = {
54
+ probeId: true,
55
+ snmpVersion: true,
56
+ snmpCommunityString: true,
57
+ snmpPort: true,
58
+ snmpV3SecurityLevel: true,
59
+ snmpV3Username: true,
60
+ snmpV3AuthProtocol: true,
61
+ snmpV3AuthKey: true,
62
+ snmpV3PrivProtocol: true,
63
+ snmpV3PrivKey: true,
64
+ };
65
+ /*
66
+ * Devices one pass will create from one scan. Deliberately conservative:
67
+ * every create runs the full service pipeline plus a detached
68
+ * label/owner/site rule chain, so hundreds per pass is already a burst of
69
+ * thousands of queries. The cap loses nothing — a truncated pass leaves the
70
+ * scan's marker unstamped, the next worker tick resumes, and idempotency
71
+ * skips what this pass already created — but it turns "a typo'd rule meets a
72
+ * /16" from an outage into a few paced ticks an operator can catch.
73
+ */
74
+ export const MAX_DEVICES_PER_AUTO_IMPORT_RUN = 500;
75
+ // Scans one manual "Run Now" will read, oldest results last.
76
+ export const MAX_SCANS_PER_AUTO_IMPORT_RULE_RUN = 100;
77
+ /*
78
+ * Results older than this are stamped processed WITHOUT importing. A late
79
+ * result can land on a reaper-Failed scan hours after its sweep ran (the
80
+ * ingest endpoint accepts it as the truth about that run), and a worker
81
+ * outage can leave a backlog — in both cases silently auto-importing an
82
+ * hours-old host list is the wrong surprise. The operator can still reach
83
+ * old results deliberately, through "Run Now".
84
+ */
85
+ export const MAX_RESULT_AGE_IN_HOURS = 24;
86
+ /*
87
+ * The Redis lock every path that CREATES devices from scan results runs
88
+ * under — the worker sweep and the manual Run Now alike. The engine's
89
+ * idempotency is check-then-create with no DB backstop (device names are
90
+ * app-level unique, addresses not unique at all), so two concurrent passes
91
+ * over the same results can both pass the check and both insert. The worker
92
+ * holds this lock for its whole sweep; a real (non-dry) manual run takes
93
+ * the SAME lock, so manual-vs-sweep and manual-vs-manual can never
94
+ * interleave — which also means the scan jsonb write-back has exactly one
95
+ * engine writer at a time. Defined here, imported by the worker, so the two
96
+ * can never drift onto different locks.
97
+ */
98
+ export const AUTO_IMPORT_SWEEP_LOCK_KEY = "NetworkDeviceDiscovery:ProcessAutoImportRules";
99
+ export const AUTO_IMPORT_SWEEP_LOCK_NAMESPACE = "Workers.Cron";
100
+ export const AUTO_IMPORT_SWEEP_LOCK_TIMEOUT_MS = 11 * 60 * 1000;
101
+ class NetworkDeviceAutoImportRuleEngineServiceClass {
102
+ /**
103
+ * The automatic path: apply the project's enabled rules to one Completed
104
+ * scan, then stamp the scan's autoImportProcessedAt marker.
105
+ *
106
+ * Returns null when there was nothing to do (scan gone, superseded,
107
+ * already processed, no import rules, or results too old) and the run's
108
+ * counters otherwise. Throws only on unexpected failures — the caller
109
+ * (the worker sweep) isolates those per scan.
110
+ *
111
+ * The marker protocol, and why every stamp is a compare-and-set on
112
+ * (status, completedAt): the ingest endpoint clears the marker in the same
113
+ * write that stores new results, so "marker is NULL" always means "the
114
+ * results now on the row have not been processed". If new results land
115
+ * while this pass is reading the old ones, the CAS misses, the marker
116
+ * stays NULL, and the next tick processes the new upload — the stamp can
117
+ * never retire results it did not see, and the jsonb write-back can never
118
+ * clobber a newer host list. A truncated pass that created something
119
+ * stamps nothing on purpose: the NULL marker is what makes the next tick
120
+ * resume. (A truncated pass where every create FAILED stamps anyway — see
121
+ * the zero-progress note at the stamp below.)
122
+ */
123
+ async processCompletedScan(data) {
124
+ var _a, _b;
125
+ const scan = await NetworkDeviceDiscoveryScanService.findOneBy({
126
+ query: {
127
+ _id: data.scanId,
128
+ status: "Completed",
129
+ },
130
+ select: Object.assign({ _id: true, projectId: true, status: true, completedAt: true, autoImportProcessedAt: true, discoveredDevices: true }, AUTO_IMPORT_SCAN_CREDENTIAL_SELECT),
131
+ props: {
132
+ isRoot: true,
133
+ },
134
+ });
135
+ /*
136
+ * Re-checked on this fresh read even though the sweep query already
137
+ * filtered: the sweep's snapshot ages while earlier scans in it are
138
+ * processed, and a scan can complete a new run (or be re-queued) in
139
+ * that window.
140
+ */
141
+ if (!scan || !scan.projectId || scan.autoImportProcessedAt) {
142
+ return null;
143
+ }
144
+ const projectId = scan.projectId;
145
+ const rules = await this.loadEnabledRules(projectId);
146
+ const hasImportRules = rules.some((rule) => {
147
+ return !rule.isExclusion;
148
+ });
149
+ if (!hasImportRules) {
150
+ // Nothing can import; retire these results so they never accumulate.
151
+ await this.stampScan({ scan: scan, createdIpAddresses: [] });
152
+ return null;
153
+ }
154
+ if (this.isTooOldToAutoImport(scan)) {
155
+ logger.warn(`Auto-import: scan ${(_a = scan.id) === null || _a === void 0 ? void 0 : _a.toString()} completed more than ${MAX_RESULT_AGE_IN_HOURS} hours ago; stamping it processed without importing. Use the rule's Run Now to import old results deliberately.`, { projectId: projectId.toString() });
156
+ await this.stampScan({ scan: scan, createdIpAddresses: [] });
157
+ return null;
158
+ }
159
+ const existingHostnames = await this.getExistingHostnames({
160
+ projectId: projectId,
161
+ cache: data.existingHostnamesByProjectId,
162
+ });
163
+ const result = this.emptyResult(false);
164
+ const attempts = { count: 0 };
165
+ const createdIpAddresses = await this.importHostsFromScan({
166
+ scan: scan,
167
+ rules: rules,
168
+ existingHostnames: existingHostnames,
169
+ result: result,
170
+ attempts: attempts,
171
+ isDryRun: false,
172
+ });
173
+ /*
174
+ * A truncated pass that made progress leaves the marker NULL so the
175
+ * next tick resumes where the cap stopped it. A truncated pass that
176
+ * created NOTHING — every attempt failed — is stamped anyway: resuming
177
+ * it would repeat the identical doomed pass every minute forever, and
178
+ * the sub-cap case already retires all-failing hosts un-imported in one
179
+ * pass, so stamping keeps the two consistent. Run Now remains the
180
+ * deliberate way back into a stamped scan.
181
+ */
182
+ const shouldResume = result.isTruncated && result.devicesCreated > 0;
183
+ if (result.isTruncated && result.devicesCreated === 0) {
184
+ logger.error(`Auto-import: scan ${(_b = scan.id) === null || _b === void 0 ? void 0 : _b.toString()} hit the per-pass cap with every create failing (${result.devicesFailed} failure(s)); stamping it processed so the sweep does not retry it forever. Inspect the failures and use the rule's Run Now to retry deliberately.`, { projectId: projectId.toString() });
185
+ }
186
+ await this.stampScan({
187
+ scan: scan,
188
+ createdIpAddresses: createdIpAddresses,
189
+ leaveMarkerUnstamped: shouldResume,
190
+ });
191
+ return result;
192
+ }
193
+ /*
194
+ * The manual path: apply ONE rule to the project's completed scans, newest
195
+ * first — or, as a dry run, evaluate everything and write nothing. The
196
+ * project's enabled exclusion rules always ride along: "Run Now" must not
197
+ * be a way around a veto.
198
+ *
199
+ * Markers are deliberately not stamped here. They belong to the automatic
200
+ * path's bookkeeping, and a manual run of one rule has not done what the
201
+ * full rule set would.
202
+ */
203
+ async applyRuleToCompletedScans(data) {
204
+ const rule = await NetworkDeviceAutoImportRuleService.findOneBy({
205
+ query: {
206
+ _id: data.ruleId,
207
+ projectId: data.projectId,
208
+ },
209
+ select: {
210
+ _id: true,
211
+ name: true,
212
+ isEnabled: true,
213
+ isExclusion: true,
214
+ ipMatchTarget: true,
215
+ sysNamePattern: true,
216
+ sysDescrPattern: true,
217
+ sysObjectIdPattern: true,
218
+ includePingOnlyHosts: true,
219
+ },
220
+ props: { isRoot: true },
221
+ });
222
+ if (!rule) {
223
+ throw new BadDataException("Auto-import rule not found.");
224
+ }
225
+ /*
226
+ * A DISABLED rule can still be dry-run — that is the point of the dry
227
+ * run: answer "what would this rule import" BEFORE enabling it against
228
+ * live scans. Only a real run of a disabled rule contradicts the toggle
229
+ * the user can see next to the button.
230
+ */
231
+ if (!rule.isEnabled && !data.isDryRun) {
232
+ throw new BadDataException("This auto-import rule is disabled. Enable it before running it, or use Dry Run to preview what it would import.");
233
+ }
234
+ if (rule.isExclusion) {
235
+ throw new BadDataException("An exclusion rule only vetoes other rules and imports nothing by itself, so running it alone would do nothing. Run one of the import rules it applies to instead.");
236
+ }
237
+ /*
238
+ * A real run creates devices, so it must hold the same lock the worker
239
+ * sweep holds (see AUTO_IMPORT_SWEEP_LOCK_KEY): without it, a Run Now
240
+ * clicked while the every-minute sweep is processing the same
241
+ * marker-NULL scan races the check-then-create idempotency and both
242
+ * paths insert the same host. A dry run writes nothing and skips the
243
+ * lock.
244
+ */
245
+ let mutex = null;
246
+ if (!data.isDryRun) {
247
+ try {
248
+ mutex = await Semaphore.lock({
249
+ key: AUTO_IMPORT_SWEEP_LOCK_KEY,
250
+ namespace: AUTO_IMPORT_SWEEP_LOCK_NAMESPACE,
251
+ lockTimeout: AUTO_IMPORT_SWEEP_LOCK_TIMEOUT_MS,
252
+ acquireAttemptsLimit: 3,
253
+ });
254
+ }
255
+ catch (err) {
256
+ logger.debug(`Auto-import Run Now: could not acquire the sweep lock: ${err}`);
257
+ throw new BadDataException("An automatic import sweep is currently running. Try again in a minute — hosts it imports are skipped by your run anyway.");
258
+ }
259
+ }
260
+ try {
261
+ return await this.runRuleAgainstCompletedScans({
262
+ rule: rule,
263
+ projectId: data.projectId,
264
+ isDryRun: data.isDryRun,
265
+ });
266
+ }
267
+ finally {
268
+ if (mutex) {
269
+ try {
270
+ await Semaphore.release(mutex);
271
+ }
272
+ catch (err) {
273
+ logger.error(`Auto-import Run Now: error releasing the sweep lock: ${err}`);
274
+ }
275
+ }
276
+ }
277
+ }
278
+ async runRuleAgainstCompletedScans(data) {
279
+ const exclusionRules = (await this.loadEnabledRules(data.projectId)).filter((candidate) => {
280
+ return Boolean(candidate.isExclusion);
281
+ });
282
+ const rules = [
283
+ data.rule,
284
+ ...exclusionRules,
285
+ ];
286
+ const existingHostnames = await this.loadExistingHostnames(data.projectId);
287
+ const result = this.emptyResult(data.isDryRun);
288
+ const attempts = { count: 0 };
289
+ /*
290
+ * Newest results first: a manual run is "bring the estate up to date",
291
+ * and the newest scan of a subnet supersedes its older runs — with
292
+ * idempotency, whichever result mentions a host first imports it and
293
+ * later mentions skip. Minimal columns only — the full row, with its
294
+ * multi-megabyte discoveredDevices jsonb, is re-read one scan at a time
295
+ * inside the loop so only one result set is ever resident (the same
296
+ * two-phase shape the worker sweep uses, for the same reason).
297
+ */
298
+ const scanStubs = await NetworkDeviceDiscoveryScanService.findBy({
299
+ query: {
300
+ projectId: data.projectId,
301
+ status: "Completed",
302
+ },
303
+ select: {
304
+ _id: true,
305
+ },
306
+ sort: {
307
+ completedAt: SortOrder.Descending,
308
+ },
309
+ limit: MAX_SCANS_PER_AUTO_IMPORT_RULE_RUN + 1,
310
+ skip: 0,
311
+ props: { isRoot: true },
312
+ });
313
+ // One extra row settles "were there more scans" honestly.
314
+ result.hasMoreScans = scanStubs.length > MAX_SCANS_PER_AUTO_IMPORT_RULE_RUN;
315
+ if (result.hasMoreScans) {
316
+ scanStubs.pop();
317
+ }
318
+ for (const scanStub of scanStubs) {
319
+ const scan = await NetworkDeviceDiscoveryScanService.findOneBy({
320
+ query: {
321
+ _id: scanStub.id,
322
+ projectId: data.projectId,
323
+ status: "Completed",
324
+ },
325
+ select: Object.assign({ _id: true, projectId: true, status: true, completedAt: true, discoveredDevices: true }, AUTO_IMPORT_SCAN_CREDENTIAL_SELECT),
326
+ props: { isRoot: true },
327
+ });
328
+ // Re-queued or deleted since the stub query — nothing to evaluate.
329
+ if (!scan) {
330
+ continue;
331
+ }
332
+ const createdIpAddresses = await this.importHostsFromScan({
333
+ scan: scan,
334
+ rules: rules,
335
+ existingHostnames: existingHostnames,
336
+ result: result,
337
+ attempts: attempts,
338
+ isDryRun: data.isDryRun,
339
+ });
340
+ if (createdIpAddresses.length > 0) {
341
+ /*
342
+ * Same CAS write-back as the automatic path, so the Review dialog
343
+ * does not re-offer (pre-checked!) hosts this run just imported. A
344
+ * miss only means fresher results arrived, which recomputed the
345
+ * flags anyway. No concurrent engine writer can clobber this: the
346
+ * write happens only on real runs, which hold the sweep lock.
347
+ */
348
+ await this.stampScan({
349
+ scan: scan,
350
+ createdIpAddresses: createdIpAddresses,
351
+ leaveMarkerUnstamped: true,
352
+ });
353
+ }
354
+ // The device cap is shared across the whole run, dry or real.
355
+ if (result.isTruncated) {
356
+ break;
357
+ }
358
+ /*
359
+ * Yield the event loop between scans: this runs inside an API request
360
+ * handler, and evaluating scan after scan of jsonb back-to-back would
361
+ * starve every other request the process is serving. setTimeout
362
+ * rather than setImmediate because the latter is a bare Node global
363
+ * that the jsdom-based test environment does not define.
364
+ */
365
+ await new Promise((resolve) => {
366
+ setTimeout(resolve, 0);
367
+ });
368
+ }
369
+ return result;
370
+ }
371
+ /*
372
+ * ------------------------------------------------------------------
373
+ * Internals
374
+ * ------------------------------------------------------------------
375
+ */
376
+ emptyResult(isDryRun) {
377
+ return {
378
+ hostsEvaluated: 0,
379
+ hostsMatched: 0,
380
+ hostsExcluded: 0,
381
+ hostsSkippedAlreadyRegistered: 0,
382
+ devicesCreated: 0,
383
+ devicesFailed: 0,
384
+ isTruncated: false,
385
+ hasMoreScans: false,
386
+ isDryRun: isDryRun,
387
+ matchedIpAddressSample: [],
388
+ };
389
+ }
390
+ /*
391
+ * ALL enabled rules of the project, paged to completeness — never a capped
392
+ * subset. The label/owner engines cap their fetch at 100 and a dropped
393
+ * rule there merely adds nothing; here a dropped EXCLUSION rule fails
394
+ * OPEN, importing the exact hosts the operator vetoed, nondeterministically
395
+ * (an uncapped unsorted fetch returns Postgres's choice of rows). Rule
396
+ * rows are a handful of scalar columns, so completeness is one query in
397
+ * any realistic project.
398
+ */
399
+ async loadEnabledRules(projectId) {
400
+ const rules = [];
401
+ for (let skip = 0;; skip += LIMIT_MAX) {
402
+ const page = await NetworkDeviceAutoImportRuleService.findBy({
403
+ query: {
404
+ projectId: projectId,
405
+ isEnabled: true,
406
+ },
407
+ select: {
408
+ _id: true,
409
+ name: true,
410
+ isExclusion: true,
411
+ ipMatchTarget: true,
412
+ sysNamePattern: true,
413
+ sysDescrPattern: true,
414
+ sysObjectIdPattern: true,
415
+ includePingOnlyHosts: true,
416
+ },
417
+ sort: {
418
+ createdAt: SortOrder.Ascending,
419
+ },
420
+ limit: LIMIT_MAX,
421
+ skip: skip,
422
+ props: { isRoot: true },
423
+ });
424
+ rules.push(...page);
425
+ if (page.length < LIMIT_MAX) {
426
+ break;
427
+ }
428
+ }
429
+ return rules;
430
+ }
431
+ isTooOldToAutoImport(scan) {
432
+ if (!scan.completedAt) {
433
+ return false;
434
+ }
435
+ return OneUptimeDate.isBefore(OneUptimeDate.fromString(scan.completedAt), OneUptimeDate.getSomeHoursAgo(MAX_RESULT_AGE_IN_HOURS));
436
+ }
437
+ async getExistingHostnames(data) {
438
+ const key = data.projectId.toString();
439
+ const cached = data.cache.get(key);
440
+ if (cached) {
441
+ return cached;
442
+ }
443
+ const loaded = await this.loadExistingHostnames(data.projectId);
444
+ data.cache.set(key, loaded);
445
+ return loaded;
446
+ }
447
+ /*
448
+ * Every device address in the project — the same paged walk the probe
449
+ * ingest endpoint does to stamp isAlreadyRegistered, for the same reason:
450
+ * a truncated answer produces duplicate devices, which is worse than a
451
+ * slow answer. Sorted for stable paging.
452
+ */
453
+ async loadExistingHostnames(projectId) {
454
+ const existingHostnames = new Set();
455
+ for (let skip = 0;; skip += LIMIT_MAX) {
456
+ const existing = await NetworkDeviceService.findBy({
457
+ query: {
458
+ projectId: projectId,
459
+ },
460
+ select: {
461
+ hostname: true,
462
+ },
463
+ sort: {
464
+ createdAt: SortOrder.Ascending,
465
+ },
466
+ limit: LIMIT_MAX,
467
+ skip: skip,
468
+ props: {
469
+ isRoot: true,
470
+ },
471
+ });
472
+ for (const device of existing) {
473
+ if (device.hostname) {
474
+ existingHostnames.add(device.hostname);
475
+ }
476
+ }
477
+ if (existing.length < LIMIT_MAX) {
478
+ break;
479
+ }
480
+ }
481
+ return existingHostnames;
482
+ }
483
+ /*
484
+ * The core loop: evaluate every discovered host of one scan against the
485
+ * rule set, create what should import, and account for every host in the
486
+ * shared result. Returns the addresses actually created (for the jsonb
487
+ * write-back). Mutates `existingHostnames` as it creates, which is what
488
+ * makes duplicate rows, overlapping scans in one sweep, and repeat runs
489
+ * idempotent.
490
+ */
491
+ async importHostsFromScan(data) {
492
+ const scan = data.scan;
493
+ const result = data.result;
494
+ if (!scan.projectId) {
495
+ return [];
496
+ }
497
+ /*
498
+ * The jsonb is the probe's payload stored verbatim; normalisation fixes
499
+ * the null rows, non-string addresses and duplicate-row flags the
500
+ * Review dialog already learned to survive. See DiscoveredHostUtil.
501
+ */
502
+ const hosts = normalizeDiscoveredHosts(scan.discoveredDevices || []);
503
+ const createdIpAddresses = [];
504
+ for (const host of hosts) {
505
+ /*
506
+ * The address becomes the device's varchar(100) hostname, on which
507
+ * the create THROWS rather than truncates — and no real address is
508
+ * anywhere near that long, so an over-long "address" is junk that
509
+ * would fail identically on every pass. Refusing it here (instead of
510
+ * counting a devicesFailed per pass forever) is what keeps a
511
+ * pathological scan from failing the same way every worker tick.
512
+ */
513
+ if (!host.ipAddress || host.ipAddress.length > 100) {
514
+ continue;
515
+ }
516
+ result.hostsEvaluated++;
517
+ const evaluation = AutoImportRuleMatcher.evaluateHost(data.rules, host);
518
+ if (evaluation.excludedByRule) {
519
+ result.hostsExcluded++;
520
+ continue;
521
+ }
522
+ if (!evaluation.shouldImport) {
523
+ continue;
524
+ }
525
+ result.hostsMatched++;
526
+ /*
527
+ * The frozen isAlreadyRegistered flag AND the live set: the flag is a
528
+ * point-in-time answer from the last upload, the set covers devices
529
+ * created since — including by this very run, which is what collapses
530
+ * duplicate rows and overlapping scans into one device.
531
+ */
532
+ if (host.isAlreadyRegistered ||
533
+ data.existingHostnames.has(host.ipAddress)) {
534
+ result.hostsSkippedAlreadyRegistered++;
535
+ continue;
536
+ }
537
+ if (data.attempts.count >= MAX_DEVICES_PER_AUTO_IMPORT_RUN) {
538
+ result.isTruncated = true;
539
+ break;
540
+ }
541
+ if (result.matchedIpAddressSample.length < MAX_MATCHED_IP_SAMPLE) {
542
+ result.matchedIpAddressSample.push(host.ipAddress);
543
+ }
544
+ if (data.isDryRun) {
545
+ /*
546
+ * A dry run reports the device as "created" in no counter at all —
547
+ * hostsMatched minus hostsSkippedAlreadyRegistered is exactly what a
548
+ * real run would attempt, and the sample above says which hosts.
549
+ * The address still joins the set, simulating the create's dedupe:
550
+ * without this, a host on duplicate rows (or in two overlapping
551
+ * scans) would be counted as importable once per appearance, and
552
+ * the dry run would promise more than the real run does.
553
+ */
554
+ data.attempts.count++;
555
+ data.existingHostnames.add(host.ipAddress);
556
+ continue;
557
+ }
558
+ const wasCreated = await this.createDeviceForHost({
559
+ projectId: scan.projectId,
560
+ scan: scan,
561
+ host: host,
562
+ existingHostnames: data.existingHostnames,
563
+ result: result,
564
+ attempts: data.attempts,
565
+ });
566
+ if (wasCreated) {
567
+ createdIpAddresses.push(host.ipAddress);
568
+ }
569
+ }
570
+ return createdIpAddresses;
571
+ }
572
+ /*
573
+ * One host -> one device, with the collision protocol the estate needs:
574
+ *
575
+ * Device names are unique per project (an app-level check, not a DB
576
+ * constraint) while addresses are not unique at all, so the failure modes
577
+ * split. A create that fails re-checks the ADDRESS first — losing a race
578
+ * to a concurrent import of the same host must read as "already
579
+ * registered", not spawn a renamed twin. Only when the address is still
580
+ * free is the name the problem (two hosts legitimately sharing a sysName),
581
+ * and one retry under the address-suffixed fallback name settles it.
582
+ */
583
+ async createDeviceForHost(data) {
584
+ var _a;
585
+ data.attempts.count++;
586
+ const attemptCreate = async (name) => {
587
+ const device = buildNetworkDeviceFromDiscoveredHost({
588
+ projectId: data.projectId,
589
+ host: data.host,
590
+ scan: data.scan,
591
+ name: name,
592
+ /*
593
+ * Zero-touch all the way down: nobody reviews an auto-imported
594
+ * device, so nobody would ever click the vendor-template banner.
595
+ * The first poll that fingerprints the vendor seeds the Health
596
+ * OIDs instead (NetworkInventoryUtil).
597
+ */
598
+ autoApplyVendorHealthTemplate: true,
599
+ });
600
+ await NetworkDeviceService.create({
601
+ data: device,
602
+ props: {
603
+ isRoot: true,
604
+ },
605
+ });
606
+ };
607
+ try {
608
+ await attemptCreate();
609
+ }
610
+ catch (firstError) {
611
+ const registeredMeanwhile = await NetworkDeviceService.findOneBy({
612
+ query: {
613
+ projectId: data.projectId,
614
+ hostname: data.host.ipAddress,
615
+ },
616
+ select: { _id: true },
617
+ props: { isRoot: true },
618
+ });
619
+ if (registeredMeanwhile) {
620
+ // Skipped is a sub-bucket of matched, same as the other run results.
621
+ data.result.hostsSkippedAlreadyRegistered++;
622
+ data.existingHostnames.add(data.host.ipAddress);
623
+ return false;
624
+ }
625
+ try {
626
+ await attemptCreate(buildFallbackDeviceName(data.host));
627
+ }
628
+ catch (secondError) {
629
+ data.result.devicesFailed++;
630
+ logger.error(`Auto-import: could not create a device for ${data.host.ipAddress} (scan ${(_a = data.scan.id) === null || _a === void 0 ? void 0 : _a.toString()}): ${firstError} / retry: ${secondError}`, { projectId: data.projectId.toString() });
631
+ return false;
632
+ }
633
+ }
634
+ data.result.devicesCreated++;
635
+ data.existingHostnames.add(data.host.ipAddress);
636
+ return true;
637
+ }
638
+ /*
639
+ * The one write this engine makes to the scan row: retire the consumed
640
+ * hosts in the stored jsonb (so the Review dialog stops offering them) and
641
+ * stamp the processed marker — in a single hook-free compare-and-set on
642
+ * (status, completedAt), for the reasons on processCompletedScan. The
643
+ * claim endpoint's hook-free contract is why this must never go through
644
+ * updateOneById: the scan service deliberately has no update-success hooks
645
+ * to piggyback on, and this write must not fire the full pipeline on every
646
+ * worker tick.
647
+ *
648
+ * The CAS defends against exactly one concurrent writer — the ingest
649
+ * endpoint, which rewrites completedAt with every upload. It cannot
650
+ * defend against another ENGINE write (which changes neither status nor
651
+ * completedAt), and does not need to: every path that reaches this method
652
+ * holds the sweep lock (the worker for its whole sweep, Run Now for a
653
+ * real run; dry runs never write), so the full-array replace below always
654
+ * has the row's only engine writer. Weaken that invariant and this
655
+ * becomes last-writer-wins on the isAlreadyRegistered flips.
656
+ */
657
+ async stampScan(data) {
658
+ const payload = {};
659
+ if (!data.leaveMarkerUnstamped) {
660
+ payload["autoImportProcessedAt"] = OneUptimeDate.getCurrentDate();
661
+ }
662
+ if (data.createdIpAddresses.length > 0) {
663
+ const createdSet = new Set(data.createdIpAddresses);
664
+ /*
665
+ * Mapped over the RAW stored rows, not the normalised copies: the
666
+ * write-back must preserve whatever shape the probe sent, only
667
+ * flipping isAlreadyRegistered on rows whose (trimmed) address was
668
+ * imported. Junk rows pass through untouched.
669
+ */
670
+ const restamped = (data.scan.discoveredDevices || []).map((row) => {
671
+ var _a;
672
+ if (!row || typeof row !== "object") {
673
+ return row;
674
+ }
675
+ const hostRow = row;
676
+ const ipAddress = String((_a = hostRow.ipAddress) !== null && _a !== void 0 ? _a : "").trim();
677
+ if (!ipAddress || !createdSet.has(ipAddress)) {
678
+ return row;
679
+ }
680
+ return Object.assign(Object.assign({}, hostRow), { isAlreadyRegistered: true });
681
+ });
682
+ payload["discoveredDevices"] = restamped;
683
+ }
684
+ if (Object.keys(payload).length === 0) {
685
+ return;
686
+ }
687
+ await NetworkDeviceDiscoveryScanService.updateColumnsByIdWithoutHooks({
688
+ id: data.scan.id,
689
+ // Cast: the model's JSON column makes DeepPartial recursion blow up.
690
+ data: payload,
691
+ expectedData: {
692
+ status: "Completed",
693
+ completedAt: data.scan.completedAt || null,
694
+ },
695
+ });
696
+ }
697
+ }
698
+ __decorate([
699
+ CaptureSpan(),
700
+ __metadata("design:type", Function),
701
+ __metadata("design:paramtypes", [Object]),
702
+ __metadata("design:returntype", Promise)
703
+ ], NetworkDeviceAutoImportRuleEngineServiceClass.prototype, "processCompletedScan", null);
704
+ __decorate([
705
+ CaptureSpan(),
706
+ __metadata("design:type", Function),
707
+ __metadata("design:paramtypes", [Object]),
708
+ __metadata("design:returntype", Promise)
709
+ ], NetworkDeviceAutoImportRuleEngineServiceClass.prototype, "applyRuleToCompletedScans", null);
710
+ export default new NetworkDeviceAutoImportRuleEngineServiceClass();
711
+ //# sourceMappingURL=NetworkDeviceAutoImportRuleEngineService.js.map