@oneuptime/common 12.0.33 → 12.0.34

package/Tests/Server/Services/NetworkDeviceAutoImportRuleEngineService.test.ts

@@ -219,9 +219,26 @@ const semaphoreLockMock: jest.Mock = Semaphore.lock as unknown as jest.Mock;
 const semaphoreReleaseMock: jest.Mock =
   Semaphore.release as unknown as jest.Mock;
 const loggerErrorMock: jest.Mock = logger.error as unknown as jest.Mock;
+const loggerWarnMock: jest.Mock = logger.warn as unknown as jest.Mock;
 
 const RECENT_COMPLETED_AT: Date = OneUptimeDate.getCurrentDate();
 
+/*
+ * The values behind a QueryHelper.any() filter.
+ *
+ * QueryHelper.any builds a raw TypeORM FindOperator whose parameters hold the
+ * list, so a status filter cannot be compared to a string any more. Reading
+ * the values back out is what lets a test still say WHICH statuses a query
+ * accepts rather than merely that it is some operator.
+ */
+function statusesMatchedBy(filter: unknown): Array<string> {
+  const parameters: Record<string, Array<string>> = (
+    filter as { _objectLiteralParameters?: Record<string, Array<string>> }
+  )._objectLiteralParameters!;
+
+  return Object.values(parameters)[0]!;
+}
+
 /*
  * The scan's credential sets (issue #3458), and the ids the probe stamps onto
  * a discovered host to record which one answered it. Opaque literals, because
@@ -542,9 +559,19 @@ describe("NetworkDeviceAutoImportRuleEngineService.processCompletedScan", () =>
     expect(updateCall.data.autoImportProcessedAt).toBeInstanceOf(Date);
     // Nothing was created, so the stored results are not rewritten.
     expect(Object.keys(updateCall.data)).toEqual(["autoImportProcessedAt"]);
+    /*
+     * The compare-and-set is over the run state this pass READ, not over the
+     * literal "Completed" it used to pin: a scan that is still sweeping is
+     * importable now (issue #3599), and its status is exactly what changes
+     * when its run ends. The two host counters ride along because they are
+     * what a running scan's partial uploads move, so a partial landing between
+     * the read and the stamp is caught the way a completion is.
+     */
     expect(updateCall.expectedData).toEqual({
       status: "Completed",
       completedAt: RECENT_COMPLETED_AT,
+      scannedHostCount: null,
+      respondedHostCount: null,
     });
   });
 
@@ -1779,7 +1806,16 @@ describe("NetworkDeviceAutoImportRuleEngineService.applyRuleToCompletedScans", (
 
     expect(scanFindOneByMock).toHaveBeenCalledTimes(1);
     const rereadCall: any = scanFindOneByMock.mock.calls[0]![0];
-    expect(rereadCall.query.status).toBe("Completed");
+    /*
+     * The status filter is a SET now, not the single "Completed" it used to
+     * be: a scan that is still sweeping already holds real results, and
+     * refusing to look at them is what left 527 discovered switches
+     * unimportable for a day (issue #3599).
+     */
+    expect(statusesMatchedBy(rereadCall.query.status)).toEqual([
+      "Completed",
+      "In Progress",
+    ]);
     expect(rereadCall.query.projectId.toString()).toBe(PROJECT_ID.toString());
     expect(rereadCall.select.discoveredDevices).toBe(true);
   });
@@ -2025,3 +2061,192 @@ describe("NetworkDeviceAutoImportRuleEngineService.applyRuleToCompletedScans", (
     });
   });
 });
+
+/*
+ * github.com/OneUptime/oneuptime/issues/3599 — "Auto Import Rule can't see
+ * already-discovered devices while the Discovery Scan is still running".
+ *
+ * A discovery sweep reports incrementally now: the probe uploads what it has
+ * found every 30 seconds, so an In Progress scan's row already holds real
+ * hosts. The engine used to read only Completed scans, which meant that on the
+ * 24-hour sweep of #3598 the 527 switches it had already found — stored,
+ * visible in the product, correct — were unimportable for the whole day.
+ *
+ * These tests pin that a running scan's results are treated as first-class,
+ * and that the marker protocol still holds when the row underneath is moving.
+ */
+describe("auto-import from a scan that is still sweeping (issue #3599)", () => {
+  /*
+   * A scan mid-sweep: no completedAt (the run has not ended), and the two
+   * host counters its partial uploads have been moving.
+   */
+  function makeInProgressScan(
+    overrides: Record<string, unknown> = {},
+  ): NetworkDeviceDiscoveryScan {
+    return makeScan({
+      status: "In Progress",
+      completedAt: undefined,
+      scannedHostCount: 4096,
+      respondedHostCount: 12,
+      ...overrides,
+    });
+  }
+
+  it("imports the hosts a running sweep has already found", async () => {
+    scanFindOneByMock.mockResolvedValue(
+      makeInProgressScan({ discoveredDevices: [makeHost()] }),
+    );
+    ruleFindByMock.mockResolvedValue([makeRule()]);
+
+    const result: AutoImportRuleRunResult | null = await processScan();
+
+    expect(result).toMatchObject({ hostsMatched: 1, devicesCreated: 1 });
+    expect(createdDevice(0).hostname).toBe("10.0.0.5");
+  });
+
+  it("asks for a running scan as well as a finished one", async () => {
+    scanFindOneByMock.mockResolvedValue(makeInProgressScan());
+    ruleFindByMock.mockResolvedValue([makeRule()]);
+
+    await processScan();
+
+    const query: any = scanFindOneByMock.mock.calls[0]![0].query;
+
+    expect(statusesMatchedBy(query.status)).toEqual([
+      "Completed",
+      "In Progress",
+    ]);
+  });
+
+  /*
+   * The freshness horizon retires results older than MAX_RESULT_AGE_IN_HOURS
+   * without importing them. It is keyed on completedAt, which a running scan
+   * does not have — and must not be read as "infinitely old". A sweep that has
+   * been running for two hours is the opposite of stale.
+   */
+  it("never treats a running scan's results as too old to import", async () => {
+    scanFindOneByMock.mockResolvedValue(
+      makeInProgressScan({ discoveredDevices: [makeHost()] }),
+    );
+    ruleFindByMock.mockResolvedValue([makeRule()]);
+
+    const result: AutoImportRuleRunResult | null = await processScan();
+
+    expect(result).toMatchObject({ devicesCreated: 1 });
+    expect(loggerWarnMock).not.toHaveBeenCalledWith(
+      expect.stringContaining("stamping it processed without importing"),
+      expect.anything(),
+    );
+  });
+
+  /*
+   * The marker protocol, on a row that is still moving.
+   *
+   * A running scan's status and completedAt do not change between uploads, so
+   * they cannot by themselves tell "the results I evaluated" from "the results
+   * that landed while I was evaluating". The host counters can, and they are
+   * what a partial upload moves.
+   */
+  it("guards the stamp on the run state it actually read, counters included", async () => {
+    scanFindOneByMock.mockResolvedValue(
+      makeInProgressScan({ discoveredDevices: [makeHost()] }),
+    );
+    ruleFindByMock.mockResolvedValue([makeRule()]);
+
+    await processScan();
+
+    const stamp: any = scanUpdateMock.mock.calls[0]![0];
+
+    expect(stamp.expectedData).toEqual({
+      status: "In Progress",
+      completedAt: null,
+      scannedHostCount: 4096,
+      respondedHostCount: 12,
+    });
+  });
+
+  // The counters are only a version token if they are actually read.
+  it("selects the counters it guards on", async () => {
+    scanFindOneByMock.mockResolvedValue(makeInProgressScan());
+    ruleFindByMock.mockResolvedValue([makeRule()]);
+
+    await processScan();
+
+    const select: any = scanFindOneByMock.mock.calls[0]![0].select;
+
+    expect(select.scannedHostCount).toBe(true);
+    expect(select.respondedHostCount).toBe(true);
+    expect(select.status).toBe(true);
+    expect(select.completedAt).toBe(true);
+  });
+
+  /*
+   * The write-back still retires the hosts this pass consumed, so the Review
+   * dialog stops offering them — on a running scan just as on a finished one.
+   */
+  it("flips isAlreadyRegistered on the rows it imported", async () => {
+    scanFindOneByMock.mockResolvedValue(
+      makeInProgressScan({
+        discoveredDevices: [makeHost(), makeHost({ ipAddress: "10.0.0.99" })],
+      }),
+    );
+    ruleFindByMock.mockResolvedValue([makeRule({ ipMatchTarget: "10.0.0.5" })]);
+
+    await processScan();
+
+    const written: Array<DiscoveredNetworkDevice> = scanUpdateMock.mock
+      .calls[0]![0].data.discoveredDevices as Array<DiscoveredNetworkDevice>;
+
+    expect(written[0]!.isAlreadyRegistered).toBe(true);
+    expect(written[1]!.isAlreadyRegistered).toBeUndefined();
+  });
+
+  /*
+   * A running scan is offered again on the next tick — its next partial upload
+   * clears the marker — so an address it already imported must read as
+   * registered rather than being created twice.
+   */
+  it("skips an address it already imported when the scan is offered again", async () => {
+    const alreadyImported: NetworkDevice = {
+      id: new ObjectID("55555555-5555-4555-8555-555555555555"),
+      projectId: PROJECT_ID,
+      hostname: "10.0.0.5",
+    } as unknown as NetworkDevice;
+
+    deviceFindByMock.mockResolvedValue([alreadyImported]);
+    scanFindOneByMock.mockResolvedValue(
+      makeInProgressScan({ discoveredDevices: [makeHost()] }),
+    );
+    ruleFindByMock.mockResolvedValue([makeRule()]);
+
+    const result: AutoImportRuleRunResult | null = await processScan();
+
+    expect(createMock).not.toHaveBeenCalled();
+    expect(result).toMatchObject({
+      hostsMatched: 1,
+      devicesCreated: 0,
+      hostsSkippedAlreadyRegistered: 1,
+    });
+  });
+
+  /*
+   * The manual path, which is where the report came from: "Run Now" said it
+   * had checked 25 hosts while 527 sat in the Devices list, because the scan
+   * that found them had not finished.
+   */
+  it("Run Now reads a running scan's results too", async () => {
+    ruleFindOneByMock.mockResolvedValue(makeRule());
+    ruleFindByMock.mockResolvedValue([]);
+    mockRunNowScans([makeInProgressScan({ discoveredDevices: [makeHost()] })]);
+
+    const result: AutoImportRuleRunResult = await runRule(true);
+
+    expect(result).toMatchObject({ hostsEvaluated: 1, hostsMatched: 1 });
+
+    const listQuery: any = scanFindByMock.mock.calls[0]![0].query;
+    expect(statusesMatchedBy(listQuery.status)).toEqual([
+      "Completed",
+      "In Progress",
+    ]);
+  });
+});

package/Tests/Server/Services/RoutineEmailSettingsPostgres.test.ts

@@ -0,0 +1,329 @@
+import RoutineEmailSettingsService from "../../../Server/Services/RoutineEmailSettingsService";
+import UserNotificationSettingService from "../../../Server/Services/UserNotificationSettingService";
+import PostgresAppInstance from "../../../Server/Infrastructure/PostgresDatabase";
+import Entities from "../../../Models/DatabaseModels/Index";
+import NotificationSettingEventType from "../../../Types/NotificationSetting/NotificationSettingEventType";
+import { ROUTINE_EMAIL_EVENT_TYPES } from "../../../Types/NotificationSetting/RoutineEmailEvents";
+import ObjectID from "../../../Types/ObjectID";
+import { DataSource } from "typeorm";
+
+/*
+ * Opt in with RUN_POSTGRES_NOTIFICATION_TESTS=true and the normal database
+ * credentials. Uses the local development Postgres port unless overridden by
+ * NOTIFICATION_TEST_DATABASE_HOST / NOTIFICATION_TEST_DATABASE_PORT.
+ * All writes go to a uniquely named schema; no project data is modified.
+ * Cloning the production table also verifies the statements against its actual
+ * defaults and column types rather than a hand-maintained test schema.
+ */
+const describePostgres: typeof describe =
+  process.env["RUN_POSTGRES_NOTIFICATION_TESTS"] === "true"
+    ? describe
+    : describe.skip;
+
+const USER_ID: ObjectID = ObjectID.generate();
+const OTHER_USER_ID: ObjectID = ObjectID.generate();
+const PROJECT_ID: ObjectID = ObjectID.generate();
+const OTHER_PROJECT_ID: ObjectID = ObjectID.generate();
+const ROUTINE_EVENT: NotificationSettingEventType =
+  NotificationSettingEventType.SEND_INCIDENT_NOTE_POSTED_OWNER_NOTIFICATION;
+const CHANNELS: Array<string> = [
+  "alertBySMS",
+  "alertByCall",
+  "alertByPush",
+  "alertByWhatsApp",
+  "alertByTelegram",
+  "alertBySlack",
+  "alertByMicrosoftTeams",
+  "alertByWebhook",
+];
+
+interface StoredSetting {
+  _id: string;
+  userId: string;
+  projectId: string;
+  eventType: NotificationSettingEventType;
+  alertByEmail: boolean;
+  version: number;
+  deletedAt: Date | null;
+  [key: string]: unknown;
+}
+
+describePostgres("reduce routine emails against Postgres", () => {
+  const schema: string = `routine_email_test_${ObjectID.generate().toString().replace(/-/g, "")}`;
+  let database: DataSource;
+
+  beforeAll(async () => {
+    database = new DataSource({
+      type: "postgres",
+      host: process.env["NOTIFICATION_TEST_DATABASE_HOST"] || "localhost",
+      port: Number(process.env["NOTIFICATION_TEST_DATABASE_PORT"] || "5400"),
+      username: process.env["DATABASE_USERNAME"] || "postgres",
+      password: process.env["DATABASE_PASSWORD"] || "password",
+      database: process.env["DATABASE_NAME"] || "oneuptimedb",
+      entities: Entities,
+      schema: schema,
+      synchronize: false,
+      extra: { options: `-c search_path=${schema},public` },
+    });
+    await database.initialize();
+    await database.query(`CREATE SCHEMA "${schema}"`);
+    await database.query(
+      `CREATE TABLE "${schema}"."UserNotificationSetting" (LIKE public."UserNotificationSetting" INCLUDING ALL)`,
+    );
+    const currentSchema: Array<{ current_schema: string }> =
+      await database.query("SELECT current_schema()");
+    expect(currentSchema[0]?.current_schema).toBe(schema);
+    jest.spyOn(PostgresAppInstance, "isConnected").mockReturnValue(true);
+    jest.spyOn(PostgresAppInstance, "getDataSource").mockReturnValue(database);
+  });
+
+  beforeEach(async () => {
+    await database.query(`TRUNCATE "${schema}"."UserNotificationSetting"`);
+  });
+
+  afterAll(async () => {
+    jest.restoreAllMocks();
+    if (database?.isInitialized) {
+      await database.query(`DROP SCHEMA IF EXISTS "${schema}" CASCADE`);
+      await database.destroy();
+    }
+  });
+
+  async function seed(data: {
+    eventType: NotificationSettingEventType;
+    email?: boolean;
+    userId?: ObjectID;
+    projectId?: ObjectID;
+    deleted?: boolean;
+  }): Promise<StoredSetting> {
+    const rows: Array<StoredSetting> = await database.query(
+      `INSERT INTO "UserNotificationSetting"
+       ("userId", "projectId", "eventType", "alertByEmail", "version", "deletedAt", ${CHANNELS.map(
+         (channel: string) => {
+           return `"${channel}"`;
+         },
+       ).join(", ")})
+       VALUES ($1, $2, $3, $4, 1, $5, ${CHANNELS.map(() => {
+         return "true";
+       }).join(", ")})
+       RETURNING *`,
+      [
+        (data.userId || USER_ID).toString(),
+        (data.projectId || PROJECT_ID).toString(),
+        data.eventType,
+        data.email ?? true,
+        data.deleted ? new Date() : null,
+      ],
+    );
+    return rows[0]!;
+  }
+
+  async function rows(): Promise<Array<StoredSetting>> {
+    return await database.query(
+      'SELECT * FROM "UserNotificationSetting" ORDER BY "_id"',
+    );
+  }
+
+  async function apply(): Promise<void> {
+    await RoutineEmailSettingsService.reduceRoutineEmails({
+      userId: USER_ID,
+      projectId: PROJECT_ID,
+    });
+  }
+
+  test("creates a persisted opt-out for every missing routine event", async () => {
+    await apply();
+    const settings: Array<StoredSetting> = await rows();
+    expect(settings).toHaveLength(ROUTINE_EMAIL_EVENT_TYPES.length);
+    expect(
+      new Set(
+        settings.map((setting: StoredSetting) => {
+          return setting.eventType;
+        }),
+      ),
+    ).toEqual(new Set(ROUTINE_EMAIL_EVENT_TYPES));
+    for (const setting of settings) {
+      expect(setting.userId).toBe(USER_ID.toString());
+      expect(setting.projectId).toBe(PROJECT_ID.toString());
+      expect(setting["createdByUserId"]).toBe(USER_ID.toString());
+      expect(setting.alertByEmail).toBe(false);
+      expect(setting.version).toBe(1);
+      for (const channel of CHANNELS) {
+        expect(setting[channel]).toBe(false);
+      }
+    }
+  });
+
+  test("disables only email while preserving every other channel on existing rows", async () => {
+    for (const eventType of ROUTINE_EMAIL_EVENT_TYPES) {
+      await seed({ eventType });
+    }
+    await apply();
+    const settings: Array<StoredSetting> = await rows();
+    expect(settings).toHaveLength(ROUTINE_EMAIL_EVENT_TYPES.length);
+    for (const setting of settings) {
+      expect(setting.alertByEmail).toBe(false);
+      expect(setting.version).toBe(2);
+      for (const channel of CHANNELS) {
+        expect(setting[channel]).toBe(true);
+      }
+    }
+  });
+
+  test("preserves every non-routine event, including existing opt-outs", async () => {
+    const preserved: Array<StoredSetting> = [];
+    for (const eventType of Object.values(NotificationSettingEventType)) {
+      if (!ROUTINE_EMAIL_EVENT_TYPES.includes(eventType)) {
+        preserved.push(await seed({ eventType }));
+        preserved.push(await seed({ eventType, email: false }));
+      }
+    }
+    await apply();
+    const settings: Array<StoredSetting> = await rows();
+    for (const before of preserved) {
+      expect(
+        settings.find((setting: StoredSetting) => {
+          return setting._id === before._id;
+        }),
+      ).toEqual(before);
+    }
+  });
+
+  test("does not create missing incident, alert or on-call preferences", async () => {
+    await apply();
+    for (const setting of await rows()) {
+      expect(ROUTINE_EMAIL_EVENT_TYPES).toContain(setting.eventType);
+    }
+  });
+
+  test("preserves an event introduced after this preset was defined", async () => {
+    const futureEvent: StoredSetting = await seed({
+      eventType: "Future notification event" as NotificationSettingEventType,
+    });
+    await apply();
+    expect(
+      (await rows()).find((setting: StoredSetting) => {
+        return setting._id === futureEvent._id;
+      }),
+    ).toEqual(futureEvent);
+  });
+
+  test("leaves other users and projects unchanged", async () => {
+    const preserved: Array<StoredSetting> = [
+      await seed({ eventType: ROUTINE_EVENT, userId: OTHER_USER_ID }),
+      await seed({ eventType: ROUTINE_EVENT, projectId: OTHER_PROJECT_ID }),
+      await seed({
+        eventType: ROUTINE_EVENT,
+        userId: OTHER_USER_ID,
+        projectId: OTHER_PROJECT_ID,
+      }),
+    ];
+    await apply();
+    const settings: Array<StoredSetting> = await rows();
+    for (const before of preserved) {
+      expect(
+        settings.find((setting: StoredSetting) => {
+          return setting._id === before._id;
+        }),
+      ).toEqual(before);
+    }
+  });
+
+  test("repeated requests do not create duplicates or change row versions and timestamps", async () => {
+    await seed({ eventType: ROUTINE_EVENT });
+    await apply();
+    const first: Array<StoredSetting> = await rows();
+    await apply();
+    expect(await rows()).toEqual(first);
+  });
+
+  test("later default seeding respects every persisted opt-out", async () => {
+    await apply();
+    const before: Array<StoredSetting> = await rows();
+    for (const eventType of ROUTINE_EMAIL_EVENT_TYPES) {
+      await UserNotificationSettingService.ensureSettingExistsForUser({
+        userId: USER_ID,
+        projectId: PROJECT_ID,
+        eventType: eventType,
+      });
+    }
+    expect(await rows()).toEqual(before);
+  });
+
+  test("simultaneous requests create only one live row per routine event", async () => {
+    await Promise.all(
+      Array.from({ length: 8 }, async () => {
+        return apply();
+      }),
+    );
+    const settings: Array<StoredSetting> = await rows();
+    expect(settings).toHaveLength(ROUTINE_EMAIL_EVENT_TYPES.length);
+    expect(
+      new Set(
+        settings.map((setting: StoredSetting) => {
+          return setting.eventType;
+        }),
+      ).size,
+    ).toBe(ROUTINE_EMAIL_EVENT_TYPES.length);
+    expect(
+      settings.every((setting: StoredSetting) => {
+        return setting.alertByEmail === false;
+      }),
+    ).toBe(true);
+  });
+
+  test("disables all live historical duplicates for the same event", async () => {
+    const first: StoredSetting = await seed({ eventType: ROUTINE_EVENT });
+    const second: StoredSetting = await seed({ eventType: ROUTINE_EVENT });
+    await apply();
+    const settings: Array<StoredSetting> = await rows();
+    for (const id of [first._id, second._id]) {
+      expect(
+        settings.find((setting: StoredSetting) => {
+          return setting._id === id;
+        })?.alertByEmail,
+      ).toBe(false);
+    }
+    expect(settings).toHaveLength(ROUTINE_EMAIL_EVENT_TYPES.length + 1);
+  });
+
+  test("retains deleted history and creates a new live opt-out", async () => {
+    const deleted: StoredSetting = await seed({
+      eventType: ROUTINE_EVENT,
+      deleted: true,
+    });
+    await apply();
+    const settings: Array<StoredSetting> = await rows();
+    expect(
+      settings.find((setting: StoredSetting) => {
+        return setting._id === deleted._id;
+      }),
+    ).toEqual(deleted);
+    expect(
+      settings.filter((setting: StoredSetting) => {
+        return (
+          setting.eventType === ROUTINE_EVENT && setting.deletedAt === null
+        );
+      }),
+    ).toEqual([expect.objectContaining({ alertByEmail: false })]);
+  });
+
+  test("rolls back existing changes when inserting a missing preference fails", async () => {
+    const original: StoredSetting = await seed({ eventType: ROUTINE_EVENT });
+    // A database constraint supplies a real statement failure after UPDATE.
+    await database.query(
+      'ALTER TABLE "UserNotificationSetting" ADD CONSTRAINT "fail_missing_preferences" CHECK ("eventType" = \'Send incident note posted notification when I am the owner of the incident\')',
+    );
+    try {
+      await expect(apply()).rejects.toThrow();
+      expect(await rows()).toEqual([original]);
+    } finally {
+      await database.query(
+        'ALTER TABLE "UserNotificationSetting" DROP CONSTRAINT "fail_missing_preferences"',
+      );
+    }
+    // The failed transaction also releases the advisory lock, so retry works.
+    await apply();
+    expect(await rows()).toHaveLength(ROUTINE_EMAIL_EVENT_TYPES.length);
+  });
+});