@oneuptime/common 14.0.9 → 14.0.11

package/Tests/Server/Services/DatabaseServerService.test.ts

@@ -1254,6 +1254,61 @@ describe("DatabaseServerService.findOrCreateByEndpoint", () => {
     expect(feed.feedInfoInMarkdown).toContain("restored from the archive");
   });
 
+  /*
+   * A lookup that saw no traffic (a resource AI agent registering) returns
+   * the owner as it is: an archived row stays archived, and neither the
+   * endpoint's last match nor the row's engine is touched.
+   */
+  test("isSighting: false returns the owner without sighting it", async () => {
+    const archived: DatabaseServer = databaseRow({
+      discoverySource: DatabaseServerDiscoverySource.Collector,
+      isArchived: true,
+      autoArchivedAt: new Date("2026-09-01T00:00:00.000Z"),
+    });
+    const owner: any = { databaseServerId: archived.id };
+    findOwner.mockResolvedValue(owner);
+    findOneBy.mockResolvedValue(archived);
+    rawQuery.mockResolvedValue([{ _id: archived.id!.toString() }]);
+    const markMatched: jest.SpyInstance = getJestSpyOn(
+      DatabaseServerEndpointService,
+      "markEndpointMatched",
+    ).mockResolvedValue(undefined);
+    const evidence: jest.SpyInstance = getJestSpyOn(
+      service,
+      "applyDatabaseSystemEvidence",
+    ).mockResolvedValue(undefined);
+
+    const result: DatabaseServer | null =
+      await DatabaseServerService.findOrCreateByEndpoint({
+        projectId: PROJECT_ID,
+        dbSystem: "postgresql",
+        endpoint: ORDERS_ENDPOINT,
+        discoverySource: DatabaseServerDiscoverySource.Collector,
+        allowCreate: true,
+        isSighting: false,
+      });
+
+    expect(result).toBe(archived);
+    expect(result!.isArchived).toBe(true);
+    expect(result!.autoArchivedAt).toBeDefined();
+    expect(rawQuery).not.toHaveBeenCalled();
+    expect(sideEffects.feed).not.toHaveBeenCalled();
+    expect(markMatched).not.toHaveBeenCalled();
+    expect(evidence).not.toHaveBeenCalled();
+    expect(create).not.toHaveBeenCalled();
+
+    // The same lookup as a sighting restores it.
+    await DatabaseServerService.findOrCreateByEndpoint({
+      projectId: PROJECT_ID,
+      dbSystem: "postgresql",
+      endpoint: ORDERS_ENDPOINT,
+      discoverySource: DatabaseServerDiscoverySource.Collector,
+      allowCreate: true,
+    });
+    expect(markMatched).toHaveBeenCalledWith(owner);
+    expect(rawQuery).toHaveBeenCalledTimes(1);
+  });
+
   test("a restore that lost to someone else writes no second feed item", async () => {
     const archived: DatabaseServer = databaseRow({
       discoverySource: DatabaseServerDiscoverySource.Collector,

package/Tests/Server/Services/GroupTelemetryUsageExcludeNames.test.ts

@@ -0,0 +1,299 @@
+import AnalyticsDatabaseService from "../../../Server/Services/AnalyticsDatabaseService";
+import { LogService } from "../../../Server/Services/LogService";
+import { MetricService } from "../../../Server/Services/MetricService";
+import { SpanService } from "../../../Server/Services/SpanService";
+import { Statement } from "../../../Server/Utils/AnalyticsDatabase/Statement";
+import AnalyticsBaseModel from "../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
+import BadDataException from "../../../Types/Exception/BadDataException";
+import { JSONObject } from "../../../Types/JSON";
+import ObjectID from "../../../Types/ObjectID";
+import ServiceType from "../../../Types/Telemetry/ServiceType";
+import SessionReplayBudgetMetricTypeUtil from "../../../Utils/SessionReplay/SessionReplayBudgetMetricType";
+import { describe, expect, test } from "@jest/globals";
+
+/*
+ * The metrics billing scan is the only place OneUptime's own session replay
+ * budget rows can be told apart from a RUM application's web vitals: both are
+ * keyed to the application with primaryEntityType RealUserMonitor, so the
+ * (primaryEntityId, primaryEntityType) grouping merges them and only their
+ * names differ. These tests pin the SQL the scan generates - the exclusion
+ * sits inside the WHERE, binds the names as a parameter, and leaves the
+ * statement byte-for-byte unchanged when there is nothing to exclude - with no
+ * ClickHouse in the loop, the same way SessionReplayMetering.test.ts pins the
+ * replay scan.
+ */
+
+const PROJECT_ID: ObjectID = new ObjectID("6512f3a9b7c4d2e108f5a3b9");
+const START_OF_DAY: Date = new Date("2026-09-28T00:00:00.000Z");
+const END_OF_DAY: Date = new Date("2026-09-28T23:59:59.999Z");
+
+/*
+ * The statement as the scan generated it before excludeNames existed,
+ * captured from that code, up to the trailing SETTINGS clause (which belongs
+ * to QuerySettingsHelper and is pinned there).
+ */
+const PRE_EXCLUSION_QUERY: string =
+  "SELECT primaryEntityId AS primaryEntityId, primaryEntityType AS primaryEntityType, count() AS rowCount, sum(byteSize(*)) AS estimatedBytes FROM {p0:Identifier}.{p1:Identifier} WHERE projectId = {p2:String} AND {p3:Identifier} >= {p4:DateTime64(9)} AND {p5:Identifier} <= {p6:DateTime64(9)} GROUP BY primaryEntityId, primaryEntityType";
+
+const BUDGET_METRIC_NAMES: Array<string> =
+  SessionReplayBudgetMetricTypeUtil.getAll();
+
+type UsageRow = {
+  primaryEntityId: string;
+  primaryEntityType: string | null;
+  rowCount: number;
+  estimatedBytes: number;
+};
+
+interface CapturedScan {
+  // Null when the scan refused before reaching ClickHouse.
+  query: string | null;
+  params: Record<string, unknown> | null;
+  result: Array<UsageRow>;
+}
+
+/*
+ * Runs the real groupTelemetryUsageByService with executeQuery replaced, so
+ * the generated statement is captured instead of sent. `excludeNames` is
+ * passed only when the key is present, so "absent" and "undefined" are
+ * separate cases.
+ */
+async function captureUsageScan<TBaseModel extends AnalyticsBaseModel>(
+  service: AnalyticsDatabaseService<TBaseModel>,
+  options: {
+    excludeNames?: Array<string> | undefined;
+    responseRows?: Array<JSONObject> | undefined;
+  } = {},
+): Promise<CapturedScan> {
+  const captured: CapturedScan = { query: null, params: null, result: [] };
+
+  service.executeQuery = ((statement: Statement): Promise<unknown> => {
+    captured.query = statement.query;
+    captured.params = statement.query_params;
+
+    return Promise.resolve({
+      json: (): Promise<{ data: Array<JSONObject> }> => {
+        return Promise.resolve({ data: options.responseRows || [] });
+      },
+    });
+  }) as unknown as typeof service.executeQuery;
+
+  captured.result = await service.groupTelemetryUsageByService({
+    projectId: PROJECT_ID,
+    timestampColumnName: "time",
+    startDate: START_OF_DAY,
+    endDate: END_OF_DAY,
+    ...("excludeNames" in options
+      ? { excludeNames: options.excludeNames }
+      : {}),
+  });
+
+  return captured;
+}
+
+function queryBeforeSettings(query: string | null): string {
+  expect(query).not.toBeNull();
+
+  const settingsIndex: number = query!.indexOf(" SETTINGS ");
+  expect(settingsIndex).toBeGreaterThan(-1);
+
+  return query!.slice(0, settingsIndex);
+}
+
+describe("groupTelemetryUsageByService without excludeNames", () => {
+  test("generates exactly the statement it did before the option existed", async () => {
+    const scan: CapturedScan = await captureUsageScan(new MetricService());
+
+    expect(queryBeforeSettings(scan.query)).toBe(PRE_EXCLUSION_QUERY);
+    expect(scan.query).not.toContain("NOT IN");
+
+    // Same parameters too - p0 is the configured database name.
+    expect(Object.keys(scan.params!).sort()).toEqual(
+      ["p0", "p1", "p2", "p3", "p4", "p5", "p6"].sort(),
+    );
+    expect(scan.params!["p1"]).toBe("MetricItemV3");
+    expect(scan.params!["p2"]).toBe(PROJECT_ID.toString());
+    expect(scan.params!["p3"]).toBe("time");
+    expect(scan.params!["p5"]).toBe("time");
+  });
+
+  test.each([
+    ["an empty list", []],
+    ["an explicit undefined", undefined],
+  ])(
+    "treats %s as absent: the same statement, the same parameters",
+    async (_name: string, excludeNames: Array<string> | undefined) => {
+      const absent: CapturedScan = await captureUsageScan(new MetricService());
+      const given: CapturedScan = await captureUsageScan(new MetricService(), {
+        excludeNames,
+      });
+
+      expect(given.query).toBe(absent.query);
+      expect(given.params).toEqual(absent.params);
+    },
+  );
+
+  test("still scans a table with no name column when there is nothing to exclude", async () => {
+    const scan: CapturedScan = await captureUsageScan(new LogService(), {
+      excludeNames: [],
+    });
+
+    expect(scan.query).not.toBeNull();
+    expect(scan.query).not.toContain("NOT IN");
+    expect(scan.params!["p1"]).toBe("LogItemV3");
+  });
+});
+
+describe("groupTelemetryUsageByService with excludeNames", () => {
+  test("adds `AND name NOT IN` inside the WHERE: after the time window, before GROUP BY", async () => {
+    const scan: CapturedScan = await captureUsageScan(new MetricService(), {
+      excludeNames: BUDGET_METRIC_NAMES,
+    });
+
+    const query: string = scan.query!;
+
+    expect(query).toContain(
+      "AND {p5:Identifier} <= {p6:DateTime64(9)} AND name NOT IN {p7:Array(String)} GROUP BY primaryEntityId, primaryEntityType SETTINGS ",
+    );
+    expect(query.indexOf(" WHERE ")).toBeLessThan(query.indexOf("NOT IN"));
+    expect(query.indexOf("NOT IN")).toBeLessThan(query.indexOf(" GROUP BY "));
+    expect(query.indexOf(" GROUP BY ")).toBeLessThan(
+      query.indexOf(" SETTINGS "),
+    );
+
+    // Exactly one exclusion clause, and nothing else about the scan moved.
+    expect(query.split("NOT IN")).toHaveLength(2);
+    expect(
+      queryBeforeSettings(query).replace(
+        " AND name NOT IN {p7:Array(String)}",
+        "",
+      ),
+    ).toBe(PRE_EXCLUSION_QUERY);
+  });
+
+  test("binds every name, in order, as one Array(String) parameter", async () => {
+    const scan: CapturedScan = await captureUsageScan(new MetricService(), {
+      excludeNames: BUDGET_METRIC_NAMES,
+    });
+
+    expect(scan.params!["p7"]).toEqual(BUDGET_METRIC_NAMES);
+
+    // The names never reach the SQL text itself.
+    for (const name of BUDGET_METRIC_NAMES) {
+      expect(scan.query).not.toContain(name);
+    }
+  });
+
+  test("keeps a hostile name a parameter value, never SQL", async () => {
+    const hostileName: string = "x') OR 1=1 GROUP BY name --";
+
+    const scan: CapturedScan = await captureUsageScan(new MetricService(), {
+      excludeNames: [hostileName],
+    });
+
+    expect(scan.params!["p7"]).toEqual([hostileName]);
+    expect(scan.query).not.toContain("OR 1=1");
+    expect(scan.query).not.toContain("--");
+    expect(scan.query!.split("GROUP BY")).toHaveLength(2);
+  });
+
+  test("leaves the caller's list untouched", async () => {
+    const excludeNames: Array<string> = [...BUDGET_METRIC_NAMES];
+
+    await captureUsageScan(new MetricService(), { excludeNames });
+
+    expect(excludeNames).toEqual(BUDGET_METRIC_NAMES);
+  });
+
+  test("parses the grouped rows exactly as before", async () => {
+    const appId: string = ObjectID.generate().toString();
+
+    const scan: CapturedScan = await captureUsageScan(new MetricService(), {
+      excludeNames: BUDGET_METRIC_NAMES,
+      responseRows: [
+        {
+          primaryEntityId: appId,
+          primaryEntityType: ServiceType.RealUserMonitor,
+          rowCount: "12",
+          estimatedBytes: "3456",
+        },
+        // Unattributed rows are skipped, as they always were.
+        {
+          primaryEntityId: "",
+          primaryEntityType: ServiceType.OpenTelemetry,
+          rowCount: "1",
+          estimatedBytes: "1",
+        },
+        // A blank type reads as a legacy row.
+        {
+          primaryEntityId: PROJECT_ID.toString(),
+          primaryEntityType: " ",
+          rowCount: 2,
+          estimatedBytes: 0,
+        },
+      ],
+    });
+
+    expect(scan.result).toEqual([
+      {
+        primaryEntityId: appId,
+        primaryEntityType: ServiceType.RealUserMonitor,
+        rowCount: 12,
+        estimatedBytes: 3456,
+      },
+      {
+        primaryEntityId: PROJECT_ID.toString(),
+        primaryEntityType: null,
+        rowCount: 2,
+        estimatedBytes: 0,
+      },
+    ]);
+  });
+
+  test("refuses a table with no name column before querying it", async () => {
+    const service: LogService = new LogService();
+    let executed: boolean = false;
+    service.executeQuery = (async (): Promise<never> => {
+      executed = true;
+      throw new Error("must not run");
+    }) as unknown as typeof service.executeQuery;
+
+    await expect(
+      service.groupTelemetryUsageByService({
+        projectId: PROJECT_ID,
+        timestampColumnName: "time",
+        startDate: START_OF_DAY,
+        endDate: END_OF_DAY,
+        excludeNames: ["anything"],
+      }),
+    ).rejects.toThrow(BadDataException);
+    expect(executed).toBe(false);
+  });
+
+  test("refuses a Nullable name column, where NOT IN would silently drop every unnamed row", async () => {
+    /*
+     * Span.name is optional. ClickHouse does not treat `NULL NOT IN (...)`
+     * as true, so the scan would lose every span without a name from the
+     * bill - an undercount nobody would notice.
+     */
+    const service: SpanService = new SpanService();
+    let executed: boolean = false;
+    service.executeQuery = (async (): Promise<never> => {
+      executed = true;
+      throw new Error("must not run");
+    }) as unknown as typeof service.executeQuery;
+
+    await expect(
+      service.groupTelemetryUsageByService({
+        projectId: PROJECT_ID,
+        timestampColumnName: "startTime",
+        startDate: START_OF_DAY,
+        endDate: END_OF_DAY,
+        excludeNames: ["anything"],
+      }),
+    ).rejects.toThrow(
+      "excludeNames needs a required name column, and SpanItemV3 has none",
+    );
+    expect(executed).toBe(false);
+  });
+});

package/Tests/Server/Services/HostServiceFindOrCreateMemo.test.ts

@@ -228,6 +228,50 @@ describe("HostService.findOrCreateByHostIdentifier L1 memo", () => {
     expect(findOneBy).toHaveBeenCalledTimes(2);
   });
 
+  /*
+   * A host deleted within the TTL keeps resolving to its old id from the
+   * memo; a caller that found that id gone (a resource AI agent
+   * registering again right after the delete) asks past the memo, gets the
+   * host created anew, and the memo then holds the new id.
+   */
+  test("bypassMemo looks the host up again and refreshes the memo with what it found", async () => {
+    const projectId: ObjectID = ObjectID.generate();
+    const deleted: Host = buildHost();
+    const recreated: Host = buildHost();
+    findOneBy.mockResolvedValue(deleted);
+
+    await HostService.findOrCreateByHostIdentifier({
+      projectId,
+      hostIdentifier: "web-01",
+    });
+
+    // The host is deleted: nothing owns the identifier any more.
+    findOneBy.mockResolvedValue(null);
+    create.mockResolvedValue(recreated);
+
+    const memoed: Host = await HostService.findOrCreateByHostIdentifier({
+      projectId,
+      hostIdentifier: "web-01",
+    });
+    expect(memoed._id).toBe(deleted._id);
+
+    const fresh: Host = await HostService.findOrCreateByHostIdentifier({
+      projectId,
+      hostIdentifier: "web-01",
+      bypassMemo: true,
+    });
+    expect(fresh._id).toBe(recreated._id);
+    expect(findOneBy).toHaveBeenCalledTimes(2);
+    expect(create).toHaveBeenCalledTimes(1);
+
+    const afterwards: Host = await HostService.findOrCreateByHostIdentifier({
+      projectId,
+      hostIdentifier: "web-01",
+    });
+    expect(afterwards._id).toBe(recreated._id);
+    expect(findOneBy).toHaveBeenCalledTimes(2);
+  });
+
   test("memoizes a host it just created", async () => {
     const projectId: ObjectID = ObjectID.generate();
     const created: Host = buildHost({ hostIdentifier: "brand-new" });

package/Tests/Server/Services/IncidentCreatedFeedTitle.test.ts

@@ -0,0 +1,212 @@
+import IncidentFeedService from "../../../Server/Services/IncidentFeedService";
+import IncidentService from "../../../Server/Services/IncidentService";
+import Markdown, { MarkdownContentType } from "../../../Server/Types/Markdown";
+import LinkedAffectedResources from "../../../Server/Utils/AffectedResources/LinkedAffectedResources";
+import IncidentWorkspaceMessages from "../../../Server/Utils/Workspace/WorkspaceMessages/Incident";
+import SlackUtil from "../../../Server/Utils/Workspace/Slack/Slack";
+import Incident from "../../../Models/DatabaseModels/Incident";
+import ObjectID from "../../../Types/ObjectID";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  jest,
+  test,
+} from "@jest/globals";
+import { Token, Tokens, marked } from "marked";
+
+/*
+ * An incident's title is plain text, typed by whoever declared it - since
+ * incident forms, that is anyone holding a form's link - and the "Incident
+ * Created" feed item places it into Markdown that the dashboard renders
+ * without its safe mode, that is posted to Slack and Teams, and that owners
+ * read in email. So the title is escaped where it is placed, as MarkdownEscape
+ * says a title must be: a title cannot become a link, an image fetched when
+ * the feed is opened, raw HTML, or a Slack channel mention - and an ordinary
+ * title still reads exactly as typed.
+ */
+
+const PROJECT_ID: ObjectID = new ObjectID(
+  "0194d4ba-0000-4000-8000-00000000f201",
+);
+const INCIDENT_ID: ObjectID = new ObjectID(
+  "0194d4ba-0000-4000-8000-00000000f202",
+);
+
+type CreateIncidentFeedAsyncFunction = (incident: Incident) => Promise<void>;
+
+let feedItem: ReturnType<typeof jest.spyOn>;
+
+beforeEach(() => {
+  feedItem = jest
+    .spyOn(IncidentFeedService, "createIncidentFeedItem")
+    .mockResolvedValue(undefined as never);
+  jest
+    .spyOn(LinkedAffectedResources, "readForIncident")
+    .mockResolvedValue([] as never);
+  jest
+    .spyOn(IncidentWorkspaceMessages, "getIncidentCreateMessageBlocks")
+    .mockResolvedValue([] as never);
+});
+
+afterEach(() => {
+  jest.restoreAllMocks();
+});
+
+async function createdFeedFor(title: string): Promise<string> {
+  const incident: Incident = new Incident();
+  incident._id = INCIDENT_ID.toString();
+  incident.projectId = PROJECT_ID;
+  incident.incidentNumber = 7;
+  incident.incidentNumberWithPrefix = "INC-7";
+  incident.title = title;
+  incident.description = "Every order fails.";
+
+  await (
+    IncidentService as unknown as {
+      createIncidentFeedAsync: CreateIncidentFeedAsyncFunction;
+    }
+  ).createIncidentFeedAsync(incident);
+
+  expect(feedItem).toHaveBeenCalledTimes(1);
+
+  return (feedItem.mock.calls[0]![0] as { feedInfoInMarkdown: string })
+    .feedInfoInMarkdown;
+}
+
+function tokensOf(markdown: string): Array<Token> {
+  const tokens: Array<Token> = [];
+
+  marked.walkTokens(marked.lexer(markdown), (token: Token): void => {
+    tokens.push(token);
+  });
+
+  return tokens;
+}
+
+function typesOf(markdown: string): Array<string> {
+  return tokensOf(markdown).map((token: Token): string => {
+    return token.type;
+  });
+}
+
+// The title line of the feed item: the bold text right after the heading.
+function titleLine(markdown: string): string {
+  return markdown.split("\n").find((line: string): boolean => {
+    return line.startsWith("**");
+  })!;
+}
+
+function slackText(markdown: string): string {
+  return JSON.stringify(
+    SlackUtil.getMarkdownBlocks({
+      payloadMarkdownBlock: {
+        _type: "WorkspacePayloadMarkdown",
+        text: markdown,
+      },
+    }),
+  );
+}
+
+describe("the incident created feed item's title", () => {
+  test("an ordinary title reads exactly as typed", async () => {
+    const markdown: string = await createdFeedFor("Site 03 - payments (EU)");
+
+    expect(titleLine(markdown)).toBe("**Site 03 - payments (EU)**:");
+    expect(slackText(markdown)).toContain("Site 03 - payments (EU)");
+    expect(
+      await Markdown.convertToHTML(markdown, MarkdownContentType.Email),
+    ).toContain("Site 03 - payments (EU)");
+  });
+
+  test("an image in the title is not fetched: it arrives as the characters typed", async () => {
+    const markdown: string = await createdFeedFor(
+      "![x](https://tracker.example/p.png)",
+    );
+
+    expect(titleLine(markdown)).toBe(
+      "**!\\[x\\](https://tracker.example/p.png)**:",
+    );
+    expect(typesOf(titleLine(markdown))).not.toContain("image");
+
+    const html: string = await Markdown.convertToHTML(
+      markdown,
+      MarkdownContentType.Email,
+    );
+
+    expect(html).not.toContain("<img");
+    /*
+     * The bare address may still become a link, which shows where it goes;
+     * the text reads as typed.
+     */
+    expect(html.replace(/<[^>]+>/g, "")).toContain(
+      "![x](https://tracker.example/p.png)",
+    );
+  });
+
+  test("a link in the title is not a link that hides where it goes", async () => {
+    const markdown: string = await createdFeedFor(
+      "[Reset your password](https://evil.example/login)",
+    );
+
+    const links: Array<Tokens.Link> = tokensOf(titleLine(markdown)).filter(
+      (token: Token): boolean => {
+        return token.type === "link";
+      },
+    ) as Array<Tokens.Link>;
+
+    // Only the bare address stays a link, showing the address it goes to.
+    expect(
+      links.map((link: Tokens.Link): string => {
+        return link.text;
+      }),
+    ).not.toContain("Reset your password");
+    expect(
+      await Markdown.convertToHTML(markdown, MarkdownContentType.Email),
+    ).toContain("[Reset your password]");
+  });
+
+  test("raw HTML in the title reads as text", async () => {
+    const markdown: string = await createdFeedFor(
+      "<img src=x onerror=alert(1)>",
+    );
+
+    expect(typesOf(titleLine(markdown))).not.toContain("html");
+  });
+
+  test.each([
+    ["<!channel> Checkout is down"],
+    ["<!here> ping"],
+    ["<#C0123ABC>"],
+  ])("%j does not reach Slack as a mention", async (title: string) => {
+    const markdown: string = await createdFeedFor(title);
+
+    expect(slackText(markdown)).not.toMatch(/<(![a-z]|#C)/);
+    expect(slackText(markdown)).toContain("&lt;");
+  });
+
+  test("a backslash at the end of the title does not undo the bold around it", async () => {
+    const markdown: string = await createdFeedFor("C:\\temp\\");
+
+    expect(titleLine(markdown)).toBe("**C:\\\\temp\\\\**:");
+    expect(
+      await Markdown.convertToHTML(
+        titleLine(markdown),
+        MarkdownContentType.Email,
+      ),
+    ).toContain("<strong>C:\\temp\\</strong>");
+  });
+
+  test("a missing title still says so", async () => {
+    const markdown: string = await createdFeedFor("");
+
+    expect(titleLine(markdown)).toBe("**No title provided.**:");
+  });
+
+  test("the description stays Markdown", async () => {
+    const markdown: string = await createdFeedFor("Checkout is down");
+
+    expect(markdown).toContain("\n\nEvery order fails.\n\n");
+  });
+});