@oneuptime/common 12.0.7 → 12.0.8

package/Tests/Server/Services/TelemetryEntityManualCreate.test.ts

@@ -0,0 +1,248 @@
+import TelemetryEntity from "../../../Models/DatabaseModels/TelemetryEntity";
+import TelemetryEntityService from "../../../Server/Services/TelemetryEntityService";
+import CreateBy from "../../../Server/Types/Database/CreateBy";
+import ObjectID from "../../../Types/ObjectID";
+import EntitySource from "../../../Types/Telemetry/EntitySource";
+import EntityType from "../../../Types/Telemetry/EntityType";
+import { keyForManualEntity } from "../../../Utils/Telemetry/EntityKey";
+import { describe, expect, test } from "@jest/globals";
+
+/*
+ * TelemetryEntityService.onBeforeCreate is the whole of the manual-CI
+ * write path: it is what decides a create is manual, validates it, and
+ * derives the identity the registry needs from the two fields a human can
+ * supply. It also has to leave machine creates — ingest and the inventory
+ * mirror, which arrive with their keys already computed — completely alone.
+ *
+ * Every assertion below is one of those two duties.
+ */
+
+const PROJECT_ID: ObjectID = new ObjectID(
+  "0f8b9c0d-e1a2-4b3c-8d5e-6f7a8b9c0d1e",
+);
+
+type CallCreate = (createBy: CreateBy<TelemetryEntity>) => Promise<unknown>;
+
+const callOnBeforeCreate: CallCreate = (
+  createBy: CreateBy<TelemetryEntity>,
+): Promise<unknown> => {
+  return (
+    TelemetryEntityService as unknown as {
+      onBeforeCreate: (c: CreateBy<TelemetryEntity>) => Promise<unknown>;
+    }
+  ).onBeforeCreate(createBy);
+};
+
+/*
+ * Explicitly admits `undefined` per field. The suite needs to build creates
+ * with a field genuinely absent (that is what the validation tests are), and
+ * `Partial<TelemetryEntity>` rejects an explicit undefined under this
+ * project's `exactOptionalPropertyTypes`.
+ */
+interface ManualCreateOverrides {
+  projectId?: ObjectID | undefined;
+  entityType?: EntityType | undefined;
+  displayName?: string | undefined;
+  description?: string | undefined;
+}
+
+function manualCreate(
+  overrides: ManualCreateOverrides = {},
+): CreateBy<TelemetryEntity> {
+  const data: TelemetryEntity = new TelemetryEntity();
+  data.projectId = PROJECT_ID;
+  data.entityType = EntityType.ExternalService;
+  data.displayName = "Stripe Payments API";
+  Object.assign(data, overrides);
+
+  return { data, props: { isRoot: true } } as CreateBy<TelemetryEntity>;
+}
+
+describe("manual entity creation", () => {
+  test("a create with no entity key is treated as manual", async () => {
+    const createBy: CreateBy<TelemetryEntity> = manualCreate();
+    await callOnBeforeCreate(createBy);
+
+    expect(createBy.data.source).toBe(EntitySource.Manual);
+  });
+
+  test("derives the entity key from project, type and name", async () => {
+    const createBy: CreateBy<TelemetryEntity> = manualCreate();
+    await callOnBeforeCreate(createBy);
+
+    expect(createBy.data.entityKey).toBe(
+      keyForManualEntity(
+        PROJECT_ID.toString(),
+        EntityType.ExternalService,
+        "Stripe Payments API",
+      ),
+    );
+  });
+
+  test("records the canonicalized name as the identifying attribute", async () => {
+    const createBy: CreateBy<TelemetryEntity> = manualCreate({
+      displayName: "  Stripe Payments API  ",
+    });
+    await callOnBeforeCreate(createBy);
+
+    expect(createBy.data.identifyingAttributes).toEqual({
+      "oneuptime.entity.name": "stripe payments api",
+    });
+  });
+
+  test("trims the stored display name but keeps its original casing", async () => {
+    const createBy: CreateBy<TelemetryEntity> = manualCreate({
+      displayName: "  Stripe Payments API  ",
+    });
+    await callOnBeforeCreate(createBy);
+
+    expect(createBy.data.displayName).toBe("Stripe Payments API");
+  });
+
+  test("two names differing only by case resolve to one identity", async () => {
+    const first: CreateBy<TelemetryEntity> = manualCreate({
+      displayName: "Stripe API",
+    });
+    const second: CreateBy<TelemetryEntity> = manualCreate({
+      displayName: "stripe api",
+    });
+
+    await callOnBeforeCreate(first);
+    await callOnBeforeCreate(second);
+
+    /*
+     * The unique index on (projectId, entityType, entityKey) then rejects
+     * the second as a duplicate rather than admitting a near-identical row.
+     */
+    expect(second.data.entityKey).toBe(first.data.entityKey);
+  });
+
+  test("stamps first and last seen so the row sorts with the rest", async () => {
+    const createBy: CreateBy<TelemetryEntity> = manualCreate();
+    await callOnBeforeCreate(createBy);
+
+    expect(createBy.data.firstSeenAt).toBeInstanceOf(Date);
+    expect(createBy.data.lastSeenAt).toBeInstanceOf(Date);
+  });
+
+  test("accepts every manually creatable type", async () => {
+    for (const entityType of [
+      EntityType.ExternalService,
+      EntityType.ExternalDatabase,
+      EntityType.Appliance,
+    ]) {
+      const createBy: CreateBy<TelemetryEntity> = manualCreate({ entityType });
+      await callOnBeforeCreate(createBy);
+
+      expect(createBy.data.entityType).toBe(entityType);
+      expect(createBy.data.entityKey).toBeTruthy();
+    }
+  });
+});
+
+describe("manual entity validation", () => {
+  test("rejects a type that OneUptime discovers on its own", async () => {
+    await expect(
+      callOnBeforeCreate(manualCreate({ entityType: EntityType.Service })),
+    ).rejects.toThrow(/cannot be created manually/);
+  });
+
+  test("rejects an inventory-mirrored type", async () => {
+    /*
+     * These arrive from the mirror with a precomputed key; a hand-made one
+     * would be a permanent duplicate of the row the mirror maintains.
+     */
+    await expect(
+      callOnBeforeCreate(
+        manualCreate({ entityType: EntityType.NetworkDevice }),
+      ),
+    ).rejects.toThrow(/cannot be created manually/);
+  });
+
+  test("rejects a missing name", async () => {
+    await expect(
+      callOnBeforeCreate(manualCreate({ displayName: undefined })),
+    ).rejects.toThrow(/Name is required/);
+  });
+
+  test("rejects a whitespace-only name", async () => {
+    await expect(
+      callOnBeforeCreate(manualCreate({ displayName: "   " })),
+    ).rejects.toThrow(/Name is required/);
+  });
+
+  test("rejects a missing entity type", async () => {
+    await expect(
+      callOnBeforeCreate(manualCreate({ entityType: undefined })),
+    ).rejects.toThrow(/Entity Type is required/);
+  });
+
+  test("rejects a missing project", async () => {
+    await expect(
+      callOnBeforeCreate(manualCreate({ projectId: undefined })),
+    ).rejects.toThrow(/Project ID is required/);
+  });
+});
+
+describe("machine creates are left alone", () => {
+  test("a discovered create keeps its precomputed key and source", async () => {
+    const data: TelemetryEntity = new TelemetryEntity();
+    data.projectId = PROJECT_ID;
+    data.entityType = EntityType.KubernetesPod;
+    data.entityKey = "210dac24142f1baa";
+    data.source = EntitySource.Discovered;
+    data.displayName = "checkout-7d9f";
+    data.identifyingAttributes = { "k8s.pod.name": "checkout-7d9f" };
+
+    const createBy: CreateBy<TelemetryEntity> = {
+      data,
+      props: { isRoot: true },
+    } as CreateBy<TelemetryEntity>;
+
+    await callOnBeforeCreate(createBy);
+
+    expect(createBy.data.entityKey).toBe("210dac24142f1baa");
+    expect(createBy.data.source).toBe(EntitySource.Discovered);
+    expect(createBy.data.identifyingAttributes).toEqual({
+      "k8s.pod.name": "checkout-7d9f",
+    });
+  });
+
+  test("a discovered create of an otherwise un-creatable type is not rejected", async () => {
+    /*
+     * The manual-type restriction must apply to users, not to ingest — which
+     * legitimately creates services, pods and hosts.
+     */
+    const data: TelemetryEntity = new TelemetryEntity();
+    data.projectId = PROJECT_ID;
+    data.entityType = EntityType.Service;
+    data.entityKey = "abcdef0123456789";
+    data.source = EntitySource.Discovered;
+
+    await expect(
+      callOnBeforeCreate({
+        data,
+        props: { isRoot: true },
+      } as CreateBy<TelemetryEntity>),
+    ).resolves.toBeDefined();
+  });
+
+  test("an inventory mirror create keeps its precomputed key and source", async () => {
+    const data: TelemetryEntity = new TelemetryEntity();
+    data.projectId = PROJECT_ID;
+    data.entityType = EntityType.NetworkDevice;
+    data.entityKey = "9f8e7d6c5b4a3210";
+    data.source = EntitySource.Inventory;
+    data.displayName = "core-switch-1";
+
+    const createBy: CreateBy<TelemetryEntity> = {
+      data,
+      props: { isRoot: true },
+    } as CreateBy<TelemetryEntity>;
+
+    await callOnBeforeCreate(createBy);
+
+    expect(createBy.data.entityKey).toBe("9f8e7d6c5b4a3210");
+    expect(createBy.data.source).toBe(EntitySource.Inventory);
+  });
+});

package/Tests/Server/Types/Workflow/Components/ApiComponentHeaders.test.ts

@@ -0,0 +1,192 @@
+/*
+ * Request headers on the API components reach sanitizeArgs in three shapes: a
+ * JSON string (every workflow written before the key/value editor), an object
+ * (what the key/value editor produces), and whatever a {{...}} substitution
+ * left behind. Every consumer downstream casts the result to
+ * Dictionary<string> and spreads it into the outgoing request, so the shape
+ * has to be settled here.
+ */
+
+jest.mock("../../../../../Utils/API", () => {
+  return {
+    __esModule: true,
+    default: {
+      get: jest.fn(),
+      post: jest.fn(),
+      put: jest.fn(),
+      patch: jest.fn(),
+      delete: jest.fn(),
+    },
+  };
+});
+
+import ApiGet from "../../../../../Server/Types/Workflow/Components/API/Get";
+import { ApiComponentUtils } from "../../../../../Server/Types/Workflow/Components/API/Utils";
+import ComponentCode, {
+  RunOptions,
+} from "../../../../../Server/Types/Workflow/ComponentCode";
+import Exception from "../../../../../Types/Exception/Exception";
+import { JSONObject } from "../../../../../Types/JSON";
+import ObjectID from "../../../../../Types/ObjectID";
+import IconProp from "../../../../../Types/Icon/IconProp";
+import ComponentMetadata, {
+  Argument,
+  ComponentInputType,
+  ComponentType,
+} from "../../../../../Types/Workflow/Component";
+import APIComponents from "../../../../../Types/Workflow/Components/API";
+import { afterEach, beforeEach, describe, expect, test } from "@jest/globals";
+import dns from "dns";
+
+type MakeOptionsFunction = () => RunOptions;
+
+const makeOptions: MakeOptionsFunction = (): RunOptions => {
+  return {
+    log: jest.fn() as RunOptions["log"],
+    workflowLogId: ObjectID.generate(),
+    workflowId: ObjectID.generate(),
+    projectId: ObjectID.generate(),
+    onError: ((exception: Exception): Exception => {
+      return exception;
+    }) as RunOptions["onError"],
+    executeWorkflow: async (): Promise<void> => {},
+  };
+};
+
+type SanitizeFunction = (headers: unknown) => Promise<JSONObject>;
+
+const sanitizeHeaders: SanitizeFunction = async (
+  headers: unknown,
+): Promise<JSONObject> => {
+  const component: ComponentCode = new ApiGet();
+
+  const result: { args: JSONObject } = await ApiComponentUtils.sanitizeArgs(
+    component.getMetadata(),
+    {
+      url: "https://api.example.com/thing",
+      "request-headers": headers as JSONObject,
+    },
+    makeOptions(),
+  );
+
+  return result.args["request-headers"] as JSONObject;
+};
+
+beforeEach(() => {
+  jest.clearAllMocks();
+  jest
+    .spyOn(dns.promises, "lookup")
+    .mockResolvedValue([{ address: "93.184.216.34", family: 4 }] as never);
+});
+
+afterEach(() => {
+  jest.restoreAllMocks();
+});
+
+describe("ApiComponentUtils.sanitizeArgs — header shapes", () => {
+  test("parses the legacy JSON-string shape", async () => {
+    await expect(
+      sanitizeHeaders('{"Authorization": "Bearer abc"}'),
+    ).resolves.toEqual({
+      Authorization: "Bearer abc",
+    });
+  });
+
+  test("parses a JSON5 string, which has always been accepted here", async () => {
+    await expect(
+      sanitizeHeaders("{Authorization: 'Bearer abc'}"),
+    ).resolves.toEqual({
+      Authorization: "Bearer abc",
+    });
+  });
+
+  test("passes the object shape through", async () => {
+    await expect(
+      sanitizeHeaders({ Authorization: "Bearer abc" }),
+    ).resolves.toEqual({ Authorization: "Bearer abc" });
+  });
+
+  test("flattens numbers and booleans the key/value editor can emit", async () => {
+    await expect(
+      sanitizeHeaders({ "X-Retries": 3, "X-Debug": true }),
+    ).resolves.toEqual({
+      "X-Retries": "3",
+      "X-Debug": "true",
+    });
+  });
+
+  test("drops null and undefined values instead of sending 'null'", async () => {
+    await expect(
+      sanitizeHeaders({ "X-A": "keep", "X-B": null }),
+    ).resolves.toEqual({ "X-A": "keep" });
+  });
+
+  test("serializes a nested value rather than sending [object Object]", async () => {
+    await expect(sanitizeHeaders({ "X-Meta": { a: 1 } })).resolves.toEqual({
+      "X-Meta": '{"a":1}',
+    });
+  });
+
+  test("rejects an array, which would spread into per-index junk headers", async () => {
+    await expect(sanitizeHeaders("[1, 2]")).rejects.toThrow(
+      /Request headers must be a JSON object/,
+    );
+  });
+
+  test("rejects a bare scalar", async () => {
+    await expect(sanitizeHeaders("42")).rejects.toThrow(
+      /Request headers must be a JSON object/,
+    );
+  });
+
+  test("leaves the request alone when no headers were given", async () => {
+    const component: ComponentCode = new ApiGet();
+
+    const result: { args: JSONObject } = await ApiComponentUtils.sanitizeArgs(
+      component.getMetadata(),
+      { url: "https://api.example.com/thing" },
+      makeOptions(),
+    );
+
+    expect(result.args["request-headers"]).toBeUndefined();
+  });
+});
+
+describe("API component metadata — headers are treated as secret", () => {
+  /*
+   * The key/value grid makes pasting a bearer token into a header the obvious
+   * thing to do. Without isSensitive the resolved value is written verbatim
+   * into the WorkflowLog for anyone who can read the project.
+   */
+  test("every API component marks request-headers sensitive", () => {
+    const componentsWithHeaders: Array<ComponentMetadata> =
+      APIComponents.filter((component: ComponentMetadata) => {
+        return component.arguments.some((arg: Argument) => {
+          return arg.id === "request-headers";
+        });
+      });
+
+    expect(componentsWithHeaders.length).toBe(5);
+
+    for (const component of componentsWithHeaders) {
+      const headerArgument: Argument = component.arguments.find(
+        (arg: Argument) => {
+          return arg.id === "request-headers";
+        },
+      ) as Argument;
+
+      expect(headerArgument.isSensitive).toBe(true);
+      expect(headerArgument.type).toBe(ComponentInputType.StringDictionary);
+    }
+  });
+
+  test("the components are the five request verbs", () => {
+    expect(
+      APIComponents.filter((component: ComponentMetadata) => {
+        return component.componentType === ComponentType.Component;
+      }).length,
+    ).toBeGreaterThanOrEqual(5);
+
+    expect(APIComponents[0]?.iconProp).toBe(IconProp.Globe);
+  });
+});

package/Tests/Server/Utils/AnalyticsDatabase/QuerySettingsHelper.test.ts

@@ -0,0 +1,152 @@
+import QuerySettingsHelper, {
+  getQuerySettings,
+  DEFAULT_MAX_MEMORY_USAGE_IN_BYTES,
+  DEFAULT_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY_IN_BYTES,
+  DEFAULT_MAX_BYTES_BEFORE_EXTERNAL_SORT_IN_BYTES,
+} from "../../../../Server/Utils/AnalyticsDatabase/QuerySettingsHelper";
+
+/*
+ * Unit tests for getQuerySettings — the builder for the trailing
+ * ` SETTINGS ...` clause appended verbatim to every ClickHouse read query.
+ *
+ * The clause is NOT parameterized; it is concatenated straight into SQL, so
+ * the exact emitted text matters. These tests lock down:
+ *   - the always-present per-query memory ceiling and spill thresholds,
+ *   - correct quoting (strings single-quoted, numbers bare),
+ *   - the conditional execution-time / overflow-mode settings, and
+ *   - the ordering of the emitted parts.
+ */
+
+describe("getQuerySettings", () => {
+  test("always emits the memory ceiling and spill thresholds by default", () => {
+    const clause: string = getQuerySettings();
+
+    expect(clause).toBe(
+      ` SETTINGS max_memory_usage = ${DEFAULT_MAX_MEMORY_USAGE_IN_BYTES}, ` +
+        `max_bytes_before_external_group_by = ${DEFAULT_MAX_BYTES_BEFORE_EXTERNAL_GROUP_BY_IN_BYTES}, ` +
+        `max_bytes_before_external_sort = ${DEFAULT_MAX_BYTES_BEFORE_EXTERNAL_SORT_IN_BYTES}`,
+    );
+  });
+
+  test("produces the same output whether called with no args or undefined", () => {
+    expect(getQuerySettings()).toBe(getQuerySettings(undefined));
+  });
+
+  test("always starts with the ` SETTINGS ` keyword and a leading space", () => {
+    expect(getQuerySettings().startsWith(" SETTINGS ")).toBe(true);
+    expect(
+      getQuerySettings({ maxExecutionTimeInSeconds: 30 }).startsWith(
+        " SETTINGS ",
+      ),
+    ).toBe(true);
+  });
+
+  test("omits execution-time settings when not provided", () => {
+    const clause: string = getQuerySettings();
+
+    expect(clause).not.toContain("max_execution_time");
+    expect(clause).not.toContain("timeout_overflow_mode");
+  });
+
+  test("includes max_execution_time when provided", () => {
+    const clause: string = getQuerySettings({ maxExecutionTimeInSeconds: 45 });
+
+    expect(clause).toContain("max_execution_time = 45");
+    // Execution time is emitted before the memory settings.
+    expect(clause.indexOf("max_execution_time")).toBeLessThan(
+      clause.indexOf("max_memory_usage"),
+    );
+  });
+
+  test("single-quotes the timeout overflow mode", () => {
+    expect(getQuerySettings({ timeoutOverflowMode: "break" })).toContain(
+      "timeout_overflow_mode = 'break'",
+    );
+    expect(getQuerySettings({ timeoutOverflowMode: "throw" })).toContain(
+      "timeout_overflow_mode = 'throw'",
+    );
+  });
+
+  test("emits execution time before overflow mode before memory settings", () => {
+    const clause: string = getQuerySettings({
+      maxExecutionTimeInSeconds: 10,
+      timeoutOverflowMode: "break",
+    });
+
+    const timePos: number = clause.indexOf("max_execution_time");
+    const overflowPos: number = clause.indexOf("timeout_overflow_mode");
+    const memoryPos: number = clause.indexOf("max_memory_usage");
+
+    expect(timePos).toBeGreaterThan(-1);
+    expect(timePos).toBeLessThan(overflowPos);
+    expect(overflowPos).toBeLessThan(memoryPos);
+  });
+
+  test("honours overrides for the memory ceiling and spill thresholds", () => {
+    const clause: string = getQuerySettings({
+      maxMemoryUsageInBytes: 100,
+      maxBytesBeforeExternalGroupByInBytes: 50,
+      maxBytesBeforeExternalSortInBytes: 25,
+    });
+
+    expect(clause).toContain("max_memory_usage = 100");
+    expect(clause).toContain("max_bytes_before_external_group_by = 50");
+    expect(clause).toContain("max_bytes_before_external_sort = 25");
+    // The defaults must not leak through when overridden.
+    expect(clause).not.toContain(`${DEFAULT_MAX_MEMORY_USAGE_IN_BYTES}`);
+  });
+
+  test("emits numeric additional settings without quotes", () => {
+    const clause: string = getQuerySettings({
+      additionalSettings: { max_threads: 4 },
+    });
+
+    expect(clause).toContain("max_threads = 4");
+    expect(clause).not.toContain("max_threads = '4'");
+  });
+
+  test("single-quotes string additional settings", () => {
+    const clause: string = getQuerySettings({
+      additionalSettings: { optimize_aggregation_in_order: "1" },
+    });
+
+    expect(clause).toContain("optimize_aggregation_in_order = '1'");
+  });
+
+  test("appends additional settings after the memory settings", () => {
+    const clause: string = getQuerySettings({
+      additionalSettings: { optimize_use_projections: 1 },
+    });
+
+    expect(clause.indexOf("max_bytes_before_external_sort")).toBeLessThan(
+      clause.indexOf("optimize_use_projections"),
+    );
+  });
+
+  test("joins every setting with a comma-and-space separator", () => {
+    const clause: string = getQuerySettings({
+      maxExecutionTimeInSeconds: 5,
+      timeoutOverflowMode: "throw",
+      additionalSettings: { max_threads: 2 },
+    });
+
+    /*
+     * The clause after " SETTINGS " is a comma-separated list; splitting it
+     * must yield one entry per emitted setting (2 conditional + 3 memory + 1
+     * extra).
+     */
+    const body: string = clause.replace(" SETTINGS ", "");
+    expect(body.split(", ").length).toBe(6);
+  });
+
+  test("class static method delegates to the standalone function", () => {
+    const options: { maxExecutionTimeInSeconds: number } = {
+      maxExecutionTimeInSeconds: 12,
+    };
+
+    expect(QuerySettingsHelper.getQuerySettings(options)).toBe(
+      getQuerySettings(options),
+    );
+    expect(QuerySettingsHelper.getQuerySettings()).toBe(getQuerySettings());
+  });
+});