@oneuptime/common 11.5.13 → 11.6.1

package/Tests/Server/Services/WorkflowService.test.ts

@@ -0,0 +1,308 @@
+/**
+ * WorkflowService trigger denormalization tests.
+ *
+ * The runner never parses workflow graphs. It finds workflows by the
+ * triggerId / triggerArguments columns, which are derived from the graph and
+ * written onto the row. Anything that puts a graph on a workflow without
+ * populating those columns produces a workflow that looks correct in the
+ * builder and silently never fires.
+ *
+ * That is exactly what creating a workflow with a graph already attached does
+ * - importing a JSON export, or duplicating an existing workflow - so the
+ * create path has to denormalize the trigger just like the update path does.
+ *
+ * updateOneById and the outbound call to the workflow service are both spied
+ * on, so no database or network is touched.
+ */
+
+import WorkflowService from "../../../Server/Services/WorkflowService";
+import Workflow from "../../../Models/DatabaseModels/Workflow";
+import { JSONObject } from "../../../Types/JSON";
+import ObjectID from "../../../Types/ObjectID";
+import API from "../../../Utils/API";
+import { afterEach, describe, expect, test, jest } from "@jest/globals";
+
+const WORKFLOW_ID: string = "550e8400-e29b-41d4-a716-446655440000";
+
+function scheduleGraph(): JSONObject {
+  return {
+    nodes: [
+      {
+        id: "node-1",
+        data: {
+          metadataId: "Schedule",
+          componentType: "Trigger",
+          nodeType: "Node",
+          arguments: { "schedule-cron": "0 0 * * *" },
+        },
+      },
+      {
+        id: "node-2",
+        data: {
+          metadataId: "Webhook",
+          componentType: "Component",
+          nodeType: "Node",
+          arguments: {},
+        },
+      },
+    ],
+    edges: [],
+  };
+}
+
+function createdWorkflow(graph: JSONObject | undefined): Workflow {
+  /*
+   * webhookSecretKey is set so onCreateSuccess skips its own key-generation
+   * update, leaving updateOneById calls attributable to trigger derivation.
+   * projectId is left undefined so the fire-and-forget label/owner rule
+   * engines are skipped.
+   */
+  return {
+    _id: WORKFLOW_ID,
+    id: new ObjectID(WORKFLOW_ID),
+    graph: graph,
+    webhookSecretKey: "already-set",
+  } as unknown as Workflow;
+}
+
+function spyOnUpdateOneById(): jest.SpyInstance {
+  return jest
+    .spyOn(WorkflowService, "updateOneById")
+    .mockResolvedValue(undefined as never) as unknown as jest.SpyInstance;
+}
+
+function spyOnApiPost(): jest.SpyInstance {
+  return jest
+    .spyOn(API, "post")
+    .mockResolvedValue({} as never) as unknown as jest.SpyInstance;
+}
+
+type OnCreateSuccessFunction = (
+  onCreate: unknown,
+  createdItem: Workflow,
+) => Promise<Workflow>;
+
+function onCreateSuccess(createdItem: Workflow): Promise<Workflow> {
+  const hook: OnCreateSuccessFunction = (
+    WorkflowService as unknown as { onCreateSuccess: OnCreateSuccessFunction }
+  ).onCreateSuccess.bind(WorkflowService);
+
+  return hook({ createBy: { data: createdItem }, carryForward: null }, {
+    ...createdItem,
+  } as Workflow);
+}
+
+type OnUpdateSuccessFunction = (
+  onUpdate: unknown,
+  updatedItemIds: Array<ObjectID>,
+) => Promise<unknown>;
+
+function onUpdateSuccess(graph: JSONObject | undefined): Promise<unknown> {
+  const hook: OnUpdateSuccessFunction = (
+    WorkflowService as unknown as { onUpdateSuccess: OnUpdateSuccessFunction }
+  ).onUpdateSuccess.bind(WorkflowService);
+
+  return hook(
+    {
+      updateBy: {
+        query: { _id: WORKFLOW_ID },
+        data: graph === undefined ? {} : { graph: graph },
+      },
+    },
+    [new ObjectID(WORKFLOW_ID)],
+  );
+}
+
+describe("WorkflowService trigger denormalization on create", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("writes the trigger from the graph when a workflow is created with one", async () => {
+    const updateSpy: jest.SpyInstance = spyOnUpdateOneById();
+    spyOnApiPost();
+
+    await onCreateSuccess(createdWorkflow(scheduleGraph()));
+
+    expect(updateSpy).toHaveBeenCalledTimes(1);
+
+    const updateArgs: JSONObject = updateSpy.mock.calls[0]![0] as JSONObject;
+    const data: JSONObject = updateArgs["data"] as JSONObject;
+
+    expect(data["triggerId"]).toBe("Schedule");
+    expect(data["triggerArguments"]).toEqual({ "schedule-cron": "0 0 * * *" });
+
+    // must not recurse back through the hooks it is running inside of.
+    expect(updateArgs["props"]).toEqual({ isRoot: true, ignoreHooks: true });
+  });
+
+  test("notifies the workflow service so schedule triggers get registered", async () => {
+    spyOnUpdateOneById();
+    const postSpy: jest.SpyInstance = spyOnApiPost();
+
+    await onCreateSuccess(createdWorkflow(scheduleGraph()));
+
+    expect(postSpy).toHaveBeenCalledTimes(1);
+
+    const postArgs: JSONObject = postSpy.mock.calls[0]![0] as JSONObject;
+
+    expect(postArgs["url"]!.toString()).toContain(
+      `/workflow/update/${WORKFLOW_ID}`,
+    );
+  });
+
+  test("clears the trigger when the graph has no trigger node", async () => {
+    const updateSpy: jest.SpyInstance = spyOnUpdateOneById();
+    spyOnApiPost();
+
+    await onCreateSuccess(
+      createdWorkflow({
+        nodes: [
+          {
+            id: "node-1",
+            data: {
+              metadataId: "Webhook",
+              componentType: "Component",
+              nodeType: "Node",
+              arguments: {},
+            },
+          },
+        ],
+        edges: [],
+      }),
+    );
+
+    const data: JSONObject = (updateSpy.mock.calls[0]![0] as JSONObject)[
+      "data"
+    ] as JSONObject;
+
+    expect(data["triggerId"]).toBeNull();
+    expect(data["triggerArguments"]).toEqual({});
+  });
+
+  test("does no trigger work for a workflow created without a graph", async () => {
+    const updateSpy: jest.SpyInstance = spyOnUpdateOneById();
+    const postSpy: jest.SpyInstance = spyOnApiPost();
+
+    await onCreateSuccess(createdWorkflow(undefined));
+
+    expect(updateSpy).not.toHaveBeenCalled();
+    expect(postSpy).not.toHaveBeenCalled();
+  });
+
+  test("does not fail the create when the workflow service is unreachable", async () => {
+    const updateSpy: jest.SpyInstance = spyOnUpdateOneById();
+
+    jest
+      .spyOn(API, "post")
+      .mockRejectedValue(new Error("connect ECONNREFUSED") as never);
+
+    /*
+     * The row and its trigger are already persisted at this point, so a
+     * workflow service outage must not turn a successful import into an error.
+     */
+    await expect(
+      onCreateSuccess(createdWorkflow(scheduleGraph())),
+    ).resolves.toBeDefined();
+
+    expect(updateSpy).toHaveBeenCalledTimes(1);
+  });
+
+  test("derives the trigger for a genuinely imported workflow, which has no webhook secret yet", async () => {
+    const updateSpy: jest.SpyInstance = spyOnUpdateOneById();
+    spyOnApiPost();
+
+    /*
+     * webhookSecretKey carries create: [] access control, so it is stripped
+     * from export files and an imported workflow reaches this hook without
+     * one. That means two updates run in sequence - key generation, then
+     * trigger derivation - and the trigger write must still happen.
+     */
+    const imported: Workflow = {
+      _id: WORKFLOW_ID,
+      id: new ObjectID(WORKFLOW_ID),
+      graph: scheduleGraph(),
+    } as unknown as Workflow;
+
+    await onCreateSuccess(imported);
+
+    expect(updateSpy).toHaveBeenCalledTimes(2);
+
+    const secretUpdate: JSONObject = (
+      updateSpy.mock.calls[0]![0] as JSONObject
+    )["data"] as JSONObject;
+    const triggerUpdate: JSONObject = (
+      updateSpy.mock.calls[1]![0] as JSONObject
+    )["data"] as JSONObject;
+
+    expect(secretUpdate["webhookSecretKey"]).toBeDefined();
+    expect(triggerUpdate["triggerId"]).toBe("Schedule");
+  });
+
+  test("writes no trigger for a graph that carries no nodes key", async () => {
+    const updateSpy: jest.SpyInstance = spyOnUpdateOneById();
+    spyOnApiPost();
+
+    await onCreateSuccess(createdWorkflow({ edges: [] }));
+
+    expect(updateSpy).not.toHaveBeenCalled();
+  });
+});
+
+describe("WorkflowService trigger denormalization on update", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("writes the trigger when a saved graph carries one", async () => {
+    const updateSpy: jest.SpyInstance = spyOnUpdateOneById();
+    spyOnApiPost();
+
+    await onUpdateSuccess(scheduleGraph());
+
+    expect(updateSpy).toHaveBeenCalledTimes(1);
+
+    const data: JSONObject = (updateSpy.mock.calls[0]![0] as JSONObject)[
+      "data"
+    ] as JSONObject;
+
+    expect(data["triggerId"]).toBe("Schedule");
+    expect(data["triggerArguments"]).toEqual({ "schedule-cron": "0 0 * * *" });
+  });
+
+  test("clears the trigger when the trigger node is removed in the builder", async () => {
+    const updateSpy: jest.SpyInstance = spyOnUpdateOneById();
+    spyOnApiPost();
+
+    await onUpdateSuccess({ nodes: [], edges: [] });
+
+    const data: JSONObject = (updateSpy.mock.calls[0]![0] as JSONObject)[
+      "data"
+    ] as JSONObject;
+
+    expect(data["triggerId"]).toBeNull();
+    expect(data["triggerArguments"]).toEqual({});
+  });
+
+  test("touches no trigger columns when an update does not carry the graph", async () => {
+    const updateSpy: jest.SpyInstance = spyOnUpdateOneById();
+    spyOnApiPost();
+
+    // e.g. renaming a workflow must leave its trigger alone.
+    await onUpdateSuccess(undefined);
+
+    expect(updateSpy).not.toHaveBeenCalled();
+  });
+
+  test("always notifies the workflow service so trigger changes take effect", async () => {
+    spyOnUpdateOneById();
+    const postSpy: jest.SpyInstance = spyOnApiPost();
+
+    await onUpdateSuccess(scheduleGraph());
+
+    expect(postSpy).toHaveBeenCalledTimes(1);
+    expect(
+      (postSpy.mock.calls[0]![0] as JSONObject)["url"]!.toString(),
+    ).toContain(`/workflow/update/${WORKFLOW_ID}`);
+  });
+});

package/Tests/Server/Utils/AnalyticsDatabase/ClusterAwareSchema.test.ts

@@ -15,13 +15,21 @@ import {
 } from "../../../../Server/Utils/AnalyticsDatabase/ClusterConfig";
 import UpdateBy from "../../../../Server/Types/AnalyticsDatabase/UpdateBy";
 import "../../TestingUtils/Init";
-import AnalyticsBaseModel from "../../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
+import AnalyticsBaseModel, {
+  AnalyticsBaseModelType,
+} from "../../../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
 import Route from "../../../../Types/API/Route";
 import AnalyticsTableEngine from "../../../../Types/AnalyticsDatabase/AnalyticsTableEngine";
 import AnalyticsTableColumn, {
   SkipIndexType,
 } from "../../../../Types/AnalyticsDatabase/TableColumn";
 import TableColumnType from "../../../../Types/AnalyticsDatabase/TableColumnType";
+import MetricItemAggMV1m from "../../../../Models/AnalyticsModels/MetricItemAggMV1m";
+import MetricItemAggMV1mByHostV2 from "../../../../Models/AnalyticsModels/MetricItemAggMV1mByHostV2";
+import MetricItemAggMV1mByService from "../../../../Models/AnalyticsModels/MetricItemAggMV1mByService";
+import MetricItemAggMV1mByK8sCluster from "../../../../Models/AnalyticsModels/MetricItemAggMV1mByK8sCluster";
+import MetricItemAggMV1mByContainer from "../../../../Models/AnalyticsModels/MetricItemAggMV1mByContainer";
+import MetricBaselineHourly from "../../../../Models/AnalyticsModels/MetricBaselineHourly";
 
 const CLUSTER_ENV_KEY: string = "CLICKHOUSE_CLUSTER_NAME";
 const SHARDING_ENV_KEY: string = "CLICKHOUSE_SHARDING_KEY";
@@ -167,12 +175,38 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
               required: true,
               type: TableColumnType.ObjectID,
             }),
+            new AnalyticsTableColumn({
+              key: "bucketTime",
+              title: "Bucket",
+              description: "Bucket",
+              required: true,
+              type: TableColumnType.Date,
+            }),
+            new AnalyticsTableColumn({
+              key: "retentionDate",
+              title: "Retention",
+              description: "Retention",
+              required: true,
+              type: TableColumnType.Date,
+              simpleAggregateFunction: "max",
+            }),
+            new AnalyticsTableColumn({
+              key: "valueState",
+              title: "Value",
+              description: "Value",
+              required: true,
+              type: TableColumnType.AggregateFunction,
+              aggregateFunctionDefinition: "sum, Float64",
+            }),
           ],
           crudApiPath: new Route("agg"),
-          primaryKeys: ["projectId"],
-          sortKeys: ["projectId"],
+          primaryKeys: ["projectId", "bucketTime"],
+          sortKeys: ["projectId", "bucketTime"],
           partitionKey: "toYYYYMM(bucketTime)",
           tableEngine: AnalyticsTableEngine.AggregatingMergeTree,
+          ttlExpression: "retentionDate DELETE",
+          includeBaseColumns: false,
+          defaultSortColumn: "bucketTime",
         });
       }
     }
@@ -215,6 +249,133 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
       expect(fullText(stmt)).toContain("MetricItemAggMV1mLocal");
     });
 
+    test("AggregatingMergeTree CREATE uses real measures without the dimension bypass", () => {
+      const stmt: Statement = aggregatingGen.toTableCreateStatement();
+      expect(stmt.query).toContain("SimpleAggregateFunction(max, DateTime)");
+      expect(stmt.query).toContain("AggregateFunction(sum, Float64)");
+      expect(fullText(stmt)).not.toContain('"_id"');
+      expect(fullText(stmt)).not.toContain('"createdAt"');
+      expect(stmt.query).not.toContain("allow_dimensions_outside_sorting_key");
+    });
+
+    test("all metric aggregate models satisfy the strict off-key measure invariant", () => {
+      const models: Array<AnalyticsBaseModel> = [
+        new MetricItemAggMV1m(),
+        new MetricItemAggMV1mByHostV2(),
+        new MetricItemAggMV1mByService(),
+        new MetricItemAggMV1mByK8sCluster(),
+        new MetricItemAggMV1mByContainer(),
+        new MetricBaselineHourly(),
+      ];
+
+      for (const model of models) {
+        expect(model.tableEngine).toBe(
+          AnalyticsTableEngine.AggregatingMergeTree,
+        );
+        expect(model.getTableColumn("_id")).toBeNull();
+        expect(model.getTableColumn("createdAt")).toBeNull();
+        expect(model.getTableColumn("updatedAt")).toBeNull();
+        expect(model.defaultSortColumn).toBeTruthy();
+        expect(model.sortKeys).toContain(model.defaultSortColumn);
+        expect(model.tableSettings).not.toContain(
+          "allow_dimensions_outside_sorting_key",
+        );
+
+        for (const column of model.tableColumns) {
+          if (model.sortKeys.includes(column.key)) {
+            continue;
+          }
+          expect(
+            column.type === TableColumnType.AggregateFunction ||
+              Boolean(column.simpleAggregateFunction),
+          ).toBe(true);
+        }
+
+        const retentionDate: AnalyticsTableColumn | null =
+          model.getTableColumn("retentionDate");
+        if (retentionDate) {
+          expect(retentionDate.simpleAggregateFunction).toBe("max");
+          expect(model.materializedViews[0]?.query).toContain(
+            "maxSimpleState(retentionDate) AS retentionDate",
+          );
+        }
+      }
+    });
+
+    test("every concrete metric aggregate model generates strict ClickHouse DDL", () => {
+      const modelTypes: Array<AnalyticsBaseModelType> = [
+        MetricItemAggMV1m,
+        MetricItemAggMV1mByHostV2,
+        MetricItemAggMV1mByService,
+        MetricItemAggMV1mByK8sCluster,
+        MetricItemAggMV1mByContainer,
+        MetricBaselineHourly,
+      ];
+
+      for (const modelType of modelTypes) {
+        const model: AnalyticsBaseModel = new modelType();
+        const generator: StatementGenerator<AnalyticsBaseModel> =
+          new StatementGenerator<AnalyticsBaseModel>({
+            modelType,
+            database: ClickhouseAppInstance,
+          });
+        const statement: Statement = generator.toTableCreateStatement();
+        const ddl: string = fullText(statement);
+
+        expect(statement.query).toContain(
+          "ENGINE = ReplicatedAggregatingMergeTree",
+        );
+        expect(ddl).toContain(`${model.tableName}Local`);
+        expect(ddl).not.toContain('"_id"');
+        expect(ddl).not.toContain('"createdAt"');
+        expect(ddl).not.toContain('"updatedAt"');
+        expect(statement.query).not.toContain(
+          "allow_dimensions_outside_sorting_key",
+        );
+
+        if (model.getTableColumn("retentionDate")) {
+          expect(statement.query).toContain(
+            "retentionDate SimpleAggregateFunction(max, DateTime)",
+          );
+          expect(statement.query).toContain("TTL retentionDate DELETE");
+        } else {
+          expect(statement.query).not.toContain("retentionDate");
+        }
+      }
+    });
+
+    test("retention materialized views emit the state type their targets declare", () => {
+      const models: Array<AnalyticsBaseModel> = [
+        new MetricItemAggMV1m(),
+        new MetricItemAggMV1mByHostV2(),
+        new MetricItemAggMV1mByService(),
+        new MetricItemAggMV1mByK8sCluster(),
+        new MetricItemAggMV1mByContainer(),
+      ];
+
+      for (const model of models) {
+        const query: string = model.materializedViews[0]?.query || "";
+        expect(query).toContain(
+          "maxSimpleState(retentionDate) AS retentionDate",
+        );
+        expect(query).not.toContain("max(retentionDate) AS retentionDate");
+        expect(
+          model.getTableColumn("retentionDate")?.simpleAggregateFunction,
+        ).toBe("max");
+      }
+
+      const baseline: MetricBaselineHourly = new MetricBaselineHourly();
+      expect(baseline.getTableColumn("retentionDate")).toBeNull();
+      expect(baseline.materializedViews[0]?.query).not.toContain(
+        "retentionDate",
+      );
+    });
+
+    test("plain MergeTree CREATE does NOT add the aggregating-only setting", () => {
+      const stmt: Statement = spanGen.toTableCreateStatement();
+      expect(stmt.query).not.toContain("allow_dimensions_outside_sorting_key");
+    });
+
     test("Distributed wrapper uses the model sharding key and local table", () => {
       const q: string = spanGen.toDistributedTableCreateStatement().query;
       expect(q).toContain("ON CLUSTER 'oneuptime'");

package/Tests/Server/Utils/AnalyticsDatabase/StatementGenerator.test.ts

@@ -1102,6 +1102,105 @@ describe("StatementGenerator", () => {
       );
     });
 
+    test("emits AggregateFunction and scalar SimpleAggregateFunction measures", () => {
+      const statement: Statement = generator.toColumnsCreateStatement([
+        new AnalyticsTableColumn({
+          key: "retentionDate",
+          title: "<title>",
+          description: "<description>",
+          required: false,
+          type: TableColumnType.Date,
+          simpleAggregateFunction: "max",
+        }),
+        new AnalyticsTableColumn({
+          key: "valueState",
+          title: "<title>",
+          description: "<description>",
+          required: true,
+          type: TableColumnType.AggregateFunction,
+          aggregateFunctionDefinition: "sum, Float64",
+        }),
+      ]);
+
+      expectStatement(
+        statement,
+        SQL`retentionDate SimpleAggregateFunction(max, DateTime), valueState AggregateFunction(sum, Float64)`,
+      );
+    });
+
+    test("supports SimpleAggregateFunction on non-DateTime scalar types and codecs", () => {
+      const statement: Statement = generator.toColumnsCreateStatement([
+        new AnalyticsTableColumn({
+          key: "sampleCount",
+          title: "<title>",
+          description: "<description>",
+          required: true,
+          type: TableColumnType.UInt64,
+          simpleAggregateFunction: "sum",
+          codec: { codec: "ZSTD", level: 1 },
+        }),
+      ]);
+
+      expectStatement(
+        statement,
+        SQL`sampleCount SimpleAggregateFunction(sum, UInt64) CODEC(ZSTD(1))`,
+      );
+    });
+
+    test("rejects a column that declares both aggregate representations", () => {
+      expect(() => {
+        return generator.toColumnsCreateStatement([
+          new AnalyticsTableColumn({
+            key: "ambiguousState",
+            title: "<title>",
+            description: "<description>",
+            required: true,
+            type: TableColumnType.AggregateFunction,
+            aggregateFunctionDefinition: "sum, Float64",
+            simpleAggregateFunction: "sum",
+          }),
+        ]);
+      }).toThrow(
+        "Column ambiguousState cannot declare both AggregateFunction and SimpleAggregateFunction",
+      );
+    });
+
+    test.each(["", " max", "max ", "max)", "max, DateTime"])(
+      "rejects unsafe SimpleAggregateFunction name %p",
+      (functionName: string) => {
+        expect(() => {
+          return generator.toColumnsCreateStatement([
+            new AnalyticsTableColumn({
+              key: "invalidMeasure",
+              title: "<title>",
+              description: "<description>",
+              required: true,
+              type: TableColumnType.Date,
+              simpleAggregateFunction: functionName,
+            }),
+          ]);
+        }).toThrow(
+          `Column invalidMeasure has invalid simpleAggregateFunction "${functionName}"`,
+        );
+      },
+    );
+
+    test("rejects AggregateFunction without its state definition", () => {
+      expect(() => {
+        return generator.toColumnsCreateStatement([
+          new AnalyticsTableColumn({
+            key: "missingStateDefinition",
+            title: "<title>",
+            description: "<description>",
+            required: true,
+            type: TableColumnType.AggregateFunction,
+          }),
+        ]);
+      }).toThrow(
+        "Column missingStateDefinition is AggregateFunction but missing aggregateFunctionDefinition",
+      );
+    });
+
     test("emits single and pipelined CODEC clauses", () => {
       const statement: Statement = generator.toColumnsCreateStatement([
         new AnalyticsTableColumn({