@oneuptime/common 11.5.13 → 11.6.0

package/Server/Services/AnalyticsDatabaseService.ts

@@ -47,7 +47,7 @@ import {
   ResponseJSON,
   ResultSet,
 } from "@clickhouse/client";
-import { AsyncLocalStorage } from "node:async_hooks";
+import { nextInsertDedupToken } from "../Utils/AnalyticsDatabase/InsertDedupContext";
 import AnalyticsBaseModel from "../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
 import { WorkflowRoute } from "../../ServiceRoute";
 import Protocol from "../../Types/API/Protocol";
@@ -140,40 +140,42 @@ export const MigrationExecuteOptions: ClickhouseExecuteOptions = {
   },
 };
 
-/**
- * Ambient context that makes ClickHouse inserts idempotent across queue
- * retries. The telemetry queue worker wraps each job in
- * `runWithInsertDedup(jobId, ...)`; every insertJsonRows call inside the
- * job then stamps `insert_deduplication_token =
- * "<tokenBase>:<table>:<chunkIndex>"` plus async_insert_deduplicate=1 /
- * wait_for_async_insert=1, so a stalled-job retry that re-processes the
- * same payload re-issues byte-identical tokens and ClickHouse drops the
- * duplicate blocks (on replicated tables; on plain MergeTree the token is
- * ignored unless non_replicated_deduplication_window is set — no harm
- * either way). The chunk counter is per table because one job inserts
- * into several tables (e.g. Span + ExceptionInstance) in a deterministic
- * order.
+/*
+ * The insert-dedup ambient context lives in
+ * Utils/AnalyticsDatabase/InsertDedupContext so that both this service and
+ * TelemetryFanInWriter can consume deterministic tokens without an import
+ * cycle. Re-exported here for existing callers.
  *
- * HTTP-path inserts run outside the context and keep the fire-and-forget
- * async insert (wait_for_async_insert=0) — dedup waiting is only
- * affordable off the request thread.
+ * HTTP-path inserts run outside the context and are always fire-and-forget
+ * (wait_for_async_insert=0) — flush waiting, when opted into via
+ * TELEMETRY_WAIT_FOR_ASYNC_INSERT, is only affordable off the request
+ * thread and therefore only applies to tokened (queue-job) inserts.
  */
-export interface InsertDedupContextStore {
-  tokenBase: string;
-  chunkIndexByTable: Map<string, number>;
-}
+export {
+  runWithInsertDedup,
+  nextInsertDedupToken,
+  type InsertDedupContextStore,
+} from "../Utils/AnalyticsDatabase/InsertDedupContext";
 
-const insertDedupContext: AsyncLocalStorage<InsertDedupContextStore> =
-  new AsyncLocalStorage<InsertDedupContextStore>();
-
-export function runWithInsertDedup<T>(
-  tokenBase: string,
-  fn: () => Promise<T>,
-): Promise<T> {
-  return insertDedupContext.run(
-    { tokenBase, chunkIndexByTable: new Map<string, number>() },
-    fn,
-  );
+/*
+ * Ack mode for telemetry ClickHouse inserts.
+ *
+ * Default (false): wait_for_async_insert=0 — fire-and-forget. ClickHouse
+ * acks as soon as the batch is accepted into its async-insert buffer and
+ * owns flushing from there; inserts release their query slot almost
+ * immediately. The trade: flush-time errors surface only in server logs
+ * (system.asynchronous_inserts / part log), and a ClickHouse crash between
+ * buffer-accept and flush loses that buffer even though callers were acked.
+ *
+ * Set TELEMETRY_WAIT_FOR_ASYNC_INSERT=true to make every tokened insert
+ * wait for the durable flush before acking (ack-after-flush end to end
+ * through the fan-in writer and writer tier) — at the cost of each waiting
+ * insert holding a ClickHouse query slot until its buffer flushes.
+ */
+export function shouldWaitForAsyncInsert(): boolean {
+  const raw: string | undefined =
+    process.env["TELEMETRY_WAIT_FOR_ASYNC_INSERT"];
+  return raw === "true" || raw === "1";
 }
 
 export default class AnalyticsDatabaseService<
@@ -259,30 +261,31 @@ export default class AnalyticsDatabaseService<
     let dedupToken: string | undefined = options?.dedupToken;
 
     if (!dedupToken) {
-      const dedupStore: InsertDedupContextStore | undefined =
-        insertDedupContext.getStore();
-      if (dedupStore) {
-        const chunkIndex: number =
-          dedupStore.chunkIndexByTable.get(tableName) ?? 0;
-        dedupStore.chunkIndexByTable.set(tableName, chunkIndex + 1);
-        dedupToken = `${dedupStore.tokenBase}:${tableName}:${chunkIndex}`;
-      }
+      dedupToken = nextInsertDedupToken(tableName);
     }
 
+    const waitForAsyncInsert: 0 | 1 = shouldWaitForAsyncInsert() ? 1 : 0;
+
     let clickhouseSettings: ClickHouseSettings = {
       async_insert: 1,
-      wait_for_async_insert: 0,
+      wait_for_async_insert: waitForAsyncInsert,
     };
 
     if (dedupToken) {
       /*
-       * wait_for_async_insert=1 so the worker only acks the job after
-       * the block actually landed (or was deduplicated) — otherwise a
-       * crash between buffer-write and flush loses data with no retry.
+       * Dedup settings ride along in both ack modes. For async inserts
+       * ClickHouse dedups by content hash of the insert body when
+       * async_insert_deduplicate=1 (the explicit token is not yet honored
+       * for async inserts — ClickHouse #52018), so byte-identical queue
+       * retries are still dropped. The ack mode is a separate, deliberate
+       * trade (see shouldWaitForAsyncInsert): by default ClickHouse owns
+       * flushing and an ack means "accepted into the async-insert buffer";
+       * with TELEMETRY_WAIT_FOR_ASYNC_INSERT=true the ack waits for the
+       * durable flush instead.
        */
       clickhouseSettings = {
         async_insert: 1,
-        wait_for_async_insert: 1,
+        wait_for_async_insert: waitForAsyncInsert,
         async_insert_deduplicate: 1,
         insert_deduplication_token: dedupToken,
       };

package/Server/Services/DatabaseService.ts

@@ -1743,10 +1743,67 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
     });
   }
 
+  /*
+   * Update `data` may arrive as a full model instance rather than a plain
+   * partial — the QueryDeepPartialEntity type structurally admits both, and
+   * callers do construct `new Model()` payloads. A model instance carries
+   * non-column own properties from DatabaseBaseModel (every column
+   * initializer plus `isPermissionIf = {}`), and _updateBy turns every data
+   * key into a select column for its internal find, so the extra keys made
+   * that find throw `TableColumnMetadata not found for isPermissionIf
+   * column` and fail the whole update. Strip a model instance down to its
+   * set table columns; plain objects pass through untouched so a typo'd
+   * column name in a literal still fails loudly instead of being silently
+   * dropped.
+   *
+   * `_id`, `createdAt`, `updatedAt` and `version` are dropped for model
+   * instances the same way BaseAPI drops them from client payloads: the row
+   * is located by the update query (spreading a foreign `_id` into the save
+   * payload would redirect the write), timestamps are database-managed, and
+   * `version` is TypeORM's optimistic-concurrency counter.
+   */
+  public sanitizeUpdateData(
+    data: UpdateBy<TBaseModel>["data"],
+  ): UpdateBy<TBaseModel>["data"] {
+    if (!(data instanceof BaseModel)) {
+      return data;
+    }
+
+    const plainData: JSONObject = {};
+
+    for (const key of Object.keys(data)) {
+      if (!this.model.isTableColumn(key)) {
+        continue;
+      }
+
+      if (
+        key === "_id" ||
+        key === "createdAt" ||
+        key === "updatedAt" ||
+        key === "version"
+      ) {
+        continue;
+      }
+
+      const value: JSONValue = (data as any)[key];
+
+      // Unset columns must not become writes (or select columns).
+      if (value === undefined) {
+        continue;
+      }
+
+      plainData[key] = value;
+    }
+
+    return plainData as UpdateBy<TBaseModel>["data"];
+  }
+
   private async _updateBy(updateBy: UpdateBy<TBaseModel>): Promise<number> {
     try {
       this.setTelemetryContextFromProps(updateBy.props);
 
+      updateBy.data = this.sanitizeUpdateData(updateBy.data);
+
       const onUpdate: OnUpdate<TBaseModel> = updateBy.props.ignoreHooks
         ? { updateBy, carryForward: [] }
         : await this.onBeforeUpdate(updateBy);
@@ -1830,9 +1887,17 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
       });
 
       for (const item of items) {
+        /*
+         * _id must be set AFTER the spread: update data can carry an
+         * explicit `_id: undefined` (sanitizeUpdateData strips it from model
+         * instances, but a plain object can still hold it), and spreading it
+         * after _id would clobber the located row's id — save() then sees no
+         * primary key, INSERTs instead of updating, and dies on the first
+         * NOT NULL column.
+         */
         const updatedItem: any = {
-          _id: item._id!,
           ...data,
+          _id: item._id!,
         } as any;
 
         logger.debug("Updated Item", {

package/Server/Services/WorkflowService.ts

@@ -49,6 +49,40 @@ export class Service extends DatabaseService<Model> {
       createdItem.webhookSecretKey = secretKey;
     }
 
+    /*
+     * A workflow that arrives with its graph already in place - imported from
+     * a JSON export, or duplicated from another workflow - never passes
+     * through onUpdateSuccess, so nothing has denormalized its trigger onto
+     * the row. The runner looks workflows up by triggerId, so without this the
+     * workflow is saved and looks correct in the builder but silently never
+     * fires.
+     */
+    if (createdItem.id && createdItem.graph) {
+      await this.saveTriggerFromGraph({
+        workflowId: createdItem.id,
+        graph: createdItem.graph,
+      });
+
+      /*
+       * Registers schedule triggers with the runner. Best effort: the row and
+       * its trigger are already persisted, so a workflow service outage must
+       * not fail the create. The trigger is already on the row, so the next
+       * save - or the runner's own startup scan in Schedule.init, which
+       * queries by triggerId - picks the workflow up.
+       */
+      try {
+        await this.notifyWorkflowService(createdItem.id);
+      } catch (error) {
+        logger.error(
+          `Error notifying workflow service of created workflow: ${error}`,
+          {
+            projectId: createdItem.projectId?.toString(),
+            workflowId: createdItem.id?.toString(),
+          } as LogAttributes,
+        );
+      }
+    }
+
     if (createdItem.projectId && createdItem.id) {
       /*
        * Run label rule first so rule-added labels are persisted before
@@ -87,39 +121,13 @@ export class Service extends DatabaseService<Model> {
   ): Promise<OnUpdate<Model>> {
     /// save trigger and trigger args.
 
-    if (
-      onUpdate.updateBy.data &&
-      (onUpdate.updateBy.data as any).graph &&
-      (((onUpdate.updateBy.data as any).graph as any)[
-        "nodes"
-      ] as Array<JSONObject>)
-    ) {
-      let trigger: NodeDataProp | null = null;
-
-      // check if it has a trigger node.
-      for (const node of ((onUpdate.updateBy.data as any).graph as any)[
-        "nodes"
-      ] as Array<JSONObject>) {
-        const nodeData: NodeDataProp = node["data"] as any;
-        if (
-          nodeData.componentType === ComponentType.Trigger &&
-          nodeData.nodeType === NodeType.Node
-        ) {
-          // found the trigger;
-          trigger = nodeData;
-        }
-      }
+    const updatedGraph: JSONObject | undefined = (onUpdate.updateBy.data as any)
+      ?.graph as JSONObject | undefined;
 
-      await this.updateOneById({
-        id: new ObjectID(onUpdate.updateBy.query._id! as any),
-        data: {
-          triggerId: trigger?.metadataId || null,
-          triggerArguments: trigger?.arguments || {},
-        } as any,
-        props: {
-          isRoot: true,
-          ignoreHooks: true,
-        },
+    if (updatedGraph) {
+      await this.saveTriggerFromGraph({
+        workflowId: new ObjectID(onUpdate.updateBy.query._id! as any),
+        graph: updatedGraph,
       });
     }
 
@@ -127,23 +135,76 @@ export class Service extends DatabaseService<Model> {
       workflowId: onUpdate.updateBy.query._id?.toString(),
     } as LogAttributes);
 
+    await this.notifyWorkflowService(
+      new ObjectID(onUpdate.updateBy.query._id! as any),
+    );
+
+    logger.debug("Updated workflow on the workflow service", {
+      workflowId: onUpdate.updateBy.query._id?.toString(),
+    } as LogAttributes);
+
+    return onUpdate;
+  }
+
+  /*
+   * The trigger node is denormalized out of the graph onto triggerId /
+   * triggerArguments because the runner queries workflows by trigger and
+   * cannot parse every graph to do it. A graph with no trigger node clears
+   * both columns, which is how a workflow stops firing when its trigger is
+   * removed in the builder.
+   */
+  private async saveTriggerFromGraph(data: {
+    workflowId: ObjectID;
+    graph: JSONObject;
+  }): Promise<void> {
+    const nodes: Array<JSONObject> | undefined = data.graph["nodes"] as
+      | Array<JSONObject>
+      | undefined;
+
+    if (!nodes || !Array.isArray(nodes)) {
+      return;
+    }
+
+    let trigger: NodeDataProp | null = null;
+
+    // check if it has a trigger node.
+    for (const node of nodes) {
+      const nodeData: NodeDataProp = node["data"] as any;
+
+      if (
+        nodeData?.componentType === ComponentType.Trigger &&
+        nodeData?.nodeType === NodeType.Node
+      ) {
+        // found the trigger;
+        trigger = nodeData;
+      }
+    }
+
+    await this.updateOneById({
+      id: data.workflowId,
+      data: {
+        triggerId: trigger?.metadataId || null,
+        triggerArguments: trigger?.arguments || {},
+      } as any,
+      props: {
+        isRoot: true,
+        ignoreHooks: true,
+      },
+    });
+  }
+
+  private async notifyWorkflowService(workflowId: ObjectID): Promise<void> {
     await API.post<EmptyResponseData>({
       url: new URL(
         Protocol.HTTP,
         WorkflowHostname,
-        new Route("/workflow/update/" + onUpdate.updateBy.query._id!),
+        new Route("/workflow/update/" + workflowId.toString()),
       ),
       data: {},
       headers: {
         ...ClusterKeyAuthorization.getClusterKeyHeaders(),
       },
     });
-
-    logger.debug("Updated workflow on the workflow service", {
-      workflowId: onUpdate.updateBy.query._id?.toString(),
-    } as LogAttributes);
-
-    return onUpdate;
   }
 }
 export default new Service();

package/Server/Types/Workflow/Components/Schedule.ts

@@ -15,6 +15,7 @@ import ComponentID from "../../../../Types/Workflow/ComponentID";
 import ScheduleComponents from "../../../../Types/Workflow/Components/Schedule";
 import Workflow from "../../../../Models/DatabaseModels/Workflow";
 import CaptureSpan from "../../../Utils/Telemetry/CaptureSpan";
+import logger from "../../../Utils/Logger";
 
 export default class WebhookTrigger extends TriggerCode {
   public constructor() {
@@ -52,24 +53,37 @@ export default class WebhookTrigger extends TriggerCode {
 
     // query all workflows.
     for (const workflow of workflows) {
-      const executeWorkflow: ExecuteWorkflowType = {
-        workflowId: new ObjectID(workflow._id!),
-        returnValues: {},
-      };
-
-      if (
-        workflow.triggerArguments &&
-        workflow.triggerArguments["schedule"] &&
-        workflow.isEnabled
-      ) {
-        await props.scheduleWorkflow(
-          executeWorkflow,
-          workflow.triggerArguments["schedule"] as string,
+      /*
+       * Isolate each workflow: a single workflow whose schedule can't be
+       * registered (e.g. an invalid cron, or a transient queue error) must not
+       * abort the startup scan and leave every other scheduled workflow
+       * unregistered.
+       */
+      try {
+        const executeWorkflow: ExecuteWorkflowType = {
+          workflowId: new ObjectID(workflow._id!),
+          returnValues: {},
+        };
+
+        if (
+          workflow.triggerArguments &&
+          workflow.triggerArguments["schedule"] &&
+          workflow.isEnabled
+        ) {
+          await props.scheduleWorkflow(
+            executeWorkflow,
+            workflow.triggerArguments["schedule"] as string,
+          );
+        }
+
+        if (!workflow.isEnabled) {
+          await props.removeWorkflow(workflow.id!);
+        }
+      } catch (err) {
+        logger.error(
+          `Failed to register schedule for workflow ${workflow._id}`,
         );
-      }
-
-      if (!workflow.isEnabled) {
-        await props.removeWorkflow(workflow.id!);
+        logger.error(err);
       }
     }
   }
@@ -128,23 +142,34 @@ export default class WebhookTrigger extends TriggerCode {
       returnValues: {},
     };
 
-    if (
-      workflow.triggerArguments &&
-      workflow.triggerArguments["schedule"] &&
-      workflow.isEnabled
-    ) {
-      await this.scheduleWorkflow(
-        executeWorkflow,
-        workflow.triggerArguments["schedule"] as string,
-      );
-    }
+    /*
+     * Guard the (re)registration so a bad schedule can't fail the workflow
+     * save that triggered this update. Invalid crons are already surfaced as a
+     * workflow log by the queue layer; this catch is a backstop for unexpected
+     * errors (e.g. a transient queue error).
+     */
+    try {
+      if (
+        workflow.triggerArguments &&
+        workflow.triggerArguments["schedule"] &&
+        workflow.isEnabled
+      ) {
+        await this.scheduleWorkflow(
+          executeWorkflow,
+          workflow.triggerArguments["schedule"] as string,
+        );
+      }
 
-    if (!this.removeWorkflow) {
-      return;
-    }
+      if (!this.removeWorkflow) {
+        return;
+      }
 
-    if (!workflow.isEnabled) {
-      await this.removeWorkflow(workflow.id!);
+      if (!workflow.isEnabled) {
+        await this.removeWorkflow(workflow.id!);
+      }
+    } catch (err) {
+      logger.error(`Failed to update schedule for workflow ${workflow._id}`);
+      logger.error(err);
     }
   }
 }

package/Server/Utils/AnalyticsDatabase/InsertDedupContext.ts

@@ -0,0 +1,59 @@
+import { AsyncLocalStorage } from "async_hooks";
+
+/**
+ * Ambient context that makes ClickHouse inserts idempotent across queue
+ * retries. The telemetry queue worker wraps each job in
+ * `runWithInsertDedup(jobId, ...)`; token consumers then derive
+ * `insert_deduplication_token = "<tokenBase>:<table>:<chunkIndex>"`, so a
+ * stalled-job recovery or attempts-based retry that re-processes the same
+ * payload re-issues byte-identical tokens and ClickHouse drops the duplicate
+ * blocks (on replicated tables; on plain MergeTree the token is ignored
+ * unless non_replicated_deduplication_window is set — no harm either way).
+ * The chunk counter is per table because one job inserts into several tables
+ * (e.g. Span + ExceptionInstance) in a deterministic order.
+ *
+ * Two consumers exist:
+ * - AnalyticsDatabaseService.insertJsonRows stamps direct inserts that run
+ *   inside the context without an explicit token.
+ * - TelemetryFanInWriter captures a token per submission AT SUBMIT TIME
+ *   (still inside the job's context) so cross-job batching downstream can
+ *   preserve per-job retry idempotence.
+ */
+export interface InsertDedupContextStore {
+  tokenBase: string;
+  chunkIndexByTable: Map<string, number>;
+}
+
+const insertDedupContext: AsyncLocalStorage<InsertDedupContextStore> =
+  new AsyncLocalStorage<InsertDedupContextStore>();
+
+export function runWithInsertDedup<T>(
+  tokenBase: string,
+  fn: () => Promise<T>,
+): Promise<T> {
+  return insertDedupContext.run(
+    { tokenBase, chunkIndexByTable: new Map<string, number>() },
+    fn,
+  );
+}
+
+/**
+ * Consume the next deterministic dedup token for `tableName` from the
+ * ambient context, or undefined when called outside a runWithInsertDedup
+ * scope. Consuming advances the per-table chunk counter, so calls must
+ * happen in a deterministic order relative to the payload — true for the
+ * ingest services, whose chunking is a pure function of the (byte-identical
+ * across retries) job body.
+ */
+export function nextInsertDedupToken(tableName: string): string | undefined {
+  const store: InsertDedupContextStore | undefined =
+    insertDedupContext.getStore();
+
+  if (!store) {
+    return undefined;
+  }
+
+  const chunkIndex: number = store.chunkIndexByTable.get(tableName) ?? 0;
+  store.chunkIndexByTable.set(tableName, chunkIndex + 1);
+  return `${store.tokenBase}:${tableName}:${chunkIndex}`;
+}

package/Server/Utils/Express.ts

@@ -103,6 +103,19 @@ class Express {
 
     if (!this.httpServer) {
       this.httpServer = createServer(this.app);
+
+      /*
+       * Keep upstream (nginx) connection reuse safe. When the ingress pools
+       * idle upstream connections (proxy keepalive), the backend must NOT be
+       * the side that closes an idle socket first — otherwise nginx can pick a
+       * socket the backend already closed and 502 a request it cannot safely
+       * retry (e.g. a POST on the telemetry ingest path). Node's default
+       * keepAliveTimeout (~5s) is well below nginx's upstream keepalive_timeout
+       * (60s), so raise it above that window. headersTimeout must stay greater
+       * than keepAliveTimeout or Node can cut an idle socket mid-request.
+       */
+      this.httpServer.keepAliveTimeout = 75 * 1000;
+      this.httpServer.headersTimeout = 76 * 1000;
     }
 
     /*