@oneuptime/common 11.5.13 → 11.6.1

package/Server/Types/Workflow/Components/Schedule.ts

@@ -15,6 +15,7 @@ import ComponentID from "../../../../Types/Workflow/ComponentID";
 import ScheduleComponents from "../../../../Types/Workflow/Components/Schedule";
 import Workflow from "../../../../Models/DatabaseModels/Workflow";
 import CaptureSpan from "../../../Utils/Telemetry/CaptureSpan";
+import logger from "../../../Utils/Logger";
 
 export default class WebhookTrigger extends TriggerCode {
   public constructor() {
@@ -52,24 +53,37 @@ export default class WebhookTrigger extends TriggerCode {
 
     // query all workflows.
     for (const workflow of workflows) {
-      const executeWorkflow: ExecuteWorkflowType = {
-        workflowId: new ObjectID(workflow._id!),
-        returnValues: {},
-      };
-
-      if (
-        workflow.triggerArguments &&
-        workflow.triggerArguments["schedule"] &&
-        workflow.isEnabled
-      ) {
-        await props.scheduleWorkflow(
-          executeWorkflow,
-          workflow.triggerArguments["schedule"] as string,
+      /*
+       * Isolate each workflow: a single workflow whose schedule can't be
+       * registered (e.g. an invalid cron, or a transient queue error) must not
+       * abort the startup scan and leave every other scheduled workflow
+       * unregistered.
+       */
+      try {
+        const executeWorkflow: ExecuteWorkflowType = {
+          workflowId: new ObjectID(workflow._id!),
+          returnValues: {},
+        };
+
+        if (
+          workflow.triggerArguments &&
+          workflow.triggerArguments["schedule"] &&
+          workflow.isEnabled
+        ) {
+          await props.scheduleWorkflow(
+            executeWorkflow,
+            workflow.triggerArguments["schedule"] as string,
+          );
+        }
+
+        if (!workflow.isEnabled) {
+          await props.removeWorkflow(workflow.id!);
+        }
+      } catch (err) {
+        logger.error(
+          `Failed to register schedule for workflow ${workflow._id}`,
         );
-      }
-
-      if (!workflow.isEnabled) {
-        await props.removeWorkflow(workflow.id!);
+        logger.error(err);
       }
     }
   }
@@ -128,23 +142,34 @@ export default class WebhookTrigger extends TriggerCode {
       returnValues: {},
     };
 
-    if (
-      workflow.triggerArguments &&
-      workflow.triggerArguments["schedule"] &&
-      workflow.isEnabled
-    ) {
-      await this.scheduleWorkflow(
-        executeWorkflow,
-        workflow.triggerArguments["schedule"] as string,
-      );
-    }
+    /*
+     * Guard the (re)registration so a bad schedule can't fail the workflow
+     * save that triggered this update. Invalid crons are already surfaced as a
+     * workflow log by the queue layer; this catch is a backstop for unexpected
+     * errors (e.g. a transient queue error).
+     */
+    try {
+      if (
+        workflow.triggerArguments &&
+        workflow.triggerArguments["schedule"] &&
+        workflow.isEnabled
+      ) {
+        await this.scheduleWorkflow(
+          executeWorkflow,
+          workflow.triggerArguments["schedule"] as string,
+        );
+      }
 
-    if (!this.removeWorkflow) {
-      return;
-    }
+      if (!this.removeWorkflow) {
+        return;
+      }
 
-    if (!workflow.isEnabled) {
-      await this.removeWorkflow(workflow.id!);
+      if (!workflow.isEnabled) {
+        await this.removeWorkflow(workflow.id!);
+      }
+    } catch (err) {
+      logger.error(`Failed to update schedule for workflow ${workflow._id}`);
+      logger.error(err);
     }
   }
 }

package/Server/Utils/AnalyticsDatabase/InsertDedupContext.ts

@@ -0,0 +1,59 @@
+import { AsyncLocalStorage } from "async_hooks";
+
+/**
+ * Ambient context that makes ClickHouse inserts idempotent across queue
+ * retries. The telemetry queue worker wraps each job in
+ * `runWithInsertDedup(jobId, ...)`; token consumers then derive
+ * `insert_deduplication_token = "<tokenBase>:<table>:<chunkIndex>"`, so a
+ * stalled-job recovery or attempts-based retry that re-processes the same
+ * payload re-issues byte-identical tokens and ClickHouse drops the duplicate
+ * blocks (on replicated tables; on plain MergeTree the token is ignored
+ * unless non_replicated_deduplication_window is set — no harm either way).
+ * The chunk counter is per table because one job inserts into several tables
+ * (e.g. Span + ExceptionInstance) in a deterministic order.
+ *
+ * Two consumers exist:
+ * - AnalyticsDatabaseService.insertJsonRows stamps direct inserts that run
+ *   inside the context without an explicit token.
+ * - TelemetryFanInWriter captures a token per submission AT SUBMIT TIME
+ *   (still inside the job's context) so cross-job batching downstream can
+ *   preserve per-job retry idempotence.
+ */
+export interface InsertDedupContextStore {
+  tokenBase: string;
+  chunkIndexByTable: Map<string, number>;
+}
+
+const insertDedupContext: AsyncLocalStorage<InsertDedupContextStore> =
+  new AsyncLocalStorage<InsertDedupContextStore>();
+
+export function runWithInsertDedup<T>(
+  tokenBase: string,
+  fn: () => Promise<T>,
+): Promise<T> {
+  return insertDedupContext.run(
+    { tokenBase, chunkIndexByTable: new Map<string, number>() },
+    fn,
+  );
+}
+
+/**
+ * Consume the next deterministic dedup token for `tableName` from the
+ * ambient context, or undefined when called outside a runWithInsertDedup
+ * scope. Consuming advances the per-table chunk counter, so calls must
+ * happen in a deterministic order relative to the payload — true for the
+ * ingest services, whose chunking is a pure function of the (byte-identical
+ * across retries) job body.
+ */
+export function nextInsertDedupToken(tableName: string): string | undefined {
+  const store: InsertDedupContextStore | undefined =
+    insertDedupContext.getStore();
+
+  if (!store) {
+    return undefined;
+  }
+
+  const chunkIndex: number = store.chunkIndexByTable.get(tableName) ?? 0;
+  store.chunkIndexByTable.set(tableName, chunkIndex + 1);
+  return `${store.tokenBase}:${tableName}:${chunkIndex}`;
+}

package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts

@@ -69,6 +69,8 @@ export interface EntityScopeQueryValue {
   attributeValue: string;
 }
 
+const SIMPLE_AGGREGATE_FUNCTION_NAME_REGEX: RegExp = /^[A-Za-z_][A-Za-z0-9_]*$/;
+
 export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
   public model!: TBaseModel;
   public modelType!: { new (): TBaseModel };
@@ -1361,6 +1363,32 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
    * column. Scalar types ignore the rest of the column.
    */
   public toColumnType(column: AnalyticsTableColumn): Statement {
+    const simpleAggregateFunction: string | undefined =
+      column.simpleAggregateFunction;
+
+    if (simpleAggregateFunction !== undefined) {
+      if (column.type === TableColumnType.AggregateFunction) {
+        throw new BadDataException(
+          `Column ${column.key} cannot declare both AggregateFunction and SimpleAggregateFunction.`,
+        );
+      }
+
+      /*
+       * The function name is emitted as raw ClickHouse DDL, so accept only a
+       * plain function identifier. Besides catching accidental whitespace /
+       * empty definitions, this prevents a model field from injecting another
+       * type or table clause into generated schema SQL.
+       */
+      if (
+        simpleAggregateFunction.trim() !== simpleAggregateFunction ||
+        !SIMPLE_AGGREGATE_FUNCTION_NAME_REGEX.test(simpleAggregateFunction)
+      ) {
+        throw new BadDataException(
+          `Column ${column.key} has invalid simpleAggregateFunction "${simpleAggregateFunction}".`,
+        );
+      }
+    }
+
     if (column.type === TableColumnType.AggregateFunction) {
       const def: string | undefined = column.aggregateFunctionDefinition;
       if (!def) {
@@ -1371,7 +1399,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
       return SQL`AggregateFunction(`.append(def).append(SQL`)`);
     }
 
-    const statement: Statement | undefined = {
+    const scalarStatement: Statement | undefined = {
       [TableColumnType.Text]: SQL`String`,
       [TableColumnType.ObjectID]: SQL`String`,
       [TableColumnType.Boolean]: SQL`Bool`,
@@ -1394,13 +1422,21 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
       [TableColumnType.UInt64]: SQL`UInt64`,
     }[column.type];
 
-    if (!statement) {
+    if (!scalarStatement) {
       throw new BadDataException(
         `Unknown column type: ${column.type}. Please add support for this column type.`,
       );
     }
 
-    return statement;
+    if (simpleAggregateFunction) {
+      return SQL`SimpleAggregateFunction(`
+        .append(simpleAggregateFunction)
+        .append(SQL`, `)
+        .append(scalarStatement)
+        .append(SQL`)`);
+    }
+
+    return scalarStatement;
   }
 
   /**
@@ -1413,7 +1449,8 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
    */
   public toFullColumnType(column: AnalyticsTableColumn): Statement {
     const isAggregateFunction: boolean =
-      column.type === TableColumnType.AggregateFunction;
+      column.type === TableColumnType.AggregateFunction ||
+      Boolean(column.simpleAggregateFunction);
 
     let typeStatement: Statement = this.toColumnType(column);
 

package/Server/Utils/Express.ts

@@ -103,6 +103,19 @@ class Express {
 
     if (!this.httpServer) {
       this.httpServer = createServer(this.app);
+
+      /*
+       * Keep upstream (nginx) connection reuse safe. When the ingress pools
+       * idle upstream connections (proxy keepalive), the backend must NOT be
+       * the side that closes an idle socket first — otherwise nginx can pick a
+       * socket the backend already closed and 502 a request it cannot safely
+       * retry (e.g. a POST on the telemetry ingest path). Node's default
+       * keepAliveTimeout (~5s) is well below nginx's upstream keepalive_timeout
+       * (60s), so raise it above that window. headersTimeout must stay greater
+       * than keepAliveTimeout or Node can cut an idle socket mid-request.
+       */
+      this.httpServer.keepAliveTimeout = 75 * 1000;
+      this.httpServer.headersTimeout = 76 * 1000;
     }
 
     /*