@oneuptime/common 12.0.33 → 13.0.0

package/Server/Utils/Telemetry/TelemetryFanInWriter.ts

@@ -125,7 +125,10 @@ export interface FanInWriterOptions {
   maxWaitMs: number;
   /** Per-pod cap on concurrent ClickHouse inserts across all tables. */
   maxConcurrentInserts: number;
-  /** High-water mark of buffered+in-flight rows; submit() blocks above it. */
+  /**
+   * High-water mark of reserved+buffered+in-flight rows. A whole submission
+   * may cross it once; later submit() calls wait for capacity to be released.
+   */
   maxPendingRows: number;
   retryMaxAttempts: number;
   retryBaseDelayMs: number;
@@ -197,6 +200,11 @@ type TableBuffer = {
   timer: NodeJS.Timeout | null;
 };
 
+type CapacityWaiter = {
+  rowCount: number;
+  resolve: () => void;
+};
+
 /*
  * ClickHouse server error codes that indicate transient overload or an
  * ambiguous outcome. All are safe to retry here because every insert
@@ -328,7 +336,7 @@ export class TelemetryFanInWriter {
   private options: FanInWriterOptions;
   private buffers: Map<string, TableBuffer> = new Map();
   private pendingRows: number = 0;
-  private capacityWaiters: Array<() => void> = [];
+  private capacityWaiters: Set<CapacityWaiter> = new Set();
   private activeInserts: number = 0;
   private insertSlotWaiters: Array<() => void> = [];
   private inFlightDispatches: Set<Promise<void>> = new Set();
@@ -428,7 +436,7 @@ export class TelemetryFanInWriter {
 
     this.acceptingSubmits++;
     try {
-      await this.waitForCapacity();
+      await this.waitForCapacity(rows.length);
 
       let buffer: TableBuffer | undefined = this.buffers.get(tableName);
       if (!buffer) {
@@ -458,7 +466,6 @@ export class TelemetryFanInWriter {
         rejectAck,
       });
       buffer.rowCount += rows.length;
-      this.pendingRows += rows.length;
 
       if (buffer.rowCount >= this.getMaxBatchRows(tableName)) {
         this.cutAndDispatch(tableName, false);
@@ -532,24 +539,39 @@ export class TelemetryFanInWriter {
     );
   }
 
-  private async waitForCapacity(): Promise<void> {
-    while (this.pendingRows >= this.options.maxPendingRows) {
-      await new Promise<void>((resolve: () => void) => {
-        this.capacityWaiters.push(resolve);
-      });
+  private async waitForCapacity(rowCount: number): Promise<void> {
+    /*
+     * Reserve before yielding: concurrent submitters otherwise all observe
+     * the same available capacity and buffer their rows after the await,
+     * multiplying the memory limit by the number of submitters. Keep whole
+     * submissions intact: one submission may cross the high-water mark,
+     * but every later submission waits until pending rows fall below it.
+     */
+    if (this.pendingRows < this.options.maxPendingRows) {
+      this.pendingRows += rowCount;
+      return;
     }
+
+    await new Promise<void>((resolve: () => void) => {
+      this.capacityWaiters.add({ rowCount, resolve });
+    });
   }
 
   private releaseCapacity(): void {
     /*
-     * Wake all waiters; each re-checks the high-water mark in its
-     * waitForCapacity loop, so overshoot stays bounded to one submission
-     * per waiter.
+     * Transfer capacity to FIFO waiters before waking them. Only admitted
+     * callers resume, so completing an insert does not wake the entire
+     * backlog to compete for the same capacity again.
+     * Set keeps FIFO order and releases each waiter without shifting the
+     * remaining queue or retaining consumed entries.
      */
-    const waiters: Array<() => void> = this.capacityWaiters;
-    this.capacityWaiters = [];
-    for (const wake of waiters) {
-      wake();
+    for (const waiter of this.capacityWaiters) {
+      if (this.pendingRows >= this.options.maxPendingRows) {
+        break;
+      }
+      this.pendingRows += waiter.rowCount;
+      this.capacityWaiters.delete(waiter);
+      waiter.resolve();
     }
   }
 

package/Server/Utils/Telemetry/TelemetryIngestionKeyGuard.ts

@@ -0,0 +1,92 @@
+import TelemetryIngestionKeyPolicy from "../../../Types/Telemetry/TelemetryIngestionKeyPolicy";
+import TelemetryIngestionKeyType from "../../../Types/Telemetry/TelemetryIngestionKeyType";
+import TelemetryIngestSurface, {
+  BROWSER_ALLOWED_INGEST_SURFACES,
+  getIngestSurfaceReadableName,
+} from "../../../Types/Telemetry/TelemetryIngestSurface";
+
+/*
+ * The three key-state checks that every ingest entry point has to make, in
+ * one place.
+ *
+ * TelemetryIngest (the Express middleware) runs these inline as part of a
+ * longer chain that also covers the Origin allowlist and the per-key rate
+ * limit, neither of which means anything off HTTP. Everything else that
+ * accepts a TelemetryIngestionKey - the gRPC OTLP server, the MQTT broker,
+ * and the two key-validation probes - is NOT an Express route and cannot use
+ * that middleware, yet must answer these same three questions or a disabled
+ * key keeps working on the pipe nobody remembered to update.
+ *
+ * The refusal SENTENCES are deliberately identical to the middleware's, so a
+ * customer who moves the same key between HTTP, gRPC and MQTT reads the same
+ * explanation each time rather than three differently-worded versions of one
+ * problem. If you change wording here, change it there too.
+ *
+ * What this deliberately does NOT do is decide the transport-level response.
+ * A refusal is a status code on HTTP, a CONNACK return code on MQTT, and a
+ * silent success on gRPC (see GrpcServer for why); each caller owns that
+ * translation because each transport's retry semantics are its own.
+ */
+export enum TelemetryIngestionKeyRefusalReason {
+  Disabled = "disabled",
+  Expired = "expired",
+  SurfaceNotAllowedForBrowserKey = "surface-not-allowed-for-browser-key",
+}
+
+export interface TelemetryIngestionKeyRefusal {
+  reason: TelemetryIngestionKeyRefusalReason;
+
+  /*
+   * Safe to return to whoever presented the key. It names the problem and
+   * never the key, the project, the allowlist or the expiry timestamp - a
+   * caller holding a dead credential is not entitled to a project's
+   * configuration.
+   */
+  message: string;
+}
+
+export default class TelemetryIngestionKeyGuard {
+  /**
+   * Why this key may not write to this surface, or null when it may.
+   *
+   * Order matters and mirrors the middleware: the kill switch is checked
+   * first so that switching a leaked key off is the one action guaranteed to
+   * stop it whatever else is true of the key, then expiry, then the
+   * browser-key surface rule.
+   */
+  public static getRefusal(data: {
+    policy: TelemetryIngestionKeyPolicy;
+    surface: TelemetryIngestSurface;
+  }): TelemetryIngestionKeyRefusal | null {
+    const policy: TelemetryIngestionKeyPolicy = data.policy;
+
+    if (policy.isEnabled === false) {
+      return {
+        reason: TelemetryIngestionKeyRefusalReason.Disabled,
+        message: "This telemetry ingestion key has been disabled.",
+      };
+    }
+
+    if (policy.expiresAt && policy.expiresAt.getTime() <= Date.now()) {
+      return {
+        reason: TelemetryIngestionKeyRefusalReason.Expired,
+        message: "This telemetry ingestion key expired.",
+      };
+    }
+
+    if (
+      policy.keyType === TelemetryIngestionKeyType.Browser &&
+      !BROWSER_ALLOWED_INGEST_SURFACES.has(data.surface)
+    ) {
+      return {
+        reason:
+          TelemetryIngestionKeyRefusalReason.SurfaceNotAllowedForBrowserKey,
+        message: `A browser ingestion key cannot be used for ${getIngestSurfaceReadableName(
+          data.surface,
+        )}. Use a server ingestion key.`,
+      };
+    }
+
+    return null;
+  }
+}

package/Server/Utils/Telemetry/TelemetryIngestionKeyRateLimiter.ts

@@ -0,0 +1,217 @@
+import Redis, { ClientType } from "../../Infrastructure/Redis";
+import logger from "../Logger";
+import ObjectID from "../../../Types/ObjectID";
+
+/*
+ * Per-ingestion-key request rate limiting.
+ *
+ * This exists because a Browser ingestion key is, by design, published in a
+ * page: anyone who views source has it. Rotation and the origin allowlist
+ * bound WHO can use a scraped key, but neither bounds HOW MUCH a caller who
+ * satisfies them can push. Without a per-key ceiling, one scraped key plus a
+ * loop is unbounded write volume into a customer's project - poisoned
+ * dashboards, forged spans and, on a metered plan, a bill. The ceiling is
+ * per key rather than per project so that turning the abuse off (rotate or
+ * disable that one key) never takes a customer's server ingest down with it.
+ *
+ * Built on raw Redis INCR + EXPIRE, mirroring
+ * App/FeatureSet/Telemetry/Utils/SessionReplayRateLimiter.ts. The same
+ * reasoning applies: a per-process in-memory counter is useless across
+ * autoscaled ingest pods, and GlobalCache has no counter primitive at all
+ * (only get/set, so a read-modify-write there would lose increments under
+ * exactly the concurrency this is meant to bound).
+ *
+ * ---------------------------------------------------------------------
+ * THIS LIMITER FAILS OPEN, AND THAT IS DIFFERENT FROM SESSION REPLAY.
+ * ---------------------------------------------------------------------
+ * When Redis is unreachable this returns CounterUnavailable, and the CALLER
+ * is expected to ADMIT the request anyway. The limit here bounds ABUSE of a
+ * key that leaked; it is not a privacy control and it is not a hard billing
+ * quota. Weigh the two failure modes during a Redis blip:
+ *
+ *   - fail closed: every paying customer's production telemetry stops for
+ *     the duration of the outage, and the gap is permanent because dropped
+ *     spans are not replayed. An infrastructure hiccup becomes a
+ *     customer-visible data loss incident, and the operator's incident
+ *     dashboards go dark at the exact moment they are needed.
+ *
+ *   - fail open: for the length of the blip an attacker who already holds a
+ *     scraped key gets an unbounded window. They already had that window up
+ *     until this feature shipped, the origin allowlist and enable/disable
+ *     switch still apply (they are resolved from Postgres, not Redis), and
+ *     Redis being down is itself alarmed on.
+ *
+ * The second is plainly the better trade for telemetry. SessionReplayRate-
+ * Limiter fails CLOSED because a recording is a video of a real person and a
+ * budget that cannot be counted cannot be honoured - accepting an unbounded
+ * volume of that is not a recoverable mistake. Same machinery, opposite
+ * answer, because the thing being protected is different.
+ */
+
+const RATE_KEY_PREFIX: string = "telemetry:ingestkey:rate:";
+
+/*
+ * Two minutes, so the counter for a minute bucket outlives the bucket it
+ * belongs to. A request landing right on a boundary cannot then read a key
+ * that was already reclaimed mid-window and be handed a fresh allowance.
+ */
+const RATE_TTL_SECONDS: number = 120;
+
+const WINDOW_MS: number = 60000;
+
+/*
+ * A Redis outage puts EVERY request on the unavailable path, so an unguarded
+ * log line there is one line per ingested request for as long as the outage
+ * lasts - the ingest path is the highest-volume surface in the product, so
+ * that buries the very incident it is reporting. One line per interval is
+ * enough to make the condition visible. Same idea as the throttle in
+ * Common/Server/Middleware/PublicDashboardRateLimit.ts.
+ */
+const COUNTER_UNAVAILABLE_LOG_INTERVAL_MS: number = 60 * 1000;
+
+export enum TelemetryIngestionKeyLimitOutcome {
+  /* Within this key's per-minute allowance. */
+  Allowed = "allowed",
+
+  /* Over the allowance. Retryable when the fixed window rolls. */
+  RateLimited = "rate-limited",
+
+  /*
+   * Redis is unreachable, so no limit can be honoured either way. The caller
+   * admits the request - see the fail-open rationale at the top of the file.
+   */
+  CounterUnavailable = "counter-unavailable",
+}
+
+export interface TelemetryIngestionKeyLimitDecision {
+  outcome: TelemetryIngestionKeyLimitOutcome;
+  retryAfterSeconds?: number | undefined;
+
+  /*
+   * True only on the request that first crossed the line in this window.
+   *
+   * The whole point of a limiter is that the rejected caller keeps knocking,
+   * so a caller that logs every refusal turns an ingest flood into a second
+   * flood in the log pipeline. Logging only the crossing gives exactly one
+   * line per key per minute, which is what an operator actually wants when
+   * they are trying to see WHICH key is being abused. Unset unless rejected.
+   */
+  isFirstRejectionInWindow?: boolean | undefined;
+}
+
+export default class TelemetryIngestionKeyRateLimiter {
+  /*
+   * Consume one request from this key's per-minute allowance.
+   *
+   * Fixed window keyed on the minute bucket, deliberately not a sliding
+   * window: the sliding version needs a sorted set per key and a read of it
+   * per request, and this runs in front of every OTLP payload the product
+   * accepts. A fixed window admits at most 2x the limit across an unlucky
+   * boundary pair, which is an entirely acceptable overshoot for an
+   * abuse ceiling and costs one INCR.
+   *
+   * The counter is incremented on rejected requests too. That is standard
+   * fixed-window behaviour and it is the point: a client hammering a scraped
+   * key keeps its own window pinned rather than being handed a fresh
+   * allowance for free by virtue of having been refused.
+   */
+  public static async consume(data: {
+    ingestionKeyId: ObjectID;
+    limitPerMinute: number;
+  }): Promise<TelemetryIngestionKeyLimitDecision> {
+    /*
+     * A non-positive or non-finite limit means "no limit" rather than "block
+     * everything", and short-circuits before Redis is touched.
+     *
+     * The column behind this is nullable and the caller resolves NULL to the
+     * shipped browser default (or to unlimited for a Server key), so a zero
+     * or garbage value reaching here is a bug or a bad row, not a customer
+     * asking for a total block. Reading it as a block would silently black
+     * out that key's ingest, and this file's whole posture (above) is that a
+     * malfunction in the limiter must not stop real telemetry.
+     */
+    if (!Number.isFinite(data.limitPerMinute) || data.limitPerMinute <= 0) {
+      return { outcome: TelemetryIngestionKeyLimitOutcome.Allowed };
+    }
+
+    const client: ClientType | null = Redis.getClient();
+
+    if (!client || !Redis.isConnected()) {
+      return { outcome: TelemetryIngestionKeyLimitOutcome.CounterUnavailable };
+    }
+
+    const minuteBucket: number = Math.floor(Date.now() / WINDOW_MS);
+    const key: string = `${RATE_KEY_PREFIX}${data.ingestionKeyId.toString()}:${minuteBucket}`;
+
+    try {
+      const total: number = await client.incr(key);
+
+      /*
+       * Set the expiry only on the write that CREATED the key. Re-issuing
+       * EXPIRE on every increment would slide the window forward for as long
+       * as the load continues, so the counter would never reset and a key
+       * that tripped its limit once under sustained traffic could never
+       * recover - which is precisely the state an attacker's loop produces.
+       */
+      if (total === 1) {
+        await client.expire(key, RATE_TTL_SECONDS);
+      }
+
+      if (total > data.limitPerMinute) {
+        return {
+          outcome: TelemetryIngestionKeyLimitOutcome.RateLimited,
+          retryAfterSeconds:
+            TelemetryIngestionKeyRateLimiter.getSecondsUntilWindowEnd(),
+
+          /*
+           * total is monotonic within the window, so the request that took
+           * it to limit + 1 is the unique first rejection.
+           */
+          isFirstRejectionInWindow: total === data.limitPerMinute + 1,
+        };
+      }
+
+      return { outcome: TelemetryIngestionKeyLimitOutcome.Allowed };
+    } catch (err) {
+      if (TelemetryIngestionKeyRateLimiter.shouldLogCounterUnavailable()) {
+        logger.warn(
+          `TelemetryIngestionKeyRateLimiter: counter failed for ingestion key ${data.ingestionKeyId.toString()}, admitting request unthrottled`,
+        );
+        logger.warn(err);
+      }
+
+      return { outcome: TelemetryIngestionKeyLimitOutcome.CounterUnavailable };
+    }
+  }
+
+  /*
+   * Remaining whole seconds in the current fixed window, floored at 1.
+   *
+   * Every rejected client is told to come back exactly when the window rolls
+   * rather than all backing off by the same constant - a constant makes every
+   * refused caller retry in lockstep and re-collide, which is a
+   * self-inflicted thundering herd on top of the burst being limited.
+   */
+  private static getSecondsUntilWindowEnd(): number {
+    const msIntoWindow: number = Date.now() % WINDOW_MS;
+
+    return Math.max(1, Math.ceil((WINDOW_MS - msIntoWindow) / 1000));
+  }
+
+  private static counterUnavailableLastLoggedAt: number = 0;
+
+  private static shouldLogCounterUnavailable(): boolean {
+    const now: number = Date.now();
+
+    if (
+      now - TelemetryIngestionKeyRateLimiter.counterUnavailableLastLoggedAt <
+      COUNTER_UNAVAILABLE_LOG_INTERVAL_MS
+    ) {
+      return false;
+    }
+
+    TelemetryIngestionKeyRateLimiter.counterUnavailableLastLoggedAt = now;
+
+    return true;
+  }
+}

package/Server/Utils/VM/VMAPI.ts

@@ -13,6 +13,7 @@ export default class VMUtil {
       timeout?: number;
       // See VMRunner.runCodeInSandbox — decided by the caller, not here.
       allowPrivateNetworkRequests?: boolean | undefined;
+      privateNetworkAccessIsAllowed?: boolean | undefined;
       privateNetworkHint?: string | undefined;
     };
   }): Promise<ReturnResult> {