@oneuptime/common 12.0.32 → 12.0.34

package/Server/Utils/Telemetry/PinServiceName.ts

@@ -0,0 +1,197 @@
+import { JSONObject, JSONValue } from "../../../Types/JSON";
+
+/*
+ * Every top-level OTLP container that carries a `resource` block. Each is an
+ * array of { resource, scopeSpans|scopeLogs|scopeMetrics|scopeProfiles }
+ * envelopes.
+ *
+ * Profiles are in the list even though a Browser ingestion key cannot reach
+ * the profiles surface at all (see BROWSER_ALLOWED_INGEST_SURFACES). Pinning
+ * is not a browser-only feature: a Server key may set a pinned service name
+ * too, and the dashboard field promises it applies to "everything the key
+ * writes". Leaving one signal out would make that promise false in exactly
+ * the quiet way nobody notices until they are reading a chart. The
+ * containment argument for browser keys is unaffected either way.
+ */
+const RESOURCE_CONTAINER_KEYS: ReadonlyArray<string> = [
+  "resourceSpans",
+  "resourceLogs",
+  "resourceMetrics",
+  "resourceProfiles",
+];
+
+const SERVICE_NAME_ATTRIBUTE_KEY: string = "service.name";
+
+/*
+ * Rewrites `service.name` on every OTLP resource in a payload so telemetry
+ * written with a given ingestion key can only ever land under the service that
+ * key is pinned to.
+ *
+ * WHY THIS EXISTS: a Browser ingestion key is published in page source by
+ * design - that is the whole point of the key class - so it must be assumed
+ * scraped. Rate limits and the origin allowlist bound HOW MUCH forged data an
+ * attacker can write and FROM WHERE, but neither stops them writing it under
+ * `service.name: "payments-api"` and poisoning the dashboards, SLOs and alert
+ * rules of a backend service that never emitted a byte of it. Pinning is the
+ * control that makes the forged data self-identifying: it can only ever appear
+ * as the browser service the customer named.
+ *
+ * WHY IT MUTATES IN PLACE: this runs on the ingest path over payloads that are
+ * routinely megabytes of spans. Deep-cloning them to return a rewritten copy
+ * would double peak heap on the hot path for no benefit - the caller owns the
+ * body and has no use for the original.
+ *
+ * WHY NOTHING IN HERE THROWS: the input is attacker-controlled JSON that has
+ * NOT been validated against the OTLP schema at this point, and this runs
+ * inside a queue worker. A TypeError on a hostile shape would fail the whole
+ * batch (and, worse, would be trivially reachable by anyone holding a scraped
+ * key). Every shape check below therefore skips or repairs rather than raises.
+ */
+export default class PinServiceName {
+  /*
+   * Pin `serviceName` onto every resource block in `body`, in place.
+   *
+   * Returns the number of resource blocks rewritten, which the caller can log
+   * or use to tell "we pinned nothing because the payload was empty" apart
+   * from "we pinned nothing because the payload was malformed".
+   */
+  public static pinInPlace(body: JSONObject, serviceName: string): number {
+    if (!body || typeof body !== "object" || Array.isArray(body)) {
+      return 0;
+    }
+
+    /*
+     * An empty pin would stamp `service.name: ""` on everything, which is
+     * strictly worse than not pinning: it destroys the customer's own service
+     * attribution AND gives the attacker's data a shape indistinguishable
+     * from it. Refuse to do that; the caller's policy layer is responsible for
+     * not configuring a blank pin in the first place.
+     */
+    const pinnedServiceName: string =
+      typeof serviceName === "string" ? serviceName.trim() : "";
+
+    if (!pinnedServiceName) {
+      return 0;
+    }
+
+    let rewrittenCount: number = 0;
+
+    for (const containerKey of RESOURCE_CONTAINER_KEYS) {
+      const container: JSONValue | undefined = body[containerKey];
+
+      if (!Array.isArray(container)) {
+        continue;
+      }
+
+      for (const resourceBlock of container) {
+        if (this.pinResourceBlock(resourceBlock, pinnedServiceName)) {
+          rewrittenCount++;
+        }
+      }
+    }
+
+    return rewrittenCount;
+  }
+
+  /*
+   * Pin one { resource, scope* } envelope. Returns whether it was rewritten.
+   *
+   * A malformed `resource` or `attributes` is REPLACED rather than skipped.
+   * That looks aggressive, but consider the alternative: a forged payload that
+   * sends `"resource": "not-an-object"` would then pass through unpinned, and
+   * the downstream decoder - which reads `resource?.attributes` defensively -
+   * would happily ingest it with whatever service attribution it falls back
+   * to. Skipping would hand the attacker a one-character bypass of the entire
+   * control. The only thing we cannot repair is an envelope that is not an
+   * object at all, because there is nowhere to write.
+   */
+  private static pinResourceBlock(
+    resourceBlock: JSONValue | undefined,
+    serviceName: string,
+  ): boolean {
+    if (
+      !resourceBlock ||
+      typeof resourceBlock !== "object" ||
+      Array.isArray(resourceBlock)
+    ) {
+      return false;
+    }
+
+    const block: JSONObject = resourceBlock as JSONObject;
+
+    const existingResource: JSONValue | undefined = block["resource"];
+
+    let resource: JSONObject;
+
+    if (
+      existingResource &&
+      typeof existingResource === "object" &&
+      !Array.isArray(existingResource)
+    ) {
+      resource = existingResource as JSONObject;
+    } else {
+      resource = {};
+      block["resource"] = resource;
+    }
+
+    const existingAttributes: JSONValue | undefined = resource["attributes"];
+
+    const attributes: Array<JSONValue> = [];
+
+    if (Array.isArray(existingAttributes)) {
+      for (const attribute of existingAttributes) {
+        /*
+         * Keep every attribute that is not a service.name in its original
+         * relative order, including entries we cannot parse - dropping those
+         * would be an unrelated, invisible edit to the customer's data.
+         */
+        if (!this.isServiceNameAttribute(attribute)) {
+          attributes.push(attribute as JSONValue);
+        }
+      }
+    }
+
+    /*
+     * Appended last, and exactly once, so there is no ambiguity for a
+     * consumer that takes either the first or the last match.
+     */
+    attributes.push({
+      key: SERVICE_NAME_ATTRIBUTE_KEY,
+      value: {
+        stringValue: serviceName,
+      },
+    } as JSONValue);
+
+    resource["attributes"] = attributes as JSONValue;
+
+    return true;
+  }
+
+  /*
+   * Matched case-insensitively and after trimming, which is deliberately
+   * looser than the OTLP spec (attribute keys are case-sensitive there).
+   *
+   * Rationale: the ONLY consequence of over-matching here is that an exotic
+   * key like "Service.Name" gets dropped, and no consumer in this codebase
+   * reads that. The consequence of under-matching is a bypass, if any
+   * consumer anywhere - now or later, ours or a customer's export - folds
+   * case before looking up the service. Defence in depth wins that trade.
+   */
+  private static isServiceNameAttribute(attribute: JSONValue): boolean {
+    if (
+      !attribute ||
+      typeof attribute !== "object" ||
+      Array.isArray(attribute)
+    ) {
+      return false;
+    }
+
+    const key: JSONValue | undefined = (attribute as JSONObject)["key"];
+
+    if (typeof key !== "string") {
+      return false;
+    }
+
+    return key.trim().toLowerCase() === SERVICE_NAME_ATTRIBUTE_KEY;
+  }
+}

package/Server/Utils/Telemetry/SpanUtil.ts

@@ -1,6 +1,7 @@
 import { DisableTelemetry } from "../../EnvironmentConfig";
 import TelemetryContext from "./TelemetryContext";
 import OpenTelemetryAPI, { Span } from "@opentelemetry/api";
+import Telemetry from "../Telemetry";
 
 export interface SpanAttributes {
   userId?: string | undefined;
@@ -51,6 +52,38 @@ export default class SpanUtil {
     this.addAttributesToSpan({ span, attributes });
   }
 
+  /**
+   * Record a thrown value on the currently active span, without ending it.
+   *
+   * For code that CATCHES an error and signals failure by another route —
+   * Express middleware calling `next(err)` being the canonical case. The
+   * @CaptureSpan frame around such a method sees a normal return, so its own
+   * recorder never runs and the error would otherwise be invisible on that
+   * span.
+   *
+   * Always prefer this to calling `span.recordException(err)` directly: the
+   * OTel SDK types the event from `exception.code` before `exception.name`,
+   * and every OneUptime ExceptionCode is an HTTP status, so the raw call
+   * produces exception groups literally named "400" and "422". This path also
+   * applies fault classification, so a rejected API key produces a `fault`
+   * event rather than an Issue.
+   *
+   * Safe to call with no active span or with telemetry disabled.
+   */
+  public static recordExceptionOnCurrentSpan(exception: unknown): void {
+    if (DisableTelemetry) {
+      return;
+    }
+
+    const span: Span | undefined = OpenTelemetryAPI.trace.getActiveSpan();
+
+    if (!span) {
+      return;
+    }
+
+    Telemetry.recordExceptionOnSpan({ span, exception });
+  }
+
   /**
    * Add attributes to a specific span.
    */

package/Server/Utils/Telemetry/TelemetryFanInWriter.ts

@@ -125,7 +125,10 @@ export interface FanInWriterOptions {
   maxWaitMs: number;
   /** Per-pod cap on concurrent ClickHouse inserts across all tables. */
   maxConcurrentInserts: number;
-  /** High-water mark of buffered+in-flight rows; submit() blocks above it. */
+  /**
+   * High-water mark of reserved+buffered+in-flight rows. A whole submission
+   * may cross it once; later submit() calls wait for capacity to be released.
+   */
   maxPendingRows: number;
   retryMaxAttempts: number;
   retryBaseDelayMs: number;
@@ -197,6 +200,11 @@ type TableBuffer = {
   timer: NodeJS.Timeout | null;
 };
 
+type CapacityWaiter = {
+  rowCount: number;
+  resolve: () => void;
+};
+
 /*
  * ClickHouse server error codes that indicate transient overload or an
  * ambiguous outcome. All are safe to retry here because every insert
@@ -328,7 +336,7 @@ export class TelemetryFanInWriter {
   private options: FanInWriterOptions;
   private buffers: Map<string, TableBuffer> = new Map();
   private pendingRows: number = 0;
-  private capacityWaiters: Array<() => void> = [];
+  private capacityWaiters: Set<CapacityWaiter> = new Set();
   private activeInserts: number = 0;
   private insertSlotWaiters: Array<() => void> = [];
   private inFlightDispatches: Set<Promise<void>> = new Set();
@@ -428,7 +436,7 @@ export class TelemetryFanInWriter {
 
     this.acceptingSubmits++;
     try {
-      await this.waitForCapacity();
+      await this.waitForCapacity(rows.length);
 
       let buffer: TableBuffer | undefined = this.buffers.get(tableName);
       if (!buffer) {
@@ -458,7 +466,6 @@ export class TelemetryFanInWriter {
         rejectAck,
       });
       buffer.rowCount += rows.length;
-      this.pendingRows += rows.length;
 
       if (buffer.rowCount >= this.getMaxBatchRows(tableName)) {
         this.cutAndDispatch(tableName, false);
@@ -532,24 +539,39 @@ export class TelemetryFanInWriter {
     );
   }
 
-  private async waitForCapacity(): Promise<void> {
-    while (this.pendingRows >= this.options.maxPendingRows) {
-      await new Promise<void>((resolve: () => void) => {
-        this.capacityWaiters.push(resolve);
-      });
+  private async waitForCapacity(rowCount: number): Promise<void> {
+    /*
+     * Reserve before yielding: concurrent submitters otherwise all observe
+     * the same available capacity and buffer their rows after the await,
+     * multiplying the memory limit by the number of submitters. Keep whole
+     * submissions intact: one submission may cross the high-water mark,
+     * but every later submission waits until pending rows fall below it.
+     */
+    if (this.pendingRows < this.options.maxPendingRows) {
+      this.pendingRows += rowCount;
+      return;
     }
+
+    await new Promise<void>((resolve: () => void) => {
+      this.capacityWaiters.add({ rowCount, resolve });
+    });
   }
 
   private releaseCapacity(): void {
     /*
-     * Wake all waiters; each re-checks the high-water mark in its
-     * waitForCapacity loop, so overshoot stays bounded to one submission
-     * per waiter.
+     * Transfer capacity to FIFO waiters before waking them. Only admitted
+     * callers resume, so completing an insert does not wake the entire
+     * backlog to compete for the same capacity again.
+     * Set keeps FIFO order and releases each waiter without shifting the
+     * remaining queue or retaining consumed entries.
      */
-    const waiters: Array<() => void> = this.capacityWaiters;
-    this.capacityWaiters = [];
-    for (const wake of waiters) {
-      wake();
+    for (const waiter of this.capacityWaiters) {
+      if (this.pendingRows >= this.options.maxPendingRows) {
+        break;
+      }
+      this.pendingRows += waiter.rowCount;
+      this.capacityWaiters.delete(waiter);
+      waiter.resolve();
     }
   }
 

package/Server/Utils/Telemetry/TelemetryIngestionKeyGuard.ts

@@ -0,0 +1,92 @@
+import TelemetryIngestionKeyPolicy from "../../../Types/Telemetry/TelemetryIngestionKeyPolicy";
+import TelemetryIngestionKeyType from "../../../Types/Telemetry/TelemetryIngestionKeyType";
+import TelemetryIngestSurface, {
+  BROWSER_ALLOWED_INGEST_SURFACES,
+  getIngestSurfaceReadableName,
+} from "../../../Types/Telemetry/TelemetryIngestSurface";
+
+/*
+ * The three key-state checks that every ingest entry point has to make, in
+ * one place.
+ *
+ * TelemetryIngest (the Express middleware) runs these inline as part of a
+ * longer chain that also covers the Origin allowlist and the per-key rate
+ * limit, neither of which means anything off HTTP. Everything else that
+ * accepts a TelemetryIngestionKey - the gRPC OTLP server, the MQTT broker,
+ * and the two key-validation probes - is NOT an Express route and cannot use
+ * that middleware, yet must answer these same three questions or a disabled
+ * key keeps working on the pipe nobody remembered to update.
+ *
+ * The refusal SENTENCES are deliberately identical to the middleware's, so a
+ * customer who moves the same key between HTTP, gRPC and MQTT reads the same
+ * explanation each time rather than three differently-worded versions of one
+ * problem. If you change wording here, change it there too.
+ *
+ * What this deliberately does NOT do is decide the transport-level response.
+ * A refusal is a status code on HTTP, a CONNACK return code on MQTT, and a
+ * silent success on gRPC (see GrpcServer for why); each caller owns that
+ * translation because each transport's retry semantics are its own.
+ */
+export enum TelemetryIngestionKeyRefusalReason {
+  Disabled = "disabled",
+  Expired = "expired",
+  SurfaceNotAllowedForBrowserKey = "surface-not-allowed-for-browser-key",
+}
+
+export interface TelemetryIngestionKeyRefusal {
+  reason: TelemetryIngestionKeyRefusalReason;
+
+  /*
+   * Safe to return to whoever presented the key. It names the problem and
+   * never the key, the project, the allowlist or the expiry timestamp - a
+   * caller holding a dead credential is not entitled to a project's
+   * configuration.
+   */
+  message: string;
+}
+
+export default class TelemetryIngestionKeyGuard {
+  /**
+   * Why this key may not write to this surface, or null when it may.
+   *
+   * Order matters and mirrors the middleware: the kill switch is checked
+   * first so that switching a leaked key off is the one action guaranteed to
+   * stop it whatever else is true of the key, then expiry, then the
+   * browser-key surface rule.
+   */
+  public static getRefusal(data: {
+    policy: TelemetryIngestionKeyPolicy;
+    surface: TelemetryIngestSurface;
+  }): TelemetryIngestionKeyRefusal | null {
+    const policy: TelemetryIngestionKeyPolicy = data.policy;
+
+    if (policy.isEnabled === false) {
+      return {
+        reason: TelemetryIngestionKeyRefusalReason.Disabled,
+        message: "This telemetry ingestion key has been disabled.",
+      };
+    }
+
+    if (policy.expiresAt && policy.expiresAt.getTime() <= Date.now()) {
+      return {
+        reason: TelemetryIngestionKeyRefusalReason.Expired,
+        message: "This telemetry ingestion key expired.",
+      };
+    }
+
+    if (
+      policy.keyType === TelemetryIngestionKeyType.Browser &&
+      !BROWSER_ALLOWED_INGEST_SURFACES.has(data.surface)
+    ) {
+      return {
+        reason:
+          TelemetryIngestionKeyRefusalReason.SurfaceNotAllowedForBrowserKey,
+        message: `A browser ingestion key cannot be used for ${getIngestSurfaceReadableName(
+          data.surface,
+        )}. Use a server ingestion key.`,
+      };
+    }
+
+    return null;
+  }
+}

package/Server/Utils/Telemetry/TelemetryIngestionKeyRateLimiter.ts

@@ -0,0 +1,217 @@
+import Redis, { ClientType } from "../../Infrastructure/Redis";
+import logger from "../Logger";
+import ObjectID from "../../../Types/ObjectID";
+
+/*
+ * Per-ingestion-key request rate limiting.
+ *
+ * This exists because a Browser ingestion key is, by design, published in a
+ * page: anyone who views source has it. Rotation and the origin allowlist
+ * bound WHO can use a scraped key, but neither bounds HOW MUCH a caller who
+ * satisfies them can push. Without a per-key ceiling, one scraped key plus a
+ * loop is unbounded write volume into a customer's project - poisoned
+ * dashboards, forged spans and, on a metered plan, a bill. The ceiling is
+ * per key rather than per project so that turning the abuse off (rotate or
+ * disable that one key) never takes a customer's server ingest down with it.
+ *
+ * Built on raw Redis INCR + EXPIRE, mirroring
+ * App/FeatureSet/Telemetry/Utils/SessionReplayRateLimiter.ts. The same
+ * reasoning applies: a per-process in-memory counter is useless across
+ * autoscaled ingest pods, and GlobalCache has no counter primitive at all
+ * (only get/set, so a read-modify-write there would lose increments under
+ * exactly the concurrency this is meant to bound).
+ *
+ * ---------------------------------------------------------------------
+ * THIS LIMITER FAILS OPEN, AND THAT IS DIFFERENT FROM SESSION REPLAY.
+ * ---------------------------------------------------------------------
+ * When Redis is unreachable this returns CounterUnavailable, and the CALLER
+ * is expected to ADMIT the request anyway. The limit here bounds ABUSE of a
+ * key that leaked; it is not a privacy control and it is not a hard billing
+ * quota. Weigh the two failure modes during a Redis blip:
+ *
+ *   - fail closed: every paying customer's production telemetry stops for
+ *     the duration of the outage, and the gap is permanent because dropped
+ *     spans are not replayed. An infrastructure hiccup becomes a
+ *     customer-visible data loss incident, and the operator's incident
+ *     dashboards go dark at the exact moment they are needed.
+ *
+ *   - fail open: for the length of the blip an attacker who already holds a
+ *     scraped key gets an unbounded window. They already had that window up
+ *     until this feature shipped, the origin allowlist and enable/disable
+ *     switch still apply (they are resolved from Postgres, not Redis), and
+ *     Redis being down is itself alarmed on.
+ *
+ * The second is plainly the better trade for telemetry. SessionReplayRate-
+ * Limiter fails CLOSED because a recording is a video of a real person and a
+ * budget that cannot be counted cannot be honoured - accepting an unbounded
+ * volume of that is not a recoverable mistake. Same machinery, opposite
+ * answer, because the thing being protected is different.
+ */
+
+const RATE_KEY_PREFIX: string = "telemetry:ingestkey:rate:";
+
+/*
+ * Two minutes, so the counter for a minute bucket outlives the bucket it
+ * belongs to. A request landing right on a boundary cannot then read a key
+ * that was already reclaimed mid-window and be handed a fresh allowance.
+ */
+const RATE_TTL_SECONDS: number = 120;
+
+const WINDOW_MS: number = 60000;
+
+/*
+ * A Redis outage puts EVERY request on the unavailable path, so an unguarded
+ * log line there is one line per ingested request for as long as the outage
+ * lasts - the ingest path is the highest-volume surface in the product, so
+ * that buries the very incident it is reporting. One line per interval is
+ * enough to make the condition visible. Same idea as the throttle in
+ * Common/Server/Middleware/PublicDashboardRateLimit.ts.
+ */
+const COUNTER_UNAVAILABLE_LOG_INTERVAL_MS: number = 60 * 1000;
+
+export enum TelemetryIngestionKeyLimitOutcome {
+  /* Within this key's per-minute allowance. */
+  Allowed = "allowed",
+
+  /* Over the allowance. Retryable when the fixed window rolls. */
+  RateLimited = "rate-limited",
+
+  /*
+   * Redis is unreachable, so no limit can be honoured either way. The caller
+   * admits the request - see the fail-open rationale at the top of the file.
+   */
+  CounterUnavailable = "counter-unavailable",
+}
+
+export interface TelemetryIngestionKeyLimitDecision {
+  outcome: TelemetryIngestionKeyLimitOutcome;
+  retryAfterSeconds?: number | undefined;
+
+  /*
+   * True only on the request that first crossed the line in this window.
+   *
+   * The whole point of a limiter is that the rejected caller keeps knocking,
+   * so a caller that logs every refusal turns an ingest flood into a second
+   * flood in the log pipeline. Logging only the crossing gives exactly one
+   * line per key per minute, which is what an operator actually wants when
+   * they are trying to see WHICH key is being abused. Unset unless rejected.
+   */
+  isFirstRejectionInWindow?: boolean | undefined;
+}
+
+export default class TelemetryIngestionKeyRateLimiter {
+  /*
+   * Consume one request from this key's per-minute allowance.
+   *
+   * Fixed window keyed on the minute bucket, deliberately not a sliding
+   * window: the sliding version needs a sorted set per key and a read of it
+   * per request, and this runs in front of every OTLP payload the product
+   * accepts. A fixed window admits at most 2x the limit across an unlucky
+   * boundary pair, which is an entirely acceptable overshoot for an
+   * abuse ceiling and costs one INCR.
+   *
+   * The counter is incremented on rejected requests too. That is standard
+   * fixed-window behaviour and it is the point: a client hammering a scraped
+   * key keeps its own window pinned rather than being handed a fresh
+   * allowance for free by virtue of having been refused.
+   */
+  public static async consume(data: {
+    ingestionKeyId: ObjectID;
+    limitPerMinute: number;
+  }): Promise<TelemetryIngestionKeyLimitDecision> {
+    /*
+     * A non-positive or non-finite limit means "no limit" rather than "block
+     * everything", and short-circuits before Redis is touched.
+     *
+     * The column behind this is nullable and the caller resolves NULL to the
+     * shipped browser default (or to unlimited for a Server key), so a zero
+     * or garbage value reaching here is a bug or a bad row, not a customer
+     * asking for a total block. Reading it as a block would silently black
+     * out that key's ingest, and this file's whole posture (above) is that a
+     * malfunction in the limiter must not stop real telemetry.
+     */
+    if (!Number.isFinite(data.limitPerMinute) || data.limitPerMinute <= 0) {
+      return { outcome: TelemetryIngestionKeyLimitOutcome.Allowed };
+    }
+
+    const client: ClientType | null = Redis.getClient();
+
+    if (!client || !Redis.isConnected()) {
+      return { outcome: TelemetryIngestionKeyLimitOutcome.CounterUnavailable };
+    }
+
+    const minuteBucket: number = Math.floor(Date.now() / WINDOW_MS);
+    const key: string = `${RATE_KEY_PREFIX}${data.ingestionKeyId.toString()}:${minuteBucket}`;
+
+    try {
+      const total: number = await client.incr(key);
+
+      /*
+       * Set the expiry only on the write that CREATED the key. Re-issuing
+       * EXPIRE on every increment would slide the window forward for as long
+       * as the load continues, so the counter would never reset and a key
+       * that tripped its limit once under sustained traffic could never
+       * recover - which is precisely the state an attacker's loop produces.
+       */
+      if (total === 1) {
+        await client.expire(key, RATE_TTL_SECONDS);
+      }
+
+      if (total > data.limitPerMinute) {
+        return {
+          outcome: TelemetryIngestionKeyLimitOutcome.RateLimited,
+          retryAfterSeconds:
+            TelemetryIngestionKeyRateLimiter.getSecondsUntilWindowEnd(),
+
+          /*
+           * total is monotonic within the window, so the request that took
+           * it to limit + 1 is the unique first rejection.
+           */
+          isFirstRejectionInWindow: total === data.limitPerMinute + 1,
+        };
+      }
+
+      return { outcome: TelemetryIngestionKeyLimitOutcome.Allowed };
+    } catch (err) {
+      if (TelemetryIngestionKeyRateLimiter.shouldLogCounterUnavailable()) {
+        logger.warn(
+          `TelemetryIngestionKeyRateLimiter: counter failed for ingestion key ${data.ingestionKeyId.toString()}, admitting request unthrottled`,
+        );
+        logger.warn(err);
+      }
+
+      return { outcome: TelemetryIngestionKeyLimitOutcome.CounterUnavailable };
+    }
+  }
+
+  /*
+   * Remaining whole seconds in the current fixed window, floored at 1.
+   *
+   * Every rejected client is told to come back exactly when the window rolls
+   * rather than all backing off by the same constant - a constant makes every
+   * refused caller retry in lockstep and re-collide, which is a
+   * self-inflicted thundering herd on top of the burst being limited.
+   */
+  private static getSecondsUntilWindowEnd(): number {
+    const msIntoWindow: number = Date.now() % WINDOW_MS;
+
+    return Math.max(1, Math.ceil((WINDOW_MS - msIntoWindow) / 1000));
+  }
+
+  private static counterUnavailableLastLoggedAt: number = 0;
+
+  private static shouldLogCounterUnavailable(): boolean {
+    const now: number = Date.now();
+
+    if (
+      now - TelemetryIngestionKeyRateLimiter.counterUnavailableLastLoggedAt <
+      COUNTER_UNAVAILABLE_LOG_INTERVAL_MS
+    ) {
+      return false;
+    }
+
+    TelemetryIngestionKeyRateLimiter.counterUnavailableLastLoggedAt = now;
+
+    return true;
+  }
+}