@oneuptime/common 12.0.33 → 12.0.34

package/Utils/API.ts

@@ -10,6 +10,7 @@ import Route from "../Types/API/Route";
 import URL from "../Types/API/URL";
 import Dictionary from "../Types/Dictionary";
 import APIException from "../Types/Exception/ApiException";
+import BadDataException from "../Types/Exception/BadDataException";
 import { JSONArray, JSONObject } from "../Types/JSON";
 import RequestFailedDetails, {
   RequestFailedPhase,
@@ -21,6 +22,9 @@ import axios, {
   AxiosResponse,
 } from "axios";
 import Sleep from "../Types/Sleep";
+import HTTPResponseBodyReader, {
+  HTTPResponseBodyBudget,
+} from "./HTTPResponseBodyReader";
 import type { Agent as HttpAgent } from "http";
 import type { Agent as HttpsAgent } from "https";
 
@@ -75,6 +79,33 @@ export interface RequestOptions {
   totalTimeoutInMs?: number | undefined;
   timeout?: number | undefined;
   doNotFollowRedirects?: boolean | undefined;
+  /*
+   * Explicit byte ceilings for callers that return tenant-selected remote
+   * content. Axios otherwise accepts an unlimited response/request body.
+   */
+  maxContentLength?: number | undefined;
+  maxBodyLength?: number | undefined;
+  /*
+   * Axios reads HTTP(S)_PROXY from the process environment on its own. A
+   * caller that supplies a destination-pinned agent must be able to turn that
+   * implicit second routing layer off, or the proxy can resolve a different
+   * address from the one the caller approved.
+   */
+  disableProxy?: boolean | undefined;
+  /* Abort active sockets when a caller-owned end-to-end deadline expires. */
+  signal?: AbortSignal | undefined;
+  /*
+   * Opt-in streaming mode used by monitor requests. All responses that share
+   * this object draw from one decompressed-byte budget.
+   */
+  responseBodyBudget?: HTTPResponseBodyBudget | undefined;
+  limitRedirectResponseBody?: boolean | undefined;
+  maximumResponseBytes?: number | undefined;
+  /**
+   * @internal Already-validated wire URL used by guarded callers when the
+   * reporting URL type cannot represent an href without normalization.
+   */
+  dispatchUrl?: string | undefined;
   // Per-request proxy agent support (Probe supplies these instead of mutating global axios defaults)
   httpAgent?: HttpAgent | undefined;
   httpsAgent?: HttpsAgent | undefined;
@@ -474,7 +505,7 @@ export default class API {
       // Identical on every attempt, so build it once.
       const axiosOptions: AxiosRequestConfig = {
         method: method,
-        url: url.toString(),
+        url: options?.dispatchUrl ?? url.toString(),
         headers: finalHeaders,
         data: finalBody,
       };
@@ -487,6 +518,35 @@ export default class API {
         axiosOptions.maxRedirects = 0;
       }
 
+      if (options?.maxContentLength !== undefined) {
+        axiosOptions.maxContentLength = options.maxContentLength;
+      }
+
+      if (options?.maxBodyLength !== undefined) {
+        axiosOptions.maxBodyLength = options.maxBodyLength;
+      }
+
+      if (options?.disableProxy) {
+        axiosOptions.proxy = false;
+      }
+
+      if (options?.signal) {
+        axiosOptions.signal = options.signal;
+      }
+
+      if (options?.responseBodyBudget) {
+        axiosOptions.responseType = "stream";
+        /*
+         * Axios's Node adapter enforces maxContentLength in an async
+         * generator before yielding the chunk that crosses the limit. That
+         * would hide already-received bytes from the shared budget and let a
+         * retry reclaim them. The streamed reader below is the sole response
+         * cap in this mode; it charges every decompressed chunk before
+         * rejecting and destroys the stream immediately.
+         */
+        axiosOptions.maxContentLength = -1;
+      }
+
       // Attach proxy agents per request if provided (avoids global side-effects)
       if (options?.httpAgent) {
         axiosOptions.httpAgent = options.httpAgent;
@@ -505,10 +565,40 @@ export default class API {
         attempts++;
 
         try {
-          result = await axios(axiosOptions);
+          const axiosResult: AxiosResponse = await axios(axiosOptions);
+
+          if (options?.responseBodyBudget) {
+            axiosResult.data = await API.readBudgetedResponseBody(
+              axiosResult,
+              options.responseBodyBudget,
+              options.limitRedirectResponseBody || false,
+              method === HTTPMethod.HEAD,
+              options.maximumResponseBytes,
+            );
+          }
+
+          result = axiosResult;
 
           break;
         } catch (e) {
+          if (e instanceof BadDataException) {
+            throw e;
+          }
+
+          if (
+            options?.responseBodyBudget &&
+            axios.isAxiosError(e) &&
+            e.response
+          ) {
+            e.response.data = await API.readBudgetedResponseBody(
+              e.response,
+              options.responseBodyBudget,
+              options.limitRedirectResponseBody || false,
+              method === HTTPMethod.HEAD,
+              options.maximumResponseBytes,
+            );
+          }
+
           if (attempt >= maxRetries) {
             throw e;
           }
@@ -590,6 +680,14 @@ export default class API {
       };
 
       if (!axios.isAxiosError(error)) {
+        if (error instanceof BadDataException) {
+          this.notifyRequestComplete(options, {
+            ...baseOutcome,
+            error: error,
+          });
+          throw error;
+        }
+
         const apiException: APIException = new APIException(error.message);
 
         this.notifyRequestComplete(options, {
@@ -625,7 +723,7 @@ export default class API {
         !options?.skipAuthRefresh &&
         !options?.hasAttemptedAuthRefresh
       ) {
-        const retryUrl: URL = URL.fromString(url.toString());
+        const retryUrl: URL = URL.fromStringLenient(url.toString());
 
         const requestContext: AuthRetryContext["request"] = {
           method,
@@ -682,6 +780,34 @@ export default class API {
     }
   }
 
+  private static async readBudgetedResponseBody(
+    response: AxiosResponse,
+    budget: HTTPResponseBodyBudget,
+    limitRedirectResponseBody: boolean,
+    isHeadResponse: boolean,
+    maximumResponseBytes?: number | undefined,
+  ): Promise<unknown> {
+    const body: Buffer = await HTTPResponseBodyReader.read(response.data, {
+      budget: budget,
+      statusCode: response.status,
+      headers: response.headers,
+      limitRedirectResponseBody: limitRedirectResponseBody,
+      isHeadResponse: isHeadResponse,
+      maximumResponseBytes: maximumResponseBytes,
+    });
+    const text: string = HTTPResponseBodyReader.decodeUtf8(body);
+
+    if (!text) {
+      return "";
+    }
+
+    try {
+      return JSON.parse(text) as JSONObject | JSONArray;
+    } catch {
+      return text;
+    }
+  }
+
   private static notifyRequestComplete(
     options: RequestOptions | undefined,
     outcome: RequestOutcome,

package/Utils/HTTPResponseBodyReader.ts

@@ -0,0 +1,221 @@
+import BadDataException from "../Types/Exception/BadDataException";
+
+interface StreamedResponseBody extends AsyncIterable<unknown> {
+  destroy?: (() => unknown) | undefined;
+}
+
+const DEFAULT_MAX_REDIRECT_RESPONSE_BYTES: number = 64 * 1024;
+
+export class HTTPResponseBodyBudget {
+  private remainingByteCount: number;
+  public readonly maxRedirectResponseBytes: number;
+
+  public constructor(
+    maximumBytes: number,
+    maxRedirectResponseBytes: number = DEFAULT_MAX_REDIRECT_RESPONSE_BYTES,
+  ) {
+    this.remainingByteCount = this.normalizeMaximumBytes(maximumBytes);
+    this.maxRedirectResponseBytes = this.normalizeMaximumBytes(
+      maxRedirectResponseBytes,
+    );
+  }
+
+  public get remainingBytes(): number {
+    return this.remainingByteCount;
+  }
+
+  public getResponseLimit(
+    statusCode: number,
+    limitRedirectResponseBody: boolean,
+  ): number {
+    if (this.remainingByteCount <= 0) {
+      throw new BadDataException(
+        "Remote response exceeded the allowed cumulative size.",
+      );
+    }
+
+    if (
+      limitRedirectResponseBody &&
+      [301, 302, 303, 307, 308].includes(statusCode)
+    ) {
+      return Math.min(this.remainingByteCount, this.maxRedirectResponseBytes);
+    }
+
+    return this.remainingByteCount;
+  }
+
+  public consume(byteCount: number): void {
+    if (
+      !Number.isFinite(byteCount) ||
+      byteCount < 0 ||
+      byteCount > this.remainingByteCount
+    ) {
+      throw new BadDataException(
+        "Remote response exceeded the allowed cumulative size.",
+      );
+    }
+
+    this.remainingByteCount -= byteCount;
+  }
+
+  private normalizeMaximumBytes(value: number): number {
+    if (!Number.isFinite(value)) {
+      return 0;
+    }
+
+    return Math.max(0, Math.floor(value));
+  }
+}
+
+export default class HTTPResponseBodyReader {
+  /*
+   * Axios removes a leading UTF-8 BOM before returning text or parsing JSON.
+   * Bounded callers request a raw stream so they can enforce a cumulative
+   * byte budget themselves; keep Axios-compatible decoding in one shared
+   * place for every streamed response path.
+   */
+  public static decodeUtf8(body: Buffer): string {
+    const text: string = body.toString("utf8");
+    return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text;
+  }
+
+  public static async read(
+    responseBody: unknown,
+    options: {
+      budget: HTTPResponseBodyBudget;
+      statusCode: number;
+      headers?: unknown;
+      limitRedirectResponseBody: boolean;
+      isHeadResponse?: boolean | undefined;
+      maximumResponseBytes?: number | undefined;
+    },
+  ): Promise<Buffer> {
+    if (!this.isStreamedResponseBody(responseBody)) {
+      throw new BadDataException("Remote response body could not be read.");
+    }
+
+    let limit: number;
+    try {
+      limit = options.budget.getResponseLimit(
+        options.statusCode,
+        options.limitRedirectResponseBody,
+      );
+
+      if (options.maximumResponseBytes !== undefined) {
+        limit = Math.min(
+          limit,
+          this.normalizeMaximumBytes(options.maximumResponseBytes),
+        );
+      }
+    } catch (error) {
+      responseBody.destroy?.();
+      throw error;
+    }
+
+    const contentLength: number | null = this.getContentLength(options.headers);
+
+    /*
+     * HEAD and 304 Content-Length values describe a selected representation,
+     * not bytes carried by this response. A 204 cannot carry a body either.
+     * Still stream-count any non-compliant bytes actually sent for all three.
+     *
+     * Keep this utility browser-bundle-safe: API.ts is shared by the web
+     * frontends, so the streamed body is deliberately duck-typed instead of
+     * importing Node's `stream` module at runtime.
+     */
+    const responseHasNoBody: boolean =
+      Boolean(options.isHeadResponse) ||
+      options.statusCode === 204 ||
+      options.statusCode === 304;
+
+    if (!responseHasNoBody && contentLength !== null && contentLength > limit) {
+      responseBody.destroy?.();
+      throw new BadDataException("Remote response exceeded the allowed size.");
+    }
+
+    const chunks: Array<Buffer> = [];
+    let byteCount: number = 0;
+
+    for await (const rawChunk of responseBody) {
+      const chunk: Buffer = Buffer.isBuffer(rawChunk)
+        ? rawChunk
+        : Buffer.from(rawChunk as Uint8Array);
+      byteCount += chunk.length;
+
+      if (byteCount > limit) {
+        /*
+         * The bytes have already crossed the socket boundary, so charge them
+         * even though this response is rejected. Otherwise a peer can send a
+         * near-limit partial response, reset the stream, and reclaim the
+         * entire cumulative budget on every monitor retry.
+         */
+        options.budget.consume(
+          Math.min(chunk.length, options.budget.remainingBytes),
+        );
+        responseBody.destroy?.();
+        throw new BadDataException(
+          "Remote response exceeded the allowed size.",
+        );
+      }
+
+      options.budget.consume(chunk.length);
+      chunks.push(chunk);
+    }
+
+    return Buffer.concat(chunks, byteCount);
+  }
+
+  private static normalizeMaximumBytes(value: number): number {
+    if (!Number.isFinite(value)) {
+      return 0;
+    }
+
+    return Math.max(0, Math.floor(value));
+  }
+
+  private static isStreamedResponseBody(
+    responseBody: unknown,
+  ): responseBody is StreamedResponseBody {
+    return Boolean(
+      responseBody &&
+        typeof responseBody === "object" &&
+        typeof (responseBody as Partial<AsyncIterable<unknown>>)[
+          Symbol.asyncIterator
+        ] === "function",
+    );
+  }
+
+  private static getContentLength(headers: unknown): number | null {
+    if (!headers || typeof headers !== "object") {
+      return null;
+    }
+
+    const getHeader: unknown = (headers as { get?: unknown }).get;
+    let value: unknown;
+
+    if (typeof getHeader === "function") {
+      value = (getHeader as (name: string) => unknown).call(
+        headers,
+        "content-length",
+      );
+    } else {
+      const headerRecord: Record<string, unknown> = headers as Record<
+        string,
+        unknown
+      >;
+      const name: string | undefined = Object.keys(headerRecord).find(
+        (headerName: string) => {
+          return headerName.toLowerCase() === "content-length";
+        },
+      );
+      value = name ? headerRecord[name] : undefined;
+    }
+
+    if (Array.isArray(value)) {
+      value = value[0];
+    }
+
+    const parsed: number = Number(value);
+    return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
+  }
+}

package/Utils/NetworkDiscovery/DiscoveryScanStatus.ts

@@ -0,0 +1,77 @@
+/*
+ * The lifecycle of one network discovery scan, as the four strings its
+ * `status` column actually holds.
+ *
+ * The column is a ShortText rather than an enum column, and the four values
+ * were written as bare literals in a dozen places: the probe-ingest claim and
+ * result endpoints, the requeue/reaper worker, the auto-import worker and its
+ * engine, the scan service's retire payload, and the Discovery page. That is
+ * fine until a question has to be answered the same way in several of them —
+ * and "which statuses may auto-import read from" is exactly such a question
+ * (OneUptime issue #3599), because the answer has to match in the worker's
+ * query, in the engine's re-read, and in the compare-and-set that stamps the
+ * scan afterwards. Three literals that agree today are three literals that
+ * can disagree tomorrow.
+ *
+ * Values are NOT a TypeScript enum: they are already persisted strings, and a
+ * plain object keeps them readable in a query, in jsonb and in a log line.
+ */
+export const DiscoveryScanStatus: {
+  Pending: string;
+  InProgress: string;
+  Completed: string;
+  Failed: string;
+} = {
+  // Queued, waiting for its probe to claim it.
+  Pending: "Pending",
+  /*
+   * Claimed by a probe and being swept RIGHT NOW. Since sweeps report
+   * incrementally, such a scan can already hold real results — see
+   * DISCOVERY_SCAN_IMPORTABLE_STATUSES.
+   */
+  InProgress: "In Progress",
+  // The sweep finished and reported. Its results are final for that run.
+  Completed: "Completed",
+  /*
+   * The sweep failed, or the server's reaper gave up on the probe. Any
+   * results on the row are from before the failure and are real, but partial
+   * in a way nobody chose.
+   */
+  Failed: "Failed",
+};
+
+/*
+ * The statuses whose stored results are worth importing from automatically.
+ *
+ * "In Progress" belongs here because a discovery sweep is no longer atomic:
+ * the probe uploads what it has found every 30 seconds while it is still
+ * working (Probe/Jobs/Discovery/FetchScans.ts). Before that, auto-import
+ * waited for the whole range — so on a 15,360-address scan the hundreds of
+ * switches already found, stored and visible in the product were unimportable
+ * for as long as the sweep ran (OneUptime issue #3599, compounded by the
+ * 24-hour sweep of #3598).
+ *
+ * "Pending" and "Failed" are deliberately absent, and for opposite reasons. A
+ * Pending row's results describe a run that has been superseded — the ingest
+ * endpoint discards results for one on exactly that ground. A Failed row's
+ * results are real, but the operator has been told that run failed, so
+ * importing from it should be their decision, through the rule's Run Now.
+ *
+ * Frozen because it is read by a query builder, a re-read and a
+ * compare-and-set that must all mean the same thing.
+ */
+export const DISCOVERY_SCAN_IMPORTABLE_STATUSES: Array<string> = Object.freeze([
+  DiscoveryScanStatus.Completed,
+  DiscoveryScanStatus.InProgress,
+]) as Array<string>;
+
+/*
+ * True when the scan is mid-sweep, so anything on its row is a running total
+ * rather than a verdict. Nullish-safe: a row read without the column, or one
+ * from a shape this code does not control, is not "in progress".
+ */
+export function isDiscoveryScanInProgress(
+  scan: { status?: string | undefined | null } | null | undefined,
+): boolean {
+  return scan?.status === DiscoveryScanStatus.InProgress;
+}

package/Utils/Rum/ChunkMath.ts

@@ -208,4 +208,110 @@ export default class ChunkMath {
       0,
     );
   }
+
+  /*
+   * Whether a manifest row is a terminator rather than footage.
+   *
+   * The recorder closes a tab with an empty chunk (payload "[]",
+   * eventCount 0) and emits the same shape when a session hits its chunk
+   * cap. The ingest stores them because they carry the end time, but they
+   * hold nothing to feed a Replayer: treating one as a chunk that "failed
+   * to decode" ends most sessions with an error banner instead of
+   * "recording ended".
+   */
+  public static isTerminatorEntry(
+    entry: SessionReplayChunkManifestEntry,
+  ): boolean {
+    return !(entry.eventCount > 0);
+  }
+
+  /*
+   * The smallest anchor strictly after a chunk index, or null. Used when a
+   * chunk fails to decode: the only safe place to resume is a later full
+   * snapshot, never the next present chunk.
+   */
+  public static findNextAnchorAfter(
+    anchors: Array<number>,
+    afterChunkIndex: number,
+  ): number | null {
+    let best: number | null = null;
+
+    for (const anchor of anchors) {
+      if (anchor > afterChunkIndex && (best === null || anchor < best)) {
+        best = anchor;
+      }
+    }
+
+    return best;
+  }
+
+  /*
+   * True when every index in [fromChunkIndex, toChunkIndex] is present, so
+   * the range can be fed into a live Replayer in order without crossing a
+   * hole. A seek that lands ahead of the fed range but inside a contiguous
+   * run feeds forward instead of tearing the Replayer down.
+   */
+  public static isContiguousRange(
+    presentIndexes: ReadonlySet<number>,
+    fromChunkIndex: number,
+    toChunkIndex: number,
+  ): boolean {
+    if (toChunkIndex < fromChunkIndex) {
+      return false;
+    }
+
+    for (let index: number = fromChunkIndex; index <= toChunkIndex; index++) {
+      if (!presentIndexes.has(index)) {
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  /*
+   * The two-chunk priority page for a first paint: the anchor and one
+   * neighbour. The neighbour after it is preferred because playback moves
+   * forward; the one before it is the fallback so a lone full snapshot
+   * (the oversized-snapshot layout) still gets rrweb its two events and,
+   * more importantly, the Meta event that lives in the chunk before it.
+   */
+  public static planPriorityPage(
+    presentIndexes: ReadonlySet<number>,
+    anchorChunkIndex: number,
+  ): Array<number> {
+    if (!presentIndexes.has(anchorChunkIndex)) {
+      return [];
+    }
+
+    if (presentIndexes.has(anchorChunkIndex + 1)) {
+      return [anchorChunkIndex, anchorChunkIndex + 1];
+    }
+
+    if (presentIndexes.has(anchorChunkIndex - 1)) {
+      return [anchorChunkIndex - 1, anchorChunkIndex];
+    }
+
+    return [anchorChunkIndex];
+  }
+
+  /*
+   * Clamp a requested playhead into what the recording can show. A seek
+   * before the first playable moment lands ON that moment rather than
+   * being refused: refusing turns "press Play at the end" and every ?t=0
+   * link into a dead end whenever chunk 0 cannot anchor playback.
+   */
+  public static clampSeekOffset(
+    offsetMs: number,
+    earliestPlayableMs: number,
+    durationMs: number,
+  ): number {
+    if (!Number.isFinite(offsetMs)) {
+      return earliestPlayableMs;
+    }
+
+    const upper: number = Math.max(earliestPlayableMs, durationMs);
+
+    return Math.min(upper, Math.max(earliestPlayableMs, offsetMs));
+  }
 }