@oneuptime/common 12.0.22 → 12.0.24

package/Server/Middleware/GzipRequestBody.ts

@@ -0,0 +1,311 @@
+import ServerException from "../../Types/Exception/ServerException";
+import {
+  ExpressRequest,
+  ExpressResponse,
+  NextFunction,
+  OneUptimeRequest,
+  headerValueToString,
+} from "../Utils/Express";
+import logger, { getLogAttributesFromRequest } from "../Utils/Logger";
+import Response from "../Utils/Response";
+import zlib from "zlib";
+
+/*
+ * Ceilings for the global `Content-Encoding: gzip` request path.
+ *
+ * Both are 50 MiB, which is exactly what body-parser is given for the
+ * identity path (`jsonBodyParserOptions.limit` / `urlEncodedBodyParserOptions.limit`,
+ * and `bytes("50mb")` is 1024-based too). Announcing gzip must not buy a
+ * caller more room than sending the same body uncompressed.
+ *
+ * They are two constants because they bound two different things and
+ * neither alone is sufficient:
+ *
+ *   - COMPRESSED bounds what arrives on the socket, so a client cannot
+ *     stream unbounded bytes into the process.
+ *   - DECOMPRESSED bounds what the inflate produces. gzip of repeated
+ *     bytes reaches four figures of amplification: 130,479 bytes of gzip
+ *     inflates to 134,217,728 bytes (1,029x, measured), so the compressed
+ *     cap on its own leaves ~50 GB of expansion on the table.
+ */
+export const MAX_COMPRESSED_REQUEST_BODY_BYTES: number = 50 * 1024 * 1024;
+export const MAX_DECOMPRESSED_REQUEST_BODY_BYTES: number = 50 * 1024 * 1024;
+
+/*
+ * Inflate a `Content-Encoding: gzip` request body under hard limits, and
+ * hand the decompressed bytes on as `req.body` (a Buffer, which is what
+ * this path has always produced).
+ *
+ * This middleware runs BEFORE routing and therefore before authentication,
+ * so every limit here is an unauthenticated-attacker limit.
+ *
+ * Two implementation notes that are easy to get wrong:
+ *
+ *   1. It streams. Buffering the whole compressed body and then calling
+ *      `zlib.gunzip` would hold the compressed cap AND the decompressed cap
+ *      at once, and would not notice a bomb until the last byte arrived.
+ *      Feeding a `createGunzip` stream instead trips the cap after ~64 KiB
+ *      of a 128 MiB bomb (measured), before most of the body is even sent.
+ *
+ *   2. It counts the output itself. `maxOutputLength` looks like the right
+ *      knob but Node only honours it on the convenience methods
+ *      (`zlib.gunzip` / `gunzipSync`); on a `createGunzip` STREAM it is
+ *      silently ignored - a stream with `maxOutputLength: 1 MiB` happily
+ *      emits 128 MiB. Verified against Node 24. Setting it here would read
+ *      as protection while providing none, so the running total below is
+ *      what actually enforces the ceiling.
+ */
+export default class GzipRequestBodyMiddleware {
+  public static parseBody(
+    req: ExpressRequest,
+    res: ExpressResponse,
+    next: NextFunction,
+  ): void {
+    /*
+     * Cheap pre-check before a byte is read. Content-Length is advisory (a
+     * chunked request carries none) but when it is present and already over
+     * the cap there is no reason to accept the body at all.
+     */
+    const contentLengthHeader: string | undefined = headerValueToString(
+      req.headers["content-length"],
+    );
+
+    if (contentLengthHeader) {
+      const declaredLength: number = parseInt(contentLengthHeader, 10);
+
+      if (
+        !isNaN(declaredLength) &&
+        declaredLength > MAX_COMPRESSED_REQUEST_BODY_BYTES
+      ) {
+        GzipRequestBodyMiddleware.rejectTooLarge(
+          req,
+          res,
+          "compressed",
+          declaredLength,
+          MAX_COMPRESSED_REQUEST_BODY_BYTES,
+        );
+        return;
+      }
+    }
+
+    const inflate: zlib.Gunzip = zlib.createGunzip();
+
+    const decoded: Array<Buffer> = [];
+
+    let compressedBytes: number = 0;
+    let decompressedBytes: number = 0;
+    let requestEnded: boolean = false;
+    let awaitingDrain: boolean = false;
+    let settled: boolean = false;
+
+    const settle: (finish: () => void) => void = (finish: () => void): void => {
+      if (settled) {
+        return;
+      }
+
+      settled = true;
+
+      req.removeListener("data", onData);
+      req.removeListener("end", onEnd);
+      req.removeListener("error", onRequestError);
+      req.removeListener("close", onClose);
+
+      inflate.removeAllListeners();
+
+      /*
+       * A stream that emits "error" with no listener throws, and an
+       * uncaught throw here would turn a memory-exhaustion fix into a
+       * process-exit one. Nothing can consume a post-teardown error, so
+       * swallow it explicitly rather than leaving the stream bare.
+       */
+      inflate.on("error", (): void => {});
+      inflate.destroy();
+
+      decoded.length = 0;
+
+      finish();
+    };
+
+    /*
+     * Stop reading rather than draining. The session replay middleware
+     * deliberately drains its oversized bodies to keep the keep-alive
+     * connection usable for the next chunk, but that is a cooperative
+     * first-party client sending a 4 MiB cap's worth of bytes. Here the
+     * body we are refusing may be gigabytes from someone who wants us to
+     * read them, so we pause: Node flushes the response we already wrote
+     * and then closes the connection because the request was never
+     * consumed, which is precisely the outcome we want.
+     */
+    const stopReading: () => void = (): void => {
+      if (typeof req.pause === "function") {
+        req.pause();
+      }
+    };
+
+    const onData: (chunk: Buffer | string) => void = (
+      chunk: Buffer | string,
+    ): void => {
+      const asBuffer: Buffer = Buffer.isBuffer(chunk)
+        ? chunk
+        : Buffer.from(chunk, "utf-8");
+
+      compressedBytes += asBuffer.length;
+
+      if (compressedBytes > MAX_COMPRESSED_REQUEST_BODY_BYTES) {
+        settle((): void => {
+          stopReading();
+          GzipRequestBodyMiddleware.rejectTooLarge(
+            req,
+            res,
+            "compressed",
+            compressedBytes,
+            MAX_COMPRESSED_REQUEST_BODY_BYTES,
+          );
+        });
+        return;
+      }
+
+      /*
+       * Honour the inflate's backpressure. zlib does its work on the
+       * threadpool, so a socket that is faster than the inflate will
+       * otherwise pile the whole compressed body into the writable queue -
+       * which is the unbounded buffering this middleware exists to remove,
+       * just one object further down. Pausing also means the caps below get
+       * to see the output before we accept the next chunk, so a bomb is
+       * refused while most of it is still on the wire.
+       */
+      if (!inflate.write(asBuffer) && !awaitingDrain) {
+        awaitingDrain = true;
+
+        if (typeof req.pause === "function") {
+          req.pause();
+        }
+
+        inflate.once("drain", (): void => {
+          awaitingDrain = false;
+
+          if (!settled && typeof req.resume === "function") {
+            req.resume();
+          }
+        });
+      }
+    };
+
+    const onEnd: () => void = (): void => {
+      requestEnded = true;
+
+      /*
+       * A request that announced gzip and then sent nothing. `inflate.end()`
+       * on an empty stream raises "unexpected end of file", which used to
+       * turn every such request - including a bare GET carrying the header -
+       * into a 500 plus an error-level log line. Treat it as the empty body
+       * it is and let the route decide.
+       */
+      if (compressedBytes === 0) {
+        settle((): void => {
+          req.body = Buffer.alloc(0);
+          next();
+        });
+        return;
+      }
+
+      inflate.end();
+    };
+
+    const onRequestError: (err: Error) => void = (err: Error): void => {
+      settle((): void => {
+        next(err);
+      });
+    };
+
+    /*
+     * The client hung up mid-body. There is nobody left to answer, so drop
+     * the inflate and everything it is holding instead of waiting for a
+     * `end` that will never come. `close` also fires on the happy path,
+     * after `end`, which is what `requestEnded` guards.
+     */
+    const onClose: () => void = (): void => {
+      if (requestEnded) {
+        return;
+      }
+
+      settle((): void => {});
+    };
+
+    inflate.on("data", (chunk: Buffer): void => {
+      decompressedBytes += chunk.length;
+
+      if (decompressedBytes > MAX_DECOMPRESSED_REQUEST_BODY_BYTES) {
+        settle((): void => {
+          stopReading();
+          GzipRequestBodyMiddleware.rejectTooLarge(
+            req,
+            res,
+            "decompressed",
+            decompressedBytes,
+            MAX_DECOMPRESSED_REQUEST_BODY_BYTES,
+          );
+        });
+        return;
+      }
+
+      decoded.push(chunk);
+    });
+
+    inflate.on("end", (): void => {
+      const body: Buffer = Buffer.concat(
+        decoded as unknown as Array<Uint8Array>,
+      );
+
+      settle((): void => {
+        req.body = body;
+        next();
+      });
+    });
+
+    inflate.on("error", (err: Error): void => {
+      settle((): void => {
+        logger.error(err, getLogAttributesFromRequest(req as OneUptimeRequest));
+
+        Response.sendErrorResponse(
+          req,
+          res,
+          new ServerException("Error decompressing data"),
+        );
+      });
+    });
+
+    req.on("data", onData);
+    req.on("end", onEnd);
+    req.on("error", onRequestError);
+    req.on("close", onClose);
+  }
+
+  private static rejectTooLarge(
+    req: ExpressRequest,
+    res: ExpressResponse,
+    stage: "compressed" | "decompressed",
+    bytes: number,
+    limit: number,
+  ): void {
+    if (res.headersSent) {
+      return;
+    }
+
+    /*
+     * debug, not warn or error. This path is unauthenticated, so a level
+     * that reaches the log sink by default would hand the same attacker a
+     * log-amplification vector in place of the memory one - while still
+     * leaving an operator something to turn on when a legitimate client
+     * starts getting 413s.
+     */
+    logger.debug(
+      `Rejected a gzip request body: ${stage} size reached ${bytes} bytes, limit is ${limit}.`,
+      getLogAttributesFromRequest(req as OneUptimeRequest),
+    );
+
+    res.status(413).json({
+      message: `Request body too large. The ${stage} body of a gzip-encoded request may not exceed ${limit} bytes.`,
+    });
+  }
+}

package/Server/Middleware/MultipartFormData.ts

@@ -1,17 +1,101 @@
+import BadRequestException from "../../Types/Exception/BadRequestException";
+import PayloadTooLargeException from "../../Types/Exception/PayloadTooLargeException";
+import {
+  ExpressRequest,
+  ExpressResponse,
+  NextFunction,
+} from "../Utils/Express";
 import multer from "multer";
 import { RequestHandler } from "express";
 
+/*
+ * Ceilings for multipart bodies.
+ *
+ * memoryStorage means every part is buffered in the process, and both
+ * routes that mount this middleware run it BEFORE their auth check -
+ * Pyroscope ingest before isAuthorizedServiceMiddleware, SendGrid inbound
+ * before the webhook secret is verified. So these are the limits an
+ * unauthenticated caller is held to, and multer's defaults leave most of
+ * them at Infinity: file size, file count, field count and part count are
+ * all unbounded out of the box (only fieldSize has a default, 1 MB).
+ *
+ * The sizes match the 50 MiB body-parser limit the rest of the app uses,
+ * and nginx cuts the shipped deployment lower still (50M on
+ * /incoming-email, its 1M default on /pyroscope). The COUNTS are the part
+ * that was genuinely missing: without them a body of ten thousand
+ * one-byte parts costs ten thousand allocations and passes every size
+ * check.
+ */
+export const MAX_MULTIPART_FILE_BYTES: number = 50 * 1024 * 1024;
+export const MAX_MULTIPART_FIELD_BYTES: number = 25 * 1024 * 1024;
+export const MAX_MULTIPART_FILES: number = 50;
+export const MAX_MULTIPART_FIELDS: number = 200;
+
 /*
  * Configure multer for handling multipart/form-data
  * Uses memory storage to store files in memory as Buffer objects
  */
-const upload: multer.Multer = multer({ storage: multer.memoryStorage() });
+const upload: multer.Multer = multer({
+  storage: multer.memoryStorage(),
+  limits: {
+    fileSize: MAX_MULTIPART_FILE_BYTES,
+    fieldSize: MAX_MULTIPART_FIELD_BYTES,
+    files: MAX_MULTIPART_FILES,
+    fields: MAX_MULTIPART_FIELDS,
+    /*
+     * Parts are files + fields, so this has to allow both in full or it
+     * would be the effective limit and the two above would never fire.
+     */
+    parts: MAX_MULTIPART_FILES + MAX_MULTIPART_FIELDS,
+  },
+});
+
+const uploadAny: RequestHandler = upload.any() as unknown as RequestHandler;
+
+/*
+ * The LIMIT_* codes that mean "you sent too much". LIMIT_UNEXPECTED_FILE
+ * is deliberately absent: it means the body named a field the route did
+ * not ask for, which is a malformed request rather than an oversized one.
+ * (`.any()` accepts every field name, so it should not arise here - the
+ * distinction is kept so this stays correct if the mount ever narrows.)
+ */
+const TOO_LARGE_CODES: Set<string> = new Set<string>([
+  "LIMIT_PART_COUNT",
+  "LIMIT_FILE_SIZE",
+  "LIMIT_FILE_COUNT",
+  "LIMIT_FIELD_KEY",
+  "LIMIT_FIELD_VALUE",
+  "LIMIT_FIELD_COUNT",
+]);
 
 /*
  * Middleware for handling any file uploads (multipart/form-data)
  * This is useful for webhooks that send data as multipart/form-data (e.g., SendGrid inbound email)
+ *
+ * Wrapped so a limit breach answers 413 rather than 500. multer signals
+ * every limit with a MulterError whose `code` is one of the LIMIT_*
+ * values, and nothing downstream knows what those mean - the generic
+ * error handler would turn a caller's oversized upload into a server
+ * error and page whoever is on call for it.
  */
-const MultipartFormDataMiddleware: RequestHandler =
-  upload.any() as unknown as RequestHandler;
+const MultipartFormDataMiddleware: RequestHandler = (
+  req: ExpressRequest,
+  res: ExpressResponse,
+  next: NextFunction,
+): void => {
+  uploadAny(req, res, (err: unknown): void => {
+    if (err instanceof multer.MulterError) {
+      const message: string = `Multipart request rejected: ${err.code}.`;
+
+      return next(
+        TOO_LARGE_CODES.has(err.code)
+          ? new PayloadTooLargeException(message)
+          : new BadRequestException(message),
+      );
+    }
+
+    return next(err);
+  });
+};
 
 export default MultipartFormDataMiddleware;

package/Server/Services/DetectionRuleService.ts

@@ -4,6 +4,10 @@ import CreateBy from "../Types/Database/CreateBy";
 import UpdateBy from "../Types/Database/UpdateBy";
 import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
 import BadDataException from "../../Types/Exception/BadDataException";
+import {
+  DETECTION_MATCH_COUNT_THRESHOLD_MAX,
+  DETECTION_MATCH_COUNT_THRESHOLD_MIN,
+} from "../../Types/SecurityEvent/DetectionFindingConstants";
 import SigmaRuleParser from "../../Utils/SecurityEvent/Sigma/SigmaRuleParser";
 import SigmaClickhouseCompiler from "../Utils/SecurityEvent/Sigma/SigmaClickhouseCompiler";
 
@@ -25,23 +29,69 @@ export class Service extends DatabaseService<Model> {
     SigmaClickhouseCompiler.compileYaml(ruleYaml);
   }
 
-  private validateEvaluationInterval(interval: number | undefined): void {
-    if (interval === undefined) {
+  // undefined always passes: an omitted field means "keep the default".
+  private validateIntegerInRange(data: {
+    value: number | undefined;
+    min: number;
+    max: number;
+    message: string;
+  }): void {
+    if (data.value === undefined) {
       return;
     }
 
-    if (!Number.isInteger(interval) || interval < 1 || interval > 1440) {
-      throw new BadDataException(
-        "Evaluation interval must be a whole number of minutes between 1 and 1440.",
-      );
+    if (
+      !Number.isInteger(data.value) ||
+      data.value < data.min ||
+      data.value > data.max
+    ) {
+      throw new BadDataException(data.message);
     }
   }
 
+  private validateEvaluationInterval(interval: number | undefined): void {
+    this.validateIntegerInRange({
+      value: interval,
+      min: 1,
+      max: 1440,
+      message:
+        "Evaluation interval must be a whole number of minutes between 1 and 1440.",
+    });
+  }
+
+  private validateMatchCountThreshold(threshold: number | undefined): void {
+    this.validateIntegerInRange({
+      value: threshold,
+      min: DETECTION_MATCH_COUNT_THRESHOLD_MIN,
+      max: DETECTION_MATCH_COUNT_THRESHOLD_MAX,
+      message: `Match count threshold must be a whole number between ${DETECTION_MATCH_COUNT_THRESHOLD_MIN} and ${DETECTION_MATCH_COUNT_THRESHOLD_MAX}.`,
+    });
+  }
+
   protected override async onBeforeCreate(
     createBy: CreateBy<Model>,
   ): Promise<OnCreate<Model>> {
     this.validateSigmaRule(createBy.data.sigmaRuleYaml);
     this.validateEvaluationInterval(createBy.data.evaluationIntervalInMinutes);
+    this.validateMatchCountThreshold(createBy.data.matchCountThreshold);
+
+    /*
+     * groupByField / distinctCountField are looked up verbatim: an
+     * unknown name silently becomes an attributes[] lookup that yields
+     * '' on every row, and for a distinct-count rule that means it never
+     * fires again with no error anywhere. Trimming here kills the whole
+     * whitespace-from-a-paste class of that outage; '' is preserved as
+     * the documented "feature off" value so clearing the field still
+     * works.
+     */
+    if (createBy.data.groupByField !== undefined) {
+      createBy.data.groupByField = createBy.data.groupByField.trim();
+    }
+
+    if (createBy.data.distinctCountField !== undefined) {
+      createBy.data.distinctCountField =
+        createBy.data.distinctCountField.trim();
+    }
 
     /*
      * Default the rule name from the Sigma title so pasting a rule is a
@@ -67,6 +117,23 @@ export class Service extends DatabaseService<Model> {
       updateBy.data.evaluationIntervalInMinutes as number | undefined,
     );
 
+    this.validateMatchCountThreshold(
+      updateBy.data.matchCountThreshold as number | undefined,
+    );
+
+    // Same trim as onBeforeCreate — see the rationale there.
+    if (updateBy.data.groupByField !== undefined) {
+      updateBy.data.groupByField = (
+        updateBy.data.groupByField as string
+      ).trim();
+    }
+
+    if (updateBy.data.distinctCountField !== undefined) {
+      updateBy.data.distinctCountField = (
+        updateBy.data.distinctCountField as string
+      ).trim();
+    }
+
     return { updateBy, carryForward: null };
   }
 }