@oneuptime/common 12.0.23 → 12.0.24

package/Tests/Server/Middleware/MultipartFormData.test.ts

@@ -0,0 +1,298 @@
+import { describe, expect, jest, test } from "@jest/globals";
+import timers from "timers";
+
+/*
+ * multer calls `setImmediate` internally (make-middleware.js and
+ * remove-uploaded-files.js), and Common's jest environment is jsdom, which
+ * does not expose it. A `@jest-environment node` docblock is not an option
+ * here - Common's shared jest.setup.ts touches `window`, which is why
+ * EsbuildConfig.test.ts spawns a subprocess instead - so lend jsdom the
+ * real one from Node.
+ */
+if (
+  typeof (globalThis as unknown as { setImmediate?: unknown }).setImmediate !==
+  "function"
+) {
+  (globalThis as unknown as { setImmediate: unknown }).setImmediate =
+    timers.setImmediate;
+}
+
+/*
+ * The limits under test are 25 and 50 MiB, so the fixtures that probe them
+ * are 25 and 50 MiB of real multipart body parsed by the real busboy.
+ * That does not fit in jest's 5s default.
+ */
+jest.setTimeout(180_000);
+
+/*
+ * The multipart reader used by Pyroscope ingest and the SendGrid inbound
+ * email webhook.
+ *
+ * Both routes mount it BEFORE their auth check - Pyroscope before
+ * isAuthorizedServiceMiddleware, SendGrid before the webhook secret is
+ * verified - and it uses multer's memoryStorage, so every part it accepts
+ * is buffered in the process on behalf of a caller who has not proved
+ * anything yet.
+ *
+ * multer's defaults leave almost all of that unbounded: fileSize, files,
+ * fields and parts are all Infinity out of the box, and only fieldSize
+ * has a default at all. These tests pin the limits and the status code a
+ * breach produces, because the failure mode of getting this wrong is
+ * silent - an unbounded upload looks exactly like a working one until the
+ * pod dies.
+ */
+
+import MultipartFormDataMiddleware, {
+  MAX_MULTIPART_FIELDS,
+  MAX_MULTIPART_FIELD_BYTES,
+  MAX_MULTIPART_FILES,
+  MAX_MULTIPART_FILE_BYTES,
+} from "../../../Server/Middleware/MultipartFormData";
+import {
+  ExpressRequest,
+  ExpressResponse,
+  NextFunction,
+} from "../../../Server/Utils/Express";
+import Exception from "../../../Types/Exception/Exception";
+import ExceptionCode from "../../../Types/Exception/ExceptionCode";
+import { Readable } from "stream";
+
+const BOUNDARY: string = "----oneuptimetestboundary";
+
+interface MultipartPart {
+  name: string;
+  value: Buffer;
+  filename?: string;
+}
+
+function buildMultipartBody(parts: Array<MultipartPart>): Buffer {
+  const chunks: Array<Buffer> = [];
+
+  for (const part of parts) {
+    const disposition: string = part.filename
+      ? `form-data; name="${part.name}"; filename="${part.filename}"`
+      : `form-data; name="${part.name}"`;
+
+    const header: string =
+      `--${BOUNDARY}\r\n` +
+      `Content-Disposition: ${disposition}\r\n` +
+      (part.filename ? "Content-Type: application/octet-stream\r\n" : "") +
+      "\r\n";
+
+    chunks.push(Buffer.from(header));
+    chunks.push(part.value);
+    chunks.push(Buffer.from("\r\n"));
+  }
+
+  chunks.push(Buffer.from(`--${BOUNDARY}--\r\n`));
+
+  return Buffer.concat(chunks as unknown as Array<Uint8Array>);
+}
+
+interface Outcome {
+  req: ExpressRequest;
+  error: unknown;
+  nextCalls: number;
+}
+
+/*
+ * Drive the middleware with a real multipart body over a real Readable, so
+ * busboy does the parsing it does in production rather than being faked.
+ */
+function run(parts: Array<MultipartPart>): Promise<Outcome> {
+  const body: Buffer = buildMultipartBody(parts);
+
+  const req: Readable & {
+    headers: Record<string, string>;
+    body?: unknown;
+    files?: unknown;
+  } = Readable.from([body]) as never;
+
+  req.headers = {
+    "content-type": `multipart/form-data; boundary=${BOUNDARY}`,
+    "content-length": String(body.length),
+  };
+
+  const res: ExpressResponse = {
+    status: (): unknown => {
+      return { send: (): void => {}, json: (): void => {} };
+    },
+    headersSent: false,
+  } as unknown as ExpressResponse;
+
+  return new Promise<Outcome>((resolve: (value: Outcome) => void): void => {
+    let nextCalls: number = 0;
+
+    MultipartFormDataMiddleware(req as unknown as ExpressRequest, res, ((
+      err?: unknown,
+    ): void => {
+      nextCalls++;
+      resolve({
+        req: req as unknown as ExpressRequest,
+        error: err,
+        nextCalls: nextCalls,
+      });
+    }) as NextFunction);
+  });
+}
+
+function files(outcome: Outcome): Array<{ fieldname: string; buffer: Buffer }> {
+  return (outcome.req.files || []) as Array<{
+    fieldname: string;
+    buffer: Buffer;
+  }>;
+}
+
+describe("MultipartFormData - the limits are set where they were reasoned about", () => {
+  test("sizes match the app-wide 50 MiB body-parser number", () => {
+    expect(MAX_MULTIPART_FILE_BYTES).toBe(50 * 1024 * 1024);
+    expect(MAX_MULTIPART_FIELD_BYTES).toBe(25 * 1024 * 1024);
+  });
+
+  test("counts are bounded at all, which is the part multer leaves at Infinity", () => {
+    expect(MAX_MULTIPART_FILES).toBe(50);
+    expect(MAX_MULTIPART_FIELDS).toBe(200);
+    expect(Number.isFinite(MAX_MULTIPART_FILES)).toBe(true);
+    expect(Number.isFinite(MAX_MULTIPART_FIELDS)).toBe(true);
+  });
+});
+
+describe("MultipartFormData - normal bodies still parse", () => {
+  test("a single file arrives as a Buffer on req.files", async () => {
+    const outcome: Outcome = await run([
+      { name: "profile", value: Buffer.from("pprof-bytes"), filename: "p.pb" },
+    ]);
+
+    expect(outcome.error).toBeUndefined();
+    expect(files(outcome)).toHaveLength(1);
+    expect(files(outcome)[0]!.fieldname).toBe("profile");
+    expect(files(outcome)[0]!.buffer.toString()).toBe("pprof-bytes");
+  });
+
+  test("fields and files together, the SendGrid shape", async () => {
+    const outcome: Outcome = await run([
+      { name: "from", value: Buffer.from("someone@example.com") },
+      { name: "subject", value: Buffer.from("hello") },
+      { name: "attachment1", value: Buffer.from("PDF"), filename: "a.pdf" },
+    ]);
+
+    expect(outcome.error).toBeUndefined();
+    expect((outcome.req.body as Record<string, string>)["from"]).toBe(
+      "someone@example.com",
+    );
+    expect(files(outcome)).toHaveLength(1);
+  });
+
+  test("a body with no parts at all is fine", async () => {
+    const outcome: Outcome = await run([]);
+
+    expect(outcome.error).toBeUndefined();
+    expect(files(outcome)).toHaveLength(0);
+  });
+
+  test("a file right at the count limit is accepted", async () => {
+    const outcome: Outcome = await run(
+      Array.from({ length: MAX_MULTIPART_FILES }, (_v: unknown, i: number) => {
+        return {
+          name: `f${i}`,
+          value: Buffer.from("x"),
+          filename: `f${i}.bin`,
+        };
+      }),
+    );
+
+    expect(outcome.error).toBeUndefined();
+    expect(files(outcome)).toHaveLength(MAX_MULTIPART_FILES);
+  });
+
+  test("fields right at the count limit are accepted", async () => {
+    const outcome: Outcome = await run(
+      Array.from({ length: MAX_MULTIPART_FIELDS }, (_v: unknown, i: number) => {
+        return { name: `k${i}`, value: Buffer.from("v") };
+      }),
+    );
+
+    expect(outcome.error).toBeUndefined();
+    expect(
+      Object.keys(outcome.req.body as Record<string, unknown>),
+    ).toHaveLength(MAX_MULTIPART_FIELDS);
+  });
+});
+
+describe("MultipartFormData - breaches answer 413, not 500", () => {
+  test("one file over the size limit", async () => {
+    const outcome: Outcome = await run([
+      {
+        name: "big",
+        value: Buffer.alloc(MAX_MULTIPART_FILE_BYTES + 1),
+        filename: "big.bin",
+      },
+    ]);
+
+    expect(outcome.error).toBeInstanceOf(Exception);
+    expect((outcome.error as Exception).code).toBe(
+      ExceptionCode.PayloadTooLargeException,
+    );
+    expect((outcome.error as Exception).code).toBe(413);
+    expect((outcome.error as Exception).message).toContain("LIMIT_FILE_SIZE");
+  });
+
+  test("one file too many", async () => {
+    const outcome: Outcome = await run(
+      Array.from(
+        { length: MAX_MULTIPART_FILES + 1 },
+        (_v: unknown, i: number) => {
+          return {
+            name: `f${i}`,
+            value: Buffer.from("x"),
+            filename: `f${i}.bin`,
+          };
+        },
+      ),
+    );
+
+    expect((outcome.error as Exception).code).toBe(
+      ExceptionCode.PayloadTooLargeException,
+    );
+    expect((outcome.error as Exception).message).toContain("LIMIT_FILE_COUNT");
+  });
+
+  test("one field too many - thousands of tiny parts is the cheap version of this attack", async () => {
+    const outcome: Outcome = await run(
+      Array.from(
+        { length: MAX_MULTIPART_FIELDS + 1 },
+        (_v: unknown, i: number) => {
+          return { name: `k${i}`, value: Buffer.from("v") };
+        },
+      ),
+    );
+
+    expect((outcome.error as Exception).code).toBe(
+      ExceptionCode.PayloadTooLargeException,
+    );
+    expect((outcome.error as Exception).message).toContain("LIMIT_FIELD_COUNT");
+  });
+
+  test("one field value over the size limit", async () => {
+    const outcome: Outcome = await run([
+      { name: "html", value: Buffer.alloc(MAX_MULTIPART_FIELD_BYTES + 1) },
+    ]);
+
+    expect((outcome.error as Exception).code).toBe(
+      ExceptionCode.PayloadTooLargeException,
+    );
+    expect((outcome.error as Exception).message).toContain("LIMIT_FIELD_VALUE");
+  });
+
+  test("next() is called exactly once on a breach", async () => {
+    const outcome: Outcome = await run([
+      {
+        name: "big",
+        value: Buffer.alloc(MAX_MULTIPART_FILE_BYTES + 1),
+        filename: "big.bin",
+      },
+    ]);
+
+    expect(outcome.nextCalls).toBe(1);
+  });
+});

package/Tests/Server/Services/AddDetectionRuleDistinctCountColumnsMigration.test.ts

@@ -0,0 +1,311 @@
+import { AddDetectionRuleDistinctCountColumns1789512000000 } from "../../../Server/Infrastructure/Postgres/SchemaMigrations/1789512000000-AddDetectionRuleDistinctCountColumns";
+import SchemaMigrations from "../../../Server/Infrastructure/Postgres/SchemaMigrations/Index";
+import DetectionRule from "../../../Models/DatabaseModels/DetectionRule";
+import ColumnLength from "../../../Types/Database/ColumnLength";
+import ColumnType from "../../../Types/Database/ColumnType";
+import { TableColumnMetadata } from "../../../Types/Database/TableColumn";
+import TableColumnType from "../../../Types/Database/TableColumnType";
+import {
+  MigrationInterface,
+  QueryRunner,
+  getMetadataArgsStorage,
+} from "typeorm";
+import type { ColumnMetadataArgs } from "typeorm/metadata-args/ColumnMetadataArgs";
+import fs from "fs";
+import path from "path";
+import { describe, expect, test } from "@jest/globals";
+
+/*
+ * The migration behind detection-rule correlation semantics (issue #3398).
+ *
+ * DetectionRule gains distinctCountField (what a rule counts: unique
+ * values of a field instead of raw matching events) and
+ * matchCountThreshold (how many of that count a group needs in one
+ * evaluation window before the rule fires). Three things have to hold or
+ * a deployed instance breaks on upgrade:
+ *
+ *   1. up() adds exactly those two columns — distinctCountField nullable
+ *      with NO default (NULL means "count raw events", the behavior every
+ *      existing rule was saved with), matchCountThreshold NOT NULL
+ *      DEFAULT '1' so every existing row keeps fire-on-any-match without
+ *      a backfill;
+ *   2. down() removes exactly what up() added;
+ *   3. the migration is registered in SchemaMigrations/Index.ts — an
+ *      unregistered migration never runs on boot, so the evaluator's
+ *      SELECT of the two columns would 500 on every evaluation cycle.
+ *
+ * Pure SQL-contract assertions against a fake QueryRunner. No Postgres.
+ * The registry-wide ordering guard lives in
+ * SchemaMigrationsOrdering.test.ts and is not repeated here; what is
+ * asserted here is about THIS migration and stays true forever.
+ */
+
+const MIGRATION_DIRECTORY: string = path.join(
+  __dirname,
+  "../../../Server/Infrastructure/Postgres/SchemaMigrations",
+);
+
+const MIGRATION_TIMESTAMP: string = "1789512000000";
+
+const MIGRATION_NAME: string = `AddDetectionRuleDistinctCountColumns${MIGRATION_TIMESTAMP}`;
+
+type MakeQueryRunnerResult = {
+  runner: QueryRunner;
+  statements: Array<string>;
+};
+
+type MakeQueryRunnerFunction = () => MakeQueryRunnerResult;
+
+const makeQueryRunner: MakeQueryRunnerFunction = (): MakeQueryRunnerResult => {
+  const statements: Array<string> = [];
+
+  const query: (...args: Array<unknown>) => Promise<undefined> = (
+    ...args: Array<unknown>
+  ): Promise<undefined> => {
+    statements.push(String(args[0]));
+    return Promise.resolve(undefined);
+  };
+
+  return {
+    runner: { query } as unknown as QueryRunner,
+    statements,
+  };
+};
+
+describe("AddDetectionRuleDistinctCountColumns1789512000000 — SQL contract", () => {
+  test("up() adds exactly the two columns with their exact types and defaults", async () => {
+    const { runner, statements } = makeQueryRunner();
+
+    await new AddDetectionRuleDistinctCountColumns1789512000000().up(runner);
+
+    expect(statements).toEqual([
+      `ALTER TABLE "DetectionRule" ADD "distinctCountField" character varying(100)`,
+      `ALTER TABLE "DetectionRule" ADD "matchCountThreshold" integer NOT NULL DEFAULT '1'`,
+    ]);
+  });
+
+  /*
+   * NULL is a meaning on distinctCountField: it is what the evaluator
+   * reads as "threshold the raw match count", i.e. the semantics every
+   * rule saved before this migration was created with. A default would
+   * silently switch existing rules to distinct counting.
+   */
+  test("distinctCountField is nullable with no default", async () => {
+    const { runner, statements } = makeQueryRunner();
+
+    await new AddDetectionRuleDistinctCountColumns1789512000000().up(runner);
+
+    const addDistinctCountField: string = statements.find((sql: string) => {
+      return sql.includes(`ADD "distinctCountField"`);
+    })!;
+
+    expect(addDistinctCountField).not.toContain("NOT NULL");
+    expect(addDistinctCountField).not.toContain("DEFAULT");
+  });
+
+  /*
+   * The opposite for matchCountThreshold: the DB default IS the
+   * compatibility story. Every pre-existing row gets threshold 1 —
+   * fire-on-any-match — the instant the column appears, with no backfill
+   * statement to forget.
+   */
+  test("matchCountThreshold is NOT NULL with DEFAULT '1'", async () => {
+    const { runner, statements } = makeQueryRunner();
+
+    await new AddDetectionRuleDistinctCountColumns1789512000000().up(runner);
+
+    const addMatchCountThreshold: string = statements.find((sql: string) => {
+      return sql.includes(`ADD "matchCountThreshold"`);
+    })!;
+
+    expect(addMatchCountThreshold).toContain("integer NOT NULL DEFAULT '1'");
+  });
+
+  test("down() drops exactly what up() added, in reverse order", async () => {
+    const { runner, statements } = makeQueryRunner();
+
+    await new AddDetectionRuleDistinctCountColumns1789512000000().down(runner);
+
+    expect(statements).toEqual([
+      `ALTER TABLE "DetectionRule" DROP COLUMN "matchCountThreshold"`,
+      `ALTER TABLE "DetectionRule" DROP COLUMN "distinctCountField"`,
+    ]);
+  });
+
+  /*
+   * Not just "same count": the exact column names, derived from the two
+   * statement sets rather than restated, so adding a column to up()
+   * without adding it to down() fails here.
+   */
+  test("up() and down() name the same columns", async () => {
+    const { runner: upRunner, statements: upStatements } = makeQueryRunner();
+    const { runner: downRunner, statements: downStatements } =
+      makeQueryRunner();
+
+    await new AddDetectionRuleDistinctCountColumns1789512000000().up(upRunner);
+    await new AddDetectionRuleDistinctCountColumns1789512000000().down(
+      downRunner,
+    );
+
+    const columnsIn: (
+      statements: Array<string>,
+      keyword: string,
+    ) => Array<string> = (
+      statements: Array<string>,
+      keyword: string,
+    ): Array<string> => {
+      return statements
+        .filter((sql: string) => {
+          return sql.includes(keyword);
+        })
+        .map((sql: string) => {
+          return sql.split(`${keyword} "`)[1]!.split('"')[0]!;
+        })
+        .sort();
+    };
+
+    expect(columnsIn(downStatements, "DROP COLUMN")).toEqual(
+      columnsIn(upStatements, "ADD"),
+    );
+  });
+});
+
+describe("registration", () => {
+  /*
+   * The step AGENTS.md calls out and the one that fails silently: a
+   * migration that is not in this array never runs, so the columns are
+   * missing in a real deployment while the evaluator's rule SELECT and
+   * the dashboard form both reference them.
+   */
+  test("the migration is registered so it runs on boot", () => {
+    const names: Array<string> = SchemaMigrations.map(
+      (migration: new () => MigrationInterface): string => {
+        return migration.name;
+      },
+    );
+
+    expect(names).toContain(MIGRATION_NAME);
+  });
+
+  test("it is registered exactly once", () => {
+    const occurrences: number = SchemaMigrations.filter(
+      (migration: new () => MigrationInterface): boolean => {
+        return migration.name === MIGRATION_NAME;
+      },
+    ).length;
+
+    expect(occurrences).toBe(1);
+  });
+
+  /*
+   * TypeORM records applied migrations by the `name` property, not the
+   * file name. A mismatch re-runs the migration on every boot.
+   */
+  test("its declared name matches its class name", () => {
+    expect(new AddDetectionRuleDistinctCountColumns1789512000000().name).toBe(
+      MIGRATION_NAME,
+    );
+    expect(AddDetectionRuleDistinctCountColumns1789512000000.name).toBe(
+      MIGRATION_NAME,
+    );
+  });
+});
+
+describe("identity on disk", () => {
+  /*
+   * The timestamp in the class name is the one on disk. A class renamed
+   * without renaming its file — or two migrations landing on the same
+   * timestamp — makes the registered ordering describe something that is
+   * not what actually runs.
+   */
+  test("exactly one file on disk carries its timestamp, and it is this one", () => {
+    const matching: Array<string> = fs
+      .readdirSync(MIGRATION_DIRECTORY)
+      .filter((file: string): boolean => {
+        return file.startsWith(`${MIGRATION_TIMESTAMP}-`);
+      });
+
+    expect(matching).toEqual([
+      `${MIGRATION_TIMESTAMP}-AddDetectionRuleDistinctCountColumns.ts`,
+    ]);
+  });
+});
+
+/*
+ * The model side of the same contract. TypeORM generates the schema from
+ * these decorators, so a column that exists in the migration but not on
+ * the model (or vice versa, or with different nullability/defaults) is
+ * exactly the drift the Postgres Schema Drift workflow fails on.
+ */
+describe("the DetectionRule model carries the columns the migration adds", () => {
+  const columnArgs: (propertyName: string) => ColumnMetadataArgs = (
+    propertyName: string,
+  ): ColumnMetadataArgs => {
+    const args: ColumnMetadataArgs | undefined = getMetadataArgsStorage()
+      .columns.filter((candidate: ColumnMetadataArgs) => {
+        return candidate.target === DetectionRule;
+      })
+      .find((candidate: ColumnMetadataArgs) => {
+        return candidate.propertyName === propertyName;
+      });
+
+    if (!args) {
+      throw new Error(
+        `DetectionRule.${propertyName} has no TypeORM @Column metadata`,
+      );
+    }
+
+    return args;
+  };
+
+  test("distinctCountField is a nullable ShortText varchar(100), matching the migration", () => {
+    const args: ColumnMetadataArgs = columnArgs("distinctCountField");
+
+    expect(args.options.type).toBe(ColumnType.ShortText);
+    expect(args.options.length).toBe(ColumnLength.ShortText);
+    // character varying(100) in the migration.
+    expect(ColumnLength.ShortText).toBe(100);
+    expect(args.options.nullable).toBe(true);
+    expect(args.options.default).toBeUndefined();
+  });
+
+  test("matchCountThreshold is a NOT NULL number defaulting to 1, matching the migration", () => {
+    const args: ColumnMetadataArgs = columnArgs("matchCountThreshold");
+
+    expect(args.options.type).toBe(ColumnType.Number);
+    expect(args.options.nullable).toBe(false);
+    expect(args.options.default).toBe(1);
+  });
+
+  test("distinctCountField's TableColumn is an optional ShortText", () => {
+    const metadata: TableColumnMetadata =
+      new DetectionRule().getTableColumnMetadata("distinctCountField");
+
+    expect(metadata.type).toBe(TableColumnType.ShortText);
+    expect(metadata.required).toBe(false);
+  });
+
+  /*
+   * isDefaultValueColumn is the API-side half of the DB default: without
+   * it, checkRequiredFields 400s every create payload written before the
+   * column existed and the DEFAULT '1' can never apply.
+   */
+  test("matchCountThreshold's TableColumn is required with default 1 as a default-value column", () => {
+    const metadata: TableColumnMetadata =
+      new DetectionRule().getTableColumnMetadata("matchCountThreshold");
+
+    expect(metadata.type).toBe(TableColumnType.Number);
+    expect(metadata.required).toBe(true);
+    expect(metadata.defaultValue).toBe(1);
+    expect(metadata.isDefaultValueColumn).toBe(true);
+  });
+
+  test("both columns default to undefined on a fresh model instance", () => {
+    const rule: DetectionRule = new DetectionRule();
+
+    expect("distinctCountField" in rule).toBe(true);
+    expect("matchCountThreshold" in rule).toBe(true);
+    expect(rule.distinctCountField).toBeUndefined();
+    expect(rule.matchCountThreshold).toBeUndefined();
+  });
+});