@oneuptime/common 13.0.6 → 13.0.8

package/Tests/Server/Infrastructure/E2EClickHouseFixtureAccess.test.ts

@@ -15,24 +15,33 @@ import yaml from "js-yaml";
  * network_mode: host and no ClickHouse settings, the stacks CI starts publish
  * no ClickHouse port, and every e2e job on master failed in beforeAll. So:
  *
- *  - production compose files publish no ClickHouse port at all;
+ *  - production compose files publish no ClickHouse port at all, and neither
+ *    does the SaaS CI stack's billing overlay;
  *  - the test-only overlay publishes ClickHouse HTTP on loopback only, on the
- *    host port docker-compose.dev.yml uses and the fixture maps to;
+ *    host port Scripts/Dev/docker-compose.dev.yml uses and the fixture maps to;
  *  - the host-networked e2e service receives the connection settings;
  *  - every workflow job that runs the e2e service starts its stack with the
  *    overlay (the release workflow is where a partial fix would slip through,
  *    since it only runs when a release is cut).
+ *
+ * The overlays live in packages/E2E but their paths are relative to the repo
+ * root, which holds only while each command passes them after
+ * `-f docker-compose.yml`. That is guarded here too.
  */
 
-const REPO_ROOT: string = path.resolve(__dirname, "../../../..");
+const REPO_ROOT: string = path.resolve(__dirname, "../../../../..");
 const WORKFLOWS_DIRECTORY: string = path.join(
   REPO_ROOT,
   ".github",
   "workflows",
 );
-const OVERLAY_FILE_NAME: string = "docker-compose.e2e-clickhouse.yml";
+const OVERLAY_FILE_NAME: string =
+  "packages/E2E/docker-compose.e2e-clickhouse.yml";
+const BILLING_OVERLAY_FILE_NAME: string =
+  "packages/E2E/docker-compose.billing.yml";
 const FIXTURE_PATH: string = path.join(
   REPO_ROOT,
+  "packages",
   "E2E",
   "Tests",
   "Dashboard",
@@ -43,7 +52,6 @@ const FIXTURE_PATH: string = path.join(
 const PRODUCTION_COMPOSE_FILES: Array<string> = [
   "docker-compose.base.yml",
   "docker-compose.yml",
-  "docker-compose.billing.yml",
 ];
 
 const CLICKHOUSE_CONNECTION_SETTINGS: Array<string> = [
@@ -56,10 +64,17 @@ const CLICKHOUSE_CONNECTION_SETTINGS: Array<string> = [
 
 const CLICKHOUSE_HTTP_CONTAINER_PORT: string = "8123";
 
-// The step that runs the suite: `docker compose -f docker-compose.dev.yml up ... e2e`.
+/*
+ * The step that runs the suite:
+ * `docker compose --project-directory . -f Scripts/Dev/docker-compose.dev.yml up ... e2e`,
+ * or `... -f packages/E2E/docker-compose.e2e.yml up ... e2e` for the release image.
+ */
 const RUNS_E2E_SERVICE: RegExp =
-  /docker compose\s[^\n]*-f docker-compose\.(?:dev|e2e)\.yml\s+up\b[^\n]*\se2e\b/;
+  /docker compose\s[^\n]*-f (?:Scripts\/Dev\/docker-compose\.dev|packages\/E2E\/docker-compose\.e2e)\.yml\s+up\b[^\n]*\se2e\b/;
 const UP_SUBCOMMAND: RegExp = /\sup(?:\s|$)/;
+const DOWN_SUBCOMMAND: RegExp = /\sdown(?:\s|$)/;
+const ROOT_COMPOSE_FILE_FLAG: RegExp = /-f docker-compose\.yml\s/;
+const E2E_OVERLAY_FLAG: string = "-f packages/E2E/docker-compose.";
 const WORKFLOW_FILE: RegExp = /\.ya?ml$/;
 const FIXTURE_LOCAL_PORT: RegExp =
   /LOCAL_CLICKHOUSE_HTTP_PORT: string = "(\d+)"/;
@@ -87,6 +102,11 @@ interface WorkflowFile {
   jobs?: Record<string, WorkflowJob>;
 }
 
+interface WorkflowJobRun {
+  job: string;
+  runText: string;
+}
+
 interface PublishedPort {
   hostIp: string;
   hostPort: string;
@@ -128,6 +148,34 @@ function clickHouseHttpHostPort(compose: ComposeFile): string | undefined {
     })?.hostPort;
 }
 
+// Every workflow job, with the run scripts of its steps joined.
+function readWorkflowJobRuns(): Array<WorkflowJobRun> {
+  const jobRuns: Array<WorkflowJobRun> = [];
+
+  for (const fileName of fs.readdirSync(WORKFLOWS_DIRECTORY).sort()) {
+    if (!WORKFLOW_FILE.test(fileName)) {
+      continue;
+    }
+
+    const workflow: WorkflowFile = readYaml<WorkflowFile>(
+      path.join(WORKFLOWS_DIRECTORY, fileName),
+    );
+
+    for (const [jobName, job] of Object.entries(workflow.jobs || {})) {
+      jobRuns.push({
+        job: `${fileName}#${jobName}`,
+        runText: (job.steps || [])
+          .map((step: WorkflowStep) => {
+            return step.run || "";
+          })
+          .join("\n"),
+      });
+    }
+  }
+
+  return jobRuns;
+}
+
 function startsStackWithOverlay(runText: string): boolean {
   return runText.split("\n").some((line: string) => {
     return (
@@ -149,6 +197,13 @@ describe("E2E ClickHouse fixture access", () => {
     },
   );
 
+  test("the billing overlay leaves ClickHouse to docker-compose.yml, so the SaaS stack publishes no port either", () => {
+    const overlay: ComposeFile = readCompose(BILLING_OVERLAY_FILE_NAME);
+
+    expect(overlay.services?.["home"]).toBeDefined();
+    expect(overlay.services?.["clickhouse"]).toBeUndefined();
+  });
+
   test("the overlay publishes only ClickHouse HTTP, and only on loopback", () => {
     const overlay: ComposeFile = readCompose(OVERLAY_FILE_NAME);
 
@@ -174,9 +229,9 @@ describe("E2E ClickHouse fixture access", () => {
     expect(clickHouseHttpHostPort(readCompose(OVERLAY_FILE_NAME))).toBe(
       fixturePort,
     );
-    expect(clickHouseHttpHostPort(readCompose("docker-compose.dev.yml"))).toBe(
-      fixturePort,
-    );
+    expect(
+      clickHouseHttpHostPort(readCompose("Scripts/Dev/docker-compose.dev.yml")),
+    ).toBe(fixturePort);
   });
 
   test("the host-networked e2e service receives the fixture's ClickHouse settings", () => {
@@ -201,30 +256,14 @@ describe("E2E ClickHouse fixture access", () => {
     const e2eJobs: Array<string> = [];
     const jobsWithoutOverlay: Array<string> = [];
 
-    for (const fileName of fs.readdirSync(WORKFLOWS_DIRECTORY).sort()) {
-      if (!WORKFLOW_FILE.test(fileName)) {
+    for (const { job, runText } of readWorkflowJobRuns()) {
+      if (!RUNS_E2E_SERVICE.test(runText)) {
         continue;
       }
 
-      const workflow: WorkflowFile = readYaml<WorkflowFile>(
-        path.join(WORKFLOWS_DIRECTORY, fileName),
-      );
-
-      for (const [jobName, job] of Object.entries(workflow.jobs || {})) {
-        const runText: string = (job.steps || [])
-          .map((step: WorkflowStep) => {
-            return step.run || "";
-          })
-          .join("\n");
-
-        if (!RUNS_E2E_SERVICE.test(runText)) {
-          continue;
-        }
-
-        e2eJobs.push(`${fileName}#${jobName}`);
-        if (!startsStackWithOverlay(runText)) {
-          jobsWithoutOverlay.push(`${fileName}#${jobName}`);
-        }
+      e2eJobs.push(job);
+      if (!startsStackWithOverlay(runText)) {
+        jobsWithoutOverlay.push(job);
       }
     }
 
@@ -237,4 +276,40 @@ describe("E2E ClickHouse fixture access", () => {
     );
     expect(jobsWithoutOverlay).toEqual([]);
   });
+
+  /*
+   * A later -f resolves its paths against the first file's folder, so an
+   * overlay passed on its own looks for docker-compose.base.yml inside
+   * packages/E2E. And `down` on an overlay pair tears down the stack it sits
+   * on, volumes included, not just the overlay's service.
+   */
+  test("workflows pass packages/E2E compose overlays after docker-compose.yml, and never run down on them", () => {
+    const overlayCommands: Array<string> = [];
+    const misusedOverlayCommands: Array<string> = [];
+
+    for (const { job, runText } of readWorkflowJobRuns()) {
+      for (const line of runText.split("\n")) {
+        const overlayIndex: number = line.indexOf(E2E_OVERLAY_FLAG);
+
+        if (!line.includes("docker compose") || overlayIndex === -1) {
+          continue;
+        }
+
+        const command: string = `${job}: ${line.trim()}`;
+        const rootIndex: number = line.search(ROOT_COMPOSE_FILE_FLAG);
+
+        overlayCommands.push(command);
+        if (
+          rootIndex === -1 ||
+          rootIndex > overlayIndex ||
+          DOWN_SUBCOMMAND.test(line)
+        ) {
+          misusedOverlayCommands.push(command);
+        }
+      }
+    }
+
+    expect(overlayCommands.length).toBeGreaterThan(0);
+    expect(misusedOverlayCommands).toEqual([]);
+  });
 });

package/Tests/Server/Infrastructure/ExampleDockerfiles.test.ts

@@ -7,9 +7,9 @@ import path from "path";
  * Regression tests for "the quick start does not work on a fresh clone".
  *
  * `.gitignore` carries a repo-wide `**\/Dockerfile` rule. That rule is correct
- * for service images: `configure.sh` renders every `Dockerfile.tpl` through
- * gomplate at install time, so the rendered `Dockerfile` really is build
- * output and really should not be committed.
+ * for service images: `Scripts/Install/configure.sh` renders every
+ * `Dockerfile.tpl` through gomplate at install time, so the rendered
+ * `Dockerfile` really is build output and really should not be committed.
  *
  * It is wrong for `Examples/`, where the Dockerfiles are hand-written source.
  * `Examples/snmp-simulator/Dockerfile` was swallowed by it: `docker compose up
@@ -23,7 +23,7 @@ import path from "path";
  * it names is neither committed nor generated from something that is.
  */
 
-const REPO_ROOT: string = path.resolve(__dirname, "..", "..", "..", "..");
+const REPO_ROOT: string = path.resolve(__dirname, "..", "..", "..", "..", "..");
 
 // Where hand-written (as opposed to gomplate-rendered) Dockerfiles live.
 const EXAMPLES_DIR: string = "Examples";
@@ -34,6 +34,14 @@ const SNMP_SIMULATOR_DOCKERFILE: string = path.join(
   "Dockerfile",
 );
 
+/*
+ * The dev stack's compose file lives in Scripts/Dev but is always run from the
+ * repo root with `--project-directory .`, so compose resolves its build paths
+ * against the repo root instead of the file's own folder.
+ */
+const DEV_COMPOSE_FILE: string = "Scripts/Dev/docker-compose.dev.yml";
+const ROOT_PROJECT_COMPOSE_FILES: Array<string> = [DEV_COMPOSE_FILE];
+
 /*
  * Run a git command against the checkout under test. A failure here is a real
  * test failure and not something to swallow: the whole point of this file is to
@@ -46,6 +54,11 @@ const git: (args: Array<string>) => string = (args: Array<string>): string => {
       cwd: REPO_ROOT,
       encoding: "utf8",
       stdio: ["ignore", "pipe", "pipe"],
+      /*
+       * `git ls-files` for the whole repository is over a megabyte, and
+       * execFileSync's default 1 MiB buffer fails it with ENOBUFS.
+       */
+      maxBuffer: 64 * 1024 * 1024,
     });
   } catch (err: unknown) {
     throw new Error(
@@ -136,7 +149,9 @@ const parseImageBuilds: (composeFile: string) => Array<ImageBuild> = (
   composeFile: string,
 ): Array<ImageBuild> => {
   const builds: Array<ImageBuild> = [];
-  const composeDir: string = path.dirname(composeFile);
+  const composeDir: string = ROOT_PROJECT_COMPOSE_FILES.includes(composeFile)
+    ? "."
+    : path.dirname(composeFile);
   const lines: Array<string> = readRepoFile(composeFile).split("\n");
 
   let inServices: boolean = false;
@@ -254,8 +269,9 @@ const imageBuilds: Array<ImageBuild> = composeFiles.flatMap(
 
 /*
  * A Dockerfile is reachable on a fresh clone if it is committed, or if
- * `configure.sh` renders it from a committed `Dockerfile.tpl` before anything
- * builds it. Anything else exists only on the machine that authored it.
+ * `Scripts/Install/configure.sh` renders it from a committed `Dockerfile.tpl`
+ * before anything builds it. Anything else exists only on the machine that
+ * authored it.
  */
 const templateFor: (dockerfile: string) => string = (
   dockerfile: string,
@@ -277,7 +293,7 @@ describe("Every Dockerfile a compose file builds survives a fresh clone", () =>
      * list is silently green -- the suite would keep passing while checking
      * nothing at all.
      */
-    expect(composeFiles).toContain("docker-compose.dev.yml");
+    expect(composeFiles).toContain(DEV_COMPOSE_FILE);
     expect(composeFiles).toContain(
       path
         .join(SNMP_SIMULATOR_DIR, "docker-compose.yml")
@@ -311,6 +327,22 @@ describe("Every Dockerfile a compose file builds survives a fresh clone", () =>
     },
   );
 
+  test("the dev compose file is run with the repo root as its project directory", () => {
+    /*
+     * Ties the build-path resolution above to the commands that run the file.
+     * Without the flag compose would resolve every build under Scripts/Dev.
+     */
+    const scripts: Record<string, string> = JSON.parse(
+      readRepoFile("package.json"),
+    ).scripts;
+
+    for (const script of ["build", "force-build", "dev"]) {
+      expect(scripts[script]).toContain(
+        `docker compose --project-directory . -f ${DEV_COMPOSE_FILE} `,
+      );
+    }
+  });
+
   test("the snmp-simulator quick start builds all three of its containers", () => {
     const builds: Array<ImageBuild> = imageBuilds.filter(
       (build: ImageBuild): boolean => {
@@ -398,8 +430,8 @@ describe("The .gitignore Dockerfile rules stay pointed at the right files", () =
       });
 
     expect(renderedDockerfiles.length).toBeGreaterThanOrEqual(10);
-    expect(renderedDockerfiles).toContain("App/Dockerfile");
-    expect(renderedDockerfiles).toContain("Probe/Dockerfile");
+    expect(renderedDockerfiles).toContain("packages/App/Dockerfile");
+    expect(renderedDockerfiles).toContain("packages/Probe/Dockerfile");
 
     for (const dockerfile of renderedDockerfiles) {
       expect({
@@ -435,13 +467,15 @@ describe("The .gitignore Dockerfile rules stay pointed at the right files", () =
     expect(templatesUnderExamples).toEqual([]);
   });
 
-  test("configure.sh is what renders the ignored Dockerfiles", () => {
+  test("Scripts/Install/configure.sh is what renders the ignored Dockerfiles", () => {
     /*
      * Ties the reasoning above to the code that makes it true. If the render
      * step moves or stops running over every template, the "ignored because
      * generated" justification stops holding and this file's premise is stale.
      */
-    const configureScript: string = readRepoFile("configure.sh");
+    const configureScript: string = readRepoFile(
+      path.join("Scripts", "Install", "configure.sh"),
+    );
 
     expect(configureScript).toContain('-name "Dockerfile.tpl"');
     expect(configureScript).toContain("gomplate");