@graphql-hive/federation-gateway-audit 0.0.0 → 0.0.1

package/src/cli.ts

@@ -0,0 +1,700 @@
+#!/usr/bin/env node
+
+process.removeAllListeners("warning");
+
+import {
+  writeFileSync,
+  createWriteStream,
+  mkdirSync,
+  existsSync,
+  readFileSync,
+} from "node:fs";
+import retry from "async-retry";
+import yargs from "yargs";
+import waitOn from "wait-on";
+import getPort from "get-port";
+import detectPort from "detect-port";
+import { spawn } from "node:child_process";
+import { hideBin } from "yargs/helpers";
+import { run } from "node:test";
+import { junit, tap } from "node:test/reporters";
+import { styleText } from "node:util";
+import { killPortProcess } from "kill-port-process";
+import { dirname, extname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { serve } from "./index.js";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const ext = extname(fileURLToPath(import.meta.url));
+
+const defaultPort = 4200;
+
+function readPort(address: string) {
+  return parseInt(new URL(address).port, 10);
+}
+
+function resolvePath(
+  argv: {
+    cwd: string;
+  },
+  path: string,
+) {
+  if (path.startsWith("/")) {
+    return path;
+  }
+  return join(argv.cwd, path);
+}
+
+yargs(hideBin(process.argv))
+  .scriptName("graphql-federation-audit")
+  .epilogue(
+    "for more information, find our manual at https://github.com/the-guild-org/federation-compatibility",
+  )
+  .version(readVersion() ?? "local")
+  .recommendCommands()
+  .option("cwd", {
+    describe: "Change the current working directory",
+    type: "string",
+    default: process.cwd(),
+  })
+  .command(
+    "serve",
+    "start the server",
+    (yargs) => {
+      return yargs.option("port", {
+        describe: "port to bind on",
+        default: defaultPort,
+      });
+    },
+    async (argv) => {
+      await serve(argv.port);
+      console.log("Server started on port", argv.port);
+    },
+  )
+  .command(
+    "supergraph",
+    "fetch a supergraph.graphql for a test group",
+    (yargs) => {
+      return yargs
+        .option("test", {
+          describe: "test group id",
+          type: "string",
+        })
+        .option("port", {
+          describe: "port to bind on",
+          default: defaultPort,
+        })
+        .demandOption("test");
+    },
+    async (argv) => {
+      const res = await fetch(
+        `http://localhost:${argv.port}/${argv.test}/supergraph`,
+      );
+
+      if (!res.ok) {
+        process.stderr.write("Failed to fetch supergraph");
+        process.stderr.write(`HTTP ${res.status} ${res.statusText}`);
+        process.stderr.write('Body: "' + (await res.text()) + '"');
+        process.exit(1);
+      }
+
+      writeFileSync(resolvePath(argv, "supergraph.graphql"), await res.text());
+      process.exit(0);
+    },
+  )
+  .command(
+    "subgraphs",
+    "fetch a subgraphs.json for a test group",
+    (yargs) => {
+      return yargs
+        .option("test", {
+          describe: "test group id",
+          type: "string",
+        })
+        .option("port", {
+          describe: "port to bind on",
+          default: defaultPort,
+        })
+        .demandOption("test");
+    },
+    async (argv) => {
+      const endpoint = `http://localhost:${argv.port}/${argv.test}/subgraphs`;
+      const res = await fetch(endpoint, {
+        headers: {
+          accept: "application/json",
+        },
+      });
+
+      if (!res.ok) {
+        process.stderr.write("Failed to fetch subgraphs: " + endpoint);
+        process.stderr.write(`HTTP ${res.status} ${res.statusText}`);
+        process.stderr.write('Body: "' + (await res.text()) + '"');
+        process.exit(1);
+      }
+
+      writeFileSync(resolvePath(argv, "subgraphs.json"), await res.text());
+      process.exit(0);
+    },
+  )
+  .command(
+    "start",
+    "start a gateway for a requested test group",
+    (yargs) => {
+      return yargs
+        .option("test", {
+          describe: "Test group id",
+          type: "string",
+        })
+        .option("run-script", {
+          describe: "Path to a bash script to run before each test",
+          type: "string",
+        })
+        .option("graphql", {
+          describe: "GraphQL endpoint serving the supergraph",
+          type: "string",
+        })
+        .option("healthcheck", {
+          describe: "Health check endpoint",
+          type: "string",
+        })
+        .option("port", {
+          describe: "Port to bind on",
+          default: defaultPort,
+        })
+        .demandOption("test")
+        .demandOption("graphql")
+        .demandOption("healthcheck")
+        .demandOption("run-script");
+    },
+    async (argv) => {
+      const abortSignal = new AbortController();
+      process.once("SIGINT", () => {
+        if (!abortSignal.signal.aborted) {
+          abortSignal.abort();
+        }
+      });
+      process.once("SIGTERM", () => {
+        if (!abortSignal.signal.aborted) {
+          abortSignal.abort();
+        }
+      });
+
+      const port = argv.port ?? (await getPort());
+      await serve(port);
+
+      process.stdout.write("\n");
+
+      await killPortIfRunning(readPort(argv.graphql)).catch(() => {});
+
+      const gatewayExit = Promise.withResolvers<void>();
+      let gatewayExited = false;
+      const gateway = spawn("sh", [argv.runScript, argv.test], {
+        signal: abortSignal.signal,
+        stdio: "inherit",
+        cwd: dirname(resolvePath(argv, argv.runScript)),
+      });
+
+      gateway.on("error", (err) => {
+        if (err.message.includes("aborted")) {
+          return;
+        }
+
+        process.stderr.write(err.message);
+      });
+
+      gateway.once("exit", () => {
+        gatewayExited = true;
+        gatewayExit.resolve();
+      });
+
+      await gatewayExit.promise;
+    },
+  )
+  .command(
+    "test-suite",
+    "run a test group",
+    (yargs) => {
+      return yargs
+        .option("test", {
+          describe: "Test group id",
+          type: "string",
+        })
+        .option("run-script", {
+          describe: "Path to a bash script to run before each test",
+          type: "string",
+        })
+        .option("graphql", {
+          describe: "GraphQL endpoint serving the supergraph",
+          type: "string",
+        })
+        .option("healthcheck", {
+          describe: "Health check endpoint",
+          type: "string",
+        })
+        .option("port", {
+          describe: "Port to bind on",
+          default: defaultPort,
+        })
+        .option("no-server", {
+          describe: "Skip starting the server",
+          type: "boolean",
+          default: false,
+        })
+        .option("reporter", {
+          describe: "Choose a reporter",
+          choices: ["dot", "tap"],
+          default: "tap",
+        })
+        .option("junit", {
+          describe: "Write test results to a JUnit XML file",
+          type: "boolean",
+        })
+        .demandOption("test")
+        .demandOption("graphql")
+        .demandOption("healthcheck")
+        .demandOption("run-script");
+    },
+    async (argv) => {
+      const abortSignal = new AbortController();
+      process.once("SIGINT", () => {
+        if (!abortSignal.signal.aborted) {
+          abortSignal.abort();
+        }
+      });
+      process.once("SIGTERM", () => {
+        if (!abortSignal.signal.aborted) {
+          abortSignal.abort();
+        }
+      });
+
+      const port = argv.port ?? (await getPort());
+      if (!argv["no-server"]) {
+        await serve(port);
+      }
+
+      process.stdout.write("\n");
+
+      if (!existsSync(resolvePath(argv, "./logs"))) {
+        mkdirSync(resolvePath(argv, "./logs"));
+      }
+
+      await killPortIfRunning(readPort(argv.graphql)).catch(() => {});
+      const logStream = createWriteStream(
+        resolvePath(argv, `./logs/${argv.test}-gateway.log`),
+        {
+          flags: "w+",
+        },
+      );
+
+      const gatewayExit = Promise.withResolvers<void>();
+      let gatewayExited = false;
+      const gateway = spawn("sh", [argv.runScript, argv.test], {
+        signal: abortSignal.signal,
+        stdio: "pipe",
+        cwd: dirname(resolvePath(argv, argv.runScript)),
+      });
+
+      gateway.once("exit", () => {
+        gatewayExited = true;
+        gatewayExit.resolve();
+      });
+
+      gateway.stdout.pipe(logStream);
+      gateway.stderr.pipe(logStream);
+
+      const result = await runTest({
+        ...argv,
+        reporter: argv.reporter === "tap" ? "tap" : "dot",
+        junit: argv.junit,
+        port,
+      });
+
+      if (!gatewayExited) {
+        gateway.kill();
+      }
+      await gatewayExit.promise;
+      process.stdout.write("\n");
+
+      if (result.includes("X")) {
+        process.exit(1);
+      } else {
+        process.exit(0);
+      }
+    },
+  )
+  .command(
+    "test",
+    "run all test groups in a sequence",
+    (yargs) => {
+      return yargs
+        .option("run-script", {
+          describe: "Path to a bash script to run before each test",
+          type: "string",
+        })
+        .option("no-server", {
+          describe: "Skip starting the server",
+          type: "boolean",
+          default: false,
+        })
+        .option("graphql", {
+          describe: "GraphQL endpoint serving the supergraph",
+          type: "string",
+        })
+        .option("healthcheck", {
+          describe: "Health check endpoint",
+          type: "string",
+        })
+        .option("port", {
+          describe: "Port to bind on",
+          default: defaultPort,
+        })
+        .option("reporter", {
+          describe: "Choose a reporter",
+          choices: ["dot", "tap"],
+          default: "dot",
+        })
+        .option("junit", {
+          describe: "Write test results to a JUnit XML file",
+          type: "boolean",
+        })
+        .option("write", {
+          describe: "Write test results to a file",
+          type: "string",
+          default: "results.txt",
+        })
+        .option("exit-on-fail", {
+          describe: "Exit with status 1 if any test fails",
+          type: "boolean",
+          default: false,
+        })
+        .demandOption("graphql")
+        .demandOption("healthcheck")
+        .demandOption("run-script");
+    },
+    async (argv) => {
+      const abortSignal = new AbortController();
+      process.once("SIGINT", () => {
+        if (!abortSignal.signal.aborted) {
+          abortSignal.abort();
+        }
+      });
+      process.once("SIGTERM", () => {
+        if (!abortSignal.signal.aborted) {
+          abortSignal.abort();
+        }
+      });
+
+      const port = argv.port ?? (await getPort());
+
+      if (!argv["no-server"]) {
+        await serve(port);
+      }
+
+      const ids = await fetch(`http://localhost:${port}/ids`, {
+        signal: abortSignal.signal,
+      }).then((res) => res.json() as Promise<string[]>);
+
+      const results: Array<{
+        id: string;
+        result: Array<"." | "X">;
+      }> = [];
+
+      process.stdout.write("Running " + ids.length + " test suites\n");
+
+      if (!existsSync(resolvePath(argv, "./logs"))) {
+        mkdirSync(resolvePath(argv, "./logs"));
+      }
+
+      process.stdout.write("\n");
+      for await (const id of ids) {
+        await killPortIfRunning(readPort(argv.graphql)).catch(() => {});
+        const logStream = createWriteStream(
+          resolvePath(argv, `./logs/${id}-gateway.log`),
+          {
+            flags: "w+",
+          },
+        );
+
+        const gatewayExit = Promise.withResolvers<void>();
+        let gatewayExited = false;
+        const gateway = spawn("sh", [argv.runScript, id], {
+          signal: abortSignal.signal,
+          stdio: "pipe",
+          cwd: dirname(resolvePath(argv, argv.runScript)),
+        });
+
+        gateway.once("exit", () => {
+          gatewayExited = true;
+          gatewayExit.resolve();
+        });
+
+        gateway.stdout.pipe(logStream);
+        gateway.stderr.pipe(logStream);
+
+        const result = await runTest({
+          test: id,
+          graphql: argv.graphql,
+          healthcheck: argv.healthcheck,
+          port: argv.port,
+          reporter: argv.reporter === "tap" ? "tap" : "dot",
+          junit: argv.junit,
+          cwd: argv.cwd,
+        });
+        results.push({ id, result });
+
+        if (!gatewayExited) {
+          gateway.kill();
+        }
+        await gatewayExit.promise;
+      }
+
+      let total = 0;
+      let passed = 0;
+      let failed = 0;
+      for (const { result } of results) {
+        for (const test of result) {
+          if (test === ".") {
+            total++;
+            passed++;
+          } else if (test === "X") {
+            total++;
+            failed++;
+          }
+        }
+      }
+
+      process.stdout.write("\n\n\n");
+
+      process.stdout.write(styleText("bold", "Results") + "\n");
+      process.stdout.write("-----------\n");
+      process.stdout.write(`Total:  ${total}\n`);
+      process.stdout.write(
+        `Passed: ${styleText("greenBright", passed + "")}\n`,
+      );
+      if (failed > 0) {
+        process.stdout.write(
+          `Failed: ${styleText("redBright", failed + "")}\n`,
+        );
+      }
+      process.stdout.write("\n");
+
+      if (failed > 0) {
+        process.stdout.write(
+          styleText("redBright", "Your gateway is not fully compatible\n"),
+        );
+      } else {
+        process.stdout.write(
+          styleText("greenBright", "Your gateway is fully compatible\n"),
+        );
+      }
+
+      writeFileSync(
+        resolvePath(argv, argv.write),
+        results
+          .flatMap((r) => [r.id, r.result.join("")])
+          .concat([
+            "",
+            "---",
+            `Total: ${total}`,
+            `Passed: ${passed}`,
+            `Failed: ${failed}`,
+          ])
+          .join("\n"),
+      );
+
+      if (argv["exit-on-fail"] && failed > 0) {
+        process.exit(1);
+      }
+      process.exit(0);
+    },
+  )
+  .demandCommand(1)
+  .parse();
+
+async function runTest(args: {
+  test: string;
+  graphql: string;
+  healthcheck?: string;
+  port: number;
+  reporter?: "dot" | "tap";
+  junit?: boolean;
+  cwd: string;
+}): Promise<Array<"." | "X">> {
+  process.stdout.write(`${args.test}\n`);
+  process.env.TESTS_ENDPOINT = `http://localhost:${args.port}/${args.test}/tests`;
+  process.env.GRAPHQL_ENDPOINT = args.graphql;
+  process.env.TEST_SUITE = args.test;
+
+  const logStream = createWriteStream(
+    resolvePath({ cwd: args.cwd }, `./logs/${args.test}-tests.log`),
+    {
+      flags: "w+",
+    },
+  );
+
+  if (args.healthcheck) {
+    try {
+      await waitOnGraphQL({
+        healthcheck: args.healthcheck,
+        graphql: args.graphql,
+      });
+    } catch (err) {
+      logStream.write("\nHealth check failed\n");
+    }
+  }
+
+  const { resolve, promise } = Promise.withResolvers<Array<"." | "X">>();
+  const dotan = createDotReporter(resolve);
+
+  const testStream = run({
+    concurrency: 1,
+    files: [join(__dirname, `test${ext}`)],
+  });
+
+  testStream.compose(args.reporter === "tap" ? tap : dot).pipe(process.stdout);
+  testStream.compose(tap).pipe(logStream);
+  testStream.compose(dotan);
+
+  if (args.junit) {
+    const reportsDir = resolvePath({ cwd: args.cwd }, "reports");
+    if (!existsSync(reportsDir)) {
+      mkdirSync(reportsDir, { recursive: true });
+    }
+    const junitPath = join(reportsDir, `${args.test}.xml`);
+    const junitStream = createWriteStream(junitPath, {
+      flags: "w+",
+    });
+    testStream.compose(junit).pipe(junitStream);
+  }
+
+  return promise;
+}
+
+function createDotReporter(resolve: (value: Array<"." | "X">) => void) {
+  const report: Array<"." | "X"> = [];
+  return async function* dot(source: Parameters<typeof tap>[0]) {
+    for await (const { type, data } of source) {
+      if (data != null && "details" in data && data.details?.type === "suite") {
+        continue;
+      }
+      if (type === "test:pass") {
+        report.push(".");
+      }
+      if (type === "test:fail") {
+        report.push("X");
+      }
+    }
+    resolve(report);
+  };
+}
+
+async function* dot(source: Parameters<typeof tap>[0]) {
+  let count = 0;
+  let columns = getLineLength();
+  const failedTests = [];
+  for await (const { type, data } of source) {
+    if (data != null && "details" in data && data.details?.type === "suite") {
+      continue;
+    }
+    if (type === "test:pass") {
+      yield styleText("green", ".");
+    }
+    if (type === "test:fail") {
+      yield styleText("red", "X");
+      failedTests.push(data);
+    }
+    if ((type === "test:fail" || type === "test:pass") && ++count === columns) {
+      yield "\n";
+      // Getting again in case the terminal was resized.
+      columns = getLineLength();
+      count = 0;
+    }
+  }
+  yield "\n";
+}
+
+function getLineLength() {
+  return Math.max(process.stdout.columns ?? 20, 20);
+}
+
+function readVersion(): string | undefined {
+  try {
+    // src/cli.ts
+    return (
+      "v" +
+      JSON.parse(readFileSync(join(__dirname, "../package.json"), "utf-8"))
+        .version
+    );
+  } catch {
+    //
+  }
+
+  try {
+    // dist/cli.js
+    return (
+      "v" +
+      JSON.parse(readFileSync(join(__dirname, "./package.json"), "utf-8"))
+        .version
+    );
+  } catch {
+    //
+  }
+}
+
+async function waitOnGraphQL(endpoints: {
+  healthcheck: string;
+  graphql: string;
+}) {
+  const timeout = process.env["GATEWAY_TIMEOUT"]
+    ? parseInt(process.env["GATEWAY_TIMEOUT"])
+    : 5000;
+  await waitOn({
+    // Make sure the health check is a GET request
+    resources: [endpoints.healthcheck.replace("http://", "http-get://")],
+    timeout,
+    httpTimeout: 200,
+    log: false,
+    verbose: false,
+  });
+
+  await retry(fetchTypename, {
+    maxRetryTime: 2_000,
+    retries: 20,
+    maxTimeout: 500,
+    minTimeout: 100,
+  });
+
+  async function fetchTypename() {
+    const response = await fetch(endpoints.graphql, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        accept: "application/json",
+      },
+      body: JSON.stringify({
+        operationName: "ping",
+        query: /* GraphQL */ `
+          query ping {
+            __typename
+          }
+        `,
+      }),
+    });
+
+    if (response.ok) {
+      const json = await response.json();
+
+      if (json.data.__typename === "Query") {
+        return;
+      }
+    }
+
+    throw new Error("Failed to fetch __typename");
+  }
+}
+
+async function killPortIfRunning(port: number) {
+  if (await detectPort(port)) {
+    await killPortProcess(port);
+  }
+}

package/src/env.ts

@@ -0,0 +1,13 @@
+let env_loaded = false;
+
+export function shouldPunishForPoorPlans() {
+  if (!env_loaded) {
+    try {
+      process.loadEnvFile();
+    } catch (e) {
+      // Ignore if `.env` is not available
+    }
+    env_loaded = true;
+  }
+  return process.env.PUNISH_FOR_POOR_PLANS === "1";
+}