@graphql-hive/federation-gateway-audit 0.0.0 → 0.0.1

package/src/index.ts

@@ -0,0 +1,262 @@
+import {
+  createRouter as createFetsRouter,
+  Response,
+  Router,
+  RouterComponentsBase,
+  RouterSDK,
+} from "fets";
+import { createServer } from "node:http";
+
+async function getTestCases(router: ReturnType<typeof createRouter>) {
+  const testCases = await Promise.all(
+    [
+      import("./test-suites/union-intersection/index.js"),
+      import("./test-suites/simple-entity-call/index.js"),
+      import("./test-suites/complex-entity-call/index.js"),
+      import("./test-suites/mysterious-external/index.js"),
+      import("./test-suites/simple-requires-provides/index.js"),
+      import("./test-suites/override-type-interface/index.js"),
+      import("./test-suites/simple-interface-object/index.js"),
+      import("./test-suites/simple-override/index.js"),
+      import("./test-suites/unavailable-override/index.js"),
+      import("./test-suites/override-with-requires/index.js"),
+      import("./test-suites/node/index.js"),
+      import("./test-suites/simple-inaccessible/index.js"),
+      import("./test-suites/enum-intersection/index.js"),
+      import("./test-suites/input-object-intersection/index.js"),
+      import("./test-suites/requires-with-fragments/index.js"),
+      import("./test-suites/child-type-mismatch/index.js"),
+      import("./test-suites/non-resolvable-interface-object/index.js"),
+      import("./test-suites/interface-object-with-requires/index.js"),
+      import("./test-suites/interface-object-indirect-extension/index.js"),
+      import("./test-suites/requires-interface/index.js"),
+      import("./test-suites/fed1-external-extends/index.js"),
+      import("./test-suites/fed2-external-extends/index.js"),
+      import("./test-suites/fed1-external-extension/index.js"),
+      import("./test-suites/fed2-external-extension/index.js"),
+      import("./test-suites/parent-entity-call/index.js"),
+      import("./test-suites/corrupted-supergraph-node-id/index.js"),
+      import("./test-suites/parent-entity-call-complex/index.js"),
+      import("./test-suites/shared-root/index.js"),
+      import("./test-suites/nested-provides/index.js"),
+      import("./test-suites/provides-on-interface/index.js"),
+      import("./test-suites/provides-on-union/index.js"),
+      import("./test-suites/requires-requires/index.js"),
+      import("./test-suites/include-skip/index.js"),
+      import("./test-suites/circular-reference-interface/index.js"),
+      import("./test-suites/typename/index.js"),
+      import("./test-suites/union-interface-distributed/index.js"),
+      import("./test-suites/mutations/index.js"),
+      import("./test-suites/abstract-types/index.js"),
+      import("./test-suites/fed1-external-extends-resolvable/index.js"),
+      import("./test-suites/requires-with-argument/index.js"),
+      import("./test-suites/requires-with-argument-conflict/index.js"),
+      import("./test-suites/keys-mashup/index.js"),
+      import("./test-suites/null-keys/index.js"),
+      import("./test-suites/requires-circular/index.js"),
+    ].map((i) => i.then((e) => e.default)),
+  );
+
+  testCases.sort((a, b) => a.id.localeCompare(b.id));
+
+  const registeredNames = new Set<string>();
+
+  for (const testCase of testCases) {
+    if (registeredNames.has(testCase.id)) {
+      throw new Error(`Duplicate test case id: ${testCase.id}`);
+    }
+
+    registeredNames.add(testCase.id);
+    testCase.createRoutes(router);
+  }
+
+  return testCases;
+}
+
+export function serve(port: number) {
+  const router = createRouter();
+  const { resolve, reject, promise } = Promise.withResolvers<{
+    close: VoidFunction;
+  }>();
+
+  let started = false;
+  const server = createServer(router).listen(port, () => {
+    started = true;
+    resolve({
+      close,
+    });
+  });
+
+  server.once("error", (error) => {
+    if (!started) {
+      reject(error);
+    } else {
+      process.stderr.write("Server error: " + String(error));
+    }
+  });
+
+  function close() {
+    if (server.listening) {
+      return new Promise<void>((resolve, reject) => {
+        server.closeAllConnections();
+        server.close((error) => {
+          if (error) {
+            reject(error);
+          } else {
+            resolve();
+          }
+        });
+      });
+    }
+  }
+
+  server.once("close", () => {
+    console.log("Server closed");
+  });
+
+  process.once("exit", () => {
+    close();
+  });
+
+  process.once("SIGINT", () => {
+    close();
+  });
+
+  process.once("SIGTERM", () => {
+    close();
+  });
+
+  return promise;
+}
+
+export function createRouter<TServerContext>(): Router<
+  TServerContext,
+  RouterComponentsBase,
+  RouterSDK
+> {
+  const router = createFetsRouter({
+    landingPage: false,
+    swaggerUI: {
+      endpoint: "/",
+      displayOperationId: false,
+    },
+    openAPI: {
+      info: {
+        title: "Federation Compatibility Test Suite",
+        description:
+          "A test suite for validating Apollo Federation v2 compatibility",
+        contact: {
+          name: "The Guild",
+          url: "https://the-guild.dev",
+          email: "contact@the-guild.dev",
+        },
+      },
+    },
+  })
+    .route({
+      method: "GET",
+      path: "/_health",
+      handler() {
+        return new Response("OK");
+      },
+    })
+    .route({
+      method: "GET",
+      path: "/ids",
+      operationId: "get_ids",
+      description: "A list of test cases",
+      tags: ["root"],
+      schemas: {
+        responses: {
+          200: {
+            type: "array",
+            items: {
+              type: "string",
+            },
+          },
+        },
+      },
+      handler() {
+        return testCases$.then((testCases) =>
+          Response.json(testCases.map((t) => t.id)),
+        );
+      },
+    })
+    .route({
+      method: "GET",
+      path: "/supergraphs",
+      description: "A list of supergraph endpoints",
+      tags: ["root"],
+      schemas: {
+        responses: {
+          200: {
+            type: "array",
+            items: {
+              type: "string",
+            },
+          },
+        },
+      },
+      handler(req) {
+        return testCases$.then((testCases) =>
+          Response.json(
+            testCases.map(
+              ({ id }) => `${req.parsedUrl.origin}/${id}/supergraph`,
+            ),
+          ),
+        );
+      },
+    })
+    .route({
+      method: "GET",
+      path: "/tests",
+      description: "A list of endpoints with tests",
+      tags: ["root"],
+      schemas: {
+        responses: {
+          200: {
+            type: "array",
+            items: {
+              type: "string",
+            },
+          },
+        },
+      },
+      handler(req) {
+        return testCases$.then((testCases) =>
+          Response.json(
+            testCases.map(({ id }) => `${req.parsedUrl.origin}/${id}/tests`),
+          ),
+        );
+      },
+    })
+    .route({
+      method: "GET",
+      path: "/subgraphs",
+      description: "A list of endpoints with subgraphs",
+      tags: ["root"],
+      schemas: {
+        responses: {
+          200: {
+            type: "array",
+            items: {
+              type: "string",
+            },
+          },
+        },
+      },
+      handler(req) {
+        return testCases$.then((testCases) =>
+          Response.json(
+            testCases.map(
+              ({ id }) => `${req.parsedUrl.origin}/${id}/subgraphs`,
+            ),
+          ),
+        );
+      },
+    });
+
+  const testCases$ = getTestCases(router);
+
+  return router;
+}

package/src/subgraph.ts

@@ -0,0 +1,113 @@
+import { createRouter, Response } from "fets";
+import { parse } from "graphql";
+import { buildSubgraphSchema } from "@apollo/subgraph";
+import { createYoga } from "graphql-yoga";
+
+export function createSubgraph(
+  name: string,
+  schemaParameters: {
+    typeDefs: string;
+    resolvers: any;
+  },
+) {
+  let schema: ReturnType<typeof buildSubgraphSchema>;
+
+  function lazySchema() {
+    if (!schema) {
+      schema = buildSubgraphSchema({
+        typeDefs: parse(schemaParameters.typeDefs),
+        resolvers: schemaParameters.resolvers,
+      });
+    }
+
+    return schema;
+  }
+
+  function lazyYoga() {
+    if (!yoga) {
+      yoga = createYoga({
+        schema: lazySchema(),
+        graphqlEndpoint: "*",
+        logging: false,
+        maskedErrors: false,
+      });
+    }
+
+    return yoga;
+  }
+
+  let yoga: ReturnType<typeof createYoga>;
+
+  return {
+    createRoutes(testCaseId: string, router: ReturnType<typeof createRouter>) {
+      router.route({
+        method: "GET",
+        path: `/${testCaseId}/${name}`,
+        tags: [testCaseId],
+        operationId: "GraphiQL",
+        schemas: {
+          request: {
+            query: {
+              type: "object",
+              properties: {
+                query: { type: "string" },
+              },
+              additionalProperties: true,
+              required: ["query"],
+            },
+          },
+        },
+        handler(req) {
+          return lazyYoga()(req) as Promise<Response>;
+        },
+      });
+
+      router.route({
+        method: "POST",
+        path: `/${testCaseId}/${name}`,
+        tags: [testCaseId],
+        operationId: `"${name}" subgraph`,
+        description: "GraphQL endpoint for the subgraph",
+        schemas: {
+          request: {
+            json: {
+              type: "object",
+              properties: {
+                query: { type: "string" },
+                variables: { type: "object" },
+              },
+              additionalProperties: true,
+              required: ["query"],
+            },
+          },
+          responses: {
+            200: {
+              type: "object",
+              properties: {
+                data: { type: "object" },
+                errors: {
+                  type: "array",
+                  items: {
+                    type: "object",
+                    properties: {
+                      message: { type: "string" },
+                    },
+                    required: ["message"],
+                    additionalProperties: false,
+                  },
+                },
+              },
+              required: ["data"],
+              additionalProperties: false,
+            },
+          },
+        },
+        handler(req) {
+          return lazyYoga()(req) as Promise<any>;
+        },
+      });
+    },
+    name,
+    typeDefs: schemaParameters.typeDefs,
+  };
+}

package/src/summary.ts

@@ -0,0 +1,237 @@
+import { readdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import prettier from "prettier";
+
+const gatewayResults = readdirSync("./gateways")
+  .filter((file) => statSync(join("./gateways", file)).isDirectory())
+  .map((id) => {
+    const gatewayDetails = JSON.parse(
+      readFileSync(join("./gateways", id, "gateway.json"), "utf-8"),
+    ) as { name: string; repository: string; website: string };
+
+    const lines = readFileSync(
+      join("./gateways", id, "results.txt"),
+      "utf-8",
+    ).split("\n");
+
+    const groups = {
+      total: 0,
+      passed: 0,
+      failed: 0,
+    };
+    const tests = {
+      total: 0,
+      passed: 0,
+      failed: 0,
+    };
+    const dot: Array<{
+      group: string;
+      results: string;
+    }> = [];
+
+    let collectingDetails = true;
+    let previousGroupName: string | null = null;
+    for (const line of lines) {
+      if (collectingDetails) {
+        if (line.charAt(0) === "." || line.charAt(0) === "X") {
+          groups.total++;
+
+          if (!line.includes("X")) {
+            groups.passed++;
+          }
+
+          if (!previousGroupName) {
+            throw new Error("No previous group found");
+          }
+
+          dot.push({
+            group: previousGroupName,
+            results: line.replaceAll(".", "🟢").replaceAll("X", "❌"),
+          });
+        } else if (line.trim() === "" || line === "---") {
+          collectingDetails = false;
+        } else {
+          previousGroupName = line;
+        }
+      } else if (line.startsWith("Passed:")) {
+        tests.passed = parseInt(line.replace("Passed:", "").trim());
+      } else if (line.startsWith("Failed:")) {
+        tests.failed = parseInt(line.replace("Failed:", "").trim());
+      } else if (line.startsWith("Total:")) {
+        tests.total = parseInt(line.replace("Total:", "").trim());
+      }
+    }
+
+    groups.failed = groups.total - groups.passed;
+
+    const name = gatewayDetails?.name ?? id;
+
+    return {
+      id,
+      name,
+      repository: gatewayDetails.repository,
+      website: gatewayDetails.website,
+      groups,
+      tests,
+      dot,
+    };
+  });
+
+gatewayResults.sort((a, b) => {
+  const diff = b.tests.passed - a.tests.passed;
+
+  if (diff !== 0) {
+    // 9 -> 1
+    return diff;
+  }
+
+  // A -> Z
+  return a.name.localeCompare(b.name);
+});
+
+function printResult(
+  result: { passed: number; failed: number },
+  format: "md" | "html",
+) {
+  const scores: string[] = [];
+
+  if (result.passed) {
+    if (format === "md") {
+      scores.push(`🟢 ${result.passed}`);
+    } else {
+      scores.push(
+        `<span class="text-emerald-700 mr-2">✓ ${result.passed}</span>`,
+      );
+    }
+  }
+
+  if (result.failed) {
+    if (format === "md") {
+      scores.push(`❌ ${result.failed}`);
+    } else {
+      scores.push(`<span class="text-red-700">✗ ${result.failed}</span>`);
+    }
+  }
+
+  return scores.join("  ");
+}
+
+let tableMd = `|  Gateway   | Compatibility | Test Cases | Test Suites |
+| :---------------------------: | :-----------: | :----------: | :---------: |`;
+let rowsHtml = ``;
+let testDetailsMd = "";
+
+const jsonReport: Array<{
+  name: string;
+  cases: { passed: number; failed: number };
+  suites: { passed: number; failed: number };
+}> = [];
+
+for (const gateway of gatewayResults) {
+  jsonReport.push({
+    name: gateway.name,
+    cases: gateway.tests,
+    suites: gateway.groups,
+  });
+
+  const score = ((gateway.tests.passed * 100) / gateway.tests.total).toFixed(2);
+  const roundedScore = Math.round(
+    (gateway.tests.passed * 100) / gateway.tests.total,
+  );
+  tableMd += `\n| [${gateway.name}](${gateway.website}) | ${score}% |  ${printResult(gateway.tests, "md")} | ${printResult(gateway.groups, "md")} |`;
+
+  rowsHtml += `
+    <tr class="border-b transition-colors hover:bg-gray-100/50">
+      <td class="p-4 align-middle font-medium  border-l-2 border-${roundedScore === 100 ? "emerald" : roundedScore >= 75 ? "yellow" : "red"}-500">
+        <a href="${gateway.website}" class="hover:underline" title="Visit ${gateway.name} website">
+          ${gateway.name}
+        </a>
+      </td>
+      <td class="p-4 align-middle font-semibold">${score}%</td>
+      <td class="p-4 align-middle">
+        ${printResult(gateway.tests, "html")}
+      </td>
+      <td class="p-4 align-middle">
+        ${printResult(gateway.groups, "html")}
+      </td>
+      <td class="p-4 align-middle"><a href="https://github.com/the-guild-org/federation-compatibility/tree/main/REPORT.md#${gateway.id}" class="text-sky-700 hover:underline">View report</a></td>
+    </tr>
+  `;
+
+  testDetailsMd += `\n<a id="${gateway.id}"></a>
+### ${gateway.name}
+
+- [Repository](${gateway.repository})
+- [Website](${gateway.website})
+
+<details>
+<summary>Results</summary>\n`;
+
+  for (const { group, results } of gateway.dot) {
+    testDetailsMd += `<a href="./src/test-suites/${group}">${group}</a>\n<pre>${results}</pre>\n`;
+  }
+
+  testDetailsMd += `</details>\n`;
+}
+
+const reportMd = `# Compatibility Results
+
+## Summary
+
+${tableMd}
+
+## Detailed Results
+
+Take a closer look at the results for each gateway.
+
+You can look at the full list of tests [here](./src/test-suites/). Every test id corresponds to a directory in the \`src/test-suites\` folder.
+${testDetailsMd}`;
+
+await writeFormatted("./REPORT.md", reportMd);
+
+const readmeMd = readFileSync("./README.md", "utf-8");
+const indexHtml = readFileSync("./website/index.html", "utf-8");
+
+const startTag = "<!-- gateways:start -->";
+const endTag = "<!-- gateways:end -->";
+const mdStartAt = readmeMd.indexOf(startTag);
+const mdEndAt = readmeMd.indexOf(endTag);
+const htmlStartAt = indexHtml.indexOf(startTag);
+const htmlEndAt = indexHtml.indexOf(endTag);
+
+const newReadmeMd =
+  readmeMd.substring(0, mdStartAt) +
+  "\n\n" +
+  startTag +
+  "\n\n" +
+  tableMd +
+  "\n\n" +
+  readmeMd.substring(mdEndAt);
+
+await writeFormatted("./README.md", newReadmeMd);
+
+const newIndexHtml =
+  indexHtml.substring(0, htmlStartAt) +
+  "\n" +
+  startTag +
+  "\n" +
+  rowsHtml +
+  "\n" +
+  indexHtml.substring(htmlEndAt);
+
+writeFormatted("./website/index.html", newIndexHtml);
+
+writeFileSync(
+  "./website/data.json",
+  JSON.stringify(jsonReport, null, 2),
+  "utf-8",
+);
+
+async function writeFormatted(filename: string, content: string) {
+  writeFileSync(
+    filename,
+    await prettier.format(content, {
+      filepath: filename,
+    }),
+  );
+}