@sentry/roach 0.0.0-stage → 0.2.0

package/src/action.ts

@@ -0,0 +1,81 @@
+/**
+ * The GitHub Action of Roach (`action.yml`). See "Use the GitHub Action" in
+ * `README.md`.
+ *
+ * It starts a run, runs the command of the step with the proxy variables of
+ * the run, and ends the run. The run config comes from `roach.json`
+ * (`config.ts`).
+ * The runner gives each input as `INPUT_<NAME>`.
+ */
+import { spawn } from "node:child_process";
+import { startRemoteRun } from "./client.ts";
+import { readEnv, readRoachFile, runConfigOf } from "./config.ts";
+import { writeRunReport } from "./report.ts";
+
+/**
+ * Variables that the command does not get: the inputs, which have the token,
+ * and the proxy variables of the job.
+ */
+const HIDDEN_VARIABLE = /^(INPUT_.*|https?_proxy|no_proxy|all_proxy)$/i;
+
+/** Run the command in bash, and return its exit code. */
+function runCommand(command: string, env: NodeJS.ProcessEnv): Promise<number> {
+  // `detached` puts bash and its children in a new process group.
+  const child = spawn("bash", ["-e", "-o", "pipefail", "-c", command], {
+    detached: true,
+    env,
+    stdio: "inherit",
+  });
+  // A canceled job sends a signal. Bash does not pass it to its children, so
+  // send it to the whole process group. Then end the run.
+  const stop = (signal: NodeJS.Signals) => {
+    try {
+      process.kill(-child.pid!, signal);
+    } catch {
+      // The process group has already stopped.
+    }
+  };
+  process.on("SIGINT", stop);
+  process.on("SIGTERM", stop);
+  return new Promise((resolve, reject) => {
+    child.once("error", reject);
+    child.once("exit", (code) => {
+      // Stop what the command left in the background, such as `cmd &`. Bash
+      // makes it ignore `SIGINT`, and the run ends next.
+      stop("SIGTERM");
+      resolve(code ?? 1);
+    });
+  });
+}
+
+const command = process.env.INPUT_RUN;
+if (!command) throw new Error("The run input is required");
+// The runner gives an empty string for an input that the step does not set.
+const token = readEnv("INPUT_TOKEN");
+const file = await readRoachFile();
+const config = runConfigOf(file, { canWrite: token !== undefined });
+const run = await startRemoteRun(
+  { url: process.env.INPUT_URL!, token },
+  config,
+  file.noProxy ? { noProxy: file.noProxy } : {},
+);
+// The proxy URL has the run token. Keep it out of the log.
+process.stdout.write(`::add-mask::${run.token}\n`);
+
+let exitCode = 1;
+try {
+  const session = await run.startSession(process.env.GITHUB_JOB ?? "command");
+  exitCode = await runCommand(command, {
+    ...Object.fromEntries(
+      Object.entries(process.env).filter(
+        ([name]) => !HIDDEN_VARIABLE.test(name),
+      ),
+    ),
+    ...run.env,
+  });
+  const { missed } = await session.end(exitCode === 0);
+  if (missed > 0 && exitCode === 0) exitCode = 1;
+} finally {
+  await writeRunReport(`Roach (${config.mode})`, await run.close());
+}
+process.exitCode = exitCode;

package/src/addresses.ts

@@ -0,0 +1,103 @@
+/**
+ * The addresses that the proxy can send requests to.
+ *
+ * A run names its own origins (`service.ts`). So without this check, a run
+ * could make the service send requests into its own network: to the GCP
+ * metadata server, which gives tokens of the VM service account, or to
+ * other hosts in the VPC. The proxy sends requests only to public
+ * addresses.
+ *
+ * The check is on the address that the connection uses, after DNS. A host
+ * name that resolves to a public address for the check and to a private
+ * one for the connection (DNS rebinding) thus does not get through.
+ */
+import { lookup as dnsLookup, type LookupAddress } from "node:dns";
+import { BlockList, isIP, type LookupFunction } from "node:net";
+
+/**
+ * Addresses that are not on the public internet: this host, private
+ * networks, link-local (with the metadata server), shared, documentation,
+ * multicast, and reserved ranges. IPv6 forms that hold an IPv4 address
+ * (NAT64, 6to4, Teredo) are refused too, because they can lead to a
+ * private IPv4 address. `BlockList` checks IPv4-mapped IPv6 addresses,
+ * such as `::ffff:127.0.0.1`, as IPv4.
+ */
+const NOT_PUBLIC = new BlockList();
+for (const [network, prefix] of [
+  ["0.0.0.0", 8],
+  ["10.0.0.0", 8],
+  ["100.64.0.0", 10],
+  ["127.0.0.0", 8],
+  ["169.254.0.0", 16],
+  ["172.16.0.0", 12],
+  ["192.0.0.0", 24],
+  ["192.0.2.0", 24],
+  ["192.88.99.0", 24],
+  ["192.168.0.0", 16],
+  ["198.18.0.0", 15],
+  ["198.51.100.0", 24],
+  ["203.0.113.0", 24],
+  ["224.0.0.0", 4],
+  ["240.0.0.0", 4],
+] as const) {
+  NOT_PUBLIC.addSubnet(network, prefix, "ipv4");
+}
+for (const [network, prefix] of [
+  ["::", 96],
+  ["64:ff9b::", 96],
+  ["64:ff9b:1::", 48],
+  ["100::", 64],
+  ["2001::", 23],
+  ["2001:db8::", 32],
+  ["2002::", 16],
+  ["fc00::", 7],
+  ["fe80::", 10],
+  ["fec0::", 10],
+  ["ff00::", 8],
+] as const) {
+  NOT_PUBLIC.addSubnet(network, prefix, "ipv6");
+}
+
+/** Check that an IP address is on the public internet. */
+export function isPublicAddress(address: string): boolean {
+  const family = isIP(address);
+  if (family === 0) return false;
+  return !NOT_PUBLIC.check(address, family === 6 ? "ipv6" : "ipv4");
+}
+
+/** The error of a request to an address that is not public. */
+export const notPublicError = (host: string, address: string) =>
+  Object.assign(
+    new Error(
+      `Roach does not send requests to ${host}, because ${address} is not a public address`,
+    ),
+    { status: 403, code: "ERR_ROACH_NOT_PUBLIC" },
+  );
+
+/**
+ * A DNS lookup for `http.request()` that fails when the host has an
+ * address that is not public. It checks all addresses of the host, so the
+ * choice of the connection does not matter.
+ */
+export const publicLookup: LookupFunction = (hostname, options, callback) => {
+  dnsLookup(hostname, { ...options, all: true }, (error, addresses) => {
+    if (error) {
+      callback(error, "", 0);
+      return;
+    }
+    const blocked = addresses.find(({ address }) => !isPublicAddress(address));
+    if (blocked) {
+      callback(notPublicError(hostname, blocked.address), "", 0);
+      return;
+    }
+    if (options.all) {
+      // The type has only the single-address form of the callback.
+      (callback as unknown as (e: null, list: LookupAddress[]) => void)(
+        null,
+        addresses,
+      );
+      return;
+    }
+    callback(null, addresses[0]!.address, addresses[0]!.family);
+  });
+};

package/src/certificates.ts

@@ -0,0 +1,169 @@
+/**
+ * The certificate authority of Roach.
+ *
+ * The proxy intercepts HTTPS. It signs one certificate for each host when a
+ * client first connects to that host, and signs it again before it
+ * expires. Runs name their own hosts, so the authority keeps the
+ * certificates of at most `MAX_HOSTS` hosts, and keeps no files for them.
+ * Clients must trust the authority certificate. The service loads
+ * a fixed one, so that clients can trust it across restarts.
+ */
+import { execFile } from "node:child_process";
+import { createHash, randomBytes } from "node:crypto";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { isIP } from "node:net";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import tls from "node:tls";
+
+function openssl(args: string[]): Promise<void> {
+  return new Promise((resolve, reject) => {
+    execFile("openssl", args, (error, _stdout, stderr) => {
+      if (error)
+        reject(new Error(`openssl failed: ${stderr || error.message}`));
+      else resolve();
+    });
+  });
+}
+
+/** A certificate authority that signs a certificate for each host. */
+export interface CertificateAuthority {
+  /** The PEM certificate that clients must trust. */
+  caCert: string;
+  /** The TLS context for a host. Signs its certificate on first use. */
+  contextFor(host: string): Promise<tls.SecureContext>;
+  /** Delete the keys and certificates. */
+  close(): Promise<void>;
+}
+
+/** A PEM certificate authority, such as the one of a deployed service. */
+export interface CertificateAuthorityPem {
+  cert: string;
+  key: string;
+}
+
+/** Days that a host certificate is valid. */
+const HOST_DAYS = 7;
+/** Sign a host again when its certificate is older than this. */
+const RESIGN_MS = (HOST_DAYS - 1) * 24 * 60 * 60 * 1000;
+/** The hosts with a certificate in memory. More hosts drop the oldest one. */
+const MAX_HOSTS = 1000;
+
+/**
+ * Create a certificate authority in a new temporary directory. Without
+ * `pem`, it creates a new authority that is valid for 7 days.
+ */
+export async function createCertificateAuthority(
+  pem?: CertificateAuthorityPem,
+): Promise<CertificateAuthority> {
+  const directory = await mkdtemp(path.join(tmpdir(), "roach-ca-"));
+  const caKey = path.join(directory, "ca.key");
+  const caCertFile = path.join(directory, "ca.crt");
+  const hostKey = path.join(directory, "host.key");
+  if (pem) {
+    await writeFile(caKey, pem.key, { mode: 0o600 });
+    await writeFile(caCertFile, pem.cert);
+  } else {
+    await createAuthority(caKey, caCertFile);
+  }
+  // All hosts share one key. Only their certificates differ.
+  await openssl(["genrsa", "-out", hostKey, "2048"]);
+  const hostKeyPem = await readFile(hostKey, "utf8");
+  const contexts = new Map<
+    string,
+    { signed: number; context: Promise<tls.SecureContext> }
+  >();
+
+  const sign = async (host: string): Promise<tls.SecureContext> => {
+    const name = createHash("sha256").update(host).digest("hex").slice(0, 16);
+    const csr = path.join(directory, `${name}.csr`);
+    const extensions = path.join(directory, `${name}.ext`);
+    const cert = path.join(directory, `${name}.crt`);
+    const bare = host.replace(/^\[|\]$/g, "");
+    const altName = isIP(bare) ? `IP:${bare}` : `DNS:${host}`;
+    await writeFile(
+      extensions,
+      `subjectAltName=${altName}\nextendedKeyUsage=serverAuth\n`,
+    );
+    await openssl([
+      "req",
+      "-new",
+      "-key",
+      hostKey,
+      "-subj",
+      "/CN=Roach host",
+      "-out",
+      csr,
+    ]);
+    await openssl([
+      "x509",
+      "-req",
+      "-in",
+      csr,
+      "-CA",
+      caCertFile,
+      "-CAkey",
+      caKey,
+      "-set_serial",
+      `0x${randomBytes(8).toString("hex")}`,
+      "-days",
+      String(HOST_DAYS),
+      "-extfile",
+      extensions,
+      "-out",
+      cert,
+    ]);
+    const certPem = await readFile(cert, "utf8");
+    await Promise.all(
+      [csr, extensions, cert].map((file) => rm(file, { force: true })),
+    );
+    return tls.createSecureContext({ cert: certPem, key: hostKeyPem });
+  };
+
+  return {
+    caCert: await readFile(caCertFile, "utf8"),
+    contextFor(host) {
+      const now = Date.now();
+      let entry = contexts.get(host);
+      if (!entry || now - entry.signed > RESIGN_MS) {
+        const fresh = { signed: now, context: sign(host) };
+        // A failed signature must not stay in the cache.
+        fresh.context.catch(() => {
+          if (contexts.get(host) === fresh) contexts.delete(host);
+        });
+        contexts.delete(host);
+        contexts.set(host, fresh);
+        // A map keeps the order of insertion, so the first key is the oldest.
+        if (contexts.size > MAX_HOSTS) {
+          contexts.delete(contexts.keys().next().value!);
+        }
+        entry = fresh;
+      }
+      return entry.context;
+    },
+    close: () => rm(directory, { force: true, recursive: true }),
+  };
+}
+
+/** Create a new authority that is valid for 7 days. */
+async function createAuthority(caKey: string, caCertFile: string) {
+  await openssl([
+    "req",
+    "-x509",
+    "-newkey",
+    "rsa:2048",
+    "-nodes",
+    "-keyout",
+    caKey,
+    "-out",
+    caCertFile,
+    "-days",
+    "7",
+    "-subj",
+    "/CN=Roach CA",
+    "-addext",
+    "basicConstraints=critical,CA:TRUE",
+    "-addext",
+    "keyUsage=critical,keyCertSign,cRLSign",
+  ]);
+}

package/src/cli.ts

@@ -0,0 +1,35 @@
+/**
+ * The command line of Roach.
+ *
+ * `cli.ts service <config.json>` starts the shared service (`service.ts`)
+ * with the configuration in the file. It prints the URL of the service as
+ * one JSON line, and stops on SIGINT or SIGTERM. The deployed container
+ * runs this command.
+ */
+import { readFile } from "node:fs/promises";
+import { startRoachService, type RoachServiceConfig } from "./service.ts";
+
+const [command, ...args] = process.argv.slice(2);
+if (command !== "service" || args.length !== 1) {
+  process.stderr.write("Usage: cli.ts service <config.json>\n");
+  process.exit(2);
+}
+
+const config = JSON.parse(
+  await readFile(args[0]!, "utf8"),
+) as RoachServiceConfig;
+const service = await startRoachService(config);
+process.stdout.write(`${JSON.stringify({ url: service.url })}\n`);
+
+/** Stop on SIGINT or SIGTERM. A failed close logs its error and exits 1. */
+const stop = () => {
+  service.close().then(
+    () => process.exit(0),
+    (error: unknown) => {
+      process.stderr.write(`[roach] Close failed: ${String(error)}\n`);
+      process.exit(1);
+    },
+  );
+};
+process.once("SIGINT", stop);
+process.once("SIGTERM", stop);

package/src/client.ts

@@ -0,0 +1,211 @@
+/**
+ * The client of Roach. See `README.md`.
+ *
+ * `startRemoteRun()` starts a run on a Roach service (`service.ts`). It
+ * returns `env`, the variables that send the traffic of a process through
+ * the run. `connectRoach()` controls a run from another process, such as a
+ * test worker.
+ */
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import http from "node:http";
+import https from "node:https";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import {
+  CONTROL_PATH,
+  type OpenedSession,
+  type RecordingStats,
+  type RemoteRun,
+  type RunConfig,
+} from "./types.ts";
+
+/** The open session of one test. */
+export interface RecordingSession {
+  /**
+   * The proxy URL of the session. Send the requests of the test through
+   * it when other sessions of the run can be open at the same time.
+   */
+  url: string;
+  /**
+   * End the session. A passed session writes its new recordings, and a
+   * failed one drops them. `missed` counts the requests of the session that
+   * `replay` mode failed. A test with a miss must fail.
+   */
+  end(passed: boolean): Promise<{ missed: number }>;
+}
+
+/** The control API of a run. */
+export interface RoachControl {
+  /**
+   * Open the session of one test. Many sessions can be open at once. A
+   * request through `url` of the run belongs to the open session only when
+   * one is open, so concurrent tests send through `url` of their session.
+   */
+  startSession(name: string): Promise<RecordingSession>;
+  /** The totals of the run. */
+  stats(): Promise<RecordingStats>;
+}
+
+/**
+ * Call the control API at `base`, and return the JSON of the response.
+ * `token` is the bearer token.
+ */
+function callControl<T>(
+  base: string,
+  token: string | undefined,
+  method: "DELETE" | "GET" | "POST",
+  route: string,
+  body?: unknown,
+): Promise<T> {
+  const payload = body === undefined ? undefined : JSON.stringify(body);
+  const url = route ? `${base}/${route}` : base;
+  const client = url.startsWith("https:") ? https : http;
+  return new Promise((resolve, reject) => {
+    const request = client.request(
+      url,
+      {
+        // Its own agent, so that a proxy agent of the process is not used.
+        agent: false,
+        method,
+        headers: {
+          ...(token ? { authorization: `Bearer ${token}` } : {}),
+          ...(payload ? { "content-type": "application/json" } : {}),
+        },
+      },
+      (response) => {
+        const chunks: Buffer[] = [];
+        response.on("data", (chunk: Buffer) => chunks.push(chunk));
+        response.on("end", () => {
+          const text = Buffer.concat(chunks).toString("utf8");
+          const status = response.statusCode ?? 500;
+          if (status >= 400) {
+            reject(
+              new Error(
+                `Roach ${method} ${route} failed with HTTP ${status}: ${text}`,
+              ),
+            );
+          } else {
+            resolve((text ? JSON.parse(text) : undefined) as T);
+          }
+        });
+        response.on("error", reject);
+      },
+    );
+    request.on("error", reject);
+    request.end(payload);
+  });
+}
+
+/** Control a run, for example from a test worker. */
+export function connectRoach({
+  controlUrl,
+  token,
+}: Pick<RemoteRun, "controlUrl" | "token">): RoachControl {
+  return {
+    async startSession(name) {
+      const { url } = await callControl<OpenedSession>(
+        controlUrl,
+        token,
+        "POST",
+        "session",
+        { name },
+      );
+      return {
+        url,
+        end: (passed) =>
+          callControl(controlUrl, token, "POST", "session/end", {
+            name,
+            passed,
+          }),
+      };
+    },
+    stats: () => callControl(controlUrl, token, "GET", "stats"),
+  };
+}
+
+/** A run on a Roach service (`service.ts`). */
+export interface RemoteRoach extends RemoteRun, RoachControl {
+  /**
+   * The variables that send the HTTP traffic of a process through the run:
+   * `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `NODE_USE_ENV_PROXY`, and
+   * `NODE_EXTRA_CA_CERTS`. Give them to a process when it starts. Node
+   * reads the last two only at startup.
+   */
+  env: Record<string, string>;
+  /** End the run. Returns its stats. */
+  close(): Promise<RecordingStats>;
+}
+
+/**
+ * Start a run on a Roach service, such as `https://roach.example.com`.
+ *
+ * `token` is the write token of the service. Only this call uses it. Without it,
+ * the service only allows `replay` mode, so CI jobs of forks can replay
+ * without a secret. The run and its workers use the run token.
+ */
+export async function startRemoteRun(
+  service: { url: string; token?: string | undefined },
+  config: RunConfig,
+  { noProxy = "localhost,127.0.0.1,::1" }: { noProxy?: string } = {},
+): Promise<RemoteRoach> {
+  const run = await callControl<RemoteRun>(
+    `${service.url.replace(/\/$/, "")}${CONTROL_PATH}`,
+    service.token,
+    "POST",
+    "runs",
+    config,
+  );
+  const proxyEnv = await createProxyEnv(run, noProxy).catch(async (error) => {
+    // End the run, or it stays open until the service times it out.
+    await callControl(run.controlUrl, run.token, "DELETE", "").catch(() => {});
+    throw error;
+  });
+  return {
+    ...run,
+    ...connectRoach(run),
+    env: proxyEnv.env,
+    async close() {
+      try {
+        return await callControl<RecordingStats>(
+          run.controlUrl,
+          run.token,
+          "DELETE",
+          "",
+        );
+      } finally {
+        await proxyEnv.remove();
+      }
+    },
+  };
+}
+
+/**
+ * The proxy variables of a process, with the CA certificate in a file. The
+ * file also has the certificates of `NODE_EXTRA_CA_CERTS` of this process,
+ * so the process still trusts them.
+ */
+async function createProxyEnv(
+  address: Pick<RemoteRun, "caCert" | "url">,
+  noProxy: string,
+): Promise<{ env: Record<string, string>; remove(): Promise<void> }> {
+  // `NODE_EXTRA_CA_CERTS` takes a file.
+  const caDirectory = await mkdtemp(path.join(tmpdir(), "roach-"));
+  const caFile = path.join(caDirectory, "ca.pem");
+  const extra = process.env.NODE_EXTRA_CA_CERTS;
+  await writeFile(
+    caFile,
+    extra
+      ? `${address.caCert.trimEnd()}\n${await readFile(extra, "utf8")}`
+      : address.caCert,
+  );
+  return {
+    env: {
+      HTTP_PROXY: address.url,
+      HTTPS_PROXY: address.url,
+      NO_PROXY: noProxy,
+      NODE_USE_ENV_PROXY: "1",
+      NODE_EXTRA_CA_CERTS: caFile,
+    },
+    remove: () => rm(caDirectory, { force: true, recursive: true }),
+  };
+}