@sentry/roach 0.0.0-stage → 0.3.0

package/src/types.ts

@@ -0,0 +1,181 @@
+/**
+ * The public contract of Roach: rules, modes, runs, sessions, and the
+ * results of a run. The client, the service, and the vitest integration
+ * share it. See `README.md`.
+ */
+import type { ValuePatterns } from "./values.ts";
+
+/** The path prefix of the control API. */
+export const CONTROL_PATH = "/__roach";
+
+/**
+ * - `auto`: replay a request that has a recording. Send other requests live
+ *   and record them.
+ * - `replay`: replay a request that has a recording. Fail other requests
+ *   with HTTP 412. Nothing goes live, and nothing is recorded.
+ * - `record`: send every request live and record it again.
+ * - `off`: record and replay nothing.
+ */
+export type RecordingMode = "auto" | "off" | "record" | "replay";
+
+/** One kind of traffic that the proxy records. */
+export interface RecordingRule {
+  /**
+   * The name of the rule, such as `model`. Its keys start with
+   * `<name>/`. Use letters, digits, `_`, `.`, and `-`.
+   */
+  name: string;
+  /** The requests of the rule. Each field that is set must match. */
+  match: {
+    /** The method, such as `POST`. */
+    method?: string;
+    /**
+     * The start of the URL, such as `https://ai-gateway.vercel.sh/`. A list
+     * matches a URL that starts with one of its prefixes, so one rule can
+     * record the same API on many hosts. The key does not use `match`, so a
+     * string and a list with that string give the same keys.
+     */
+    url?: string | string[];
+    /** Header values. Names are lowercase. */
+    headers?: Record<string, string>;
+  };
+  /**
+   * Request headers that are in the key, with the method, the URL, and the
+   * body. Names are lowercase.
+   */
+  keyHeaders?: string[];
+  /**
+   * Values in the request body that change from run to run, such as ids and
+   * times. See `values.ts`.
+   */
+  values?: ValuePatterns;
+}
+
+/**
+ * Where a request without a recording first differs from the request of
+ * the closest recording, in the texts that the keys hash. It has no text:
+ * the texts go only to the `roach.request` span in Sentry, as
+ * `miss.recorded` and `miss.request`.
+ */
+export interface RecordingChange {
+  /** The key of the recording that the request is compared with. */
+  file: string;
+  /** The offset of the first different character. */
+  at: number;
+}
+
+/** A request that a rule matched, but that had no recording. */
+export interface RecordingMiss {
+  rule: string;
+  /** The session of the request. */
+  session?: string | undefined;
+  /** The key of the recording that the request needed. */
+  file: string;
+  /**
+   * The first change from the request of the closest recording. Only the
+   * first miss of each session in a run with the write token has it, and
+   * only when the proxy has the text of that request.
+   */
+  change?: RecordingChange;
+}
+
+/** The totals of a run. */
+export interface RecordingStats {
+  /**
+   * Requests by rule name. `missed` counts the requests that `replay` mode
+   * failed, because they had no recording.
+   */
+  counts: Record<string, { live: number; missed: number; replayed: number }>;
+  /** The requests without a recording, in order. The first 500 are kept. */
+  misses: RecordingMiss[];
+  /** Recordings that the proxy wrote and that were new or changed. */
+  written: number;
+  /** New recordings that the proxy dropped, because their session failed. */
+  discarded: number;
+  /**
+   * Requests that matched no rule, by origin. They went live and were not
+   * recorded. Use this to find traffic that a rule misses.
+   */
+  passthrough: Record<string, number>;
+  /**
+   * Requests that the proxy refused with HTTP 403 and sent nowhere, by
+   * reason and origin. An older service does not send it.
+   */
+  refused?: Partial<Record<RefusalReason, Record<string, number>>>;
+}
+
+/**
+ * Why the proxy refused a request:
+ * - `notAllowed`: its origin is not in `allow`.
+ * - `notPublic`: its origin is in `allow`, but its address is not public.
+ * - `replayOnly`: the run can only replay, and no rule records the request.
+ */
+export type RefusalReason = "notAllowed" | "notPublic" | "replayOnly";
+
+/** The configuration of one run. The client sends it. */
+export interface RunConfig {
+  /**
+   * The tenant of the run, as `owner/repo`, such as
+   * `process.env.GITHUB_REPOSITORY`.
+   */
+  tenant: string;
+  mode: RecordingMode;
+  rules: RecordingRule[];
+  /**
+   * The only origins that the run can reach, such as
+   * `https://ai-gateway.vercel.sh`. The service sends requests only to
+   * their public addresses.
+   */
+  allow: string[];
+  /**
+   * The name of the run in Sentry, such as
+   * `${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}`. Default: the run id.
+   */
+  name?: string;
+  /**
+   * Credentials that this run must never write, such as the API keys of
+   * its CI job. The service redacts them with the `secrets` of the service
+   * config, keeps them only in memory, and forgets them when the run ends.
+   */
+  secrets?: string[];
+  /**
+   * The CI job of the run, for Sentry. `runConfigOf()` takes it from
+   * `GITHUB_RUN_ID`, `GITHUB_JOB`, `GITHUB_REPOSITORY`, and `GITHUB_REF`.
+   * An older service ignores it.
+   */
+  ci?: RunCi;
+}
+
+/** The CI job of a run. Each field is a string of at most 256 characters. */
+export interface RunCi {
+  runId?: string;
+  job?: string;
+  repo?: string;
+  ref?: string;
+}
+
+/** A run that the service created. */
+export interface RemoteRun {
+  /** The id of the run. */
+  id: string;
+  /**
+   * The proxy URL, with the run id and token as its user and password. A
+   * request through it belongs to the open session when only one is open.
+   */
+  url: string;
+  /** The run token. Session and stats calls use it. */
+  token: string;
+  /** The base URL of the control API of the run. */
+  controlUrl: string;
+  /** The PEM certificate of the authority that signs intercepted hosts. */
+  caCert: string;
+}
+
+/** A session that the service opened. */
+export interface OpenedSession {
+  /**
+   * The proxy URL of the session. Each request through it belongs to the
+   * session, also when other sessions are open.
+   */
+  url: string;
+}

package/src/values.ts

@@ -0,0 +1,267 @@
+/**
+ * Changing values in recorded requests, such as ids and clock times.
+ *
+ * A rule names the values that change from run to run. The proxy handles
+ * them in two directions:
+ *
+ * - The key sees each value as `<<name>>`, so two runs with other ids or
+ *   times send the same request.
+ * - A response often repeats a value of its request, for example when the
+ *   model archives the memory with id `<<uuid>>`. When the proxy records a
+ *   response, it writes each value of the request as `<<name:n>>`, the n-th
+ *   value of that name in the request. On replay, it writes back the n-th
+ *   value of the current request. So a replayed response uses the ids of
+ *   this run, not the ids of the recording run.
+ *
+ * A value that the response makes itself, such as a date that the model
+ * calculates, stays as it was recorded.
+ */
+import vm from "node:vm";
+
+/**
+ * Lookbehind: the value does not continue a word or number. The proxy
+ * matches values in JSON text, where `\nab12` is a newline and then
+ * `ab12`. So a letter after a backslash, such as the `n` of `\n`, does not
+ * count as a word. Use it in a pattern that starts with a letter or digit.
+ */
+export const NOT_AFTER_WORD = String.raw`(?<!(?:^|[^\\])[0-9A-Za-z])`;
+
+/**
+ * Patterns for common changing values. Use them in `values` of a rule. Each
+ * pattern is a regular expression source.
+ */
+export const VALUE_PATTERNS = {
+  /** A UUID, also inside a name such as `junior-ws-<uuid>`. */
+  uuid: String.raw`${NOT_AFTER_WORD}[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}(?![0-9A-Za-z])`,
+  /**
+   * An ISO time, with `T` or a space, with or without seconds, and in UTC,
+   * with an offset, or in local time.
+   */
+  isoTime: String.raw`(?<![\d-])\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2}| ?UTC)?(?![\d:])`,
+  /** An ISO date, such as `2026-10-07`. */
+  date: String.raw`(?<![\d-])\d{4}-\d{2}-\d{2}(?![\d-])`,
+  /**
+   * Unix milliseconds from 2023 to 2033. Unix seconds, such as Slack
+   * timestamps, do not match.
+   */
+  epochMs: String.raw`(?<!\d)1[789]\d{11}(?!\d)`,
+  /**
+   * A SHA-256 hex digest. Ids that the product hashes from other changing
+   * values, such as attachment ids, change with them.
+   */
+  sha256: String.raw`${NOT_AFTER_WORD}[0-9a-f]{64}(?![0-9A-Za-z])`,
+  /**
+   * A git commit id. Its hash covers the commit time, so it changes on each
+   * run. A full id has 40 hex digits. A short id only counts in the output
+   * of `git commit`, such as `[main (root-commit) a7a9f1d]`, and in the range
+   * of `git push`, such as `a5fd79f..a73e959`, because seven hex digits can
+   * also be a word.
+   */
+  gitCommit: String.raw`${NOT_AFTER_WORD}[0-9a-f]{40}(?![0-9A-Za-z])|(?<=\[[^\]\s]+ (?:\(root-commit\) )?)[0-9a-f]{7,12}(?=\])|${NOT_AFTER_WORD}[0-9a-f]{7,12}(?=\.\.\.?[0-9a-f]{7,12}(?![0-9A-Za-z]))|(?<=${NOT_AFTER_WORD}[0-9a-f]{7,12}\.\.\.?)[0-9a-f]{7,12}(?![0-9A-Za-z])`,
+} as const;
+
+/** Changing values by name. Each value is a regular expression source. */
+export type ValuePatterns = Record<string, string>;
+
+/** The values of one request, by name, in the order of the request. */
+export type RequestValues = Map<string, string[]>;
+
+interface CompiledValues {
+  names: string[];
+  /** One expression for all patterns, with one group per name. */
+  pattern: RegExp;
+  /** The index of the group of each name. */
+  groups: number[];
+}
+
+const compiled = new WeakMap<ValuePatterns, CompiledValues>();
+
+/** The number of capturing groups in a pattern. */
+const groupCount = (source: string) =>
+  new RegExp(`${source}|`).exec("")!.length - 1;
+
+function compile(patterns: ValuePatterns): CompiledValues {
+  const cached = compiled.get(patterns);
+  if (cached) return cached;
+  const names = Object.keys(patterns);
+  const groups: number[] = [];
+  let next = 1;
+  for (const name of names) {
+    groups.push(next);
+    next += 1 + groupCount(patterns[name]!);
+  }
+  const result = {
+    names,
+    // Earlier names win when two patterns match at the same place.
+    pattern: new RegExp(
+      names.map((name) => `(${patterns[name]})`).join("|"),
+      "g",
+    ),
+    groups,
+  };
+  compiled.set(patterns, result);
+  return result;
+}
+
+/**
+ * Run `fn` with a time limit. A slow pattern cannot stop itself, but the
+ * `timeout` of `vm` stops any code that runs too long, also code outside
+ * the context that the script calls. A timeout throws an error with HTTP
+ * status 400 and `timedOut`.
+ */
+export function withTimeLimit<T>(ms: number, fn: () => T): T {
+  try {
+    return vm.runInContext("fn()", vm.createContext({ fn }), {
+      timeout: ms,
+    }) as T;
+  } catch (error) {
+    if ((error as { code?: string }).code === "ERR_SCRIPT_EXECUTION_TIMEOUT") {
+      throw Object.assign(
+        new Error(`the value patterns took over ${ms} ms. Make them simpler.`),
+        { status: 400, timedOut: true },
+      );
+    }
+    throw error;
+  }
+}
+
+/**
+ * Check that the patterns of each rule compile. Returns the error, or
+ * `undefined`. A pattern can be slow to compile, so all of them together
+ * have 1 s.
+ */
+export function invalidValuePatterns(
+  list: ValuePatterns[],
+): string | undefined {
+  try {
+    withTimeLimit(1000, () => list.forEach((patterns) => compile(patterns)));
+    return undefined;
+  } catch (error) {
+    return (error as Error).message;
+  }
+}
+
+/** Values that an earlier request of the same session had, by value. */
+export type KnownValues = ReadonlyMap<string, string>;
+
+/**
+ * Replace each changing value in `text` with `<<name>>`. Returns the text
+ * and the values in their order. With no patterns, nothing changes.
+ *
+ * `known` maps values of earlier requests in the session to their names. A
+ * known value also counts where no pattern finds it. For example, the
+ * model repeats the short commit id of a `git push` range in its reply,
+ * and a later request sends that reply back. A pattern wins over a known
+ * value at the same place.
+ *
+ * Past `maxValues` values, it stops and throws an error with HTTP status
+ * 400. It counts each value when it finds it, so a text with millions of
+ * values cannot fill the memory before the error.
+ */
+export function extractValues(
+  text: string,
+  patterns: ValuePatterns | undefined,
+  known?: KnownValues,
+  maxValues = Infinity,
+): { text: string; values: RequestValues } {
+  const values: RequestValues = new Map();
+  if (!patterns || Object.keys(patterns).length === 0) {
+    return { text, values };
+  }
+  const { names, pattern, groups } = compile(patterns);
+  const hasKnown = known !== undefined && known.size > 0;
+  // Known values are the last group, after the groups of the patterns.
+  const knownGroup = groupCount(pattern.source) + 1;
+  const search = hasKnown
+    ? new RegExp(`${pattern.source}|(${literals([...known.keys()])})`, "g")
+    : pattern;
+  // An `exec()` loop, not `replace()`: `replace()` finds all matches
+  // before it calls its function, so it cannot stop at `maxValues`. The
+  // loop finds the same matches, also the empty ones.
+  const parts: string[] = [];
+  let count = 0;
+  let end = 0;
+  search.lastIndex = 0;
+  try {
+    for (let match = search.exec(text); match; match = search.exec(text)) {
+      count += 1;
+      if (count > maxValues) {
+        throw Object.assign(
+          new Error("the request has too many changing values"),
+          { status: 400 },
+        );
+      }
+      const value = match[0];
+      const name =
+        hasKnown && match[knownGroup] !== undefined
+          ? known.get(value)!
+          : names[groups.findIndex((group) => match[group] !== undefined)]!;
+      const list = values.get(name) ?? [];
+      list.push(value);
+      values.set(name, list);
+      parts.push(text.slice(end, match.index), `<<${name}>>`);
+      end = match.index + value.length;
+      if (value.length === 0) search.lastIndex += 1;
+    }
+  } finally {
+    // `pattern` is shared, so it must not keep a position.
+    search.lastIndex = 0;
+  }
+  parts.push(text.slice(end));
+  return { text: parts.join(""), values };
+}
+
+/**
+ * Whether `value` can be in `known` or in a template (`templateValues()`).
+ * V8 cannot run a pattern with a literal of over 32,767 characters on
+ * any text, also on a text of 5 characters. (A literal with a character
+ * over U+00FF fails only on a text with such a character, and only such a
+ * text can hold it.) Such a value in `known` failed each later request of
+ * its session, so those requests had no key, and to leave it out changes
+ * no key.
+ */
+export const canBeKnown = (value: string): boolean => value.length <= 32_767;
+
+const escape = (text: string) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/**
+ * A pattern source for any of `values`, where it is not part of a longer
+ * word or number. Longer values come first, so a date never replaces part
+ * of a time.
+ */
+const literals = (values: string[]) =>
+  `${NOT_AFTER_WORD}(?:${values
+    .toSorted((a, b) => b.length - a.length)
+    .map(escape)
+    .join("|")})(?![0-9A-Za-z])`;
+
+/**
+ * Write each value of the request that a recorded response repeats as
+ * `<<name:n>>`. A value is replaced only where it is not part of a longer
+ * word or number. A value that `canBeKnown()` refuses stays as it is.
+ */
+export function templateValues(text: string, values: RequestValues): string {
+  const placeholders = new Map<string, string>();
+  for (const [name, list] of values) {
+    list.forEach((value, index) => {
+      if (!placeholders.has(value) && canBeKnown(value)) {
+        placeholders.set(value, `<<${name}:${index + 1}>>`);
+      }
+    });
+  }
+  if (placeholders.size === 0) return text;
+  const pattern = new RegExp(literals([...placeholders.keys()]), "g");
+  return text.replace(pattern, (value) => placeholders.get(value)!);
+}
+
+/**
+ * Write back the values of the current request into a recorded response.
+ * This undoes `templateValues()` with the values of this run.
+ */
+export function fillValues(text: string, values: RequestValues): string {
+  return text.replace(
+    /<<([^<>:\s]+):(\d+)>>/g,
+    (placeholder, name: string, index: string) =>
+      values.get(name)?.[Number(index) - 1] ?? placeholder,
+  );
+}

package/src/vitest-setup.ts

@@ -0,0 +1,62 @@
+/**
+ * The vitest setup file of Roach (`@sentry/roach/vitest/setup`). See "Use
+ * with vitest" in `README.md`.
+ *
+ * It opens one session for each test of the run that `vitest.ts` started.
+ * Requests of the test go through the proxy URL of its session, so tests
+ * in other workers record at the same time. Its child processes get that
+ * URL too. A passed test writes its new recordings, and a failed test
+ * drops them. A test with a request that `replay` mode failed fails.
+ *
+ * The tests of one file share one process, so they must run one at a
+ * time. Do not use `test.concurrent` with Roach.
+ */
+import http from "node:http";
+import {
+  beforeEach,
+  inject,
+  onTestFinished,
+  type ProvidedContext,
+} from "vitest";
+import { connectRoach } from "./client.ts";
+import { swapEnv } from "./config.ts";
+
+const provided = inject("roach");
+if (provided === undefined) {
+  throw new Error(
+    "Roach: add @sentry/roach/vitest to globalSetup to use the setup file",
+  );
+}
+// `null` is `off` mode: no run, so requests go direct.
+if (provided) useRun(provided);
+
+/** Open a session for each test of the run. */
+function useRun(run: NonNullable<ProvidedContext["roach"]>): void {
+  const control = connectRoach(run);
+  beforeEach(async ({ task }) => {
+    // The full name starts with the file, so tests of two files never share
+    // a session.
+    const session = await control.startSession(task.fullName);
+    const restoreEnv = swapEnv({
+      HTTP_PROXY: session.url,
+      HTTPS_PROXY: session.url,
+    });
+    const restoreProxy = http.setGlobalProxyFromEnv({
+      HTTP_PROXY: session.url,
+      HTTPS_PROXY: session.url,
+      NO_PROXY: run.noProxy,
+    });
+    onTestFinished(async ({ task: finished }) => {
+      restoreProxy();
+      restoreEnv();
+      const { missed } = await session.end(finished.result?.state === "pass");
+      // `replay` mode failed these requests, so nothing went live. The test
+      // must fail, also when it recovered from the error.
+      if (missed > 0) {
+        throw new Error(
+          `Roach: ${missed} request(s) had no recording in replay mode. Run with ROACH_TOKEN to record them.`,
+        );
+      }
+    });
+  });
+}

package/src/vitest.ts

@@ -0,0 +1,105 @@
+/**
+ * The vitest global setup of Roach (`@sentry/roach/vitest`). See "Use with
+ * vitest" in `README.md`.
+ *
+ * It starts one run with the config of `roach.json` (`config.ts`), and
+ * gives the proxy variables of the run to the test workers, which start
+ * after it. `vitest-setup.ts` opens one session for each test, so the
+ * tests of many workers record at the same time. At the end it closes the
+ * run and reports it.
+ *
+ * - `ROACH_TOKEN` is the write token. Only this file reads it, and the
+ *   workers do not get it. Without it, the run can only replay.
+ * - `ROACH_URL` is the service. Default: the Roach service of Sentry.
+ * - `ROACH_MODE` sets the mode. See `runConfigOf()`. With `off`, it starts
+ *   no run and does not use the service, so requests go direct.
+ */
+import http from "node:http";
+import tls from "node:tls";
+import type { TestProject } from "vitest/node";
+import { startRemoteRun } from "./client.ts";
+import {
+  DEFAULT_NO_PROXY,
+  DEFAULT_SERVICE_URL,
+  readEnv,
+  readRoachFile,
+  runConfigOf,
+  swapEnv,
+} from "./config.ts";
+import { writeRunReport } from "./report.ts";
+import type { RemoteRun } from "./types.ts";
+
+declare module "vitest" {
+  export interface ProvidedContext {
+    /** The run of `@sentry/roach/vitest`, for the test workers. */
+    roach:
+      | (Pick<RemoteRun, "controlUrl" | "token"> & { noProxy: string })
+      | null;
+  }
+}
+
+/**
+ * Start a Roach run for this vitest invocation. Returns the teardown,
+ * which ends the run and reports it. Call it from a global setup of your
+ * own when you have one, or use this module as a global setup.
+ */
+export async function startRoach(
+  project: Pick<TestProject, "provide">,
+): Promise<() => Promise<void>> {
+  const token = readEnv("ROACH_TOKEN");
+  delete process.env.ROACH_TOKEN;
+  const file = await readRoachFile();
+  const config = runConfigOf(file, { canWrite: token !== undefined });
+  if (config.mode === "off") {
+    project.provide("roach", null);
+    return async () => {};
+  }
+  // Only a run needs it, so `off` mode works on older Node too.
+  if (typeof http.setGlobalProxyFromEnv !== "function") {
+    throw new Error("Roach: vitest needs Node 24.14 or later");
+  }
+  const noProxy = file.noProxy ?? DEFAULT_NO_PROXY;
+  const run = await startRemoteRun(
+    { url: readEnv("ROACH_URL") ?? DEFAULT_SERVICE_URL, token },
+    config,
+    { noProxy },
+  );
+  // The proxy URL has the run token. Keep it out of the CI log.
+  if (process.env.GITHUB_ACTIONS) {
+    process.stdout.write(`::add-mask::${run.token}\n`);
+  }
+  // Test workers and their child processes start later and inherit these.
+  const restoreEnv = swapEnv(run.env);
+  // This process started before them, so it reads neither the proxy nor
+  // the CA file. Set both here.
+  const previousCa = tls.getCACertificates("default");
+  tls.setDefaultCACertificates([...previousCa, run.caCert]);
+  const restoreProxy = http.setGlobalProxyFromEnv(run.env);
+  project.provide("roach", {
+    controlUrl: run.controlUrl,
+    token: run.token,
+    noProxy,
+  });
+
+  return async () => {
+    try {
+      const stats = await run.close();
+      await writeRunReport(`Roach (${config.mode})`, stats);
+      // A miss in a test fails that test. A miss outside a test, such as in
+      // a global setup, must fail the invocation too.
+      const missed = Object.values(stats.counts).some(
+        (count) => count.missed > 0,
+      );
+      if (missed) process.exitCode = 1;
+    } finally {
+      restoreProxy();
+      tls.setDefaultCACertificates(previousCa);
+      restoreEnv();
+    }
+  };
+}
+
+/** The vitest global setup. */
+export default function setup(project: TestProject) {
+  return startRoach(project);
+}