@mizchi/playwright-faults 0.1.0

package/src/runtime-faults.test.ts

@@ -0,0 +1,282 @@
+import { describe, expect, it } from "vitest";
+import { createRng } from "./random.js";
+import {
+  buildRuntimeFaultsScript,
+  compileRuntimeFaults,
+  mergeRuntimeStats,
+  runtimeFaultName,
+  runtimeMatchesUrl,
+  shouldFireProbability,
+} from "./runtime-faults.js";
+import type { RuntimeFault } from "./types.js";
+
+describe("runtimeFaultName", () => {
+  it("uses an explicit name when set", () => {
+    expect(
+      runtimeFaultName({ name: "custom", action: { kind: "flaky-fetch" } }),
+    ).toBe("custom");
+  });
+
+  it("auto-derives a name for flaky-fetch", () => {
+    expect(runtimeFaultName({ action: { kind: "flaky-fetch" } })).toBe("flaky-fetch");
+  });
+
+  it("auto-derives a name for clock-skew including the skew amount", () => {
+    expect(
+      runtimeFaultName({ action: { kind: "clock-skew", skewMs: 60_000 } }),
+    ).toBe("clock-skew:60000ms");
+  });
+});
+
+describe("compileRuntimeFaults", () => {
+  it("returns empty array for empty input", () => {
+    expect(compileRuntimeFaults(undefined)).toEqual([]);
+    expect(compileRuntimeFaults([])).toEqual([]);
+  });
+
+  it("compiles string urlPattern to RegExp", () => {
+    const compiled = compileRuntimeFaults([
+      { urlPattern: "api", action: { kind: "flaky-fetch" } },
+    ]);
+    expect(compiled[0]!.pattern).toBeInstanceOf(RegExp);
+    expect(compiled[0]!.pattern!.test("http://x/api/users")).toBe(true);
+  });
+
+  it("preserves RegExp urlPattern verbatim", () => {
+    const re = /\/api\//i;
+    const compiled = compileRuntimeFaults([{ urlPattern: re, action: { kind: "flaky-fetch" } }]);
+    expect(compiled[0]!.pattern).toBe(re);
+  });
+
+  it("leaves pattern null when urlPattern is omitted", () => {
+    const compiled = compileRuntimeFaults([{ action: { kind: "flaky-fetch" } }]);
+    expect(compiled[0]!.pattern).toBeNull();
+  });
+
+  it("seeds counters at zero", () => {
+    const compiled = compileRuntimeFaults([{ action: { kind: "flaky-fetch" } }]);
+    expect(compiled[0]!.matched).toBe(0);
+    expect(compiled[0]!.fired).toBe(0);
+  });
+});
+
+describe("runtimeMatchesUrl", () => {
+  it("matches everything when pattern is null", () => {
+    expect(runtimeMatchesUrl({ pattern: null }, "http://x/")).toBe(true);
+  });
+
+  it("matches via the regex", () => {
+    expect(runtimeMatchesUrl({ pattern: /\/api\// }, "http://x/api/users")).toBe(true);
+    expect(runtimeMatchesUrl({ pattern: /\/api\// }, "http://x/static/foo")).toBe(false);
+  });
+});
+
+describe("shouldFireProbability", () => {
+  it("always fires when probability is undefined or >= 1", () => {
+    const rng = createRng(1);
+    expect(shouldFireProbability(undefined, rng)).toBe(true);
+    expect(shouldFireProbability(1, rng)).toBe(true);
+    expect(shouldFireProbability(2, rng)).toBe(true);
+  });
+
+  it("never fires when probability is 0 or negative", () => {
+    const rng = createRng(1);
+    expect(shouldFireProbability(0, rng)).toBe(false);
+    expect(shouldFireProbability(-1, rng)).toBe(false);
+  });
+
+  it("rolls deterministically against the RNG", () => {
+    const a = createRng(42);
+    const b = createRng(42);
+    const aFires: boolean[] = [];
+    const bFires: boolean[] = [];
+    for (let i = 0; i < 20; i++) {
+      aFires.push(shouldFireProbability(0.5, a));
+      bFires.push(shouldFireProbability(0.5, b));
+    }
+    expect(aFires).toEqual(bFires);
+  });
+});
+
+describe("buildRuntimeFaultsScript", () => {
+  it("returns a string starting with an IIFE", () => {
+    const s = buildRuntimeFaultsScript(
+      [{ action: { kind: "flaky-fetch" } }],
+      42,
+    );
+    expect(s.startsWith("(() => {")).toBe(true);
+    expect(s).toContain("__chaosbringerRuntimeFaultsInstalled");
+    expect(s).toContain("__chaosbringerRuntimeStats");
+  });
+
+  it("guards against re-installation", () => {
+    const s = buildRuntimeFaultsScript([{ action: { kind: "flaky-fetch" } }], 42);
+    expect(s).toContain("if (window.__chaosbringerRuntimeFaultsInstalled) return");
+  });
+
+  it("inlines the seed verbatim", () => {
+    const s = buildRuntimeFaultsScript([{ action: { kind: "flaky-fetch" } }], 12345);
+    expect(s).toContain("12345");
+  });
+
+  it("serializes RegExp urlPattern as { source, flags }", () => {
+    const s = buildRuntimeFaultsScript(
+      [{ urlPattern: /\/api\//i, action: { kind: "flaky-fetch" } }],
+      1,
+    );
+    expect(s).toContain('"source":"\\\\/api\\\\/"');
+    expect(s).toContain('"flags":"i"');
+  });
+
+  it("serializes string urlPattern with empty flags", () => {
+    const s = buildRuntimeFaultsScript(
+      [{ urlPattern: "/api/", action: { kind: "flaky-fetch" } }],
+      1,
+    );
+    expect(s).toContain('"source":"/api/"');
+    expect(s).toContain('"flags":""');
+  });
+
+  it("includes the fetch patch only when a flaky-fetch fault is present", () => {
+    const withFetch = buildRuntimeFaultsScript(
+      [{ action: { kind: "flaky-fetch" } }],
+      1,
+    );
+    expect(withFetch).toContain("flaky-fetch");
+    expect(withFetch).toContain("Promise.reject");
+  });
+
+  it("includes the clock-skew patch only when a clock-skew fault is present", () => {
+    const skewed = buildRuntimeFaultsScript(
+      [{ action: { kind: "clock-skew", skewMs: 5000 } }],
+      1,
+    );
+    expect(skewed).toContain("clock-skew");
+    expect(skewed).toContain("performance.now");
+    expect(skewed).toContain("Date.now");
+  });
+
+  it("uses the fault's auto-derived name as a display label", () => {
+    const s = buildRuntimeFaultsScript(
+      [{ action: { kind: "flaky-fetch" } }, { action: { kind: "clock-skew", skewMs: 1000 } }],
+      1,
+    );
+    expect(s).toContain('"name":"flaky-fetch"');
+    expect(s).toContain('"name":"clock-skew:1000ms"');
+  });
+
+  it("assigns sequential id keys so duplicate names don't collide in stats", () => {
+    const s = buildRuntimeFaultsScript(
+      [{ action: { kind: "flaky-fetch" } }, { action: { kind: "flaky-fetch" } }],
+      1,
+    );
+    expect(s).toContain('"id":0');
+    expect(s).toContain('"id":1');
+    // The stats accessor uses String(f.id), not f.name.
+    expect(s).toContain("stats[String(f.id)]");
+  });
+
+  it("does not 32-bit-truncate large skewMs values", () => {
+    // 30 days = 2,592,000,000 ms — exceeds int32 max. The script must
+    // accumulate via Number(...), not `| 0`.
+    const s = buildRuntimeFaultsScript(
+      [{ action: { kind: "clock-skew", skewMs: 2_592_000_000 } }],
+      1,
+    );
+    expect(s).not.toContain("skewMs | 0");
+    expect(s).toContain("Number(f.action.skewMs)");
+  });
+
+  it("respects an explicit fault name", () => {
+    const s = buildRuntimeFaultsScript(
+      [{ name: "spy", action: { kind: "flaky-fetch" } }],
+      1,
+    );
+    expect(s).toContain('"name":"spy"');
+  });
+
+  it("is a no-op when no faults are configured (still emits the IIFE shell)", () => {
+    const s = buildRuntimeFaultsScript([], 1);
+    expect(s.startsWith("(() => {")).toBe(true);
+    expect(s).toContain("__chaosbringerRuntimeFaultsInstalled");
+  });
+
+  it("accepts negative seed by coercing into an unsigned 32-bit", () => {
+    const s = buildRuntimeFaultsScript([{ action: { kind: "flaky-fetch" } }], -1);
+    // -1 coerced to uint32 is 4294967295.
+    expect(s).toContain("4294967295");
+  });
+});
+
+describe("mergeRuntimeStats", () => {
+  it("accumulates per-page counters by index into the compiled fault counters", () => {
+    const compiled = compileRuntimeFaults([
+      { name: "f1", action: { kind: "flaky-fetch" } },
+      { name: "c1", action: { kind: "clock-skew", skewMs: 100 } },
+    ]);
+    mergeRuntimeStats(compiled, { "0": { matched: 3, fired: 2 } });
+    expect(compiled[0]!.matched).toBe(3);
+    expect(compiled[0]!.fired).toBe(2);
+    expect(compiled[1]!.matched).toBe(0);
+    expect(compiled[1]!.fired).toBe(0);
+    mergeRuntimeStats(compiled, { "0": { matched: 1, fired: 1 } });
+    expect(compiled[0]!.matched).toBe(4);
+    expect(compiled[0]!.fired).toBe(3);
+  });
+
+  it("does not collapse counters when two faults share a name", () => {
+    // Both default to name="flaky-fetch"; index keys keep them apart.
+    const compiled = compileRuntimeFaults([
+      { action: { kind: "flaky-fetch" }, urlPattern: /\/api\/a/ },
+      { action: { kind: "flaky-fetch" }, urlPattern: /\/api\/b/ },
+    ]);
+    mergeRuntimeStats(compiled, {
+      "0": { matched: 5, fired: 5 },
+      "1": { matched: 1, fired: 0 },
+    });
+    expect(compiled[0]!.matched).toBe(5);
+    expect(compiled[0]!.fired).toBe(5);
+    expect(compiled[1]!.matched).toBe(1);
+    expect(compiled[1]!.fired).toBe(0);
+  });
+
+  it("ignores stats keys that don't correspond to a compiled fault", () => {
+    const compiled = compileRuntimeFaults([{ name: "known", action: { kind: "flaky-fetch" } }]);
+    mergeRuntimeStats(compiled, { "99": { matched: 99, fired: 99 } });
+    expect(compiled[0]!.matched).toBe(0);
+  });
+
+  it("returns a stats array shaped like RuntimeFaultStats", () => {
+    const compiled = compileRuntimeFaults([{ name: "f", action: { kind: "flaky-fetch" } }]);
+    const out = mergeRuntimeStats(compiled, { "0": { matched: 5, fired: 3 } });
+    expect(out).toEqual([{ rule: "f", matched: 5, fired: 3 }]);
+  });
+
+  it("falls back to a name-keyed snapshot for the first matching fault", () => {
+    // Backwards-compat path: when reading legacy snapshots that keyed
+    // by name, the merge applies the count to the first compiled fault
+    // with that name and skips the rest.
+    const compiled = compileRuntimeFaults([
+      { name: "shared", action: { kind: "flaky-fetch" } },
+      { name: "shared", action: { kind: "flaky-fetch" } },
+    ]);
+    mergeRuntimeStats(compiled, { shared: { matched: 4, fired: 2 } });
+    expect(compiled[0]!.matched).toBe(4);
+    expect(compiled[0]!.fired).toBe(2);
+    expect(compiled[1]!.matched).toBe(0);
+    expect(compiled[1]!.fired).toBe(0);
+  });
+});
+
+describe("faults.flakyFetch / faults.clockSkew round-trip via compile", () => {
+  it("compiles a programmatic fault config without errors", () => {
+    const faults: RuntimeFault[] = [
+      { action: { kind: "flaky-fetch", rejectionMessage: "oops" }, probability: 0.5 },
+      { action: { kind: "clock-skew", skewMs: 30_000 } },
+    ];
+    const compiled = compileRuntimeFaults(faults);
+    expect(compiled).toHaveLength(2);
+    expect(compiled[0]!.name).toBe("flaky-fetch");
+    expect(compiled[1]!.name).toBe("clock-skew:30000ms");
+  });
+});

package/src/runtime-faults.ts

@@ -0,0 +1,255 @@
+/**
+ * JS-runtime fault injection.
+ *
+ * Distinct from `FaultRule` (request-scoped, applied via Playwright `route()`)
+ * and `LifecycleFault` (one-shot at named stages of a page visit). These are
+ * persistent monkey-patches injected into every page via `addInitScript`,
+ * subverting in-page JS APIs (fetch / Date / storage / addEventListener) so
+ * the app sees client-side failures that no network mock would expose.
+ *
+ * Examples:
+ *   - `flaky-fetch`: `window.fetch` rejects with a TypeError before any
+ *     network round-trip — simulates "Failed to fetch" / DNS down / Service
+ *     Worker reject. Different from `faults.status(500)`, which still
+ *     resolves the promise.
+ *   - `clock-skew`: `Date.now` / `performance.now` are shifted forward by N
+ *     ms — exposes token-expiry / cache-bust bugs on long sessions.
+ *
+ * Pure helpers (`buildRuntimeFaultsScript`, `runtimeFaultName`,
+ * `compileRuntimeFaults`) generate / serialize the init script and roll
+ * probability — unit-testable without a browser. Stats are reported by the
+ * in-page script via a known `window.__chaosbringerRuntimeStats` global; the
+ * crawler reads it after each page visit.
+ */
+
+import type { Rng, RuntimeFault, RuntimeFaultStats, UrlMatcher } from "./types.js";
+
+/** Compiled form: regex pre-compiled, name pre-derived. */
+export interface CompiledRuntimeFault {
+  fault: RuntimeFault;
+  /** null when `fault.urlPattern` was omitted. */
+  pattern: RegExp | null;
+  name: string;
+  matched: number;
+  fired: number;
+}
+
+/** Auto-derive a stats label when the user didn't set `fault.name`. */
+export function runtimeFaultName(fault: RuntimeFault): string {
+  if (fault.name) return fault.name;
+  const a = fault.action;
+  switch (a.kind) {
+    case "flaky-fetch":
+      return "flaky-fetch";
+    case "clock-skew":
+      return `clock-skew:${a.skewMs}ms`;
+  }
+}
+
+function compilePattern(matcher: UrlMatcher | undefined): RegExp | null {
+  if (matcher === undefined) return null;
+  return matcher instanceof RegExp ? matcher : new RegExp(matcher);
+}
+
+export function compileRuntimeFaults(
+  faults: RuntimeFault[] | undefined,
+): CompiledRuntimeFault[] {
+  if (!faults || faults.length === 0) return [];
+  return faults.map((fault) => ({
+    fault,
+    pattern: compilePattern(fault.urlPattern),
+    name: runtimeFaultName(fault),
+    matched: 0,
+    fired: 0,
+  }));
+}
+
+/** True when `compiled.pattern` matches `url` (or no pattern was set). */
+export function runtimeMatchesUrl(
+  compiled: Pick<CompiledRuntimeFault, "pattern">,
+  url: string,
+): boolean {
+  return compiled.pattern === null || compiled.pattern.test(url);
+}
+
+/**
+ * Decide whether a probabilistic fault fires this time. Mirrors the
+ * lifecycle / network helpers so all three layers share a deterministic
+ * roll behaviour given the same RNG.
+ */
+export function shouldFireProbability(probability: number | undefined, rng: Rng): boolean {
+  const p = probability ?? 1;
+  if (p >= 1) return true;
+  if (p <= 0) return false;
+  return rng.next() < p;
+}
+
+/**
+ * Serialize a UrlMatcher into a structure the in-page script can rebuild
+ * without `eval`. Strings stay strings; RegExp becomes `{ source, flags }`.
+ */
+function serializeMatcher(m: UrlMatcher | undefined): { source: string; flags: string } | null {
+  if (m === undefined) return null;
+  if (m instanceof RegExp) return { source: m.source, flags: m.flags };
+  return { source: m, flags: "" };
+}
+
+/**
+ * Build the init script body. Self-contained IIFE — no closure over the
+ * caller's scope, no external imports — because Playwright serializes init
+ * scripts as plain text and runs them in a fresh frame on every navigation.
+ *
+ * `seed` lets each page roll deterministic probabilities. Pass the
+ * crawler's seed so a `(seed, runtimeFaults)` pair always produces the same
+ * pattern of injections.
+ */
+export function buildRuntimeFaultsScript(
+  faults: ReadonlyArray<RuntimeFault>,
+  seed: number,
+): string {
+  // Stats keys are indices, not names — two faults can legitimately share
+  // a name (`flaky-fetch` x2 with different urlPatterns) and we mustn't
+  // collapse their counters.
+  const serialized = faults.map((f, i) => ({
+    id: i,
+    name: runtimeFaultName(f),
+    pattern: serializeMatcher(f.urlPattern),
+    probability: typeof f.probability === "number" ? f.probability : 1,
+    action: f.action,
+  }));
+
+  // Body of the init script. Indented for readability; whitespace is fine
+  // because Playwright won't minify it.
+  return `(() => {
+  if (typeof window === "undefined") return;
+  if (window.__chaosbringerRuntimeFaultsInstalled) return;
+  window.__chaosbringerRuntimeFaultsInstalled = true;
+  window.__chaosbringerRuntimeStats = {};
+
+  // Park-Miller LCG — small, deterministic, good enough for fault rolls.
+  let __rng = ${seed >>> 0} || 1;
+  const __nextRoll = () => {
+    __rng = ((__rng * 16807) % 2147483647) | 0;
+    if (__rng <= 0) __rng += 2147483647;
+    return (__rng - 1) / 2147483646;
+  };
+
+  const faults = ${JSON.stringify(serialized)};
+  const stats = window.__chaosbringerRuntimeStats;
+  for (const f of faults) stats[String(f.id)] = { matched: 0, fired: 0 };
+
+  const matchUrl = (pattern, url) => {
+    if (!pattern) return true;
+    try {
+      return new RegExp(pattern.source, pattern.flags).test(url);
+    } catch {
+      return false;
+    }
+  };
+
+  const roll = (f) => {
+    const slot = stats[String(f.id)];
+    slot.matched++;
+    if (f.probability >= 1) {
+      slot.fired++;
+      return true;
+    }
+    if (f.probability <= 0) return false;
+    const fired = __nextRoll() < f.probability;
+    if (fired) slot.fired++;
+    return fired;
+  };
+
+  // --- flaky-fetch ---
+  const fetchFaults = faults.filter((f) => f.action.kind === "flaky-fetch");
+  if (fetchFaults.length > 0 && typeof window.fetch === "function") {
+    const realFetch = window.fetch.bind(window);
+    window.fetch = function chaosFetch(input, init) {
+      const url =
+        typeof input === "string" ? input :
+        input instanceof URL ? input.toString() :
+        (input && typeof input.url === "string") ? input.url :
+        "";
+      for (const f of fetchFaults) {
+        if (matchUrl(f.pattern, url) && roll(f)) {
+          const msg = f.action.rejectionMessage || "chaosbringer: simulated fetch failure";
+          return Promise.reject(new TypeError(msg));
+        }
+      }
+      return realFetch(input, init);
+    };
+  }
+
+  // --- clock-skew ---
+  const skewFaults = faults.filter((f) => f.action.kind === "clock-skew");
+  if (skewFaults.length > 0) {
+    // Use a plain accumulator. Multi-day skews (e.g. 30 days ~ 2.6e9 ms)
+    // exceed int32 max, so any '| 0' truncation here would flip them negative.
+    let totalSkew = 0;
+    for (const f of skewFaults) {
+      if (matchUrl(f.pattern, location.href) && roll(f)) {
+        totalSkew += Number(f.action.skewMs);
+      }
+    }
+    if (totalSkew !== 0) {
+      const realDateNow = Date.now.bind(Date);
+      Date.now = () => realDateNow() + totalSkew;
+      const realPerfNow = performance.now.bind(performance);
+      performance.now = () => realPerfNow() + totalSkew;
+      // Patch the Date constructor so \`new Date()\` (no args) also skews.
+      const RealDate = Date;
+      const SkewedDate = function (...args) {
+        if (args.length === 0) return new RealDate(realDateNow() + totalSkew);
+        // @ts-ignore
+        return new RealDate(...args);
+      };
+      SkewedDate.now = Date.now;
+      SkewedDate.UTC = RealDate.UTC;
+      SkewedDate.parse = RealDate.parse;
+      SkewedDate.prototype = RealDate.prototype;
+      // @ts-ignore
+      window.Date = SkewedDate;
+    }
+  }
+})();`;
+}
+
+/**
+ * Read the in-page stats counter and merge into the compiled-fault counters
+ * (`matched` and `fired`). Returns the merged stats; the compiled-fault
+ * objects are mutated in place so the next page picks up where this one
+ * left off.
+ *
+ * Stats keys are array indices so two faults with the same name don't
+ * collide their counters. Both legacy name-keyed snapshots and the
+ * current index-keyed shape are accepted: name-keyed entries are applied
+ * to the first compiled fault with that name (the safe, non-collapsing
+ * fallback used when reading older traces).
+ */
+export function mergeRuntimeStats(
+  compiled: CompiledRuntimeFault[],
+  pageStats: Record<string, { matched: number; fired: number }>,
+): RuntimeFaultStats[] {
+  for (let i = 0; i < compiled.length; i++) {
+    const c = compiled[i]!;
+    // Index-keyed (current shape).
+    const ps = pageStats[String(i)];
+    if (ps) {
+      c.matched += ps.matched;
+      c.fired += ps.fired;
+      continue;
+    }
+    // Backwards-compat: name-keyed (one slot per distinct name only;
+    // applied to the first compiled fault that wears that name).
+    const byName = pageStats[c.name];
+    if (byName && !compiled.slice(0, i).some((c2) => c2.name === c.name)) {
+      c.matched += byName.matched;
+      c.fired += byName.fired;
+    }
+  }
+  return compiled.map((c) => ({
+    rule: c.name,
+    matched: c.matched,
+    fired: c.fired,
+  }));
+}