@gigzen/populace 0.1.0

package/src/instrument.mjs

@@ -0,0 +1,299 @@
+// Wraps an adapter so every call through it is timed, counted, and its failures
+// grouped by message.
+//
+// This is what a customer actually buys. The simulation being interesting to
+// watch is not the product — the product is the evidence it leaves behind:
+// which of your endpoints broke, how often, under how many concurrent users,
+// and how slow they got while it was happening.
+//
+// Neither the engine nor the adapter knows this exists. The engine calls
+// `adapter.post(...)`; the adapter does its thing; the numbers accumulate in
+// between.
+
+import {
+  backoffMs,
+  CircuitBreaker,
+  isTransportError,
+  sleep,
+  TargetUnreachableError,
+} from "./net.mjs";
+
+const PERCENTILES = [50, 95, 99];
+
+// A call that never comes back is the worst failure mode this tool has, because
+// it does not look like a failure — it looks like nothing. A real run against a
+// flaky network froze at tick 24 of 60 and sat there silently until an external
+// timeout killed it 9 minutes later, producing no report at all. The customer's
+// API was fine; one socket died and the whole run went with it.
+//
+// So every adapter call gets a deadline. Past it we stop waiting, record a
+// normal failure, and let the other agents carry on. A slow API then shows up
+// as a timeout in the report — which is a finding — instead of a hung process,
+// which is nothing.
+export const DEFAULT_TIMEOUT_MS = 20_000;
+
+// Transport failures get retried; application failures never do. Three extra
+// attempts absorbs the ordinary blips of a real network — a CI runner losing a
+// socket, a staging box behind a flaky VPN — without papering over an endpoint
+// that is genuinely down, which still fails after the last attempt.
+export const DEFAULT_RETRIES = 3;
+
+// Consecutive transport failures before Populace concludes the target is gone
+// and stops the run. Twelve is roughly two full ticks' worth of calls for a
+// small population — long enough that a brief outage does not abort a good run,
+// short enough that a dead host is called in under a minute instead of grinding
+// out the full duration and producing nothing.
+export const DEFAULT_GIVE_UP_AFTER = 12;
+
+export class TimeoutError extends Error {
+  constructor(method, ms) {
+    super(`${method} timed out after ${ms}ms`);
+    this.name = "TimeoutError";
+    this.isTimeout = true;
+  }
+}
+
+/**
+ * Reject once `ms` has passed. Resolves to a cancel() so the timer is always
+ * cleared — an uncleared timer keeps the process alive past the end of a run,
+ * which would make `populace run` hang on exit for every fast call.
+ */
+function deadline(method, ms) {
+  let timer;
+  const promise = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new TimeoutError(method, ms)), ms);
+  });
+  return { promise, cancel: () => clearTimeout(timer) };
+}
+
+export function createMetrics() {
+  return { methods: new Map(), startedAt: Date.now(), endedAt: null };
+}
+
+function bucket(metrics, method) {
+  if (!metrics.methods.has(method)) {
+    metrics.methods.set(method, {
+      method,
+      calls: 0,
+      failures: 0,
+      // Failures split by whose problem they are. `apiFailures` are findings
+      // about the customer's code; `transportFailures` are the network between
+      // us and them, and must never be presented as the same thing.
+      apiFailures: 0,
+      transportFailures: 0,
+      // Attempts that failed at transport level and were retried. Kept and
+      // reported: a run that only survived on its fifth try is not the same as
+      // one that worked first time, and hiding that would inflate reliability.
+      retries: 0,
+      durations: [],
+      errors: new Map(),
+      firstErrorAt: null,
+    });
+  }
+  return metrics.methods.get(method);
+}
+
+/**
+ * Group errors by SHAPE, not by exact text. Two failures that differ only by a
+ * uuid or a row count are the same bug, and a report that lists them separately
+ * buries the signal it was written to surface.
+ */
+export function normaliseError(error) {
+  return String(error?.message || error || "unknown error")
+    .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, "<id>")
+    .replace(/\b\d+\b/g, "<n>")
+    .replace(/\s+/g, " ")
+    .trim()
+    .slice(0, 200);
+}
+
+export function instrument(
+  adapter,
+  metrics,
+  {
+    timeoutMs = DEFAULT_TIMEOUT_MS,
+    retries = DEFAULT_RETRIES,
+    giveUpAfter = DEFAULT_GIVE_UP_AFTER,
+    breaker = new CircuitBreaker({ threshold: giveUpAfter }),
+  } = {},
+) {
+  const wrapped = { name: adapter.name };
+  metrics.breaker = breaker;
+  // 0 or Infinity disables the deadline, for adapters whose work is legitimately
+  // long (a batch import, a deliberate slow-endpoint probe).
+  const limited = Number.isFinite(timeoutMs) && timeoutMs > 0;
+  const maxAttempts = Math.max(1, Number(retries) + 1);
+
+  for (const key of Object.keys(adapter)) {
+    const value = adapter[key];
+    if (typeof value !== "function") {
+      wrapped[key] = value;
+      continue;
+    }
+
+    /** One attempt, with the deadline applied. Timing is per-attempt. */
+    const attemptOnce = async (args) => {
+      const clock = limited ? deadline(key, timeoutMs) : null;
+      const started = performance.now();
+      try {
+        // We stop WAITING at the deadline; we cannot cancel the adapter's own
+        // work, so a late reply may still land and is simply ignored. Recording
+        // the timeout as the outcome is honest — the run did not get an answer
+        // in time, which is exactly what a user of the API would experience.
+        const call = value.apply(adapter, args);
+        // Never leave a late rejection unhandled: once we have raced past it,
+        // nothing is awaiting `call`, and an unhandled rejection would crash the
+        // run with a stack trace that has nothing to do with the real problem.
+        if (clock) call.then?.(undefined, () => {});
+        const result = clock ? await Promise.race([call, clock.promise]) : await call;
+        return { ok: true, result, ms: performance.now() - started };
+      } catch (error) {
+        return { ok: false, error, ms: performance.now() - started };
+      } finally {
+        clock?.cancel();
+      }
+    };
+
+    wrapped[key] = async (...args) => {
+      const entry = bucket(metrics, key);
+      entry.calls += 1;
+
+      // Open and still inside its cooldown: fail instantly rather than spend
+      // the full deadline rediscovering the same thing. allows() lets a single
+      // probe through once the cooldown expires, so a passing outage recovers
+      // instead of ending the run.
+      if (!breaker.allows()) {
+        const error = new TargetUnreachableError(breaker.openedAfter);
+        error.fromAdapter = true;
+        entry.failures += 1;
+        entry.transportFailures += 1;
+        entry.durations.push(0);
+        if (entry.firstErrorAt === null) entry.firstErrorAt = Date.now();
+        const shape = normaliseError(error);
+        entry.errors.set(shape, (entry.errors.get(shape) || 0) + 1);
+        throw error;
+      }
+
+      for (let attempt = 1; ; attempt++) {
+        const outcome = await attemptOnce(args);
+
+        if (outcome.ok) {
+          breaker.recordSuccess();
+          // Only the SUCCESSFUL attempt's duration is recorded. Including the
+          // failed attempts before it would fold network problems into the
+          // customer's latency figures and make p50/p95 unpublishable — which
+          // is exactly what went wrong in an earlier run of this tool.
+          entry.durations.push(outcome.ms);
+          return outcome.result;
+        }
+
+        const error = outcome.error;
+        const transport = isTransportError(error);
+
+        // An error the server RETURNED proves the link is alive, whatever it
+        // says about their code. That must reset the breaker, or a genuinely
+        // broken endpoint would look like a dead network and abort the run.
+        if (transport) breaker.recordTransportFailure();
+        else breaker.recordApiFailure();
+
+        // Retry ONLY when the server never answered. Any response the server
+        // actually produced — including a 500 — is a finding about their code,
+        // and retrying it would quietly turn a real bug into a green tick.
+        if (transport && attempt < maxAttempts && !breaker.open) {
+          entry.retries += 1;
+          await sleep(backoffMs(attempt));
+          continue;
+        }
+
+        // Mark it as the adapter's, so the engine can tell a customer's API
+        // failing apart from a bug of our own. Untagged failures reaching the
+        // agent loop are Populace's fault and must not be reported as theirs.
+        if (error && typeof error === "object") error.fromAdapter = true;
+        entry.durations.push(outcome.ms);
+        entry.failures += 1;
+        if (transport) entry.transportFailures += 1;
+        else entry.apiFailures += 1;
+        if (entry.firstErrorAt === null) entry.firstErrorAt = Date.now();
+        const shape = normaliseError(error);
+        entry.errors.set(shape, (entry.errors.get(shape) || 0) + 1);
+        throw error;
+      }
+    };
+  }
+  return wrapped;
+}
+
+function percentile(sorted, p) {
+  if (!sorted.length) return 0;
+  const i = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);
+  return sorted[Math.max(0, i)];
+}
+
+export function summarise(metrics) {
+  const methods = [...metrics.methods.values()].map((entry) => {
+    const sorted = [...entry.durations].sort((a, b) => a - b);
+    const latency = Object.fromEntries(
+      PERCENTILES.map((p) => [`p${p}`, Math.round(percentile(sorted, p))]),
+    );
+    return {
+      method: entry.method,
+      calls: entry.calls,
+      failures: entry.failures,
+      apiFailures: entry.apiFailures,
+      transportFailures: entry.transportFailures,
+      retries: entry.retries,
+      failureRate: entry.calls ? entry.failures / entry.calls : 0,
+      latencyMs: { ...latency, max: Math.round(sorted[sorted.length - 1] || 0) },
+      errors: [...entry.errors.entries()]
+        .map(([message, count]) => ({ message, count }))
+        .sort((a, b) => b.count - a.count),
+    };
+  });
+
+  // Sort by the customer's own failures first. On a bad link transport noise
+  // can dwarf a single real bug, and the bug is what they need to see at the
+  // top of the table.
+  methods.sort(
+    (a, b) => b.apiFailures - a.apiFailures || b.failures - a.failures || b.calls - a.calls,
+  );
+
+  const sum = (f) => methods.reduce((n, m) => n + f(m), 0);
+  const calls = sum((m) => m.calls);
+  const failures = sum((m) => m.failures);
+  const apiFailures = sum((m) => m.apiFailures);
+  const transportFailures = sum((m) => m.transportFailures);
+  const retries = sum((m) => m.retries);
+
+  return {
+    calls,
+    failures,
+    apiFailures,
+    transportFailures,
+    retries,
+    failureRate: calls ? failures / calls : 0,
+    // The rate that actually says something about their software. Reported
+    // separately so a bad link cannot inflate the number they are judged on.
+    apiFailureRate: calls ? apiFailures / calls : 0,
+    network: {
+      retries,
+      transportFailures,
+      // Attempts that had to be repeated, as a share of all attempts made.
+      // A high number here means the link was bad, NOT that the API was.
+      retryRate: calls + retries ? retries / (calls + retries) : 0,
+      healthy: retries === 0 && transportFailures === 0,
+      // Set when Populace concluded the target was gone and stopped early.
+      // Without this the report would show a short run full of failures and
+      // give no clue that it was cut short deliberately.
+      // gaveUp means ABANDONED, not merely open. A breaker that opened and
+      // then recovered is a run that survived an outage — reporting that as
+      // "gave up" would call a completed run incomplete.
+      gaveUp: Boolean(metrics.breaker?.abandoned),
+      gaveUpAfter: metrics.breaker?.openedAfter ?? null,
+      // Outages ridden out and recovered from. Worth reporting: it is the
+      // difference between a clean network and one that merely held together.
+      outages: metrics.breaker?.trips ?? 0,
+    },
+    durationMs: (metrics.endedAt || Date.now()) - metrics.startedAt,
+    methods,
+  };
+}

package/src/net.mjs

@@ -0,0 +1,175 @@
+// Telling "your API said no" apart from "we never reached your API".
+//
+// This distinction is the difference between a report a customer trusts and one
+// they argue with. A dropped socket is not a bug in their code, and a tool that
+// reports it as one will be dismissed the first time it cries wolf. Equally, a
+// 500 from their server IS their bug and must never be quietly retried away.
+//
+// So the rule is narrow on purpose: only failures that happened BELOW the HTTP
+// layer count as transport. Anything the server actually answered — any status
+// code at all — is the application's, and is reported exactly once.
+
+const TRANSPORT = [
+  "fetch failed", // undici's blanket wrapper; cause carries the detail
+  "ENOTFOUND", // DNS did not resolve
+  "EAI_AGAIN", // DNS temporary failure
+  "ECONNREFUSED",
+  "ECONNRESET",
+  "EPIPE",
+  "ETIMEDOUT",
+  "EHOSTUNREACH",
+  "ENETUNREACH",
+  "ENETDOWN",
+  "UND_ERR", // undici's own timeouts (connect, headers, body)
+  "socket hang up",
+  "network",
+  "Client network socket disconnected",
+];
+
+/**
+ * True when a call failed without the server ever answering.
+ *
+ * Matches against `cause.code` first — undici hides the real reason there and
+ * flattens everything to the message "fetch failed", which on its own tells you
+ * nothing.
+ */
+export function isTransportError(error) {
+  if (!error) return false;
+  if (error.isTimeout) return true; // our own deadline: no answer arrived
+  const text = String(
+    error.cause?.code || error.code || error.cause?.message || error.message || error,
+  );
+  return TRANSPORT.some((t) => text.toLowerCase().includes(t.toLowerCase()));
+}
+
+/**
+ * Exponential backoff with jitter.
+ *
+ * Jitter matters here more than usual: Populace runs N agents in lockstep on a
+ * tick, so a blip tends to hit all of them at once. Without jitter they would
+ * retry in unison and hammer an already-struggling API in synchronised waves —
+ * turning a small outage into a self-inflicted load test.
+ */
+export function backoffMs(attempt, base = 250, cap = 4000) {
+  const exponential = Math.min(cap, base * 2 ** (attempt - 1));
+  return Math.round(exponential / 2 + Math.random() * (exponential / 2));
+}
+
+export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+
+/**
+ * Stops retrying once the target is clearly gone rather than merely flaky.
+ *
+ * Retries fix a flaky link and make a dead one agonising: every call costs the
+ * full deadline times every attempt, so a run against an unreachable host
+ * crawls for twenty minutes and then produces nothing. Observed exactly that —
+ * the retry fix removed the hang and replaced it with a grind.
+ *
+ * So: after `threshold` transport failures in a row with no success in between,
+ * the breaker opens. Calls then fail immediately, the run ends early, and the
+ * report says the target became unreachable — in seconds instead of the best
+ * part of an hour.
+ *
+ * Any single success closes it again. A brief outage in the middle of an
+ * otherwise fine run must not abort that run; only a sustained one should.
+ */
+export class CircuitBreaker {
+  constructor({ threshold = 12, cooldownMs = 15_000, giveUpAfterProbes = 4 } = {}) {
+    this.threshold = threshold;
+    this.cooldownMs = cooldownMs;
+    // How many cooldowns may expire with the target still dead before we stop
+    // trying. Four at 15s means a run survives roughly a minute of outage and
+    // still abandons a genuinely dead host promptly.
+    this.giveUpAfterProbes = giveUpAfterProbes;
+
+    this.consecutive = 0;
+    this.open = false;
+    this.openedAfter = null;
+    this.openedAt = 0;
+    this.probes = 0;      // cooldowns elapsed without a success
+    this.trips = 0;       // how many times it opened during the run
+    this.abandoned = false; // permanently given up
+  }
+
+  /**
+   * Whether a call may go through right now.
+   *
+   * This is the half-open state, and leaving it out was a real bug: the breaker
+   * opened on the first burst of a flaky link and stayed open for the rest of
+   * the run, so a twenty-second blip killed a five-minute run that could have
+   * carried on. Network loss arrives in bursts, not as independent coin flips,
+   * so "N consecutive failures" is reached by any ordinary outage — the breaker
+   * has to be able to come back.
+   */
+  allows(now = Date.now()) {
+    if (this.abandoned) return false;
+    if (!this.open) return true;
+    if (now - this.openedAt < this.cooldownMs) return false;
+    // Cooldown elapsed: let exactly one call through to test the water.
+    this.openedAt = now;
+    this.probes += 1;
+    if (this.probes > this.giveUpAfterProbes) {
+      this.abandoned = true;
+      return false;
+    }
+    return true;
+  }
+
+  recordSuccess() {
+    this.consecutive = 0;
+    this.open = false;
+    // A probe that got through means the target is back. Clear the probe budget
+    // so a later, unrelated outage gets the full allowance again rather than
+    // inheriting the last one's.
+    this.probes = 0;
+  }
+
+  /**
+   * Force it shut for a new phase of work.
+   *
+   * Cleanup needs this. An open breaker made teardown fail all five deletions
+   * instantly and left five invented accounts live in a customer's project —
+   * the safety mechanism causing the exact harm the product promises to avoid.
+   * Cleanup is when you most need to keep trying, so it starts with a fresh
+   * budget rather than inheriting the run's verdict on the network.
+   */
+  reset() {
+    this.consecutive = 0;
+    this.open = false;
+    this.openedAfter = null;
+    this.openedAt = 0;
+    this.probes = 0;
+    this.abandoned = false;
+  }
+
+  /** @returns {boolean} true when this failure was the one that opened it. */
+  recordTransportFailure(now = Date.now()) {
+    if (this.threshold <= 0) return false; // disabled
+    this.consecutive += 1;
+    if (!this.open && this.consecutive >= this.threshold) {
+      this.open = true;
+      this.openedAfter = this.consecutive;
+      this.openedAt = now;
+      this.trips += 1;
+      return true;
+    }
+    return false;
+  }
+
+  /** An application error says the server IS answering, so the link is alive. */
+  recordApiFailure() {
+    this.recordSuccess();
+  }
+}
+
+export class TargetUnreachableError extends Error {
+  constructor(consecutive) {
+    super(
+      `Target became unreachable — ${consecutive} consecutive calls failed at the network ` +
+        `level with no response. Stopping rather than retrying for the rest of the run.`,
+    );
+    this.name = "TargetUnreachableError";
+    this.isTransport = true;
+    this.circuitOpen = true;
+  }
+}