@gigzen/populace 0.1.0 → 1.1.0

package/src/ai.mjs

@@ -0,0 +1,158 @@
+// The model layer: explanations for failures no rule could name.
+//
+// Deliberately the SECOND thing tried, never the first. src/explain.mjs handles
+// the recognisable shapes — permissions, foreign keys, quotas, sockets — with
+// rules that are instant, free, offline and auditable. This is for the
+// remainder, and it only ever sees the remainder.
+//
+// Raw fetch rather than @anthropic-ai/sdk, and that is a deliberate trade. The
+// SDK is the better tool in almost any other project; here it would be
+// Populace's first runtime dependency, and "zero dependencies" is not a
+// slogan — it is why `npx @gigzen/populace` works with nothing to install.
+// One POST is a fair price for keeping that true.
+//
+// WHAT IS SENT, exactly:
+//   · the method name        e.g. "sendMessage"
+//   · the error message      e.g. "TypeError: fetch failed"
+//   · how many times         e.g. 17
+//   · p50/p95 for that method
+// Nothing else. No target URL, no keys, no tokens, no request bodies, no
+// simulated users' content. Everything sent is printed first when --verbose is
+// on, because a tool that quietly ships your data somewhere is not one you
+// should trust with your staging credentials.
+
+const ENDPOINT = "https://api.anthropic.com/v1/messages";
+const MODEL = "claude-opus-5";
+const VERSION = "2023-06-01";
+
+/** The key, from the environment or the config. Never written to disk by us. */
+export function apiKey(config) {
+  return process.env.ANTHROPIC_API_KEY || config?.ai?.apiKey || null;
+}
+
+export const isConfigured = (config) => Boolean(apiKey(config));
+
+const SYSTEM = `You explain failures from a load-testing tool called Populace, which drives an
+application through its own API as a population of simulated users.
+
+You are the fallback. A rule engine already handled every failure it recognised —
+row-level security, foreign keys, expired sessions, provider rate limits, socket
+errors. What reaches you is what it could not name.
+
+For each failure, answer three things:
+  headline  one sentence: what happened. No preamble.
+  why       two or three sentences: the mechanism. What would produce this?
+  fix       one or two sentences: what to change. Concrete.
+  blame     exactly one of: app | environment | harness | unknown
+            app         the application under test is at fault
+            environment the platform or provider refused it (quotas, limits)
+            harness     it never reached the server (sockets, DNS, the client)
+            unknown     you genuinely cannot tell
+
+Rules you must follow:
+- If you cannot tell, say so and use blame "unknown". A confident wrong cause
+  costs more to debug than an honest blank.
+- Never invent a status code, table name, policy name or line number that was
+  not in the input.
+- Prefer the boring explanation. Under concurrency, most failures are
+  contention, permissions, or a resource limit — not exotic.
+
+Reply with JSON only, no prose around it:
+{"explanations":[{"method":"...","headline":"...","why":"...","fix":"...","blame":"..."}]}`;
+
+/**
+ * Ask the model about failures the rules could not name.
+ *
+ * Returns [] on any problem — a missing key, a refusal, a network error, a
+ * malformed reply. This is an enhancement to a report that is already complete
+ * and useful; it must never be the reason a run fails or a report is not
+ * written.
+ */
+export async function explainWithAI(unexplained, { config, verbose = false, timeoutMs = 30000 } = {}) {
+  const key = apiKey(config);
+  if (!key || !unexplained?.length) return [];
+
+  const payload = unexplained.map((e) => ({
+    method: e.method,
+    error: String(e.message ?? "").slice(0, 500),
+    occurrences: e.count ?? 1,
+    p50ms: e.p50 ?? null,
+    p95ms: e.p95 ?? null,
+  }));
+
+  if (verbose) {
+    console.log("\n  Sending to the model — this is everything, nothing else leaves this machine:");
+    console.log(JSON.stringify(payload, null, 2).split("\n").map((l) => `    ${l}`).join("\n"));
+  }
+
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+
+  try {
+    const res = await fetch(ENDPOINT, {
+      method: "POST",
+      signal: controller.signal,
+      headers: {
+        "content-type": "application/json",
+        "x-api-key": key,
+        "anthropic-version": VERSION,
+      },
+      body: JSON.stringify({
+        model: MODEL,
+        max_tokens: 2000,
+        // Effort low: this is a small, bounded task on a handful of short
+        // strings, and a CLI that pauses for twenty seconds to explain an
+        // error is a CLI people turn off.
+        output_config: { effort: "low" },
+        system: SYSTEM,
+        messages: [{
+          role: "user",
+          content:
+            `Failures a rule engine could not classify:\n\n${JSON.stringify(payload, null, 2)}\n\n` +
+            `Explain each one.`,
+        }],
+      }),
+    });
+
+    if (!res.ok) {
+      const body = await res.text().catch(() => "");
+      return fail(res.status === 401
+        ? "the API key was rejected"
+        : res.status === 429
+          ? "rate limited by the API"
+          : `the API returned ${res.status}${body ? `: ${body.slice(0, 160)}` : ""}`);
+    }
+
+    const data = await res.json();
+
+    // A safety refusal is a normal outcome, not a crash.
+    if (data.stop_reason === "refusal") return fail("the model declined to answer");
+
+    const text = (data.content || []).filter((b) => b.type === "text").map((b) => b.text).join("");
+    const json = text.match(/\{[\s\S]*\}/);
+    if (!json) return fail("the reply was not JSON");
+
+    const parsed = JSON.parse(json[0]);
+    const BLAME = new Set(["app", "environment", "harness", "unknown"]);
+
+    return (parsed.explanations || []).map((e) => ({
+      method: String(e.method ?? "unknown"),
+      headline: String(e.headline ?? "").trim(),
+      why: String(e.why ?? "").trim(),
+      fix: e.fix ? String(e.fix).trim() : undefined,
+      // An unrecognised blame value becomes "unknown" rather than being trusted.
+      blame: BLAME.has(e.blame) ? e.blame : "unknown",
+      rule: "model",
+      source: "model",
+    })).filter((e) => e.headline);
+  } catch (error) {
+    return fail(error.name === "AbortError" ? `no reply within ${timeoutMs / 1000}s` : error.message);
+  } finally {
+    clearTimeout(timer);
+  }
+
+  function fail(why) {
+    console.log(`  (the model layer was skipped — ${why})`);
+    return [];
+  }
+}

package/src/cli.mjs

@@ -12,6 +12,7 @@ import { fileURLToPath } from "node:url";
 import { ConfigError, loadAdapter, loadConfig } from "./config.mjs";
 import { createMetrics, instrument } from "./instrument.mjs";
 import { buildReport, renderReport, writeReport } from "./report.mjs";
+import { createProgress } from "./progress.mjs";
 import { canSignInOnly } from "./contract.mjs";
 import { isTransportError } from "./net.mjs";
 import { diagnose } from "./diagnose.mjs";
@@ -19,6 +20,10 @@ import { renderSmoke, smoke, smokePersona } from "./smoke.mjs";
 import { PACKAGE_NAME, VERSION } from "./version.mjs";
 import { World } from "./engine/world.mjs";
 import { buildPersonas, CITIES } from "./engine/personas.mjs";
+import * as openapi from "./openapi.mjs";
+import { explainReport, verdictLine } from "./explain.mjs";
+import { explainWithAI, isConfigured } from "./ai.mjs";
+import { updateCommand, updateNotice } from "./update.mjs";
 import { Agent } from "./engine/agent.mjs";
 
 const here = path.dirname(fileURLToPath(import.meta.url));
@@ -49,7 +54,9 @@ function overridesFromFlags() {
   if (flag("minutes")) o.minutes = Number(flag("minutes"));
   if (flag("tick")) o.tickSeconds = Number(flag("tick"));
   if (flag("cities")) o.cities = String(flag("cities")).split(",").map((s) => s.trim()).filter(Boolean);
+  if (flag("engagement")) o.engagement = Number(flag("engagement"));
   if (flag("report")) o.reportPath = flag("report");
+  if (flag("stagger")) o.signupStaggerMs = Number(flag("stagger"));
   return o;
 }
 
@@ -90,7 +97,20 @@ async function init() {
 
   fs.mkdirSync(adapterDir, { recursive: true });
   fs.copyFileSync(path.join(here, "..", "populace.config.example.mjs"), configFile);
-  if (!fs.existsSync(adapterFile) || has("force")) {
+
+  // --from-openapi fills the template's paths in from an API description. The
+  // adapter is still a draft afterwards; the point is to remove the half hour
+  // of looking up thirteen endpoints by hand, not to finish the job.
+  const specPath = flag("from-openapi", null);
+  let matched = null;
+  if (specPath && !blank) {
+    const doc = openapi.load(specPath);
+    const { operationCount, results } = openapi.match(doc);
+    const template = fs.readFileSync(path.join(here, "..", "adapters", templateName), "utf8");
+    const { source, applied } = openapi.fill(template, results);
+    if (!fs.existsSync(adapterFile) || has("force")) fs.writeFileSync(adapterFile, source);
+    matched = { operationCount, results, applied };
+  } else if (!fs.existsSync(adapterFile) || has("force")) {
     fs.copyFileSync(path.join(here, "..", "adapters", templateName), adapterFile);
   }
 
@@ -100,6 +120,32 @@ async function init() {
     adapters/my-app.mjs      ← ${blank ? "an empty contract to fill in" : "a working REST adapter to edit"}
 `);
 
+  if (matched) {
+    const { operationCount, results } = matched;
+    const order = { high: 0, medium: 1, low: 2, none: 3 };
+    const rows = Object.entries(results).sort((a, b) => order[a[1].confidence] - order[b[1].confidence]);
+    const found = rows.filter(([, r]) => r.op).length;
+
+    console.log(`  Read ${operationCount} operations from ${path.basename(specPath)} and matched ${found} of 13.
+`);
+    for (const [method, r] of rows) {
+      const mark = { high: "✔", medium: "~", low: "?", none: "✖" }[r.confidence];
+      const where = r.op ? `${r.op.verb.toUpperCase()} ${r.op.path}` : "left as the template default";
+      console.log(`    ${mark} ${method.padEnd(21)} ${where}`);
+      if (r.confidence === "low" || r.confidence === "none") {
+        console.log(`        ${r.why[0]} — check this one`);
+      }
+    }
+    console.log(`
+  These are guesses from endpoint names. Request bodies and field names are
+  still the template's defaults, so the adapter is a draft, not finished.
+
+  Next:  populace smoke      ← exercises each method once and names the first
+                               one that is wrong
+`);
+    return;
+  }
+
   if (blank) {
     console.log(`  Every method throws until you write it. Start with createUser and
   deleteUser — those two are required.
@@ -143,6 +189,28 @@ async function doctor() {
 
   const d = diagnose({ config, adapter: raw, reachable });
 
+  // --json for anything that has to describe a config without printing it -
+  // the desktop app shows this on the Run screen, so you can see what you are
+  // about to point a population at before you start one.
+  if (has("json")) {
+    console.log(JSON.stringify({
+      app: config.app || null,
+      environment: config.environment,
+      adapter: config.adapter,
+      target: config.target?.url || null,
+      coverage: d.coverage,
+      guarded: d.guarded,
+      cleanup: d.cleanup,
+      reachable,
+      reachError: reachable === false ? reachError : null,
+      ready: d.ready,
+      blockers: d.blockers,
+      population: config.population,
+    }));
+    if (!d.ready) process.exitCode = 1;
+    return;
+  }
+
   console.log(`
   Config    ${path.basename(config._file)}`);
   console.log(`  App       ${config.app || raw.name || "(unnamed)"}`);
@@ -187,13 +255,26 @@ async function run() {
   const startedAt = Date.now();
 
   const { agents, cities, minutes, tickSeconds } = config.population;
+
+  // Anything watching this run rather than reading it - the desktop app today,
+  // a dashboard tomorrow - gets the same numbers the terminal table shows,
+  // every tick, as JSON. Off unless asked, so ordinary output is untouched.
+  const progress = createProgress({ enabled: flag("progress") === "json" });
+  progress.start(config);
+
   console.log(`\n  Bringing ${agents} people to life across ${cities.join(", ")}…\n`);
 
   const world = World.fromConfig(config, adapter, {
     joined: (a) => console.log(`   ✓ ${a.persona.name} (${a.persona.city.name}, ${a.persona.platform})`),
     joinFailed: (p, e) => console.log(`   ✖ ${p.name}: ${e.message}`),
+    // Said out loud, because waiting silently is indistinguishable from hanging.
+    // Retrying a throttled sign-up can add seconds per person, and a run that
+    // pauses without explanation is a run somebody kills.
+    joinThrottled: (p, attempt) =>
+      console.log(`   … ${p.name}: rate limited, waiting (attempt ${attempt})`),
     tick: (n, total, w) => {
       render(config, n, total, w);
+      progress.tick(n, total, w, metrics);
       // Stop as soon as the target is judged gone. Grinding out the remaining
       // ticks against a dead host wastes the operator's time and adds nothing
       // to the report.
@@ -236,11 +317,18 @@ async function run() {
 
   metrics.endedAt = Date.now();
   const report = buildReport({ config, adapter: raw, world, metrics, teardown, startedAt });
+  progress.done(report);
   const files = writeReport(report, config);
   console.log(renderReport(report));
   console.log(`  Report:  ${displayPath(files.json)}`);
   console.log(`  Shareable page:  ${displayPath(files.html)}\n`);
 
+  // After the report a person is already reading, never before it, and at most
+  // once a day. Silent if the registry cannot be reached — a version check is
+  // never worth an error message at the end of a successful run.
+  const notice = await updateNotice();
+  if (notice) console.log(`${notice}\n`);
+
   if (report.verdict.status !== "clean") process.exitCode = 1;
 }
 
@@ -544,7 +632,104 @@ async function version() {
   console.log(`${PACKAGE_NAME} ${VERSION}  ·  node ${process.version}  ·  ${process.platform}`);
 }
 
-const commands = { init, doctor, run, clean, demo, report, version, smoke: smokeCmd };
+
+// ---------------------------------------------------------------- explain
+
+/**
+ * Explain a report's failures: rules first, then the model for the remainder.
+ *
+ * A separate command rather than part of `run` on purpose. A run should not
+ * wait on a network call to something else, and it should never fail because
+ * an API key was wrong. The report is complete without this.
+ */
+async function explainCmd() {
+  const reportPath = flag("file", flag("report", "populace-report.json"));
+  let report;
+  try {
+    report = JSON.parse(fs.readFileSync(reportPath, "utf8"));
+  } catch (error) {
+    console.log(`
+  No readable report at ${displayPath(reportPath)} — ${error.message}
+`);
+    process.exit(1);
+  }
+
+  const explained = explainReport(report);
+
+  // --json for anything rendering this itself rather than reading a terminal.
+  // Same explanations, same order; only the presentation differs, so a window
+  // cannot show a cause the command line would not.
+  if (has("json")) {
+    console.log(JSON.stringify({
+      report: reportPath,
+      verdict: explained.length ? verdictLine(explained) : null,
+      explained,
+    }));
+    return;
+  }
+
+  if (!explained.length) {
+    console.log(`
+  Nothing failed in ${displayPath(reportPath)}. Nothing to explain.
+`);
+    return;
+  }
+
+  const BLAME = { app: "YOUR APP", environment: "THE PLATFORM", harness: "THE TEST CLIENT", unknown: "UNKNOWN" };
+  const show = (e) => {
+    const tag = e.source === "model" ? "  ·  explained by the model" : "";
+    console.log(`
+  [${BLAME[e.blame]}]  ${e.method} × ${e.count ?? 1}${tag}`);
+    console.log(`    ${e.headline}`);
+    console.log(`    ${e.why}`);
+    if (e.fix) console.log(`    Fix: ${e.fix}`);
+  };
+
+  console.log(`
+  ${verdictLine(explained)}`);
+  const named = explained.filter((e) => e.rule !== "none");
+  const unnamed = explained.filter((e) => e.rule === "none");
+  for (const e of named) show(e);
+
+  if (!unnamed.length) {
+    console.log(`
+  Every failure matched a rule. The model was not needed.
+`);
+    return;
+  }
+
+  console.log(`
+  ${unnamed.length} failure${unnamed.length === 1 ? "" : "s"} matched no rule.`);
+
+  const { config } = await open().catch(() => ({ config: null }));
+  if (!isConfigured(config)) {
+    console.log(`
+  Set ANTHROPIC_API_KEY to have the model explain these. It is sent the method
+  name, the error text, how many times it happened and that method's latency —
+  and nothing else. Run with --verbose to see exactly what leaves the machine.
+`);
+    for (const e of unnamed) show(e);
+    return;
+  }
+
+  console.log(`  Asking the model…`);
+  const ai = await explainWithAI(
+    unnamed.map((e) => {
+      const m = (report.api?.methods || []).find((x) => x.method === e.method);
+      return { ...e, p50: m?.latencyMs?.p50, p95: m?.latencyMs?.p95 };
+    }),
+    { config, verbose: has("verbose") },
+  );
+
+  const byMethod = new Map(ai.map((e) => [e.method, e]));
+  for (const e of unnamed) {
+    const better = byMethod.get(e.method);
+    show(better ? { ...e, ...better } : e);
+  }
+  console.log("");
+}
+
+const commands = { init, doctor, run, clean, demo, report, version, smoke: smokeCmd, explain: explainCmd, update: updateCommand };
 
 // `--version` and `-v` are what people actually type.
 if (has("version") || argv[0] === "-v") {
@@ -563,6 +748,8 @@ if (!commands[command]) {
     populace run                      bring the population to life
     populace clean                    delete accounts a run created
     populace report                   re-open the report from an earlier run
+    populace explain                  say what each failure means and how to fix it
+    populace update                   check whether a newer Populace is out
     populace version                  print version and environment
 
   Options
@@ -570,6 +757,9 @@ if (!commands[command]) {
     --agents <n>  --minutes <n>       override the config
     --tick <seconds>                  simulated seconds per step
     --cities <a,b>                    ${Object.keys(CITIES).join(", ")}
+    --engagement <x>                  how busy people are; 1 = normal, 5 = relentless
+    --stagger <ms>                    gap between sign-ups; raise it for a
+                                      throttled auth endpoint (default 400)
     --report <path>                   where to write the report
     --keep                            leave accounts in place after a run
     --file <path>                     which report to re-open (report)

package/src/config.mjs

@@ -36,6 +36,19 @@ const DEFAULTS = {
   // Consecutive unreachable calls before Populace declares the target down and
   // stops, instead of retrying every call for the rest of the run. 0 disables.
   giveUpAfter: 12,
+  // Gap between sign-ups. Auth endpoints are throttled far harder than the rest
+  // of an API — Supabase's default is 30 sign-ups per five minutes per address,
+  // and 400ms is 2.5 per second. That default is right for real users, who each
+  // arrive from their own address, and impossible for a simulation, where every
+  // request shares one. Raise this to fit a target you do not control; a hosted
+  // Supabase project needs about 10_000.
+  signupStaggerMs: 400,
+  // When a sign-up is refused *for being too fast*, wait and try that person
+  // again rather than recording them as a failure. A rate limit says nothing
+  // about the application under test, so counting it as a finding is a lie.
+  // Backoff is this value times the attempt number. 0 disables retrying.
+  signupRateLimitBackoffMs: 5_000,
+  signupRateLimitRetries: 2,
   population: { agents: 6, cities: ["manila", "mumbai"], tickSeconds: 5, minutes: 10 },
   // Comfortably inside a 1-hour token, which is the common default.
   session: { refreshEveryMinutes: 30 },
@@ -43,8 +56,8 @@ const DEFAULTS = {
 };
 
 export async function loadConfig({ configPath, cwd = process.cwd(), overrides = {} } = {}) {
-  // reportPath is not a population setting; keep it out of that spread.
-  const { reportPath: _reportPath, ...populationOverrides } = overrides;
+  // Neither of these is a population setting; keep them out of that spread.
+  const { reportPath: _reportPath, signupStaggerMs: _stagger, ...populationOverrides } = overrides;
   const file = path.resolve(cwd, configPath || "populace.config.mjs");
 
   if (!fs.existsSync(file)) {
@@ -63,6 +76,9 @@ export async function loadConfig({ configPath, cwd = process.cwd(), overrides =
   const config = {
     ...DEFAULTS,
     ...loaded,
+    // --stagger wins over the config file, so a run can be paced to fit a target
+    // whose rate limit is not yours to change.
+    ...(overrides.signupStaggerMs ? { signupStaggerMs: overrides.signupStaggerMs } : {}),
     population: {
       ...DEFAULTS.population,
       ...(loaded.population || {}),