@moda-ai/cli 1.41.2 → 1.42.0

package/dist/cli.js

@@ -15,6 +15,7 @@ import {
   HARNESS_REPORT_PATH,
   SIGNALS_MANAGEMENT_SUBCOMMANDS,
   assertFlags,
+  assertPositionals,
   callControlAPI,
   callDataAPI,
   getApiRequestCount,
@@ -2500,6 +2501,183 @@ function isDefined(value) {
   return value !== undefined;
 }
 
+// src/replays.ts
+var UUID_RE2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+var PLAYOUT_ID_RE = /^[0-9a-f]{64}$/i;
+var ARMS = ["prod", "proposed"];
+var REPLAYS_SUBCOMMANDS = ["sets", "runs", "run", "playouts", "playout"];
+var REPLAYS_SUBCOMMAND_HELP = [
+  {
+    name: "sets",
+    description: "List replay sets (ids, names, case counts)",
+    usage: "moda replays sets",
+    examples: ["moda replays sets --json"]
+  },
+  {
+    name: "runs",
+    description: "Run history for a set, newest first, every status",
+    usage: "moda replays runs <set_id> [--limit=N]",
+    flags: [{ name: "--limit=N", description: "Runs to return (default 20, max 100)." }],
+    examples: ["moda replays runs <set_id> --limit=5 --json"]
+  },
+  {
+    name: "run",
+    description: "One run: summary, per-case aggregate, every case x arm x seed verdict, playout count",
+    usage: "moda replays run <set_id> <run_id|latest>",
+    examples: ["moda replays run <set_id> <run_id> --json", "moda replays run <set_id> latest --json"]
+  },
+  {
+    name: "playouts",
+    description: "Playouts of a run (one per case x arm x seed) with verdicts; --transcript inlines what was said and every tool call",
+    usage: "moda replays playouts <set_id> <run_id> [--case=<case_id>] [--arm=prod|proposed] [--seed=N] [--transcript] [--max-chars=N] [--limit=N] [--cursor=<next_cursor>]",
+    flags: [
+      { name: "--case=<case_id>", description: "Only this case." },
+      { name: "--arm=prod|proposed", description: "Only this arm (prod = baseline, proposed = candidate)." },
+      { name: "--seed=N", description: "Only this seed." },
+      { name: "--transcript", description: "Inline each transcript (messages, tool calls with served output and provenance, criteria). Page cap 10." },
+      { name: "--max-chars=N", description: "Per-field cap for message text and tool JSON (default 4000, 100-32000)." },
+      { name: "--limit=N", description: "Page size (default 50, max 200; 10 with --transcript)." },
+      { name: "--cursor=<next_cursor>", description: "Next page: the previous response's nextCursor." }
+    ],
+    examples: [
+      "moda replays playouts <set_id> <run_id> --json",
+      "moda replays playouts <set_id> <run_id> --case=<case_id> --arm=proposed --seed=1 --transcript --json"
+    ]
+  },
+  {
+    name: "playout",
+    description: "One playout with its full transcript",
+    usage: "moda replays playout <set_id> <run_id> <playout_id> [--max-chars=N]",
+    flags: [{ name: "--max-chars=N", description: "Per-field cap (default 4000, 100-32000)." }],
+    examples: ["moda replays playout <set_id> <run_id> <playout_id> --json"]
+  }
+];
+function profileOptions(context) {
+  return { profile: context.profile, env: context.env };
+}
+function requireTenant(context) {
+  const key = resolveApiKey(profileOptions(context));
+  if (!key) {
+    throw new CliInputError("No Moda API key found for moda replays.", "Set MODA_API_KEY, run `moda auth login --api-key`, or select a profile with an API key.");
+  }
+  const keyTenant = embeddedTenantIdFromApiKey(key);
+  const flagTenant = context.flags["tenant-id"];
+  if (flagTenant && keyTenant && flagTenant !== keyTenant) {
+    throw new CliInputError(`--tenant-id=${flagTenant} does not match this API key's tenant (${keyTenant}), so nothing was sent.`, "Drop --tenant-id (the key names its tenant) or use a key for that tenant.");
+  }
+  const tenantId = flagTenant || resolveApiTenantId(profileOptions(context));
+  if (!tenantId) {
+    throw new CliInputError("Could not determine the tenant.", "Use a signed API key (it names its tenant), set MODA_TENANT_ID, or pass --tenant-id=<id>.");
+  }
+  return tenantId;
+}
+function intFlag(name, value, min, max) {
+  if (value === undefined)
+    return;
+  const parsed = Number(value);
+  if (!Number.isInteger(parsed) || parsed < min || parsed > max) {
+    throw new CliInputError(`--${name} must be an integer from ${min} to ${max} (got '${value}').`);
+  }
+  return String(parsed);
+}
+function query(params) {
+  const search = new URLSearchParams(Object.entries(params).filter((entry) => entry[1] !== undefined)).toString();
+  return search ? `?${search}` : "";
+}
+function needArg(value, name, usage) {
+  if (!value)
+    throw new CliInputError(`<${name}> is required.`, `Usage: ${usage}`);
+  return value;
+}
+function needRunId(value, usage, allowLatest = false) {
+  const runId = needArg(value, "run_id", usage);
+  if (allowLatest && runId === "latest")
+    return runId;
+  if (!UUID_RE2.test(runId)) {
+    throw new CliInputError(`<run_id> must be a run UUID (got '${runId}').`, "List runs with `moda replays runs <set_id>`.");
+  }
+  return runId;
+}
+async function runReplaysCommand(context) {
+  const [subcommand, ...args] = context.positionals;
+  const flags = context.flags;
+  const help = REPLAYS_SUBCOMMAND_HELP.find((entry) => entry.name === subcommand);
+  if (!subcommand || !help) {
+    throw new CliInputError(subcommand ? `Unknown replays command '${subcommand}'.` : "A replays subcommand is required.", `Use one of: ${REPLAYS_SUBCOMMANDS.join(", ")}. See \`moda replays --help\`.`);
+  }
+  const usage = help.usage;
+  const allowed = {
+    sets: [],
+    runs: ["limit"],
+    run: [],
+    playouts: ["case", "arm", "seed", "transcript", "max-chars", "limit", "cursor"],
+    playout: ["max-chars"]
+  };
+  const name = subcommand;
+  assertFlags(`replays ${name}`, flags, [...allowed[name], "tenant-id"]);
+  const maxArgs = { sets: 0, runs: 1, run: 2, playouts: 2, playout: 3 }[name];
+  assertPositionals(`replays ${name}`, args, maxArgs, usage);
+  const tenantId = requireTenant(context);
+  const base = `/tenants/${encodeURIComponent(tenantId)}/replay-sets`;
+  let endpoint;
+  switch (name) {
+    case "sets":
+      endpoint = base;
+      break;
+    case "runs": {
+      const setId = needArg(args[0], "set_id", usage);
+      endpoint = `${base}/${encodeURIComponent(setId)}/runs${query({ limit: intFlag("limit", flags.limit, 1, 100) })}`;
+      break;
+    }
+    case "run": {
+      const setId = needArg(args[0], "set_id", usage);
+      const runId = needRunId(args[1], usage, true);
+      endpoint = runId === "latest" ? `${base}/${encodeURIComponent(setId)}/runs/latest` : `${base}/${encodeURIComponent(setId)}/runs/${encodeURIComponent(runId)}`;
+      break;
+    }
+    case "playouts": {
+      const setId = needArg(args[0], "set_id", usage);
+      const runId = needRunId(args[1], usage);
+      const arm = flags.arm;
+      if (arm !== undefined && !ARMS.includes(arm)) {
+        throw new CliInputError(`--arm must be one of ${ARMS.join(", ")} (got '${arm}').`);
+      }
+      if (flags.case !== undefined && (flags.case.length === 0 || flags.case.length > 128)) {
+        throw new CliInputError("--case must be a case id of 1 to 128 characters.");
+      }
+      if (flags.cursor !== undefined && !/^\d{1,9}$/.test(flags.cursor)) {
+        throw new CliInputError(`--cursor must be the nextCursor of a previous page (got '${flags.cursor}').`);
+      }
+      const transcript = flags.transcript === "true";
+      endpoint = `${base}/${encodeURIComponent(setId)}/runs/${encodeURIComponent(runId)}/playouts${query({
+        caseId: flags.case,
+        arm,
+        seed: intFlag("seed", flags.seed, 0, 1000),
+        include: transcript ? "transcript" : undefined,
+        maxChars: intFlag("max-chars", flags["max-chars"], 100, 32000),
+        limit: intFlag("limit", flags.limit, 1, transcript ? 10 : 200),
+        cursor: flags.cursor
+      })}`;
+      break;
+    }
+    case "playout": {
+      const setId = needArg(args[0], "set_id", usage);
+      const runId = needRunId(args[1], usage);
+      const playoutId = needArg(args[2], "playout_id", usage);
+      if (!PLAYOUT_ID_RE.test(playoutId)) {
+        throw new CliInputError(`<playout_id> must be a 64-character hex id (got '${playoutId}').`, "List playouts with `moda replays playouts <set_id> <run_id>`.");
+      }
+      endpoint = `${base}/${encodeURIComponent(setId)}/runs/${encodeURIComponent(runId)}/playouts/${encodeURIComponent(playoutId)}${query({
+        maxChars: intFlag("max-chars", flags["max-chars"], 100, 32000)
+      })}`;
+      break;
+    }
+  }
+  const data = await callControlAPI(endpoint, { method: "GET" }, profileOptions(context), { timeoutMs: 60000 });
+  context.output.writeData(data ?? {});
+  return { exitCode: 0 };
+}
+
 // src/telemetry.ts
 import { randomUUID } from "node:crypto";
 var TELEMETRY_TIMEOUT_MS = 500;
@@ -2514,8 +2692,8 @@ var SEARCH_SURFACE_BY_COMMAND = {
   context: "context"
 };
 var SEARCH_MODES2 = ["keyword", "semantic", "hybrid"];
-function termCount(query) {
-  return query.trim().split(/\s+/).filter(Boolean).length;
+function termCount(query2) {
+  return query2.trim().split(/\s+/).filter(Boolean).length;
 }
 function clampInt(value, min, max) {
   if (typeof value !== "number" || !Number.isFinite(value))
@@ -5180,8 +5358,8 @@ function parsePayload(payload) {
 }
 var DEFAULT_STREAM_TIMEOUT_MS = 250000;
 async function streamAsk(options) {
-  const profileOptions = options.profile ? { profile: options.profile } : {};
-  const url = `${resolveModaBaseUrl(profileOptions)}/api/cli/ask/stream`;
+  const profileOptions2 = options.profile ? { profile: options.profile } : {};
+  const url = `${resolveModaBaseUrl(profileOptions2)}/api/cli/ask/stream`;
   const controller = new AbortController;
   const timeout = setTimeout(() => controller.abort(), options.timeoutMs ?? DEFAULT_STREAM_TIMEOUT_MS);
   const startedAt = Date.now();
@@ -5197,7 +5375,7 @@ async function streamAsk(options) {
       response = await fetchImpl(url, {
         method: "POST",
         headers: {
-          "x-api-key": resolveApiKey(profileOptions) ?? "",
+          "x-api-key": resolveApiKey(profileOptions2) ?? "",
           "Content-Type": "application/json",
           Accept: "text/event-stream"
         },
@@ -7005,6 +7183,15 @@ Commands:
   prompts ab                       Run a judged prompt A/B replay experiment
   prompts propose                  Revise a prompt from a run's failures (--gate to auto-A/B, --promote-on-win)
 
+  Replays (read-only results):
+  replays sets                     List replay sets
+  replays runs <set_id>            Run history for a set (every status)
+  replays run <set_id> <run_id>    One run: per-case and per-seed verdicts, playout count
+  replays playouts <set_id> <run_id>
+                                   Playouts per case/arm/seed (--transcript inlines messages and tool calls)
+  replays playout <set_id> <run_id> <playout_id>
+                                   One playout transcript
+
   Skills:
   skills gen                       Kick off tenant-wide skill generation
   skills status [run-id]           Show a generation run's status, outcome, and recent events (default: latest run)
@@ -7383,10 +7570,10 @@ async function attachWindows(rows, windowSize) {
       return { ...row, window: conversation };
     }
     try {
-      const query = new URLSearchParams;
-      query.set("msg_index", msg_index.toString());
-      query.set("window", windowSize.toString());
-      const data = await callDataAPI(`/conversations/${conv_id}/context?${query.toString()}`);
+      const query2 = new URLSearchParams;
+      query2.set("msg_index", msg_index.toString());
+      query2.set("window", windowSize.toString());
+      const data = await callDataAPI(`/conversations/${conv_id}/context?${query2.toString()}`);
       return { ...row, window: data };
     } catch (err) {
       const message = err instanceof Error ? err.message : String(err);
@@ -7787,15 +7974,15 @@ function shellArg(value) {
 async function searchFallbackToTraces(params) {
   const limit = Math.min(params.limit ?? 20, 100);
   const offset = params.offset ?? 0;
-  const query = new URLSearchParams;
-  query.set("search", params.q);
+  const query2 = new URLSearchParams;
+  query2.set("search", params.q);
   if (params.user_id)
-    query.set("user_id", params.user_id);
+    query2.set("user_id", params.user_id);
   if (params.time_range)
-    query.set("time_range", params.time_range);
-  query.set("limit", String(limit));
+    query2.set("time_range", params.time_range);
+  query2.set("limit", String(limit));
   if (offset > 0)
-    query.set("offset", String(offset));
+    query2.set("offset", String(offset));
   const tracesCommand = [
     `moda traces --search=${shellArg(params.q)}`,
     params.user_id ? `--user-id=${shellArg(params.user_id)}` : "",
@@ -7804,7 +7991,7 @@ async function searchFallbackToTraces(params) {
   ].filter(Boolean).join(" ");
   let traces;
   try {
-    traces = asRecord(await callDataAPI(`/conversations?${query.toString()}`)) ?? {};
+    traces = asRecord(await callDataAPI(`/conversations?${query2.toString()}`)) ?? {};
   } catch (error) {
     const reason = error instanceof Error ? error.message : String(error);
     return {
@@ -7953,16 +8140,16 @@ function assertHandParsedFlags(command, flags, positionals) {
 var PROFILE_MANAGING_COMMANDS = new Set(["profiles", "auth", "init", "provision", "status", "config", "keys", "manifest", "agent", "doctor"]);
 async function runProblemsRemainder(flags, context) {
   const params = parseCommandFlags(ProblemsRemainderSchema, flagsToArgs(flags), "problems remainder");
-  const query = new URLSearchParams;
+  const query2 = new URLSearchParams;
   if (params.days_back)
-    query.set("days_back", params.days_back.toString());
+    query2.set("days_back", params.days_back.toString());
   if (params.limit)
-    query.set("limit", params.limit.toString());
+    query2.set("limit", params.limit.toString());
   if (params.family)
-    query.set("family", params.family);
+    query2.set("family", params.family);
   if (params.cursor)
-    query.set("cursor", params.cursor);
-  const queryString = query.toString() ? `?${query.toString()}` : "";
+    query2.set("cursor", params.cursor);
+  const queryString = query2.toString() ? `?${query2.toString()}` : "";
   context.output.writeData(await callDataAPI(`/problems/remainder${queryString}`));
 }
 function processingDisabledWarning(data, family) {
@@ -8158,28 +8345,28 @@ async function runCommand(command, positional, flags, positionals = positional ?
       const args = flagsToArgs(flags);
       const params = parseCommandFlags(ClustersSchema, args, command);
       if (params.search !== undefined || params.node_id !== undefined) {
-        const query2 = new URLSearchParams;
+        const query3 = new URLSearchParams;
         if (params.search !== undefined)
-          query2.set("q", params.search);
+          query3.set("q", params.search);
         if (params.node_id !== undefined)
-          query2.set("node_id", params.node_id);
+          query3.set("node_id", params.node_id);
         if (params.mode)
-          query2.set("mode", params.mode);
+          query3.set("mode", params.mode);
         if (params.limit)
-          query2.set("limit", params.limit.toString());
-        const data2 = await callDataAPI(`/task-clusters/search?${query2.toString()}`);
+          query3.set("limit", params.limit.toString());
+        const data2 = await callDataAPI(`/task-clusters/search?${query3.toString()}`);
         const record = asRecord(data2) ?? {};
         const matches = Array.isArray(record.matches) ? record.matches : [];
         const warnings = params.node_id !== undefined && matches.length === 0 ? notFoundWarning("cluster", params.node_id, true) : [];
         context.output.writeData(data2, warnings.length > 0 ? { warnings } : undefined);
         break;
       }
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.parent_id)
-        query.set("parent_id", params.parent_id);
+        query2.set("parent_id", params.parent_id);
       if (params.time_range)
-        query.set("time_range", params.time_range);
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+        query2.set("time_range", params.time_range);
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/clusters${queryString}`);
       context.output.writeData(data);
       break;
@@ -8191,12 +8378,12 @@ async function runCommand(command, positional, flags, positionals = positional ?
       }
       const args = flagsToArgs(flags, "node_id", positional);
       const params = parseCommandFlags(ClusterConversationsSchema, args, command, { positionalKey: "node_id" });
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.limit)
-        query.set("limit", params.limit.toString());
+        query2.set("limit", params.limit.toString());
       if (params.offset !== undefined)
-        query.set("offset", params.offset.toString());
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+        query2.set("offset", params.offset.toString());
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/clusters/${params.node_id}/conversations${queryString}`);
       const clusterRecord = asRecord(data) ?? {};
       const warnings = notFoundWarning("cluster", params.node_id, clusterRecord.cluster === null || clusterRecord.cluster === undefined);
@@ -8207,28 +8394,28 @@ async function runCommand(command, positional, flags, positionals = positional ?
     case "conversations": {
       const args = withStringIdFlags(flagsToArgs(flags), flags);
       const params = parseCommandFlags(ConversationsSchema, args, command);
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.search)
-        query.set("search", params.search);
+        query2.set("search", params.search);
       if (params.cluster_id)
-        query.set("cluster_id", params.cluster_id);
+        query2.set("cluster_id", params.cluster_id);
       if (params.user_id)
-        query.set("user_id", params.user_id);
+        query2.set("user_id", params.user_id);
       if (params.time_range)
-        query.set("time_range", params.time_range);
+        query2.set("time_range", params.time_range);
       if (params.environment)
-        query.set("environment", params.environment);
+        query2.set("environment", params.environment);
       if (params.world_state)
-        query.set("world_state", params.world_state);
+        query2.set("world_state", params.world_state);
       if (params.outcome && params.outcome !== "any")
-        query.set("outcome", params.outcome);
+        query2.set("outcome", params.outcome);
       if (params.sort)
-        query.set("sort_by", params.sort);
+        query2.set("sort_by", params.sort);
       if (params.limit)
-        query.set("limit", params.limit.toString());
+        query2.set("limit", params.limit.toString());
       if (params.offset !== undefined)
-        query.set("offset", params.offset.toString());
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+        query2.set("offset", params.offset.toString());
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const raw = await callDataAPI(`/conversations${queryString}`);
       recordConversationsTelemetry(context, raw);
       const searchWarnings = [];
@@ -8265,19 +8452,19 @@ async function runCommand(command, positional, flags, positionals = positional ?
       }
       const args = withStringIdFlags(flagsToArgs(flags, "q", positional), flags);
       const params = parseCommandFlags(SearchSchema, args, command, { positionalKey: "q" });
-      const query = new URLSearchParams;
-      query.set("q", params.q);
+      const query2 = new URLSearchParams;
+      query2.set("q", params.q);
       if (params.mode)
-        query.set("mode", params.mode);
+        query2.set("mode", params.mode);
       if (params.user_id)
-        query.set("user_id", params.user_id);
+        query2.set("user_id", params.user_id);
       if (params.time_range)
-        query.set("time_range", params.time_range);
+        query2.set("time_range", params.time_range);
       if (params.limit)
-        query.set("limit", params.limit.toString());
+        query2.set("limit", params.limit.toString());
       if (params.include_tool_io)
-        query.set("include_tool_io", "true");
-      const data = await callDataAPI(`/search?${query.toString()}`);
+        query2.set("include_tool_io", "true");
+      const data = await callDataAPI(`/search?${query2.toString()}`);
       recordSearchTelemetry(context, data);
       const searchRecord = asRecord(data) ?? {};
       const degradeReason = asString(searchRecord.degrade_reason);
@@ -8325,13 +8512,13 @@ async function runCommand(command, positional, flags, positionals = positional ?
         throw new CliInputError("Choose either --snapshot or --replay, not both.", "Usage: moda world-state <conversation_id> [--snapshot --msg-index=N [--block-index=M] [--segment-index=K]] | [--replay --message-count=N [--format=frames|fold-inputs]]");
       }
       if (params.snapshot === true) {
-        const query2 = new URLSearchParams;
-        query2.set("msg_index", (params.msg_index ?? 0).toString());
+        const query3 = new URLSearchParams;
+        query3.set("msg_index", (params.msg_index ?? 0).toString());
         if (params.block_index !== undefined)
-          query2.set("block_index", params.block_index.toString());
+          query3.set("block_index", params.block_index.toString());
         if (params.segment_index !== undefined)
-          query2.set("segment_index", params.segment_index.toString());
-        const data2 = await callDataAPI(`/conversations/${params.conversation_id}/world-state/snapshot?${query2.toString()}`);
+          query3.set("segment_index", params.segment_index.toString());
+        const data2 = await callDataAPI(`/conversations/${params.conversation_id}/world-state/snapshot?${query3.toString()}`);
         const snapshotRecord = asRecord(data2) ?? {};
         const warnings2 = notFoundWarning("world state", params.conversation_id, snapshotRecord.found === false);
         context.output.writeData(data2, warnings2.length > 0 ? { warnings: warnings2 } : undefined);
@@ -8342,22 +8529,22 @@ async function runCommand(command, positional, flags, positionals = positional ?
         if (format === "frames" && params.message_count === undefined) {
           throw new CliInputError("--message-count=N is required with --replay in frames mode (the API returns no frames without it). Use --format=fold-inputs for the raw keyframes/events without a count.");
         }
-        const query2 = new URLSearchParams;
-        query2.set("format", format === "fold-inputs" ? "fold_inputs" : "frames");
+        const query3 = new URLSearchParams;
+        query3.set("format", format === "fold-inputs" ? "fold_inputs" : "frames");
         if (params.message_count !== undefined)
-          query2.set("message_count", params.message_count.toString());
-        const data2 = await callDataAPI(`/conversations/${params.conversation_id}/world-state/replay?${query2.toString()}`);
+          query3.set("message_count", params.message_count.toString());
+        const data2 = await callDataAPI(`/conversations/${params.conversation_id}/world-state/replay?${query3.toString()}`);
         const replayRecord = asRecord(data2) ?? {};
         const warnings2 = notFoundWarning("world state", params.conversation_id, replayRecord.hasData === false);
         context.output.writeData(data2, warnings2.length > 0 ? { warnings: warnings2 } : undefined);
         break;
       }
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.summary_only === true)
-        query.set("summary_only", "1");
+        query2.set("summary_only", "1");
       if (params.event_limit)
-        query.set("event_limit", params.event_limit.toString());
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+        query2.set("event_limit", params.event_limit.toString());
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/conversations/${params.conversation_id}/world-state${queryString}`);
       const wsRecord = asRecord(data) ?? {};
       const warnings = notFoundWarning("world state", params.conversation_id, wsRecord.hasData === false);
@@ -8389,12 +8576,12 @@ async function runCommand(command, positional, flags, positionals = positional ?
       if (params.from !== undefined || params.max_messages !== undefined) {
         throw new CliInputError("--from and --max-messages page the whole trace and need --all.", "Usage: moda context <conversation_id> --all [--from=N] [--max-messages=N]");
       }
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.msg_index !== undefined)
-        query.set("msg_index", params.msg_index.toString());
+        query2.set("msg_index", params.msg_index.toString());
       if (params.window)
-        query.set("window", params.window.toString());
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+        query2.set("window", params.window.toString());
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/conversations/${params.conversation_id}/context${queryString}`);
       const ctxRecord = asRecord(data) ?? {};
       const warnings = notFoundWarning("trace", params.conversation_id, asNumber(ctxRecord.total_messages) === 0);
@@ -8408,14 +8595,14 @@ async function runCommand(command, positional, flags, positionals = positional ?
       }
       const args = flagsToArgs(flags, "id", positional);
       const params = parseCommandFlags(AuditSchema, args, command, { positionalKey: "id" });
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.kind && params.kind !== "auto")
-        query.set("kind", params.kind);
+        query2.set("kind", params.kind);
       if (params.include_raw === true)
-        query.set("include_raw", "1");
+        query2.set("include_raw", "1");
       if (params.limit)
-        query.set("limit", params.limit.toString());
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+        query2.set("limit", params.limit.toString());
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/audit/${encodeURIComponent(params.id)}${queryString}`);
       const capped = auditCapStatus(data, params.limit ?? AUDIT_DEFAULT_LIMIT);
       const auditRecord = asRecord(data);
@@ -8448,19 +8635,19 @@ async function runCommand(command, positional, flags, positionals = positional ?
     case "frustrations": {
       const args = withStringIdFlags(flagsToArgs(flags), flags);
       const params = parseCommandFlags(FrustrationsSchema, args, command);
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.days_back)
-        query.set("days_back", params.days_back.toString());
+        query2.set("days_back", params.days_back.toString());
       if (params.limit)
-        query.set("limit", params.limit.toString());
+        query2.set("limit", params.limit.toString());
       if (params.offset !== undefined)
-        query.set("offset", params.offset.toString());
+        query2.set("offset", params.offset.toString());
       if (params.user_id)
-        query.set("user_id", params.user_id);
+        query2.set("user_id", params.user_id);
       if (params.conversation_id)
-        query.set("conversation_id", params.conversation_id);
-      query.set("family", "frustration");
-      const data = frustrationsFromEmotions(await callDataAPI(`/emotions?${query.toString()}`));
+        query2.set("conversation_id", params.conversation_id);
+      query2.set("family", "frustration");
+      const data = frustrationsFromEmotions(await callDataAPI(`/emotions?${query2.toString()}`));
       await printWithOptionalWindows(withFrustrationAnchors(data), "frustrations", params.include_window === true, params.window ?? 1, context.output, processingDisabledWarning(data, "frustration"));
       break;
     }
@@ -8489,15 +8676,15 @@ async function runCommand(command, positional, flags, positionals = positional ?
       if (params.compact === true && params.include_window === true) {
         throw new CliInputError("--compact drops the conversation window; drop --include-window, or drop --compact.");
       }
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.subtype)
-        query.set("subtype", params.subtype);
+        query2.set("subtype", params.subtype);
       if (params.days_back)
-        query.set("days_back", params.days_back.toString());
-      query.set("limit", (params.limit ?? TOOL_FAILURE_DETAIL_DEFAULT_LIMIT).toString());
+        query2.set("days_back", params.days_back.toString());
+      query2.set("limit", (params.limit ?? TOOL_FAILURE_DETAIL_DEFAULT_LIMIT).toString());
       if (params.offset !== undefined)
-        query.set("offset", params.offset.toString());
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+        query2.set("offset", params.offset.toString());
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/tool-failures/${params.tool_name}${queryString}`);
       const tfRecord = asRecord(data) ?? {};
       const subtypeCount = Array.isArray(tfRecord.subtypes) ? tfRecord.subtypes.length : 0;
@@ -8531,7 +8718,7 @@ async function runCommand(command, positional, flags, positionals = positional ?
     }
     case "problem-impact": {
       const params = ProblemImpactSchema.parse(Object.fromEntries(Object.entries(flags).map(([key, value]) => [key.replace(/-/g, "_"), value])));
-      const query = new URLSearchParams({ days: String(params.days_back), mode: params.mode });
+      const query2 = new URLSearchParams({ days: String(params.days_back), mode: params.mode });
       for (const [key, value] of Object.entries({
         nodeId: params.node_id,
         problemId: params.problem_id,
@@ -8541,8 +8728,8 @@ async function runCommand(command, positional, flags, positionals = positional ?
         conversationIds: params.conversation_ids ? JSON.stringify(params.conversation_ids) : undefined
       }))
         if (value !== undefined)
-          query.set(key, value);
-      const data = asRecord(await callDataAPI(`/problem-impact?${query}`)) ?? {};
+          query2.set(key, value);
+      const data = asRecord(await callDataAPI(`/problem-impact?${query2}`)) ?? {};
       const link = new URLSearchParams({ impactDays: String(params.days_back) });
       const window = asRecord(data.window);
       const resolvedRun = asString(data.runId);
@@ -8569,12 +8756,12 @@ async function runCommand(command, positional, flags, positionals = positional ?
       }
       const args = flagsToArgs(flags);
       const params = parseCommandFlags(ProblemsSchema, args, command);
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.days_back)
-        query.set("days_back", params.days_back.toString());
+        query2.set("days_back", params.days_back.toString());
       if (params.limit)
-        query.set("limit", params.limit.toString());
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+        query2.set("limit", params.limit.toString());
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/problems${queryString}`);
       const { payload, truncated } = capProblemsBlocks(asRecord(data) ?? {}, params.limit ?? 25);
       const withLink = {
@@ -8616,18 +8803,18 @@ async function runCommand(command, positional, flags, positionals = positional ?
         const viewFlag = view === "conversations" ? "--traces" : `--${view}`;
         throw new CliInputError(`${viewFlag} requires a canonical problem UUID; got "${params.id}". Run \`moda problems\` or \`moda problem <id>\` first to resolve the id.`);
       }
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.limit)
-        query.set("limit", params.limit.toString());
+        query2.set("limit", params.limit.toString());
       if (params.cursor)
-        query.set("cursor", params.cursor);
+        query2.set("cursor", params.cursor);
       if (view === "conversations") {
         if (params.family)
-          query.set("family", params.family);
+          query2.set("family", params.family);
         if (params.door)
-          query.set("door", params.door);
+          query2.set("door", params.door);
       }
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/problems/${params.id}/${view}${queryString}`);
       context.output.writeData(data);
       break;
@@ -8763,14 +8950,14 @@ async function runCommand(command, positional, flags, positionals = positional ?
         context.output.writeData(data2);
         break;
       }
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.days_back)
-        query.set("days_back", params.days_back.toString());
+        query2.set("days_back", params.days_back.toString());
       if (params.limit)
-        query.set("limit", params.limit.toString());
+        query2.set("limit", params.limit.toString());
       if (params.offset !== undefined)
-        query.set("offset", params.offset.toString());
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+        query2.set("offset", params.offset.toString());
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/custom-signals/${id}/detections${queryString}`);
       context.output.writeData(data);
       break;
@@ -8922,20 +9109,20 @@ async function runCommand(command, positional, flags, positionals = positional ?
     case "false-positives": {
       if (!positional) {
         const kinds = { quote: "frustration_quote", tool: "tool_call", laziness: "laziness", hallucination: "hallucination", "custom-signal": "custom_signal" };
-        const query = new URLSearchParams;
+        const query2 = new URLSearchParams;
         if (flags.kind !== undefined) {
           if (!kinds[flags.kind])
             throw new CliInputError(`--kind must be one of ${Object.keys(kinds).join("|")}`);
-          query.set("kind", kinds[flags.kind]);
+          query2.set("kind", kinds[flags.kind]);
         }
         if (flags.limit !== undefined) {
           if (!/^\d+$/.test(flags.limit) || Number(flags.limit) < 1 || Number(flags.limit) > 200)
             throw new CliInputError("--limit must be 1-200");
-          query.set("limit", flags.limit);
+          query2.set("limit", flags.limit);
         }
         if (flags.all !== undefined && flags.all !== "false")
-          query.set("include_cleared", "true");
-        const queryString = query.toString() ? `?${query.toString()}` : "";
+          query2.set("include_cleared", "true");
+        const queryString = query2.toString() ? `?${query2.toString()}` : "";
         const data2 = await callDataAPI(`/false-positives${queryString}`);
         context.output.writeData(data2);
         break;
@@ -8947,20 +9134,20 @@ async function runCommand(command, positional, flags, positionals = positional ?
     case "emotions": {
       const args = withStringIdFlags(flagsToArgs(flags), flags);
       const params = parseCommandFlags(EmotionsSchema, args, command);
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.days_back)
-        query.set("days_back", params.days_back.toString());
+        query2.set("days_back", params.days_back.toString());
       if (params.limit)
-        query.set("limit", params.limit.toString());
+        query2.set("limit", params.limit.toString());
       if (params.offset !== undefined)
-        query.set("offset", params.offset.toString());
+        query2.set("offset", params.offset.toString());
       if (params.family)
-        query.set("family", params.family);
+        query2.set("family", params.family);
       if (params.user_id)
-        query.set("user_id", params.user_id);
+        query2.set("user_id", params.user_id);
       if (params.conversation_id)
-        query.set("conversation_id", params.conversation_id);
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+        query2.set("conversation_id", params.conversation_id);
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/emotions${queryString}`);
       const processingWarnings = processingDisabledWarning(data, "emotion");
       context.output.writeData(data, processingWarnings.length > 0 ? { warnings: processingWarnings } : undefined);
@@ -8972,18 +9159,18 @@ async function runCommand(command, positional, flags, positionals = positional ?
         args.conversation_id = flags["conversation-id"];
       }
       const params = parseCommandFlags(LazinessSchema, args, command);
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.days_back)
-        query.set("days_back", params.days_back.toString());
+        query2.set("days_back", params.days_back.toString());
       if (params.limit)
-        query.set("limit", params.limit.toString());
+        query2.set("limit", params.limit.toString());
       if (params.offset !== undefined)
-        query.set("offset", params.offset.toString());
+        query2.set("offset", params.offset.toString());
       if (params.conversation_id)
-        query.set("conversation_id", params.conversation_id);
+        query2.set("conversation_id", params.conversation_id);
       if (params.family)
-        query.set("family", params.family);
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+        query2.set("family", params.family);
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/laziness${queryString}`);
       context.output.writeData(data);
       break;
@@ -8994,18 +9181,18 @@ async function runCommand(command, positional, flags, positionals = positional ?
         args.conversation_id = flags["conversation-id"];
       }
       const params = parseCommandFlags(HallucinationsSchema, args, command);
-      const query = new URLSearchParams;
+      const query2 = new URLSearchParams;
       if (params.days_back)
-        query.set("days_back", params.days_back.toString());
+        query2.set("days_back", params.days_back.toString());
       if (params.limit)
-        query.set("limit", params.limit.toString());
+        query2.set("limit", params.limit.toString());
       if (params.offset !== undefined)
-        query.set("offset", params.offset.toString());
+        query2.set("offset", params.offset.toString());
       if (params.conversation_id)
-        query.set("conversation_id", params.conversation_id);
+        query2.set("conversation_id", params.conversation_id);
       if (params.kind)
-        query.set("kind", params.kind);
-      const queryString = query.toString() ? `?${query.toString()}` : "";
+        query2.set("kind", params.kind);
+      const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/hallucinations${queryString}`);
       context.output.writeData(data);
       break;
@@ -9481,6 +9668,23 @@ var commandRegistry = createCommandRegistry([
     telemetry: "result-and-error",
     handler: runUserCommand
   },
+  {
+    name: "replays",
+    description: "Read replay results: sets, run history, per-seed verdicts, and playout transcripts",
+    examples: [
+      "moda replays runs <set_id>",
+      "moda replays run <set_id> <run_id> --json",
+      "moda replays playouts <set_id> <run_id> --case=<case_id> --arm=proposed --transcript --json"
+    ],
+    subcommands: REPLAYS_SUBCOMMAND_HELP,
+    auth: "api-key",
+    outputModes: ["human", "json", "agent"],
+    mutability: "read",
+    validateConfigBeforeRun: false,
+    resetApiRequestCountBeforeRun: false,
+    telemetry: "result-and-error",
+    handler: runReplaysCommand
+  },
   {
     name: "problem-reopen",
     description: "Reopen a problem that was marked fixed (resolved problems return only via re-detection)",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moda-ai/cli",
-  "version": "1.41.2",
+  "version": "1.42.0",
   "description": "CLI for Moda - AI agent analytics and observability",
   "type": "module",
   "bin": {

package/skills/integration/index.json

@@ -1,7 +1,7 @@
 {
   "schema_version": "moda.skill_index.v1",
-  "bundled_at": "2026-10-09T01:27:15.161Z",
-  "cli_version": "1.41.2",
+  "bundled_at": "2026-10-09T02:07:16.635Z",
+  "cli_version": "1.42.0",
   "skills": [
     {
       "id": "integration-cloudflare-think",

package/skills/moda-cli/SKILL.md

@@ -785,6 +785,43 @@ Do not copy prompt text from the dashboard into source code. The repo prompt
 files are the source of truth; the dashboard is for visibility, usage, labels,
 and future evals.
 
+### 7a. Check a replay verdict (`moda replays`)
+
+Use after `prompts ab`, `prompts propose --gate`, or a dashboard replay when
+you need to see *why* an arm passed or failed. Read-only; no replay spend.
+
+```bash
+moda replays sets --json                              # set ids
+moda replays runs <set_id> --limit=5 --json           # newest runs, every status
+moda replays run <set_id> <run_id> --json             # per-case + per-seed verdicts
+moda replays playouts <set_id> <run_id> --json        # one row per case x arm x seed
+moda replays playouts <set_id> <run_id> --case=<case_id> --arm=proposed --seed=1 --transcript --json
+moda replays playout <set_id> <run_id> <playout_id> --json
+```
+
+`run` returns `results[]` (one per case × arm × seed: `pass`, `abstained`,
+`coverageReason`, `score`, `rationale`, `criteria[]`) next to the per-case
+majority in `cases[]`. `arm=prod` is the baseline and `arm=proposed` the
+candidate. An `abstained: true` row is not a verdict: the pair hit a replay
+coverage hole and is excluded from pass counts.
+
+A playout's `transcript.events[]` holds the simulated user and agent messages
+and every tool call with its `input`, the `output` the replay served, and
+`provenance.source`: `replay_runtime` (compiled mock over the case's recorded
+source state), `agent_tool_simulator` (LLM simulator grounded in recorded
+receipts; `simulator.model`), `counterfactual_reconstruction`, or
+`generic_handler`. `provenance.errorSubtype` such as `missing_source_state`
+marks an unservable call. `judge` is the judgement stored with the playout and
+`verdict` the stored eval row (null when `seed` is null, which happens on
+older runs whose graphs cannot be matched to a seed).
+
+Limits: `--transcript` pages at most 10 playouts (`--limit` default 50, max
+200 without it); text and tool JSON are clipped at `--max-chars` (default
+4000, max 32000) and `transcript.truncated.fields` counts the clipped fields.
+Page with `--cursor=<nextCursor>`. Playouts exist only for runs whose
+playouts were captured; `playoutCount: 0` on `run` means there is no
+transcript to read.
+
 ### 7b. Push prompts, skills, and tools without a repo scan (`moda registry`)
 
 Use when the user wants their prompts, skills, and tools visible in Moda but
@@ -1026,6 +1063,7 @@ Writes listed here accept `--dry-run` (prints the exact request, sends nothing)
 | Custom signals: edit, lifecycle | `moda signals update <id> …`, `moda signals activate\|pause\|delete <id>` (delete: drafts only; archiving is dashboard-only) |
 | Custom signals: review & tune | `moda signals examples <id> add\|update\|remove --conversation-id=…`, `refine <id> --request="…"`, `revise <id> --conversation-ids=…`, `proposals <id>`, `accept\|reject <id> <proposal_id>` |
 | Custom signals: find matches | `moda signals search <id>`, `search-status <id> <search_id>`, `search-stop <id> <search_id>`, `preview --file=detector.json` |
+| Replays: run detail, playout transcripts | `moda replays runs <set_id>`, `run <set_id> <run_id>`, `playouts <set_id> <run_id> [--case --arm --seed --transcript]`, `playout <set_id> <run_id> <playout_id>` |
 | Prompts registry | `moda prompts list [--label --search]`, `show <key>`, `proposals <key>`, `usage <key>`, `replay-runs <key>`, `decide <key> <proposal_id> --action=merge\|reject\|reopen` |
 | Skills library | `moda skills list`, `show <key>`, `policy <key> --live\|--local` |
 | Harness pages | `moda harness list`, `show <id>`, `versions <id>`, `diff <id> --from=… --to=…` |
@@ -1234,6 +1272,7 @@ npx: `npx -p @moda-ai/cli moda <command>`.
 | `moda problem-reopen <pid>` | Reopen a Problem marked fixed |
 | `moda signals <create\|update\|activate\|pause\|delete\|examples\|refine\|revise\|proposals\|accept\|reject\|search\|preview>` | Manage custom signals (writes take `--dry-run`; LLM-backed ones are rate-limited per key) |
 | `moda prompts list\|show\|proposals\|usage\|replay-runs\|decide` | Read the prompt registry; merge/reject proposals |
+| `moda replays sets\|runs\|run\|playouts\|playout` | Replay results: run history, per-seed verdicts, playout transcripts with tool provenance (read-only) |
 | `moda skills list\|show\|policy` / `moda harness list\|show\|versions\|diff` | Skill library and synced harnesses |
 | `moda fixes` | Ranked queue of Fixes ("a fix with proof"); `--status=S`, `--limit=N`, `--cursor=TOKEN` |
 | `moda fix <fix_id>` | One Fix: status + gate result; `--packet` prints `moda.fix_packet.v1` and writes tool/skill candidates under `.moda/fixes/<ref>/`; `--wait` drives a running pipeline to rest |