@moda-ai/cli 1.41.2 → 1.43.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5586,6 +5586,11 @@ var REPLAY_MODEL_ALLOWLIST = [
5586
5586
  var DEFAULT_POLL_INTERVAL_MS = 15000;
5587
5587
  var DEFAULT_WAIT_TIMEOUT_MS = 2 * 60 * 60 * 1000;
5588
5588
  var TERMINAL_RUN_STATUSES = new Set(["completed", "skipped", "error"]);
5589
+ var FALLBACK_SUCCESS_CRITERIA = [
5590
+ "The agent understands the user request",
5591
+ "The agent uses tools appropriately when needed",
5592
+ "The agent provides a helpful final answer"
5593
+ ];
5589
5594
  async function runPromptAb(flags, profileOptions, context) {
5590
5595
  validateConfig();
5591
5596
  const tenantId = flags["tenant-id"] || resolveApiTenantId(profileOptions);
@@ -5626,14 +5631,20 @@ async function runPromptAb(flags, profileOptions, context) {
5626
5631
  prod: resolvePromptArmSpec(baselineSource, "baseline"),
5627
5632
  proposed: resolvePromptArmSpec(candidateSource, "candidate")
5628
5633
  };
5629
- const replaySetId = await ensureReplaySet(plan, flags, tenantId, profileOptions);
5634
+ const { setId: replaySetId, caseSources, warnings } = await ensureReplaySet(plan, flags, tenantId, profileOptions);
5635
+ if (context.outputMode === "human") {
5636
+ for (const warning of warnings)
5637
+ process.stderr.write(`Warning: ${warning}
5638
+ `);
5639
+ }
5640
+ const writeOptions = warnings.length ? { warnings } : undefined;
5630
5641
  const enqueue = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/run`, {
5631
5642
  method: "POST",
5632
5643
  body: JSON.stringify({
5633
5644
  promptArms,
5634
5645
  seedsPerCase,
5635
5646
  assistantModel: assistantModel || undefined,
5636
- promotePrimary: flags["no-promote"] !== "true",
5647
+ promotePrimary: flags.promote === "true" && flags["no-promote"] !== "true",
5637
5648
  ...plan.kind === "existing" ? replayCasePin(plan.caseIds) : {}
5638
5649
  })
5639
5650
  }, profileOptions, { timeoutMs: 120000, retries: 0 });
@@ -5643,11 +5654,12 @@ async function runPromptAb(flags, profileOptions, context) {
5643
5654
  if (noWait) {
5644
5655
  context.output.writeData({
5645
5656
  replaySetId,
5657
+ ...caseSources ? { caseSources } : {},
5646
5658
  runId: enqueue.runId,
5647
5659
  status: enqueue.status,
5648
5660
  message: enqueue.message ?? "Replay comparison queued",
5649
5661
  plannedPlayouts
5650
- });
5662
+ }, writeOptions);
5651
5663
  return;
5652
5664
  }
5653
5665
  const deadline = Date.now() + timeoutMs;
@@ -5670,6 +5682,7 @@ async function runPromptAb(flags, profileOptions, context) {
5670
5682
  }
5671
5683
  const result = {
5672
5684
  replaySetId,
5685
+ ...caseSources ? { caseSources } : {},
5673
5686
  plannedPlayouts,
5674
5687
  runId: latest.run.runId,
5675
5688
  status: latest.run.status,
@@ -5680,7 +5693,7 @@ async function runPromptAb(flags, profileOptions, context) {
5680
5693
  if (context.outputMode === "human") {
5681
5694
  printHumanVerdict(result);
5682
5695
  }
5683
- context.output.writeData(result);
5696
+ context.output.writeData(result, writeOptions);
5684
5697
  }
5685
5698
  function warnIfQueued(enqueue) {
5686
5699
  const { runsAhead, maxActiveRuns } = enqueue;
@@ -5793,7 +5806,7 @@ async function planReplaySet(flags, tenantId, profileOptions) {
5793
5806
  const snapshot = await fetchReplaySetCases(tenantId, existingSetId, profileOptions);
5794
5807
  return { kind: "existing", setId: existingSetId, cases: snapshot?.ids.length ?? null, caseIds: snapshot?.ids };
5795
5808
  }
5796
- const conversations = parseCsv(flags.traces ?? flags.conversations ?? flags["conversation-ids"]);
5809
+ const conversations = [...new Set(parseCsv(flags.traces ?? flags.conversations ?? flags["conversation-ids"]))];
5797
5810
  if (conversations.length > MAX_REPLAY_TRACES) {
5798
5811
  throw new CliInputError(`--traces has ${conversations.length} ids; the limit is ${MAX_REPLAY_TRACES} per replay set, so nothing was sent.`, "Split the traces across several runs, or use --auto-generate.");
5799
5812
  }
@@ -5861,9 +5874,9 @@ function rerunWithYes(command, positionals, flags) {
5861
5874
  }
5862
5875
  async function ensureReplaySet(plan, flags, tenantId, profileOptions) {
5863
5876
  if (plan.kind === "existing")
5864
- return plan.setId;
5877
+ return { setId: plan.setId, warnings: [] };
5865
5878
  if (plan.kind === "traces") {
5866
- return createSetFromConversations(plan.traces, flags, tenantId, profileOptions);
5879
+ return createSetFromTraces(plan.traces, flags, tenantId, profileOptions);
5867
5880
  }
5868
5881
  const caseCount = plan.cases;
5869
5882
  const lookbackDays = parsePositiveInt(flags["lookback-days"], 30, 365, "--lookback-days");
@@ -5875,7 +5888,7 @@ async function ensureReplaySet(plan, flags, tenantId, profileOptions) {
5875
5888
  if (!generated?.id) {
5876
5889
  throw new Error("Auto-generate did not return a replay set id");
5877
5890
  }
5878
- return generated.id;
5891
+ return { setId: generated.id, warnings: [] };
5879
5892
  }
5880
5893
  function parseTraceRef(ref) {
5881
5894
  const match = /^(.+)@(\d+)$/.exec(ref.trim());
@@ -5883,38 +5896,75 @@ function parseTraceRef(ref) {
5883
5896
  return { conversationId: ref.trim() };
5884
5897
  return { conversationId: match[1], startMsgIndex: Number(match[2]) };
5885
5898
  }
5886
- async function createSetFromConversations(conversationIds, flags, tenantId, profileOptions) {
5887
- const name = (flags.name ?? flags["set-name"] ?? `Prompt A/B ${conversationIds.length} trace(s)`).trim();
5888
- const created = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets`, {
5889
- method: "POST",
5890
- body: JSON.stringify({ name, description: "Created by moda prompts ab" })
5891
- }, profileOptions);
5892
- let position = 0;
5893
- for (const ref of conversationIds) {
5894
- const { conversationId, startMsgIndex } = parseTraceRef(ref);
5899
+ async function createSetFromTraces(conversationIds, flags, tenantId, profileOptions) {
5900
+ const refs = [...new Set(conversationIds)].map(parseTraceRef);
5901
+ const traces = [...new Set(refs.map((ref) => ref.conversationId))];
5902
+ const startByConversation = new Map;
5903
+ for (const ref of refs) {
5904
+ if (ref.startMsgIndex !== undefined)
5905
+ startByConversation.set(ref.conversationId, ref.startMsgIndex);
5906
+ }
5907
+ const name = (flags.name ?? flags["set-name"] ?? `Prompt A/B ${traces.length} trace(s)`).trim();
5908
+ const base = `/tenants/${encodeURIComponent(tenantId)}/replay-sets`;
5909
+ let set = null;
5910
+ let unavailable = null;
5911
+ try {
5912
+ set = await callControlAPI(`${base}/auto-generate`, { method: "POST", body: JSON.stringify({ name, conversationIds: traces }) }, profileOptions, { timeoutMs: 300000, retries: 0 });
5913
+ if (!set?.id)
5914
+ throw new Error("Auto-generate did not return a replay set id");
5915
+ } catch (error) {
5916
+ const status = error?.statusCode;
5917
+ if (status === 401 || status === 403)
5918
+ throw error;
5919
+ unavailable = error instanceof Error ? error.message : String(error);
5920
+ set = null;
5921
+ }
5922
+ const distilledIds = new Set((set?.cases ?? []).map((c) => String(c?.sourceConversationId ?? "")).filter(Boolean));
5923
+ const distilled = traces.filter((id) => distilledIds.has(id));
5924
+ const fallback = traces.filter((id) => !distilledIds.has(id));
5925
+ if (!set) {
5926
+ set = await callControlAPI(base, { method: "POST", body: JSON.stringify({ name, description: "Created by moda prompts ab" }) }, profileOptions);
5927
+ }
5928
+ const requestedIndex = new Map(traces.map((id, index) => [id, index]));
5929
+ for (const existing of set.cases ?? []) {
5930
+ const conversationId = String(existing?.sourceConversationId ?? "");
5931
+ const index = requestedIndex.get(conversationId);
5932
+ const startMsgIndex = startByConversation.get(conversationId);
5933
+ const update = {};
5934
+ if (fallback.length && index !== undefined && existing.position !== index)
5935
+ update.position = index;
5936
+ if (startMsgIndex !== undefined)
5937
+ update.sourceStartMsgIndex = startMsgIndex;
5938
+ if (!Object.keys(update).length)
5939
+ continue;
5940
+ await callControlAPI(`${base}/${encodeURIComponent(set.id)}/cases/${encodeURIComponent(existing.id)}`, { method: "PUT", body: JSON.stringify(update) }, profileOptions);
5941
+ }
5942
+ for (const conversationId of fallback) {
5895
5943
  const scenario = await loadScenarioFromConversation(conversationId);
5896
- await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(created.id)}/cases`, {
5944
+ await callControlAPI(`${base}/${encodeURIComponent(set.id)}/cases`, {
5897
5945
  method: "POST",
5898
5946
  body: JSON.stringify({
5899
5947
  title: conversationId,
5900
5948
  scenario,
5901
5949
  sourceConversationId: conversationId,
5902
- ...startMsgIndex !== undefined ? { sourceStartMsgIndex: startMsgIndex } : {},
5903
- successCriteria: [
5904
- "The agent understands the user request",
5905
- "The agent uses tools appropriately when needed",
5906
- "The agent provides a helpful final answer"
5907
- ],
5908
- position: position++
5950
+ ...startByConversation.has(conversationId) ? { sourceStartMsgIndex: startByConversation.get(conversationId) } : {},
5951
+ successCriteria: FALLBACK_SUCCESS_CRITERIA,
5952
+ position: requestedIndex.get(conversationId)
5909
5953
  })
5910
5954
  }, profileOptions);
5911
5955
  }
5912
- return created.id;
5956
+ const warnings = [];
5957
+ if (fallback.length) {
5958
+ const why = unavailable ? `Moda could not distill these traces (${unavailable.slice(0, 200)})` : "Moda could not distill a scenario for these traces";
5959
+ warnings.push(`${why}, so ${fallback.length} of ${traces.length} case(s) use only the opening user message and three generic ` + `success criteria, which long traces pass easily: ${fallback.slice(0, 10).join(", ")}${fallback.length > 10 ? ", …" : ""}. ` + "Edit their scenario/criteria in the dashboard, or pass --set-id with a curated set.");
5960
+ }
5961
+ return { setId: set.id, caseSources: { distilled, fallback }, warnings };
5913
5962
  }
5914
5963
  async function loadScenarioFromConversation(conversationId) {
5915
5964
  try {
5916
- const data = await callDataAPI(`/conversations/${encodeURIComponent(conversationId)}/context?msg_index=0&window=0`);
5917
- const firstUser = (data.messages ?? []).find((message) => message.role === "user" && String(message.content ?? "").trim());
5965
+ const data = await callDataAPI(`/conversations/${encodeURIComponent(conversationId)}/context?msg_index=0&window=5`);
5966
+ const messages = data.context?.messages ?? data.messages ?? [];
5967
+ const firstUser = messages.find((message) => message.role === "user" && String(message.content ?? "").trim());
5918
5968
  const opening = String(firstUser?.content ?? "").trim();
5919
5969
  if (opening) {
5920
5970
  return `The user says: "${opening}"
@@ -6446,6 +6496,7 @@ var PROMPT_WRITE_SUBCOMMAND_FLAGS = {
6446
6496
  "lookback-days",
6447
6497
  "no-promote",
6448
6498
  "no-wait",
6499
+ "promote",
6449
6500
  "poll-interval",
6450
6501
  "replay-set-id",
6451
6502
  "seeds-per-case",
package/dist/cli.js CHANGED
@@ -8,13 +8,14 @@ import {
8
8
  runPromptsCommand,
9
9
  runSkillsCommand,
10
10
  runStatusCommand
11
- } from "./cli-pr6s56fc.js";
11
+ } from "./cli-ecrx32mm.js";
12
12
  import {
13
13
  ApiError,
14
14
  HARNESS_REPORT_APPROVAL_PATH,
15
15
  HARNESS_REPORT_PATH,
16
16
  SIGNALS_MANAGEMENT_SUBCOMMANDS,
17
17
  assertFlags,
18
+ assertPositionals,
18
19
  callControlAPI,
19
20
  callDataAPI,
20
21
  getApiRequestCount,
@@ -150,7 +151,7 @@ var WorldStateSchema = z.object({
150
151
  var ContextSchema = z.object({
151
152
  conversation_id: z.string(),
152
153
  msg_index: z.number().min(0).optional(),
153
- window: z.number().min(1).max(5).default(2).optional(),
154
+ window: z.number().int().min(0).optional(),
154
155
  all: boolFlag(),
155
156
  from: z.number().int().min(0).optional(),
156
157
  max_messages: z.number().int().min(1).max(5000).optional()
@@ -2500,6 +2501,183 @@ function isDefined(value) {
2500
2501
  return value !== undefined;
2501
2502
  }
2502
2503
 
2504
+ // src/replays.ts
2505
+ var UUID_RE2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
2506
+ var PLAYOUT_ID_RE = /^[0-9a-f]{64}$/i;
2507
+ var ARMS = ["prod", "proposed"];
2508
+ var REPLAYS_SUBCOMMANDS = ["sets", "runs", "run", "playouts", "playout"];
2509
+ var REPLAYS_SUBCOMMAND_HELP = [
2510
+ {
2511
+ name: "sets",
2512
+ description: "List replay sets (ids, names, case counts)",
2513
+ usage: "moda replays sets",
2514
+ examples: ["moda replays sets --json"]
2515
+ },
2516
+ {
2517
+ name: "runs",
2518
+ description: "Run history for a set, newest first, every status",
2519
+ usage: "moda replays runs <set_id> [--limit=N]",
2520
+ flags: [{ name: "--limit=N", description: "Runs to return (default 20, max 100)." }],
2521
+ examples: ["moda replays runs <set_id> --limit=5 --json"]
2522
+ },
2523
+ {
2524
+ name: "run",
2525
+ description: "One run: summary, per-case aggregate, every case x arm x seed verdict, playout count",
2526
+ usage: "moda replays run <set_id> <run_id|latest>",
2527
+ examples: ["moda replays run <set_id> <run_id> --json", "moda replays run <set_id> latest --json"]
2528
+ },
2529
+ {
2530
+ name: "playouts",
2531
+ description: "Playouts of a run (one per case x arm x seed) with verdicts; --transcript inlines what was said and every tool call",
2532
+ usage: "moda replays playouts <set_id> <run_id> [--case=<case_id>] [--arm=prod|proposed] [--seed=N] [--transcript] [--max-chars=N] [--limit=N] [--cursor=<next_cursor>]",
2533
+ flags: [
2534
+ { name: "--case=<case_id>", description: "Only this case." },
2535
+ { name: "--arm=prod|proposed", description: "Only this arm (prod = baseline, proposed = candidate)." },
2536
+ { name: "--seed=N", description: "Only this seed." },
2537
+ { name: "--transcript", description: "Inline each transcript (messages, tool calls with served output and provenance, criteria). Page cap 10." },
2538
+ { name: "--max-chars=N", description: "Per-field cap for message text and tool JSON (default 4000, 100-32000)." },
2539
+ { name: "--limit=N", description: "Page size (default 50, max 200; 10 with --transcript)." },
2540
+ { name: "--cursor=<next_cursor>", description: "Next page: the previous response's nextCursor." }
2541
+ ],
2542
+ examples: [
2543
+ "moda replays playouts <set_id> <run_id> --json",
2544
+ "moda replays playouts <set_id> <run_id> --case=<case_id> --arm=proposed --seed=1 --transcript --json"
2545
+ ]
2546
+ },
2547
+ {
2548
+ name: "playout",
2549
+ description: "One playout with its full transcript",
2550
+ usage: "moda replays playout <set_id> <run_id> <playout_id> [--max-chars=N]",
2551
+ flags: [{ name: "--max-chars=N", description: "Per-field cap (default 4000, 100-32000)." }],
2552
+ examples: ["moda replays playout <set_id> <run_id> <playout_id> --json"]
2553
+ }
2554
+ ];
2555
+ function profileOptions(context) {
2556
+ return { profile: context.profile, env: context.env };
2557
+ }
2558
+ function requireTenant(context) {
2559
+ const key = resolveApiKey(profileOptions(context));
2560
+ if (!key) {
2561
+ throw new CliInputError("No Moda API key found for moda replays.", "Set MODA_API_KEY, run `moda auth login --api-key`, or select a profile with an API key.");
2562
+ }
2563
+ const keyTenant = embeddedTenantIdFromApiKey(key);
2564
+ const flagTenant = context.flags["tenant-id"];
2565
+ if (flagTenant && keyTenant && flagTenant !== keyTenant) {
2566
+ throw new CliInputError(`--tenant-id=${flagTenant} does not match this API key's tenant (${keyTenant}), so nothing was sent.`, "Drop --tenant-id (the key names its tenant) or use a key for that tenant.");
2567
+ }
2568
+ const tenantId = flagTenant || resolveApiTenantId(profileOptions(context));
2569
+ if (!tenantId) {
2570
+ throw new CliInputError("Could not determine the tenant.", "Use a signed API key (it names its tenant), set MODA_TENANT_ID, or pass --tenant-id=<id>.");
2571
+ }
2572
+ return tenantId;
2573
+ }
2574
+ function intFlag(name, value, min, max) {
2575
+ if (value === undefined)
2576
+ return;
2577
+ const parsed = Number(value);
2578
+ if (!Number.isInteger(parsed) || parsed < min || parsed > max) {
2579
+ throw new CliInputError(`--${name} must be an integer from ${min} to ${max} (got '${value}').`);
2580
+ }
2581
+ return String(parsed);
2582
+ }
2583
+ function query(params) {
2584
+ const search = new URLSearchParams(Object.entries(params).filter((entry) => entry[1] !== undefined)).toString();
2585
+ return search ? `?${search}` : "";
2586
+ }
2587
+ function needArg(value, name, usage) {
2588
+ if (!value)
2589
+ throw new CliInputError(`<${name}> is required.`, `Usage: ${usage}`);
2590
+ return value;
2591
+ }
2592
+ function needRunId(value, usage, allowLatest = false) {
2593
+ const runId = needArg(value, "run_id", usage);
2594
+ if (allowLatest && runId === "latest")
2595
+ return runId;
2596
+ if (!UUID_RE2.test(runId)) {
2597
+ throw new CliInputError(`<run_id> must be a run UUID (got '${runId}').`, "List runs with `moda replays runs <set_id>`.");
2598
+ }
2599
+ return runId;
2600
+ }
2601
+ async function runReplaysCommand(context) {
2602
+ const [subcommand, ...args] = context.positionals;
2603
+ const flags = context.flags;
2604
+ const help = REPLAYS_SUBCOMMAND_HELP.find((entry) => entry.name === subcommand);
2605
+ if (!subcommand || !help) {
2606
+ throw new CliInputError(subcommand ? `Unknown replays command '${subcommand}'.` : "A replays subcommand is required.", `Use one of: ${REPLAYS_SUBCOMMANDS.join(", ")}. See \`moda replays --help\`.`);
2607
+ }
2608
+ const usage = help.usage;
2609
+ const allowed = {
2610
+ sets: [],
2611
+ runs: ["limit"],
2612
+ run: [],
2613
+ playouts: ["case", "arm", "seed", "transcript", "max-chars", "limit", "cursor"],
2614
+ playout: ["max-chars"]
2615
+ };
2616
+ const name = subcommand;
2617
+ assertFlags(`replays ${name}`, flags, [...allowed[name], "tenant-id"]);
2618
+ const maxArgs = { sets: 0, runs: 1, run: 2, playouts: 2, playout: 3 }[name];
2619
+ assertPositionals(`replays ${name}`, args, maxArgs, usage);
2620
+ const tenantId = requireTenant(context);
2621
+ const base = `/tenants/${encodeURIComponent(tenantId)}/replay-sets`;
2622
+ let endpoint;
2623
+ switch (name) {
2624
+ case "sets":
2625
+ endpoint = base;
2626
+ break;
2627
+ case "runs": {
2628
+ const setId = needArg(args[0], "set_id", usage);
2629
+ endpoint = `${base}/${encodeURIComponent(setId)}/runs${query({ limit: intFlag("limit", flags.limit, 1, 100) })}`;
2630
+ break;
2631
+ }
2632
+ case "run": {
2633
+ const setId = needArg(args[0], "set_id", usage);
2634
+ const runId = needRunId(args[1], usage, true);
2635
+ endpoint = runId === "latest" ? `${base}/${encodeURIComponent(setId)}/runs/latest` : `${base}/${encodeURIComponent(setId)}/runs/${encodeURIComponent(runId)}`;
2636
+ break;
2637
+ }
2638
+ case "playouts": {
2639
+ const setId = needArg(args[0], "set_id", usage);
2640
+ const runId = needRunId(args[1], usage);
2641
+ const arm = flags.arm;
2642
+ if (arm !== undefined && !ARMS.includes(arm)) {
2643
+ throw new CliInputError(`--arm must be one of ${ARMS.join(", ")} (got '${arm}').`);
2644
+ }
2645
+ if (flags.case !== undefined && (flags.case.length === 0 || flags.case.length > 128)) {
2646
+ throw new CliInputError("--case must be a case id of 1 to 128 characters.");
2647
+ }
2648
+ if (flags.cursor !== undefined && !/^\d{1,9}$/.test(flags.cursor)) {
2649
+ throw new CliInputError(`--cursor must be the nextCursor of a previous page (got '${flags.cursor}').`);
2650
+ }
2651
+ const transcript = flags.transcript === "true";
2652
+ endpoint = `${base}/${encodeURIComponent(setId)}/runs/${encodeURIComponent(runId)}/playouts${query({
2653
+ caseId: flags.case,
2654
+ arm,
2655
+ seed: intFlag("seed", flags.seed, 0, 1000),
2656
+ include: transcript ? "transcript" : undefined,
2657
+ maxChars: intFlag("max-chars", flags["max-chars"], 100, 32000),
2658
+ limit: intFlag("limit", flags.limit, 1, transcript ? 10 : 200),
2659
+ cursor: flags.cursor
2660
+ })}`;
2661
+ break;
2662
+ }
2663
+ case "playout": {
2664
+ const setId = needArg(args[0], "set_id", usage);
2665
+ const runId = needRunId(args[1], usage);
2666
+ const playoutId = needArg(args[2], "playout_id", usage);
2667
+ if (!PLAYOUT_ID_RE.test(playoutId)) {
2668
+ throw new CliInputError(`<playout_id> must be a 64-character hex id (got '${playoutId}').`, "List playouts with `moda replays playouts <set_id> <run_id>`.");
2669
+ }
2670
+ endpoint = `${base}/${encodeURIComponent(setId)}/runs/${encodeURIComponent(runId)}/playouts/${encodeURIComponent(playoutId)}${query({
2671
+ maxChars: intFlag("max-chars", flags["max-chars"], 100, 32000)
2672
+ })}`;
2673
+ break;
2674
+ }
2675
+ }
2676
+ const data = await callControlAPI(endpoint, { method: "GET" }, profileOptions(context), { timeoutMs: 60000 });
2677
+ context.output.writeData(data ?? {});
2678
+ return { exitCode: 0 };
2679
+ }
2680
+
2503
2681
  // src/telemetry.ts
2504
2682
  import { randomUUID } from "node:crypto";
2505
2683
  var TELEMETRY_TIMEOUT_MS = 500;
@@ -2514,8 +2692,8 @@ var SEARCH_SURFACE_BY_COMMAND = {
2514
2692
  context: "context"
2515
2693
  };
2516
2694
  var SEARCH_MODES2 = ["keyword", "semantic", "hybrid"];
2517
- function termCount(query) {
2518
- return query.trim().split(/\s+/).filter(Boolean).length;
2695
+ function termCount(query2) {
2696
+ return query2.trim().split(/\s+/).filter(Boolean).length;
2519
2697
  }
2520
2698
  function clampInt(value, min, max) {
2521
2699
  if (typeof value !== "number" || !Number.isFinite(value))
@@ -5180,8 +5358,8 @@ function parsePayload(payload) {
5180
5358
  }
5181
5359
  var DEFAULT_STREAM_TIMEOUT_MS = 250000;
5182
5360
  async function streamAsk(options) {
5183
- const profileOptions = options.profile ? { profile: options.profile } : {};
5184
- const url = `${resolveModaBaseUrl(profileOptions)}/api/cli/ask/stream`;
5361
+ const profileOptions2 = options.profile ? { profile: options.profile } : {};
5362
+ const url = `${resolveModaBaseUrl(profileOptions2)}/api/cli/ask/stream`;
5185
5363
  const controller = new AbortController;
5186
5364
  const timeout = setTimeout(() => controller.abort(), options.timeoutMs ?? DEFAULT_STREAM_TIMEOUT_MS);
5187
5365
  const startedAt = Date.now();
@@ -5197,7 +5375,7 @@ async function streamAsk(options) {
5197
5375
  response = await fetchImpl(url, {
5198
5376
  method: "POST",
5199
5377
  headers: {
5200
- "x-api-key": resolveApiKey(profileOptions) ?? "",
5378
+ "x-api-key": resolveApiKey(profileOptions2) ?? "",
5201
5379
  "Content-Type": "application/json",
5202
5380
  Accept: "text/event-stream"
5203
5381
  },
@@ -7005,6 +7183,15 @@ Commands:
7005
7183
  prompts ab Run a judged prompt A/B replay experiment
7006
7184
  prompts propose Revise a prompt from a run's failures (--gate to auto-A/B, --promote-on-win)
7007
7185
 
7186
+ Replays (read-only results):
7187
+ replays sets List replay sets
7188
+ replays runs <set_id> Run history for a set (every status)
7189
+ replays run <set_id> <run_id> One run: per-case and per-seed verdicts, playout count
7190
+ replays playouts <set_id> <run_id>
7191
+ Playouts per case/arm/seed (--transcript inlines messages and tool calls)
7192
+ replays playout <set_id> <run_id> <playout_id>
7193
+ One playout transcript
7194
+
7008
7195
  Skills:
7009
7196
  skills gen Kick off tenant-wide skill generation
7010
7197
  skills status [run-id] Show a generation run's status, outcome, and recent events (default: latest run)
@@ -7383,10 +7570,10 @@ async function attachWindows(rows, windowSize) {
7383
7570
  return { ...row, window: conversation };
7384
7571
  }
7385
7572
  try {
7386
- const query = new URLSearchParams;
7387
- query.set("msg_index", msg_index.toString());
7388
- query.set("window", windowSize.toString());
7389
- const data = await callDataAPI(`/conversations/${conv_id}/context?${query.toString()}`);
7573
+ const query2 = new URLSearchParams;
7574
+ query2.set("msg_index", msg_index.toString());
7575
+ query2.set("window", windowSize.toString());
7576
+ const data = await callDataAPI(`/conversations/${conv_id}/context?${query2.toString()}`);
7390
7577
  return { ...row, window: data };
7391
7578
  } catch (err) {
7392
7579
  const message = err instanceof Error ? err.message : String(err);
@@ -7787,15 +7974,15 @@ function shellArg(value) {
7787
7974
  async function searchFallbackToTraces(params) {
7788
7975
  const limit = Math.min(params.limit ?? 20, 100);
7789
7976
  const offset = params.offset ?? 0;
7790
- const query = new URLSearchParams;
7791
- query.set("search", params.q);
7977
+ const query2 = new URLSearchParams;
7978
+ query2.set("search", params.q);
7792
7979
  if (params.user_id)
7793
- query.set("user_id", params.user_id);
7980
+ query2.set("user_id", params.user_id);
7794
7981
  if (params.time_range)
7795
- query.set("time_range", params.time_range);
7796
- query.set("limit", String(limit));
7982
+ query2.set("time_range", params.time_range);
7983
+ query2.set("limit", String(limit));
7797
7984
  if (offset > 0)
7798
- query.set("offset", String(offset));
7985
+ query2.set("offset", String(offset));
7799
7986
  const tracesCommand = [
7800
7987
  `moda traces --search=${shellArg(params.q)}`,
7801
7988
  params.user_id ? `--user-id=${shellArg(params.user_id)}` : "",
@@ -7804,7 +7991,7 @@ async function searchFallbackToTraces(params) {
7804
7991
  ].filter(Boolean).join(" ");
7805
7992
  let traces;
7806
7993
  try {
7807
- traces = asRecord(await callDataAPI(`/conversations?${query.toString()}`)) ?? {};
7994
+ traces = asRecord(await callDataAPI(`/conversations?${query2.toString()}`)) ?? {};
7808
7995
  } catch (error) {
7809
7996
  const reason = error instanceof Error ? error.message : String(error);
7810
7997
  return {
@@ -7953,16 +8140,16 @@ function assertHandParsedFlags(command, flags, positionals) {
7953
8140
  var PROFILE_MANAGING_COMMANDS = new Set(["profiles", "auth", "init", "provision", "status", "config", "keys", "manifest", "agent", "doctor"]);
7954
8141
  async function runProblemsRemainder(flags, context) {
7955
8142
  const params = parseCommandFlags(ProblemsRemainderSchema, flagsToArgs(flags), "problems remainder");
7956
- const query = new URLSearchParams;
8143
+ const query2 = new URLSearchParams;
7957
8144
  if (params.days_back)
7958
- query.set("days_back", params.days_back.toString());
8145
+ query2.set("days_back", params.days_back.toString());
7959
8146
  if (params.limit)
7960
- query.set("limit", params.limit.toString());
8147
+ query2.set("limit", params.limit.toString());
7961
8148
  if (params.family)
7962
- query.set("family", params.family);
8149
+ query2.set("family", params.family);
7963
8150
  if (params.cursor)
7964
- query.set("cursor", params.cursor);
7965
- const queryString = query.toString() ? `?${query.toString()}` : "";
8151
+ query2.set("cursor", params.cursor);
8152
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
7966
8153
  context.output.writeData(await callDataAPI(`/problems/remainder${queryString}`));
7967
8154
  }
7968
8155
  function processingDisabledWarning(data, family) {
@@ -8007,6 +8194,8 @@ function pageMessages(page) {
8007
8194
  const messages = asRecord(page.context)?.messages ?? page.messages;
8008
8195
  return (Array.isArray(messages) ? messages : []).map((m) => asRecord(m) ?? {});
8009
8196
  }
8197
+ var CONTEXT_MAX_WINDOW = 5;
8198
+ var CONTEXT_ALL_MAX_MESSAGES = 5000;
8010
8199
  async function readFullTranscript(conversationId, opts = {}) {
8011
8200
  const start = opts.from ?? 0;
8012
8201
  const requested = opts.maxMessages ?? FULL_TRANSCRIPT_DEFAULT_MAX_MESSAGES;
@@ -8158,28 +8347,28 @@ async function runCommand(command, positional, flags, positionals = positional ?
8158
8347
  const args = flagsToArgs(flags);
8159
8348
  const params = parseCommandFlags(ClustersSchema, args, command);
8160
8349
  if (params.search !== undefined || params.node_id !== undefined) {
8161
- const query2 = new URLSearchParams;
8350
+ const query3 = new URLSearchParams;
8162
8351
  if (params.search !== undefined)
8163
- query2.set("q", params.search);
8352
+ query3.set("q", params.search);
8164
8353
  if (params.node_id !== undefined)
8165
- query2.set("node_id", params.node_id);
8354
+ query3.set("node_id", params.node_id);
8166
8355
  if (params.mode)
8167
- query2.set("mode", params.mode);
8356
+ query3.set("mode", params.mode);
8168
8357
  if (params.limit)
8169
- query2.set("limit", params.limit.toString());
8170
- const data2 = await callDataAPI(`/task-clusters/search?${query2.toString()}`);
8358
+ query3.set("limit", params.limit.toString());
8359
+ const data2 = await callDataAPI(`/task-clusters/search?${query3.toString()}`);
8171
8360
  const record = asRecord(data2) ?? {};
8172
8361
  const matches = Array.isArray(record.matches) ? record.matches : [];
8173
8362
  const warnings = params.node_id !== undefined && matches.length === 0 ? notFoundWarning("cluster", params.node_id, true) : [];
8174
8363
  context.output.writeData(data2, warnings.length > 0 ? { warnings } : undefined);
8175
8364
  break;
8176
8365
  }
8177
- const query = new URLSearchParams;
8366
+ const query2 = new URLSearchParams;
8178
8367
  if (params.parent_id)
8179
- query.set("parent_id", params.parent_id);
8368
+ query2.set("parent_id", params.parent_id);
8180
8369
  if (params.time_range)
8181
- query.set("time_range", params.time_range);
8182
- const queryString = query.toString() ? `?${query.toString()}` : "";
8370
+ query2.set("time_range", params.time_range);
8371
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8183
8372
  const data = await callDataAPI(`/clusters${queryString}`);
8184
8373
  context.output.writeData(data);
8185
8374
  break;
@@ -8191,12 +8380,12 @@ async function runCommand(command, positional, flags, positionals = positional ?
8191
8380
  }
8192
8381
  const args = flagsToArgs(flags, "node_id", positional);
8193
8382
  const params = parseCommandFlags(ClusterConversationsSchema, args, command, { positionalKey: "node_id" });
8194
- const query = new URLSearchParams;
8383
+ const query2 = new URLSearchParams;
8195
8384
  if (params.limit)
8196
- query.set("limit", params.limit.toString());
8385
+ query2.set("limit", params.limit.toString());
8197
8386
  if (params.offset !== undefined)
8198
- query.set("offset", params.offset.toString());
8199
- const queryString = query.toString() ? `?${query.toString()}` : "";
8387
+ query2.set("offset", params.offset.toString());
8388
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8200
8389
  const data = await callDataAPI(`/clusters/${params.node_id}/conversations${queryString}`);
8201
8390
  const clusterRecord = asRecord(data) ?? {};
8202
8391
  const warnings = notFoundWarning("cluster", params.node_id, clusterRecord.cluster === null || clusterRecord.cluster === undefined);
@@ -8207,28 +8396,28 @@ async function runCommand(command, positional, flags, positionals = positional ?
8207
8396
  case "conversations": {
8208
8397
  const args = withStringIdFlags(flagsToArgs(flags), flags);
8209
8398
  const params = parseCommandFlags(ConversationsSchema, args, command);
8210
- const query = new URLSearchParams;
8399
+ const query2 = new URLSearchParams;
8211
8400
  if (params.search)
8212
- query.set("search", params.search);
8401
+ query2.set("search", params.search);
8213
8402
  if (params.cluster_id)
8214
- query.set("cluster_id", params.cluster_id);
8403
+ query2.set("cluster_id", params.cluster_id);
8215
8404
  if (params.user_id)
8216
- query.set("user_id", params.user_id);
8405
+ query2.set("user_id", params.user_id);
8217
8406
  if (params.time_range)
8218
- query.set("time_range", params.time_range);
8407
+ query2.set("time_range", params.time_range);
8219
8408
  if (params.environment)
8220
- query.set("environment", params.environment);
8409
+ query2.set("environment", params.environment);
8221
8410
  if (params.world_state)
8222
- query.set("world_state", params.world_state);
8411
+ query2.set("world_state", params.world_state);
8223
8412
  if (params.outcome && params.outcome !== "any")
8224
- query.set("outcome", params.outcome);
8413
+ query2.set("outcome", params.outcome);
8225
8414
  if (params.sort)
8226
- query.set("sort_by", params.sort);
8415
+ query2.set("sort_by", params.sort);
8227
8416
  if (params.limit)
8228
- query.set("limit", params.limit.toString());
8417
+ query2.set("limit", params.limit.toString());
8229
8418
  if (params.offset !== undefined)
8230
- query.set("offset", params.offset.toString());
8231
- const queryString = query.toString() ? `?${query.toString()}` : "";
8419
+ query2.set("offset", params.offset.toString());
8420
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8232
8421
  const raw = await callDataAPI(`/conversations${queryString}`);
8233
8422
  recordConversationsTelemetry(context, raw);
8234
8423
  const searchWarnings = [];
@@ -8265,19 +8454,19 @@ async function runCommand(command, positional, flags, positionals = positional ?
8265
8454
  }
8266
8455
  const args = withStringIdFlags(flagsToArgs(flags, "q", positional), flags);
8267
8456
  const params = parseCommandFlags(SearchSchema, args, command, { positionalKey: "q" });
8268
- const query = new URLSearchParams;
8269
- query.set("q", params.q);
8457
+ const query2 = new URLSearchParams;
8458
+ query2.set("q", params.q);
8270
8459
  if (params.mode)
8271
- query.set("mode", params.mode);
8460
+ query2.set("mode", params.mode);
8272
8461
  if (params.user_id)
8273
- query.set("user_id", params.user_id);
8462
+ query2.set("user_id", params.user_id);
8274
8463
  if (params.time_range)
8275
- query.set("time_range", params.time_range);
8464
+ query2.set("time_range", params.time_range);
8276
8465
  if (params.limit)
8277
- query.set("limit", params.limit.toString());
8466
+ query2.set("limit", params.limit.toString());
8278
8467
  if (params.include_tool_io)
8279
- query.set("include_tool_io", "true");
8280
- const data = await callDataAPI(`/search?${query.toString()}`);
8468
+ query2.set("include_tool_io", "true");
8469
+ const data = await callDataAPI(`/search?${query2.toString()}`);
8281
8470
  recordSearchTelemetry(context, data);
8282
8471
  const searchRecord = asRecord(data) ?? {};
8283
8472
  const degradeReason = asString(searchRecord.degrade_reason);
@@ -8325,13 +8514,13 @@ async function runCommand(command, positional, flags, positionals = positional ?
8325
8514
  throw new CliInputError("Choose either --snapshot or --replay, not both.", "Usage: moda world-state <conversation_id> [--snapshot --msg-index=N [--block-index=M] [--segment-index=K]] | [--replay --message-count=N [--format=frames|fold-inputs]]");
8326
8515
  }
8327
8516
  if (params.snapshot === true) {
8328
- const query2 = new URLSearchParams;
8329
- query2.set("msg_index", (params.msg_index ?? 0).toString());
8517
+ const query3 = new URLSearchParams;
8518
+ query3.set("msg_index", (params.msg_index ?? 0).toString());
8330
8519
  if (params.block_index !== undefined)
8331
- query2.set("block_index", params.block_index.toString());
8520
+ query3.set("block_index", params.block_index.toString());
8332
8521
  if (params.segment_index !== undefined)
8333
- query2.set("segment_index", params.segment_index.toString());
8334
- const data2 = await callDataAPI(`/conversations/${params.conversation_id}/world-state/snapshot?${query2.toString()}`);
8522
+ query3.set("segment_index", params.segment_index.toString());
8523
+ const data2 = await callDataAPI(`/conversations/${params.conversation_id}/world-state/snapshot?${query3.toString()}`);
8335
8524
  const snapshotRecord = asRecord(data2) ?? {};
8336
8525
  const warnings2 = notFoundWarning("world state", params.conversation_id, snapshotRecord.found === false);
8337
8526
  context.output.writeData(data2, warnings2.length > 0 ? { warnings: warnings2 } : undefined);
@@ -8342,22 +8531,22 @@ async function runCommand(command, positional, flags, positionals = positional ?
8342
8531
  if (format === "frames" && params.message_count === undefined) {
8343
8532
  throw new CliInputError("--message-count=N is required with --replay in frames mode (the API returns no frames without it). Use --format=fold-inputs for the raw keyframes/events without a count.");
8344
8533
  }
8345
- const query2 = new URLSearchParams;
8346
- query2.set("format", format === "fold-inputs" ? "fold_inputs" : "frames");
8534
+ const query3 = new URLSearchParams;
8535
+ query3.set("format", format === "fold-inputs" ? "fold_inputs" : "frames");
8347
8536
  if (params.message_count !== undefined)
8348
- query2.set("message_count", params.message_count.toString());
8349
- const data2 = await callDataAPI(`/conversations/${params.conversation_id}/world-state/replay?${query2.toString()}`);
8537
+ query3.set("message_count", params.message_count.toString());
8538
+ const data2 = await callDataAPI(`/conversations/${params.conversation_id}/world-state/replay?${query3.toString()}`);
8350
8539
  const replayRecord = asRecord(data2) ?? {};
8351
8540
  const warnings2 = notFoundWarning("world state", params.conversation_id, replayRecord.hasData === false);
8352
8541
  context.output.writeData(data2, warnings2.length > 0 ? { warnings: warnings2 } : undefined);
8353
8542
  break;
8354
8543
  }
8355
- const query = new URLSearchParams;
8544
+ const query2 = new URLSearchParams;
8356
8545
  if (params.summary_only === true)
8357
- query.set("summary_only", "1");
8546
+ query2.set("summary_only", "1");
8358
8547
  if (params.event_limit)
8359
- query.set("event_limit", params.event_limit.toString());
8360
- const queryString = query.toString() ? `?${query.toString()}` : "";
8548
+ query2.set("event_limit", params.event_limit.toString());
8549
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8361
8550
  const data = await callDataAPI(`/conversations/${params.conversation_id}/world-state${queryString}`);
8362
8551
  const wsRecord = asRecord(data) ?? {};
8363
8552
  const warnings = notFoundWarning("world state", params.conversation_id, wsRecord.hasData === false);
@@ -8389,16 +8578,29 @@ async function runCommand(command, positional, flags, positionals = positional ?
8389
8578
  if (params.from !== undefined || params.max_messages !== undefined) {
8390
8579
  throw new CliInputError("--from and --max-messages page the whole trace and need --all.", "Usage: moda context <conversation_id> --all [--from=N] [--max-messages=N]");
8391
8580
  }
8392
- const query = new URLSearchParams;
8581
+ const query2 = new URLSearchParams;
8393
8582
  if (params.msg_index !== undefined)
8394
- query.set("msg_index", params.msg_index.toString());
8395
- if (params.window)
8396
- query.set("window", params.window.toString());
8397
- const queryString = query.toString() ? `?${query.toString()}` : "";
8583
+ query2.set("msg_index", params.msg_index.toString());
8584
+ const window = params.window === undefined ? undefined : Math.min(params.window, CONTEXT_MAX_WINDOW);
8585
+ if (window !== undefined)
8586
+ query2.set("window", window.toString());
8587
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8398
8588
  const data = await callDataAPI(`/conversations/${params.conversation_id}/context${queryString}`);
8399
8589
  const ctxRecord = asRecord(data) ?? {};
8400
8590
  const warnings = notFoundWarning("trace", params.conversation_id, asNumber(ctxRecord.total_messages) === 0);
8401
- context.output.writeData(data, warnings.length > 0 ? { warnings } : undefined);
8591
+ const nextCommands = [];
8592
+ if (params.window !== undefined && params.window > CONTEXT_MAX_WINDOW) {
8593
+ const center = asNumber(asRecord(ctxRecord.context)?.center_index) ?? params.msg_index ?? 0;
8594
+ const span = Math.min(params.window * 2 + 1, CONTEXT_ALL_MAX_MESSAGES);
8595
+ const from = Math.max(0, center - Math.floor((span - 1) / 2));
8596
+ const wide = `moda context ${shellArg(params.conversation_id)} --all --from=${from} --max-messages=${span}`;
8597
+ warnings.push(`--window=${params.window} is above the maximum of ${CONTEXT_MAX_WINDOW}, so this shows ±${CONTEXT_MAX_WINDOW} messages. For a wider slice run: ${wide}`);
8598
+ nextCommands.push({ command: wide, purpose: `Read ±${params.window} messages around the anchor.`, mutability: "read", requires_approval: false });
8599
+ }
8600
+ context.output.writeData(data, {
8601
+ ...warnings.length > 0 ? { warnings } : {},
8602
+ ...nextCommands.length > 0 ? { nextCommands } : {}
8603
+ });
8402
8604
  break;
8403
8605
  }
8404
8606
  case "audit":
@@ -8408,14 +8610,14 @@ async function runCommand(command, positional, flags, positionals = positional ?
8408
8610
  }
8409
8611
  const args = flagsToArgs(flags, "id", positional);
8410
8612
  const params = parseCommandFlags(AuditSchema, args, command, { positionalKey: "id" });
8411
- const query = new URLSearchParams;
8613
+ const query2 = new URLSearchParams;
8412
8614
  if (params.kind && params.kind !== "auto")
8413
- query.set("kind", params.kind);
8615
+ query2.set("kind", params.kind);
8414
8616
  if (params.include_raw === true)
8415
- query.set("include_raw", "1");
8617
+ query2.set("include_raw", "1");
8416
8618
  if (params.limit)
8417
- query.set("limit", params.limit.toString());
8418
- const queryString = query.toString() ? `?${query.toString()}` : "";
8619
+ query2.set("limit", params.limit.toString());
8620
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8419
8621
  const data = await callDataAPI(`/audit/${encodeURIComponent(params.id)}${queryString}`);
8420
8622
  const capped = auditCapStatus(data, params.limit ?? AUDIT_DEFAULT_LIMIT);
8421
8623
  const auditRecord = asRecord(data);
@@ -8448,19 +8650,19 @@ async function runCommand(command, positional, flags, positionals = positional ?
8448
8650
  case "frustrations": {
8449
8651
  const args = withStringIdFlags(flagsToArgs(flags), flags);
8450
8652
  const params = parseCommandFlags(FrustrationsSchema, args, command);
8451
- const query = new URLSearchParams;
8653
+ const query2 = new URLSearchParams;
8452
8654
  if (params.days_back)
8453
- query.set("days_back", params.days_back.toString());
8655
+ query2.set("days_back", params.days_back.toString());
8454
8656
  if (params.limit)
8455
- query.set("limit", params.limit.toString());
8657
+ query2.set("limit", params.limit.toString());
8456
8658
  if (params.offset !== undefined)
8457
- query.set("offset", params.offset.toString());
8659
+ query2.set("offset", params.offset.toString());
8458
8660
  if (params.user_id)
8459
- query.set("user_id", params.user_id);
8661
+ query2.set("user_id", params.user_id);
8460
8662
  if (params.conversation_id)
8461
- query.set("conversation_id", params.conversation_id);
8462
- query.set("family", "frustration");
8463
- const data = frustrationsFromEmotions(await callDataAPI(`/emotions?${query.toString()}`));
8663
+ query2.set("conversation_id", params.conversation_id);
8664
+ query2.set("family", "frustration");
8665
+ const data = frustrationsFromEmotions(await callDataAPI(`/emotions?${query2.toString()}`));
8464
8666
  await printWithOptionalWindows(withFrustrationAnchors(data), "frustrations", params.include_window === true, params.window ?? 1, context.output, processingDisabledWarning(data, "frustration"));
8465
8667
  break;
8466
8668
  }
@@ -8489,15 +8691,15 @@ async function runCommand(command, positional, flags, positionals = positional ?
8489
8691
  if (params.compact === true && params.include_window === true) {
8490
8692
  throw new CliInputError("--compact drops the conversation window; drop --include-window, or drop --compact.");
8491
8693
  }
8492
- const query = new URLSearchParams;
8694
+ const query2 = new URLSearchParams;
8493
8695
  if (params.subtype)
8494
- query.set("subtype", params.subtype);
8696
+ query2.set("subtype", params.subtype);
8495
8697
  if (params.days_back)
8496
- query.set("days_back", params.days_back.toString());
8497
- query.set("limit", (params.limit ?? TOOL_FAILURE_DETAIL_DEFAULT_LIMIT).toString());
8698
+ query2.set("days_back", params.days_back.toString());
8699
+ query2.set("limit", (params.limit ?? TOOL_FAILURE_DETAIL_DEFAULT_LIMIT).toString());
8498
8700
  if (params.offset !== undefined)
8499
- query.set("offset", params.offset.toString());
8500
- const queryString = query.toString() ? `?${query.toString()}` : "";
8701
+ query2.set("offset", params.offset.toString());
8702
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8501
8703
  const data = await callDataAPI(`/tool-failures/${params.tool_name}${queryString}`);
8502
8704
  const tfRecord = asRecord(data) ?? {};
8503
8705
  const subtypeCount = Array.isArray(tfRecord.subtypes) ? tfRecord.subtypes.length : 0;
@@ -8531,7 +8733,7 @@ async function runCommand(command, positional, flags, positionals = positional ?
8531
8733
  }
8532
8734
  case "problem-impact": {
8533
8735
  const params = ProblemImpactSchema.parse(Object.fromEntries(Object.entries(flags).map(([key, value]) => [key.replace(/-/g, "_"), value])));
8534
- const query = new URLSearchParams({ days: String(params.days_back), mode: params.mode });
8736
+ const query2 = new URLSearchParams({ days: String(params.days_back), mode: params.mode });
8535
8737
  for (const [key, value] of Object.entries({
8536
8738
  nodeId: params.node_id,
8537
8739
  problemId: params.problem_id,
@@ -8541,8 +8743,8 @@ async function runCommand(command, positional, flags, positionals = positional ?
8541
8743
  conversationIds: params.conversation_ids ? JSON.stringify(params.conversation_ids) : undefined
8542
8744
  }))
8543
8745
  if (value !== undefined)
8544
- query.set(key, value);
8545
- const data = asRecord(await callDataAPI(`/problem-impact?${query}`)) ?? {};
8746
+ query2.set(key, value);
8747
+ const data = asRecord(await callDataAPI(`/problem-impact?${query2}`)) ?? {};
8546
8748
  const link = new URLSearchParams({ impactDays: String(params.days_back) });
8547
8749
  const window = asRecord(data.window);
8548
8750
  const resolvedRun = asString(data.runId);
@@ -8569,12 +8771,12 @@ async function runCommand(command, positional, flags, positionals = positional ?
8569
8771
  }
8570
8772
  const args = flagsToArgs(flags);
8571
8773
  const params = parseCommandFlags(ProblemsSchema, args, command);
8572
- const query = new URLSearchParams;
8774
+ const query2 = new URLSearchParams;
8573
8775
  if (params.days_back)
8574
- query.set("days_back", params.days_back.toString());
8776
+ query2.set("days_back", params.days_back.toString());
8575
8777
  if (params.limit)
8576
- query.set("limit", params.limit.toString());
8577
- const queryString = query.toString() ? `?${query.toString()}` : "";
8778
+ query2.set("limit", params.limit.toString());
8779
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8578
8780
  const data = await callDataAPI(`/problems${queryString}`);
8579
8781
  const { payload, truncated } = capProblemsBlocks(asRecord(data) ?? {}, params.limit ?? 25);
8580
8782
  const withLink = {
@@ -8616,18 +8818,18 @@ async function runCommand(command, positional, flags, positionals = positional ?
8616
8818
  const viewFlag = view === "conversations" ? "--traces" : `--${view}`;
8617
8819
  throw new CliInputError(`${viewFlag} requires a canonical problem UUID; got "${params.id}". Run \`moda problems\` or \`moda problem <id>\` first to resolve the id.`);
8618
8820
  }
8619
- const query = new URLSearchParams;
8821
+ const query2 = new URLSearchParams;
8620
8822
  if (params.limit)
8621
- query.set("limit", params.limit.toString());
8823
+ query2.set("limit", params.limit.toString());
8622
8824
  if (params.cursor)
8623
- query.set("cursor", params.cursor);
8825
+ query2.set("cursor", params.cursor);
8624
8826
  if (view === "conversations") {
8625
8827
  if (params.family)
8626
- query.set("family", params.family);
8828
+ query2.set("family", params.family);
8627
8829
  if (params.door)
8628
- query.set("door", params.door);
8830
+ query2.set("door", params.door);
8629
8831
  }
8630
- const queryString = query.toString() ? `?${query.toString()}` : "";
8832
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8631
8833
  const data = await callDataAPI(`/problems/${params.id}/${view}${queryString}`);
8632
8834
  context.output.writeData(data);
8633
8835
  break;
@@ -8763,14 +8965,14 @@ async function runCommand(command, positional, flags, positionals = positional ?
8763
8965
  context.output.writeData(data2);
8764
8966
  break;
8765
8967
  }
8766
- const query = new URLSearchParams;
8968
+ const query2 = new URLSearchParams;
8767
8969
  if (params.days_back)
8768
- query.set("days_back", params.days_back.toString());
8970
+ query2.set("days_back", params.days_back.toString());
8769
8971
  if (params.limit)
8770
- query.set("limit", params.limit.toString());
8972
+ query2.set("limit", params.limit.toString());
8771
8973
  if (params.offset !== undefined)
8772
- query.set("offset", params.offset.toString());
8773
- const queryString = query.toString() ? `?${query.toString()}` : "";
8974
+ query2.set("offset", params.offset.toString());
8975
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8774
8976
  const data = await callDataAPI(`/custom-signals/${id}/detections${queryString}`);
8775
8977
  context.output.writeData(data);
8776
8978
  break;
@@ -8922,20 +9124,20 @@ async function runCommand(command, positional, flags, positionals = positional ?
8922
9124
  case "false-positives": {
8923
9125
  if (!positional) {
8924
9126
  const kinds = { quote: "frustration_quote", tool: "tool_call", laziness: "laziness", hallucination: "hallucination", "custom-signal": "custom_signal" };
8925
- const query = new URLSearchParams;
9127
+ const query2 = new URLSearchParams;
8926
9128
  if (flags.kind !== undefined) {
8927
9129
  if (!kinds[flags.kind])
8928
9130
  throw new CliInputError(`--kind must be one of ${Object.keys(kinds).join("|")}`);
8929
- query.set("kind", kinds[flags.kind]);
9131
+ query2.set("kind", kinds[flags.kind]);
8930
9132
  }
8931
9133
  if (flags.limit !== undefined) {
8932
9134
  if (!/^\d+$/.test(flags.limit) || Number(flags.limit) < 1 || Number(flags.limit) > 200)
8933
9135
  throw new CliInputError("--limit must be 1-200");
8934
- query.set("limit", flags.limit);
9136
+ query2.set("limit", flags.limit);
8935
9137
  }
8936
9138
  if (flags.all !== undefined && flags.all !== "false")
8937
- query.set("include_cleared", "true");
8938
- const queryString = query.toString() ? `?${query.toString()}` : "";
9139
+ query2.set("include_cleared", "true");
9140
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8939
9141
  const data2 = await callDataAPI(`/false-positives${queryString}`);
8940
9142
  context.output.writeData(data2);
8941
9143
  break;
@@ -8947,20 +9149,20 @@ async function runCommand(command, positional, flags, positionals = positional ?
8947
9149
  case "emotions": {
8948
9150
  const args = withStringIdFlags(flagsToArgs(flags), flags);
8949
9151
  const params = parseCommandFlags(EmotionsSchema, args, command);
8950
- const query = new URLSearchParams;
9152
+ const query2 = new URLSearchParams;
8951
9153
  if (params.days_back)
8952
- query.set("days_back", params.days_back.toString());
9154
+ query2.set("days_back", params.days_back.toString());
8953
9155
  if (params.limit)
8954
- query.set("limit", params.limit.toString());
9156
+ query2.set("limit", params.limit.toString());
8955
9157
  if (params.offset !== undefined)
8956
- query.set("offset", params.offset.toString());
9158
+ query2.set("offset", params.offset.toString());
8957
9159
  if (params.family)
8958
- query.set("family", params.family);
9160
+ query2.set("family", params.family);
8959
9161
  if (params.user_id)
8960
- query.set("user_id", params.user_id);
9162
+ query2.set("user_id", params.user_id);
8961
9163
  if (params.conversation_id)
8962
- query.set("conversation_id", params.conversation_id);
8963
- const queryString = query.toString() ? `?${query.toString()}` : "";
9164
+ query2.set("conversation_id", params.conversation_id);
9165
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8964
9166
  const data = await callDataAPI(`/emotions${queryString}`);
8965
9167
  const processingWarnings = processingDisabledWarning(data, "emotion");
8966
9168
  context.output.writeData(data, processingWarnings.length > 0 ? { warnings: processingWarnings } : undefined);
@@ -8972,18 +9174,18 @@ async function runCommand(command, positional, flags, positionals = positional ?
8972
9174
  args.conversation_id = flags["conversation-id"];
8973
9175
  }
8974
9176
  const params = parseCommandFlags(LazinessSchema, args, command);
8975
- const query = new URLSearchParams;
9177
+ const query2 = new URLSearchParams;
8976
9178
  if (params.days_back)
8977
- query.set("days_back", params.days_back.toString());
9179
+ query2.set("days_back", params.days_back.toString());
8978
9180
  if (params.limit)
8979
- query.set("limit", params.limit.toString());
9181
+ query2.set("limit", params.limit.toString());
8980
9182
  if (params.offset !== undefined)
8981
- query.set("offset", params.offset.toString());
9183
+ query2.set("offset", params.offset.toString());
8982
9184
  if (params.conversation_id)
8983
- query.set("conversation_id", params.conversation_id);
9185
+ query2.set("conversation_id", params.conversation_id);
8984
9186
  if (params.family)
8985
- query.set("family", params.family);
8986
- const queryString = query.toString() ? `?${query.toString()}` : "";
9187
+ query2.set("family", params.family);
9188
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
8987
9189
  const data = await callDataAPI(`/laziness${queryString}`);
8988
9190
  context.output.writeData(data);
8989
9191
  break;
@@ -8994,18 +9196,18 @@ async function runCommand(command, positional, flags, positionals = positional ?
8994
9196
  args.conversation_id = flags["conversation-id"];
8995
9197
  }
8996
9198
  const params = parseCommandFlags(HallucinationsSchema, args, command);
8997
- const query = new URLSearchParams;
9199
+ const query2 = new URLSearchParams;
8998
9200
  if (params.days_back)
8999
- query.set("days_back", params.days_back.toString());
9201
+ query2.set("days_back", params.days_back.toString());
9000
9202
  if (params.limit)
9001
- query.set("limit", params.limit.toString());
9203
+ query2.set("limit", params.limit.toString());
9002
9204
  if (params.offset !== undefined)
9003
- query.set("offset", params.offset.toString());
9205
+ query2.set("offset", params.offset.toString());
9004
9206
  if (params.conversation_id)
9005
- query.set("conversation_id", params.conversation_id);
9207
+ query2.set("conversation_id", params.conversation_id);
9006
9208
  if (params.kind)
9007
- query.set("kind", params.kind);
9008
- const queryString = query.toString() ? `?${query.toString()}` : "";
9209
+ query2.set("kind", params.kind);
9210
+ const queryString = query2.toString() ? `?${query2.toString()}` : "";
9009
9211
  const data = await callDataAPI(`/hallucinations${queryString}`);
9010
9212
  context.output.writeData(data);
9011
9213
  break;
@@ -9173,7 +9375,7 @@ var commandRegistry = createCommandRegistry([
9173
9375
  resetApiRequestCountBeforeRun: true,
9174
9376
  telemetry: "result",
9175
9377
  handler: async (context) => {
9176
- const { runInit } = await import("./index-ah6yf5x2.js");
9378
+ const { runInit } = await import("./index-r1egcg99.js");
9177
9379
  if (context.outputMode === "agent-stream") {
9178
9380
  context.output.writeEvent({
9179
9381
  event: "started",
@@ -9481,6 +9683,23 @@ var commandRegistry = createCommandRegistry([
9481
9683
  telemetry: "result-and-error",
9482
9684
  handler: runUserCommand
9483
9685
  },
9686
+ {
9687
+ name: "replays",
9688
+ description: "Read replay results: sets, run history, per-seed verdicts, and playout transcripts",
9689
+ examples: [
9690
+ "moda replays runs <set_id>",
9691
+ "moda replays run <set_id> <run_id> --json",
9692
+ "moda replays playouts <set_id> <run_id> --case=<case_id> --arm=proposed --transcript --json"
9693
+ ],
9694
+ subcommands: REPLAYS_SUBCOMMAND_HELP,
9695
+ auth: "api-key",
9696
+ outputModes: ["human", "json", "agent"],
9697
+ mutability: "read",
9698
+ validateConfigBeforeRun: false,
9699
+ resetApiRequestCountBeforeRun: false,
9700
+ telemetry: "result-and-error",
9701
+ handler: runReplaysCommand
9702
+ },
9484
9703
  {
9485
9704
  name: "problem-reopen",
9486
9705
  description: "Reopen a problem that was marked fixed (resolved problems return only via re-detection)",
@@ -9624,6 +9843,7 @@ var commandRegistry = createCommandRegistry([
9624
9843
  description: "Get windowed trace context (messages around one turn), or the whole trace with --all",
9625
9844
  examples: [
9626
9845
  "moda context <conversation_id> --window=3",
9846
+ "moda context <conversation_id> --msg-index=40 --window=0",
9627
9847
  "moda context <conversation_id> --all",
9628
9848
  "moda context <conversation_id> --all --from=500 --max-messages=500"
9629
9849
  ],
@@ -9921,11 +10141,12 @@ var commandRegistry = createCommandRegistry([
9921
10141
  flags: [
9922
10142
  { name: "--baseline=<path>", description: "Prompt file to treat as the control." },
9923
10143
  { name: "--candidate=<path>", description: "Prompt file to treat as the variant." },
9924
- { name: "--traces=<ids>", description: "Comma-separated trace ids (conversation_id values) to replay, at most 100. Legacy alias: --conversations=<ids>." },
10144
+ { name: "--traces=<ids>", description: "Comma-separated trace ids (conversation_id values) to replay, at most 100. Each case uses Moda's distilled scenario and success criteria for the trace; a trace that cannot be distilled falls back to its opening user message with generic criteria, and the run warns. Legacy alias: --conversations=<ids>." },
9925
10145
  { name: "--cases=<n>", description: "Cases to auto-generate (default 5, max 100)." },
9926
10146
  { name: "--seeds=<n>", description: "Repeats per case per arm (default 3, max 10)." },
9927
10147
  { name: "--model=<slug>", description: "Assistant model for both arms; must be in the replay allowlist unless --allow-any-model." },
9928
- { name: "--yes", description: "Confirm a run above 200 planned playouts (cases x seeds x 2 arms), or one whose case count is unknown." }
10148
+ { name: "--yes", description: "Confirm a run above 200 planned playouts (cases x seeds x 2 arms), or one whose case count is unknown." },
10149
+ { name: "--promote", description: "Make this run's replay set the tenant's primary replay set. Off by default, so an ad-hoc A/B never replaces it." }
9929
10150
  ],
9930
10151
  examples: [
9931
10152
  "moda prompts ab --baseline=prompts/agent.prompt.md --candidate=prompts/agent-v2.prompt.md --traces=conv_1,conv_2"
@@ -3,7 +3,7 @@ import {
3
3
  initPrompts,
4
4
  runPromptSync,
5
5
  runSkillSync
6
- } from "./cli-pr6s56fc.js";
6
+ } from "./cli-ecrx32mm.js";
7
7
  import {
8
8
  codingAgentDisplayName,
9
9
  describeCodingAgentEvent,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@moda-ai/cli",
3
- "version": "1.41.2",
3
+ "version": "1.43.0",
4
4
  "description": "CLI for Moda - AI agent analytics and observability",
5
5
  "type": "module",
6
6
  "bin": {
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "schema_version": "moda.skill_index.v1",
3
- "bundled_at": "2026-10-09T01:27:15.161Z",
4
- "cli_version": "1.41.2",
3
+ "bundled_at": "2026-10-09T02:13:14.110Z",
4
+ "cli_version": "1.43.0",
5
5
  "skills": [
6
6
  {
7
7
  "id": "integration-cloudflare-think",
@@ -442,7 +442,9 @@ fail instead of being clamped.
442
442
 
443
443
  `--window=N` on `context`, `frustrations --include-window` and
444
444
  `tool-failure-detail --include-window` is a message half-width (1–5), not a
445
- time window.
445
+ time window. On `context`, `--window=0` returns only the anchor message (plus
446
+ `total_messages`), and a value above 5 is clamped to 5 with a warning that
447
+ prints the `--all --from --max-messages` command for a wider slice.
446
448
 
447
449
  ### Schema introspection
448
450
 
@@ -631,6 +633,7 @@ Filters: `--search`, `--cluster-id`, `--user-id`, `--time-range`
631
633
  moda context <conversation_id> # default window around middle
632
634
  moda context <conversation_id> --msg-index=5 # center on message 5
633
635
  moda context <conversation_id> --window=3 # 3 messages each side (max 5)
636
+ moda context <conversation_id> --msg-index=40 --window=0 # just message 40 + total_messages
634
637
  moda context <conversation_id> --all # whole trace in order, first 500 messages
635
638
  moda context <conversation_id> --all --from=500 --max-messages=500 # continue (max 5000)
636
639
  ```
@@ -767,6 +770,13 @@ earlier turns as history);
767
770
  `anthropic/claude-haiku-4-5`, `google/gemini-2.5-flash`,
768
771
  `google/gemini-2.5-pro` unless you pass `--allow-any-model`.
769
772
 
773
+ `prompts ab --traces=<ids>` builds one case per trace from Moda's distilled
774
+ scenario and success criteria for that trace. A trace Moda cannot distill
775
+ falls back to its opening user message with three generic criteria (which long
776
+ traces pass easily); the run warns and lists it under `caseSources.fallback`.
777
+ `prompts ab` never makes its replay set the tenant's primary set unless you
778
+ pass `--promote`.
779
+
770
780
  Runtime code should render synced prompts through the SDK:
771
781
 
772
782
  ```typescript
@@ -785,6 +795,43 @@ Do not copy prompt text from the dashboard into source code. The repo prompt
785
795
  files are the source of truth; the dashboard is for visibility, usage, labels,
786
796
  and future evals.
787
797
 
798
+ ### 7a. Check a replay verdict (`moda replays`)
799
+
800
+ Use after `prompts ab`, `prompts propose --gate`, or a dashboard replay when
801
+ you need to see *why* an arm passed or failed. Read-only; no replay spend.
802
+
803
+ ```bash
804
+ moda replays sets --json # set ids
805
+ moda replays runs <set_id> --limit=5 --json # newest runs, every status
806
+ moda replays run <set_id> <run_id> --json # per-case + per-seed verdicts
807
+ moda replays playouts <set_id> <run_id> --json # one row per case x arm x seed
808
+ moda replays playouts <set_id> <run_id> --case=<case_id> --arm=proposed --seed=1 --transcript --json
809
+ moda replays playout <set_id> <run_id> <playout_id> --json
810
+ ```
811
+
812
+ `run` returns `results[]` (one per case × arm × seed: `pass`, `abstained`,
813
+ `coverageReason`, `score`, `rationale`, `criteria[]`) next to the per-case
814
+ majority in `cases[]`. `arm=prod` is the baseline and `arm=proposed` the
815
+ candidate. An `abstained: true` row is not a verdict: the pair hit a replay
816
+ coverage hole and is excluded from pass counts.
817
+
818
+ A playout's `transcript.events[]` holds the simulated user and agent messages
819
+ and every tool call with its `input`, the `output` the replay served, and
820
+ `provenance.source`: `replay_runtime` (compiled mock over the case's recorded
821
+ source state), `agent_tool_simulator` (LLM simulator grounded in recorded
822
+ receipts; `simulator.model`), `counterfactual_reconstruction`, or
823
+ `generic_handler`. `provenance.errorSubtype` such as `missing_source_state`
824
+ marks an unservable call. `judge` is the judgement stored with the playout and
825
+ `verdict` the stored eval row (null when `seed` is null, which happens on
826
+ older runs whose graphs cannot be matched to a seed).
827
+
828
+ Limits: `--transcript` pages at most 10 playouts (`--limit` default 50, max
829
+ 200 without it); text and tool JSON are clipped at `--max-chars` (default
830
+ 4000, max 32000) and `transcript.truncated.fields` counts the clipped fields.
831
+ Page with `--cursor=<nextCursor>`. Playouts exist only for runs whose
832
+ playouts were captured; `playoutCount: 0` on `run` means there is no
833
+ transcript to read.
834
+
788
835
  ### 7b. Push prompts, skills, and tools without a repo scan (`moda registry`)
789
836
 
790
837
  Use when the user wants their prompts, skills, and tools visible in Moda but
@@ -1026,6 +1073,7 @@ Writes listed here accept `--dry-run` (prints the exact request, sends nothing)
1026
1073
  | Custom signals: edit, lifecycle | `moda signals update <id> …`, `moda signals activate\|pause\|delete <id>` (delete: drafts only; archiving is dashboard-only) |
1027
1074
  | Custom signals: review & tune | `moda signals examples <id> add\|update\|remove --conversation-id=…`, `refine <id> --request="…"`, `revise <id> --conversation-ids=…`, `proposals <id>`, `accept\|reject <id> <proposal_id>` |
1028
1075
  | Custom signals: find matches | `moda signals search <id>`, `search-status <id> <search_id>`, `search-stop <id> <search_id>`, `preview --file=detector.json` |
1076
+ | Replays: run detail, playout transcripts | `moda replays runs <set_id>`, `run <set_id> <run_id>`, `playouts <set_id> <run_id> [--case --arm --seed --transcript]`, `playout <set_id> <run_id> <playout_id>` |
1029
1077
  | Prompts registry | `moda prompts list [--label --search]`, `show <key>`, `proposals <key>`, `usage <key>`, `replay-runs <key>`, `decide <key> <proposal_id> --action=merge\|reject\|reopen` |
1030
1078
  | Skills library | `moda skills list`, `show <key>`, `policy <key> --live\|--local` |
1031
1079
  | Harness pages | `moda harness list`, `show <id>`, `versions <id>`, `diff <id> --from=… --to=…` |
@@ -1234,6 +1282,7 @@ npx: `npx -p @moda-ai/cli moda <command>`.
1234
1282
  | `moda problem-reopen <pid>` | Reopen a Problem marked fixed |
1235
1283
  | `moda signals <create\|update\|activate\|pause\|delete\|examples\|refine\|revise\|proposals\|accept\|reject\|search\|preview>` | Manage custom signals (writes take `--dry-run`; LLM-backed ones are rate-limited per key) |
1236
1284
  | `moda prompts list\|show\|proposals\|usage\|replay-runs\|decide` | Read the prompt registry; merge/reject proposals |
1285
+ | `moda replays sets\|runs\|run\|playouts\|playout` | Replay results: run history, per-seed verdicts, playout transcripts with tool provenance (read-only) |
1237
1286
  | `moda skills list\|show\|policy` / `moda harness list\|show\|versions\|diff` | Skill library and synced harnesses |
1238
1287
  | `moda fixes` | Ranked queue of Fixes ("a fix with proof"); `--status=S`, `--limit=N`, `--cursor=TOKEN` |
1239
1288
  | `moda fix <fix_id>` | One Fix: status + gate result; `--packet` prints `moda.fix_packet.v1` and writes tool/skill candidates under `.moda/fixes/<ref>/`; `--wait` drives a running pipeline to rest |