@moda-ai/cli 1.42.0 → 1.43.0

package/dist/{cli-pr6s56fc.js → cli-ecrx32mm.js}

@@ -5586,6 +5586,11 @@ var REPLAY_MODEL_ALLOWLIST = [
 var DEFAULT_POLL_INTERVAL_MS = 15000;
 var DEFAULT_WAIT_TIMEOUT_MS = 2 * 60 * 60 * 1000;
 var TERMINAL_RUN_STATUSES = new Set(["completed", "skipped", "error"]);
+var FALLBACK_SUCCESS_CRITERIA = [
+  "The agent understands the user request",
+  "The agent uses tools appropriately when needed",
+  "The agent provides a helpful final answer"
+];
 async function runPromptAb(flags, profileOptions, context) {
   validateConfig();
   const tenantId = flags["tenant-id"] || resolveApiTenantId(profileOptions);
@@ -5626,14 +5631,20 @@ async function runPromptAb(flags, profileOptions, context) {
     prod: resolvePromptArmSpec(baselineSource, "baseline"),
     proposed: resolvePromptArmSpec(candidateSource, "candidate")
   };
-  const replaySetId = await ensureReplaySet(plan, flags, tenantId, profileOptions);
+  const { setId: replaySetId, caseSources, warnings } = await ensureReplaySet(plan, flags, tenantId, profileOptions);
+  if (context.outputMode === "human") {
+    for (const warning of warnings)
+      process.stderr.write(`Warning: ${warning}
+`);
+  }
+  const writeOptions = warnings.length ? { warnings } : undefined;
   const enqueue = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/run`, {
     method: "POST",
     body: JSON.stringify({
       promptArms,
       seedsPerCase,
       assistantModel: assistantModel || undefined,
-      promotePrimary: flags["no-promote"] !== "true",
+      promotePrimary: flags.promote === "true" && flags["no-promote"] !== "true",
       ...plan.kind === "existing" ? replayCasePin(plan.caseIds) : {}
     })
   }, profileOptions, { timeoutMs: 120000, retries: 0 });
@@ -5643,11 +5654,12 @@ async function runPromptAb(flags, profileOptions, context) {
   if (noWait) {
     context.output.writeData({
       replaySetId,
+      ...caseSources ? { caseSources } : {},
       runId: enqueue.runId,
       status: enqueue.status,
       message: enqueue.message ?? "Replay comparison queued",
       plannedPlayouts
-    });
+    }, writeOptions);
     return;
   }
   const deadline = Date.now() + timeoutMs;
@@ -5670,6 +5682,7 @@ async function runPromptAb(flags, profileOptions, context) {
   }
   const result = {
     replaySetId,
+    ...caseSources ? { caseSources } : {},
     plannedPlayouts,
     runId: latest.run.runId,
     status: latest.run.status,
@@ -5680,7 +5693,7 @@ async function runPromptAb(flags, profileOptions, context) {
   if (context.outputMode === "human") {
     printHumanVerdict(result);
   }
-  context.output.writeData(result);
+  context.output.writeData(result, writeOptions);
 }
 function warnIfQueued(enqueue) {
   const { runsAhead, maxActiveRuns } = enqueue;
@@ -5793,7 +5806,7 @@ async function planReplaySet(flags, tenantId, profileOptions) {
     const snapshot = await fetchReplaySetCases(tenantId, existingSetId, profileOptions);
     return { kind: "existing", setId: existingSetId, cases: snapshot?.ids.length ?? null, caseIds: snapshot?.ids };
   }
-  const conversations = parseCsv(flags.traces ?? flags.conversations ?? flags["conversation-ids"]);
+  const conversations = [...new Set(parseCsv(flags.traces ?? flags.conversations ?? flags["conversation-ids"]))];
   if (conversations.length > MAX_REPLAY_TRACES) {
     throw new CliInputError(`--traces has ${conversations.length} ids; the limit is ${MAX_REPLAY_TRACES} per replay set, so nothing was sent.`, "Split the traces across several runs, or use --auto-generate.");
   }
@@ -5861,9 +5874,9 @@ function rerunWithYes(command, positionals, flags) {
 }
 async function ensureReplaySet(plan, flags, tenantId, profileOptions) {
   if (plan.kind === "existing")
-    return plan.setId;
+    return { setId: plan.setId, warnings: [] };
   if (plan.kind === "traces") {
-    return createSetFromConversations(plan.traces, flags, tenantId, profileOptions);
+    return createSetFromTraces(plan.traces, flags, tenantId, profileOptions);
   }
   const caseCount = plan.cases;
   const lookbackDays = parsePositiveInt(flags["lookback-days"], 30, 365, "--lookback-days");
@@ -5875,7 +5888,7 @@ async function ensureReplaySet(plan, flags, tenantId, profileOptions) {
   if (!generated?.id) {
     throw new Error("Auto-generate did not return a replay set id");
   }
-  return generated.id;
+  return { setId: generated.id, warnings: [] };
 }
 function parseTraceRef(ref) {
   const match = /^(.+)@(\d+)$/.exec(ref.trim());
@@ -5883,38 +5896,75 @@ function parseTraceRef(ref) {
     return { conversationId: ref.trim() };
   return { conversationId: match[1], startMsgIndex: Number(match[2]) };
 }
-async function createSetFromConversations(conversationIds, flags, tenantId, profileOptions) {
-  const name = (flags.name ?? flags["set-name"] ?? `Prompt A/B ${conversationIds.length} trace(s)`).trim();
-  const created = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets`, {
-    method: "POST",
-    body: JSON.stringify({ name, description: "Created by moda prompts ab" })
-  }, profileOptions);
-  let position = 0;
-  for (const ref of conversationIds) {
-    const { conversationId, startMsgIndex } = parseTraceRef(ref);
+async function createSetFromTraces(conversationIds, flags, tenantId, profileOptions) {
+  const refs = [...new Set(conversationIds)].map(parseTraceRef);
+  const traces = [...new Set(refs.map((ref) => ref.conversationId))];
+  const startByConversation = new Map;
+  for (const ref of refs) {
+    if (ref.startMsgIndex !== undefined)
+      startByConversation.set(ref.conversationId, ref.startMsgIndex);
+  }
+  const name = (flags.name ?? flags["set-name"] ?? `Prompt A/B ${traces.length} trace(s)`).trim();
+  const base = `/tenants/${encodeURIComponent(tenantId)}/replay-sets`;
+  let set = null;
+  let unavailable = null;
+  try {
+    set = await callControlAPI(`${base}/auto-generate`, { method: "POST", body: JSON.stringify({ name, conversationIds: traces }) }, profileOptions, { timeoutMs: 300000, retries: 0 });
+    if (!set?.id)
+      throw new Error("Auto-generate did not return a replay set id");
+  } catch (error) {
+    const status = error?.statusCode;
+    if (status === 401 || status === 403)
+      throw error;
+    unavailable = error instanceof Error ? error.message : String(error);
+    set = null;
+  }
+  const distilledIds = new Set((set?.cases ?? []).map((c) => String(c?.sourceConversationId ?? "")).filter(Boolean));
+  const distilled = traces.filter((id) => distilledIds.has(id));
+  const fallback = traces.filter((id) => !distilledIds.has(id));
+  if (!set) {
+    set = await callControlAPI(base, { method: "POST", body: JSON.stringify({ name, description: "Created by moda prompts ab" }) }, profileOptions);
+  }
+  const requestedIndex = new Map(traces.map((id, index) => [id, index]));
+  for (const existing of set.cases ?? []) {
+    const conversationId = String(existing?.sourceConversationId ?? "");
+    const index = requestedIndex.get(conversationId);
+    const startMsgIndex = startByConversation.get(conversationId);
+    const update = {};
+    if (fallback.length && index !== undefined && existing.position !== index)
+      update.position = index;
+    if (startMsgIndex !== undefined)
+      update.sourceStartMsgIndex = startMsgIndex;
+    if (!Object.keys(update).length)
+      continue;
+    await callControlAPI(`${base}/${encodeURIComponent(set.id)}/cases/${encodeURIComponent(existing.id)}`, { method: "PUT", body: JSON.stringify(update) }, profileOptions);
+  }
+  for (const conversationId of fallback) {
     const scenario = await loadScenarioFromConversation(conversationId);
-    await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(created.id)}/cases`, {
+    await callControlAPI(`${base}/${encodeURIComponent(set.id)}/cases`, {
       method: "POST",
       body: JSON.stringify({
         title: conversationId,
         scenario,
         sourceConversationId: conversationId,
-        ...startMsgIndex !== undefined ? { sourceStartMsgIndex: startMsgIndex } : {},
-        successCriteria: [
-          "The agent understands the user request",
-          "The agent uses tools appropriately when needed",
-          "The agent provides a helpful final answer"
-        ],
-        position: position++
+        ...startByConversation.has(conversationId) ? { sourceStartMsgIndex: startByConversation.get(conversationId) } : {},
+        successCriteria: FALLBACK_SUCCESS_CRITERIA,
+        position: requestedIndex.get(conversationId)
       })
     }, profileOptions);
   }
-  return created.id;
+  const warnings = [];
+  if (fallback.length) {
+    const why = unavailable ? `Moda could not distill these traces (${unavailable.slice(0, 200)})` : "Moda could not distill a scenario for these traces";
+    warnings.push(`${why}, so ${fallback.length} of ${traces.length} case(s) use only the opening user message and three generic ` + `success criteria, which long traces pass easily: ${fallback.slice(0, 10).join(", ")}${fallback.length > 10 ? ", …" : ""}. ` + "Edit their scenario/criteria in the dashboard, or pass --set-id with a curated set.");
+  }
+  return { setId: set.id, caseSources: { distilled, fallback }, warnings };
 }
 async function loadScenarioFromConversation(conversationId) {
   try {
-    const data = await callDataAPI(`/conversations/${encodeURIComponent(conversationId)}/context?msg_index=0&window=0`);
-    const firstUser = (data.messages ?? []).find((message) => message.role === "user" && String(message.content ?? "").trim());
+    const data = await callDataAPI(`/conversations/${encodeURIComponent(conversationId)}/context?msg_index=0&window=5`);
+    const messages = data.context?.messages ?? data.messages ?? [];
+    const firstUser = messages.find((message) => message.role === "user" && String(message.content ?? "").trim());
     const opening = String(firstUser?.content ?? "").trim();
     if (opening) {
       return `The user says: "${opening}"
@@ -6446,6 +6496,7 @@ var PROMPT_WRITE_SUBCOMMAND_FLAGS = {
     "lookback-days",
     "no-promote",
     "no-wait",
+    "promote",
     "poll-interval",
     "replay-set-id",
     "seeds-per-case",

package/dist/cli.js

@@ -8,7 +8,7 @@ import {
   runPromptsCommand,
   runSkillsCommand,
   runStatusCommand
-} from "./cli-pr6s56fc.js";
+} from "./cli-ecrx32mm.js";
 import {
   ApiError,
   HARNESS_REPORT_APPROVAL_PATH,
@@ -151,7 +151,7 @@ var WorldStateSchema = z.object({
 var ContextSchema = z.object({
   conversation_id: z.string(),
   msg_index: z.number().min(0).optional(),
-  window: z.number().min(1).max(5).default(2).optional(),
+  window: z.number().int().min(0).optional(),
   all: boolFlag(),
   from: z.number().int().min(0).optional(),
   max_messages: z.number().int().min(1).max(5000).optional()
@@ -8194,6 +8194,8 @@ function pageMessages(page) {
   const messages = asRecord(page.context)?.messages ?? page.messages;
   return (Array.isArray(messages) ? messages : []).map((m) => asRecord(m) ?? {});
 }
+var CONTEXT_MAX_WINDOW = 5;
+var CONTEXT_ALL_MAX_MESSAGES = 5000;
 async function readFullTranscript(conversationId, opts = {}) {
   const start = opts.from ?? 0;
   const requested = opts.maxMessages ?? FULL_TRANSCRIPT_DEFAULT_MAX_MESSAGES;
@@ -8579,13 +8581,26 @@ async function runCommand(command, positional, flags, positionals = positional ?
       const query2 = new URLSearchParams;
       if (params.msg_index !== undefined)
         query2.set("msg_index", params.msg_index.toString());
-      if (params.window)
-        query2.set("window", params.window.toString());
+      const window = params.window === undefined ? undefined : Math.min(params.window, CONTEXT_MAX_WINDOW);
+      if (window !== undefined)
+        query2.set("window", window.toString());
       const queryString = query2.toString() ? `?${query2.toString()}` : "";
       const data = await callDataAPI(`/conversations/${params.conversation_id}/context${queryString}`);
       const ctxRecord = asRecord(data) ?? {};
       const warnings = notFoundWarning("trace", params.conversation_id, asNumber(ctxRecord.total_messages) === 0);
-      context.output.writeData(data, warnings.length > 0 ? { warnings } : undefined);
+      const nextCommands = [];
+      if (params.window !== undefined && params.window > CONTEXT_MAX_WINDOW) {
+        const center = asNumber(asRecord(ctxRecord.context)?.center_index) ?? params.msg_index ?? 0;
+        const span = Math.min(params.window * 2 + 1, CONTEXT_ALL_MAX_MESSAGES);
+        const from = Math.max(0, center - Math.floor((span - 1) / 2));
+        const wide = `moda context ${shellArg(params.conversation_id)} --all --from=${from} --max-messages=${span}`;
+        warnings.push(`--window=${params.window} is above the maximum of ${CONTEXT_MAX_WINDOW}, so this shows ±${CONTEXT_MAX_WINDOW} messages. For a wider slice run: ${wide}`);
+        nextCommands.push({ command: wide, purpose: `Read ±${params.window} messages around the anchor.`, mutability: "read", requires_approval: false });
+      }
+      context.output.writeData(data, {
+        ...warnings.length > 0 ? { warnings } : {},
+        ...nextCommands.length > 0 ? { nextCommands } : {}
+      });
       break;
     }
     case "audit":
@@ -9360,7 +9375,7 @@ var commandRegistry = createCommandRegistry([
     resetApiRequestCountBeforeRun: true,
     telemetry: "result",
     handler: async (context) => {
-      const { runInit } = await import("./index-ah6yf5x2.js");
+      const { runInit } = await import("./index-r1egcg99.js");
       if (context.outputMode === "agent-stream") {
         context.output.writeEvent({
           event: "started",
@@ -9828,6 +9843,7 @@ var commandRegistry = createCommandRegistry([
     description: "Get windowed trace context (messages around one turn), or the whole trace with --all",
     examples: [
       "moda context <conversation_id> --window=3",
+      "moda context <conversation_id> --msg-index=40 --window=0",
       "moda context <conversation_id> --all",
       "moda context <conversation_id> --all --from=500 --max-messages=500"
     ],
@@ -10125,11 +10141,12 @@ var commandRegistry = createCommandRegistry([
         flags: [
           { name: "--baseline=<path>", description: "Prompt file to treat as the control." },
           { name: "--candidate=<path>", description: "Prompt file to treat as the variant." },
-          { name: "--traces=<ids>", description: "Comma-separated trace ids (conversation_id values) to replay, at most 100. Legacy alias: --conversations=<ids>." },
+          { name: "--traces=<ids>", description: "Comma-separated trace ids (conversation_id values) to replay, at most 100. Each case uses Moda's distilled scenario and success criteria for the trace; a trace that cannot be distilled falls back to its opening user message with generic criteria, and the run warns. Legacy alias: --conversations=<ids>." },
           { name: "--cases=<n>", description: "Cases to auto-generate (default 5, max 100)." },
           { name: "--seeds=<n>", description: "Repeats per case per arm (default 3, max 10)." },
           { name: "--model=<slug>", description: "Assistant model for both arms; must be in the replay allowlist unless --allow-any-model." },
-          { name: "--yes", description: "Confirm a run above 200 planned playouts (cases x seeds x 2 arms), or one whose case count is unknown." }
+          { name: "--yes", description: "Confirm a run above 200 planned playouts (cases x seeds x 2 arms), or one whose case count is unknown." },
+          { name: "--promote", description: "Make this run's replay set the tenant's primary replay set. Off by default, so an ad-hoc A/B never replaces it." }
         ],
         examples: [
           "moda prompts ab --baseline=prompts/agent.prompt.md --candidate=prompts/agent-v2.prompt.md --traces=conv_1,conv_2"

package/dist/{index-ah6yf5x2.js → index-r1egcg99.js}

@@ -3,7 +3,7 @@ import {
   initPrompts,
   runPromptSync,
   runSkillSync
-} from "./cli-pr6s56fc.js";
+} from "./cli-ecrx32mm.js";
 import {
   codingAgentDisplayName,
   describeCodingAgentEvent,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moda-ai/cli",
-  "version": "1.42.0",
+  "version": "1.43.0",
   "description": "CLI for Moda - AI agent analytics and observability",
   "type": "module",
   "bin": {

package/skills/integration/index.json

@@ -1,7 +1,7 @@
 {
   "schema_version": "moda.skill_index.v1",
-  "bundled_at": "2026-10-09T02:07:16.635Z",
-  "cli_version": "1.42.0",
+  "bundled_at": "2026-10-09T02:13:14.110Z",
+  "cli_version": "1.43.0",
   "skills": [
     {
       "id": "integration-cloudflare-think",

package/skills/moda-cli/SKILL.md

@@ -442,7 +442,9 @@ fail instead of being clamped.
 
 `--window=N` on `context`, `frustrations --include-window` and
 `tool-failure-detail --include-window` is a message half-width (1–5), not a
-time window.
+time window. On `context`, `--window=0` returns only the anchor message (plus
+`total_messages`), and a value above 5 is clamped to 5 with a warning that
+prints the `--all --from --max-messages` command for a wider slice.
 
 ### Schema introspection
 
@@ -631,6 +633,7 @@ Filters: `--search`, `--cluster-id`, `--user-id`, `--time-range`
 moda context <conversation_id>                # default window around middle
 moda context <conversation_id> --msg-index=5  # center on message 5
 moda context <conversation_id> --window=3     # 3 messages each side (max 5)
+moda context <conversation_id> --msg-index=40 --window=0  # just message 40 + total_messages
 moda context <conversation_id> --all          # whole trace in order, first 500 messages
 moda context <conversation_id> --all --from=500 --max-messages=500  # continue (max 5000)
 ```
@@ -767,6 +770,13 @@ earlier turns as history);
 `anthropic/claude-haiku-4-5`, `google/gemini-2.5-flash`,
 `google/gemini-2.5-pro` unless you pass `--allow-any-model`.
 
+`prompts ab --traces=<ids>` builds one case per trace from Moda's distilled
+scenario and success criteria for that trace. A trace Moda cannot distill
+falls back to its opening user message with three generic criteria (which long
+traces pass easily); the run warns and lists it under `caseSources.fallback`.
+`prompts ab` never makes its replay set the tenant's primary set unless you
+pass `--promote`.
+
 Runtime code should render synced prompts through the SDK:
 
 ```typescript