@moda-ai/cli 1.41.2 → 1.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -5586,6 +5586,11 @@ var REPLAY_MODEL_ALLOWLIST = [
|
|
|
5586
5586
|
var DEFAULT_POLL_INTERVAL_MS = 15000;
|
|
5587
5587
|
var DEFAULT_WAIT_TIMEOUT_MS = 2 * 60 * 60 * 1000;
|
|
5588
5588
|
var TERMINAL_RUN_STATUSES = new Set(["completed", "skipped", "error"]);
|
|
5589
|
+
var FALLBACK_SUCCESS_CRITERIA = [
|
|
5590
|
+
"The agent understands the user request",
|
|
5591
|
+
"The agent uses tools appropriately when needed",
|
|
5592
|
+
"The agent provides a helpful final answer"
|
|
5593
|
+
];
|
|
5589
5594
|
async function runPromptAb(flags, profileOptions, context) {
|
|
5590
5595
|
validateConfig();
|
|
5591
5596
|
const tenantId = flags["tenant-id"] || resolveApiTenantId(profileOptions);
|
|
@@ -5626,14 +5631,20 @@ async function runPromptAb(flags, profileOptions, context) {
|
|
|
5626
5631
|
prod: resolvePromptArmSpec(baselineSource, "baseline"),
|
|
5627
5632
|
proposed: resolvePromptArmSpec(candidateSource, "candidate")
|
|
5628
5633
|
};
|
|
5629
|
-
const replaySetId = await ensureReplaySet(plan, flags, tenantId, profileOptions);
|
|
5634
|
+
const { setId: replaySetId, caseSources, warnings } = await ensureReplaySet(plan, flags, tenantId, profileOptions);
|
|
5635
|
+
if (context.outputMode === "human") {
|
|
5636
|
+
for (const warning of warnings)
|
|
5637
|
+
process.stderr.write(`Warning: ${warning}
|
|
5638
|
+
`);
|
|
5639
|
+
}
|
|
5640
|
+
const writeOptions = warnings.length ? { warnings } : undefined;
|
|
5630
5641
|
const enqueue = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/run`, {
|
|
5631
5642
|
method: "POST",
|
|
5632
5643
|
body: JSON.stringify({
|
|
5633
5644
|
promptArms,
|
|
5634
5645
|
seedsPerCase,
|
|
5635
5646
|
assistantModel: assistantModel || undefined,
|
|
5636
|
-
promotePrimary: flags["no-promote"] !== "true",
|
|
5647
|
+
promotePrimary: flags.promote === "true" && flags["no-promote"] !== "true",
|
|
5637
5648
|
...plan.kind === "existing" ? replayCasePin(plan.caseIds) : {}
|
|
5638
5649
|
})
|
|
5639
5650
|
}, profileOptions, { timeoutMs: 120000, retries: 0 });
|
|
@@ -5643,11 +5654,12 @@ async function runPromptAb(flags, profileOptions, context) {
|
|
|
5643
5654
|
if (noWait) {
|
|
5644
5655
|
context.output.writeData({
|
|
5645
5656
|
replaySetId,
|
|
5657
|
+
...caseSources ? { caseSources } : {},
|
|
5646
5658
|
runId: enqueue.runId,
|
|
5647
5659
|
status: enqueue.status,
|
|
5648
5660
|
message: enqueue.message ?? "Replay comparison queued",
|
|
5649
5661
|
plannedPlayouts
|
|
5650
|
-
});
|
|
5662
|
+
}, writeOptions);
|
|
5651
5663
|
return;
|
|
5652
5664
|
}
|
|
5653
5665
|
const deadline = Date.now() + timeoutMs;
|
|
@@ -5670,6 +5682,7 @@ async function runPromptAb(flags, profileOptions, context) {
|
|
|
5670
5682
|
}
|
|
5671
5683
|
const result = {
|
|
5672
5684
|
replaySetId,
|
|
5685
|
+
...caseSources ? { caseSources } : {},
|
|
5673
5686
|
plannedPlayouts,
|
|
5674
5687
|
runId: latest.run.runId,
|
|
5675
5688
|
status: latest.run.status,
|
|
@@ -5680,7 +5693,7 @@ async function runPromptAb(flags, profileOptions, context) {
|
|
|
5680
5693
|
if (context.outputMode === "human") {
|
|
5681
5694
|
printHumanVerdict(result);
|
|
5682
5695
|
}
|
|
5683
|
-
context.output.writeData(result);
|
|
5696
|
+
context.output.writeData(result, writeOptions);
|
|
5684
5697
|
}
|
|
5685
5698
|
function warnIfQueued(enqueue) {
|
|
5686
5699
|
const { runsAhead, maxActiveRuns } = enqueue;
|
|
@@ -5793,7 +5806,7 @@ async function planReplaySet(flags, tenantId, profileOptions) {
|
|
|
5793
5806
|
const snapshot = await fetchReplaySetCases(tenantId, existingSetId, profileOptions);
|
|
5794
5807
|
return { kind: "existing", setId: existingSetId, cases: snapshot?.ids.length ?? null, caseIds: snapshot?.ids };
|
|
5795
5808
|
}
|
|
5796
|
-
const conversations = parseCsv(flags.traces ?? flags.conversations ?? flags["conversation-ids"]);
|
|
5809
|
+
const conversations = [...new Set(parseCsv(flags.traces ?? flags.conversations ?? flags["conversation-ids"]))];
|
|
5797
5810
|
if (conversations.length > MAX_REPLAY_TRACES) {
|
|
5798
5811
|
throw new CliInputError(`--traces has ${conversations.length} ids; the limit is ${MAX_REPLAY_TRACES} per replay set, so nothing was sent.`, "Split the traces across several runs, or use --auto-generate.");
|
|
5799
5812
|
}
|
|
@@ -5861,9 +5874,9 @@ function rerunWithYes(command, positionals, flags) {
|
|
|
5861
5874
|
}
|
|
5862
5875
|
async function ensureReplaySet(plan, flags, tenantId, profileOptions) {
|
|
5863
5876
|
if (plan.kind === "existing")
|
|
5864
|
-
return plan.setId;
|
|
5877
|
+
return { setId: plan.setId, warnings: [] };
|
|
5865
5878
|
if (plan.kind === "traces") {
|
|
5866
|
-
return
|
|
5879
|
+
return createSetFromTraces(plan.traces, flags, tenantId, profileOptions);
|
|
5867
5880
|
}
|
|
5868
5881
|
const caseCount = plan.cases;
|
|
5869
5882
|
const lookbackDays = parsePositiveInt(flags["lookback-days"], 30, 365, "--lookback-days");
|
|
@@ -5875,7 +5888,7 @@ async function ensureReplaySet(plan, flags, tenantId, profileOptions) {
|
|
|
5875
5888
|
if (!generated?.id) {
|
|
5876
5889
|
throw new Error("Auto-generate did not return a replay set id");
|
|
5877
5890
|
}
|
|
5878
|
-
return generated.id;
|
|
5891
|
+
return { setId: generated.id, warnings: [] };
|
|
5879
5892
|
}
|
|
5880
5893
|
function parseTraceRef(ref) {
|
|
5881
5894
|
const match = /^(.+)@(\d+)$/.exec(ref.trim());
|
|
@@ -5883,38 +5896,75 @@ function parseTraceRef(ref) {
|
|
|
5883
5896
|
return { conversationId: ref.trim() };
|
|
5884
5897
|
return { conversationId: match[1], startMsgIndex: Number(match[2]) };
|
|
5885
5898
|
}
|
|
5886
|
-
async function
|
|
5887
|
-
const
|
|
5888
|
-
const
|
|
5889
|
-
|
|
5890
|
-
|
|
5891
|
-
|
|
5892
|
-
|
|
5893
|
-
|
|
5894
|
-
|
|
5899
|
+
async function createSetFromTraces(conversationIds, flags, tenantId, profileOptions) {
|
|
5900
|
+
const refs = [...new Set(conversationIds)].map(parseTraceRef);
|
|
5901
|
+
const traces = [...new Set(refs.map((ref) => ref.conversationId))];
|
|
5902
|
+
const startByConversation = new Map;
|
|
5903
|
+
for (const ref of refs) {
|
|
5904
|
+
if (ref.startMsgIndex !== undefined)
|
|
5905
|
+
startByConversation.set(ref.conversationId, ref.startMsgIndex);
|
|
5906
|
+
}
|
|
5907
|
+
const name = (flags.name ?? flags["set-name"] ?? `Prompt A/B ${traces.length} trace(s)`).trim();
|
|
5908
|
+
const base = `/tenants/${encodeURIComponent(tenantId)}/replay-sets`;
|
|
5909
|
+
let set = null;
|
|
5910
|
+
let unavailable = null;
|
|
5911
|
+
try {
|
|
5912
|
+
set = await callControlAPI(`${base}/auto-generate`, { method: "POST", body: JSON.stringify({ name, conversationIds: traces }) }, profileOptions, { timeoutMs: 300000, retries: 0 });
|
|
5913
|
+
if (!set?.id)
|
|
5914
|
+
throw new Error("Auto-generate did not return a replay set id");
|
|
5915
|
+
} catch (error) {
|
|
5916
|
+
const status = error?.statusCode;
|
|
5917
|
+
if (status === 401 || status === 403)
|
|
5918
|
+
throw error;
|
|
5919
|
+
unavailable = error instanceof Error ? error.message : String(error);
|
|
5920
|
+
set = null;
|
|
5921
|
+
}
|
|
5922
|
+
const distilledIds = new Set((set?.cases ?? []).map((c) => String(c?.sourceConversationId ?? "")).filter(Boolean));
|
|
5923
|
+
const distilled = traces.filter((id) => distilledIds.has(id));
|
|
5924
|
+
const fallback = traces.filter((id) => !distilledIds.has(id));
|
|
5925
|
+
if (!set) {
|
|
5926
|
+
set = await callControlAPI(base, { method: "POST", body: JSON.stringify({ name, description: "Created by moda prompts ab" }) }, profileOptions);
|
|
5927
|
+
}
|
|
5928
|
+
const requestedIndex = new Map(traces.map((id, index) => [id, index]));
|
|
5929
|
+
for (const existing of set.cases ?? []) {
|
|
5930
|
+
const conversationId = String(existing?.sourceConversationId ?? "");
|
|
5931
|
+
const index = requestedIndex.get(conversationId);
|
|
5932
|
+
const startMsgIndex = startByConversation.get(conversationId);
|
|
5933
|
+
const update = {};
|
|
5934
|
+
if (fallback.length && index !== undefined && existing.position !== index)
|
|
5935
|
+
update.position = index;
|
|
5936
|
+
if (startMsgIndex !== undefined)
|
|
5937
|
+
update.sourceStartMsgIndex = startMsgIndex;
|
|
5938
|
+
if (!Object.keys(update).length)
|
|
5939
|
+
continue;
|
|
5940
|
+
await callControlAPI(`${base}/${encodeURIComponent(set.id)}/cases/${encodeURIComponent(existing.id)}`, { method: "PUT", body: JSON.stringify(update) }, profileOptions);
|
|
5941
|
+
}
|
|
5942
|
+
for (const conversationId of fallback) {
|
|
5895
5943
|
const scenario = await loadScenarioFromConversation(conversationId);
|
|
5896
|
-
await callControlAPI(
|
|
5944
|
+
await callControlAPI(`${base}/${encodeURIComponent(set.id)}/cases`, {
|
|
5897
5945
|
method: "POST",
|
|
5898
5946
|
body: JSON.stringify({
|
|
5899
5947
|
title: conversationId,
|
|
5900
5948
|
scenario,
|
|
5901
5949
|
sourceConversationId: conversationId,
|
|
5902
|
-
...
|
|
5903
|
-
successCriteria:
|
|
5904
|
-
|
|
5905
|
-
"The agent uses tools appropriately when needed",
|
|
5906
|
-
"The agent provides a helpful final answer"
|
|
5907
|
-
],
|
|
5908
|
-
position: position++
|
|
5950
|
+
...startByConversation.has(conversationId) ? { sourceStartMsgIndex: startByConversation.get(conversationId) } : {},
|
|
5951
|
+
successCriteria: FALLBACK_SUCCESS_CRITERIA,
|
|
5952
|
+
position: requestedIndex.get(conversationId)
|
|
5909
5953
|
})
|
|
5910
5954
|
}, profileOptions);
|
|
5911
5955
|
}
|
|
5912
|
-
|
|
5956
|
+
const warnings = [];
|
|
5957
|
+
if (fallback.length) {
|
|
5958
|
+
const why = unavailable ? `Moda could not distill these traces (${unavailable.slice(0, 200)})` : "Moda could not distill a scenario for these traces";
|
|
5959
|
+
warnings.push(`${why}, so ${fallback.length} of ${traces.length} case(s) use only the opening user message and three generic ` + `success criteria, which long traces pass easily: ${fallback.slice(0, 10).join(", ")}${fallback.length > 10 ? ", …" : ""}. ` + "Edit their scenario/criteria in the dashboard, or pass --set-id with a curated set.");
|
|
5960
|
+
}
|
|
5961
|
+
return { setId: set.id, caseSources: { distilled, fallback }, warnings };
|
|
5913
5962
|
}
|
|
5914
5963
|
async function loadScenarioFromConversation(conversationId) {
|
|
5915
5964
|
try {
|
|
5916
|
-
const data = await callDataAPI(`/conversations/${encodeURIComponent(conversationId)}/context?msg_index=0&window=
|
|
5917
|
-
const
|
|
5965
|
+
const data = await callDataAPI(`/conversations/${encodeURIComponent(conversationId)}/context?msg_index=0&window=5`);
|
|
5966
|
+
const messages = data.context?.messages ?? data.messages ?? [];
|
|
5967
|
+
const firstUser = messages.find((message) => message.role === "user" && String(message.content ?? "").trim());
|
|
5918
5968
|
const opening = String(firstUser?.content ?? "").trim();
|
|
5919
5969
|
if (opening) {
|
|
5920
5970
|
return `The user says: "${opening}"
|
|
@@ -6446,6 +6496,7 @@ var PROMPT_WRITE_SUBCOMMAND_FLAGS = {
|
|
|
6446
6496
|
"lookback-days",
|
|
6447
6497
|
"no-promote",
|
|
6448
6498
|
"no-wait",
|
|
6499
|
+
"promote",
|
|
6449
6500
|
"poll-interval",
|
|
6450
6501
|
"replay-set-id",
|
|
6451
6502
|
"seeds-per-case",
|
package/dist/cli.js
CHANGED
|
@@ -8,13 +8,14 @@ import {
|
|
|
8
8
|
runPromptsCommand,
|
|
9
9
|
runSkillsCommand,
|
|
10
10
|
runStatusCommand
|
|
11
|
-
} from "./cli-
|
|
11
|
+
} from "./cli-ecrx32mm.js";
|
|
12
12
|
import {
|
|
13
13
|
ApiError,
|
|
14
14
|
HARNESS_REPORT_APPROVAL_PATH,
|
|
15
15
|
HARNESS_REPORT_PATH,
|
|
16
16
|
SIGNALS_MANAGEMENT_SUBCOMMANDS,
|
|
17
17
|
assertFlags,
|
|
18
|
+
assertPositionals,
|
|
18
19
|
callControlAPI,
|
|
19
20
|
callDataAPI,
|
|
20
21
|
getApiRequestCount,
|
|
@@ -150,7 +151,7 @@ var WorldStateSchema = z.object({
|
|
|
150
151
|
var ContextSchema = z.object({
|
|
151
152
|
conversation_id: z.string(),
|
|
152
153
|
msg_index: z.number().min(0).optional(),
|
|
153
|
-
window: z.number().
|
|
154
|
+
window: z.number().int().min(0).optional(),
|
|
154
155
|
all: boolFlag(),
|
|
155
156
|
from: z.number().int().min(0).optional(),
|
|
156
157
|
max_messages: z.number().int().min(1).max(5000).optional()
|
|
@@ -2500,6 +2501,183 @@ function isDefined(value) {
|
|
|
2500
2501
|
return value !== undefined;
|
|
2501
2502
|
}
|
|
2502
2503
|
|
|
2504
|
+
// src/replays.ts
|
|
2505
|
+
var UUID_RE2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
2506
|
+
var PLAYOUT_ID_RE = /^[0-9a-f]{64}$/i;
|
|
2507
|
+
var ARMS = ["prod", "proposed"];
|
|
2508
|
+
var REPLAYS_SUBCOMMANDS = ["sets", "runs", "run", "playouts", "playout"];
|
|
2509
|
+
var REPLAYS_SUBCOMMAND_HELP = [
|
|
2510
|
+
{
|
|
2511
|
+
name: "sets",
|
|
2512
|
+
description: "List replay sets (ids, names, case counts)",
|
|
2513
|
+
usage: "moda replays sets",
|
|
2514
|
+
examples: ["moda replays sets --json"]
|
|
2515
|
+
},
|
|
2516
|
+
{
|
|
2517
|
+
name: "runs",
|
|
2518
|
+
description: "Run history for a set, newest first, every status",
|
|
2519
|
+
usage: "moda replays runs <set_id> [--limit=N]",
|
|
2520
|
+
flags: [{ name: "--limit=N", description: "Runs to return (default 20, max 100)." }],
|
|
2521
|
+
examples: ["moda replays runs <set_id> --limit=5 --json"]
|
|
2522
|
+
},
|
|
2523
|
+
{
|
|
2524
|
+
name: "run",
|
|
2525
|
+
description: "One run: summary, per-case aggregate, every case x arm x seed verdict, playout count",
|
|
2526
|
+
usage: "moda replays run <set_id> <run_id|latest>",
|
|
2527
|
+
examples: ["moda replays run <set_id> <run_id> --json", "moda replays run <set_id> latest --json"]
|
|
2528
|
+
},
|
|
2529
|
+
{
|
|
2530
|
+
name: "playouts",
|
|
2531
|
+
description: "Playouts of a run (one per case x arm x seed) with verdicts; --transcript inlines what was said and every tool call",
|
|
2532
|
+
usage: "moda replays playouts <set_id> <run_id> [--case=<case_id>] [--arm=prod|proposed] [--seed=N] [--transcript] [--max-chars=N] [--limit=N] [--cursor=<next_cursor>]",
|
|
2533
|
+
flags: [
|
|
2534
|
+
{ name: "--case=<case_id>", description: "Only this case." },
|
|
2535
|
+
{ name: "--arm=prod|proposed", description: "Only this arm (prod = baseline, proposed = candidate)." },
|
|
2536
|
+
{ name: "--seed=N", description: "Only this seed." },
|
|
2537
|
+
{ name: "--transcript", description: "Inline each transcript (messages, tool calls with served output and provenance, criteria). Page cap 10." },
|
|
2538
|
+
{ name: "--max-chars=N", description: "Per-field cap for message text and tool JSON (default 4000, 100-32000)." },
|
|
2539
|
+
{ name: "--limit=N", description: "Page size (default 50, max 200; 10 with --transcript)." },
|
|
2540
|
+
{ name: "--cursor=<next_cursor>", description: "Next page: the previous response's nextCursor." }
|
|
2541
|
+
],
|
|
2542
|
+
examples: [
|
|
2543
|
+
"moda replays playouts <set_id> <run_id> --json",
|
|
2544
|
+
"moda replays playouts <set_id> <run_id> --case=<case_id> --arm=proposed --seed=1 --transcript --json"
|
|
2545
|
+
]
|
|
2546
|
+
},
|
|
2547
|
+
{
|
|
2548
|
+
name: "playout",
|
|
2549
|
+
description: "One playout with its full transcript",
|
|
2550
|
+
usage: "moda replays playout <set_id> <run_id> <playout_id> [--max-chars=N]",
|
|
2551
|
+
flags: [{ name: "--max-chars=N", description: "Per-field cap (default 4000, 100-32000)." }],
|
|
2552
|
+
examples: ["moda replays playout <set_id> <run_id> <playout_id> --json"]
|
|
2553
|
+
}
|
|
2554
|
+
];
|
|
2555
|
+
function profileOptions(context) {
|
|
2556
|
+
return { profile: context.profile, env: context.env };
|
|
2557
|
+
}
|
|
2558
|
+
function requireTenant(context) {
|
|
2559
|
+
const key = resolveApiKey(profileOptions(context));
|
|
2560
|
+
if (!key) {
|
|
2561
|
+
throw new CliInputError("No Moda API key found for moda replays.", "Set MODA_API_KEY, run `moda auth login --api-key`, or select a profile with an API key.");
|
|
2562
|
+
}
|
|
2563
|
+
const keyTenant = embeddedTenantIdFromApiKey(key);
|
|
2564
|
+
const flagTenant = context.flags["tenant-id"];
|
|
2565
|
+
if (flagTenant && keyTenant && flagTenant !== keyTenant) {
|
|
2566
|
+
throw new CliInputError(`--tenant-id=${flagTenant} does not match this API key's tenant (${keyTenant}), so nothing was sent.`, "Drop --tenant-id (the key names its tenant) or use a key for that tenant.");
|
|
2567
|
+
}
|
|
2568
|
+
const tenantId = flagTenant || resolveApiTenantId(profileOptions(context));
|
|
2569
|
+
if (!tenantId) {
|
|
2570
|
+
throw new CliInputError("Could not determine the tenant.", "Use a signed API key (it names its tenant), set MODA_TENANT_ID, or pass --tenant-id=<id>.");
|
|
2571
|
+
}
|
|
2572
|
+
return tenantId;
|
|
2573
|
+
}
|
|
2574
|
+
function intFlag(name, value, min, max) {
|
|
2575
|
+
if (value === undefined)
|
|
2576
|
+
return;
|
|
2577
|
+
const parsed = Number(value);
|
|
2578
|
+
if (!Number.isInteger(parsed) || parsed < min || parsed > max) {
|
|
2579
|
+
throw new CliInputError(`--${name} must be an integer from ${min} to ${max} (got '${value}').`);
|
|
2580
|
+
}
|
|
2581
|
+
return String(parsed);
|
|
2582
|
+
}
|
|
2583
|
+
function query(params) {
|
|
2584
|
+
const search = new URLSearchParams(Object.entries(params).filter((entry) => entry[1] !== undefined)).toString();
|
|
2585
|
+
return search ? `?${search}` : "";
|
|
2586
|
+
}
|
|
2587
|
+
function needArg(value, name, usage) {
|
|
2588
|
+
if (!value)
|
|
2589
|
+
throw new CliInputError(`<${name}> is required.`, `Usage: ${usage}`);
|
|
2590
|
+
return value;
|
|
2591
|
+
}
|
|
2592
|
+
function needRunId(value, usage, allowLatest = false) {
|
|
2593
|
+
const runId = needArg(value, "run_id", usage);
|
|
2594
|
+
if (allowLatest && runId === "latest")
|
|
2595
|
+
return runId;
|
|
2596
|
+
if (!UUID_RE2.test(runId)) {
|
|
2597
|
+
throw new CliInputError(`<run_id> must be a run UUID (got '${runId}').`, "List runs with `moda replays runs <set_id>`.");
|
|
2598
|
+
}
|
|
2599
|
+
return runId;
|
|
2600
|
+
}
|
|
2601
|
+
async function runReplaysCommand(context) {
|
|
2602
|
+
const [subcommand, ...args] = context.positionals;
|
|
2603
|
+
const flags = context.flags;
|
|
2604
|
+
const help = REPLAYS_SUBCOMMAND_HELP.find((entry) => entry.name === subcommand);
|
|
2605
|
+
if (!subcommand || !help) {
|
|
2606
|
+
throw new CliInputError(subcommand ? `Unknown replays command '${subcommand}'.` : "A replays subcommand is required.", `Use one of: ${REPLAYS_SUBCOMMANDS.join(", ")}. See \`moda replays --help\`.`);
|
|
2607
|
+
}
|
|
2608
|
+
const usage = help.usage;
|
|
2609
|
+
const allowed = {
|
|
2610
|
+
sets: [],
|
|
2611
|
+
runs: ["limit"],
|
|
2612
|
+
run: [],
|
|
2613
|
+
playouts: ["case", "arm", "seed", "transcript", "max-chars", "limit", "cursor"],
|
|
2614
|
+
playout: ["max-chars"]
|
|
2615
|
+
};
|
|
2616
|
+
const name = subcommand;
|
|
2617
|
+
assertFlags(`replays ${name}`, flags, [...allowed[name], "tenant-id"]);
|
|
2618
|
+
const maxArgs = { sets: 0, runs: 1, run: 2, playouts: 2, playout: 3 }[name];
|
|
2619
|
+
assertPositionals(`replays ${name}`, args, maxArgs, usage);
|
|
2620
|
+
const tenantId = requireTenant(context);
|
|
2621
|
+
const base = `/tenants/${encodeURIComponent(tenantId)}/replay-sets`;
|
|
2622
|
+
let endpoint;
|
|
2623
|
+
switch (name) {
|
|
2624
|
+
case "sets":
|
|
2625
|
+
endpoint = base;
|
|
2626
|
+
break;
|
|
2627
|
+
case "runs": {
|
|
2628
|
+
const setId = needArg(args[0], "set_id", usage);
|
|
2629
|
+
endpoint = `${base}/${encodeURIComponent(setId)}/runs${query({ limit: intFlag("limit", flags.limit, 1, 100) })}`;
|
|
2630
|
+
break;
|
|
2631
|
+
}
|
|
2632
|
+
case "run": {
|
|
2633
|
+
const setId = needArg(args[0], "set_id", usage);
|
|
2634
|
+
const runId = needRunId(args[1], usage, true);
|
|
2635
|
+
endpoint = runId === "latest" ? `${base}/${encodeURIComponent(setId)}/runs/latest` : `${base}/${encodeURIComponent(setId)}/runs/${encodeURIComponent(runId)}`;
|
|
2636
|
+
break;
|
|
2637
|
+
}
|
|
2638
|
+
case "playouts": {
|
|
2639
|
+
const setId = needArg(args[0], "set_id", usage);
|
|
2640
|
+
const runId = needRunId(args[1], usage);
|
|
2641
|
+
const arm = flags.arm;
|
|
2642
|
+
if (arm !== undefined && !ARMS.includes(arm)) {
|
|
2643
|
+
throw new CliInputError(`--arm must be one of ${ARMS.join(", ")} (got '${arm}').`);
|
|
2644
|
+
}
|
|
2645
|
+
if (flags.case !== undefined && (flags.case.length === 0 || flags.case.length > 128)) {
|
|
2646
|
+
throw new CliInputError("--case must be a case id of 1 to 128 characters.");
|
|
2647
|
+
}
|
|
2648
|
+
if (flags.cursor !== undefined && !/^\d{1,9}$/.test(flags.cursor)) {
|
|
2649
|
+
throw new CliInputError(`--cursor must be the nextCursor of a previous page (got '${flags.cursor}').`);
|
|
2650
|
+
}
|
|
2651
|
+
const transcript = flags.transcript === "true";
|
|
2652
|
+
endpoint = `${base}/${encodeURIComponent(setId)}/runs/${encodeURIComponent(runId)}/playouts${query({
|
|
2653
|
+
caseId: flags.case,
|
|
2654
|
+
arm,
|
|
2655
|
+
seed: intFlag("seed", flags.seed, 0, 1000),
|
|
2656
|
+
include: transcript ? "transcript" : undefined,
|
|
2657
|
+
maxChars: intFlag("max-chars", flags["max-chars"], 100, 32000),
|
|
2658
|
+
limit: intFlag("limit", flags.limit, 1, transcript ? 10 : 200),
|
|
2659
|
+
cursor: flags.cursor
|
|
2660
|
+
})}`;
|
|
2661
|
+
break;
|
|
2662
|
+
}
|
|
2663
|
+
case "playout": {
|
|
2664
|
+
const setId = needArg(args[0], "set_id", usage);
|
|
2665
|
+
const runId = needRunId(args[1], usage);
|
|
2666
|
+
const playoutId = needArg(args[2], "playout_id", usage);
|
|
2667
|
+
if (!PLAYOUT_ID_RE.test(playoutId)) {
|
|
2668
|
+
throw new CliInputError(`<playout_id> must be a 64-character hex id (got '${playoutId}').`, "List playouts with `moda replays playouts <set_id> <run_id>`.");
|
|
2669
|
+
}
|
|
2670
|
+
endpoint = `${base}/${encodeURIComponent(setId)}/runs/${encodeURIComponent(runId)}/playouts/${encodeURIComponent(playoutId)}${query({
|
|
2671
|
+
maxChars: intFlag("max-chars", flags["max-chars"], 100, 32000)
|
|
2672
|
+
})}`;
|
|
2673
|
+
break;
|
|
2674
|
+
}
|
|
2675
|
+
}
|
|
2676
|
+
const data = await callControlAPI(endpoint, { method: "GET" }, profileOptions(context), { timeoutMs: 60000 });
|
|
2677
|
+
context.output.writeData(data ?? {});
|
|
2678
|
+
return { exitCode: 0 };
|
|
2679
|
+
}
|
|
2680
|
+
|
|
2503
2681
|
// src/telemetry.ts
|
|
2504
2682
|
import { randomUUID } from "node:crypto";
|
|
2505
2683
|
var TELEMETRY_TIMEOUT_MS = 500;
|
|
@@ -2514,8 +2692,8 @@ var SEARCH_SURFACE_BY_COMMAND = {
|
|
|
2514
2692
|
context: "context"
|
|
2515
2693
|
};
|
|
2516
2694
|
var SEARCH_MODES2 = ["keyword", "semantic", "hybrid"];
|
|
2517
|
-
function termCount(
|
|
2518
|
-
return
|
|
2695
|
+
function termCount(query2) {
|
|
2696
|
+
return query2.trim().split(/\s+/).filter(Boolean).length;
|
|
2519
2697
|
}
|
|
2520
2698
|
function clampInt(value, min, max) {
|
|
2521
2699
|
if (typeof value !== "number" || !Number.isFinite(value))
|
|
@@ -5180,8 +5358,8 @@ function parsePayload(payload) {
|
|
|
5180
5358
|
}
|
|
5181
5359
|
var DEFAULT_STREAM_TIMEOUT_MS = 250000;
|
|
5182
5360
|
async function streamAsk(options) {
|
|
5183
|
-
const
|
|
5184
|
-
const url = `${resolveModaBaseUrl(
|
|
5361
|
+
const profileOptions2 = options.profile ? { profile: options.profile } : {};
|
|
5362
|
+
const url = `${resolveModaBaseUrl(profileOptions2)}/api/cli/ask/stream`;
|
|
5185
5363
|
const controller = new AbortController;
|
|
5186
5364
|
const timeout = setTimeout(() => controller.abort(), options.timeoutMs ?? DEFAULT_STREAM_TIMEOUT_MS);
|
|
5187
5365
|
const startedAt = Date.now();
|
|
@@ -5197,7 +5375,7 @@ async function streamAsk(options) {
|
|
|
5197
5375
|
response = await fetchImpl(url, {
|
|
5198
5376
|
method: "POST",
|
|
5199
5377
|
headers: {
|
|
5200
|
-
"x-api-key": resolveApiKey(
|
|
5378
|
+
"x-api-key": resolveApiKey(profileOptions2) ?? "",
|
|
5201
5379
|
"Content-Type": "application/json",
|
|
5202
5380
|
Accept: "text/event-stream"
|
|
5203
5381
|
},
|
|
@@ -7005,6 +7183,15 @@ Commands:
|
|
|
7005
7183
|
prompts ab Run a judged prompt A/B replay experiment
|
|
7006
7184
|
prompts propose Revise a prompt from a run's failures (--gate to auto-A/B, --promote-on-win)
|
|
7007
7185
|
|
|
7186
|
+
Replays (read-only results):
|
|
7187
|
+
replays sets List replay sets
|
|
7188
|
+
replays runs <set_id> Run history for a set (every status)
|
|
7189
|
+
replays run <set_id> <run_id> One run: per-case and per-seed verdicts, playout count
|
|
7190
|
+
replays playouts <set_id> <run_id>
|
|
7191
|
+
Playouts per case/arm/seed (--transcript inlines messages and tool calls)
|
|
7192
|
+
replays playout <set_id> <run_id> <playout_id>
|
|
7193
|
+
One playout transcript
|
|
7194
|
+
|
|
7008
7195
|
Skills:
|
|
7009
7196
|
skills gen Kick off tenant-wide skill generation
|
|
7010
7197
|
skills status [run-id] Show a generation run's status, outcome, and recent events (default: latest run)
|
|
@@ -7383,10 +7570,10 @@ async function attachWindows(rows, windowSize) {
|
|
|
7383
7570
|
return { ...row, window: conversation };
|
|
7384
7571
|
}
|
|
7385
7572
|
try {
|
|
7386
|
-
const
|
|
7387
|
-
|
|
7388
|
-
|
|
7389
|
-
const data = await callDataAPI(`/conversations/${conv_id}/context?${
|
|
7573
|
+
const query2 = new URLSearchParams;
|
|
7574
|
+
query2.set("msg_index", msg_index.toString());
|
|
7575
|
+
query2.set("window", windowSize.toString());
|
|
7576
|
+
const data = await callDataAPI(`/conversations/${conv_id}/context?${query2.toString()}`);
|
|
7390
7577
|
return { ...row, window: data };
|
|
7391
7578
|
} catch (err) {
|
|
7392
7579
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -7787,15 +7974,15 @@ function shellArg(value) {
|
|
|
7787
7974
|
async function searchFallbackToTraces(params) {
|
|
7788
7975
|
const limit = Math.min(params.limit ?? 20, 100);
|
|
7789
7976
|
const offset = params.offset ?? 0;
|
|
7790
|
-
const
|
|
7791
|
-
|
|
7977
|
+
const query2 = new URLSearchParams;
|
|
7978
|
+
query2.set("search", params.q);
|
|
7792
7979
|
if (params.user_id)
|
|
7793
|
-
|
|
7980
|
+
query2.set("user_id", params.user_id);
|
|
7794
7981
|
if (params.time_range)
|
|
7795
|
-
|
|
7796
|
-
|
|
7982
|
+
query2.set("time_range", params.time_range);
|
|
7983
|
+
query2.set("limit", String(limit));
|
|
7797
7984
|
if (offset > 0)
|
|
7798
|
-
|
|
7985
|
+
query2.set("offset", String(offset));
|
|
7799
7986
|
const tracesCommand = [
|
|
7800
7987
|
`moda traces --search=${shellArg(params.q)}`,
|
|
7801
7988
|
params.user_id ? `--user-id=${shellArg(params.user_id)}` : "",
|
|
@@ -7804,7 +7991,7 @@ async function searchFallbackToTraces(params) {
|
|
|
7804
7991
|
].filter(Boolean).join(" ");
|
|
7805
7992
|
let traces;
|
|
7806
7993
|
try {
|
|
7807
|
-
traces = asRecord(await callDataAPI(`/conversations?${
|
|
7994
|
+
traces = asRecord(await callDataAPI(`/conversations?${query2.toString()}`)) ?? {};
|
|
7808
7995
|
} catch (error) {
|
|
7809
7996
|
const reason = error instanceof Error ? error.message : String(error);
|
|
7810
7997
|
return {
|
|
@@ -7953,16 +8140,16 @@ function assertHandParsedFlags(command, flags, positionals) {
|
|
|
7953
8140
|
var PROFILE_MANAGING_COMMANDS = new Set(["profiles", "auth", "init", "provision", "status", "config", "keys", "manifest", "agent", "doctor"]);
|
|
7954
8141
|
async function runProblemsRemainder(flags, context) {
|
|
7955
8142
|
const params = parseCommandFlags(ProblemsRemainderSchema, flagsToArgs(flags), "problems remainder");
|
|
7956
|
-
const
|
|
8143
|
+
const query2 = new URLSearchParams;
|
|
7957
8144
|
if (params.days_back)
|
|
7958
|
-
|
|
8145
|
+
query2.set("days_back", params.days_back.toString());
|
|
7959
8146
|
if (params.limit)
|
|
7960
|
-
|
|
8147
|
+
query2.set("limit", params.limit.toString());
|
|
7961
8148
|
if (params.family)
|
|
7962
|
-
|
|
8149
|
+
query2.set("family", params.family);
|
|
7963
8150
|
if (params.cursor)
|
|
7964
|
-
|
|
7965
|
-
const queryString =
|
|
8151
|
+
query2.set("cursor", params.cursor);
|
|
8152
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
7966
8153
|
context.output.writeData(await callDataAPI(`/problems/remainder${queryString}`));
|
|
7967
8154
|
}
|
|
7968
8155
|
function processingDisabledWarning(data, family) {
|
|
@@ -8007,6 +8194,8 @@ function pageMessages(page) {
|
|
|
8007
8194
|
const messages = asRecord(page.context)?.messages ?? page.messages;
|
|
8008
8195
|
return (Array.isArray(messages) ? messages : []).map((m) => asRecord(m) ?? {});
|
|
8009
8196
|
}
|
|
8197
|
+
var CONTEXT_MAX_WINDOW = 5;
|
|
8198
|
+
var CONTEXT_ALL_MAX_MESSAGES = 5000;
|
|
8010
8199
|
async function readFullTranscript(conversationId, opts = {}) {
|
|
8011
8200
|
const start = opts.from ?? 0;
|
|
8012
8201
|
const requested = opts.maxMessages ?? FULL_TRANSCRIPT_DEFAULT_MAX_MESSAGES;
|
|
@@ -8158,28 +8347,28 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8158
8347
|
const args = flagsToArgs(flags);
|
|
8159
8348
|
const params = parseCommandFlags(ClustersSchema, args, command);
|
|
8160
8349
|
if (params.search !== undefined || params.node_id !== undefined) {
|
|
8161
|
-
const
|
|
8350
|
+
const query3 = new URLSearchParams;
|
|
8162
8351
|
if (params.search !== undefined)
|
|
8163
|
-
|
|
8352
|
+
query3.set("q", params.search);
|
|
8164
8353
|
if (params.node_id !== undefined)
|
|
8165
|
-
|
|
8354
|
+
query3.set("node_id", params.node_id);
|
|
8166
8355
|
if (params.mode)
|
|
8167
|
-
|
|
8356
|
+
query3.set("mode", params.mode);
|
|
8168
8357
|
if (params.limit)
|
|
8169
|
-
|
|
8170
|
-
const data2 = await callDataAPI(`/task-clusters/search?${
|
|
8358
|
+
query3.set("limit", params.limit.toString());
|
|
8359
|
+
const data2 = await callDataAPI(`/task-clusters/search?${query3.toString()}`);
|
|
8171
8360
|
const record = asRecord(data2) ?? {};
|
|
8172
8361
|
const matches = Array.isArray(record.matches) ? record.matches : [];
|
|
8173
8362
|
const warnings = params.node_id !== undefined && matches.length === 0 ? notFoundWarning("cluster", params.node_id, true) : [];
|
|
8174
8363
|
context.output.writeData(data2, warnings.length > 0 ? { warnings } : undefined);
|
|
8175
8364
|
break;
|
|
8176
8365
|
}
|
|
8177
|
-
const
|
|
8366
|
+
const query2 = new URLSearchParams;
|
|
8178
8367
|
if (params.parent_id)
|
|
8179
|
-
|
|
8368
|
+
query2.set("parent_id", params.parent_id);
|
|
8180
8369
|
if (params.time_range)
|
|
8181
|
-
|
|
8182
|
-
const queryString =
|
|
8370
|
+
query2.set("time_range", params.time_range);
|
|
8371
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8183
8372
|
const data = await callDataAPI(`/clusters${queryString}`);
|
|
8184
8373
|
context.output.writeData(data);
|
|
8185
8374
|
break;
|
|
@@ -8191,12 +8380,12 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8191
8380
|
}
|
|
8192
8381
|
const args = flagsToArgs(flags, "node_id", positional);
|
|
8193
8382
|
const params = parseCommandFlags(ClusterConversationsSchema, args, command, { positionalKey: "node_id" });
|
|
8194
|
-
const
|
|
8383
|
+
const query2 = new URLSearchParams;
|
|
8195
8384
|
if (params.limit)
|
|
8196
|
-
|
|
8385
|
+
query2.set("limit", params.limit.toString());
|
|
8197
8386
|
if (params.offset !== undefined)
|
|
8198
|
-
|
|
8199
|
-
const queryString =
|
|
8387
|
+
query2.set("offset", params.offset.toString());
|
|
8388
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8200
8389
|
const data = await callDataAPI(`/clusters/${params.node_id}/conversations${queryString}`);
|
|
8201
8390
|
const clusterRecord = asRecord(data) ?? {};
|
|
8202
8391
|
const warnings = notFoundWarning("cluster", params.node_id, clusterRecord.cluster === null || clusterRecord.cluster === undefined);
|
|
@@ -8207,28 +8396,28 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8207
8396
|
case "conversations": {
|
|
8208
8397
|
const args = withStringIdFlags(flagsToArgs(flags), flags);
|
|
8209
8398
|
const params = parseCommandFlags(ConversationsSchema, args, command);
|
|
8210
|
-
const
|
|
8399
|
+
const query2 = new URLSearchParams;
|
|
8211
8400
|
if (params.search)
|
|
8212
|
-
|
|
8401
|
+
query2.set("search", params.search);
|
|
8213
8402
|
if (params.cluster_id)
|
|
8214
|
-
|
|
8403
|
+
query2.set("cluster_id", params.cluster_id);
|
|
8215
8404
|
if (params.user_id)
|
|
8216
|
-
|
|
8405
|
+
query2.set("user_id", params.user_id);
|
|
8217
8406
|
if (params.time_range)
|
|
8218
|
-
|
|
8407
|
+
query2.set("time_range", params.time_range);
|
|
8219
8408
|
if (params.environment)
|
|
8220
|
-
|
|
8409
|
+
query2.set("environment", params.environment);
|
|
8221
8410
|
if (params.world_state)
|
|
8222
|
-
|
|
8411
|
+
query2.set("world_state", params.world_state);
|
|
8223
8412
|
if (params.outcome && params.outcome !== "any")
|
|
8224
|
-
|
|
8413
|
+
query2.set("outcome", params.outcome);
|
|
8225
8414
|
if (params.sort)
|
|
8226
|
-
|
|
8415
|
+
query2.set("sort_by", params.sort);
|
|
8227
8416
|
if (params.limit)
|
|
8228
|
-
|
|
8417
|
+
query2.set("limit", params.limit.toString());
|
|
8229
8418
|
if (params.offset !== undefined)
|
|
8230
|
-
|
|
8231
|
-
const queryString =
|
|
8419
|
+
query2.set("offset", params.offset.toString());
|
|
8420
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8232
8421
|
const raw = await callDataAPI(`/conversations${queryString}`);
|
|
8233
8422
|
recordConversationsTelemetry(context, raw);
|
|
8234
8423
|
const searchWarnings = [];
|
|
@@ -8265,19 +8454,19 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8265
8454
|
}
|
|
8266
8455
|
const args = withStringIdFlags(flagsToArgs(flags, "q", positional), flags);
|
|
8267
8456
|
const params = parseCommandFlags(SearchSchema, args, command, { positionalKey: "q" });
|
|
8268
|
-
const
|
|
8269
|
-
|
|
8457
|
+
const query2 = new URLSearchParams;
|
|
8458
|
+
query2.set("q", params.q);
|
|
8270
8459
|
if (params.mode)
|
|
8271
|
-
|
|
8460
|
+
query2.set("mode", params.mode);
|
|
8272
8461
|
if (params.user_id)
|
|
8273
|
-
|
|
8462
|
+
query2.set("user_id", params.user_id);
|
|
8274
8463
|
if (params.time_range)
|
|
8275
|
-
|
|
8464
|
+
query2.set("time_range", params.time_range);
|
|
8276
8465
|
if (params.limit)
|
|
8277
|
-
|
|
8466
|
+
query2.set("limit", params.limit.toString());
|
|
8278
8467
|
if (params.include_tool_io)
|
|
8279
|
-
|
|
8280
|
-
const data = await callDataAPI(`/search?${
|
|
8468
|
+
query2.set("include_tool_io", "true");
|
|
8469
|
+
const data = await callDataAPI(`/search?${query2.toString()}`);
|
|
8281
8470
|
recordSearchTelemetry(context, data);
|
|
8282
8471
|
const searchRecord = asRecord(data) ?? {};
|
|
8283
8472
|
const degradeReason = asString(searchRecord.degrade_reason);
|
|
@@ -8325,13 +8514,13 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8325
8514
|
throw new CliInputError("Choose either --snapshot or --replay, not both.", "Usage: moda world-state <conversation_id> [--snapshot --msg-index=N [--block-index=M] [--segment-index=K]] | [--replay --message-count=N [--format=frames|fold-inputs]]");
|
|
8326
8515
|
}
|
|
8327
8516
|
if (params.snapshot === true) {
|
|
8328
|
-
const
|
|
8329
|
-
|
|
8517
|
+
const query3 = new URLSearchParams;
|
|
8518
|
+
query3.set("msg_index", (params.msg_index ?? 0).toString());
|
|
8330
8519
|
if (params.block_index !== undefined)
|
|
8331
|
-
|
|
8520
|
+
query3.set("block_index", params.block_index.toString());
|
|
8332
8521
|
if (params.segment_index !== undefined)
|
|
8333
|
-
|
|
8334
|
-
const data2 = await callDataAPI(`/conversations/${params.conversation_id}/world-state/snapshot?${
|
|
8522
|
+
query3.set("segment_index", params.segment_index.toString());
|
|
8523
|
+
const data2 = await callDataAPI(`/conversations/${params.conversation_id}/world-state/snapshot?${query3.toString()}`);
|
|
8335
8524
|
const snapshotRecord = asRecord(data2) ?? {};
|
|
8336
8525
|
const warnings2 = notFoundWarning("world state", params.conversation_id, snapshotRecord.found === false);
|
|
8337
8526
|
context.output.writeData(data2, warnings2.length > 0 ? { warnings: warnings2 } : undefined);
|
|
@@ -8342,22 +8531,22 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8342
8531
|
if (format === "frames" && params.message_count === undefined) {
|
|
8343
8532
|
throw new CliInputError("--message-count=N is required with --replay in frames mode (the API returns no frames without it). Use --format=fold-inputs for the raw keyframes/events without a count.");
|
|
8344
8533
|
}
|
|
8345
|
-
const
|
|
8346
|
-
|
|
8534
|
+
const query3 = new URLSearchParams;
|
|
8535
|
+
query3.set("format", format === "fold-inputs" ? "fold_inputs" : "frames");
|
|
8347
8536
|
if (params.message_count !== undefined)
|
|
8348
|
-
|
|
8349
|
-
const data2 = await callDataAPI(`/conversations/${params.conversation_id}/world-state/replay?${
|
|
8537
|
+
query3.set("message_count", params.message_count.toString());
|
|
8538
|
+
const data2 = await callDataAPI(`/conversations/${params.conversation_id}/world-state/replay?${query3.toString()}`);
|
|
8350
8539
|
const replayRecord = asRecord(data2) ?? {};
|
|
8351
8540
|
const warnings2 = notFoundWarning("world state", params.conversation_id, replayRecord.hasData === false);
|
|
8352
8541
|
context.output.writeData(data2, warnings2.length > 0 ? { warnings: warnings2 } : undefined);
|
|
8353
8542
|
break;
|
|
8354
8543
|
}
|
|
8355
|
-
const
|
|
8544
|
+
const query2 = new URLSearchParams;
|
|
8356
8545
|
if (params.summary_only === true)
|
|
8357
|
-
|
|
8546
|
+
query2.set("summary_only", "1");
|
|
8358
8547
|
if (params.event_limit)
|
|
8359
|
-
|
|
8360
|
-
const queryString =
|
|
8548
|
+
query2.set("event_limit", params.event_limit.toString());
|
|
8549
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8361
8550
|
const data = await callDataAPI(`/conversations/${params.conversation_id}/world-state${queryString}`);
|
|
8362
8551
|
const wsRecord = asRecord(data) ?? {};
|
|
8363
8552
|
const warnings = notFoundWarning("world state", params.conversation_id, wsRecord.hasData === false);
|
|
@@ -8389,16 +8578,29 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8389
8578
|
if (params.from !== undefined || params.max_messages !== undefined) {
|
|
8390
8579
|
throw new CliInputError("--from and --max-messages page the whole trace and need --all.", "Usage: moda context <conversation_id> --all [--from=N] [--max-messages=N]");
|
|
8391
8580
|
}
|
|
8392
|
-
const
|
|
8581
|
+
const query2 = new URLSearchParams;
|
|
8393
8582
|
if (params.msg_index !== undefined)
|
|
8394
|
-
|
|
8395
|
-
|
|
8396
|
-
|
|
8397
|
-
|
|
8583
|
+
query2.set("msg_index", params.msg_index.toString());
|
|
8584
|
+
const window = params.window === undefined ? undefined : Math.min(params.window, CONTEXT_MAX_WINDOW);
|
|
8585
|
+
if (window !== undefined)
|
|
8586
|
+
query2.set("window", window.toString());
|
|
8587
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8398
8588
|
const data = await callDataAPI(`/conversations/${params.conversation_id}/context${queryString}`);
|
|
8399
8589
|
const ctxRecord = asRecord(data) ?? {};
|
|
8400
8590
|
const warnings = notFoundWarning("trace", params.conversation_id, asNumber(ctxRecord.total_messages) === 0);
|
|
8401
|
-
|
|
8591
|
+
const nextCommands = [];
|
|
8592
|
+
if (params.window !== undefined && params.window > CONTEXT_MAX_WINDOW) {
|
|
8593
|
+
const center = asNumber(asRecord(ctxRecord.context)?.center_index) ?? params.msg_index ?? 0;
|
|
8594
|
+
const span = Math.min(params.window * 2 + 1, CONTEXT_ALL_MAX_MESSAGES);
|
|
8595
|
+
const from = Math.max(0, center - Math.floor((span - 1) / 2));
|
|
8596
|
+
const wide = `moda context ${shellArg(params.conversation_id)} --all --from=${from} --max-messages=${span}`;
|
|
8597
|
+
warnings.push(`--window=${params.window} is above the maximum of ${CONTEXT_MAX_WINDOW}, so this shows ±${CONTEXT_MAX_WINDOW} messages. For a wider slice run: ${wide}`);
|
|
8598
|
+
nextCommands.push({ command: wide, purpose: `Read ±${params.window} messages around the anchor.`, mutability: "read", requires_approval: false });
|
|
8599
|
+
}
|
|
8600
|
+
context.output.writeData(data, {
|
|
8601
|
+
...warnings.length > 0 ? { warnings } : {},
|
|
8602
|
+
...nextCommands.length > 0 ? { nextCommands } : {}
|
|
8603
|
+
});
|
|
8402
8604
|
break;
|
|
8403
8605
|
}
|
|
8404
8606
|
case "audit":
|
|
@@ -8408,14 +8610,14 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8408
8610
|
}
|
|
8409
8611
|
const args = flagsToArgs(flags, "id", positional);
|
|
8410
8612
|
const params = parseCommandFlags(AuditSchema, args, command, { positionalKey: "id" });
|
|
8411
|
-
const
|
|
8613
|
+
const query2 = new URLSearchParams;
|
|
8412
8614
|
if (params.kind && params.kind !== "auto")
|
|
8413
|
-
|
|
8615
|
+
query2.set("kind", params.kind);
|
|
8414
8616
|
if (params.include_raw === true)
|
|
8415
|
-
|
|
8617
|
+
query2.set("include_raw", "1");
|
|
8416
8618
|
if (params.limit)
|
|
8417
|
-
|
|
8418
|
-
const queryString =
|
|
8619
|
+
query2.set("limit", params.limit.toString());
|
|
8620
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8419
8621
|
const data = await callDataAPI(`/audit/${encodeURIComponent(params.id)}${queryString}`);
|
|
8420
8622
|
const capped = auditCapStatus(data, params.limit ?? AUDIT_DEFAULT_LIMIT);
|
|
8421
8623
|
const auditRecord = asRecord(data);
|
|
@@ -8448,19 +8650,19 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8448
8650
|
case "frustrations": {
|
|
8449
8651
|
const args = withStringIdFlags(flagsToArgs(flags), flags);
|
|
8450
8652
|
const params = parseCommandFlags(FrustrationsSchema, args, command);
|
|
8451
|
-
const
|
|
8653
|
+
const query2 = new URLSearchParams;
|
|
8452
8654
|
if (params.days_back)
|
|
8453
|
-
|
|
8655
|
+
query2.set("days_back", params.days_back.toString());
|
|
8454
8656
|
if (params.limit)
|
|
8455
|
-
|
|
8657
|
+
query2.set("limit", params.limit.toString());
|
|
8456
8658
|
if (params.offset !== undefined)
|
|
8457
|
-
|
|
8659
|
+
query2.set("offset", params.offset.toString());
|
|
8458
8660
|
if (params.user_id)
|
|
8459
|
-
|
|
8661
|
+
query2.set("user_id", params.user_id);
|
|
8460
8662
|
if (params.conversation_id)
|
|
8461
|
-
|
|
8462
|
-
|
|
8463
|
-
const data = frustrationsFromEmotions(await callDataAPI(`/emotions?${
|
|
8663
|
+
query2.set("conversation_id", params.conversation_id);
|
|
8664
|
+
query2.set("family", "frustration");
|
|
8665
|
+
const data = frustrationsFromEmotions(await callDataAPI(`/emotions?${query2.toString()}`));
|
|
8464
8666
|
await printWithOptionalWindows(withFrustrationAnchors(data), "frustrations", params.include_window === true, params.window ?? 1, context.output, processingDisabledWarning(data, "frustration"));
|
|
8465
8667
|
break;
|
|
8466
8668
|
}
|
|
@@ -8489,15 +8691,15 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8489
8691
|
if (params.compact === true && params.include_window === true) {
|
|
8490
8692
|
throw new CliInputError("--compact drops the conversation window; drop --include-window, or drop --compact.");
|
|
8491
8693
|
}
|
|
8492
|
-
const
|
|
8694
|
+
const query2 = new URLSearchParams;
|
|
8493
8695
|
if (params.subtype)
|
|
8494
|
-
|
|
8696
|
+
query2.set("subtype", params.subtype);
|
|
8495
8697
|
if (params.days_back)
|
|
8496
|
-
|
|
8497
|
-
|
|
8698
|
+
query2.set("days_back", params.days_back.toString());
|
|
8699
|
+
query2.set("limit", (params.limit ?? TOOL_FAILURE_DETAIL_DEFAULT_LIMIT).toString());
|
|
8498
8700
|
if (params.offset !== undefined)
|
|
8499
|
-
|
|
8500
|
-
const queryString =
|
|
8701
|
+
query2.set("offset", params.offset.toString());
|
|
8702
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8501
8703
|
const data = await callDataAPI(`/tool-failures/${params.tool_name}${queryString}`);
|
|
8502
8704
|
const tfRecord = asRecord(data) ?? {};
|
|
8503
8705
|
const subtypeCount = Array.isArray(tfRecord.subtypes) ? tfRecord.subtypes.length : 0;
|
|
@@ -8531,7 +8733,7 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8531
8733
|
}
|
|
8532
8734
|
case "problem-impact": {
|
|
8533
8735
|
const params = ProblemImpactSchema.parse(Object.fromEntries(Object.entries(flags).map(([key, value]) => [key.replace(/-/g, "_"), value])));
|
|
8534
|
-
const
|
|
8736
|
+
const query2 = new URLSearchParams({ days: String(params.days_back), mode: params.mode });
|
|
8535
8737
|
for (const [key, value] of Object.entries({
|
|
8536
8738
|
nodeId: params.node_id,
|
|
8537
8739
|
problemId: params.problem_id,
|
|
@@ -8541,8 +8743,8 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8541
8743
|
conversationIds: params.conversation_ids ? JSON.stringify(params.conversation_ids) : undefined
|
|
8542
8744
|
}))
|
|
8543
8745
|
if (value !== undefined)
|
|
8544
|
-
|
|
8545
|
-
const data = asRecord(await callDataAPI(`/problem-impact?${
|
|
8746
|
+
query2.set(key, value);
|
|
8747
|
+
const data = asRecord(await callDataAPI(`/problem-impact?${query2}`)) ?? {};
|
|
8546
8748
|
const link = new URLSearchParams({ impactDays: String(params.days_back) });
|
|
8547
8749
|
const window = asRecord(data.window);
|
|
8548
8750
|
const resolvedRun = asString(data.runId);
|
|
@@ -8569,12 +8771,12 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8569
8771
|
}
|
|
8570
8772
|
const args = flagsToArgs(flags);
|
|
8571
8773
|
const params = parseCommandFlags(ProblemsSchema, args, command);
|
|
8572
|
-
const
|
|
8774
|
+
const query2 = new URLSearchParams;
|
|
8573
8775
|
if (params.days_back)
|
|
8574
|
-
|
|
8776
|
+
query2.set("days_back", params.days_back.toString());
|
|
8575
8777
|
if (params.limit)
|
|
8576
|
-
|
|
8577
|
-
const queryString =
|
|
8778
|
+
query2.set("limit", params.limit.toString());
|
|
8779
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8578
8780
|
const data = await callDataAPI(`/problems${queryString}`);
|
|
8579
8781
|
const { payload, truncated } = capProblemsBlocks(asRecord(data) ?? {}, params.limit ?? 25);
|
|
8580
8782
|
const withLink = {
|
|
@@ -8616,18 +8818,18 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8616
8818
|
const viewFlag = view === "conversations" ? "--traces" : `--${view}`;
|
|
8617
8819
|
throw new CliInputError(`${viewFlag} requires a canonical problem UUID; got "${params.id}". Run \`moda problems\` or \`moda problem <id>\` first to resolve the id.`);
|
|
8618
8820
|
}
|
|
8619
|
-
const
|
|
8821
|
+
const query2 = new URLSearchParams;
|
|
8620
8822
|
if (params.limit)
|
|
8621
|
-
|
|
8823
|
+
query2.set("limit", params.limit.toString());
|
|
8622
8824
|
if (params.cursor)
|
|
8623
|
-
|
|
8825
|
+
query2.set("cursor", params.cursor);
|
|
8624
8826
|
if (view === "conversations") {
|
|
8625
8827
|
if (params.family)
|
|
8626
|
-
|
|
8828
|
+
query2.set("family", params.family);
|
|
8627
8829
|
if (params.door)
|
|
8628
|
-
|
|
8830
|
+
query2.set("door", params.door);
|
|
8629
8831
|
}
|
|
8630
|
-
const queryString =
|
|
8832
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8631
8833
|
const data = await callDataAPI(`/problems/${params.id}/${view}${queryString}`);
|
|
8632
8834
|
context.output.writeData(data);
|
|
8633
8835
|
break;
|
|
@@ -8763,14 +8965,14 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8763
8965
|
context.output.writeData(data2);
|
|
8764
8966
|
break;
|
|
8765
8967
|
}
|
|
8766
|
-
const
|
|
8968
|
+
const query2 = new URLSearchParams;
|
|
8767
8969
|
if (params.days_back)
|
|
8768
|
-
|
|
8970
|
+
query2.set("days_back", params.days_back.toString());
|
|
8769
8971
|
if (params.limit)
|
|
8770
|
-
|
|
8972
|
+
query2.set("limit", params.limit.toString());
|
|
8771
8973
|
if (params.offset !== undefined)
|
|
8772
|
-
|
|
8773
|
-
const queryString =
|
|
8974
|
+
query2.set("offset", params.offset.toString());
|
|
8975
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8774
8976
|
const data = await callDataAPI(`/custom-signals/${id}/detections${queryString}`);
|
|
8775
8977
|
context.output.writeData(data);
|
|
8776
8978
|
break;
|
|
@@ -8922,20 +9124,20 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8922
9124
|
case "false-positives": {
|
|
8923
9125
|
if (!positional) {
|
|
8924
9126
|
const kinds = { quote: "frustration_quote", tool: "tool_call", laziness: "laziness", hallucination: "hallucination", "custom-signal": "custom_signal" };
|
|
8925
|
-
const
|
|
9127
|
+
const query2 = new URLSearchParams;
|
|
8926
9128
|
if (flags.kind !== undefined) {
|
|
8927
9129
|
if (!kinds[flags.kind])
|
|
8928
9130
|
throw new CliInputError(`--kind must be one of ${Object.keys(kinds).join("|")}`);
|
|
8929
|
-
|
|
9131
|
+
query2.set("kind", kinds[flags.kind]);
|
|
8930
9132
|
}
|
|
8931
9133
|
if (flags.limit !== undefined) {
|
|
8932
9134
|
if (!/^\d+$/.test(flags.limit) || Number(flags.limit) < 1 || Number(flags.limit) > 200)
|
|
8933
9135
|
throw new CliInputError("--limit must be 1-200");
|
|
8934
|
-
|
|
9136
|
+
query2.set("limit", flags.limit);
|
|
8935
9137
|
}
|
|
8936
9138
|
if (flags.all !== undefined && flags.all !== "false")
|
|
8937
|
-
|
|
8938
|
-
const queryString =
|
|
9139
|
+
query2.set("include_cleared", "true");
|
|
9140
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8939
9141
|
const data2 = await callDataAPI(`/false-positives${queryString}`);
|
|
8940
9142
|
context.output.writeData(data2);
|
|
8941
9143
|
break;
|
|
@@ -8947,20 +9149,20 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8947
9149
|
case "emotions": {
|
|
8948
9150
|
const args = withStringIdFlags(flagsToArgs(flags), flags);
|
|
8949
9151
|
const params = parseCommandFlags(EmotionsSchema, args, command);
|
|
8950
|
-
const
|
|
9152
|
+
const query2 = new URLSearchParams;
|
|
8951
9153
|
if (params.days_back)
|
|
8952
|
-
|
|
9154
|
+
query2.set("days_back", params.days_back.toString());
|
|
8953
9155
|
if (params.limit)
|
|
8954
|
-
|
|
9156
|
+
query2.set("limit", params.limit.toString());
|
|
8955
9157
|
if (params.offset !== undefined)
|
|
8956
|
-
|
|
9158
|
+
query2.set("offset", params.offset.toString());
|
|
8957
9159
|
if (params.family)
|
|
8958
|
-
|
|
9160
|
+
query2.set("family", params.family);
|
|
8959
9161
|
if (params.user_id)
|
|
8960
|
-
|
|
9162
|
+
query2.set("user_id", params.user_id);
|
|
8961
9163
|
if (params.conversation_id)
|
|
8962
|
-
|
|
8963
|
-
const queryString =
|
|
9164
|
+
query2.set("conversation_id", params.conversation_id);
|
|
9165
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8964
9166
|
const data = await callDataAPI(`/emotions${queryString}`);
|
|
8965
9167
|
const processingWarnings = processingDisabledWarning(data, "emotion");
|
|
8966
9168
|
context.output.writeData(data, processingWarnings.length > 0 ? { warnings: processingWarnings } : undefined);
|
|
@@ -8972,18 +9174,18 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8972
9174
|
args.conversation_id = flags["conversation-id"];
|
|
8973
9175
|
}
|
|
8974
9176
|
const params = parseCommandFlags(LazinessSchema, args, command);
|
|
8975
|
-
const
|
|
9177
|
+
const query2 = new URLSearchParams;
|
|
8976
9178
|
if (params.days_back)
|
|
8977
|
-
|
|
9179
|
+
query2.set("days_back", params.days_back.toString());
|
|
8978
9180
|
if (params.limit)
|
|
8979
|
-
|
|
9181
|
+
query2.set("limit", params.limit.toString());
|
|
8980
9182
|
if (params.offset !== undefined)
|
|
8981
|
-
|
|
9183
|
+
query2.set("offset", params.offset.toString());
|
|
8982
9184
|
if (params.conversation_id)
|
|
8983
|
-
|
|
9185
|
+
query2.set("conversation_id", params.conversation_id);
|
|
8984
9186
|
if (params.family)
|
|
8985
|
-
|
|
8986
|
-
const queryString =
|
|
9187
|
+
query2.set("family", params.family);
|
|
9188
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
8987
9189
|
const data = await callDataAPI(`/laziness${queryString}`);
|
|
8988
9190
|
context.output.writeData(data);
|
|
8989
9191
|
break;
|
|
@@ -8994,18 +9196,18 @@ async function runCommand(command, positional, flags, positionals = positional ?
|
|
|
8994
9196
|
args.conversation_id = flags["conversation-id"];
|
|
8995
9197
|
}
|
|
8996
9198
|
const params = parseCommandFlags(HallucinationsSchema, args, command);
|
|
8997
|
-
const
|
|
9199
|
+
const query2 = new URLSearchParams;
|
|
8998
9200
|
if (params.days_back)
|
|
8999
|
-
|
|
9201
|
+
query2.set("days_back", params.days_back.toString());
|
|
9000
9202
|
if (params.limit)
|
|
9001
|
-
|
|
9203
|
+
query2.set("limit", params.limit.toString());
|
|
9002
9204
|
if (params.offset !== undefined)
|
|
9003
|
-
|
|
9205
|
+
query2.set("offset", params.offset.toString());
|
|
9004
9206
|
if (params.conversation_id)
|
|
9005
|
-
|
|
9207
|
+
query2.set("conversation_id", params.conversation_id);
|
|
9006
9208
|
if (params.kind)
|
|
9007
|
-
|
|
9008
|
-
const queryString =
|
|
9209
|
+
query2.set("kind", params.kind);
|
|
9210
|
+
const queryString = query2.toString() ? `?${query2.toString()}` : "";
|
|
9009
9211
|
const data = await callDataAPI(`/hallucinations${queryString}`);
|
|
9010
9212
|
context.output.writeData(data);
|
|
9011
9213
|
break;
|
|
@@ -9173,7 +9375,7 @@ var commandRegistry = createCommandRegistry([
|
|
|
9173
9375
|
resetApiRequestCountBeforeRun: true,
|
|
9174
9376
|
telemetry: "result",
|
|
9175
9377
|
handler: async (context) => {
|
|
9176
|
-
const { runInit } = await import("./index-
|
|
9378
|
+
const { runInit } = await import("./index-r1egcg99.js");
|
|
9177
9379
|
if (context.outputMode === "agent-stream") {
|
|
9178
9380
|
context.output.writeEvent({
|
|
9179
9381
|
event: "started",
|
|
@@ -9481,6 +9683,23 @@ var commandRegistry = createCommandRegistry([
|
|
|
9481
9683
|
telemetry: "result-and-error",
|
|
9482
9684
|
handler: runUserCommand
|
|
9483
9685
|
},
|
|
9686
|
+
{
|
|
9687
|
+
name: "replays",
|
|
9688
|
+
description: "Read replay results: sets, run history, per-seed verdicts, and playout transcripts",
|
|
9689
|
+
examples: [
|
|
9690
|
+
"moda replays runs <set_id>",
|
|
9691
|
+
"moda replays run <set_id> <run_id> --json",
|
|
9692
|
+
"moda replays playouts <set_id> <run_id> --case=<case_id> --arm=proposed --transcript --json"
|
|
9693
|
+
],
|
|
9694
|
+
subcommands: REPLAYS_SUBCOMMAND_HELP,
|
|
9695
|
+
auth: "api-key",
|
|
9696
|
+
outputModes: ["human", "json", "agent"],
|
|
9697
|
+
mutability: "read",
|
|
9698
|
+
validateConfigBeforeRun: false,
|
|
9699
|
+
resetApiRequestCountBeforeRun: false,
|
|
9700
|
+
telemetry: "result-and-error",
|
|
9701
|
+
handler: runReplaysCommand
|
|
9702
|
+
},
|
|
9484
9703
|
{
|
|
9485
9704
|
name: "problem-reopen",
|
|
9486
9705
|
description: "Reopen a problem that was marked fixed (resolved problems return only via re-detection)",
|
|
@@ -9624,6 +9843,7 @@ var commandRegistry = createCommandRegistry([
|
|
|
9624
9843
|
description: "Get windowed trace context (messages around one turn), or the whole trace with --all",
|
|
9625
9844
|
examples: [
|
|
9626
9845
|
"moda context <conversation_id> --window=3",
|
|
9846
|
+
"moda context <conversation_id> --msg-index=40 --window=0",
|
|
9627
9847
|
"moda context <conversation_id> --all",
|
|
9628
9848
|
"moda context <conversation_id> --all --from=500 --max-messages=500"
|
|
9629
9849
|
],
|
|
@@ -9921,11 +10141,12 @@ var commandRegistry = createCommandRegistry([
|
|
|
9921
10141
|
flags: [
|
|
9922
10142
|
{ name: "--baseline=<path>", description: "Prompt file to treat as the control." },
|
|
9923
10143
|
{ name: "--candidate=<path>", description: "Prompt file to treat as the variant." },
|
|
9924
|
-
{ name: "--traces=<ids>", description: "Comma-separated trace ids (conversation_id values) to replay, at most 100. Legacy alias: --conversations=<ids>." },
|
|
10144
|
+
{ name: "--traces=<ids>", description: "Comma-separated trace ids (conversation_id values) to replay, at most 100. Each case uses Moda's distilled scenario and success criteria for the trace; a trace that cannot be distilled falls back to its opening user message with generic criteria, and the run warns. Legacy alias: --conversations=<ids>." },
|
|
9925
10145
|
{ name: "--cases=<n>", description: "Cases to auto-generate (default 5, max 100)." },
|
|
9926
10146
|
{ name: "--seeds=<n>", description: "Repeats per case per arm (default 3, max 10)." },
|
|
9927
10147
|
{ name: "--model=<slug>", description: "Assistant model for both arms; must be in the replay allowlist unless --allow-any-model." },
|
|
9928
|
-
{ name: "--yes", description: "Confirm a run above 200 planned playouts (cases x seeds x 2 arms), or one whose case count is unknown." }
|
|
10148
|
+
{ name: "--yes", description: "Confirm a run above 200 planned playouts (cases x seeds x 2 arms), or one whose case count is unknown." },
|
|
10149
|
+
{ name: "--promote", description: "Make this run's replay set the tenant's primary replay set. Off by default, so an ad-hoc A/B never replaces it." }
|
|
9929
10150
|
],
|
|
9930
10151
|
examples: [
|
|
9931
10152
|
"moda prompts ab --baseline=prompts/agent.prompt.md --candidate=prompts/agent-v2.prompt.md --traces=conv_1,conv_2"
|
package/package.json
CHANGED
package/skills/moda-cli/SKILL.md
CHANGED
|
@@ -442,7 +442,9 @@ fail instead of being clamped.
|
|
|
442
442
|
|
|
443
443
|
`--window=N` on `context`, `frustrations --include-window` and
|
|
444
444
|
`tool-failure-detail --include-window` is a message half-width (1–5), not a
|
|
445
|
-
time window.
|
|
445
|
+
time window. On `context`, `--window=0` returns only the anchor message (plus
|
|
446
|
+
`total_messages`), and a value above 5 is clamped to 5 with a warning that
|
|
447
|
+
prints the `--all --from --max-messages` command for a wider slice.
|
|
446
448
|
|
|
447
449
|
### Schema introspection
|
|
448
450
|
|
|
@@ -631,6 +633,7 @@ Filters: `--search`, `--cluster-id`, `--user-id`, `--time-range`
|
|
|
631
633
|
moda context <conversation_id> # default window around middle
|
|
632
634
|
moda context <conversation_id> --msg-index=5 # center on message 5
|
|
633
635
|
moda context <conversation_id> --window=3 # 3 messages each side (max 5)
|
|
636
|
+
moda context <conversation_id> --msg-index=40 --window=0 # just message 40 + total_messages
|
|
634
637
|
moda context <conversation_id> --all # whole trace in order, first 500 messages
|
|
635
638
|
moda context <conversation_id> --all --from=500 --max-messages=500 # continue (max 5000)
|
|
636
639
|
```
|
|
@@ -767,6 +770,13 @@ earlier turns as history);
|
|
|
767
770
|
`anthropic/claude-haiku-4-5`, `google/gemini-2.5-flash`,
|
|
768
771
|
`google/gemini-2.5-pro` unless you pass `--allow-any-model`.
|
|
769
772
|
|
|
773
|
+
`prompts ab --traces=<ids>` builds one case per trace from Moda's distilled
|
|
774
|
+
scenario and success criteria for that trace. A trace Moda cannot distill
|
|
775
|
+
falls back to its opening user message with three generic criteria (which long
|
|
776
|
+
traces pass easily); the run warns and lists it under `caseSources.fallback`.
|
|
777
|
+
`prompts ab` never makes its replay set the tenant's primary set unless you
|
|
778
|
+
pass `--promote`.
|
|
779
|
+
|
|
770
780
|
Runtime code should render synced prompts through the SDK:
|
|
771
781
|
|
|
772
782
|
```typescript
|
|
@@ -785,6 +795,43 @@ Do not copy prompt text from the dashboard into source code. The repo prompt
|
|
|
785
795
|
files are the source of truth; the dashboard is for visibility, usage, labels,
|
|
786
796
|
and future evals.
|
|
787
797
|
|
|
798
|
+
### 7a. Check a replay verdict (`moda replays`)
|
|
799
|
+
|
|
800
|
+
Use after `prompts ab`, `prompts propose --gate`, or a dashboard replay when
|
|
801
|
+
you need to see *why* an arm passed or failed. Read-only; no replay spend.
|
|
802
|
+
|
|
803
|
+
```bash
|
|
804
|
+
moda replays sets --json # set ids
|
|
805
|
+
moda replays runs <set_id> --limit=5 --json # newest runs, every status
|
|
806
|
+
moda replays run <set_id> <run_id> --json # per-case + per-seed verdicts
|
|
807
|
+
moda replays playouts <set_id> <run_id> --json # one row per case x arm x seed
|
|
808
|
+
moda replays playouts <set_id> <run_id> --case=<case_id> --arm=proposed --seed=1 --transcript --json
|
|
809
|
+
moda replays playout <set_id> <run_id> <playout_id> --json
|
|
810
|
+
```
|
|
811
|
+
|
|
812
|
+
`run` returns `results[]` (one per case × arm × seed: `pass`, `abstained`,
|
|
813
|
+
`coverageReason`, `score`, `rationale`, `criteria[]`) next to the per-case
|
|
814
|
+
majority in `cases[]`. `arm=prod` is the baseline and `arm=proposed` the
|
|
815
|
+
candidate. An `abstained: true` row is not a verdict: the pair hit a replay
|
|
816
|
+
coverage hole and is excluded from pass counts.
|
|
817
|
+
|
|
818
|
+
A playout's `transcript.events[]` holds the simulated user and agent messages
|
|
819
|
+
and every tool call with its `input`, the `output` the replay served, and
|
|
820
|
+
`provenance.source`: `replay_runtime` (compiled mock over the case's recorded
|
|
821
|
+
source state), `agent_tool_simulator` (LLM simulator grounded in recorded
|
|
822
|
+
receipts; `simulator.model`), `counterfactual_reconstruction`, or
|
|
823
|
+
`generic_handler`. `provenance.errorSubtype` such as `missing_source_state`
|
|
824
|
+
marks an unservable call. `judge` is the judgement stored with the playout and
|
|
825
|
+
`verdict` the stored eval row (null when `seed` is null, which happens on
|
|
826
|
+
older runs whose graphs cannot be matched to a seed).
|
|
827
|
+
|
|
828
|
+
Limits: `--transcript` pages at most 10 playouts (`--limit` default 50, max
|
|
829
|
+
200 without it); text and tool JSON are clipped at `--max-chars` (default
|
|
830
|
+
4000, max 32000) and `transcript.truncated.fields` counts the clipped fields.
|
|
831
|
+
Page with `--cursor=<nextCursor>`. Playouts exist only for runs whose
|
|
832
|
+
playouts were captured; `playoutCount: 0` on `run` means there is no
|
|
833
|
+
transcript to read.
|
|
834
|
+
|
|
788
835
|
### 7b. Push prompts, skills, and tools without a repo scan (`moda registry`)
|
|
789
836
|
|
|
790
837
|
Use when the user wants their prompts, skills, and tools visible in Moda but
|
|
@@ -1026,6 +1073,7 @@ Writes listed here accept `--dry-run` (prints the exact request, sends nothing)
|
|
|
1026
1073
|
| Custom signals: edit, lifecycle | `moda signals update <id> …`, `moda signals activate\|pause\|delete <id>` (delete: drafts only; archiving is dashboard-only) |
|
|
1027
1074
|
| Custom signals: review & tune | `moda signals examples <id> add\|update\|remove --conversation-id=…`, `refine <id> --request="…"`, `revise <id> --conversation-ids=…`, `proposals <id>`, `accept\|reject <id> <proposal_id>` |
|
|
1028
1075
|
| Custom signals: find matches | `moda signals search <id>`, `search-status <id> <search_id>`, `search-stop <id> <search_id>`, `preview --file=detector.json` |
|
|
1076
|
+
| Replays: run detail, playout transcripts | `moda replays runs <set_id>`, `run <set_id> <run_id>`, `playouts <set_id> <run_id> [--case --arm --seed --transcript]`, `playout <set_id> <run_id> <playout_id>` |
|
|
1029
1077
|
| Prompts registry | `moda prompts list [--label --search]`, `show <key>`, `proposals <key>`, `usage <key>`, `replay-runs <key>`, `decide <key> <proposal_id> --action=merge\|reject\|reopen` |
|
|
1030
1078
|
| Skills library | `moda skills list`, `show <key>`, `policy <key> --live\|--local` |
|
|
1031
1079
|
| Harness pages | `moda harness list`, `show <id>`, `versions <id>`, `diff <id> --from=… --to=…` |
|
|
@@ -1234,6 +1282,7 @@ npx: `npx -p @moda-ai/cli moda <command>`.
|
|
|
1234
1282
|
| `moda problem-reopen <pid>` | Reopen a Problem marked fixed |
|
|
1235
1283
|
| `moda signals <create\|update\|activate\|pause\|delete\|examples\|refine\|revise\|proposals\|accept\|reject\|search\|preview>` | Manage custom signals (writes take `--dry-run`; LLM-backed ones are rate-limited per key) |
|
|
1236
1284
|
| `moda prompts list\|show\|proposals\|usage\|replay-runs\|decide` | Read the prompt registry; merge/reject proposals |
|
|
1285
|
+
| `moda replays sets\|runs\|run\|playouts\|playout` | Replay results: run history, per-seed verdicts, playout transcripts with tool provenance (read-only) |
|
|
1237
1286
|
| `moda skills list\|show\|policy` / `moda harness list\|show\|versions\|diff` | Skill library and synced harnesses |
|
|
1238
1287
|
| `moda fixes` | Ranked queue of Fixes ("a fix with proof"); `--status=S`, `--limit=N`, `--cursor=TOKEN` |
|
|
1239
1288
|
| `moda fix <fix_id>` | One Fix: status + gate result; `--packet` prints `moda.fix_packet.v1` and writes tool/skill candidates under `.moda/fixes/<ref>/`; `--wait` drives a running pipeline to rest |
|