@moda-ai/cli 1.55.23 → 1.55.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{agent-f8a1kejk.js → agent-t7dxmpdq.js} +8 -8
- package/dist/{cli-9g8vz4sv.js → cli-0wr3483h.js} +1 -1
- package/dist/{cli-4f6jkrnm.js → cli-7xc585hc.js} +5 -5
- package/dist/{cli-wkmdw40s.js → cli-9btx12wp.js} +1 -1
- package/dist/{cli-4yarp5s1.js → cli-9hc5bg2r.js} +1 -1
- package/dist/{cli-cv3jt1ms.js → cli-dn91fb56.js} +10 -10
- package/dist/{cli-t3a6eknn.js → cli-dve50x7p.js} +233 -28
- package/dist/{cli-54zhv6cx.js → cli-m0045npe.js} +17 -14
- package/dist/{cli-s6tw1fje.js → cli-m805msp6.js} +210 -23
- package/dist/{cli-z3g07t65.js → cli-mebf5vtr.js} +5 -5
- package/dist/{cli-2cggp8ww.js → cli-qvg26twp.js} +5 -4
- package/dist/{cli-d2zews6c.js → cli-qycsmxev.js} +3 -3
- package/dist/cli.js +1292 -335
- package/dist/{doctor-snb87jvy.js → doctor-q21mr5ya.js} +8 -8
- package/dist/{fix-fr48gr0w.js → fix-0qdtdz5h.js} +6 -6
- package/dist/{harness-github-actions-cfkx9ty9.js → harness-github-actions-atc5bbkj.js} +5 -5
- package/dist/{harness-m63b6s0k.js → harness-vzy7n7ea.js} +5 -5
- package/dist/{index-nsaph4as.js → index-4b1ybz51.js} +17 -17
- package/dist/{production-intelligence-fwjh572m.js → production-intelligence-c8e0xw3n.js} +11 -11
- package/dist/{profile-commands-548ts402.js → profile-commands-6e2dgv72.js} +3 -3
- package/dist/prompt-source-mcp.js +5 -5
- package/dist/{prompts-p66y7wep.js → prompts-3h5jcjrw.js} +6 -6
- package/dist/{provision-r9fhrvpw.js → provision-q86phh4a.js} +5 -5
- package/dist/{registry-n7dsdxmg.js → registry-s2vdsfw8.js} +3 -3
- package/dist/{skills-c17gg69y.js → skills-vq3njc8a.js} +5 -5
- package/dist/{sync-bne90xvj.js → sync-hq9dpqpr.js} +7 -7
- package/package.json +1 -1
- package/skills/integration/index.json +2 -2
- package/skills/moda-cli/SKILL.md +35 -30
- package/skills/moda-cli/references/commands.md +22 -22
- package/skills/moda-cli/references/dashboard-parity.md +4 -4
- package/skills/moda-cli/references/detections.md +10 -10
- package/skills/moda-cli/references/feedback.md +2 -2
- package/skills/moda-cli/references/output-protocol.md +2 -2
- package/skills/moda-cli/references/production-intelligence.md +13 -12
- package/skills/moda-cli/references/prompts.md +6 -2
- package/skills/moda-cli/references/recipes.md +14 -14
- package/skills/moda-cli/references/search-and-traces.md +32 -32
- package/skills/moda-cli/references/signals-and-false-positives.md +29 -29
- package/skills/moda-cli/references/troubleshooting.md +4 -4
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createCommandContext
|
|
3
|
-
} from "./cli-
|
|
3
|
+
} from "./cli-qvg26twp.js";
|
|
4
4
|
import {
|
|
5
5
|
assertFlags,
|
|
6
6
|
assertPositionals
|
|
7
|
-
} from "./cli-
|
|
7
|
+
} from "./cli-dn91fb56.js";
|
|
8
8
|
import {
|
|
9
9
|
callControlAPI
|
|
10
|
-
} from "./cli-
|
|
10
|
+
} from "./cli-9btx12wp.js";
|
|
11
11
|
import {
|
|
12
12
|
CliInputError,
|
|
13
13
|
recordApiRequest,
|
|
14
14
|
resolveApiKey,
|
|
15
15
|
resolveIngestUrl,
|
|
16
16
|
resolveTenantId
|
|
17
|
-
} from "./cli-
|
|
17
|
+
} from "./cli-dve50x7p.js";
|
|
18
18
|
import {
|
|
19
19
|
SKILL_SOURCE_HARNESS,
|
|
20
20
|
SKILL_SOURCE_INTEGRATION_BUNDLED,
|
|
@@ -56,25 +56,28 @@ function renderFixesBlock() {
|
|
|
56
56
|
"The `moda` CLI needs `MODA_API_KEY` (or `npx -y -p @moda-ai/cli moda ...` when",
|
|
57
57
|
"not installed); `moda <command> --help` lists its flags.",
|
|
58
58
|
"",
|
|
59
|
-
'- Find where something happened in production: `moda search "<words>"`, then',
|
|
60
|
-
" `moda
|
|
59
|
+
'- Find where something happened in production: `moda traces search "<words>"`, then',
|
|
60
|
+
" `moda traces get <conversation_id> --msg-index=<message_index>` to read the turn.",
|
|
61
61
|
"- Find what to fix: `moda investigate` ranks tool failures, user frustrations,",
|
|
62
62
|
" and top Problems with evidence; `moda problems` ranks cross-signal Problems",
|
|
63
63
|
" by root cause; `moda failures` lists tool failures only.",
|
|
64
|
-
"- Read the evidence before editing: `moda
|
|
65
|
-
" `moda
|
|
66
|
-
" `moda
|
|
67
|
-
" `moda
|
|
68
|
-
" `moda
|
|
64
|
+
"- Read the evidence before editing: `moda problems get <problem_id>` (dossier),",
|
|
65
|
+
" `moda problems get <problem_id> --evidence` (attribution rationales and anchors),",
|
|
66
|
+
" `moda problems get <problem_id> --traces` (affected traces),",
|
|
67
|
+
" `moda detections tools <tool_name>` (failing calls in context), and",
|
|
68
|
+
" `moda traces get <conversation_id>` (the messages around a turn; `--all` for",
|
|
69
69
|
" the whole trace).",
|
|
70
70
|
"- Propose the change in the code, prompt, tool definition, or skill that",
|
|
71
71
|
" causes the issue, and cite the Moda problem or trace ids it is based on",
|
|
72
72
|
" in the PR description.",
|
|
73
|
+
"- Test a prompt change on the Problem's failing traces before it ships (a paid",
|
|
74
|
+
" replay; prompt changes only today): `moda prompts ab --problem=<problem_id>",
|
|
75
|
+
" --baseline=<prompt_key> --candidate=<edited prompt file>`.",
|
|
73
76
|
"- Prompts live in `.moda/prompts.yml` (version 2: `prompt_paths` globs plus",
|
|
74
77
|
" `sources:` mappings for prompts defined in code); after `moda prompts sync`,",
|
|
75
78
|
" pass `prompt: lock.prompts['<key>']` to createThinkTelemetry/createAI7Telemetry.",
|
|
76
79
|
"- After the fix ships, record it with",
|
|
77
|
-
" `moda
|
|
80
|
+
" `moda problems feedback <problem_id> --action=mark_fixed`; if the issue",
|
|
78
81
|
" is detected again, the Problem comes back.",
|
|
79
82
|
FIXES_BLOCK_END
|
|
80
83
|
].join(`
|
|
@@ -111,7 +114,7 @@ function upsertManagedBlock(existing, block) {
|
|
|
111
114
|
const problem = begins === 1 && ends === 1 ? `its ${FIXES_BLOCK_END} marker comes before its ${FIXES_BLOCK_BEGIN} marker` : `it has ${begins} ${FIXES_BLOCK_BEGIN} and ${ends} ${FIXES_BLOCK_END} marker(s) instead of one of each`;
|
|
112
115
|
return {
|
|
113
116
|
ok: false,
|
|
114
|
-
reason: `The Moda block in ${AGENTS_MD_FILENAME} was not updated because ${problem}. ` + "The file was left unchanged so no text is lost. Keep exactly one begin marker followed by one end marker " + "(or delete the stray markers), then rerun `moda skills
|
|
117
|
+
reason: `The Moda block in ${AGENTS_MD_FILENAME} was not updated because ${problem}. ` + "The file was left unchanged so no text is lost. Keep exactly one begin marker followed by one end marker " + "(or delete the stray markers), then rerun `moda skills push`."
|
|
115
118
|
};
|
|
116
119
|
}
|
|
117
120
|
function writeFileIfChanged(filePath, content) {
|
|
@@ -757,7 +760,7 @@ async function runSkillsCommand(input) {
|
|
|
757
760
|
return runSkillsInstall(context, positionals.slice(1));
|
|
758
761
|
}
|
|
759
762
|
if (sub !== "pull") {
|
|
760
|
-
throw new CliInputError(sub ? `Unknown skills subcommand '${sub}'` : "A skills subcommand is required", "Usage: moda skills gen [--source=all|sdk] [--max-sessions=N (default 200)] [--lookback-hours=N (default 720)] [--max-traces=N] [--improvement-rounds=1-3] [--start-at=ISO] [--end-at=ISO] [--replay] [--replay-set-id=ID] [--reprocess-segments] [--force-reprocess] [--wait] | moda skills status [run-id] | moda skills proposals list [--status=ready_for_pr] | moda skills proposal apply <proposal-id> | moda skills pull [--status=approved|proposed|all] | moda skills
|
|
763
|
+
throw new CliInputError(sub ? `Unknown skills subcommand '${sub}'` : "A skills subcommand is required", "Usage: moda skills gen [--source=all|sdk] [--max-sessions=N (default 200)] [--lookback-hours=N (default 720)] [--max-traces=N] [--improvement-rounds=1-3] [--start-at=ISO] [--end-at=ISO] [--replay] [--replay-set-id=ID] [--reprocess-segments] [--force-reprocess] [--wait] | moda skills status [run-id] | moda skills proposals list [--status=ready_for_pr] | moda skills proposal apply <proposal-id> | moda skills pull [--status=approved|proposed|all] | moda skills push [--dry-run] | moda skills install [--list] <id...>");
|
|
761
764
|
}
|
|
762
765
|
const status = (flags.status ?? "approved").toLowerCase();
|
|
763
766
|
if (!VALID_STATUSES.has(status)) {
|
|
@@ -12,18 +12,19 @@ import {
|
|
|
12
12
|
require_typescript,
|
|
13
13
|
resolvePromptSource,
|
|
14
14
|
safePromptEvidencePath
|
|
15
|
-
} from "./cli-
|
|
15
|
+
} from "./cli-7xc585hc.js";
|
|
16
16
|
import {
|
|
17
17
|
createCommandContext
|
|
18
|
-
} from "./cli-
|
|
18
|
+
} from "./cli-qvg26twp.js";
|
|
19
19
|
import {
|
|
20
|
+
UUID_RE,
|
|
20
21
|
assertFlags,
|
|
21
22
|
assertPositionals
|
|
22
|
-
} from "./cli-
|
|
23
|
+
} from "./cli-dn91fb56.js";
|
|
23
24
|
import {
|
|
24
25
|
callControlAPI,
|
|
25
26
|
callDataAPI
|
|
26
|
-
} from "./cli-
|
|
27
|
+
} from "./cli-9btx12wp.js";
|
|
27
28
|
import {
|
|
28
29
|
CliInputError,
|
|
29
30
|
EXIT_ERROR,
|
|
@@ -31,7 +32,7 @@ import {
|
|
|
31
32
|
resolveApiTenantId,
|
|
32
33
|
setPlatformTenantOverride,
|
|
33
34
|
validateConfig
|
|
34
|
-
} from "./cli-
|
|
35
|
+
} from "./cli-dve50x7p.js";
|
|
35
36
|
import {
|
|
36
37
|
__commonJS,
|
|
37
38
|
__require,
|
|
@@ -5561,6 +5562,9 @@ var DEFAULT_SEEDS_PER_CASE = 3;
|
|
|
5561
5562
|
var DEFAULT_AUTO_GENERATE_CASES = 5;
|
|
5562
5563
|
var MAX_AUTO_GENERATE_CASES = 100;
|
|
5563
5564
|
var MAX_REPLAY_TRACES = 100;
|
|
5565
|
+
var DEFAULT_PROBLEM_MAX_CASES = 10;
|
|
5566
|
+
var PROBLEM_PAGE_LIMIT = 50;
|
|
5567
|
+
var PROBLEM_MAX_PAGES = 4;
|
|
5564
5568
|
var PLAYOUT_CONFIRM_THRESHOLD = 200;
|
|
5565
5569
|
var REPLAY_MODEL_ALLOWLIST = [
|
|
5566
5570
|
"openai/gpt-5.6-luna",
|
|
@@ -5589,7 +5593,7 @@ async function runPromptAb(flags, profileOptions, context) {
|
|
|
5589
5593
|
const baselineSource = flags.baseline || flags["baseline-file"] || flags["baseline-key"];
|
|
5590
5594
|
const candidateSource = flags.candidate || flags["candidate-file"] || flags["candidate-key"];
|
|
5591
5595
|
if (!baselineSource || !candidateSource) {
|
|
5592
|
-
throw new Error("Usage: moda prompts ab --baseline=<file|key> --candidate=<file|key> [--auto-generate [--cases=1-100]|--set-id=ID|--traces=id1,id2[@msg_index] (max 100)] [--seeds=1-10] [--model=<slug>] [--yes] (legacy alias: --conversations=)");
|
|
5596
|
+
throw new Error("Usage: moda prompts ab --baseline=<file|key> --candidate=<file|key> [--auto-generate [--cases=1-100]|--set-id=ID|--traces=id1,id2[@msg_index] (max 100)|--problem=<problem_id> [--max-cases=1-100]] [--seeds=1-10] [--model=<slug>] [--yes] (legacy alias: --conversations=)");
|
|
5593
5597
|
}
|
|
5594
5598
|
assertArmSourceShape(baselineSource, "baseline");
|
|
5595
5599
|
assertArmSourceShape(candidateSource, "candidate");
|
|
@@ -5599,6 +5603,10 @@ async function runPromptAb(flags, profileOptions, context) {
|
|
|
5599
5603
|
const pollIntervalMs = parsePositiveInt(flags["poll-interval"], DEFAULT_POLL_INTERVAL_MS, 120000, "--poll-interval");
|
|
5600
5604
|
const assistantModel = assertAllowedModel(flags.model ?? flags["assistant-model"], flags, "--model");
|
|
5601
5605
|
const plan = await planReplaySet(flags, tenantId, profileOptions);
|
|
5606
|
+
const problemCases = plan.kind === "traces" ? plan.problem : undefined;
|
|
5607
|
+
if (problemCases && context.outputMode === "human") {
|
|
5608
|
+
process.stderr.write(describeProblemCases(problemCases));
|
|
5609
|
+
}
|
|
5602
5610
|
const plannedPlayouts = plan.cases === null ? null : plan.cases * seedsPerCase * 2;
|
|
5603
5611
|
assertPlayoutBudget(plannedPlayouts, { seedsPerCase, cases: plan.cases }, context.yes, rerunWithYes("prompts ab", [], flags));
|
|
5604
5612
|
if (context.outputMode === "human") {
|
|
@@ -5644,6 +5652,7 @@ async function runPromptAb(flags, profileOptions, context) {
|
|
|
5644
5652
|
if (noWait) {
|
|
5645
5653
|
context.output.writeData({
|
|
5646
5654
|
replaySetId,
|
|
5655
|
+
...problemCases ? { problemCases } : {},
|
|
5647
5656
|
...caseSources ? { caseSources } : {},
|
|
5648
5657
|
...warnings.length ? { warnings } : {},
|
|
5649
5658
|
runId: enqueue.runId,
|
|
@@ -5673,6 +5682,7 @@ async function runPromptAb(flags, profileOptions, context) {
|
|
|
5673
5682
|
}
|
|
5674
5683
|
const result = {
|
|
5675
5684
|
replaySetId,
|
|
5685
|
+
...problemCases ? { problemCases } : {},
|
|
5676
5686
|
...caseSources ? { caseSources } : {},
|
|
5677
5687
|
...warnings.length ? { warnings } : {},
|
|
5678
5688
|
plannedPlayouts,
|
|
@@ -5801,6 +5811,9 @@ function resolvePromptArmSpec(source, label, opts = {}) {
|
|
|
5801
5811
|
};
|
|
5802
5812
|
}
|
|
5803
5813
|
async function planReplaySet(flags, tenantId, profileOptions) {
|
|
5814
|
+
const problemPlan = await planProblemCases(flags);
|
|
5815
|
+
if (problemPlan)
|
|
5816
|
+
return problemPlan;
|
|
5804
5817
|
const existingSetId = (flags["set-id"] || flags["replay-set-id"] || "").trim();
|
|
5805
5818
|
if (existingSetId) {
|
|
5806
5819
|
const snapshot = await fetchReplaySetCases(tenantId, existingSetId, profileOptions);
|
|
@@ -5821,6 +5834,151 @@ async function planReplaySet(flags, tenantId, profileOptions) {
|
|
|
5821
5834
|
cases: parseCappedInt(flags.cases ?? flags["case-count"], DEFAULT_AUTO_GENERATE_CASES, MAX_AUTO_GENERATE_CASES, "--cases")
|
|
5822
5835
|
};
|
|
5823
5836
|
}
|
|
5837
|
+
var PROBLEM_CASE_CONFLICTS = [
|
|
5838
|
+
["traces", "--traces"],
|
|
5839
|
+
["conversations", "--conversations"],
|
|
5840
|
+
["conversation-ids", "--conversation-ids"],
|
|
5841
|
+
["set-id", "--set-id"],
|
|
5842
|
+
["replay-set-id", "--replay-set-id"],
|
|
5843
|
+
["cases", "--cases"],
|
|
5844
|
+
["case-count", "--case-count"]
|
|
5845
|
+
];
|
|
5846
|
+
async function planProblemCases(flags) {
|
|
5847
|
+
const raw = flags.problem;
|
|
5848
|
+
if (raw === undefined) {
|
|
5849
|
+
if (flags["max-cases"] !== undefined) {
|
|
5850
|
+
throw new CliInputError("--max-cases applies only with --problem, so nothing was sent.", "Use --cases=N with --auto-generate, or pass --problem=<problem_id>.");
|
|
5851
|
+
}
|
|
5852
|
+
return null;
|
|
5853
|
+
}
|
|
5854
|
+
const problemId = raw.trim();
|
|
5855
|
+
const conflicts = PROBLEM_CASE_CONFLICTS.filter(([key]) => flags[key] !== undefined).map(([, name]) => name);
|
|
5856
|
+
if (flags["auto-generate"] === "true")
|
|
5857
|
+
conflicts.push("--auto-generate");
|
|
5858
|
+
if (conflicts.length) {
|
|
5859
|
+
throw new CliInputError(`--problem cannot be combined with ${conflicts.join(", ")}: each picks the replay cases, so nothing was sent.`, "Pass one of --problem=<problem_id> [--max-cases=N], --traces=<ids>, --set-id=<id>, or --auto-generate [--cases=N].");
|
|
5860
|
+
}
|
|
5861
|
+
if (!UUID_RE.test(problemId)) {
|
|
5862
|
+
throw new CliInputError(`--problem needs a canonical problem UUID; got "${problemId}", so nothing was sent.`, "Run `moda problems` to find the problem_id.");
|
|
5863
|
+
}
|
|
5864
|
+
const maxCases = parsePositiveInt(flags["max-cases"], DEFAULT_PROBLEM_MAX_CASES, MAX_REPLAY_TRACES, "--max-cases");
|
|
5865
|
+
const selection = await fetchProblemCases(problemId, maxCases);
|
|
5866
|
+
if (!selection.traces.length) {
|
|
5867
|
+
throw new CliInputError(`Problem ${problemId} has no affected traces to replay, so nothing was sent.`, `Check the id with \`moda problems get ${problemId}\`, or pass --traces=<ids>.`);
|
|
5868
|
+
}
|
|
5869
|
+
return {
|
|
5870
|
+
kind: "traces",
|
|
5871
|
+
traces: selection.traces.map((c) => c.msg_index === null ? c.conversation_id : `${c.conversation_id}@${c.msg_index}`),
|
|
5872
|
+
cases: selection.traces.length,
|
|
5873
|
+
problem: selection
|
|
5874
|
+
};
|
|
5875
|
+
}
|
|
5876
|
+
async function readProblemPages(problemId, view, enough, drain = false) {
|
|
5877
|
+
const rows = [];
|
|
5878
|
+
let enoughAt = null;
|
|
5879
|
+
let cursor = null;
|
|
5880
|
+
let resolvedId = null;
|
|
5881
|
+
let hasMore = false;
|
|
5882
|
+
for (let page = 0;page < PROBLEM_MAX_PAGES; page++) {
|
|
5883
|
+
const query = new URLSearchParams({ limit: String(PROBLEM_PAGE_LIMIT) });
|
|
5884
|
+
if (cursor)
|
|
5885
|
+
query.set("cursor", cursor);
|
|
5886
|
+
const data = await callDataAPI(`/problems/${encodeURIComponent(problemId)}/${view}?${query.toString()}`);
|
|
5887
|
+
if (typeof data?.problem_id === "string" && data.problem_id)
|
|
5888
|
+
resolvedId = data.problem_id;
|
|
5889
|
+
const pageRows = data?.[view];
|
|
5890
|
+
if (Array.isArray(pageRows))
|
|
5891
|
+
rows.push(...pageRows);
|
|
5892
|
+
const next = data?.pagination?.next_cursor;
|
|
5893
|
+
hasMore = data?.pagination?.has_more === true && typeof next === "string" && next !== "";
|
|
5894
|
+
if (enoughAt === null && enough(rows))
|
|
5895
|
+
enoughAt = rows.length;
|
|
5896
|
+
if (!hasMore || enoughAt !== null && !drain)
|
|
5897
|
+
break;
|
|
5898
|
+
cursor = next;
|
|
5899
|
+
}
|
|
5900
|
+
return { rows, enoughAt, resolvedId, hasMore };
|
|
5901
|
+
}
|
|
5902
|
+
async function fetchProblemCases(problemId, maxCases) {
|
|
5903
|
+
const anchored = (evidenceRows, traceRows2) => selectProblemCases(evidenceRows, traceRows2, MAX_REPLAY_TRACES).traces.filter((c) => c.msg_index !== null).length;
|
|
5904
|
+
const evidence = await readProblemPages(problemId, "evidence", (rows) => anchored(rows, []) >= maxCases, true);
|
|
5905
|
+
const choiceRows = evidence.enoughAt === null ? evidence.rows : evidence.rows.slice(0, evidence.enoughAt);
|
|
5906
|
+
let traceRows = [];
|
|
5907
|
+
let tracesHaveMore = false;
|
|
5908
|
+
let resolvedId = evidence.resolvedId;
|
|
5909
|
+
if (anchored(choiceRows, []) < maxCases) {
|
|
5910
|
+
const traces2 = await readProblemPages(problemId, "conversations", (rows) => anchored(evidence.rows, rows) >= maxCases);
|
|
5911
|
+
traceRows = traces2.rows;
|
|
5912
|
+
tracesHaveMore = traces2.hasMore;
|
|
5913
|
+
resolvedId ??= traces2.resolvedId;
|
|
5914
|
+
}
|
|
5915
|
+
const selected = selectProblemCases(choiceRows, traceRows, maxCases);
|
|
5916
|
+
const all = selectProblemCases(evidence.rows, traceRows, Number.MAX_SAFE_INTEGER);
|
|
5917
|
+
const earliestByTrace = new Map(all.traces.map((c) => [c.conversation_id, c]));
|
|
5918
|
+
const traces = selected.traces.map((c) => {
|
|
5919
|
+
const first = earliestByTrace.get(c.conversation_id);
|
|
5920
|
+
return first && first.msg_index !== null && (c.msg_index === null || first.msg_index < c.msg_index) ? first : c;
|
|
5921
|
+
});
|
|
5922
|
+
const found = all.found;
|
|
5923
|
+
const resolved = resolvedId ?? problemId;
|
|
5924
|
+
return {
|
|
5925
|
+
problemId: resolved,
|
|
5926
|
+
...resolved !== problemId ? { requestedProblemId: problemId } : {},
|
|
5927
|
+
maxCases,
|
|
5928
|
+
traces,
|
|
5929
|
+
found,
|
|
5930
|
+
moreAvailable: found > traces.length || evidence.hasMore || tracesHaveMore
|
|
5931
|
+
};
|
|
5932
|
+
}
|
|
5933
|
+
function anchorIndex(value) {
|
|
5934
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
5935
|
+
}
|
|
5936
|
+
function selectProblemCases(evidenceRows, traceRows, maxCases) {
|
|
5937
|
+
const byTrace = new Map;
|
|
5938
|
+
const consider = (conversationId, msgIndex, source, family) => {
|
|
5939
|
+
if (typeof conversationId !== "string" || !conversationId.trim())
|
|
5940
|
+
return;
|
|
5941
|
+
const id = conversationId.trim();
|
|
5942
|
+
const existing = byTrace.get(id);
|
|
5943
|
+
const signalFamily = typeof family === "string" && family ? family : undefined;
|
|
5944
|
+
if (existing && (msgIndex === null || existing.msg_index !== null && existing.msg_index <= msgIndex))
|
|
5945
|
+
return;
|
|
5946
|
+
byTrace.set(id, {
|
|
5947
|
+
conversation_id: id,
|
|
5948
|
+
msg_index: msgIndex,
|
|
5949
|
+
source,
|
|
5950
|
+
...signalFamily ? { signal_family: signalFamily } : {}
|
|
5951
|
+
});
|
|
5952
|
+
};
|
|
5953
|
+
for (const row of evidenceRows) {
|
|
5954
|
+
const r = row ?? {};
|
|
5955
|
+
consider(r.conversation_id, anchorIndex(r.anchor_msg_index), "evidence", r.signal_family);
|
|
5956
|
+
}
|
|
5957
|
+
for (const row of traceRows) {
|
|
5958
|
+
const r = row ?? {};
|
|
5959
|
+
const anchor = r.latest_anchor ?? {};
|
|
5960
|
+
consider(r.conversation_id, anchorIndex(anchor.anchor_msg_index), "trace", undefined);
|
|
5961
|
+
}
|
|
5962
|
+
const all = [...byTrace.values()];
|
|
5963
|
+
const ordered = [...all.filter((c) => c.msg_index !== null), ...all.filter((c) => c.msg_index === null)];
|
|
5964
|
+
return { traces: ordered.slice(0, maxCases), found: all.length };
|
|
5965
|
+
}
|
|
5966
|
+
function describeProblemCases(selection) {
|
|
5967
|
+
const requested = selection.requestedProblemId ? ` (requested ${selection.requestedProblemId})` : "";
|
|
5968
|
+
const lines = [
|
|
5969
|
+
`Replaying ${selection.traces.length} trace(s) from Problem ${selection.problemId}${requested}, each from its evidence anchor:`
|
|
5970
|
+
];
|
|
5971
|
+
for (const c of selection.traces) {
|
|
5972
|
+
const where = c.msg_index === null ? " (no anchor: replays from the first message)" : ` @ msg ${c.msg_index}`;
|
|
5973
|
+
lines.push(` ${c.conversation_id}${where}${c.signal_family ? ` [${c.signal_family}]` : ""}`);
|
|
5974
|
+
}
|
|
5975
|
+
if (selection.moreAvailable) {
|
|
5976
|
+
lines.push(`Capped at --max-cases=${selection.maxCases}; the Problem has more affected traces.`);
|
|
5977
|
+
}
|
|
5978
|
+
return `${lines.join(`
|
|
5979
|
+
`)}
|
|
5980
|
+
`;
|
|
5981
|
+
}
|
|
5824
5982
|
async function fetchReplaySetCases(tenantId, setId, profileOptions) {
|
|
5825
5983
|
try {
|
|
5826
5984
|
const set = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(setId)}`, { method: "GET" }, profileOptions);
|
|
@@ -5876,7 +6034,8 @@ async function ensureReplaySet(plan, flags, tenantId, profileOptions) {
|
|
|
5876
6034
|
if (plan.kind === "existing")
|
|
5877
6035
|
return { setId: plan.setId, warnings: [] };
|
|
5878
6036
|
if (plan.kind === "traces") {
|
|
5879
|
-
|
|
6037
|
+
const defaultName = plan.problem ? `Prompt A/B problem ${plan.problem.problemId.slice(0, 8)} (${plan.traces.length} trace(s))` : undefined;
|
|
6038
|
+
return createSetFromTraces(plan.traces, flags, tenantId, profileOptions, defaultName);
|
|
5880
6039
|
}
|
|
5881
6040
|
const caseCount = plan.cases;
|
|
5882
6041
|
const lookbackDays = parsePositiveInt(flags["lookback-days"], 30, 365, "--lookback-days");
|
|
@@ -5896,7 +6055,7 @@ function parseTraceRef(ref) {
|
|
|
5896
6055
|
return { conversationId: ref.trim() };
|
|
5897
6056
|
return { conversationId: match[1], startMsgIndex: Number(match[2]) };
|
|
5898
6057
|
}
|
|
5899
|
-
async function createSetFromTraces(conversationIds, flags, tenantId, profileOptions) {
|
|
6058
|
+
async function createSetFromTraces(conversationIds, flags, tenantId, profileOptions, defaultName) {
|
|
5900
6059
|
const refs = [...new Set(conversationIds)].map(parseTraceRef);
|
|
5901
6060
|
const traces = [...new Set(refs.map((ref) => ref.conversationId))];
|
|
5902
6061
|
const startByConversation = new Map;
|
|
@@ -5904,12 +6063,16 @@ async function createSetFromTraces(conversationIds, flags, tenantId, profileOpti
|
|
|
5904
6063
|
if (ref.startMsgIndex !== undefined)
|
|
5905
6064
|
startByConversation.set(ref.conversationId, ref.startMsgIndex);
|
|
5906
6065
|
}
|
|
5907
|
-
const name = (flags.name ?? flags["set-name"] ?? `Prompt A/B ${traces.length} trace(s)`).trim();
|
|
6066
|
+
const name = (flags.name ?? flags["set-name"] ?? defaultName ?? `Prompt A/B ${traces.length} trace(s)`).trim();
|
|
5908
6067
|
const base = `/tenants/${encodeURIComponent(tenantId)}/replay-sets`;
|
|
6068
|
+
const traceAnchors = traces.filter((id) => startByConversation.has(id)).map((id) => ({ conversationId: id, msgIndex: startByConversation.get(id) }));
|
|
5909
6069
|
let set = null;
|
|
5910
6070
|
let unavailable = null;
|
|
5911
6071
|
try {
|
|
5912
|
-
set = await callControlAPI(`${base}/auto-generate`, {
|
|
6072
|
+
set = await callControlAPI(`${base}/auto-generate`, {
|
|
6073
|
+
method: "POST",
|
|
6074
|
+
body: JSON.stringify({ name, conversationIds: traces, ...traceAnchors.length ? { traceAnchors } : {} })
|
|
6075
|
+
}, profileOptions, { timeoutMs: 300000, retries: 0 });
|
|
5913
6076
|
if (!set?.id)
|
|
5914
6077
|
throw new Error("Auto-generate did not return a replay set id");
|
|
5915
6078
|
} catch (error) {
|
|
@@ -5933,14 +6096,15 @@ async function createSetFromTraces(conversationIds, flags, tenantId, profileOpti
|
|
|
5933
6096
|
const update = {};
|
|
5934
6097
|
if (fallback.length && index !== undefined && existing.position !== index)
|
|
5935
6098
|
update.position = index;
|
|
5936
|
-
if (startMsgIndex !== undefined)
|
|
6099
|
+
if (startMsgIndex !== undefined && existing.sourceStartMsgIndex !== startMsgIndex) {
|
|
5937
6100
|
update.sourceStartMsgIndex = startMsgIndex;
|
|
6101
|
+
}
|
|
5938
6102
|
if (!Object.keys(update).length)
|
|
5939
6103
|
continue;
|
|
5940
6104
|
await callControlAPI(`${base}/${encodeURIComponent(set.id)}/cases/${encodeURIComponent(existing.id)}`, { method: "PUT", body: JSON.stringify(update) }, profileOptions);
|
|
5941
6105
|
}
|
|
5942
6106
|
for (const conversationId of fallback) {
|
|
5943
|
-
const scenario = await loadScenarioFromConversation(conversationId);
|
|
6107
|
+
const scenario = await loadScenarioFromConversation(conversationId, startByConversation.get(conversationId));
|
|
5944
6108
|
await callControlAPI(`${base}/${encodeURIComponent(set.id)}/cases`, {
|
|
5945
6109
|
method: "POST",
|
|
5946
6110
|
body: JSON.stringify({
|
|
@@ -5960,19 +6124,40 @@ async function createSetFromTraces(conversationIds, flags, tenantId, profileOpti
|
|
|
5960
6124
|
}
|
|
5961
6125
|
return { setId: set.id, caseSources: { distilled, fallback }, warnings };
|
|
5962
6126
|
}
|
|
5963
|
-
|
|
6127
|
+
var ANCHOR_REQUEST_MAX_READS = 20;
|
|
6128
|
+
var CONTEXT_WINDOW = 10;
|
|
6129
|
+
async function loadScenarioFromConversation(conversationId, startMsgIndex) {
|
|
6130
|
+
const anchored = startMsgIndex !== undefined && startMsgIndex > 0;
|
|
6131
|
+
const userRequest = (messages, atOrBefore) => messages.filter((message) => message.role === "user" && String(message.content ?? "").trim() && (atOrBefore === undefined || typeof message.index === "number" && message.index <= atOrBefore));
|
|
6132
|
+
const readContext = async (msgIndex, window2) => {
|
|
6133
|
+
const data = await callDataAPI(`/conversations/${encodeURIComponent(conversationId)}/context?msg_index=${msgIndex}&window=${window2}`);
|
|
6134
|
+
return data.context?.messages ?? data.messages ?? [];
|
|
6135
|
+
};
|
|
5964
6136
|
try {
|
|
5965
|
-
|
|
5966
|
-
|
|
5967
|
-
|
|
5968
|
-
|
|
5969
|
-
|
|
5970
|
-
|
|
6137
|
+
if (anchored) {
|
|
6138
|
+
let center = startMsgIndex;
|
|
6139
|
+
for (let read = 0;read < ANCHOR_REQUEST_MAX_READS; read++) {
|
|
6140
|
+
const lead = userRequest(await readContext(center, CONTEXT_WINDOW), startMsgIndex).pop();
|
|
6141
|
+
const request = String(lead?.content ?? "").trim();
|
|
6142
|
+
if (request) {
|
|
6143
|
+
return `Earlier in the conversation the user and the agent already talked; the replay keeps those turns as history. ` + `Now the user says: "${request}"
|
|
6144
|
+
|
|
6145
|
+
Help them with this request using the available tools.`;
|
|
6146
|
+
}
|
|
6147
|
+
if (center - CONTEXT_WINDOW <= 0)
|
|
6148
|
+
break;
|
|
6149
|
+
center = Math.max(0, center - (2 * CONTEXT_WINDOW + 1));
|
|
6150
|
+
}
|
|
6151
|
+
} else {
|
|
6152
|
+
const request = String(userRequest(await readContext(0, 5))[0]?.content ?? "").trim();
|
|
6153
|
+
if (request) {
|
|
6154
|
+
return `The user says: "${request}"
|
|
5971
6155
|
|
|
5972
6156
|
Replay the conversation faithfully and help them accomplish their goal.`;
|
|
6157
|
+
}
|
|
5973
6158
|
}
|
|
5974
6159
|
} catch {}
|
|
5975
|
-
return `Replay conversation ${conversationId}: help the user accomplish their original goal using the available tools.`;
|
|
6160
|
+
return anchored ? `Replay conversation ${conversationId} from message ${startMsgIndex}: help the user with their request at that point using the available tools.` : `Replay conversation ${conversationId}: help the user accomplish their original goal using the available tools.`;
|
|
5976
6161
|
}
|
|
5977
6162
|
async function fetchLatestRun(tenantId, replaySetId, runId, profileOptions) {
|
|
5978
6163
|
return callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/runs/latest?runId=${encodeURIComponent(runId)}`, { method: "GET" }, profileOptions, { timeoutMs: 60000, retries: 2 });
|
|
@@ -6505,7 +6690,9 @@ var PROMPT_WRITE_SUBCOMMAND_FLAGS = {
|
|
|
6505
6690
|
"set-id",
|
|
6506
6691
|
"set-name",
|
|
6507
6692
|
"tenant-id",
|
|
6508
|
-
"allow-any-model"
|
|
6693
|
+
"allow-any-model",
|
|
6694
|
+
"problem",
|
|
6695
|
+
"max-cases"
|
|
6509
6696
|
],
|
|
6510
6697
|
propose: [
|
|
6511
6698
|
"gate",
|
|
@@ -6729,7 +6916,7 @@ async function runPromptSyncExclusive(flags, profileOptions, options) {
|
|
|
6729
6916
|
const analyzedDefinitions = new Map;
|
|
6730
6917
|
if (flags["no-analyze"] !== "true") {
|
|
6731
6918
|
options.onProgress?.("Analyzing prompt sources before sync.");
|
|
6732
|
-
const { analyzeHarnessPrompts } = await import("./harness-
|
|
6919
|
+
const { analyzeHarnessPrompts } = await import("./harness-vzy7n7ea.js");
|
|
6733
6920
|
const result = await analyzeHarnessPrompts({
|
|
6734
6921
|
rootDir: process.cwd(),
|
|
6735
6922
|
existing: established.flatMap((prompt) => prompt.sourceDefinition ? [prompt.sourceDefinition.source] : []),
|
|
@@ -7059,7 +7246,7 @@ async function migratePromptManifest(options = {}) {
|
|
|
7059
7246
|
let addedSources = [];
|
|
7060
7247
|
let pendingSources = [];
|
|
7061
7248
|
if (options.discover) {
|
|
7062
|
-
const { analyzeHarnessPrompts } = await import("./harness-
|
|
7249
|
+
const { analyzeHarnessPrompts } = await import("./harness-vzy7n7ea.js");
|
|
7063
7250
|
const result = await analyzeHarnessPrompts({
|
|
7064
7251
|
rootDir: root,
|
|
7065
7252
|
existing: manifest.sources,
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
import {
|
|
2
2
|
isAuthSessionValid,
|
|
3
3
|
loadAuthSession
|
|
4
|
-
} from "./cli-
|
|
4
|
+
} from "./cli-0wr3483h.js";
|
|
5
5
|
import {
|
|
6
6
|
inspectPromptManifest
|
|
7
|
-
} from "./cli-
|
|
7
|
+
} from "./cli-m805msp6.js";
|
|
8
8
|
import {
|
|
9
9
|
readHarnessSyncState,
|
|
10
10
|
renderDoctorHuman,
|
|
11
11
|
renderStatusHuman,
|
|
12
12
|
scanHarness
|
|
13
|
-
} from "./cli-
|
|
13
|
+
} from "./cli-7xc585hc.js";
|
|
14
14
|
import {
|
|
15
15
|
callDataAPI
|
|
16
|
-
} from "./cli-
|
|
16
|
+
} from "./cli-9btx12wp.js";
|
|
17
17
|
import {
|
|
18
18
|
ensureSecretFilePermissions,
|
|
19
19
|
getActiveProfile,
|
|
@@ -27,7 +27,7 @@ import {
|
|
|
27
27
|
resolveSecretReference,
|
|
28
28
|
resolveTenantId,
|
|
29
29
|
secretFilePathFromRef
|
|
30
|
-
} from "./cli-
|
|
30
|
+
} from "./cli-dve50x7p.js";
|
|
31
31
|
|
|
32
32
|
// src/doctor.ts
|
|
33
33
|
import { createHash } from "node:crypto";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
createCliOutput,
|
|
3
3
|
outputModeFromFlags
|
|
4
|
-
} from "./cli-
|
|
4
|
+
} from "./cli-dve50x7p.js";
|
|
5
5
|
|
|
6
6
|
// src/command-context.ts
|
|
7
7
|
function flagEnabled(flags, key) {
|
|
@@ -48,15 +48,16 @@ function createCommandContext(input, options = {}) {
|
|
|
48
48
|
env: options.env ?? process.env,
|
|
49
49
|
outputMode,
|
|
50
50
|
output: createCliOutput({
|
|
51
|
-
command: options.commandName ?? input.command,
|
|
51
|
+
command: options.outputCommand ?? options.commandName ?? input.command,
|
|
52
52
|
mode: outputMode,
|
|
53
53
|
quiet,
|
|
54
|
-
invocation: {
|
|
54
|
+
invocation: options.invocation ?? {
|
|
55
55
|
command: options.commandName ?? input.command,
|
|
56
56
|
positionals: input.positionals,
|
|
57
57
|
flags: rawFlags,
|
|
58
58
|
...input.bareFlags ? { bareFlags: input.bareFlags } : {}
|
|
59
|
-
}
|
|
59
|
+
},
|
|
60
|
+
...options.deprecation ? { deprecation: options.deprecation } : {}
|
|
60
61
|
}),
|
|
61
62
|
profile: flags.profile,
|
|
62
63
|
dryRun: flagEnabled(flags, "dry-run"),
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import {
|
|
2
2
|
fetchApiKeyProbe
|
|
3
|
-
} from "./cli-
|
|
3
|
+
} from "./cli-9hc5bg2r.js";
|
|
4
4
|
import {
|
|
5
5
|
authFetch,
|
|
6
6
|
getCachedLoginContext
|
|
7
|
-
} from "./cli-
|
|
7
|
+
} from "./cli-0wr3483h.js";
|
|
8
8
|
import {
|
|
9
9
|
CliAccessError,
|
|
10
10
|
CliInputError,
|
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
resolveTenantIdFromActiveProfile,
|
|
19
19
|
saveCliConfig,
|
|
20
20
|
writeSecretJsonFile
|
|
21
|
-
} from "./cli-
|
|
21
|
+
} from "./cli-dve50x7p.js";
|
|
22
22
|
|
|
23
23
|
// src/init/tenant.ts
|
|
24
24
|
import { randomUUID } from "node:crypto";
|