@moda-ai/cli 1.55.27 → 1.55.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{agent-vp9prdma.js → agent-0w9rtdsw.js} +22 -16
- package/dist/cli-3n0gz4d0.js +17971 -0
- package/dist/{cli-q9nsvqqg.js → cli-3retgh64.js} +85 -1165
- package/dist/{cli-ccjfwzqx.js → cli-97jew729.js} +1 -1
- package/dist/{cli-r5pp7qny.js → cli-99p2jh28.js} +42 -456
- package/dist/{cli-0pkc8jhg.js → cli-9bgk5jeh.js} +8 -6
- package/dist/{cli-hmjdgxrn.js → cli-9bgnvth2.js} +5 -3
- package/dist/cli-9kfxkxmk.js +6994 -0
- package/dist/cli-cesds766.js +1516 -0
- package/dist/{cli-v8q7sd7g.js → cli-dr6x0zv7.js} +1 -1
- package/dist/{cli-pg5qfmqc.js → cli-e23g9ywa.js} +1 -1
- package/dist/{cli-vdkwkkp5.js → cli-fwpqghgc.js} +4 -2
- package/dist/cli-k4tqwah4.js +89 -0
- package/dist/{cli-6et4z504.js → cli-mg5zv4s6.js} +15 -13
- package/dist/{cli-wzds257x.js → cli-n2hng3gv.js} +6 -4
- package/dist/cli-ns5fcnkz.js +341 -0
- package/dist/{cli-4h0ave0f.js → cli-ny48a8de.js} +12 -9
- package/dist/{cli-qwgct6x7.js → cli-qpq84eee.js} +5220 -32716
- package/dist/cli-xq45etg1.js +1098 -0
- package/dist/cli-yf39avhh.js +754 -0
- package/dist/cli-yz8xcpyk.js +12 -0
- package/dist/cli.js +77 -153
- package/dist/doctor-k3ta6h2y.js +24 -0
- package/dist/{fix-75s9s0d6.js → fix-w5z02wt4.js} +8 -9
- package/dist/{harness-a7tb9a23.js → harness-7azmzytc.js} +15 -7
- package/dist/{harness-github-actions-6v4wydb8.js → harness-github-actions-bv2qrnqb.js} +11 -5
- package/dist/harness-report-rd9c4e9v.js +54 -0
- package/dist/{index-91bpc8g8.js → index-d1z50dsq.js} +41 -33
- package/dist/{production-intelligence-7nqddd4f.js → production-intelligence-wvsgjxth.js} +9 -6
- package/dist/{profile-commands-49yn02yg.js → profile-commands-h8evz9x0.js} +7 -4
- package/dist/prompt-attribution-1nytc9wj.js +148 -0
- package/dist/prompt-source-mcp.js +14 -7
- package/dist/prompt-sources-07tczyqa.js +19 -0
- package/dist/{prompts-17wyrst6.js → prompts-6y9wfq5z.js} +8 -7
- package/dist/prompts-ab-ezrd63sk.js +70 -0
- package/dist/prompts-propose-2k3enm2q.js +255 -0
- package/dist/{provision-bc3h68e1.js → provision-y5bdbh4y.js} +9 -6
- package/dist/{registry-fpzhjznj.js → registry-96aze5wp.js} +7 -4
- package/dist/{skills-ccdbqghj.js → skills-j8865009.js} +7 -5
- package/dist/{sync-hkxce5zh.js → sync-ew7yvy0e.js} +16 -10
- package/package.json +1 -1
- package/skills/integration/index.json +2 -2
- package/dist/doctor-wmre3mr7.js +0 -22
|
@@ -2,38 +2,24 @@ import {
|
|
|
2
2
|
computePromptContentHash,
|
|
3
3
|
stableJson
|
|
4
4
|
} from "./cli-mm48br00.js";
|
|
5
|
-
import {
|
|
6
|
-
$isSeq,
|
|
7
|
-
$parseDocument,
|
|
8
|
-
$stringify,
|
|
9
|
-
backfillPromptSource,
|
|
10
|
-
inventoryFiles,
|
|
11
|
-
normalizeHarnessReport,
|
|
12
|
-
require_typescript,
|
|
13
|
-
resolvePromptSource,
|
|
14
|
-
safePromptEvidencePath
|
|
15
|
-
} from "./cli-qwgct6x7.js";
|
|
16
5
|
import {
|
|
17
6
|
createCommandContext
|
|
18
|
-
} from "./cli-
|
|
7
|
+
} from "./cli-dr6x0zv7.js";
|
|
19
8
|
import {
|
|
20
|
-
UUID_RE,
|
|
21
9
|
assertFlags,
|
|
22
10
|
assertPositionals
|
|
23
|
-
} from "./cli-
|
|
11
|
+
} from "./cli-n2hng3gv.js";
|
|
12
|
+
import {
|
|
13
|
+
callControlAPI
|
|
14
|
+
} from "./cli-e23g9ywa.js";
|
|
24
15
|
import {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
} from "./cli-pg5qfmqc.js";
|
|
16
|
+
CliInputError
|
|
17
|
+
} from "./cli-k4tqwah4.js";
|
|
28
18
|
import {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
setPlatformTenantOverride,
|
|
34
|
-
shellArg,
|
|
35
|
-
validateConfig
|
|
36
|
-
} from "./cli-r5pp7qny.js";
|
|
19
|
+
$isSeq,
|
|
20
|
+
$parseDocument,
|
|
21
|
+
$stringify
|
|
22
|
+
} from "./cli-9kfxkxmk.js";
|
|
37
23
|
import {
|
|
38
24
|
__commonJS,
|
|
39
25
|
__require,
|
|
@@ -5545,1093 +5531,21 @@ var import_fast_glob = __toESM(require_out4(), 1);
|
|
|
5545
5531
|
import { execFileSync } from "node:child_process";
|
|
5546
5532
|
import { createHash } from "node:crypto";
|
|
5547
5533
|
import {
|
|
5548
|
-
existsSync
|
|
5534
|
+
existsSync,
|
|
5549
5535
|
lstatSync,
|
|
5550
5536
|
mkdirSync,
|
|
5551
5537
|
readFileSync,
|
|
5552
5538
|
realpathSync,
|
|
5553
5539
|
renameSync,
|
|
5554
5540
|
rmSync,
|
|
5555
|
-
writeFileSync
|
|
5541
|
+
writeFileSync
|
|
5556
5542
|
} from "node:fs";
|
|
5557
|
-
import { dirname
|
|
5558
|
-
|
|
5559
|
-
|
|
5560
|
-
|
|
5561
|
-
|
|
5562
|
-
var DEFAULT_SEEDS_PER_CASE = 3;
|
|
5563
|
-
var DEFAULT_AUTO_GENERATE_CASES = 5;
|
|
5564
|
-
var MAX_AUTO_GENERATE_CASES = 100;
|
|
5565
|
-
var MAX_REPLAY_TRACES = 100;
|
|
5566
|
-
var DEFAULT_PROBLEM_MAX_CASES = 10;
|
|
5567
|
-
var PROBLEM_PAGE_LIMIT = 50;
|
|
5568
|
-
var PROBLEM_MAX_PAGES = 4;
|
|
5569
|
-
var PLAYOUT_CONFIRM_THRESHOLD = 200;
|
|
5570
|
-
var REPLAY_MODEL_ALLOWLIST = [
|
|
5571
|
-
"openai/gpt-5.6-luna",
|
|
5572
|
-
"openai/gpt-4o-mini",
|
|
5573
|
-
"openai/gpt-4o",
|
|
5574
|
-
"anthropic/claude-sonnet-4-5",
|
|
5575
|
-
"anthropic/claude-haiku-4-5",
|
|
5576
|
-
"google/gemini-2.5-flash",
|
|
5577
|
-
"google/gemini-2.5-pro"
|
|
5578
|
-
];
|
|
5579
|
-
var DEFAULT_POLL_INTERVAL_MS = 15000;
|
|
5580
|
-
var DEFAULT_WAIT_TIMEOUT_MS = 2 * 60 * 60 * 1000;
|
|
5581
|
-
var TERMINAL_RUN_STATUSES = new Set(["completed", "skipped", "error"]);
|
|
5582
|
-
var FALLBACK_SUCCESS_CRITERIA = [
|
|
5583
|
-
"The agent understands the user request",
|
|
5584
|
-
"The agent uses tools appropriately when needed",
|
|
5585
|
-
"The agent provides a helpful final answer"
|
|
5586
|
-
];
|
|
5587
|
-
async function runPromptAb(flags, profileOptions, context) {
|
|
5588
|
-
validateConfig();
|
|
5589
|
-
const tenantId = flags["tenant-id"] || resolveApiTenantId(profileOptions);
|
|
5590
|
-
if (!tenantId) {
|
|
5591
|
-
throw new Error("Missing tenant id. Run `moda init` or pass --tenant-id=<id>.");
|
|
5592
|
-
}
|
|
5593
|
-
setPlatformTenantOverride(tenantId);
|
|
5594
|
-
const baselineSource = flags.baseline || flags["baseline-file"] || flags["baseline-key"];
|
|
5595
|
-
const candidateSource = flags.candidate || flags["candidate-file"] || flags["candidate-key"];
|
|
5596
|
-
if (!baselineSource || !candidateSource) {
|
|
5597
|
-
throw new Error("Usage: moda prompts ab --baseline=<file|key> --candidate=<file|key> [--auto-generate [--cases=1-100]|--set-id=ID|--traces=id1,id2[@msg_index] (max 100)|--problem=<problem_id> [--max-cases=1-100]] [--seeds=1-10] [--model=<slug>] [--yes] (legacy alias: --conversations=)");
|
|
5598
|
-
}
|
|
5599
|
-
assertArmSourceShape(baselineSource, "baseline");
|
|
5600
|
-
assertArmSourceShape(candidateSource, "candidate");
|
|
5601
|
-
const seedsPerCase = parsePositiveInt(flags.seeds ?? flags["seeds-per-case"], DEFAULT_SEEDS_PER_CASE, 10, flags.seeds !== undefined ? "--seeds" : "--seeds-per-case");
|
|
5602
|
-
parsePositiveInt(flags["lookback-days"], 30, 365, "--lookback-days");
|
|
5603
|
-
const timeoutMs = parsePositiveInt(flags.timeout, DEFAULT_WAIT_TIMEOUT_MS, 24 * 60 * 60 * 1000, "--timeout");
|
|
5604
|
-
const pollIntervalMs = parsePositiveInt(flags["poll-interval"], DEFAULT_POLL_INTERVAL_MS, 120000, "--poll-interval");
|
|
5605
|
-
const assistantModel = assertAllowedModel(flags.model ?? flags["assistant-model"], flags, "--model");
|
|
5606
|
-
const plan = await planReplaySet(flags, tenantId, profileOptions);
|
|
5607
|
-
const problemCases = plan.kind === "traces" ? plan.problem : undefined;
|
|
5608
|
-
if (problemCases && context.outputMode === "human") {
|
|
5609
|
-
process.stderr.write(describeProblemCases(problemCases));
|
|
5610
|
-
}
|
|
5611
|
-
const plannedPlayouts = plan.cases === null ? null : plan.cases * seedsPerCase * 2;
|
|
5612
|
-
assertPlayoutBudget(plannedPlayouts, { seedsPerCase, cases: plan.cases }, context.yes, rerunWithYes("prompts ab", [], flags));
|
|
5613
|
-
if (context.outputMode === "human") {
|
|
5614
|
-
process.stderr.write(`${describePlayouts(plannedPlayouts, plan.cases, seedsPerCase)}
|
|
5615
|
-
`);
|
|
5616
|
-
}
|
|
5617
|
-
if (flags.sync === "true") {
|
|
5618
|
-
for (const [source, label] of [[baselineSource, "baseline"], [candidateSource, "candidate"]]) {
|
|
5619
|
-
try {
|
|
5620
|
-
resolvePromptArmSpec(source, label);
|
|
5621
|
-
} catch (error) {
|
|
5622
|
-
if (!(error instanceof UnresolvedPromptError))
|
|
5623
|
-
throw error;
|
|
5624
|
-
}
|
|
5625
|
-
}
|
|
5626
|
-
await runPromptSync({ ...flags, watch: "false" }, profileOptions);
|
|
5627
|
-
}
|
|
5628
|
-
const promptArms = {
|
|
5629
|
-
prod: resolvePromptArmSpec(baselineSource, "baseline"),
|
|
5630
|
-
proposed: resolvePromptArmSpec(candidateSource, "candidate")
|
|
5631
|
-
};
|
|
5632
|
-
const { setId: replaySetId, caseSources, warnings } = await ensureReplaySet(plan, flags, tenantId, profileOptions);
|
|
5633
|
-
if (context.outputMode === "human") {
|
|
5634
|
-
for (const warning of warnings)
|
|
5635
|
-
process.stderr.write(`Warning: ${warning}
|
|
5636
|
-
`);
|
|
5637
|
-
}
|
|
5638
|
-
const writeOptions = warnings.length ? { warnings } : undefined;
|
|
5639
|
-
try {
|
|
5640
|
-
const enqueue = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/run`, {
|
|
5641
|
-
method: "POST",
|
|
5642
|
-
body: JSON.stringify({
|
|
5643
|
-
promptArms,
|
|
5644
|
-
seedsPerCase,
|
|
5645
|
-
assistantModel: assistantModel || undefined,
|
|
5646
|
-
promotePrimary: flags.promote === "true" && flags["no-promote"] !== "true",
|
|
5647
|
-
...plan.kind === "existing" ? replayCasePin(plan.caseIds) : {}
|
|
5648
|
-
})
|
|
5649
|
-
}, profileOptions, { timeoutMs: 120000, retries: 0 });
|
|
5650
|
-
if (context.outputMode === "human")
|
|
5651
|
-
warnIfQueued(enqueue);
|
|
5652
|
-
const noWait = flags.wait === "false" || flags["no-wait"] === "true";
|
|
5653
|
-
if (noWait) {
|
|
5654
|
-
context.output.writeData({
|
|
5655
|
-
replaySetId,
|
|
5656
|
-
...problemCases ? { problemCases } : {},
|
|
5657
|
-
...caseSources ? { caseSources } : {},
|
|
5658
|
-
...warnings.length ? { warnings } : {},
|
|
5659
|
-
runId: enqueue.runId,
|
|
5660
|
-
status: enqueue.status,
|
|
5661
|
-
message: enqueue.message ?? "Replay comparison queued",
|
|
5662
|
-
plannedPlayouts
|
|
5663
|
-
}, writeOptions);
|
|
5664
|
-
return EXIT_OK;
|
|
5665
|
-
}
|
|
5666
|
-
const deadline = Date.now() + timeoutMs;
|
|
5667
|
-
let latest = null;
|
|
5668
|
-
while (Date.now() < deadline) {
|
|
5669
|
-
latest = await fetchLatestRun(tenantId, replaySetId, enqueue.runId, profileOptions);
|
|
5670
|
-
const status = latest.run?.status ?? "";
|
|
5671
|
-
const runId = latest.run?.runId ?? "";
|
|
5672
|
-
if (TERMINAL_RUN_STATUSES.has(status) && runId === enqueue.runId) {
|
|
5673
|
-
break;
|
|
5674
|
-
}
|
|
5675
|
-
if (context.outputMode === "human") {
|
|
5676
|
-
process.stderr.write(`Waiting for replay run (${waitLabel(latest.run)})...
|
|
5677
|
-
`);
|
|
5678
|
-
}
|
|
5679
|
-
await sleep(pollIntervalMs);
|
|
5680
|
-
}
|
|
5681
|
-
if (!latest?.run || latest.run.runId !== enqueue.runId || !TERMINAL_RUN_STATUSES.has(latest.run.status)) {
|
|
5682
|
-
throw new Error(`Timed out after ${timeoutMs}ms waiting for replay run ${enqueue.runId} to finish`);
|
|
5683
|
-
}
|
|
5684
|
-
const result = {
|
|
5685
|
-
replaySetId,
|
|
5686
|
-
...problemCases ? { problemCases } : {},
|
|
5687
|
-
...caseSources ? { caseSources } : {},
|
|
5688
|
-
...warnings.length ? { warnings } : {},
|
|
5689
|
-
plannedPlayouts,
|
|
5690
|
-
runId: latest.run.runId,
|
|
5691
|
-
status: latest.run.status,
|
|
5692
|
-
verdict: formatVerdict(latest),
|
|
5693
|
-
run: { ...latest.run },
|
|
5694
|
-
cases: latest.cases.map((row) => ({ ...row }))
|
|
5695
|
-
};
|
|
5696
|
-
if (context.outputMode === "human") {
|
|
5697
|
-
printHumanVerdict(result);
|
|
5698
|
-
}
|
|
5699
|
-
const failed = result.status === "error";
|
|
5700
|
-
context.output.writeData(result, { ...writeOptions, ...failed ? { status: "error" } : {} });
|
|
5701
|
-
return failed ? EXIT_ERROR : EXIT_OK;
|
|
5702
|
-
} catch (error) {
|
|
5703
|
-
if (warnings.length && error && typeof error === "object") {
|
|
5704
|
-
Object.assign(error, { warnings });
|
|
5705
|
-
}
|
|
5706
|
-
throw error;
|
|
5707
|
-
}
|
|
5708
|
-
}
|
|
5709
|
-
function warnIfQueued(enqueue) {
|
|
5710
|
-
const { runsAhead, maxActiveRuns } = enqueue;
|
|
5711
|
-
if (typeof runsAhead !== "number" || typeof maxActiveRuns !== "number")
|
|
5712
|
-
return;
|
|
5713
|
-
if (runsAhead < maxActiveRuns)
|
|
5714
|
-
return;
|
|
5715
|
-
process.stderr.write(`Note: ${runsAhead} replay runs for this tenant are already running or queued ` + `(${maxActiveRuns} at a time); run ${enqueue.runId} waits in the queue until a slot frees.
|
|
5716
|
-
`);
|
|
5717
|
-
}
|
|
5718
|
-
function waitLabel(run) {
|
|
5719
|
-
if (!run?.status)
|
|
5720
|
-
return "pending";
|
|
5721
|
-
if (run.status === "accepted") {
|
|
5722
|
-
return typeof run.runsAhead === "number" ? `queued, ${run.runsAhead} run${run.runsAhead === 1 ? "" : "s"} ahead` : "queued";
|
|
5723
|
-
}
|
|
5724
|
-
return run.status;
|
|
5725
|
-
}
|
|
5726
|
-
var FILE_ARM_RE = /[\\/]|\.(?:md|txt|json|ya?ml|jinja2?|j2|hbs|tmpl|prompt|[cm]?[jt]sx?|py)$/i;
|
|
5727
|
-
var PROMPT_KEY_RE = /^[^\s"'`\x00-\x1f\x7f]+$/;
|
|
5728
|
-
function assertArmSourceShape(source, label) {
|
|
5729
|
-
const trimmed = source.trim();
|
|
5730
|
-
if (!trimmed) {
|
|
5731
|
-
throw new CliInputError(`Empty prompt source for ${label}, so nothing was sent.`);
|
|
5732
|
-
}
|
|
5733
|
-
const absPath = isAbsolute(trimmed) ? trimmed : resolve(process.cwd(), trimmed);
|
|
5734
|
-
const exists = existsSync(absPath);
|
|
5735
|
-
if (!exists && FILE_ARM_RE.test(trimmed)) {
|
|
5736
|
-
if (isDiscoveredKey(trimmed))
|
|
5737
|
-
return;
|
|
5738
|
-
throw new CliInputError(`--${label} file not found: ${trimmed}, so nothing was sent.`);
|
|
5739
|
-
}
|
|
5740
|
-
if (exists) {
|
|
5741
|
-
if (!statSync(absPath).isFile()) {
|
|
5742
|
-
throw new CliInputError(`--${label} is not a file: ${trimmed}, so nothing was sent.`);
|
|
5743
|
-
}
|
|
5744
|
-
try {
|
|
5745
|
-
accessSync(absPath, fsConstants.R_OK);
|
|
5746
|
-
} catch {
|
|
5747
|
-
throw new CliInputError(`--${label} file is not readable: ${trimmed}, so nothing was sent.`);
|
|
5748
|
-
}
|
|
5749
|
-
if (/\.(?:[cm]?[jt]sx?|py)$/i.test(absPath)) {
|
|
5750
|
-
throw new CliInputError(`--${label}=${trimmed} is a source module; source modules cannot be replayed as instructions, so nothing was sent.`, "Use the registered prompt key for this code source.");
|
|
5751
|
-
}
|
|
5752
|
-
return;
|
|
5753
|
-
}
|
|
5754
|
-
if (!PROMPT_KEY_RE.test(trimmed)) {
|
|
5755
|
-
throw new CliInputError(`--${label}=${JSON.stringify(trimmed)} is neither an existing prompt file nor a valid prompt key, so nothing was sent.`, "Pass a .prompt.md path or a prompt key (no spaces or quotes).");
|
|
5756
|
-
}
|
|
5757
|
-
}
|
|
5758
|
-
function isDiscoveredKey(key) {
|
|
5759
|
-
try {
|
|
5760
|
-
return discoverPrompts().some((prompt) => prompt.key === key);
|
|
5761
|
-
} catch {
|
|
5762
|
-
return false;
|
|
5763
|
-
}
|
|
5764
|
-
}
|
|
5765
|
-
|
|
5766
|
-
class UnresolvedPromptError extends Error {
|
|
5767
|
-
}
|
|
5768
|
-
function resolvePromptArmSpec(source, label, opts = {}) {
|
|
5769
|
-
const trimmed = source.trim();
|
|
5770
|
-
if (!trimmed) {
|
|
5771
|
-
throw new Error(`Empty prompt source for ${label}`);
|
|
5772
|
-
}
|
|
5773
|
-
const fromDiscovery = () => {
|
|
5774
|
-
const discovered = discoverPrompts();
|
|
5775
|
-
const matches = discovered.filter((prompt) => prompt.key === trimmed || prompt.sourcePath === trimmed);
|
|
5776
|
-
if (matches.length > 1)
|
|
5777
|
-
throw new Error(`Multiple prompts use '${trimmed}'; select a stable prompt key.`);
|
|
5778
|
-
const match = matches[0];
|
|
5779
|
-
if (match)
|
|
5780
|
-
assertReplayablePrompt(match);
|
|
5781
|
-
return match ? { key: match.key, content: match.content } : null;
|
|
5782
|
-
};
|
|
5783
|
-
const absPath = isAbsolute(trimmed) ? trimmed : resolve(process.cwd(), trimmed);
|
|
5784
|
-
let key;
|
|
5785
|
-
let content;
|
|
5786
|
-
const preferred = opts.preferKey ? fromDiscovery() : null;
|
|
5787
|
-
if (preferred) {
|
|
5788
|
-
key = preferred.key;
|
|
5789
|
-
content = preferred.content;
|
|
5790
|
-
} else if (opts.preferKey) {
|
|
5791
|
-
throw new Error(`Could not resolve prompt key '${trimmed}' — no discovered prompt matched this key`);
|
|
5792
|
-
} else if (existsSync(absPath)) {
|
|
5793
|
-
if (/\.(?:[cm]?[jt]sx?|py)$/.test(absPath))
|
|
5794
|
-
throw new Error("Use a registered prompt key for code sources; source modules cannot be replayed as instructions.");
|
|
5795
|
-
const parsed = parsePromptFileForAb(absPath);
|
|
5796
|
-
key = parsed.key;
|
|
5797
|
-
content = parsed.content;
|
|
5798
|
-
} else {
|
|
5799
|
-
const match = fromDiscovery();
|
|
5800
|
-
if (!match) {
|
|
5801
|
-
throw new UnresolvedPromptError(`Could not resolve prompt '${trimmed}' — pass a .prompt.md path or a discovered prompt key`);
|
|
5802
|
-
}
|
|
5803
|
-
key = match.key;
|
|
5804
|
-
content = match.content;
|
|
5805
|
-
}
|
|
5806
|
-
const locked = readPromptLock()?.prompts[key];
|
|
5807
|
-
return {
|
|
5808
|
-
content,
|
|
5809
|
-
prompt_version_id: locked?.versionId ?? "",
|
|
5810
|
-
prompt_id: locked?.promptId ?? "",
|
|
5811
|
-
prompt_label: label
|
|
5812
|
-
};
|
|
5813
|
-
}
|
|
5814
|
-
async function planReplaySet(flags, tenantId, profileOptions) {
|
|
5815
|
-
const problemPlan = await planProblemCases(flags);
|
|
5816
|
-
if (problemPlan)
|
|
5817
|
-
return problemPlan;
|
|
5818
|
-
const existingSetId = (flags["set-id"] || flags["replay-set-id"] || "").trim();
|
|
5819
|
-
if (existingSetId) {
|
|
5820
|
-
const snapshot = await fetchReplaySetCases(tenantId, existingSetId, profileOptions);
|
|
5821
|
-
return { kind: "existing", setId: existingSetId, cases: snapshot?.ids.length ?? null, caseIds: snapshot?.ids };
|
|
5822
|
-
}
|
|
5823
|
-
const conversations = [...new Set(parseCsv(flags.traces ?? flags.conversations ?? flags["conversation-ids"]))];
|
|
5824
|
-
if (conversations.length > MAX_REPLAY_TRACES) {
|
|
5825
|
-
throw new CliInputError(`--traces has ${conversations.length} ids; the limit is ${MAX_REPLAY_TRACES} per replay set, so nothing was sent.`, "Split the traces across several runs, or use --auto-generate.");
|
|
5826
|
-
}
|
|
5827
|
-
if (conversations.length) {
|
|
5828
|
-
return { kind: "traces", traces: conversations, cases: conversations.length };
|
|
5829
|
-
}
|
|
5830
|
-
if (flags["auto-generate"] === "false") {
|
|
5831
|
-
throw new Error("Provide --set-id=, --traces= (legacy alias: --conversations=), or allow --auto-generate (default)");
|
|
5832
|
-
}
|
|
5833
|
-
return {
|
|
5834
|
-
kind: "auto-generate",
|
|
5835
|
-
cases: parseCappedInt(flags.cases ?? flags["case-count"], DEFAULT_AUTO_GENERATE_CASES, MAX_AUTO_GENERATE_CASES, "--cases")
|
|
5836
|
-
};
|
|
5837
|
-
}
|
|
5838
|
-
var PROBLEM_CASE_CONFLICTS = [
|
|
5839
|
-
["traces", "--traces"],
|
|
5840
|
-
["conversations", "--conversations"],
|
|
5841
|
-
["conversation-ids", "--conversation-ids"],
|
|
5842
|
-
["set-id", "--set-id"],
|
|
5843
|
-
["replay-set-id", "--replay-set-id"],
|
|
5844
|
-
["cases", "--cases"],
|
|
5845
|
-
["case-count", "--case-count"]
|
|
5846
|
-
];
|
|
5847
|
-
async function planProblemCases(flags) {
|
|
5848
|
-
const raw = flags.problem;
|
|
5849
|
-
if (raw === undefined) {
|
|
5850
|
-
if (flags["max-cases"] !== undefined) {
|
|
5851
|
-
throw new CliInputError("--max-cases applies only with --problem, so nothing was sent.", "Use --cases=N with --auto-generate, or pass --problem=<problem_id>.");
|
|
5852
|
-
}
|
|
5853
|
-
return null;
|
|
5854
|
-
}
|
|
5855
|
-
const problemId = raw.trim();
|
|
5856
|
-
const conflicts = PROBLEM_CASE_CONFLICTS.filter(([key]) => flags[key] !== undefined).map(([, name]) => name);
|
|
5857
|
-
if (flags["auto-generate"] === "true")
|
|
5858
|
-
conflicts.push("--auto-generate");
|
|
5859
|
-
if (conflicts.length) {
|
|
5860
|
-
throw new CliInputError(`--problem cannot be combined with ${conflicts.join(", ")}: each picks the replay cases, so nothing was sent.`, "Pass one of --problem=<problem_id> [--max-cases=N], --traces=<ids>, --set-id=<id>, or --auto-generate [--cases=N].");
|
|
5861
|
-
}
|
|
5862
|
-
if (!UUID_RE.test(problemId)) {
|
|
5863
|
-
throw new CliInputError(`--problem needs a canonical problem UUID; got "${problemId}", so nothing was sent.`, "Run `moda problems` to find the problem_id.");
|
|
5864
|
-
}
|
|
5865
|
-
const maxCases = parsePositiveInt(flags["max-cases"], DEFAULT_PROBLEM_MAX_CASES, MAX_REPLAY_TRACES, "--max-cases");
|
|
5866
|
-
const selection = await fetchProblemCases(problemId, maxCases);
|
|
5867
|
-
if (!selection.traces.length) {
|
|
5868
|
-
throw new CliInputError(`Problem ${problemId} has no affected traces to replay, so nothing was sent.`, `Check the id with \`moda problems get ${problemId}\`, or pass --traces=<ids>.`);
|
|
5869
|
-
}
|
|
5870
|
-
return {
|
|
5871
|
-
kind: "traces",
|
|
5872
|
-
traces: selection.traces.map((c) => c.msg_index === null ? c.conversation_id : `${c.conversation_id}@${c.msg_index}`),
|
|
5873
|
-
cases: selection.traces.length,
|
|
5874
|
-
problem: selection
|
|
5875
|
-
};
|
|
5876
|
-
}
|
|
5877
|
-
async function readProblemPages(problemId, view, enough, drain = false) {
|
|
5878
|
-
const rows = [];
|
|
5879
|
-
let enoughAt = null;
|
|
5880
|
-
let cursor = null;
|
|
5881
|
-
let resolvedId = null;
|
|
5882
|
-
let hasMore = false;
|
|
5883
|
-
for (let page = 0;page < PROBLEM_MAX_PAGES; page++) {
|
|
5884
|
-
const query = new URLSearchParams({ limit: String(PROBLEM_PAGE_LIMIT) });
|
|
5885
|
-
if (cursor)
|
|
5886
|
-
query.set("cursor", cursor);
|
|
5887
|
-
const data = await callDataAPI(`/problems/${encodeURIComponent(problemId)}/${view}?${query.toString()}`);
|
|
5888
|
-
if (typeof data?.problem_id === "string" && data.problem_id)
|
|
5889
|
-
resolvedId = data.problem_id;
|
|
5890
|
-
const pageRows = data?.[view];
|
|
5891
|
-
if (Array.isArray(pageRows))
|
|
5892
|
-
rows.push(...pageRows);
|
|
5893
|
-
const next = data?.pagination?.next_cursor;
|
|
5894
|
-
hasMore = data?.pagination?.has_more === true && typeof next === "string" && next !== "";
|
|
5895
|
-
if (enoughAt === null && enough(rows))
|
|
5896
|
-
enoughAt = rows.length;
|
|
5897
|
-
if (!hasMore || enoughAt !== null && !drain)
|
|
5898
|
-
break;
|
|
5899
|
-
cursor = next;
|
|
5900
|
-
}
|
|
5901
|
-
return { rows, enoughAt, resolvedId, hasMore };
|
|
5902
|
-
}
|
|
5903
|
-
async function fetchProblemCases(problemId, maxCases) {
|
|
5904
|
-
const anchored = (evidenceRows, traceRows2) => selectProblemCases(evidenceRows, traceRows2, MAX_REPLAY_TRACES).traces.filter((c) => c.msg_index !== null).length;
|
|
5905
|
-
const evidence = await readProblemPages(problemId, "evidence", (rows) => anchored(rows, []) >= maxCases, true);
|
|
5906
|
-
const choiceRows = evidence.enoughAt === null ? evidence.rows : evidence.rows.slice(0, evidence.enoughAt);
|
|
5907
|
-
let traceRows = [];
|
|
5908
|
-
let tracesHaveMore = false;
|
|
5909
|
-
let resolvedId = evidence.resolvedId;
|
|
5910
|
-
if (anchored(choiceRows, []) < maxCases) {
|
|
5911
|
-
const traces2 = await readProblemPages(problemId, "conversations", (rows) => anchored(evidence.rows, rows) >= maxCases);
|
|
5912
|
-
traceRows = traces2.rows;
|
|
5913
|
-
tracesHaveMore = traces2.hasMore;
|
|
5914
|
-
resolvedId ??= traces2.resolvedId;
|
|
5915
|
-
}
|
|
5916
|
-
const selected = selectProblemCases(choiceRows, traceRows, maxCases);
|
|
5917
|
-
const all = selectProblemCases(evidence.rows, traceRows, Number.MAX_SAFE_INTEGER);
|
|
5918
|
-
const earliestByTrace = new Map(all.traces.map((c) => [c.conversation_id, c]));
|
|
5919
|
-
const traces = selected.traces.map((c) => {
|
|
5920
|
-
const first = earliestByTrace.get(c.conversation_id);
|
|
5921
|
-
return first && first.msg_index !== null && (c.msg_index === null || first.msg_index < c.msg_index) ? first : c;
|
|
5922
|
-
});
|
|
5923
|
-
const found = all.found;
|
|
5924
|
-
const resolved = resolvedId ?? problemId;
|
|
5925
|
-
return {
|
|
5926
|
-
problemId: resolved,
|
|
5927
|
-
...resolved !== problemId ? { requestedProblemId: problemId } : {},
|
|
5928
|
-
maxCases,
|
|
5929
|
-
traces,
|
|
5930
|
-
found,
|
|
5931
|
-
moreAvailable: found > traces.length || evidence.hasMore || tracesHaveMore
|
|
5932
|
-
};
|
|
5933
|
-
}
|
|
5934
|
-
function anchorIndex(value) {
|
|
5935
|
-
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
5936
|
-
}
|
|
5937
|
-
function selectProblemCases(evidenceRows, traceRows, maxCases) {
|
|
5938
|
-
const byTrace = new Map;
|
|
5939
|
-
const consider = (conversationId, msgIndex, source, family) => {
|
|
5940
|
-
if (typeof conversationId !== "string" || !conversationId.trim())
|
|
5941
|
-
return;
|
|
5942
|
-
const id = conversationId.trim();
|
|
5943
|
-
const existing = byTrace.get(id);
|
|
5944
|
-
const signalFamily = typeof family === "string" && family ? family : undefined;
|
|
5945
|
-
if (existing && (msgIndex === null || existing.msg_index !== null && existing.msg_index <= msgIndex))
|
|
5946
|
-
return;
|
|
5947
|
-
byTrace.set(id, {
|
|
5948
|
-
conversation_id: id,
|
|
5949
|
-
msg_index: msgIndex,
|
|
5950
|
-
source,
|
|
5951
|
-
...signalFamily ? { signal_family: signalFamily } : {}
|
|
5952
|
-
});
|
|
5953
|
-
};
|
|
5954
|
-
for (const row of evidenceRows) {
|
|
5955
|
-
const r = row ?? {};
|
|
5956
|
-
consider(r.conversation_id, anchorIndex(r.anchor_msg_index), "evidence", r.signal_family);
|
|
5957
|
-
}
|
|
5958
|
-
for (const row of traceRows) {
|
|
5959
|
-
const r = row ?? {};
|
|
5960
|
-
const anchor = r.latest_anchor ?? {};
|
|
5961
|
-
consider(r.conversation_id, anchorIndex(anchor.anchor_msg_index), "trace", undefined);
|
|
5962
|
-
}
|
|
5963
|
-
const all = [...byTrace.values()];
|
|
5964
|
-
const ordered = [...all.filter((c) => c.msg_index !== null), ...all.filter((c) => c.msg_index === null)];
|
|
5965
|
-
return { traces: ordered.slice(0, maxCases), found: all.length };
|
|
5966
|
-
}
|
|
5967
|
-
function describeProblemCases(selection) {
|
|
5968
|
-
const requested = selection.requestedProblemId ? ` (requested ${selection.requestedProblemId})` : "";
|
|
5969
|
-
const lines = [
|
|
5970
|
-
`Replaying ${selection.traces.length} trace(s) from Problem ${selection.problemId}${requested}, each from its evidence anchor:`
|
|
5971
|
-
];
|
|
5972
|
-
for (const c of selection.traces) {
|
|
5973
|
-
const where = c.msg_index === null ? " (no anchor: replays from the first message)" : ` @ msg ${c.msg_index}`;
|
|
5974
|
-
lines.push(` ${c.conversation_id}${where}${c.signal_family ? ` [${c.signal_family}]` : ""}`);
|
|
5975
|
-
}
|
|
5976
|
-
if (selection.moreAvailable) {
|
|
5977
|
-
lines.push(`Capped at --max-cases=${selection.maxCases}; the Problem has more affected traces.`);
|
|
5978
|
-
}
|
|
5979
|
-
return `${lines.join(`
|
|
5980
|
-
`)}
|
|
5981
|
-
`;
|
|
5982
|
-
}
|
|
5983
|
-
async function fetchReplaySetCases(tenantId, setId, profileOptions) {
|
|
5984
|
-
try {
|
|
5985
|
-
const set = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(setId)}`, { method: "GET" }, profileOptions);
|
|
5986
|
-
if (!Array.isArray(set?.cases))
|
|
5987
|
-
return null;
|
|
5988
|
-
return {
|
|
5989
|
-
ids: set.cases.filter((c) => String(c?.scenario ?? "").trim()).map((c, index) => String(c?.id ?? `case_${index}`))
|
|
5990
|
-
};
|
|
5991
|
-
} catch (error) {
|
|
5992
|
-
if (error?.statusCode === 404) {
|
|
5993
|
-
throw new CliInputError(`Replay set '${setId}' not found for tenant ${tenantId}, so nothing was sent.`, "Check --set-id (list sets in the dashboard, or omit it to auto-generate).");
|
|
5994
|
-
}
|
|
5995
|
-
return null;
|
|
5996
|
-
}
|
|
5997
|
-
}
|
|
5998
|
-
function replayCasePin(caseIds) {
|
|
5999
|
-
if (!caseIds)
|
|
6000
|
-
return {};
|
|
6001
|
-
return caseIds.length <= REPLAY_CASE_IDS_PIN_MAX ? { caseIds, expectedCases: caseIds.length } : { expectedCases: caseIds.length };
|
|
6002
|
-
}
|
|
6003
|
-
var REPLAY_CASE_IDS_PIN_MAX = 500;
|
|
6004
|
-
function describePlayouts(planned, cases, seeds) {
|
|
6005
|
-
return planned === null ? `Planned replay playouts: unknown (case count unavailable) × ${seeds} seeds × 2 arms` : `Planned replay playouts: ${planned} (${cases} cases × ${seeds} seeds × 2 arms)`;
|
|
6006
|
-
}
|
|
6007
|
-
function assertPlayoutBudget(planned, shape, yes, rerun) {
|
|
6008
|
-
if (yes)
|
|
6009
|
-
return;
|
|
6010
|
-
if (planned !== null && planned <= PLAYOUT_CONFIRM_THRESHOLD)
|
|
6011
|
-
return;
|
|
6012
|
-
const what = planned === null ? "an unknown number of replay playouts (the replay set's case count could not be read)" : `${planned} replay playouts (${shape.cases} cases × ${shape.seedsPerCase} seeds × 2 arms)`;
|
|
6013
|
-
throw new CliInputError(`This would run ${what}, above the ${PLAYOUT_CONFIRM_THRESHOLD}-playout limit that needs confirmation. Nothing was sent.`, `Re-run with --yes to confirm: ${rerun}`);
|
|
6014
|
-
}
|
|
6015
|
-
function assertAllowedModel(raw, flags, flagName) {
|
|
6016
|
-
const model = (raw ?? "").trim();
|
|
6017
|
-
if (!model || flags["allow-any-model"] === "true")
|
|
6018
|
-
return model;
|
|
6019
|
-
if (REPLAY_MODEL_ALLOWLIST.includes(model))
|
|
6020
|
-
return model;
|
|
6021
|
-
throw new CliInputError(`${flagName}=${model} is not in the replay model allowlist, so nothing was sent.`, `Allowed: ${REPLAY_MODEL_ALLOWLIST.join(", ")}. Pass --allow-any-model to use another OpenRouter slug.`);
|
|
6022
|
-
}
|
|
6023
|
-
function rerunWithYes(command, positionals, flags) {
|
|
6024
|
-
const parts = ["moda", ...command.split(" "), ...positionals.map(shellArg)];
|
|
6025
|
-
for (const [key, value] of Object.entries(flags)) {
|
|
6026
|
-
if (key === "yes")
|
|
6027
|
-
continue;
|
|
6028
|
-
parts.push(value === "true" ? `--${key}` : `--${key}=${shellArg(value)}`);
|
|
6029
|
-
}
|
|
6030
|
-
parts.push("--yes");
|
|
6031
|
-
return parts.join(" ");
|
|
6032
|
-
}
|
|
6033
|
-
async function ensureReplaySet(plan, flags, tenantId, profileOptions) {
|
|
6034
|
-
if (plan.kind === "existing")
|
|
6035
|
-
return { setId: plan.setId, warnings: [] };
|
|
6036
|
-
if (plan.kind === "traces") {
|
|
6037
|
-
const defaultName = plan.problem ? `Prompt A/B problem ${plan.problem.problemId.slice(0, 8)} (${plan.traces.length} trace(s))` : undefined;
|
|
6038
|
-
return createSetFromTraces(plan.traces, flags, tenantId, profileOptions, defaultName);
|
|
6039
|
-
}
|
|
6040
|
-
const caseCount = plan.cases;
|
|
6041
|
-
const lookbackDays = parsePositiveInt(flags["lookback-days"], 30, 365, "--lookback-days");
|
|
6042
|
-
const name = (flags.name ?? flags["set-name"] ?? `Prompt A/B ${new Date().toISOString().slice(0, 10)}`).trim();
|
|
6043
|
-
const generated = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/auto-generate`, {
|
|
6044
|
-
method: "POST",
|
|
6045
|
-
body: JSON.stringify({ name, caseCount, lookbackDays })
|
|
6046
|
-
}, profileOptions, { timeoutMs: 300000, retries: 0 });
|
|
6047
|
-
if (!generated?.id) {
|
|
6048
|
-
throw new Error("Auto-generate did not return a replay set id");
|
|
6049
|
-
}
|
|
6050
|
-
return { setId: generated.id, warnings: [] };
|
|
6051
|
-
}
|
|
6052
|
-
function parseTraceRef(ref) {
|
|
6053
|
-
const match = /^(.+)@(\d+)$/.exec(ref.trim());
|
|
6054
|
-
if (!match)
|
|
6055
|
-
return { conversationId: ref.trim() };
|
|
6056
|
-
return { conversationId: match[1], startMsgIndex: Number(match[2]) };
|
|
6057
|
-
}
|
|
6058
|
-
async function createSetFromTraces(conversationIds, flags, tenantId, profileOptions, defaultName) {
|
|
6059
|
-
const refs = [...new Set(conversationIds)].map(parseTraceRef);
|
|
6060
|
-
const traces = [...new Set(refs.map((ref) => ref.conversationId))];
|
|
6061
|
-
const startByConversation = new Map;
|
|
6062
|
-
for (const ref of refs) {
|
|
6063
|
-
if (ref.startMsgIndex !== undefined)
|
|
6064
|
-
startByConversation.set(ref.conversationId, ref.startMsgIndex);
|
|
6065
|
-
}
|
|
6066
|
-
const name = (flags.name ?? flags["set-name"] ?? defaultName ?? `Prompt A/B ${traces.length} trace(s)`).trim();
|
|
6067
|
-
const base = `/tenants/${encodeURIComponent(tenantId)}/replay-sets`;
|
|
6068
|
-
const traceAnchors = traces.filter((id) => startByConversation.has(id)).map((id) => ({ conversationId: id, msgIndex: startByConversation.get(id) }));
|
|
6069
|
-
let set = null;
|
|
6070
|
-
let unavailable = null;
|
|
6071
|
-
try {
|
|
6072
|
-
set = await callControlAPI(`${base}/auto-generate`, {
|
|
6073
|
-
method: "POST",
|
|
6074
|
-
body: JSON.stringify({ name, conversationIds: traces, ...traceAnchors.length ? { traceAnchors } : {} })
|
|
6075
|
-
}, profileOptions, { timeoutMs: 300000, retries: 0 });
|
|
6076
|
-
if (!set?.id)
|
|
6077
|
-
throw new Error("Auto-generate did not return a replay set id");
|
|
6078
|
-
} catch (error) {
|
|
6079
|
-
const status = error?.statusCode;
|
|
6080
|
-
if (status === 401 || status === 403)
|
|
6081
|
-
throw error;
|
|
6082
|
-
unavailable = error instanceof Error ? error.message : String(error);
|
|
6083
|
-
set = null;
|
|
6084
|
-
}
|
|
6085
|
-
const distilledIds = new Set((set?.cases ?? []).map((c) => String(c?.sourceConversationId ?? "")).filter(Boolean));
|
|
6086
|
-
const distilled = traces.filter((id) => distilledIds.has(id));
|
|
6087
|
-
const fallback = traces.filter((id) => !distilledIds.has(id));
|
|
6088
|
-
if (!set) {
|
|
6089
|
-
set = await callControlAPI(base, { method: "POST", body: JSON.stringify({ name, description: "Created by moda prompts ab" }) }, profileOptions);
|
|
6090
|
-
}
|
|
6091
|
-
const requestedIndex = new Map(traces.map((id, index) => [id, index]));
|
|
6092
|
-
for (const existing of set.cases ?? []) {
|
|
6093
|
-
const conversationId = String(existing?.sourceConversationId ?? "");
|
|
6094
|
-
const index = requestedIndex.get(conversationId);
|
|
6095
|
-
const startMsgIndex = startByConversation.get(conversationId);
|
|
6096
|
-
const update = {};
|
|
6097
|
-
if (fallback.length && index !== undefined && existing.position !== index)
|
|
6098
|
-
update.position = index;
|
|
6099
|
-
if (startMsgIndex !== undefined && existing.sourceStartMsgIndex !== startMsgIndex) {
|
|
6100
|
-
update.sourceStartMsgIndex = startMsgIndex;
|
|
6101
|
-
}
|
|
6102
|
-
if (!Object.keys(update).length)
|
|
6103
|
-
continue;
|
|
6104
|
-
await callControlAPI(`${base}/${encodeURIComponent(set.id)}/cases/${encodeURIComponent(existing.id)}`, { method: "PUT", body: JSON.stringify(update) }, profileOptions);
|
|
6105
|
-
}
|
|
6106
|
-
for (const conversationId of fallback) {
|
|
6107
|
-
const scenario = await loadScenarioFromConversation(conversationId, startByConversation.get(conversationId));
|
|
6108
|
-
await callControlAPI(`${base}/${encodeURIComponent(set.id)}/cases`, {
|
|
6109
|
-
method: "POST",
|
|
6110
|
-
body: JSON.stringify({
|
|
6111
|
-
title: conversationId,
|
|
6112
|
-
scenario,
|
|
6113
|
-
sourceConversationId: conversationId,
|
|
6114
|
-
...startByConversation.has(conversationId) ? { sourceStartMsgIndex: startByConversation.get(conversationId) } : {},
|
|
6115
|
-
successCriteria: FALLBACK_SUCCESS_CRITERIA,
|
|
6116
|
-
position: requestedIndex.get(conversationId)
|
|
6117
|
-
})
|
|
6118
|
-
}, profileOptions);
|
|
6119
|
-
}
|
|
6120
|
-
const warnings = [];
|
|
6121
|
-
if (fallback.length) {
|
|
6122
|
-
const why = unavailable ? `Moda could not distill these traces (${unavailable.slice(0, 600)})` : "Moda could not distill a scenario for these traces";
|
|
6123
|
-
warnings.push(`${why}, so ${fallback.length} of ${traces.length} case(s) use only the opening user message and three generic ` + `success criteria, which long traces pass easily: ${fallback.slice(0, 10).join(", ")}${fallback.length > 10 ? ", …" : ""}. ` + "Edit their scenario/criteria in the dashboard, or pass --set-id with a curated set.");
|
|
6124
|
-
}
|
|
6125
|
-
return { setId: set.id, caseSources: { distilled, fallback }, warnings };
|
|
6126
|
-
}
|
|
6127
|
-
var ANCHOR_REQUEST_MAX_READS = 20;
|
|
6128
|
-
var CONTEXT_WINDOW = 10;
|
|
6129
|
-
async function loadScenarioFromConversation(conversationId, startMsgIndex) {
|
|
6130
|
-
const anchored = startMsgIndex !== undefined && startMsgIndex > 0;
|
|
6131
|
-
const userRequest = (messages, atOrBefore) => messages.filter((message) => message.role === "user" && String(message.content ?? "").trim() && (atOrBefore === undefined || typeof message.index === "number" && message.index <= atOrBefore));
|
|
6132
|
-
const readContext = async (msgIndex, window2) => {
|
|
6133
|
-
const data = await callDataAPI(`/conversations/${encodeURIComponent(conversationId)}/context?msg_index=${msgIndex}&window=${window2}`);
|
|
6134
|
-
return data.context?.messages ?? data.messages ?? [];
|
|
6135
|
-
};
|
|
6136
|
-
try {
|
|
6137
|
-
if (anchored) {
|
|
6138
|
-
let center = startMsgIndex;
|
|
6139
|
-
for (let read = 0;read < ANCHOR_REQUEST_MAX_READS; read++) {
|
|
6140
|
-
const lead = userRequest(await readContext(center, CONTEXT_WINDOW), startMsgIndex).pop();
|
|
6141
|
-
const request = String(lead?.content ?? "").trim();
|
|
6142
|
-
if (request) {
|
|
6143
|
-
return `Earlier in the conversation the user and the agent already talked; the replay keeps those turns as history. ` + `Now the user says: "${request}"
|
|
6144
|
-
|
|
6145
|
-
Help them with this request using the available tools.`;
|
|
6146
|
-
}
|
|
6147
|
-
if (center - CONTEXT_WINDOW <= 0)
|
|
6148
|
-
break;
|
|
6149
|
-
center = Math.max(0, center - (2 * CONTEXT_WINDOW + 1));
|
|
6150
|
-
}
|
|
6151
|
-
} else {
|
|
6152
|
-
const request = String(userRequest(await readContext(0, 5))[0]?.content ?? "").trim();
|
|
6153
|
-
if (request) {
|
|
6154
|
-
return `The user says: "${request}"
|
|
6155
|
-
|
|
6156
|
-
Replay the conversation faithfully and help them accomplish their goal.`;
|
|
6157
|
-
}
|
|
6158
|
-
}
|
|
6159
|
-
} catch {}
|
|
6160
|
-
return anchored ? `Replay conversation ${conversationId} from message ${startMsgIndex}: help the user with their request at that point using the available tools.` : `Replay conversation ${conversationId}: help the user accomplish their original goal using the available tools.`;
|
|
6161
|
-
}
|
|
6162
|
-
async function fetchLatestRun(tenantId, replaySetId, runId, profileOptions) {
|
|
6163
|
-
return callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/runs/latest?runId=${encodeURIComponent(runId)}`, { method: "GET" }, profileOptions, { timeoutMs: 60000, retries: 2 });
|
|
6164
|
-
}
|
|
6165
|
-
function candidateOutcome(payload) {
|
|
6166
|
-
const run = payload.run;
|
|
6167
|
-
if (!run || run.status !== "completed")
|
|
6168
|
-
return "no-run";
|
|
6169
|
-
const delta = run.proposedPassCount - run.prodPassCount;
|
|
6170
|
-
if (delta > 0)
|
|
6171
|
-
return "candidate";
|
|
6172
|
-
if (delta < 0)
|
|
6173
|
-
return "baseline";
|
|
6174
|
-
return "tie";
|
|
6175
|
-
}
|
|
6176
|
-
function formatVerdict(payload) {
|
|
6177
|
-
const run = payload.run;
|
|
6178
|
-
if (!run)
|
|
6179
|
-
return "no run";
|
|
6180
|
-
if (run.status === "skipped")
|
|
6181
|
-
return `skipped: ${run.notes || "no reason"}`;
|
|
6182
|
-
if (run.status === "error")
|
|
6183
|
-
return `error: ${run.notes || "unknown error"}`;
|
|
6184
|
-
if (run.status !== "completed")
|
|
6185
|
-
return `status=${run.status}`;
|
|
6186
|
-
const delta = run.proposedPassCount - run.prodPassCount;
|
|
6187
|
-
const winner = delta > 0 ? "candidate wins" : delta < 0 ? "baseline wins" : "tie on case passes";
|
|
6188
|
-
const rates = `baseline ${run.prodPassCount}/${run.caseCount} (${pct(run.prodPassRate)}), candidate ${run.proposedPassCount}/${run.caseCount} (${pct(run.proposedPassRate)})`;
|
|
6189
|
-
const notes = run.notes?.trim();
|
|
6190
|
-
return `${winner}; ${rates}${notes ? `; ${notes}` : ""}`;
|
|
6191
|
-
}
|
|
6192
|
-
function printHumanVerdict(result) {
|
|
6193
|
-
process.stderr.write(`
|
|
6194
|
-
Prompt A/B replay verdict
|
|
6195
|
-
`);
|
|
6196
|
-
process.stderr.write(`-------------------------
|
|
6197
|
-
`);
|
|
6198
|
-
process.stderr.write(`Replay set: ${result.replaySetId}
|
|
6199
|
-
`);
|
|
6200
|
-
process.stderr.write(`Run id: ${result.runId}
|
|
6201
|
-
`);
|
|
6202
|
-
process.stderr.write(`Status: ${result.status}
|
|
6203
|
-
`);
|
|
6204
|
-
process.stderr.write(`Verdict: ${result.verdict}
|
|
6205
|
-
`);
|
|
6206
|
-
if (result.cases.length) {
|
|
6207
|
-
process.stderr.write(`
|
|
6208
|
-
Per-case:
|
|
6209
|
-
`);
|
|
6210
|
-
for (const row of result.cases) {
|
|
6211
|
-
const caseId = String(row.caseId ?? "");
|
|
6212
|
-
const prod = formatPass(row.prodPass);
|
|
6213
|
-
const proposed = formatPass(row.proposedPass);
|
|
6214
|
-
process.stderr.write(` ${caseId}: baseline=${prod} candidate=${proposed}
|
|
6215
|
-
`);
|
|
6216
|
-
}
|
|
6217
|
-
}
|
|
6218
|
-
process.stderr.write(`
|
|
6219
|
-
`);
|
|
6220
|
-
}
|
|
6221
|
-
function formatPass(value) {
|
|
6222
|
-
if (value === null || value === undefined)
|
|
6223
|
-
return "abstained";
|
|
6224
|
-
return value ? "pass" : "fail";
|
|
6225
|
-
}
|
|
6226
|
-
function pct(rate) {
|
|
6227
|
-
return `${Math.round(Math.max(0, Math.min(1, rate)) * 100)}%`;
|
|
6228
|
-
}
|
|
6229
|
-
async function enqueueAndPollComparison(args, profileOptions) {
|
|
6230
|
-
const enqueue = await callControlAPI(`/tenants/${encodeURIComponent(args.tenantId)}/replay-sets/${encodeURIComponent(args.replaySetId)}/run`, {
|
|
6231
|
-
method: "POST",
|
|
6232
|
-
body: JSON.stringify({
|
|
6233
|
-
promptArms: args.promptArms,
|
|
6234
|
-
seedsPerCase: args.seedsPerCase ?? DEFAULT_SEEDS_PER_CASE,
|
|
6235
|
-
assistantModel: args.assistantModel || undefined,
|
|
6236
|
-
promotePrimary: args.promotePrimary ?? false,
|
|
6237
|
-
...replayCasePin(args.caseIds)
|
|
6238
|
-
})
|
|
6239
|
-
}, profileOptions, { timeoutMs: 120000, retries: 0 });
|
|
6240
|
-
if (args.onWaiting)
|
|
6241
|
-
warnIfQueued(enqueue);
|
|
6242
|
-
const timeoutMs = args.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS;
|
|
6243
|
-
const pollIntervalMs = args.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
6244
|
-
const deadline = Date.now() + timeoutMs;
|
|
6245
|
-
let latest = null;
|
|
6246
|
-
while (Date.now() < deadline) {
|
|
6247
|
-
latest = await fetchLatestRun(args.tenantId, args.replaySetId, enqueue.runId, profileOptions);
|
|
6248
|
-
const status = latest.run?.status ?? "";
|
|
6249
|
-
const runId = latest.run?.runId ?? "";
|
|
6250
|
-
if (TERMINAL_RUN_STATUSES.has(status) && runId === enqueue.runId) {
|
|
6251
|
-
return latest;
|
|
6252
|
-
}
|
|
6253
|
-
args.onWaiting?.(waitLabel(latest.run));
|
|
6254
|
-
await sleep(pollIntervalMs);
|
|
6255
|
-
}
|
|
6256
|
-
if (!latest?.run || latest.run.runId !== enqueue.runId || !TERMINAL_RUN_STATUSES.has(latest.run.status)) {
|
|
6257
|
-
throw new Error(`Timed out after ${timeoutMs}ms waiting for replay run ${enqueue.runId} to finish`);
|
|
6258
|
-
}
|
|
6259
|
-
return latest;
|
|
6260
|
-
}
|
|
6261
|
-
function parsePositiveInt(raw, fallback, max, flagName) {
|
|
6262
|
-
if (raw === undefined)
|
|
6263
|
-
return fallback;
|
|
6264
|
-
const text = String(raw).trim();
|
|
6265
|
-
const parsed = Number(text);
|
|
6266
|
-
if (!/^\d+$/.test(text) || !Number.isSafeInteger(parsed) || parsed < 1) {
|
|
6267
|
-
throw new CliInputError(`${flagName}=${raw} must be a whole number of at least 1, so nothing was sent.`, `Use ${flagName}=N with 1 <= N <= ${max}.`);
|
|
6268
|
-
}
|
|
6269
|
-
if (parsed > max) {
|
|
6270
|
-
throw new CliInputError(`${flagName}=${parsed} is above the maximum of ${max}, so nothing was sent.`, `Use ${flagName}=${max} or less.`);
|
|
6271
|
-
}
|
|
6272
|
-
return parsed;
|
|
6273
|
-
}
|
|
6274
|
-
function parseCappedInt(raw, fallback, max, flagName) {
|
|
6275
|
-
return parsePositiveInt(raw, fallback, max, flagName);
|
|
6276
|
-
}
|
|
6277
|
-
function parseCsv(raw) {
|
|
6278
|
-
if (!raw?.trim())
|
|
6279
|
-
return [];
|
|
6280
|
-
return raw.split(",").map((part) => part.trim()).filter(Boolean);
|
|
6281
|
-
}
|
|
6282
|
-
function sleep(ms) {
|
|
6283
|
-
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
|
6284
|
-
}
|
|
6285
|
-
|
|
6286
|
-
// src/prompts-propose.ts
|
|
6287
|
-
import { writeFileSync } from "node:fs";
|
|
6288
|
-
import { isAbsolute as isAbsolute2, resolve as resolve2 } from "node:path";
|
|
6289
|
-
var PROPOSE_TIMEOUT_MS = 300000;
|
|
6290
|
-
async function runPromptPropose(positionals, flags, profileOptions, context) {
|
|
6291
|
-
validateConfig();
|
|
6292
|
-
const promptKey = flags["prompt-key"] || positionals[1];
|
|
6293
|
-
const fromRunId = flags["from-run"] || flags["from-run-id"] || flags["run-id"];
|
|
6294
|
-
const replaySetId = flags["set-id"] || flags["replay-set-id"];
|
|
6295
|
-
if (!promptKey || !fromRunId || !replaySetId) {
|
|
6296
|
-
throw new Error("Usage: moda prompts propose --prompt-key=<key> --from-run=<run_id> --set-id=<replay_set_id> [--max-dossiers=8] [--model=<slug>] [--out=path.prompt.md] [--gate [--seeds=1-10] [--assistant-model=<slug>] [--yes]] [--promote-on-win] [--allow-any-model]");
|
|
6297
|
-
}
|
|
6298
|
-
const maxDossiers = parsePositiveInt2(flags["max-dossiers"], 8, 16, "--max-dossiers");
|
|
6299
|
-
parsePositiveInt2(flags["gate-timeout"] ?? flags.timeout, 2 * 60 * 60 * 1000, 24 * 60 * 60 * 1000, flags["gate-timeout"] !== undefined ? "--gate-timeout" : "--timeout");
|
|
6300
|
-
parsePositiveInt2(flags["poll-interval"], 15000, 120000, "--poll-interval");
|
|
6301
|
-
const model = assertAllowedModel(flags.model ?? flags["revise-model"], flags, "--model");
|
|
6302
|
-
const wantGate = flags.gate === "true";
|
|
6303
|
-
let gatePlan;
|
|
6304
|
-
if (wantGate) {
|
|
6305
|
-
const tenantId = flags["tenant-id"] || resolveApiTenantId(profileOptions);
|
|
6306
|
-
if (!tenantId) {
|
|
6307
|
-
throw new Error("Missing tenant id for --gate. Run `moda init` or pass --tenant-id=<id>.");
|
|
6308
|
-
}
|
|
6309
|
-
const seedsPerCase = parsePositiveInt2(flags.seeds ?? flags["seeds-per-case"], 3, 10, flags.seeds !== undefined ? "--seeds" : "--seeds-per-case");
|
|
6310
|
-
const assistantModel = assertAllowedModel(flags["assistant-model"], flags, "--assistant-model");
|
|
6311
|
-
const snapshot = await fetchGateSetCases(tenantId, replaySetId, profileOptions);
|
|
6312
|
-
const cases = snapshot?.ids.length ?? null;
|
|
6313
|
-
const plannedPlayouts = cases === null ? null : cases * seedsPerCase * 2;
|
|
6314
|
-
assertPlayoutBudget(plannedPlayouts, { seedsPerCase, cases }, context.yes, rerunWithYes("prompts propose", positionals.slice(1), flags));
|
|
6315
|
-
if (context.outputMode === "human") {
|
|
6316
|
-
process.stderr.write(`Gate: ${describePlayouts(plannedPlayouts, cases, seedsPerCase)}
|
|
6317
|
-
`);
|
|
6318
|
-
}
|
|
6319
|
-
gatePlan = { tenantId, seedsPerCase, assistantModel, plannedPlayouts, cases, caseIds: snapshot?.ids };
|
|
6320
|
-
}
|
|
6321
|
-
const result = await callControlAPI(`/prompts/${encodeURIComponent(promptKey)}/propose`, {
|
|
6322
|
-
method: "POST",
|
|
6323
|
-
body: JSON.stringify({ fromRunId, replaySetId, maxDossiers, model: model || undefined })
|
|
6324
|
-
}, profileOptions, { timeoutMs: PROPOSE_TIMEOUT_MS, retries: 0 });
|
|
6325
|
-
let outPath;
|
|
6326
|
-
if (result.status === "proposed" && result.content && flags.out) {
|
|
6327
|
-
outPath = isAbsolute2(flags.out) ? flags.out : resolve2(process.cwd(), flags.out);
|
|
6328
|
-
writeFileSync(outPath, result.content, "utf8");
|
|
6329
|
-
}
|
|
6330
|
-
const promoteOnWin = flags["promote-on-win"] === "true";
|
|
6331
|
-
let gate;
|
|
6332
|
-
if (gatePlan) {
|
|
6333
|
-
if (result.status !== "proposed" || !result.content) {
|
|
6334
|
-
gate = {
|
|
6335
|
-
ran: false,
|
|
6336
|
-
scope: "full-set",
|
|
6337
|
-
scopeNote: `no candidate to gate (${result.status})`,
|
|
6338
|
-
verdict: "skipped",
|
|
6339
|
-
outcome: "no-run",
|
|
6340
|
-
runId: null,
|
|
6341
|
-
promoted: false,
|
|
6342
|
-
plannedPlayouts: gatePlan.plannedPlayouts
|
|
6343
|
-
};
|
|
6344
|
-
} else {
|
|
6345
|
-
gate = await runGate({
|
|
6346
|
-
result,
|
|
6347
|
-
promptKey,
|
|
6348
|
-
replaySetId,
|
|
6349
|
-
flags,
|
|
6350
|
-
plan: gatePlan,
|
|
6351
|
-
promoteOnWin,
|
|
6352
|
-
profileOptions,
|
|
6353
|
-
context
|
|
6354
|
-
});
|
|
6355
|
-
}
|
|
6356
|
-
}
|
|
6357
|
-
if (context.outputMode === "human") {
|
|
6358
|
-
printHuman(result, outPath, gate);
|
|
6359
|
-
}
|
|
6360
|
-
context.output.writeData({
|
|
6361
|
-
...result,
|
|
6362
|
-
...outPath ? { outPath } : {},
|
|
6363
|
-
...gate ? { gate } : {}
|
|
6364
|
-
});
|
|
6365
|
-
}
|
|
6366
|
-
async function fetchGateSetCases(tenantId, replaySetId, profileOptions) {
|
|
6367
|
-
let set;
|
|
6368
|
-
try {
|
|
6369
|
-
set = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}`, { method: "GET" }, profileOptions);
|
|
6370
|
-
} catch (error) {
|
|
6371
|
-
if (error?.statusCode === 404) {
|
|
6372
|
-
throw new CliInputError(`Replay set '${replaySetId}' not found for tenant ${tenantId}, so nothing was sent.`, "Check --set-id: use the replay set the --from-run run was scored on (see `moda prompts replay-runs <key>`).");
|
|
6373
|
-
}
|
|
6374
|
-
return null;
|
|
6375
|
-
}
|
|
6376
|
-
if (!Array.isArray(set?.cases))
|
|
6377
|
-
return null;
|
|
6378
|
-
return {
|
|
6379
|
-
ids: set.cases.filter((c) => String(c?.scenario ?? "").trim()).map((c, index) => String(c?.id ?? `case_${index}`))
|
|
6380
|
-
};
|
|
6381
|
-
}
|
|
6382
|
-
async function runGate(args) {
|
|
6383
|
-
const { result, promptKey, replaySetId, flags, plan, promoteOnWin, profileOptions, context } = args;
|
|
6384
|
-
const { tenantId, seedsPerCase, assistantModel } = plan;
|
|
6385
|
-
const prodArm = resolvePromptArmSpec(promptKey, "baseline", { preferKey: true });
|
|
6386
|
-
const proposedArm = {
|
|
6387
|
-
content: result.content ?? "",
|
|
6388
|
-
prompt_version_id: result.versionId ?? "",
|
|
6389
|
-
prompt_id: "",
|
|
6390
|
-
prompt_label: "candidate"
|
|
6391
|
-
};
|
|
6392
|
-
const timeoutMs = parsePositiveInt2(flags["gate-timeout"] ?? flags.timeout, 2 * 60 * 60 * 1000, 24 * 60 * 60 * 1000, flags["gate-timeout"] !== undefined ? "--gate-timeout" : "--timeout");
|
|
6393
|
-
const pollIntervalMs = parsePositiveInt2(flags["poll-interval"], 15000, 120000, "--poll-interval");
|
|
6394
|
-
const scopeNote = result.holdoutCaseIds.length ? `gated over the FULL replay set (${result.holdoutCaseIds.length} holdout case(s) could not be isolated — the run endpoint scores every case)` : "gated over the FULL replay set";
|
|
6395
|
-
if (context.outputMode === "human") {
|
|
6396
|
-
process.stderr.write(`
|
|
6397
|
-
Gating candidate for '${promptKey}' — ${scopeNote}...
|
|
6398
|
-
`);
|
|
6399
|
-
}
|
|
6400
|
-
const payload = await enqueueAndPollComparison({
|
|
6401
|
-
tenantId,
|
|
6402
|
-
replaySetId,
|
|
6403
|
-
promptArms: { prod: prodArm, proposed: proposedArm },
|
|
6404
|
-
seedsPerCase,
|
|
6405
|
-
assistantModel: assistantModel || undefined,
|
|
6406
|
-
promotePrimary: false,
|
|
6407
|
-
caseIds: plan.caseIds,
|
|
6408
|
-
timeoutMs,
|
|
6409
|
-
pollIntervalMs,
|
|
6410
|
-
onWaiting: context.outputMode === "human" ? (status) => process.stderr.write(`Waiting for gate run (${status})...
|
|
6411
|
-
`) : undefined
|
|
6412
|
-
}, profileOptions);
|
|
6413
|
-
const outcome = candidateOutcome(payload);
|
|
6414
|
-
const gate = {
|
|
6415
|
-
ran: true,
|
|
6416
|
-
scope: "full-set",
|
|
6417
|
-
scopeNote,
|
|
6418
|
-
verdict: formatVerdict(payload),
|
|
6419
|
-
outcome,
|
|
6420
|
-
runId: payload.run?.runId ?? null,
|
|
6421
|
-
promoted: false,
|
|
6422
|
-
plannedPlayouts: plan.plannedPlayouts
|
|
6423
|
-
};
|
|
6424
|
-
if (promoteOnWin) {
|
|
6425
|
-
if (outcome === "candidate" && result.versionId) {
|
|
6426
|
-
await callControlAPI(`/prompts/${encodeURIComponent(promptKey)}/labels`, { method: "POST", body: JSON.stringify({ label: "prod", versionId: result.versionId }) }, profileOptions);
|
|
6427
|
-
gate.promoted = true;
|
|
6428
|
-
} else {
|
|
6429
|
-
gate.promoteSkippedReason = outcome === "candidate" ? "no candidate versionId to promote" : `candidate did not strictly win (${outcome})`;
|
|
6430
|
-
}
|
|
6431
|
-
}
|
|
6432
|
-
return gate;
|
|
6433
|
-
}
|
|
6434
|
-
function printHuman(result, outPath, gate) {
|
|
6435
|
-
const w = (s) => process.stderr.write(s + `
|
|
6436
|
-
`);
|
|
6437
|
-
if (result.status !== "proposed") {
|
|
6438
|
-
w(`No candidate produced (${result.status}): ${result.note}`);
|
|
6439
|
-
return;
|
|
6440
|
-
}
|
|
6441
|
-
w(`Proposed candidate for '${result.promptKey}'`);
|
|
6442
|
-
w(` new version: ${result.versionId} (unlabeled — not promoted)`);
|
|
6443
|
-
w(` base version: ${result.baseVersionId ?? "(none)"}`);
|
|
6444
|
-
w(` from run: built from ${result.dossierCount} failure dossier(s)`);
|
|
6445
|
-
if (result.changelog)
|
|
6446
|
-
w(` changelog: ${result.changelog}`);
|
|
6447
|
-
if (result.risks.length)
|
|
6448
|
-
w(` risks: ${result.risks.map((r) => `- ${r}`).join(`
|
|
6449
|
-
`)}`);
|
|
6450
|
-
if (outPath)
|
|
6451
|
-
w(` written to: ${outPath}`);
|
|
6452
|
-
if (result.lowN) {
|
|
6453
|
-
w("");
|
|
6454
|
-
w(` ⚠ low sample: ${result.lowN}`);
|
|
6455
|
-
}
|
|
6456
|
-
w("");
|
|
6457
|
-
if (gate?.ran) {
|
|
6458
|
-
w("Gate (automatic A/B of candidate vs baseline):");
|
|
6459
|
-
w(` scope: ${gate.scopeNote}`);
|
|
6460
|
-
w(` verdict: ${gate.verdict}`);
|
|
6461
|
-
if (gate.plannedPlayouts !== undefined)
|
|
6462
|
-
w(` playouts: ${gate.plannedPlayouts ?? "unknown"} planned`);
|
|
6463
|
-
if (gate.runId)
|
|
6464
|
-
w(` run id: ${gate.runId}`);
|
|
6465
|
-
if (gate.promoted) {
|
|
6466
|
-
w(` promoted: candidate won — promoted version ${result.versionId} to label 'prod'`);
|
|
6467
|
-
} else if (gate.promoteSkippedReason) {
|
|
6468
|
-
w(` promote: skipped (${gate.promoteSkippedReason})`);
|
|
6469
|
-
}
|
|
6470
|
-
w("");
|
|
6471
|
-
if (!gate.promoted) {
|
|
6472
|
-
w(` moda prompts promote ${shellArg(result.promptKey)} --label=prod --version=${result.versionId ? shellArg(result.versionId) : "<version_id>"}`);
|
|
6473
|
-
}
|
|
6474
|
-
return;
|
|
6475
|
-
}
|
|
6476
|
-
if (gate && !gate.ran) {
|
|
6477
|
-
w(`Gate not run: ${gate.scopeNote}`);
|
|
6478
|
-
w("");
|
|
6479
|
-
}
|
|
6480
|
-
w("Next — gate on holdout cases the revision never saw, then promote if it wins:");
|
|
6481
|
-
if (result.holdoutCaseIds.length) {
|
|
6482
|
-
w(` holdout cases (${result.holdoutCaseIds.length}): ${result.holdoutCaseIds.join(", ")}`);
|
|
6483
|
-
} else {
|
|
6484
|
-
w(" (too few failures to reserve a holdout — gate on a fresh --auto-generate set)");
|
|
6485
|
-
}
|
|
6486
|
-
if (outPath) {
|
|
6487
|
-
w(` moda prompts ab --baseline-key=${shellArg(result.promptKey)} --candidate=${shellArg(outPath)} --auto-generate`);
|
|
6488
|
-
} else {
|
|
6489
|
-
w(` # re-run with --out=<path.prompt.md> to materialize the candidate, then:`);
|
|
6490
|
-
w(` moda prompts ab --baseline-key=${shellArg(result.promptKey)} --candidate=<path.prompt.md> --auto-generate`);
|
|
6491
|
-
}
|
|
6492
|
-
w(` # or: moda prompts propose ... --gate --promote-on-win (auto-A/B, promote on a strict win)`);
|
|
6493
|
-
w(` moda prompts promote ${shellArg(result.promptKey)} --label=prod --version=${result.versionId ? shellArg(result.versionId) : "<version_id>"}`);
|
|
6494
|
-
}
|
|
6495
|
-
function parsePositiveInt2(raw, fallback, max, flagName) {
|
|
6496
|
-
if (raw === undefined)
|
|
6497
|
-
return fallback;
|
|
6498
|
-
const text = String(raw).trim();
|
|
6499
|
-
const parsed = Number(text);
|
|
6500
|
-
if (!/^\d+$/.test(text) || !Number.isSafeInteger(parsed) || parsed < 1) {
|
|
6501
|
-
throw new CliInputError(`${flagName}=${raw} must be a whole number of at least 1, so nothing was sent.`, `Use ${flagName}=N with 1 <= N <= ${max}.`);
|
|
6502
|
-
}
|
|
6503
|
-
if (parsed > max) {
|
|
6504
|
-
throw new CliInputError(`${flagName}=${parsed} is above the maximum of ${max}, so nothing was sent.`, `Use ${flagName}=${max} or less.`);
|
|
6505
|
-
}
|
|
6506
|
-
return parsed;
|
|
6507
|
-
}
|
|
6508
|
-
|
|
6509
|
-
// src/prompt-attribution.ts
|
|
6510
|
-
var import_typescript = __toESM(require_typescript(), 1);
|
|
6511
|
-
import { dirname, extname, resolve as resolve3 } from "node:path";
|
|
6512
|
-
function checkPromptAttribution(rootDir, keys) {
|
|
6513
|
-
const result = { references: [], coverage: "partial", issues: [] };
|
|
6514
|
-
const known = new Set(keys), lock = resolve3(rootDir, ".moda/prompts.lock.json");
|
|
6515
|
-
let files;
|
|
6516
|
-
try {
|
|
6517
|
-
files = inventoryFiles(rootDir, []);
|
|
6518
|
-
} catch (error) {
|
|
6519
|
-
result.issues.push({ path: ".", reason: `Attribution scan incomplete: ${error instanceof Error ? error.message : String(error)}` });
|
|
6520
|
-
return result;
|
|
6521
|
-
}
|
|
6522
|
-
for (const file of files) {
|
|
6523
|
-
if (![".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs"].includes(extname(file.path)) || !file.text?.includes("prompts.lock.json"))
|
|
6524
|
-
continue;
|
|
6525
|
-
if (/(^|\/)(?:tests?|__tests__|fixtures?)(\/|$)|\.(?:test|spec)\.[^.]+$/.test(file.path))
|
|
6526
|
-
continue;
|
|
6527
|
-
try {
|
|
6528
|
-
const absolute = safePromptEvidencePath(rootDir, file.path);
|
|
6529
|
-
const program = import_typescript.default.createProgram([absolute], { noResolve: true, noLib: true, types: [], allowJs: true });
|
|
6530
|
-
const sf = program.getSourceFile(absolute);
|
|
6531
|
-
if (!sf || program.getSyntacticDiagnostics(sf).length) {
|
|
6532
|
-
result.issues.push({ path: file.path, reason: "Attribution source could not be parsed" });
|
|
6533
|
-
continue;
|
|
6534
|
-
}
|
|
6535
|
-
const checker = program.getTypeChecker();
|
|
6536
|
-
const declaration = (node) => {
|
|
6537
|
-
const all = checker.getSymbolAtLocation(node)?.declarations;
|
|
6538
|
-
return all?.length === 1 ? all[0] : undefined;
|
|
6539
|
-
};
|
|
6540
|
-
const imported = (node) => {
|
|
6541
|
-
if (import_typescript.default.isImportSpecifier(node) && node.isTypeOnly)
|
|
6542
|
-
return false;
|
|
6543
|
-
let parent = node;
|
|
6544
|
-
while (parent && !import_typescript.default.isImportDeclaration(parent))
|
|
6545
|
-
parent = parent.parent;
|
|
6546
|
-
return !!parent && import_typescript.default.isImportDeclaration(parent) && !parent.importClause?.isTypeOnly && import_typescript.default.isStringLiteral(parent.moduleSpecifier) && parent.moduleSpecifier.text.startsWith(".") && resolve3(dirname(absolute), parent.moduleSpecifier.text) === lock;
|
|
6547
|
-
};
|
|
6548
|
-
const initializer = (node) => {
|
|
6549
|
-
const decl = declaration(node);
|
|
6550
|
-
return decl && import_typescript.default.isVariableDeclaration(decl) && import_typescript.default.isVariableDeclarationList(decl.parent) && !!(decl.parent.flags & import_typescript.default.NodeFlags.Const) ? decl.initializer : undefined;
|
|
6551
|
-
};
|
|
6552
|
-
const unwrap = (node) => {
|
|
6553
|
-
while (import_typescript.default.isParenthesizedExpression(node) || import_typescript.default.isAsExpression(node) || import_typescript.default.isTypeAssertionExpression(node) || import_typescript.default.isNonNullExpression(node))
|
|
6554
|
-
node = node.expression;
|
|
6555
|
-
return node;
|
|
6556
|
-
};
|
|
6557
|
-
const literal = (expression, seen = new Set) => {
|
|
6558
|
-
const node = unwrap(expression);
|
|
6559
|
-
if (seen.has(node))
|
|
6560
|
-
return;
|
|
6561
|
-
seen.add(node);
|
|
6562
|
-
if (import_typescript.default.isStringLiteral(node) || import_typescript.default.isNoSubstitutionTemplateLiteral(node))
|
|
6563
|
-
return node.text;
|
|
6564
|
-
if (import_typescript.default.isIdentifier(node)) {
|
|
6565
|
-
const init = initializer(node);
|
|
6566
|
-
return init ? literal(init, seen) : undefined;
|
|
6567
|
-
}
|
|
6568
|
-
if (import_typescript.default.isBinaryExpression(node) && node.operatorToken.kind === import_typescript.default.SyntaxKind.PlusToken) {
|
|
6569
|
-
const a = literal(node.left, new Set(seen)), b = literal(node.right, new Set(seen));
|
|
6570
|
-
return a !== undefined && b !== undefined ? a + b : undefined;
|
|
6571
|
-
}
|
|
6572
|
-
};
|
|
6573
|
-
const lockPath = (node) => import_typescript.default.isStringLiteral(node) && node.text.startsWith(".") && resolve3(dirname(absolute), node.text) === lock;
|
|
6574
|
-
const isConstBinding = (node) => import_typescript.default.isVariableDeclarationList(node.parent) && !!(node.parent.flags & import_typescript.default.NodeFlags.Const);
|
|
6575
|
-
const isLock = (expression, seen = new Set) => {
|
|
6576
|
-
const node = unwrap(expression);
|
|
6577
|
-
if (seen.has(node))
|
|
6578
|
-
return false;
|
|
6579
|
-
seen.add(node);
|
|
6580
|
-
if (import_typescript.default.isCallExpression(node) && import_typescript.default.isIdentifier(node.expression) && node.expression.text === "require" && !declaration(node.expression) && node.arguments.length === 1)
|
|
6581
|
-
return lockPath(node.arguments[0]);
|
|
6582
|
-
if (!import_typescript.default.isIdentifier(node))
|
|
6583
|
-
return false;
|
|
6584
|
-
const decl = declaration(node);
|
|
6585
|
-
if (decl && (import_typescript.default.isImportClause(decl) || import_typescript.default.isNamespaceImport(decl)))
|
|
6586
|
-
return imported(decl);
|
|
6587
|
-
const init = initializer(node);
|
|
6588
|
-
return !!init && isLock(init, seen);
|
|
6589
|
-
};
|
|
6590
|
-
const isTable = (expression, seen = new Set) => {
|
|
6591
|
-
const node = unwrap(expression);
|
|
6592
|
-
if (seen.has(node))
|
|
6593
|
-
return false;
|
|
6594
|
-
seen.add(node);
|
|
6595
|
-
if (import_typescript.default.isPropertyAccessExpression(node))
|
|
6596
|
-
return node.name.text === "prompts" && isLock(node.expression);
|
|
6597
|
-
if (import_typescript.default.isElementAccessExpression(node))
|
|
6598
|
-
return literal(node.argumentExpression) === "prompts" && isLock(node.expression);
|
|
6599
|
-
if (import_typescript.default.isIdentifier(node)) {
|
|
6600
|
-
const decl = declaration(node);
|
|
6601
|
-
if (decl && import_typescript.default.isBindingElement(decl) && import_typescript.default.isObjectBindingPattern(decl.parent) && import_typescript.default.isVariableDeclaration(decl.parent.parent)) {
|
|
6602
|
-
const owner = decl.parent.parent;
|
|
6603
|
-
const name = decl.propertyName ?? decl.name;
|
|
6604
|
-
if (!decl.dotDotDotToken && !decl.initializer && (import_typescript.default.isIdentifier(name) || import_typescript.default.isStringLiteral(name)) && name.text === "prompts" && isConstBinding(owner) && owner.initializer)
|
|
6605
|
-
return isLock(owner.initializer);
|
|
6606
|
-
}
|
|
6607
|
-
if (decl && import_typescript.default.isImportSpecifier(decl))
|
|
6608
|
-
return (decl.propertyName?.text ?? decl.name.text) === "prompts" && imported(decl);
|
|
6609
|
-
const init = initializer(node);
|
|
6610
|
-
return !!init && isTable(init, seen);
|
|
6611
|
-
}
|
|
6612
|
-
return false;
|
|
6613
|
-
};
|
|
6614
|
-
const visit = (node) => {
|
|
6615
|
-
if ((import_typescript.default.isElementAccessExpression(node) || import_typescript.default.isPropertyAccessExpression(node)) && isTable(node.expression)) {
|
|
6616
|
-
const key = import_typescript.default.isElementAccessExpression(node) ? literal(node.argumentExpression) : node.name.text;
|
|
6617
|
-
result.references.push({
|
|
6618
|
-
path: file.path,
|
|
6619
|
-
line: sf.getLineAndCharacterOfPosition(node.getStart(sf)).line + 1,
|
|
6620
|
-
key,
|
|
6621
|
-
status: key === undefined ? "unresolved" : known.has(key) ? "mapped" : "missing"
|
|
6622
|
-
});
|
|
6623
|
-
}
|
|
6624
|
-
import_typescript.default.forEachChild(node, visit);
|
|
6625
|
-
};
|
|
6626
|
-
visit(sf);
|
|
6627
|
-
} catch (error) {
|
|
6628
|
-
result.issues.push({ path: file.path, reason: error instanceof Error ? error.message : String(error) });
|
|
6629
|
-
}
|
|
6630
|
-
}
|
|
6631
|
-
return result;
|
|
5543
|
+
import { dirname, isAbsolute, join, relative, resolve } from "node:path";
|
|
5544
|
+
var promptSourcesModule;
|
|
5545
|
+
async function loadPromptSources() {
|
|
5546
|
+
promptSourcesModule ??= await import("./prompt-sources-07tczyqa.js");
|
|
5547
|
+
return promptSourcesModule;
|
|
6632
5548
|
}
|
|
6633
|
-
|
|
6634
|
-
// src/prompts.ts
|
|
6635
5549
|
var MODA_DIR = ".moda";
|
|
6636
5550
|
var MANIFEST_PATH = join(MODA_DIR, "prompts.yml");
|
|
6637
5551
|
var LOCK_PATH = join(MODA_DIR, "prompts.lock.json");
|
|
@@ -6737,7 +5651,7 @@ async function runPromptsCommand(input, legacyFlags, legacyProfileOptions) {
|
|
|
6737
5651
|
assertFlags(`prompts ${subcommand}`, flags, writeFlags);
|
|
6738
5652
|
switch (subcommand) {
|
|
6739
5653
|
case "init": {
|
|
6740
|
-
const imported = flags["from-harness"] ? importHarnessPromptSources(flags["from-harness"]) : undefined;
|
|
5654
|
+
const imported = flags["from-harness"] ? await importHarnessPromptSources(flags["from-harness"]) : undefined;
|
|
6741
5655
|
const init = await initPrompts({ silent: true });
|
|
6742
5656
|
context.output.writeData({
|
|
6743
5657
|
manifest: MANIFEST_PATH,
|
|
@@ -6786,7 +5700,7 @@ async function runPromptsCommand(input, legacyFlags, legacyProfileOptions) {
|
|
|
6786
5700
|
case "diff": {
|
|
6787
5701
|
const status = await getPromptStatus();
|
|
6788
5702
|
const differing = status.prompts.filter((row) => row.state !== "unchanged");
|
|
6789
|
-
const tracked =
|
|
5703
|
+
const tracked = existsSync(MANIFEST_PATH);
|
|
6790
5704
|
const diffWarnings = tracked ? promptManifestWarnings(status) : [
|
|
6791
5705
|
`No ${MANIFEST_PATH} here, so diff has no last sync to compare against: it only covers prompts tracked by \`moda prompts sync\`${differing.length > 0 ? ", and every discovered prompt file lists as new" : ""}. For a registry manifest, preview with \`moda registry push --dry-run\`.`
|
|
6792
5706
|
];
|
|
@@ -6811,9 +5725,9 @@ async function runPromptsCommand(input, legacyFlags, legacyProfileOptions) {
|
|
|
6811
5725
|
await promotePrompt(positionals, flags, profileOptions, context);
|
|
6812
5726
|
break;
|
|
6813
5727
|
case "ab":
|
|
6814
|
-
return runPromptAb(flags, profileOptions, context);
|
|
5728
|
+
return (await import("./prompts-ab-ezrd63sk.js")).runPromptAb(flags, profileOptions, context);
|
|
6815
5729
|
case "propose":
|
|
6816
|
-
await runPromptPropose(positionals, flags, profileOptions, context);
|
|
5730
|
+
await (await import("./prompts-propose-2k3enm2q.js")).runPromptPropose(positionals, flags, profileOptions, context);
|
|
6817
5731
|
break;
|
|
6818
5732
|
case "list":
|
|
6819
5733
|
case "show":
|
|
@@ -6830,16 +5744,16 @@ async function runPromptsCommand(input, legacyFlags, legacyProfileOptions) {
|
|
|
6830
5744
|
}
|
|
6831
5745
|
async function initPrompts(options = {}) {
|
|
6832
5746
|
const cwd = process.cwd();
|
|
6833
|
-
const manifestPath =
|
|
6834
|
-
const lockPath =
|
|
6835
|
-
if (
|
|
5747
|
+
const manifestPath = resolve(cwd, MANIFEST_PATH);
|
|
5748
|
+
const lockPath = resolve(cwd, LOCK_PATH);
|
|
5749
|
+
if (existsSync(MODA_DIR) && lstatSync(MODA_DIR).isSymbolicLink())
|
|
6836
5750
|
throw new CliInputError(".moda must not be a symlink.");
|
|
6837
5751
|
mkdirSync(MODA_DIR, { recursive: true });
|
|
6838
5752
|
let created = false;
|
|
6839
|
-
if (!
|
|
5753
|
+
if (!existsSync(MANIFEST_PATH)) {
|
|
6840
5754
|
assertContainedSource(cwd, MODA_DIR);
|
|
6841
5755
|
try {
|
|
6842
|
-
|
|
5756
|
+
writeFileSync(manifestPath, ["version: 2", "prompt_paths:", ...DEFAULT_PROMPT_PATHS.map((path) => ` - "${path}"`), "sources: []", ""].join(`
|
|
6843
5757
|
`), { flag: "wx" });
|
|
6844
5758
|
created = true;
|
|
6845
5759
|
} catch (error) {
|
|
@@ -6847,7 +5761,7 @@ async function initPrompts(options = {}) {
|
|
|
6847
5761
|
throw error;
|
|
6848
5762
|
}
|
|
6849
5763
|
}
|
|
6850
|
-
if (!
|
|
5764
|
+
if (!existsSync(manifestPath)) {
|
|
6851
5765
|
throw new Error(`Could not write ${manifestPath}: the file is missing after init. Check directory permissions and available disk space.`);
|
|
6852
5766
|
}
|
|
6853
5767
|
const result = {
|
|
@@ -6880,8 +5794,8 @@ async function syncPrompts(flags, profileOptions, context) {
|
|
|
6880
5794
|
async function runPromptSync(flags, profileOptions, options = {}) {
|
|
6881
5795
|
if (flags["dry-run"] === "true")
|
|
6882
5796
|
return runPromptSyncExclusive(flags, profileOptions, options);
|
|
6883
|
-
mkdirSync(
|
|
6884
|
-
const syncLock =
|
|
5797
|
+
mkdirSync(dirname(MANIFEST_PATH), { recursive: true });
|
|
5798
|
+
const syncLock = resolve(dirname(MANIFEST_PATH), "prompts.sync.lock");
|
|
6885
5799
|
try {
|
|
6886
5800
|
mkdirSync(syncLock);
|
|
6887
5801
|
} catch (error) {
|
|
@@ -6899,24 +5813,24 @@ async function runPromptSync(flags, profileOptions, options = {}) {
|
|
|
6899
5813
|
async function runPromptSyncExclusive(flags, profileOptions, options) {
|
|
6900
5814
|
let harnessImport;
|
|
6901
5815
|
if (flags["from-harness"] && flags["dry-run"] !== "true")
|
|
6902
|
-
harnessImport = importHarnessPromptSources(flags["from-harness"]);
|
|
5816
|
+
harnessImport = await importHarnessPromptSources(flags["from-harness"]);
|
|
6903
5817
|
if (flags["dry-run"] !== "true")
|
|
6904
5818
|
await initPrompts({ silent: true });
|
|
6905
5819
|
const manifest = readManifest();
|
|
6906
5820
|
const originalManifestHash = hashFile(MANIFEST_PATH);
|
|
6907
5821
|
let extraSources = [];
|
|
6908
5822
|
if (flags["from-harness"] && flags["dry-run"] === "true") {
|
|
6909
|
-
const read = readHarnessSources(flags["from-harness"]);
|
|
5823
|
+
const read = await readHarnessSources(flags["from-harness"]);
|
|
6910
5824
|
extraSources = read.sources;
|
|
6911
5825
|
harnessImport = read.result;
|
|
6912
5826
|
}
|
|
6913
5827
|
mergePromptSources(manifest.sources, extraSources);
|
|
6914
|
-
const established = discoverPrompts(extraSources);
|
|
5828
|
+
const established = await discoverPrompts(extraSources);
|
|
6915
5829
|
let discovery;
|
|
6916
5830
|
const analyzedDefinitions = new Map;
|
|
6917
5831
|
if (flags["no-analyze"] !== "true") {
|
|
6918
5832
|
options.onProgress?.("Analyzing prompt sources before sync.");
|
|
6919
|
-
const { analyzeHarnessPrompts } = await import("./harness-
|
|
5833
|
+
const { analyzeHarnessPrompts } = await import("./harness-7azmzytc.js");
|
|
6920
5834
|
const result = await analyzeHarnessPrompts({
|
|
6921
5835
|
rootDir: process.cwd(),
|
|
6922
5836
|
existing: established.flatMap((prompt) => prompt.sourceDefinition ? [prompt.sourceDefinition.source] : []),
|
|
@@ -6925,10 +5839,10 @@ async function runPromptSyncExclusive(flags, profileOptions, options) {
|
|
|
6925
5839
|
dryRun: flags["dry-run"] === "true",
|
|
6926
5840
|
onProgress: options.onProgress
|
|
6927
5841
|
});
|
|
6928
|
-
const legacyPaths = new Set(established.filter((prompt) => !prompt.sourceDefinition).map((prompt) =>
|
|
5842
|
+
const legacyPaths = new Set(established.filter((prompt) => !prompt.sourceDefinition).map((prompt) => resolve(prompt.sourcePath)));
|
|
6929
5843
|
const exclusions = manifest.promptPaths.filter((pattern) => pattern.startsWith("!")).map((pattern) => pattern.slice(1));
|
|
6930
|
-
const excludedPaths = new Set(exclusions.length ? import_fast_glob.default.sync(exclusions, { cwd: process.cwd(), onlyFiles: true, dot: true, followSymbolicLinks: false }).map((path) =>
|
|
6931
|
-
const found = result.sources.filter((source) => !legacyPaths.has(
|
|
5844
|
+
const excludedPaths = new Set(exclusions.length ? import_fast_glob.default.sync(exclusions, { cwd: process.cwd(), onlyFiles: true, dot: true, followSymbolicLinks: false }).map((path) => resolve(path)) : []);
|
|
5845
|
+
const found = result.sources.filter((source) => !legacyPaths.has(resolve(source.path)) && !excludedPaths.has(resolve(source.path)));
|
|
6932
5846
|
for (const source of found) {
|
|
6933
5847
|
const analyzed = result.resolved.find((item) => item.key === source.key);
|
|
6934
5848
|
if (!analyzed)
|
|
@@ -6943,7 +5857,8 @@ async function runPromptSyncExclusive(flags, profileOptions, options) {
|
|
|
6943
5857
|
throw new Error("Prompt manifest changed during analysis. Retry with a stable checkout.");
|
|
6944
5858
|
const mergedSources = mergePromptSources(manifest.sources, extraSources);
|
|
6945
5859
|
const manifestUpdate = stableJson(mergedSources) === stableJson(mergePromptSources(manifest.sources, [])) ? undefined : $stringify({ version: 2, prompt_paths: manifest.promptPaths, sources: mergedSources });
|
|
6946
|
-
const prompts = discoverPrompts(extraSources);
|
|
5860
|
+
const prompts = await discoverPrompts(extraSources);
|
|
5861
|
+
const { checkPromptAttribution } = await import("./prompt-attribution-1nytc9wj.js");
|
|
6947
5862
|
const attribution = checkPromptAttribution(process.cwd(), prompts.map((prompt) => prompt.key));
|
|
6948
5863
|
const brokenAttribution = attribution.references.filter((reference) => reference.status === "missing");
|
|
6949
5864
|
if (brokenAttribution.length) {
|
|
@@ -6982,7 +5897,7 @@ async function runPromptSyncExclusive(flags, profileOptions, options) {
|
|
|
6982
5897
|
dryRun
|
|
6983
5898
|
})
|
|
6984
5899
|
}, profileOptions);
|
|
6985
|
-
if (signature !== stableJson(discoverPrompts(extraSources)) || originalManifestHash !== hashFile(MANIFEST_PATH)) {
|
|
5900
|
+
if (signature !== stableJson(await discoverPrompts(extraSources)) || originalManifestHash !== hashFile(MANIFEST_PATH)) {
|
|
6986
5901
|
throw new Error("Prompt sources changed during sync. Remote writes may have completed; the prior lockfile was preserved. Retry with a stable checkout.");
|
|
6987
5902
|
}
|
|
6988
5903
|
const synced = response.synced;
|
|
@@ -7045,7 +5960,7 @@ async function watchPromptSync(flags, profileOptions) {
|
|
|
7045
5960
|
await syncPromptsOnce({ ...flags, watch: "false", "no-analyze": "true" }, profileOptions);
|
|
7046
5961
|
}
|
|
7047
5962
|
}
|
|
7048
|
-
await new Promise((
|
|
5963
|
+
await new Promise((resolve2) => setTimeout(resolve2, intervalMs));
|
|
7049
5964
|
}
|
|
7050
5965
|
}
|
|
7051
5966
|
async function promotePrompt(positionals, flags, profileOptions, context) {
|
|
@@ -7065,7 +5980,7 @@ async function promotePrompt(positionals, flags, profileOptions, context) {
|
|
|
7065
5980
|
context.output.writeData(response);
|
|
7066
5981
|
}
|
|
7067
5982
|
async function getPromptStatus() {
|
|
7068
|
-
const prompts = discoverPrompts();
|
|
5983
|
+
const prompts = await discoverPrompts();
|
|
7069
5984
|
const lock = readLock();
|
|
7070
5985
|
const rows = prompts.map((prompt) => {
|
|
7071
5986
|
const locked = lock?.prompts[prompt.key];
|
|
@@ -7108,16 +6023,17 @@ async function getPromptStatus() {
|
|
|
7108
6023
|
prompts: rows
|
|
7109
6024
|
};
|
|
7110
6025
|
}
|
|
7111
|
-
function discoverPrompts(extraSources = []) {
|
|
6026
|
+
async function discoverPrompts(extraSources = []) {
|
|
7112
6027
|
return discoverManifestPrompts(readManifest(), extraSources);
|
|
7113
6028
|
}
|
|
7114
|
-
|
|
6029
|
+
var NATIVE_PROMPT_FILE_RE = /\.prompt\.(md|json)$/i;
|
|
6030
|
+
async function discoverManifestPrompts(manifest, extraSources = []) {
|
|
7115
6031
|
const root = process.cwd();
|
|
7116
6032
|
for (const pattern of manifest.promptPaths) {
|
|
7117
6033
|
const positive = pattern.startsWith("!") ? pattern.slice(1) : pattern;
|
|
7118
|
-
if (
|
|
6034
|
+
if (isAbsolute(positive) || positive.split(/[\\/]/).includes(".."))
|
|
7119
6035
|
throw new CliInputError(`Prompt pattern must stay inside the repository: ${pattern}`);
|
|
7120
|
-
if (!pattern.startsWith("!") && !import_fast_glob.default.isDynamicPattern(pattern) && !
|
|
6036
|
+
if (!pattern.startsWith("!") && !import_fast_glob.default.isDynamicPattern(pattern) && !existsSync(resolve(root, pattern))) {
|
|
7121
6037
|
throw new CliInputError(`Configured prompt file is missing: ${pattern}`);
|
|
7122
6038
|
}
|
|
7123
6039
|
}
|
|
@@ -7132,15 +6048,17 @@ function discoverManifestPrompts(manifest, extraSources = []) {
|
|
|
7132
6048
|
const sources = new Map(manifest.sources.map((source) => [source.key, source]));
|
|
7133
6049
|
for (const source of extraSources)
|
|
7134
6050
|
sources.set(source.key, source);
|
|
7135
|
-
const selectedPaths = new Set([...sources.values()].map((source) =>
|
|
7136
|
-
const
|
|
6051
|
+
const selectedPaths = new Set([...sources.values()].map((source) => resolve(root, source.path)));
|
|
6052
|
+
const owned = paths.filter((path) => !selectedPaths.has(resolve(root, path)));
|
|
6053
|
+
const resolver = sources.size > 0 || owned.some((path) => !NATIVE_PROMPT_FILE_RE.test(path)) ? (await loadPromptSources()).resolvePromptSource : undefined;
|
|
6054
|
+
const prompts = owned.map((path) => {
|
|
7137
6055
|
assertContainedSource(root, path);
|
|
7138
|
-
if (
|
|
7139
|
-
return parsePromptFile(
|
|
7140
|
-
return normalizePrompt(
|
|
6056
|
+
if (NATIVE_PROMPT_FILE_RE.test(path))
|
|
6057
|
+
return parsePromptFile(resolve(root, path));
|
|
6058
|
+
return normalizePrompt(resolver(root, { key: keyFromPath(path), path }));
|
|
7141
6059
|
});
|
|
7142
6060
|
for (const source of sources.values())
|
|
7143
|
-
prompts.push(normalizePrompt(
|
|
6061
|
+
prompts.push(normalizePrompt(resolver(root, source)));
|
|
7144
6062
|
const seen = new Set;
|
|
7145
6063
|
for (const prompt of prompts) {
|
|
7146
6064
|
if (seen.has(prompt.key))
|
|
@@ -7151,7 +6069,7 @@ function discoverManifestPrompts(manifest, extraSources = []) {
|
|
|
7151
6069
|
}
|
|
7152
6070
|
var LEGACY_PROMPT_SUFFIXES = [".prompt.md", ".prompt.json", ".prompt.yaml", ".prompt.yml"];
|
|
7153
6071
|
function repoPath(root, path) {
|
|
7154
|
-
return relative(root,
|
|
6072
|
+
return relative(root, resolve(root, path)).replaceAll("\\", "/");
|
|
7155
6073
|
}
|
|
7156
6074
|
function globPromptFiles(root, patterns, ignore = []) {
|
|
7157
6075
|
if (patterns.length === 0)
|
|
@@ -7167,7 +6085,7 @@ function globPromptFiles(root, patterns, ignore = []) {
|
|
|
7167
6085
|
}
|
|
7168
6086
|
function inspectPromptManifest(root = process.cwd()) {
|
|
7169
6087
|
const manifest = readManifest(root);
|
|
7170
|
-
const safe = (pattern) => !
|
|
6088
|
+
const safe = (pattern) => !isAbsolute(pattern) && !pattern.split(/[\\/]/).includes("..");
|
|
7171
6089
|
const positives = manifest.promptPaths.filter((pattern) => !pattern.startsWith("!") && safe(pattern));
|
|
7172
6090
|
const negations = manifest.promptPaths.filter((pattern) => pattern.startsWith("!")).map((pattern) => pattern.slice(1));
|
|
7173
6091
|
const patterns = manifest.promptPaths.filter((pattern) => !pattern.startsWith("!")).map((pattern) => ({
|
|
@@ -7192,8 +6110,8 @@ function inspectPromptManifest(root = process.cwd()) {
|
|
|
7192
6110
|
};
|
|
7193
6111
|
}
|
|
7194
6112
|
function lockedSourcePaths(root) {
|
|
7195
|
-
const lockPath =
|
|
7196
|
-
if (!
|
|
6113
|
+
const lockPath = resolve(root, LOCK_PATH);
|
|
6114
|
+
if (!existsSync(lockPath))
|
|
7197
6115
|
return new Set;
|
|
7198
6116
|
try {
|
|
7199
6117
|
const lock = JSON.parse(readFileSync(lockPath, "utf8"));
|
|
@@ -7232,8 +6150,8 @@ function migrateWarnings(result) {
|
|
|
7232
6150
|
}
|
|
7233
6151
|
async function migratePromptManifest(options = {}) {
|
|
7234
6152
|
const root = process.cwd();
|
|
7235
|
-
const manifestPath =
|
|
7236
|
-
if (!
|
|
6153
|
+
const manifestPath = resolve(root, MANIFEST_PATH);
|
|
6154
|
+
if (!existsSync(manifestPath)) {
|
|
7237
6155
|
throw new CliInputError(`No ${MANIFEST_PATH} to migrate in ${root}.`, "Run `moda prompts init` to create a version 2 manifest.");
|
|
7238
6156
|
}
|
|
7239
6157
|
const original = readFileSync(manifestPath, "utf8");
|
|
@@ -7246,7 +6164,7 @@ async function migratePromptManifest(options = {}) {
|
|
|
7246
6164
|
let addedSources = [];
|
|
7247
6165
|
let pendingSources = [];
|
|
7248
6166
|
if (options.discover) {
|
|
7249
|
-
const { analyzeHarnessPrompts } = await import("./harness-
|
|
6167
|
+
const { analyzeHarnessPrompts } = await import("./harness-7azmzytc.js");
|
|
7250
6168
|
const result = await analyzeHarnessPrompts({
|
|
7251
6169
|
rootDir: root,
|
|
7252
6170
|
existing: manifest.sources,
|
|
@@ -7271,7 +6189,7 @@ async function migratePromptManifest(options = {}) {
|
|
|
7271
6189
|
})).sort((a, b) => a.key.localeCompare(b.key));
|
|
7272
6190
|
}
|
|
7273
6191
|
try {
|
|
7274
|
-
discoverManifestPrompts({ version: 2, promptPaths, sources });
|
|
6192
|
+
await discoverManifestPrompts({ version: 2, promptPaths, sources });
|
|
7275
6193
|
} catch (error) {
|
|
7276
6194
|
throw new CliInputError(`Cannot migrate ${MANIFEST_PATH}: ${error instanceof Error ? error.message : String(error)}`, `Give the conflicting prompt a unique \`key:\` (frontmatter or ${MANIFEST_PATH} sources), then rerun \`moda prompts migrate --dry-run\`. Nothing was written.`);
|
|
7277
6195
|
}
|
|
@@ -7329,15 +6247,15 @@ async function migratePromptManifest(options = {}) {
|
|
|
7329
6247
|
};
|
|
7330
6248
|
}
|
|
7331
6249
|
function assertContainedSource(root, path) {
|
|
7332
|
-
const abs =
|
|
6250
|
+
const abs = resolve(root, path);
|
|
7333
6251
|
const rel = relative(realpathSync(root), realpathSync(abs));
|
|
7334
|
-
if (rel === ".." || rel.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`) ||
|
|
6252
|
+
if (rel === ".." || rel.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`) || isAbsolute(rel))
|
|
7335
6253
|
throw new CliInputError(`Prompt source escapes repository: ${path}`);
|
|
7336
6254
|
let cursor = abs;
|
|
7337
|
-
while (cursor !==
|
|
6255
|
+
while (cursor !== resolve(root)) {
|
|
7338
6256
|
if (lstatSync(cursor).isSymbolicLink())
|
|
7339
6257
|
throw new CliInputError(`Prompt source uses a symlink: ${path}`);
|
|
7340
|
-
const next =
|
|
6258
|
+
const next = dirname(cursor);
|
|
7341
6259
|
if (next === cursor)
|
|
7342
6260
|
throw new CliInputError(`Prompt source escapes repository: ${path}`);
|
|
7343
6261
|
cursor = next;
|
|
@@ -7404,10 +6322,10 @@ function normalizePrompt(prompt) {
|
|
|
7404
6322
|
};
|
|
7405
6323
|
}
|
|
7406
6324
|
function readManifest(root = process.cwd()) {
|
|
7407
|
-
if (!
|
|
6325
|
+
if (!existsSync(resolve(root, MANIFEST_PATH)))
|
|
7408
6326
|
return { version: null, promptPaths: DEFAULT_PROMPT_PATHS, sources: [] };
|
|
7409
6327
|
assertContainedSource(root, MANIFEST_PATH);
|
|
7410
|
-
const doc = $parseDocument(readFileSync(
|
|
6328
|
+
const doc = $parseDocument(readFileSync(resolve(root, MANIFEST_PATH), "utf8"), { uniqueKeys: true });
|
|
7411
6329
|
if (doc.errors.length)
|
|
7412
6330
|
throw new CliInputError(`Invalid ${MANIFEST_PATH}: ${doc.errors[0].message}`);
|
|
7413
6331
|
const parsed = doc.toJS({ maxAliasCount: 20 });
|
|
@@ -7431,12 +6349,12 @@ function readManifest(root = process.cwd()) {
|
|
|
7431
6349
|
return { version: parsed.version, promptPaths: parsed.prompt_paths ?? (parsed.version === 1 ? DEFAULT_PROMPT_PATHS : []), sources };
|
|
7432
6350
|
}
|
|
7433
6351
|
var HARNESS_REANALYZE_COMMAND = "moda harness analyze --experimental --concern=prompts";
|
|
7434
|
-
function readHarnessSources(reportPath) {
|
|
6352
|
+
async function readHarnessSources(reportPath) {
|
|
7435
6353
|
const selected = reportPath === "true" ? ".moda/harness-report.json" : reportPath;
|
|
7436
6354
|
assertContainedSource(process.cwd(), selected);
|
|
7437
6355
|
let raw;
|
|
7438
6356
|
try {
|
|
7439
|
-
raw = JSON.parse(readFileSync(
|
|
6357
|
+
raw = JSON.parse(readFileSync(resolve(selected), "utf8"));
|
|
7440
6358
|
} catch (error) {
|
|
7441
6359
|
throw new CliInputError(`Could not read harness report ${selected}: ${error instanceof Error ? error.message : String(error)}.`, `Regenerate it with \`${HARNESS_REANALYZE_COMMAND}\`.`);
|
|
7442
6360
|
}
|
|
@@ -7447,6 +6365,8 @@ function readHarnessSources(reportPath) {
|
|
|
7447
6365
|
if (!graph) {
|
|
7448
6366
|
throw new CliInputError(`${selected} is not a harness report with a graph (schema: ${schemaLabel}). Regenerate it with \`${HARNESS_REANALYZE_COMMAND}\`.`);
|
|
7449
6367
|
}
|
|
6368
|
+
const { normalizeHarnessReport } = await import("./harness-report-rd9c4e9v.js");
|
|
6369
|
+
const { backfillPromptSource, resolvePromptSource } = await loadPromptSources();
|
|
7450
6370
|
const report = normalizeHarnessReport(raw);
|
|
7451
6371
|
const artifacts = (report.graph.artifacts ?? []).filter((artifact) => artifact.type === "prompt");
|
|
7452
6372
|
const sources = [];
|
|
@@ -7481,7 +6401,7 @@ function readHarnessSources(reportPath) {
|
|
|
7481
6401
|
let key = ownedKey ?? keyFromPath(path);
|
|
7482
6402
|
try {
|
|
7483
6403
|
if (ownedKey === undefined && nativeFile)
|
|
7484
|
-
key = parsePromptFile(
|
|
6404
|
+
key = parsePromptFile(resolve(root, path)).key;
|
|
7485
6405
|
} catch {}
|
|
7486
6406
|
tracked.push({ artifact: label, key, path });
|
|
7487
6407
|
continue;
|
|
@@ -7511,11 +6431,11 @@ function readHarnessSources(reportPath) {
|
|
|
7511
6431
|
function harnessSkipWarning(result) {
|
|
7512
6432
|
return `${result.skipped.length} harness prompt(s) skipped (registered ${result.registered.length}): ${result.skipped.slice(0, 3).map((skip) => `'${skip.artifact}' (${skip.reason})`).join("; ")}${result.skipped.length > 3 ? "; ..." : ""}. See harnessImport.skipped; regenerate the report with \`${HARNESS_REANALYZE_COMMAND}\` to map them.`;
|
|
7513
6433
|
}
|
|
7514
|
-
function importHarnessPromptSources(reportPath = ".moda/harness-report.json") {
|
|
7515
|
-
const { sources, result } = readHarnessSources(reportPath);
|
|
6434
|
+
async function importHarnessPromptSources(reportPath = ".moda/harness-report.json") {
|
|
6435
|
+
const { sources, result } = await readHarnessSources(reportPath);
|
|
7516
6436
|
const manifest = readManifest();
|
|
7517
6437
|
const merged = mergePromptSources(manifest.sources, sources);
|
|
7518
|
-
discoverPrompts(sources);
|
|
6438
|
+
await discoverPrompts(sources);
|
|
7519
6439
|
atomicWrite(MANIFEST_PATH, $stringify({ version: 2, prompt_paths: manifest.promptPaths, sources: merged }));
|
|
7520
6440
|
return result;
|
|
7521
6441
|
}
|
|
@@ -7531,13 +6451,13 @@ function mergePromptSources(existingSources, sources) {
|
|
|
7531
6451
|
return [...merged.values()].sort((a, b) => a.key.localeCompare(b.key));
|
|
7532
6452
|
}
|
|
7533
6453
|
function atomicWrite(path, content) {
|
|
7534
|
-
mkdirSync(
|
|
7535
|
-
assertContainedSource(process.cwd(),
|
|
7536
|
-
if (
|
|
6454
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
6455
|
+
assertContainedSource(process.cwd(), dirname(path));
|
|
6456
|
+
if (existsSync(path) && lstatSync(path).isSymbolicLink())
|
|
7537
6457
|
throw new CliInputError(`Refusing to overwrite symlink: ${path}`);
|
|
7538
6458
|
const temp = `${path}.${process.pid}.tmp`;
|
|
7539
6459
|
try {
|
|
7540
|
-
|
|
6460
|
+
writeFileSync(temp, content, { flag: "wx" });
|
|
7541
6461
|
renameSync(temp, path);
|
|
7542
6462
|
} finally {
|
|
7543
6463
|
rmSync(temp, { force: true });
|
|
@@ -7552,7 +6472,7 @@ function parsePromptFileForAb(absPath) {
|
|
|
7552
6472
|
return { key: parsed.key, content: parsed.content };
|
|
7553
6473
|
}
|
|
7554
6474
|
function readLock() {
|
|
7555
|
-
if (!
|
|
6475
|
+
if (!existsSync(LOCK_PATH))
|
|
7556
6476
|
return null;
|
|
7557
6477
|
assertContainedSource(process.cwd(), LOCK_PATH);
|
|
7558
6478
|
return JSON.parse(readFileSync(LOCK_PATH, "utf8"));
|
|
@@ -7573,7 +6493,7 @@ function writeLock(prompts, synced) {
|
|
|
7573
6493
|
}];
|
|
7574
6494
|
}))
|
|
7575
6495
|
};
|
|
7576
|
-
mkdirSync(
|
|
6496
|
+
mkdirSync(dirname(LOCK_PATH), { recursive: true });
|
|
7577
6497
|
atomicWrite(LOCK_PATH, `${JSON.stringify(lock, null, 2)}
|
|
7578
6498
|
`);
|
|
7579
6499
|
}
|
|
@@ -7598,7 +6518,7 @@ function keyFromPath(path) {
|
|
|
7598
6518
|
return path.replace(/^prompts\//, "").replace(/(?:\.prompt)?\.(md|txt|json|ya?ml|jinja2?|j2|hbs|tmpl|prompt)$/i, "").replace(/[\\/]/g, ".");
|
|
7599
6519
|
}
|
|
7600
6520
|
function hashFile(path) {
|
|
7601
|
-
return
|
|
6521
|
+
return existsSync(path) ? sha256(readFileSync(path, "utf8")) : "";
|
|
7602
6522
|
}
|
|
7603
6523
|
function getGitCommit() {
|
|
7604
6524
|
try {
|
|
@@ -7691,4 +6611,4 @@ function asRecordValue(value) {
|
|
|
7691
6611
|
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
7692
6612
|
}
|
|
7693
6613
|
|
|
7694
|
-
export {
|
|
6614
|
+
export { runPromptsCommand, initPrompts, runPromptSync, writePromptSyncProgress, discoverPrompts, inspectPromptManifest, promptManifestWarnings, migratePromptManifest, importHarnessPromptSources, readPromptLock, parsePromptFileForAb, assertReplayablePrompt };
|