@moda-ai/cli 1.13.6 → 1.13.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +5 -2
- package/dist/prompts-ab.d.ts +18 -0
- package/dist/prompts-ab.js +243 -0
- package/dist/prompts.d.ts +35 -0
- package/dist/prompts.js +14 -2
- package/dist/skills.js +123 -3
- package/package.json +1 -1
- package/skills/integration/index.json +2 -2
package/dist/cli.js
CHANGED
|
@@ -181,9 +181,11 @@ Commands:
|
|
|
181
181
|
prompts sync Upload changed prompt versions
|
|
182
182
|
prompts sync --watch Upload again when prompt files change
|
|
183
183
|
prompts promote <key> Move a dev/staging/prod label
|
|
184
|
+
prompts ab Run a judged prompt A/B replay experiment
|
|
184
185
|
|
|
185
186
|
Skills:
|
|
186
187
|
skills gen Kick off tenant-wide skill generation
|
|
188
|
+
skills status [run-id] Show a generation run's status, outcome, and recent events (default: latest run)
|
|
187
189
|
skills pull Pull tenant-generated skills into .claude/skills (and .cursor/rules)
|
|
188
190
|
skills pull --status=proposed Include not-yet-promoted skills (default: approved)
|
|
189
191
|
skills sync Push local .claude/skills/**/SKILL.md up to Moda
|
|
@@ -309,6 +311,7 @@ Examples:
|
|
|
309
311
|
moda prompts status
|
|
310
312
|
moda prompts sync
|
|
311
313
|
moda prompts promote support.triage --label=prod --version=pver_abc123
|
|
314
|
+
moda prompts ab --baseline=prompts/agent.prompt.md --candidate=prompts/agent-v2.prompt.md --conversations=conv_1,conv_2
|
|
312
315
|
moda skills pull`);
|
|
313
316
|
}
|
|
314
317
|
// --- Command Handlers ---
|
|
@@ -1056,7 +1059,7 @@ export const commandRegistry = createCommandRegistry([
|
|
|
1056
1059
|
legacyCommand({
|
|
1057
1060
|
name: 'prompts',
|
|
1058
1061
|
description: 'Manage prompt versions and labels',
|
|
1059
|
-
examples: ['moda prompts status', 'moda prompts sync'],
|
|
1062
|
+
examples: ['moda prompts status', 'moda prompts sync', 'moda prompts ab --baseline=old.md --candidate=new.md'],
|
|
1060
1063
|
auth: 'api-key',
|
|
1061
1064
|
outputModes: ['human', 'json', 'agent'],
|
|
1062
1065
|
mutability: 'write',
|
|
@@ -1121,7 +1124,7 @@ export const commandRegistry = createCommandRegistry([
|
|
|
1121
1124
|
{
|
|
1122
1125
|
name: 'skills',
|
|
1123
1126
|
description: 'Generate, pull, and sync Moda skills',
|
|
1124
|
-
examples: ['moda skills gen', 'moda skills pull', 'moda skills sync'],
|
|
1127
|
+
examples: ['moda skills gen', 'moda skills status', 'moda skills pull', 'moda skills sync'],
|
|
1125
1128
|
auth: 'none',
|
|
1126
1129
|
outputModes: ['human', 'json', 'agent'],
|
|
1127
1130
|
mutability: 'write',
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { CommandContext } from './command-context.js';
|
|
2
|
+
import type { ProfileIOOptions } from './profiles.js';
|
|
3
|
+
export interface PromptArmPayload {
|
|
4
|
+
content: string;
|
|
5
|
+
prompt_version_id?: string;
|
|
6
|
+
prompt_id?: string;
|
|
7
|
+
prompt_label?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface PromptAbResult {
|
|
10
|
+
replaySetId: string;
|
|
11
|
+
runId: string;
|
|
12
|
+
status: string;
|
|
13
|
+
verdict: string;
|
|
14
|
+
run: Record<string, unknown> | null;
|
|
15
|
+
cases: Array<Record<string, unknown>>;
|
|
16
|
+
}
|
|
17
|
+
export declare function runPromptAb(flags: Record<string, string>, profileOptions: ProfileIOOptions, context: CommandContext): Promise<void>;
|
|
18
|
+
//# sourceMappingURL=prompts-ab.d.ts.map
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { isAbsolute, resolve } from 'node:path';
|
|
3
|
+
import { callControlAPI, callDataAPI } from './api-client.js';
|
|
4
|
+
import { resolveTenantId, validateConfig } from './config.js';
|
|
5
|
+
import { discoverPrompts, parsePromptFileForAb, readPromptLock, runPromptSync, } from './prompts.js';
|
|
6
|
+
const DEFAULT_SEEDS_PER_CASE = 3;
|
|
7
|
+
const DEFAULT_POLL_INTERVAL_MS = 15_000;
|
|
8
|
+
const DEFAULT_WAIT_TIMEOUT_MS = 2 * 60 * 60 * 1000;
|
|
9
|
+
const TERMINAL_RUN_STATUSES = new Set(['completed', 'skipped', 'error']);
|
|
10
|
+
export async function runPromptAb(flags, profileOptions, context) {
|
|
11
|
+
validateConfig();
|
|
12
|
+
const tenantId = flags['tenant-id'] || resolveTenantId(profileOptions);
|
|
13
|
+
if (!tenantId) {
|
|
14
|
+
throw new Error('Missing tenant id. Run `moda init` or pass --tenant-id=<id>.');
|
|
15
|
+
}
|
|
16
|
+
const baselineSource = flags.baseline || flags['baseline-file'] || flags['baseline-key'];
|
|
17
|
+
const candidateSource = flags.candidate || flags['candidate-file'] || flags['candidate-key'];
|
|
18
|
+
if (!baselineSource || !candidateSource) {
|
|
19
|
+
throw new Error('Usage: moda prompts ab --baseline=<file|key> --candidate=<file|key> [--auto-generate|--set-id=ID|--conversations=id1,id2]');
|
|
20
|
+
}
|
|
21
|
+
if (flags.sync === 'true') {
|
|
22
|
+
await runPromptSync({ ...flags, watch: 'false' }, profileOptions);
|
|
23
|
+
}
|
|
24
|
+
const promptArms = {
|
|
25
|
+
prod: resolvePromptArmSpec(baselineSource, 'baseline'),
|
|
26
|
+
proposed: resolvePromptArmSpec(candidateSource, 'candidate'),
|
|
27
|
+
};
|
|
28
|
+
const replaySetId = await ensureReplaySet(flags, tenantId, profileOptions);
|
|
29
|
+
const seedsPerCase = parsePositiveInt(flags.seeds ?? flags['seeds-per-case'], DEFAULT_SEEDS_PER_CASE, 10);
|
|
30
|
+
const assistantModel = (flags.model ?? flags['assistant-model'] ?? '').trim();
|
|
31
|
+
const enqueue = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/run`, {
|
|
32
|
+
method: 'POST',
|
|
33
|
+
body: JSON.stringify({
|
|
34
|
+
promptArms,
|
|
35
|
+
seedsPerCase,
|
|
36
|
+
assistantModel: assistantModel || undefined,
|
|
37
|
+
promotePrimary: flags['no-promote'] !== 'true',
|
|
38
|
+
}),
|
|
39
|
+
}, profileOptions, { timeoutMs: 120_000, retries: 1 });
|
|
40
|
+
const noWait = flags.wait === 'false' || flags['no-wait'] === 'true';
|
|
41
|
+
if (noWait) {
|
|
42
|
+
context.output.writeData({
|
|
43
|
+
replaySetId,
|
|
44
|
+
runId: enqueue.runId,
|
|
45
|
+
status: enqueue.status,
|
|
46
|
+
message: enqueue.message ?? 'Replay comparison queued',
|
|
47
|
+
});
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
const timeoutMs = parsePositiveInt(flags.timeout, DEFAULT_WAIT_TIMEOUT_MS, 24 * 60 * 60 * 1000);
|
|
51
|
+
const pollIntervalMs = parsePositiveInt(flags['poll-interval'], DEFAULT_POLL_INTERVAL_MS, 120_000);
|
|
52
|
+
const deadline = Date.now() + timeoutMs;
|
|
53
|
+
let latest = null;
|
|
54
|
+
while (Date.now() < deadline) {
|
|
55
|
+
latest = await fetchLatestRun(tenantId, replaySetId, enqueue.runId, profileOptions);
|
|
56
|
+
const status = latest.run?.status ?? '';
|
|
57
|
+
const runId = latest.run?.runId ?? '';
|
|
58
|
+
if (TERMINAL_RUN_STATUSES.has(status) && runId === enqueue.runId) {
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
if (context.outputMode === 'human') {
|
|
62
|
+
process.stderr.write(`Waiting for replay run (${status || 'pending'})...\n`);
|
|
63
|
+
}
|
|
64
|
+
await sleep(pollIntervalMs);
|
|
65
|
+
}
|
|
66
|
+
if (!latest?.run
|
|
67
|
+
|| latest.run.runId !== enqueue.runId
|
|
68
|
+
|| !TERMINAL_RUN_STATUSES.has(latest.run.status)) {
|
|
69
|
+
throw new Error(`Timed out after ${timeoutMs}ms waiting for replay run ${enqueue.runId} to finish`);
|
|
70
|
+
}
|
|
71
|
+
const result = {
|
|
72
|
+
replaySetId,
|
|
73
|
+
runId: latest.run.runId,
|
|
74
|
+
status: latest.run.status,
|
|
75
|
+
verdict: formatVerdict(latest),
|
|
76
|
+
run: { ...latest.run },
|
|
77
|
+
cases: latest.cases.map((row) => ({ ...row })),
|
|
78
|
+
};
|
|
79
|
+
if (context.outputMode === 'human') {
|
|
80
|
+
printHumanVerdict(result);
|
|
81
|
+
}
|
|
82
|
+
context.output.writeData(result);
|
|
83
|
+
}
|
|
84
|
+
function resolvePromptArmSpec(source, label) {
|
|
85
|
+
const trimmed = source.trim();
|
|
86
|
+
if (!trimmed) {
|
|
87
|
+
throw new Error(`Empty prompt source for ${label}`);
|
|
88
|
+
}
|
|
89
|
+
const absPath = isAbsolute(trimmed) ? trimmed : resolve(process.cwd(), trimmed);
|
|
90
|
+
let key;
|
|
91
|
+
let content;
|
|
92
|
+
if (existsSync(absPath)) {
|
|
93
|
+
const parsed = parsePromptFileForAb(absPath);
|
|
94
|
+
key = parsed.key;
|
|
95
|
+
content = parsed.content;
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
const discovered = discoverPrompts();
|
|
99
|
+
const match = discovered.find((prompt) => prompt.key === trimmed)
|
|
100
|
+
?? discovered.find((prompt) => prompt.sourcePath === trimmed);
|
|
101
|
+
if (!match) {
|
|
102
|
+
throw new Error(`Could not resolve prompt '${trimmed}' — pass a .prompt.md path or a discovered prompt key`);
|
|
103
|
+
}
|
|
104
|
+
key = match.key;
|
|
105
|
+
content = match.content;
|
|
106
|
+
}
|
|
107
|
+
const locked = readPromptLock()?.prompts[key];
|
|
108
|
+
return {
|
|
109
|
+
content,
|
|
110
|
+
prompt_version_id: locked?.versionId ?? '',
|
|
111
|
+
prompt_id: locked?.promptId ?? '',
|
|
112
|
+
prompt_label: label,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
async function ensureReplaySet(flags, tenantId, profileOptions) {
|
|
116
|
+
const existingSetId = (flags['set-id'] || flags['replay-set-id'] || '').trim();
|
|
117
|
+
if (existingSetId) {
|
|
118
|
+
return existingSetId;
|
|
119
|
+
}
|
|
120
|
+
const conversations = parseCsv(flags.conversations ?? flags['conversation-ids']);
|
|
121
|
+
if (conversations.length) {
|
|
122
|
+
return createSetFromConversations(conversations, flags, tenantId, profileOptions);
|
|
123
|
+
}
|
|
124
|
+
if (flags['auto-generate'] === 'false' && !existingSetId) {
|
|
125
|
+
throw new Error('Provide --set-id=, --conversations=, or allow --auto-generate (default)');
|
|
126
|
+
}
|
|
127
|
+
const caseCount = parsePositiveInt(flags.cases ?? flags['case-count'], 5, 25);
|
|
128
|
+
const lookbackDays = parsePositiveInt(flags['lookback-days'], 30, 365);
|
|
129
|
+
const name = (flags.name ?? flags['set-name'] ?? `Prompt A/B ${new Date().toISOString().slice(0, 10)}`).trim();
|
|
130
|
+
const generated = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/auto-generate`, {
|
|
131
|
+
method: 'POST',
|
|
132
|
+
body: JSON.stringify({ name, caseCount, lookbackDays }),
|
|
133
|
+
}, profileOptions, { timeoutMs: 300_000, retries: 0 });
|
|
134
|
+
if (!generated?.id) {
|
|
135
|
+
throw new Error('Auto-generate did not return a replay set id');
|
|
136
|
+
}
|
|
137
|
+
return generated.id;
|
|
138
|
+
}
|
|
139
|
+
async function createSetFromConversations(conversationIds, flags, tenantId, profileOptions) {
|
|
140
|
+
const name = (flags.name ?? flags['set-name'] ?? `Prompt A/B ${conversationIds.length} conv`).trim();
|
|
141
|
+
const created = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets`, {
|
|
142
|
+
method: 'POST',
|
|
143
|
+
body: JSON.stringify({ name, description: 'Created by moda prompts ab' }),
|
|
144
|
+
}, profileOptions);
|
|
145
|
+
let position = 0;
|
|
146
|
+
for (const conversationId of conversationIds) {
|
|
147
|
+
const scenario = await loadScenarioFromConversation(conversationId);
|
|
148
|
+
await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(created.id)}/cases`, {
|
|
149
|
+
method: 'POST',
|
|
150
|
+
body: JSON.stringify({
|
|
151
|
+
title: conversationId,
|
|
152
|
+
scenario,
|
|
153
|
+
sourceConversationId: conversationId,
|
|
154
|
+
successCriteria: [
|
|
155
|
+
'The agent understands the user request',
|
|
156
|
+
'The agent uses tools appropriately when needed',
|
|
157
|
+
'The agent provides a helpful final answer',
|
|
158
|
+
],
|
|
159
|
+
position: position++,
|
|
160
|
+
}),
|
|
161
|
+
}, profileOptions);
|
|
162
|
+
}
|
|
163
|
+
return created.id;
|
|
164
|
+
}
|
|
165
|
+
async function loadScenarioFromConversation(conversationId) {
|
|
166
|
+
try {
|
|
167
|
+
const data = await callDataAPI(`/conversations/${encodeURIComponent(conversationId)}/context?msg_index=0&window=0`);
|
|
168
|
+
const firstUser = (data.messages ?? []).find((message) => message.role === 'user' && String(message.content ?? '').trim());
|
|
169
|
+
const opening = String(firstUser?.content ?? '').trim();
|
|
170
|
+
if (opening) {
|
|
171
|
+
return `The user says: "${opening}"\n\nReplay the conversation faithfully and help them accomplish their goal.`;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
// Fall through to generic scenario.
|
|
176
|
+
}
|
|
177
|
+
return `Replay conversation ${conversationId}: help the user accomplish their original goal using the available tools.`;
|
|
178
|
+
}
|
|
179
|
+
async function fetchLatestRun(tenantId, replaySetId, runId, profileOptions) {
|
|
180
|
+
// Pin the poll to OUR run id — otherwise a concurrent comparison on the same
|
|
181
|
+
// set would make `/runs/latest` return the newer run forever and this command
|
|
182
|
+
// would stall until timeout even after its own run finished.
|
|
183
|
+
return callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/runs/latest?runId=${encodeURIComponent(runId)}`, { method: 'GET' }, profileOptions, { timeoutMs: 60_000, retries: 2 });
|
|
184
|
+
}
|
|
185
|
+
function formatVerdict(payload) {
|
|
186
|
+
const run = payload.run;
|
|
187
|
+
if (!run)
|
|
188
|
+
return 'no run';
|
|
189
|
+
if (run.status === 'skipped')
|
|
190
|
+
return `skipped: ${run.notes || 'no reason'}`;
|
|
191
|
+
if (run.status === 'error')
|
|
192
|
+
return `error: ${run.notes || 'unknown error'}`;
|
|
193
|
+
if (run.status !== 'completed')
|
|
194
|
+
return `status=${run.status}`;
|
|
195
|
+
const delta = run.proposedPassCount - run.prodPassCount;
|
|
196
|
+
const winner = delta > 0 ? 'candidate wins' : delta < 0 ? 'baseline wins' : 'tie on case passes';
|
|
197
|
+
const rates = `baseline ${run.prodPassCount}/${run.caseCount} (${pct(run.prodPassRate)}), candidate ${run.proposedPassCount}/${run.caseCount} (${pct(run.proposedPassRate)})`;
|
|
198
|
+
const notes = run.notes?.trim();
|
|
199
|
+
return `${winner}; ${rates}${notes ? `; ${notes}` : ''}`;
|
|
200
|
+
}
|
|
201
|
+
function printHumanVerdict(result) {
|
|
202
|
+
process.stderr.write('\nPrompt A/B replay verdict\n');
|
|
203
|
+
process.stderr.write('-------------------------\n');
|
|
204
|
+
process.stderr.write(`Replay set: ${result.replaySetId}\n`);
|
|
205
|
+
process.stderr.write(`Run id: ${result.runId}\n`);
|
|
206
|
+
process.stderr.write(`Status: ${result.status}\n`);
|
|
207
|
+
process.stderr.write(`Verdict: ${result.verdict}\n`);
|
|
208
|
+
if (result.cases.length) {
|
|
209
|
+
process.stderr.write('\nPer-case:\n');
|
|
210
|
+
for (const row of result.cases) {
|
|
211
|
+
const caseId = String(row.caseId ?? '');
|
|
212
|
+
const prod = formatPass(row.prodPass);
|
|
213
|
+
const proposed = formatPass(row.proposedPass);
|
|
214
|
+
process.stderr.write(` ${caseId}: baseline=${prod} candidate=${proposed}\n`);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
process.stderr.write('\n');
|
|
218
|
+
}
|
|
219
|
+
function formatPass(value) {
|
|
220
|
+
if (value === null || value === undefined)
|
|
221
|
+
return 'abstained';
|
|
222
|
+
return value ? 'pass' : 'fail';
|
|
223
|
+
}
|
|
224
|
+
function pct(rate) {
|
|
225
|
+
return `${Math.round(Math.max(0, Math.min(1, rate)) * 100)}%`;
|
|
226
|
+
}
|
|
227
|
+
function parsePositiveInt(raw, fallback, max) {
|
|
228
|
+
if (!raw)
|
|
229
|
+
return fallback;
|
|
230
|
+
const parsed = Number.parseInt(raw, 10);
|
|
231
|
+
if (!Number.isFinite(parsed) || parsed < 1)
|
|
232
|
+
return fallback;
|
|
233
|
+
return Math.min(parsed, max);
|
|
234
|
+
}
|
|
235
|
+
function parseCsv(raw) {
|
|
236
|
+
if (!raw?.trim())
|
|
237
|
+
return [];
|
|
238
|
+
return raw.split(',').map((part) => part.trim()).filter(Boolean);
|
|
239
|
+
}
|
|
240
|
+
function sleep(ms) {
|
|
241
|
+
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
|
242
|
+
}
|
|
243
|
+
//# sourceMappingURL=prompts-ab.js.map
|
package/dist/prompts.d.ts
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
import { type CommandContext } from './command-context.js';
|
|
2
2
|
import type { ProfileIOOptions } from './profiles.js';
|
|
3
|
+
interface PromptFile {
|
|
4
|
+
key: string;
|
|
5
|
+
name: string;
|
|
6
|
+
description?: string;
|
|
7
|
+
category?: string;
|
|
8
|
+
sourcePath: string;
|
|
9
|
+
content: string;
|
|
10
|
+
systemPrompt?: string;
|
|
11
|
+
messages?: unknown[];
|
|
12
|
+
variables?: unknown[];
|
|
13
|
+
variablesSchema?: Record<string, unknown>;
|
|
14
|
+
modelConfig?: Record<string, unknown>;
|
|
15
|
+
toolRefs?: unknown[];
|
|
16
|
+
responseSchema?: Record<string, unknown>;
|
|
17
|
+
contentHash: string;
|
|
18
|
+
}
|
|
19
|
+
interface PromptLock {
|
|
20
|
+
version: 1;
|
|
21
|
+
generatedAt: string;
|
|
22
|
+
prompts: Record<string, {
|
|
23
|
+
key: string;
|
|
24
|
+
sourcePath: string;
|
|
25
|
+
contentHash: string;
|
|
26
|
+
versionId?: string;
|
|
27
|
+
promptId?: string;
|
|
28
|
+
}>;
|
|
29
|
+
}
|
|
3
30
|
export declare function runPromptsCommand(input: CommandContext | string[], legacyFlags?: Record<string, string>, legacyProfileOptions?: ProfileIOOptions): Promise<void>;
|
|
4
31
|
export declare function initPrompts(options?: {
|
|
5
32
|
silent?: boolean;
|
|
@@ -20,4 +47,12 @@ export interface PromptSyncResult {
|
|
|
20
47
|
* control their own output. `moda prompts sync` wraps this with JSON output.
|
|
21
48
|
*/
|
|
22
49
|
export declare function runPromptSync(flags: Record<string, string>, profileOptions: ProfileIOOptions): Promise<PromptSyncResult>;
|
|
50
|
+
export declare function discoverPrompts(): PromptFile[];
|
|
51
|
+
export declare function readPromptLock(): PromptLock | null;
|
|
52
|
+
/** Parse a prompt file for A/B runs (body text + registry key). */
|
|
53
|
+
export declare function parsePromptFileForAb(absPath: string): {
|
|
54
|
+
key: string;
|
|
55
|
+
content: string;
|
|
56
|
+
};
|
|
57
|
+
export {};
|
|
23
58
|
//# sourceMappingURL=prompts.d.ts.map
|
package/dist/prompts.js
CHANGED
|
@@ -4,6 +4,7 @@ import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, writeFileS
|
|
|
4
4
|
import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
|
|
5
5
|
import { callControlAPI } from './api-client.js';
|
|
6
6
|
import { createCommandContext } from './command-context.js';
|
|
7
|
+
import { runPromptAb } from './prompts-ab.js';
|
|
7
8
|
const MODA_DIR = '.moda';
|
|
8
9
|
const MANIFEST_PATH = join(MODA_DIR, 'prompts.yml');
|
|
9
10
|
const LOCK_PATH = join(MODA_DIR, 'prompts.lock.json');
|
|
@@ -51,8 +52,11 @@ export async function runPromptsCommand(input, legacyFlags, legacyProfileOptions
|
|
|
51
52
|
case 'promote':
|
|
52
53
|
await promotePrompt(positionals, flags, profileOptions, context);
|
|
53
54
|
break;
|
|
55
|
+
case 'ab':
|
|
56
|
+
await runPromptAb(flags, profileOptions, context);
|
|
57
|
+
break;
|
|
54
58
|
default:
|
|
55
|
-
throw new Error(`Unknown prompts command '${subcommand}'. Use init, status, diff, sync, or
|
|
59
|
+
throw new Error(`Unknown prompts command '${subcommand}'. Use init, status, diff, sync, promote, or ab.`);
|
|
56
60
|
}
|
|
57
61
|
}
|
|
58
62
|
export async function initPrompts(options = {}) {
|
|
@@ -195,7 +199,7 @@ async function getPromptStatus() {
|
|
|
195
199
|
prompts: rows,
|
|
196
200
|
};
|
|
197
201
|
}
|
|
198
|
-
function discoverPrompts() {
|
|
202
|
+
export function discoverPrompts() {
|
|
199
203
|
const manifest = readManifest();
|
|
200
204
|
// A pattern like `src/prompts/**/*.prompt.md` should *only* match files under
|
|
201
205
|
// src/prompts. Extract both the base directory (everything up to the first
|
|
@@ -290,6 +294,14 @@ function readManifest() {
|
|
|
290
294
|
.filter((path) => path.includes('.prompt.'));
|
|
291
295
|
return { promptPaths: promptPaths.length ? promptPaths : DEFAULT_PROMPT_PATHS };
|
|
292
296
|
}
|
|
297
|
+
export function readPromptLock() {
|
|
298
|
+
return readLock();
|
|
299
|
+
}
|
|
300
|
+
/** Parse a prompt file for A/B runs (body text + registry key). */
|
|
301
|
+
export function parsePromptFileForAb(absPath) {
|
|
302
|
+
const parsed = parsePromptFile(absPath);
|
|
303
|
+
return { key: parsed.key, content: parsed.content };
|
|
304
|
+
}
|
|
293
305
|
function readLock() {
|
|
294
306
|
if (!existsSync(LOCK_PATH))
|
|
295
307
|
return null;
|
package/dist/skills.js
CHANGED
|
@@ -74,6 +74,108 @@ async function fetchSkills(status) {
|
|
|
74
74
|
status,
|
|
75
75
|
};
|
|
76
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* Authenticated GET against the ingest worker's `/v1/skills/runs` surface.
|
|
79
|
+
* Returns `{ status, body }` so callers can treat 404 (no such run) as data;
|
|
80
|
+
* every other non-2xx throws.
|
|
81
|
+
*/
|
|
82
|
+
async function skillRunsRequest(url) {
|
|
83
|
+
const apiKey = resolveApiKey();
|
|
84
|
+
if (!apiKey) {
|
|
85
|
+
throw new CliInputError('No Moda API key found. Run `moda init` first (or set MODA_API_KEY).');
|
|
86
|
+
}
|
|
87
|
+
const controller = new AbortController();
|
|
88
|
+
const timeout = setTimeout(() => controller.abort(), SKILLS_REQUEST_TIMEOUT_MS);
|
|
89
|
+
let response;
|
|
90
|
+
try {
|
|
91
|
+
response = await fetch(url, {
|
|
92
|
+
headers: { Authorization: `Bearer ${apiKey}` },
|
|
93
|
+
signal: controller.signal,
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
98
|
+
throw new Error(`Network error: failed to reach ${url} (${message})`);
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
clearTimeout(timeout);
|
|
102
|
+
}
|
|
103
|
+
const text = await response.text().catch(() => '');
|
|
104
|
+
if (!response.ok && response.status !== 404) {
|
|
105
|
+
throw new Error(`Failed to fetch run status (HTTP ${response.status}): ${text.slice(0, 200)}`);
|
|
106
|
+
}
|
|
107
|
+
let body = {};
|
|
108
|
+
try {
|
|
109
|
+
body = text ? JSON.parse(text) : {};
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
body = {};
|
|
113
|
+
}
|
|
114
|
+
return { status: response.status, body };
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Resolve a run's status + recent events. Without a run id, the tenant's
|
|
118
|
+
* latest run (`GET /v1/skills/runs?limit=1`) is used; the detail form
|
|
119
|
+
* (`GET /v1/skills/runs/:id`) always supplies the event breadcrumbs.
|
|
120
|
+
*/
|
|
121
|
+
async function fetchSkillRunStatus(runId) {
|
|
122
|
+
const ingestUrl = resolveIngestUrl().replace(/\/$/, '');
|
|
123
|
+
let resolvedRunId = runId;
|
|
124
|
+
if (!resolvedRunId) {
|
|
125
|
+
const list = await skillRunsRequest(`${ingestUrl}/v1/skills/runs?limit=1`);
|
|
126
|
+
const listBody = list.body;
|
|
127
|
+
const latest = Array.isArray(listBody.runs) ? listBody.runs[0] : undefined;
|
|
128
|
+
if (!latest)
|
|
129
|
+
return { run: null, events: [] };
|
|
130
|
+
resolvedRunId = latest.generation_run_id;
|
|
131
|
+
// Defensive: a row without an id can still be shown, just without events.
|
|
132
|
+
if (!resolvedRunId)
|
|
133
|
+
return { run: latest, events: [] };
|
|
134
|
+
}
|
|
135
|
+
const detail = await skillRunsRequest(`${ingestUrl}/v1/skills/runs/${encodeURIComponent(resolvedRunId)}`);
|
|
136
|
+
if (detail.status === 404) {
|
|
137
|
+
if (runId) {
|
|
138
|
+
throw new CliInputError(`No skill generation run found with id '${runId}'`);
|
|
139
|
+
}
|
|
140
|
+
return { run: null, events: [] };
|
|
141
|
+
}
|
|
142
|
+
const detailBody = detail.body;
|
|
143
|
+
return {
|
|
144
|
+
run: detailBody.run ?? null,
|
|
145
|
+
events: Array.isArray(detailBody.events) ? detailBody.events : [],
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
const STATUS_HUMAN_EVENT_LIMIT = 10;
|
|
149
|
+
/** Human rendering for `moda skills status` (key: value lines, gen-style). */
|
|
150
|
+
function formatRunStatusHuman(run, events) {
|
|
151
|
+
const counters = run.counters ?? {};
|
|
152
|
+
const lines = [
|
|
153
|
+
run.generation_run_id && `run: ${run.generation_run_id}`,
|
|
154
|
+
`status: ${run.status || 'unknown'}`,
|
|
155
|
+
run.current_stage && `stage: ${run.current_stage}`,
|
|
156
|
+
run.outcome && `outcome: ${run.outcome}`,
|
|
157
|
+
`processed: ${Number(counters.sessions_processed ?? 0)}`,
|
|
158
|
+
`failed: ${Number(counters.sessions_failed ?? 0)}`,
|
|
159
|
+
Number(counters.distilled_skills_generated ?? 0) > 0
|
|
160
|
+
&& `distilled: ${Number(counters.distilled_skills_generated ?? 0)}`,
|
|
161
|
+
Number(counters.skills_indexed ?? 0) > 0
|
|
162
|
+
&& `indexed: ${Number(counters.skills_indexed ?? 0)}`,
|
|
163
|
+
(run.generated_skill_ids?.length ?? 0) > 0
|
|
164
|
+
&& `skills: ${(run.generated_skill_ids ?? []).join(', ')}`,
|
|
165
|
+
run.error && `error: ${run.error}`,
|
|
166
|
+
run.started_at && `started: ${run.started_at}`,
|
|
167
|
+
run.updated_at && `updated: ${run.updated_at}`,
|
|
168
|
+
].filter(Boolean);
|
|
169
|
+
if (events.length > 0) {
|
|
170
|
+
lines.push('recent events (newest first):');
|
|
171
|
+
for (const event of events.slice(0, STATUS_HUMAN_EVENT_LIMIT)) {
|
|
172
|
+
lines.push(` ${[event.created_at, event.level, event.stage, event.message]
|
|
173
|
+
.filter(Boolean)
|
|
174
|
+
.join(' ')}`);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return lines.join('\n');
|
|
178
|
+
}
|
|
77
179
|
/**
|
|
78
180
|
* Write `content` to `filePath`, creating parent dirs. Returns whether the
|
|
79
181
|
* file was created, updated (content differed), or left unchanged.
|
|
@@ -463,6 +565,8 @@ export async function runSkillsCommand(input) {
|
|
|
463
565
|
'accepted: true',
|
|
464
566
|
result.generation_run_id && `generation_run_id: ${result.generation_run_id}`,
|
|
465
567
|
result.message && `message: ${result.message}`,
|
|
568
|
+
result.generation_run_id
|
|
569
|
+
&& `follow progress: moda skills status ${result.generation_run_id}`,
|
|
466
570
|
].filter(Boolean).join('\n'));
|
|
467
571
|
}
|
|
468
572
|
else {
|
|
@@ -473,10 +577,26 @@ export async function runSkillsCommand(input) {
|
|
|
473
577
|
`pending_found: ${result.pending_sessions_found ?? 0}`,
|
|
474
578
|
`distilled: ${Number(result.cluster_result?.distilled_skills_generated ?? 0)}`,
|
|
475
579
|
`indexed: ${Number(result.cluster_result?.skills_indexed ?? 0)}`,
|
|
476
|
-
|
|
580
|
+
result.distillation_skipped
|
|
581
|
+
&& `distillation skipped: ${result.distillation_skip_reason || 'unknown'}`,
|
|
582
|
+
].filter(Boolean).join('\n'));
|
|
477
583
|
}
|
|
478
584
|
return Number(result.sessions_failed ?? 0) > 0 ? 1 : 0;
|
|
479
585
|
}
|
|
586
|
+
if (sub === 'status') {
|
|
587
|
+
const runId = positionals[1] || flags['run-id'] || undefined;
|
|
588
|
+
const { run, events } = await fetchSkillRunStatus(runId);
|
|
589
|
+
if (context.outputMode !== 'human') {
|
|
590
|
+
context.output.writeData({ run, events }, { status: run?.status === 'failed' ? 'error' : 'success' });
|
|
591
|
+
}
|
|
592
|
+
else if (!run) {
|
|
593
|
+
console.log('No skill generation runs found yet. Start one with `moda skills gen`.');
|
|
594
|
+
}
|
|
595
|
+
else {
|
|
596
|
+
console.log(formatRunStatusHuman(run, events));
|
|
597
|
+
}
|
|
598
|
+
return run?.status === 'failed' ? 1 : 0;
|
|
599
|
+
}
|
|
480
600
|
if (sub === 'proposals' && positionals[1] === 'list') {
|
|
481
601
|
const proposals = await fetchSkillProposals(flags, { profile: context.profile });
|
|
482
602
|
if (flags.json === 'true') {
|
|
@@ -533,7 +653,7 @@ export async function runSkillsCommand(input) {
|
|
|
533
653
|
return 0;
|
|
534
654
|
}
|
|
535
655
|
if (sub !== 'pull') {
|
|
536
|
-
throw new CliInputError(sub ? `Unknown skills subcommand '${sub}'` : 'A skills subcommand is required', 'Usage: moda skills gen [--source=all|sdk|coding-agent] [--max-sessions=N] [--start-at=ISO] [--end-at=ISO] [--replay] [--replay-set-id=ID] [--reprocess-segments] [--force-reprocess] [--wait] | moda skills proposals list [--status=ready_for_pr] | moda skills proposal apply <proposal-id> | moda skills pull [--status=approved|proposed|all] | moda skills sync [--dry-run]');
|
|
656
|
+
throw new CliInputError(sub ? `Unknown skills subcommand '${sub}'` : 'A skills subcommand is required', 'Usage: moda skills gen [--source=all|sdk|coding-agent] [--max-sessions=N] [--start-at=ISO] [--end-at=ISO] [--replay] [--replay-set-id=ID] [--reprocess-segments] [--force-reprocess] [--wait] | moda skills status [run-id] | moda skills proposals list [--status=ready_for_pr] | moda skills proposal apply <proposal-id> | moda skills pull [--status=approved|proposed|all] | moda skills sync [--dry-run]');
|
|
537
657
|
}
|
|
538
658
|
const status = (flags.status ?? 'approved').toLowerCase();
|
|
539
659
|
if (!VALID_STATUSES.has(status)) {
|
|
@@ -580,7 +700,7 @@ async function generateSkills(flags) {
|
|
|
580
700
|
replay_set_id: flags['replay-set-id'] || flags['eval-set-id'] || undefined,
|
|
581
701
|
run_clustering: flags['no-clustering'] !== 'true',
|
|
582
702
|
source: normalizeSkillSource(flags.source),
|
|
583
|
-
clustering_timeout_seconds: parsePositiveInt(flags['clustering-timeout-seconds'],
|
|
703
|
+
clustering_timeout_seconds: parsePositiveInt(flags['clustering-timeout-seconds'], 7200),
|
|
584
704
|
clustering_poll_seconds: parsePositiveInt(flags['clustering-poll-seconds'], 15),
|
|
585
705
|
max_cluster_pct: parseOptionalFloat(flags['max-cluster-pct']),
|
|
586
706
|
reprocess_segments: flags['reprocess-segments'] === 'true' || forceReprocess,
|
package/package.json
CHANGED