@moda-ai/cli 1.13.6 → 1.13.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth.d.ts +16 -0
- package/dist/auth.js +27 -4
- package/dist/cli.js +5 -2
- package/dist/doctor.js +78 -1
- package/dist/harness-artifact-bodies.d.ts +7 -0
- package/dist/harness-artifact-bodies.js +78 -11
- package/dist/harness.d.ts +6 -0
- package/dist/harness.js +28 -12
- package/dist/init/index.d.ts +3 -2
- package/dist/init/index.js +11 -3
- package/dist/init/tenant.js +2 -1
- package/dist/manifest.js +1 -0
- package/dist/prompts-ab.d.ts +18 -0
- package/dist/prompts-ab.js +243 -0
- package/dist/prompts.d.ts +35 -0
- package/dist/prompts.js +14 -2
- package/dist/skills.js +123 -3
- package/package.json +1 -1
- package/skills/integration/index.json +2 -2
package/dist/auth.d.ts
CHANGED
|
@@ -2,6 +2,17 @@ import { type ProfileIOOptions } from './profiles.js';
|
|
|
2
2
|
import { type AuthSessionOptions } from './auth-storage.js';
|
|
3
3
|
export { clearAuthSession, isAuthSessionValid, loadAuthSession, loadToken, saveAuthSession, type AuthSessionOptions, type StoredAuth, type StoredAuthTokenKind, } from './auth-storage.js';
|
|
4
4
|
export { revokeCliSessionToken } from './cli-session.js';
|
|
5
|
+
/** Default wait for browser Auth0/consent during `moda auth login` / init. */
|
|
6
|
+
export declare const DEFAULT_AUTH_TIMEOUT_MS = 600000;
|
|
7
|
+
/**
|
|
8
|
+
* Resolve the browser-auth wait. Explicit LoginOptions.timeoutMs wins;
|
|
9
|
+
* otherwise MODA_AUTH_TIMEOUT_MS (clamped); otherwise a 10-minute default that
|
|
10
|
+
* covers real Auth0 + MFA + slow consent without abandoning mid-login.
|
|
11
|
+
*/
|
|
12
|
+
export declare function resolveAuthTimeoutMs(options?: {
|
|
13
|
+
timeoutMs?: number;
|
|
14
|
+
env?: NodeJS.ProcessEnv;
|
|
15
|
+
}): number;
|
|
5
16
|
/** base64url(sha256(verifier)) — the S256 code challenge for a verifier. */
|
|
6
17
|
export declare function computePkceChallenge(codeVerifier: string): string;
|
|
7
18
|
export interface CliLoginContext {
|
|
@@ -20,6 +31,11 @@ export declare function getCachedLoginContext(options?: ProfileIOOptions): CliLo
|
|
|
20
31
|
export interface LoginOptions extends ProfileIOOptions {
|
|
21
32
|
/** Use this exact port for the localhost callback (no fallback ladder). */
|
|
22
33
|
callbackPort?: number;
|
|
34
|
+
/**
|
|
35
|
+
* Explicit browser-auth wait in ms. Unlike `MODA_AUTH_TIMEOUT_MS` (clamped
|
|
36
|
+
* 30s–30m), an explicit value bypasses that clamp (only floored at 1ms) so
|
|
37
|
+
* programmatic/test callers can set very short or very long waits.
|
|
38
|
+
*/
|
|
23
39
|
timeoutMs?: number;
|
|
24
40
|
openUrl?: (url: string) => Promise<void>;
|
|
25
41
|
/**
|
package/dist/auth.js
CHANGED
|
@@ -20,10 +20,33 @@ const CALLBACK_PORT = 8432;
|
|
|
20
20
|
// callback redirect is port-parameterized, so any of these works as long as
|
|
21
21
|
// the auth URL carries the port that actually got bound.
|
|
22
22
|
const CALLBACK_PORT_LADDER = [CALLBACK_PORT, 8433, 8434, 8435];
|
|
23
|
-
|
|
23
|
+
/** Default wait for browser Auth0/consent during `moda auth login` / init. */
|
|
24
|
+
export const DEFAULT_AUTH_TIMEOUT_MS = 600_000;
|
|
25
|
+
/** Floor / ceiling for MODA_AUTH_TIMEOUT_MS (30s .. 30m). */
|
|
26
|
+
const AUTH_TIMEOUT_MIN_MS = 30_000;
|
|
27
|
+
const AUTH_TIMEOUT_MAX_MS = 30 * 60_000;
|
|
24
28
|
const DEFAULT_EXPIRES_IN_SECONDS = 86_400;
|
|
25
29
|
// PKCE code exchange against {base}/api/cli/auth/exchange.
|
|
26
30
|
const EXCHANGE_TIMEOUT_MS = 10_000;
|
|
31
|
+
/**
|
|
32
|
+
* Resolve the browser-auth wait. Explicit LoginOptions.timeoutMs wins;
|
|
33
|
+
* otherwise MODA_AUTH_TIMEOUT_MS (clamped); otherwise a 10-minute default that
|
|
34
|
+
* covers real Auth0 + MFA + slow consent without abandoning mid-login.
|
|
35
|
+
*/
|
|
36
|
+
export function resolveAuthTimeoutMs(options = {}) {
|
|
37
|
+
if (typeof options.timeoutMs === 'number' && Number.isFinite(options.timeoutMs)) {
|
|
38
|
+
return Math.max(1, Math.floor(options.timeoutMs));
|
|
39
|
+
}
|
|
40
|
+
const env = options.env ?? process.env;
|
|
41
|
+
const raw = env.MODA_AUTH_TIMEOUT_MS?.trim();
|
|
42
|
+
if (raw) {
|
|
43
|
+
const parsed = Number.parseInt(raw, 10);
|
|
44
|
+
if (Number.isFinite(parsed) && parsed > 0) {
|
|
45
|
+
return Math.min(AUTH_TIMEOUT_MAX_MS, Math.max(AUTH_TIMEOUT_MIN_MS, parsed));
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
return DEFAULT_AUTH_TIMEOUT_MS;
|
|
49
|
+
}
|
|
27
50
|
function escapeHtml(value) {
|
|
28
51
|
return value
|
|
29
52
|
.replace(/&/g, '&')
|
|
@@ -209,7 +232,7 @@ export async function login(screen = 'login', options = {}) {
|
|
|
209
232
|
// flips. The flag only needs to clear on failure, where a
|
|
210
233
|
// retry paste is still legitimate.
|
|
211
234
|
res.writeHead(302, {
|
|
212
|
-
Location: `${resolveModaBaseUrl()}/auth/cli/done?status=success`,
|
|
235
|
+
Location: `${resolveModaBaseUrl(storageOptions)}/auth/cli/done?status=success`,
|
|
213
236
|
Connection: 'close',
|
|
214
237
|
});
|
|
215
238
|
res.end(() => {
|
|
@@ -220,7 +243,7 @@ export async function login(screen = 'login', options = {}) {
|
|
|
220
243
|
exchangeInFlight = false;
|
|
221
244
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
222
245
|
res.writeHead(302, {
|
|
223
|
-
Location: `${resolveModaBaseUrl()}/auth/cli/done` +
|
|
246
|
+
Location: `${resolveModaBaseUrl(storageOptions)}/auth/cli/done` +
|
|
224
247
|
`?status=error&reason=${exchangeFailureReason(err)}`,
|
|
225
248
|
Connection: 'close',
|
|
226
249
|
});
|
|
@@ -538,7 +561,7 @@ export async function login(screen = 'login', options = {}) {
|
|
|
538
561
|
}
|
|
539
562
|
timeout = setTimeout(() => {
|
|
540
563
|
fail(new Error('Authentication timed out. Please try again.'));
|
|
541
|
-
}, options.timeoutMs
|
|
564
|
+
}, resolveAuthTimeoutMs({ timeoutMs: options.timeoutMs, env: options.env }));
|
|
542
565
|
server.on('error', (err) => {
|
|
543
566
|
if (settled)
|
|
544
567
|
return;
|
package/dist/cli.js
CHANGED
|
@@ -181,9 +181,11 @@ Commands:
|
|
|
181
181
|
prompts sync Upload changed prompt versions
|
|
182
182
|
prompts sync --watch Upload again when prompt files change
|
|
183
183
|
prompts promote <key> Move a dev/staging/prod label
|
|
184
|
+
prompts ab Run a judged prompt A/B replay experiment
|
|
184
185
|
|
|
185
186
|
Skills:
|
|
186
187
|
skills gen Kick off tenant-wide skill generation
|
|
188
|
+
skills status [run-id] Show a generation run's status, outcome, and recent events (default: latest run)
|
|
187
189
|
skills pull Pull tenant-generated skills into .claude/skills (and .cursor/rules)
|
|
188
190
|
skills pull --status=proposed Include not-yet-promoted skills (default: approved)
|
|
189
191
|
skills sync Push local .claude/skills/**/SKILL.md up to Moda
|
|
@@ -309,6 +311,7 @@ Examples:
|
|
|
309
311
|
moda prompts status
|
|
310
312
|
moda prompts sync
|
|
311
313
|
moda prompts promote support.triage --label=prod --version=pver_abc123
|
|
314
|
+
moda prompts ab --baseline=prompts/agent.prompt.md --candidate=prompts/agent-v2.prompt.md --conversations=conv_1,conv_2
|
|
312
315
|
moda skills pull`);
|
|
313
316
|
}
|
|
314
317
|
// --- Command Handlers ---
|
|
@@ -1056,7 +1059,7 @@ export const commandRegistry = createCommandRegistry([
|
|
|
1056
1059
|
legacyCommand({
|
|
1057
1060
|
name: 'prompts',
|
|
1058
1061
|
description: 'Manage prompt versions and labels',
|
|
1059
|
-
examples: ['moda prompts status', 'moda prompts sync'],
|
|
1062
|
+
examples: ['moda prompts status', 'moda prompts sync', 'moda prompts ab --baseline=old.md --candidate=new.md'],
|
|
1060
1063
|
auth: 'api-key',
|
|
1061
1064
|
outputModes: ['human', 'json', 'agent'],
|
|
1062
1065
|
mutability: 'write',
|
|
@@ -1121,7 +1124,7 @@ export const commandRegistry = createCommandRegistry([
|
|
|
1121
1124
|
{
|
|
1122
1125
|
name: 'skills',
|
|
1123
1126
|
description: 'Generate, pull, and sync Moda skills',
|
|
1124
|
-
examples: ['moda skills gen', 'moda skills pull', 'moda skills sync'],
|
|
1127
|
+
examples: ['moda skills gen', 'moda skills status', 'moda skills pull', 'moda skills sync'],
|
|
1125
1128
|
auth: 'none',
|
|
1126
1129
|
outputModes: ['human', 'json', 'agent'],
|
|
1127
1130
|
mutability: 'write',
|
package/dist/doctor.js
CHANGED
|
@@ -7,7 +7,7 @@ import { isAuthSessionValid, loadAuthSession } from './auth.js';
|
|
|
7
7
|
import { resolveApiKey, resolveIngestUrl, resolveModaBaseUrl, resolveTenantId } from './config.js';
|
|
8
8
|
import { readHarnessSyncState, scanHarness } from './harness.js';
|
|
9
9
|
import { HARNESS_REPORT_APPROVAL_PATH, HARNESS_REPORT_APPROVAL_SCHEMA_VERSION, HARNESS_REPORT_PATH, hashHarnessReport, readHarnessReport, validateHarnessReport, } from './harness-report.js';
|
|
10
|
-
import { ensureSecretFilePermissions, getActiveProfile, getModaPaths, loadProfileConfig, resolveSecretReference, secretFilePathFromRef, } from './profiles.js';
|
|
10
|
+
import { ensureSecretFilePermissions, getActiveProfile, getModaPaths, isLoopbackBaseUrl, loadProfileConfig, resolveSecretReference, secretFilePathFromRef, } from './profiles.js';
|
|
11
11
|
import { renderDoctorHuman, renderStatusHuman } from './tui.js';
|
|
12
12
|
export async function runDoctorCommand(context) {
|
|
13
13
|
const report = await buildDoctorReport(context, {
|
|
@@ -132,6 +132,7 @@ export async function buildDoctorReport(context, options) {
|
|
|
132
132
|
}
|
|
133
133
|
checks.push(urlCheck('base_url_valid', 'config', 'Base URL is valid', baseUrl));
|
|
134
134
|
checks.push(urlCheck('ingest_url_valid', 'config', 'Ingest URL is valid', ingestUrl));
|
|
135
|
+
checks.push(localBaseUrlCheck(activeProfile.environment, baseUrl));
|
|
135
136
|
const manifestPath = join(context.cwd, '.moda', 'harness.json');
|
|
136
137
|
const manifestExists = existsSync(manifestPath);
|
|
137
138
|
checks.push({
|
|
@@ -287,6 +288,82 @@ function urlCheck(id, group, title, value) {
|
|
|
287
288
|
nextCommand: valid ? undefined : 'moda profiles create',
|
|
288
289
|
};
|
|
289
290
|
}
|
|
291
|
+
/**
|
|
292
|
+
* Local stacks mount `/auth/cli` and `/api/*` CLI/control proxies on the
|
|
293
|
+
* frontend (`:3000` by default). Pointing `base_url` at the backend (`:3001`)
|
|
294
|
+
* breaks browser login even though some Nest routes still answer there.
|
|
295
|
+
*/
|
|
296
|
+
function localBaseUrlCheck(environment, baseUrl) {
|
|
297
|
+
// Only the `local` profile targets a loopback worktree stack. `dev` defaults to
|
|
298
|
+
// remote `https://moda.dev`, so including it here warns valid remote configs.
|
|
299
|
+
if (environment !== 'local') {
|
|
300
|
+
return {
|
|
301
|
+
id: 'local_base_url_frontend',
|
|
302
|
+
group: 'config',
|
|
303
|
+
title: 'Local base URL targets the frontend',
|
|
304
|
+
status: 'skipped',
|
|
305
|
+
severity: 'info',
|
|
306
|
+
evidence: [{ label: 'environment', value: environment }],
|
|
307
|
+
};
|
|
308
|
+
}
|
|
309
|
+
let port = null;
|
|
310
|
+
let loopback = false;
|
|
311
|
+
try {
|
|
312
|
+
const parsed = new URL(baseUrl);
|
|
313
|
+
loopback = isLoopbackBaseUrl(baseUrl);
|
|
314
|
+
port = parsed.port || (parsed.protocol === 'https:' ? '443' : '80');
|
|
315
|
+
}
|
|
316
|
+
catch {
|
|
317
|
+
return {
|
|
318
|
+
id: 'local_base_url_frontend',
|
|
319
|
+
group: 'config',
|
|
320
|
+
title: 'Local base URL targets the frontend',
|
|
321
|
+
status: 'fail',
|
|
322
|
+
severity: 'critical',
|
|
323
|
+
evidence: [{ label: 'base_url', value: baseUrl }],
|
|
324
|
+
remediation: 'Set the local profile base URL to the frontend origin (http://localhost:3000).',
|
|
325
|
+
nextCommand: 'moda profiles create local --environment local --base-url http://localhost:3000',
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
const looksLikeBackend = loopback && (port === '3001' || port === '3002');
|
|
329
|
+
if (looksLikeBackend) {
|
|
330
|
+
return {
|
|
331
|
+
id: 'local_base_url_frontend',
|
|
332
|
+
group: 'config',
|
|
333
|
+
title: 'Local base URL targets the frontend',
|
|
334
|
+
status: 'warning',
|
|
335
|
+
severity: 'warning',
|
|
336
|
+
evidence: [
|
|
337
|
+
{ label: 'base_url', value: baseUrl },
|
|
338
|
+
{ label: 'port', value: port ?? 'unknown' },
|
|
339
|
+
],
|
|
340
|
+
remediation: 'Local CLI auth and control APIs are served by the frontend. ' +
|
|
341
|
+
'Use http://localhost:3000 (or your worktree frontend port), not the backend :3001.',
|
|
342
|
+
nextCommand: 'moda profiles create local --environment local --base-url http://localhost:3000 --use',
|
|
343
|
+
};
|
|
344
|
+
}
|
|
345
|
+
if (!loopback) {
|
|
346
|
+
return {
|
|
347
|
+
id: 'local_base_url_frontend',
|
|
348
|
+
group: 'config',
|
|
349
|
+
title: 'Local base URL targets the frontend',
|
|
350
|
+
status: 'warning',
|
|
351
|
+
severity: 'warning',
|
|
352
|
+
evidence: [{ label: 'base_url', value: baseUrl }],
|
|
353
|
+
remediation: 'Local profile should use a loopback frontend URL (http://localhost:3000). ' +
|
|
354
|
+
'Activate with `moda profiles use local` or set MODA_PROFILE=local.',
|
|
355
|
+
nextCommand: 'moda profiles use local',
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
return {
|
|
359
|
+
id: 'local_base_url_frontend',
|
|
360
|
+
group: 'config',
|
|
361
|
+
title: 'Local base URL targets the frontend',
|
|
362
|
+
status: 'pass',
|
|
363
|
+
severity: 'info',
|
|
364
|
+
evidence: [{ label: 'base_url', value: baseUrl }],
|
|
365
|
+
};
|
|
366
|
+
}
|
|
290
367
|
function harnessCurrentCheck(cwd, manifestPath, manifestExists, tenantId) {
|
|
291
368
|
if (!manifestExists) {
|
|
292
369
|
return {
|
|
@@ -11,6 +11,13 @@ export interface ArtifactBodyFileItem {
|
|
|
11
11
|
text?: string;
|
|
12
12
|
}
|
|
13
13
|
export declare function extractToolBodyFromText(text: string, toolName: string): string | undefined;
|
|
14
|
+
export declare function annotatedSourceCandidates(sourcePath: string, annotation: ArtifactBodyAnnotation): string[];
|
|
15
|
+
/**
|
|
16
|
+
* Resolve an annotated artifact `path=` to a repo-relative sourcePath.
|
|
17
|
+
* Prefer the nested annotation-relative file when present so later whole-file
|
|
18
|
+
* hydration and content hashing look in the right place.
|
|
19
|
+
*/
|
|
20
|
+
export declare function resolveAnnotatedSourcePath(sourcePath: string, annotation: ArtifactBodyAnnotation): string;
|
|
14
21
|
export declare function readPromptOrToolBody(type: HarnessArtifactType, sourcePath: string, annotation: ArtifactBodyAnnotation): string | undefined;
|
|
15
22
|
export declare function extractToolBody(file: ArtifactBodyFileItem, tool: {
|
|
16
23
|
name: string;
|
|
@@ -26,6 +26,21 @@ function isSecretishPath(path) {
|
|
|
26
26
|
function isOutsideRoot(rel) {
|
|
27
27
|
return rel === '' || rel === '..' || rel.startsWith('../') || isAbsolute(rel);
|
|
28
28
|
}
|
|
29
|
+
/** Walk from `fromAbs` up to `rootAbs` (inclusive), bounded by `maxDepth`. */
|
|
30
|
+
function walkAncestorDirs(fromAbs, rootAbs, maxDepth = 8) {
|
|
31
|
+
const dirs = [];
|
|
32
|
+
let cursor = fromAbs;
|
|
33
|
+
for (let depth = 0; depth < maxDepth; depth += 1) {
|
|
34
|
+
dirs.push(cursor);
|
|
35
|
+
if (normalizePath(cursor) === normalizePath(rootAbs))
|
|
36
|
+
break;
|
|
37
|
+
const parent = dirname(cursor);
|
|
38
|
+
if (parent === cursor)
|
|
39
|
+
break;
|
|
40
|
+
cursor = parent;
|
|
41
|
+
}
|
|
42
|
+
return dirs;
|
|
43
|
+
}
|
|
29
44
|
function readBoundedRepoText(absPath, rootDir) {
|
|
30
45
|
if (!existsSync(absPath))
|
|
31
46
|
return undefined;
|
|
@@ -113,18 +128,31 @@ export function extractToolBodyFromText(text, toolName) {
|
|
|
113
128
|
}
|
|
114
129
|
return undefined;
|
|
115
130
|
}
|
|
116
|
-
function
|
|
131
|
+
export function annotatedSourceCandidates(sourcePath, annotation) {
|
|
117
132
|
const explicitSourcePath = normalizePath(sourcePath) !== normalizePath(annotation.evidence.path);
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
133
|
+
// Relative `path=` annotations in nested packages usually mean "relative to the
|
|
134
|
+
// package/harness root that owns the annotated file", not the file's own directory.
|
|
135
|
+
// Example: harnesses/boardy-ops/src/ops.ts with path=prompts/ops-system.prompt.md
|
|
136
|
+
// resolves to harnesses/boardy-ops/prompts/ops-system.prompt.md (one level up from
|
|
137
|
+
// src/), not harnesses/boardy-ops/src/prompts/... Walk parents between the
|
|
138
|
+
// annotated file and the scan root, then fall back to the scan-root candidate
|
|
139
|
+
// for flat repos that annotate from the repository root.
|
|
140
|
+
if (!explicitSourcePath) {
|
|
141
|
+
return [
|
|
142
|
+
resolve(annotation.rootDir, sourcePath),
|
|
143
|
+
resolve(dirname(annotation.sourceAbsPath), sourcePath),
|
|
144
|
+
annotation.sourceAbsPath,
|
|
145
|
+
];
|
|
146
|
+
}
|
|
147
|
+
const rootAbs = resolve(annotation.rootDir);
|
|
148
|
+
const candidates = walkAncestorDirs(dirname(annotation.sourceAbsPath), rootAbs)
|
|
149
|
+
.map((dir) => resolve(dir, sourcePath));
|
|
150
|
+
candidates.push(resolve(annotation.rootDir, sourcePath));
|
|
151
|
+
return candidates;
|
|
152
|
+
}
|
|
153
|
+
function readAnnotatedSourceText(sourcePath, annotation) {
|
|
126
154
|
const seen = new Set();
|
|
127
|
-
for (const candidate of
|
|
155
|
+
for (const candidate of annotatedSourceCandidates(sourcePath, annotation)) {
|
|
128
156
|
if (seen.has(candidate))
|
|
129
157
|
continue;
|
|
130
158
|
seen.add(candidate);
|
|
@@ -134,6 +162,25 @@ function readAnnotatedSourceText(sourcePath, annotation) {
|
|
|
134
162
|
}
|
|
135
163
|
return undefined;
|
|
136
164
|
}
|
|
165
|
+
/**
|
|
166
|
+
* Resolve an annotated artifact `path=` to a repo-relative sourcePath.
|
|
167
|
+
* Prefer the nested annotation-relative file when present so later whole-file
|
|
168
|
+
* hydration and content hashing look in the right place.
|
|
169
|
+
*/
|
|
170
|
+
export function resolveAnnotatedSourcePath(sourcePath, annotation) {
|
|
171
|
+
const seen = new Set();
|
|
172
|
+
for (const candidate of annotatedSourceCandidates(sourcePath, annotation)) {
|
|
173
|
+
if (seen.has(candidate))
|
|
174
|
+
continue;
|
|
175
|
+
seen.add(candidate);
|
|
176
|
+
if (readBoundedRepoText(candidate, annotation.rootDir) === undefined)
|
|
177
|
+
continue;
|
|
178
|
+
const rel = normalizePath(relative(annotation.rootDir, candidate));
|
|
179
|
+
if (!isOutsideRoot(rel))
|
|
180
|
+
return rel;
|
|
181
|
+
}
|
|
182
|
+
return sourcePath;
|
|
183
|
+
}
|
|
137
184
|
export function readPromptOrToolBody(type, sourcePath, annotation) {
|
|
138
185
|
if (!requiresArtifactBody(type))
|
|
139
186
|
return undefined;
|
|
@@ -154,7 +201,8 @@ export function extractToolBody(file, tool) {
|
|
|
154
201
|
}
|
|
155
202
|
function hydrateArtifactBody(artifact, rootDir) {
|
|
156
203
|
if (artifact.sourcePath) {
|
|
157
|
-
const wholeFile = readRepoRelativeText(rootDir, artifact.sourcePath)
|
|
204
|
+
const wholeFile = readRepoRelativeText(rootDir, artifact.sourcePath)
|
|
205
|
+
?? readSourcePathRelativeToEvidence(artifact, rootDir);
|
|
158
206
|
if (wholeFile !== undefined) {
|
|
159
207
|
if (artifact.type === 'prompt' || isDedicatedToolFilePath(artifact.sourcePath)) {
|
|
160
208
|
return wholeFile;
|
|
@@ -174,6 +222,25 @@ function hydrateArtifactBody(artifact, rootDir) {
|
|
|
174
222
|
}
|
|
175
223
|
return undefined;
|
|
176
224
|
}
|
|
225
|
+
/**
|
|
226
|
+
* Fallback for nested relative `sourcePath` values that were recorded before the
|
|
227
|
+
* annotation-relative resolver existed: walk parent dirs beside the evidence file
|
|
228
|
+
* looking for the package-local prompt/tool path.
|
|
229
|
+
*/
|
|
230
|
+
function readSourcePathRelativeToEvidence(artifact, rootDir) {
|
|
231
|
+
if (!artifact.sourcePath)
|
|
232
|
+
return undefined;
|
|
233
|
+
const evidencePath = firstEvidencePath(artifact);
|
|
234
|
+
if (!evidencePath)
|
|
235
|
+
return undefined;
|
|
236
|
+
const rootAbs = resolve(rootDir);
|
|
237
|
+
for (const dir of walkAncestorDirs(dirname(resolve(rootDir, evidencePath)), rootAbs)) {
|
|
238
|
+
const text = readBoundedRepoText(resolve(dir, artifact.sourcePath), rootDir);
|
|
239
|
+
if (text !== undefined)
|
|
240
|
+
return text;
|
|
241
|
+
}
|
|
242
|
+
return undefined;
|
|
243
|
+
}
|
|
177
244
|
export function hydratePromptAndToolBodies(graph, rootDir) {
|
|
178
245
|
for (const artifact of graph.artifacts) {
|
|
179
246
|
if (!requiresArtifactBody(artifact.type) || artifactHasBody(artifact)) {
|
package/dist/harness.d.ts
CHANGED
|
@@ -273,6 +273,12 @@ export declare function emitWrittenArtifacts(context: CommandContext, written: s
|
|
|
273
273
|
export declare function repairExternalAnalystReport(report: HarnessReport, rootDir: string): HarnessReport;
|
|
274
274
|
export declare function sleep(ms: number): Promise<void>;
|
|
275
275
|
export declare function summarizeInvalidAnalystReport(report: unknown): Record<string, unknown>;
|
|
276
|
+
/**
|
|
277
|
+
* The analyst spellings that select the deterministic local scanner (no LLM,
|
|
278
|
+
* no remote analyze). Shared by `normalizeAnalystAdapter` and `moda init`'s
|
|
279
|
+
* `resolveHarnessRemote` so the set cannot drift between the two.
|
|
280
|
+
*/
|
|
281
|
+
export declare function isDeterministicLocalAnalyst(raw: string | undefined): boolean;
|
|
276
282
|
export type CodingAgentAdapter = ExternalHarnessAnalystAdapter;
|
|
277
283
|
export interface CodingAgentRun {
|
|
278
284
|
adapter: CodingAgentAdapter;
|
package/dist/harness.js
CHANGED
|
@@ -10,7 +10,7 @@ import { resolveApiKey, resolveTenantId } from './config.js';
|
|
|
10
10
|
import { CliInputError, CliInputRequiredError } from './errors.js';
|
|
11
11
|
import { isRemoteHarnessAnalyze, runRemoteHarnessAnalyst } from './harness-remote.js';
|
|
12
12
|
import { getActiveProfile } from './profiles.js';
|
|
13
|
-
import { enrichScanCaptureMetadata, extractToolBody, hydratePromptAndToolBodies, readPromptOrToolBody, } from './harness-artifact-bodies.js';
|
|
13
|
+
import { enrichScanCaptureMetadata, annotatedSourceCandidates, extractToolBody, hydratePromptAndToolBodies, readPromptOrToolBody, resolveAnnotatedSourcePath, } from './harness-artifact-bodies.js';
|
|
14
14
|
import { approveHarnessReport, createHarnessReport, graphForReport, hashHarnessReport, hashStableJson as hashHarnessReportStableJson, HARNESS_REPORT_PATH, loadHarnessReportForSync, normalizeHarnessReport, readHarnessReport, resolveHarnessReportPath, validateHarnessReport, writeHarnessReportArtifacts, } from './harness-report.js';
|
|
15
15
|
import { assertHarnessGraphSyncSize } from './harness-sync-limits.js';
|
|
16
16
|
import { renderHarnessAnalystChoice, renderHarnessAnalystLaunch, renderHarnessAnalystPreflight, renderHarnessAnalyzeHumanResult, renderHarnessProgress, } from './tui.js';
|
|
@@ -1201,6 +1201,15 @@ function shouldPromptForHarnessAnalyst(context) {
|
|
|
1201
1201
|
!context.flags['analyst-command'] &&
|
|
1202
1202
|
!process.env.MODA_HARNESS_ANALYST_COMMAND;
|
|
1203
1203
|
}
|
|
1204
|
+
/**
|
|
1205
|
+
* The analyst spellings that select the deterministic local scanner (no LLM,
|
|
1206
|
+
* no remote analyze). Shared by `normalizeAnalystAdapter` and `moda init`'s
|
|
1207
|
+
* `resolveHarnessRemote` so the set cannot drift between the two.
|
|
1208
|
+
*/
|
|
1209
|
+
export function isDeterministicLocalAnalyst(raw) {
|
|
1210
|
+
const value = raw?.trim().toLowerCase();
|
|
1211
|
+
return value === 'local-scan' || value === 'local' || value === 'scripted' || value === 'moda';
|
|
1212
|
+
}
|
|
1204
1213
|
function normalizeAnalystAdapter(raw) {
|
|
1205
1214
|
const value = (raw && raw !== 'true' ? raw : 'auto').trim().toLowerCase();
|
|
1206
1215
|
if (value === 'auto')
|
|
@@ -1211,7 +1220,7 @@ function normalizeAnalystAdapter(raw) {
|
|
|
1211
1220
|
return 'codex';
|
|
1212
1221
|
if (value === 'cursor' || value === 'cursor-cli' || value === 'cursor_cli')
|
|
1213
1222
|
return 'cursor';
|
|
1214
|
-
if (value
|
|
1223
|
+
if (isDeterministicLocalAnalyst(value))
|
|
1215
1224
|
return 'local-scan';
|
|
1216
1225
|
throw new CliInputError(`Unknown harness analyst '${raw}'.`, 'Use `--analyst=claude|codex|cursor|local-scan`.');
|
|
1217
1226
|
}
|
|
@@ -2439,9 +2448,17 @@ function channelFromAnnotation(annotation) {
|
|
|
2439
2448
|
function artifactFromAnnotation(annotation) {
|
|
2440
2449
|
const values = annotation.values;
|
|
2441
2450
|
const type = artifactType(first(values.kind) ?? first(values.type));
|
|
2442
|
-
const
|
|
2451
|
+
const annotatedPath = first(values.path) ?? annotation.evidence.path;
|
|
2452
|
+
// Canonicalize nested relative path= annotations to a repo-relative location
|
|
2453
|
+
// so later whole-file body hydration and hashing find the real prompt/tool file.
|
|
2454
|
+
const sourcePath = resolveAnnotatedSourcePath(annotatedPath, annotation);
|
|
2443
2455
|
const id = first(values.id) ?? stableId(type, first(values.name) ?? sourcePath);
|
|
2444
|
-
|
|
2456
|
+
// Body reads from the RAW annotated path on purpose: readPromptOrToolBody walks
|
|
2457
|
+
// the annotation's ancestor dirs itself to locate nested-package files, so it
|
|
2458
|
+
// must not be handed the already-canonicalized sourcePath. sourcePath and
|
|
2459
|
+
// contentHash use the canonical path so downstream whole-file hydration and
|
|
2460
|
+
// hashing resolve the same file the body came from.
|
|
2461
|
+
const body = readPromptOrToolBody(type, annotatedPath, annotation);
|
|
2445
2462
|
return {
|
|
2446
2463
|
id,
|
|
2447
2464
|
type,
|
|
@@ -3420,14 +3437,13 @@ function upsertDeployment(graph, next) {
|
|
|
3420
3437
|
existing.environmentId = existing.environmentId ?? next.environmentId;
|
|
3421
3438
|
}
|
|
3422
3439
|
function contentHashForAnnotatedSource(sourcePath, annotation) {
|
|
3423
|
-
|
|
3424
|
-
|
|
3425
|
-
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
3429
|
-
|
|
3430
|
-
annotation.sourceAbsPath,
|
|
3440
|
+
// Reuse the annotation-relative candidate resolver; prefer the canonicalized
|
|
3441
|
+
// repo-relative path first so hashing matches hydrated artifact bodies.
|
|
3442
|
+
const bodyAnnotation = annotation;
|
|
3443
|
+
const rootCandidate = resolve(annotation.rootDir, sourcePath);
|
|
3444
|
+
const candidates = [
|
|
3445
|
+
rootCandidate,
|
|
3446
|
+
...annotatedSourceCandidates(sourcePath, bodyAnnotation).filter((c) => c !== rootCandidate),
|
|
3431
3447
|
];
|
|
3432
3448
|
const seen = new Set();
|
|
3433
3449
|
for (const candidate of candidates) {
|
package/dist/init/index.d.ts
CHANGED
|
@@ -30,8 +30,9 @@ export interface InitRunOptions {
|
|
|
30
30
|
export declare function runInit(flags: Record<string, unknown>, options?: InitRunOptions): Promise<InitResult>;
|
|
31
31
|
/**
|
|
32
32
|
* Onboarding runs harness discovery SERVER-SIDE by default so the customer
|
|
33
|
-
* needs no local coding agent. Opt out with `--local`, `--no-remote`,
|
|
34
|
-
* `--remote=false
|
|
33
|
+
* needs no local coding agent. Opt out with `--local`, `--no-remote`,
|
|
34
|
+
* `--remote=false`, or an explicit deterministic local analyst
|
|
35
|
+
* (`--analyst=local-scan|local|scripted|moda`).
|
|
35
36
|
*/
|
|
36
37
|
export declare function resolveHarnessRemote(flags: Record<string, unknown>): boolean;
|
|
37
38
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/init/index.js
CHANGED
|
@@ -15,7 +15,7 @@ import { runSdkIntegration } from './sdk-integration.js';
|
|
|
15
15
|
import { createCommandContext } from '../command-context.js';
|
|
16
16
|
import { VERSION } from '../config.js';
|
|
17
17
|
import { CliAuthError, CliInputError } from '../errors.js';
|
|
18
|
-
import { runHarnessCommand } from '../harness.js';
|
|
18
|
+
import { isDeterministicLocalAnalyst, runHarnessCommand } from '../harness.js';
|
|
19
19
|
import { buildDoctorReport } from '../doctor.js';
|
|
20
20
|
import { renderInitBanner, renderInitComplete, renderInitNote, renderInitSetupPlan, renderInitStep, } from '../tui.js';
|
|
21
21
|
export async function runInit(flags, options = {}) {
|
|
@@ -676,8 +676,9 @@ function isTruthyFlag(value) {
|
|
|
676
676
|
}
|
|
677
677
|
/**
|
|
678
678
|
* Onboarding runs harness discovery SERVER-SIDE by default so the customer
|
|
679
|
-
* needs no local coding agent. Opt out with `--local`, `--no-remote`,
|
|
680
|
-
* `--remote=false
|
|
679
|
+
* needs no local coding agent. Opt out with `--local`, `--no-remote`,
|
|
680
|
+
* `--remote=false`, or an explicit deterministic local analyst
|
|
681
|
+
* (`--analyst=local-scan|local|scripted|moda`).
|
|
681
682
|
*/
|
|
682
683
|
export function resolveHarnessRemote(flags) {
|
|
683
684
|
if (isTruthyFlag(flagValue(flags, 'local')))
|
|
@@ -687,6 +688,13 @@ export function resolveHarnessRemote(flags) {
|
|
|
687
688
|
const remote = parseOptionalBoolean(flagValue(flags, 'remote'));
|
|
688
689
|
if (remote !== undefined)
|
|
689
690
|
return remote;
|
|
691
|
+
// Scenarios and users that pin `--analyst=local-scan` mean "run the local
|
|
692
|
+
// deterministic scanner", not "upload to hosted analyze and poll". Treat that
|
|
693
|
+
// as an implicit local opt-out so moda-test live/onboarding suites and
|
|
694
|
+
// developer workflows do not hang waiting on remote analyze.
|
|
695
|
+
if (isDeterministicLocalAnalyst(stringFlag(flagValue(flags, 'analyst')))) {
|
|
696
|
+
return false;
|
|
697
|
+
}
|
|
690
698
|
return true;
|
|
691
699
|
}
|
|
692
700
|
function flagValue(flags, ...keys) {
|
package/dist/init/tenant.js
CHANGED
|
@@ -360,7 +360,8 @@ export async function selectTenantAndCreateKey(options = {}) {
|
|
|
360
360
|
'This organization has reached its CLI API key limit.', {
|
|
361
361
|
// Command-agnostic: this path is shared by `moda init` and
|
|
362
362
|
// `moda provision`.
|
|
363
|
-
guidance: 'Revoke
|
|
363
|
+
guidance: 'Revoke unused "CLI - ..." keys in the dashboard (Settings → API Keys), ' +
|
|
364
|
+
'or reuse the existing key already saved in ~/.moda/config.json from a previous moda init / moda provision --save, then retry.',
|
|
364
365
|
});
|
|
365
366
|
}
|
|
366
367
|
throw new Error(`Failed to create API key (409): ${body}`);
|
package/dist/manifest.js
CHANGED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { CommandContext } from './command-context.js';
|
|
2
|
+
import type { ProfileIOOptions } from './profiles.js';
|
|
3
|
+
export interface PromptArmPayload {
|
|
4
|
+
content: string;
|
|
5
|
+
prompt_version_id?: string;
|
|
6
|
+
prompt_id?: string;
|
|
7
|
+
prompt_label?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface PromptAbResult {
|
|
10
|
+
replaySetId: string;
|
|
11
|
+
runId: string;
|
|
12
|
+
status: string;
|
|
13
|
+
verdict: string;
|
|
14
|
+
run: Record<string, unknown> | null;
|
|
15
|
+
cases: Array<Record<string, unknown>>;
|
|
16
|
+
}
|
|
17
|
+
export declare function runPromptAb(flags: Record<string, string>, profileOptions: ProfileIOOptions, context: CommandContext): Promise<void>;
|
|
18
|
+
//# sourceMappingURL=prompts-ab.d.ts.map
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { isAbsolute, resolve } from 'node:path';
|
|
3
|
+
import { callControlAPI, callDataAPI } from './api-client.js';
|
|
4
|
+
import { resolveTenantId, validateConfig } from './config.js';
|
|
5
|
+
import { discoverPrompts, parsePromptFileForAb, readPromptLock, runPromptSync, } from './prompts.js';
|
|
6
|
+
const DEFAULT_SEEDS_PER_CASE = 3;
|
|
7
|
+
const DEFAULT_POLL_INTERVAL_MS = 15_000;
|
|
8
|
+
const DEFAULT_WAIT_TIMEOUT_MS = 2 * 60 * 60 * 1000;
|
|
9
|
+
const TERMINAL_RUN_STATUSES = new Set(['completed', 'skipped', 'error']);
|
|
10
|
+
export async function runPromptAb(flags, profileOptions, context) {
|
|
11
|
+
validateConfig();
|
|
12
|
+
const tenantId = flags['tenant-id'] || resolveTenantId(profileOptions);
|
|
13
|
+
if (!tenantId) {
|
|
14
|
+
throw new Error('Missing tenant id. Run `moda init` or pass --tenant-id=<id>.');
|
|
15
|
+
}
|
|
16
|
+
const baselineSource = flags.baseline || flags['baseline-file'] || flags['baseline-key'];
|
|
17
|
+
const candidateSource = flags.candidate || flags['candidate-file'] || flags['candidate-key'];
|
|
18
|
+
if (!baselineSource || !candidateSource) {
|
|
19
|
+
throw new Error('Usage: moda prompts ab --baseline=<file|key> --candidate=<file|key> [--auto-generate|--set-id=ID|--conversations=id1,id2]');
|
|
20
|
+
}
|
|
21
|
+
if (flags.sync === 'true') {
|
|
22
|
+
await runPromptSync({ ...flags, watch: 'false' }, profileOptions);
|
|
23
|
+
}
|
|
24
|
+
const promptArms = {
|
|
25
|
+
prod: resolvePromptArmSpec(baselineSource, 'baseline'),
|
|
26
|
+
proposed: resolvePromptArmSpec(candidateSource, 'candidate'),
|
|
27
|
+
};
|
|
28
|
+
const replaySetId = await ensureReplaySet(flags, tenantId, profileOptions);
|
|
29
|
+
const seedsPerCase = parsePositiveInt(flags.seeds ?? flags['seeds-per-case'], DEFAULT_SEEDS_PER_CASE, 10);
|
|
30
|
+
const assistantModel = (flags.model ?? flags['assistant-model'] ?? '').trim();
|
|
31
|
+
const enqueue = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/run`, {
|
|
32
|
+
method: 'POST',
|
|
33
|
+
body: JSON.stringify({
|
|
34
|
+
promptArms,
|
|
35
|
+
seedsPerCase,
|
|
36
|
+
assistantModel: assistantModel || undefined,
|
|
37
|
+
promotePrimary: flags['no-promote'] !== 'true',
|
|
38
|
+
}),
|
|
39
|
+
}, profileOptions, { timeoutMs: 120_000, retries: 1 });
|
|
40
|
+
const noWait = flags.wait === 'false' || flags['no-wait'] === 'true';
|
|
41
|
+
if (noWait) {
|
|
42
|
+
context.output.writeData({
|
|
43
|
+
replaySetId,
|
|
44
|
+
runId: enqueue.runId,
|
|
45
|
+
status: enqueue.status,
|
|
46
|
+
message: enqueue.message ?? 'Replay comparison queued',
|
|
47
|
+
});
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
const timeoutMs = parsePositiveInt(flags.timeout, DEFAULT_WAIT_TIMEOUT_MS, 24 * 60 * 60 * 1000);
|
|
51
|
+
const pollIntervalMs = parsePositiveInt(flags['poll-interval'], DEFAULT_POLL_INTERVAL_MS, 120_000);
|
|
52
|
+
const deadline = Date.now() + timeoutMs;
|
|
53
|
+
let latest = null;
|
|
54
|
+
while (Date.now() < deadline) {
|
|
55
|
+
latest = await fetchLatestRun(tenantId, replaySetId, enqueue.runId, profileOptions);
|
|
56
|
+
const status = latest.run?.status ?? '';
|
|
57
|
+
const runId = latest.run?.runId ?? '';
|
|
58
|
+
if (TERMINAL_RUN_STATUSES.has(status) && runId === enqueue.runId) {
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
if (context.outputMode === 'human') {
|
|
62
|
+
process.stderr.write(`Waiting for replay run (${status || 'pending'})...\n`);
|
|
63
|
+
}
|
|
64
|
+
await sleep(pollIntervalMs);
|
|
65
|
+
}
|
|
66
|
+
if (!latest?.run
|
|
67
|
+
|| latest.run.runId !== enqueue.runId
|
|
68
|
+
|| !TERMINAL_RUN_STATUSES.has(latest.run.status)) {
|
|
69
|
+
throw new Error(`Timed out after ${timeoutMs}ms waiting for replay run ${enqueue.runId} to finish`);
|
|
70
|
+
}
|
|
71
|
+
const result = {
|
|
72
|
+
replaySetId,
|
|
73
|
+
runId: latest.run.runId,
|
|
74
|
+
status: latest.run.status,
|
|
75
|
+
verdict: formatVerdict(latest),
|
|
76
|
+
run: { ...latest.run },
|
|
77
|
+
cases: latest.cases.map((row) => ({ ...row })),
|
|
78
|
+
};
|
|
79
|
+
if (context.outputMode === 'human') {
|
|
80
|
+
printHumanVerdict(result);
|
|
81
|
+
}
|
|
82
|
+
context.output.writeData(result);
|
|
83
|
+
}
|
|
84
|
+
function resolvePromptArmSpec(source, label) {
|
|
85
|
+
const trimmed = source.trim();
|
|
86
|
+
if (!trimmed) {
|
|
87
|
+
throw new Error(`Empty prompt source for ${label}`);
|
|
88
|
+
}
|
|
89
|
+
const absPath = isAbsolute(trimmed) ? trimmed : resolve(process.cwd(), trimmed);
|
|
90
|
+
let key;
|
|
91
|
+
let content;
|
|
92
|
+
if (existsSync(absPath)) {
|
|
93
|
+
const parsed = parsePromptFileForAb(absPath);
|
|
94
|
+
key = parsed.key;
|
|
95
|
+
content = parsed.content;
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
const discovered = discoverPrompts();
|
|
99
|
+
const match = discovered.find((prompt) => prompt.key === trimmed)
|
|
100
|
+
?? discovered.find((prompt) => prompt.sourcePath === trimmed);
|
|
101
|
+
if (!match) {
|
|
102
|
+
throw new Error(`Could not resolve prompt '${trimmed}' — pass a .prompt.md path or a discovered prompt key`);
|
|
103
|
+
}
|
|
104
|
+
key = match.key;
|
|
105
|
+
content = match.content;
|
|
106
|
+
}
|
|
107
|
+
const locked = readPromptLock()?.prompts[key];
|
|
108
|
+
return {
|
|
109
|
+
content,
|
|
110
|
+
prompt_version_id: locked?.versionId ?? '',
|
|
111
|
+
prompt_id: locked?.promptId ?? '',
|
|
112
|
+
prompt_label: label,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
async function ensureReplaySet(flags, tenantId, profileOptions) {
|
|
116
|
+
const existingSetId = (flags['set-id'] || flags['replay-set-id'] || '').trim();
|
|
117
|
+
if (existingSetId) {
|
|
118
|
+
return existingSetId;
|
|
119
|
+
}
|
|
120
|
+
const conversations = parseCsv(flags.conversations ?? flags['conversation-ids']);
|
|
121
|
+
if (conversations.length) {
|
|
122
|
+
return createSetFromConversations(conversations, flags, tenantId, profileOptions);
|
|
123
|
+
}
|
|
124
|
+
if (flags['auto-generate'] === 'false' && !existingSetId) {
|
|
125
|
+
throw new Error('Provide --set-id=, --conversations=, or allow --auto-generate (default)');
|
|
126
|
+
}
|
|
127
|
+
const caseCount = parsePositiveInt(flags.cases ?? flags['case-count'], 5, 25);
|
|
128
|
+
const lookbackDays = parsePositiveInt(flags['lookback-days'], 30, 365);
|
|
129
|
+
const name = (flags.name ?? flags['set-name'] ?? `Prompt A/B ${new Date().toISOString().slice(0, 10)}`).trim();
|
|
130
|
+
const generated = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/auto-generate`, {
|
|
131
|
+
method: 'POST',
|
|
132
|
+
body: JSON.stringify({ name, caseCount, lookbackDays }),
|
|
133
|
+
}, profileOptions, { timeoutMs: 300_000, retries: 0 });
|
|
134
|
+
if (!generated?.id) {
|
|
135
|
+
throw new Error('Auto-generate did not return a replay set id');
|
|
136
|
+
}
|
|
137
|
+
return generated.id;
|
|
138
|
+
}
|
|
139
|
+
async function createSetFromConversations(conversationIds, flags, tenantId, profileOptions) {
|
|
140
|
+
const name = (flags.name ?? flags['set-name'] ?? `Prompt A/B ${conversationIds.length} conv`).trim();
|
|
141
|
+
const created = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets`, {
|
|
142
|
+
method: 'POST',
|
|
143
|
+
body: JSON.stringify({ name, description: 'Created by moda prompts ab' }),
|
|
144
|
+
}, profileOptions);
|
|
145
|
+
let position = 0;
|
|
146
|
+
for (const conversationId of conversationIds) {
|
|
147
|
+
const scenario = await loadScenarioFromConversation(conversationId);
|
|
148
|
+
await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(created.id)}/cases`, {
|
|
149
|
+
method: 'POST',
|
|
150
|
+
body: JSON.stringify({
|
|
151
|
+
title: conversationId,
|
|
152
|
+
scenario,
|
|
153
|
+
sourceConversationId: conversationId,
|
|
154
|
+
successCriteria: [
|
|
155
|
+
'The agent understands the user request',
|
|
156
|
+
'The agent uses tools appropriately when needed',
|
|
157
|
+
'The agent provides a helpful final answer',
|
|
158
|
+
],
|
|
159
|
+
position: position++,
|
|
160
|
+
}),
|
|
161
|
+
}, profileOptions);
|
|
162
|
+
}
|
|
163
|
+
return created.id;
|
|
164
|
+
}
|
|
165
|
+
async function loadScenarioFromConversation(conversationId) {
|
|
166
|
+
try {
|
|
167
|
+
const data = await callDataAPI(`/conversations/${encodeURIComponent(conversationId)}/context?msg_index=0&window=0`);
|
|
168
|
+
const firstUser = (data.messages ?? []).find((message) => message.role === 'user' && String(message.content ?? '').trim());
|
|
169
|
+
const opening = String(firstUser?.content ?? '').trim();
|
|
170
|
+
if (opening) {
|
|
171
|
+
return `The user says: "${opening}"\n\nReplay the conversation faithfully and help them accomplish their goal.`;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
// Fall through to generic scenario.
|
|
176
|
+
}
|
|
177
|
+
return `Replay conversation ${conversationId}: help the user accomplish their original goal using the available tools.`;
|
|
178
|
+
}
|
|
179
|
+
async function fetchLatestRun(tenantId, replaySetId, runId, profileOptions) {
|
|
180
|
+
// Pin the poll to OUR run id — otherwise a concurrent comparison on the same
|
|
181
|
+
// set would make `/runs/latest` return the newer run forever and this command
|
|
182
|
+
// would stall until timeout even after its own run finished.
|
|
183
|
+
return callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/runs/latest?runId=${encodeURIComponent(runId)}`, { method: 'GET' }, profileOptions, { timeoutMs: 60_000, retries: 2 });
|
|
184
|
+
}
|
|
185
|
+
function formatVerdict(payload) {
|
|
186
|
+
const run = payload.run;
|
|
187
|
+
if (!run)
|
|
188
|
+
return 'no run';
|
|
189
|
+
if (run.status === 'skipped')
|
|
190
|
+
return `skipped: ${run.notes || 'no reason'}`;
|
|
191
|
+
if (run.status === 'error')
|
|
192
|
+
return `error: ${run.notes || 'unknown error'}`;
|
|
193
|
+
if (run.status !== 'completed')
|
|
194
|
+
return `status=${run.status}`;
|
|
195
|
+
const delta = run.proposedPassCount - run.prodPassCount;
|
|
196
|
+
const winner = delta > 0 ? 'candidate wins' : delta < 0 ? 'baseline wins' : 'tie on case passes';
|
|
197
|
+
const rates = `baseline ${run.prodPassCount}/${run.caseCount} (${pct(run.prodPassRate)}), candidate ${run.proposedPassCount}/${run.caseCount} (${pct(run.proposedPassRate)})`;
|
|
198
|
+
const notes = run.notes?.trim();
|
|
199
|
+
return `${winner}; ${rates}${notes ? `; ${notes}` : ''}`;
|
|
200
|
+
}
|
|
201
|
+
function printHumanVerdict(result) {
|
|
202
|
+
process.stderr.write('\nPrompt A/B replay verdict\n');
|
|
203
|
+
process.stderr.write('-------------------------\n');
|
|
204
|
+
process.stderr.write(`Replay set: ${result.replaySetId}\n`);
|
|
205
|
+
process.stderr.write(`Run id: ${result.runId}\n`);
|
|
206
|
+
process.stderr.write(`Status: ${result.status}\n`);
|
|
207
|
+
process.stderr.write(`Verdict: ${result.verdict}\n`);
|
|
208
|
+
if (result.cases.length) {
|
|
209
|
+
process.stderr.write('\nPer-case:\n');
|
|
210
|
+
for (const row of result.cases) {
|
|
211
|
+
const caseId = String(row.caseId ?? '');
|
|
212
|
+
const prod = formatPass(row.prodPass);
|
|
213
|
+
const proposed = formatPass(row.proposedPass);
|
|
214
|
+
process.stderr.write(` ${caseId}: baseline=${prod} candidate=${proposed}\n`);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
process.stderr.write('\n');
|
|
218
|
+
}
|
|
219
|
+
function formatPass(value) {
|
|
220
|
+
if (value === null || value === undefined)
|
|
221
|
+
return 'abstained';
|
|
222
|
+
return value ? 'pass' : 'fail';
|
|
223
|
+
}
|
|
224
|
+
function pct(rate) {
|
|
225
|
+
return `${Math.round(Math.max(0, Math.min(1, rate)) * 100)}%`;
|
|
226
|
+
}
|
|
227
|
+
function parsePositiveInt(raw, fallback, max) {
|
|
228
|
+
if (!raw)
|
|
229
|
+
return fallback;
|
|
230
|
+
const parsed = Number.parseInt(raw, 10);
|
|
231
|
+
if (!Number.isFinite(parsed) || parsed < 1)
|
|
232
|
+
return fallback;
|
|
233
|
+
return Math.min(parsed, max);
|
|
234
|
+
}
|
|
235
|
+
function parseCsv(raw) {
|
|
236
|
+
if (!raw?.trim())
|
|
237
|
+
return [];
|
|
238
|
+
return raw.split(',').map((part) => part.trim()).filter(Boolean);
|
|
239
|
+
}
|
|
240
|
+
function sleep(ms) {
|
|
241
|
+
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
|
242
|
+
}
|
|
243
|
+
//# sourceMappingURL=prompts-ab.js.map
|
package/dist/prompts.d.ts
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
import { type CommandContext } from './command-context.js';
|
|
2
2
|
import type { ProfileIOOptions } from './profiles.js';
|
|
3
|
+
interface PromptFile {
|
|
4
|
+
key: string;
|
|
5
|
+
name: string;
|
|
6
|
+
description?: string;
|
|
7
|
+
category?: string;
|
|
8
|
+
sourcePath: string;
|
|
9
|
+
content: string;
|
|
10
|
+
systemPrompt?: string;
|
|
11
|
+
messages?: unknown[];
|
|
12
|
+
variables?: unknown[];
|
|
13
|
+
variablesSchema?: Record<string, unknown>;
|
|
14
|
+
modelConfig?: Record<string, unknown>;
|
|
15
|
+
toolRefs?: unknown[];
|
|
16
|
+
responseSchema?: Record<string, unknown>;
|
|
17
|
+
contentHash: string;
|
|
18
|
+
}
|
|
19
|
+
interface PromptLock {
|
|
20
|
+
version: 1;
|
|
21
|
+
generatedAt: string;
|
|
22
|
+
prompts: Record<string, {
|
|
23
|
+
key: string;
|
|
24
|
+
sourcePath: string;
|
|
25
|
+
contentHash: string;
|
|
26
|
+
versionId?: string;
|
|
27
|
+
promptId?: string;
|
|
28
|
+
}>;
|
|
29
|
+
}
|
|
3
30
|
export declare function runPromptsCommand(input: CommandContext | string[], legacyFlags?: Record<string, string>, legacyProfileOptions?: ProfileIOOptions): Promise<void>;
|
|
4
31
|
export declare function initPrompts(options?: {
|
|
5
32
|
silent?: boolean;
|
|
@@ -20,4 +47,12 @@ export interface PromptSyncResult {
|
|
|
20
47
|
* control their own output. `moda prompts sync` wraps this with JSON output.
|
|
21
48
|
*/
|
|
22
49
|
export declare function runPromptSync(flags: Record<string, string>, profileOptions: ProfileIOOptions): Promise<PromptSyncResult>;
|
|
50
|
+
export declare function discoverPrompts(): PromptFile[];
|
|
51
|
+
export declare function readPromptLock(): PromptLock | null;
|
|
52
|
+
/** Parse a prompt file for A/B runs (body text + registry key). */
|
|
53
|
+
export declare function parsePromptFileForAb(absPath: string): {
|
|
54
|
+
key: string;
|
|
55
|
+
content: string;
|
|
56
|
+
};
|
|
57
|
+
export {};
|
|
23
58
|
//# sourceMappingURL=prompts.d.ts.map
|
package/dist/prompts.js
CHANGED
|
@@ -4,6 +4,7 @@ import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, writeFileS
|
|
|
4
4
|
import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
|
|
5
5
|
import { callControlAPI } from './api-client.js';
|
|
6
6
|
import { createCommandContext } from './command-context.js';
|
|
7
|
+
import { runPromptAb } from './prompts-ab.js';
|
|
7
8
|
const MODA_DIR = '.moda';
|
|
8
9
|
const MANIFEST_PATH = join(MODA_DIR, 'prompts.yml');
|
|
9
10
|
const LOCK_PATH = join(MODA_DIR, 'prompts.lock.json');
|
|
@@ -51,8 +52,11 @@ export async function runPromptsCommand(input, legacyFlags, legacyProfileOptions
|
|
|
51
52
|
case 'promote':
|
|
52
53
|
await promotePrompt(positionals, flags, profileOptions, context);
|
|
53
54
|
break;
|
|
55
|
+
case 'ab':
|
|
56
|
+
await runPromptAb(flags, profileOptions, context);
|
|
57
|
+
break;
|
|
54
58
|
default:
|
|
55
|
-
throw new Error(`Unknown prompts command '${subcommand}'. Use init, status, diff, sync, or
|
|
59
|
+
throw new Error(`Unknown prompts command '${subcommand}'. Use init, status, diff, sync, promote, or ab.`);
|
|
56
60
|
}
|
|
57
61
|
}
|
|
58
62
|
export async function initPrompts(options = {}) {
|
|
@@ -195,7 +199,7 @@ async function getPromptStatus() {
|
|
|
195
199
|
prompts: rows,
|
|
196
200
|
};
|
|
197
201
|
}
|
|
198
|
-
function discoverPrompts() {
|
|
202
|
+
export function discoverPrompts() {
|
|
199
203
|
const manifest = readManifest();
|
|
200
204
|
// A pattern like `src/prompts/**/*.prompt.md` should *only* match files under
|
|
201
205
|
// src/prompts. Extract both the base directory (everything up to the first
|
|
@@ -290,6 +294,14 @@ function readManifest() {
|
|
|
290
294
|
.filter((path) => path.includes('.prompt.'));
|
|
291
295
|
return { promptPaths: promptPaths.length ? promptPaths : DEFAULT_PROMPT_PATHS };
|
|
292
296
|
}
|
|
297
|
+
export function readPromptLock() {
|
|
298
|
+
return readLock();
|
|
299
|
+
}
|
|
300
|
+
/** Parse a prompt file for A/B runs (body text + registry key). */
|
|
301
|
+
export function parsePromptFileForAb(absPath) {
|
|
302
|
+
const parsed = parsePromptFile(absPath);
|
|
303
|
+
return { key: parsed.key, content: parsed.content };
|
|
304
|
+
}
|
|
293
305
|
function readLock() {
|
|
294
306
|
if (!existsSync(LOCK_PATH))
|
|
295
307
|
return null;
|
package/dist/skills.js
CHANGED
|
@@ -74,6 +74,108 @@ async function fetchSkills(status) {
|
|
|
74
74
|
status,
|
|
75
75
|
};
|
|
76
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* Authenticated GET against the ingest worker's `/v1/skills/runs` surface.
|
|
79
|
+
* Returns `{ status, body }` so callers can treat 404 (no such run) as data;
|
|
80
|
+
* every other non-2xx throws.
|
|
81
|
+
*/
|
|
82
|
+
async function skillRunsRequest(url) {
|
|
83
|
+
const apiKey = resolveApiKey();
|
|
84
|
+
if (!apiKey) {
|
|
85
|
+
throw new CliInputError('No Moda API key found. Run `moda init` first (or set MODA_API_KEY).');
|
|
86
|
+
}
|
|
87
|
+
const controller = new AbortController();
|
|
88
|
+
const timeout = setTimeout(() => controller.abort(), SKILLS_REQUEST_TIMEOUT_MS);
|
|
89
|
+
let response;
|
|
90
|
+
try {
|
|
91
|
+
response = await fetch(url, {
|
|
92
|
+
headers: { Authorization: `Bearer ${apiKey}` },
|
|
93
|
+
signal: controller.signal,
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
98
|
+
throw new Error(`Network error: failed to reach ${url} (${message})`);
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
clearTimeout(timeout);
|
|
102
|
+
}
|
|
103
|
+
const text = await response.text().catch(() => '');
|
|
104
|
+
if (!response.ok && response.status !== 404) {
|
|
105
|
+
throw new Error(`Failed to fetch run status (HTTP ${response.status}): ${text.slice(0, 200)}`);
|
|
106
|
+
}
|
|
107
|
+
let body = {};
|
|
108
|
+
try {
|
|
109
|
+
body = text ? JSON.parse(text) : {};
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
body = {};
|
|
113
|
+
}
|
|
114
|
+
return { status: response.status, body };
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Resolve a run's status + recent events. Without a run id, the tenant's
|
|
118
|
+
* latest run (`GET /v1/skills/runs?limit=1`) is used; the detail form
|
|
119
|
+
* (`GET /v1/skills/runs/:id`) always supplies the event breadcrumbs.
|
|
120
|
+
*/
|
|
121
|
+
async function fetchSkillRunStatus(runId) {
|
|
122
|
+
const ingestUrl = resolveIngestUrl().replace(/\/$/, '');
|
|
123
|
+
let resolvedRunId = runId;
|
|
124
|
+
if (!resolvedRunId) {
|
|
125
|
+
const list = await skillRunsRequest(`${ingestUrl}/v1/skills/runs?limit=1`);
|
|
126
|
+
const listBody = list.body;
|
|
127
|
+
const latest = Array.isArray(listBody.runs) ? listBody.runs[0] : undefined;
|
|
128
|
+
if (!latest)
|
|
129
|
+
return { run: null, events: [] };
|
|
130
|
+
resolvedRunId = latest.generation_run_id;
|
|
131
|
+
// Defensive: a row without an id can still be shown, just without events.
|
|
132
|
+
if (!resolvedRunId)
|
|
133
|
+
return { run: latest, events: [] };
|
|
134
|
+
}
|
|
135
|
+
const detail = await skillRunsRequest(`${ingestUrl}/v1/skills/runs/${encodeURIComponent(resolvedRunId)}`);
|
|
136
|
+
if (detail.status === 404) {
|
|
137
|
+
if (runId) {
|
|
138
|
+
throw new CliInputError(`No skill generation run found with id '${runId}'`);
|
|
139
|
+
}
|
|
140
|
+
return { run: null, events: [] };
|
|
141
|
+
}
|
|
142
|
+
const detailBody = detail.body;
|
|
143
|
+
return {
|
|
144
|
+
run: detailBody.run ?? null,
|
|
145
|
+
events: Array.isArray(detailBody.events) ? detailBody.events : [],
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
const STATUS_HUMAN_EVENT_LIMIT = 10;
|
|
149
|
+
/** Human rendering for `moda skills status` (key: value lines, gen-style). */
|
|
150
|
+
function formatRunStatusHuman(run, events) {
|
|
151
|
+
const counters = run.counters ?? {};
|
|
152
|
+
const lines = [
|
|
153
|
+
run.generation_run_id && `run: ${run.generation_run_id}`,
|
|
154
|
+
`status: ${run.status || 'unknown'}`,
|
|
155
|
+
run.current_stage && `stage: ${run.current_stage}`,
|
|
156
|
+
run.outcome && `outcome: ${run.outcome}`,
|
|
157
|
+
`processed: ${Number(counters.sessions_processed ?? 0)}`,
|
|
158
|
+
`failed: ${Number(counters.sessions_failed ?? 0)}`,
|
|
159
|
+
Number(counters.distilled_skills_generated ?? 0) > 0
|
|
160
|
+
&& `distilled: ${Number(counters.distilled_skills_generated ?? 0)}`,
|
|
161
|
+
Number(counters.skills_indexed ?? 0) > 0
|
|
162
|
+
&& `indexed: ${Number(counters.skills_indexed ?? 0)}`,
|
|
163
|
+
(run.generated_skill_ids?.length ?? 0) > 0
|
|
164
|
+
&& `skills: ${(run.generated_skill_ids ?? []).join(', ')}`,
|
|
165
|
+
run.error && `error: ${run.error}`,
|
|
166
|
+
run.started_at && `started: ${run.started_at}`,
|
|
167
|
+
run.updated_at && `updated: ${run.updated_at}`,
|
|
168
|
+
].filter(Boolean);
|
|
169
|
+
if (events.length > 0) {
|
|
170
|
+
lines.push('recent events (newest first):');
|
|
171
|
+
for (const event of events.slice(0, STATUS_HUMAN_EVENT_LIMIT)) {
|
|
172
|
+
lines.push(` ${[event.created_at, event.level, event.stage, event.message]
|
|
173
|
+
.filter(Boolean)
|
|
174
|
+
.join(' ')}`);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return lines.join('\n');
|
|
178
|
+
}
|
|
77
179
|
/**
|
|
78
180
|
* Write `content` to `filePath`, creating parent dirs. Returns whether the
|
|
79
181
|
* file was created, updated (content differed), or left unchanged.
|
|
@@ -463,6 +565,8 @@ export async function runSkillsCommand(input) {
|
|
|
463
565
|
'accepted: true',
|
|
464
566
|
result.generation_run_id && `generation_run_id: ${result.generation_run_id}`,
|
|
465
567
|
result.message && `message: ${result.message}`,
|
|
568
|
+
result.generation_run_id
|
|
569
|
+
&& `follow progress: moda skills status ${result.generation_run_id}`,
|
|
466
570
|
].filter(Boolean).join('\n'));
|
|
467
571
|
}
|
|
468
572
|
else {
|
|
@@ -473,10 +577,26 @@ export async function runSkillsCommand(input) {
|
|
|
473
577
|
`pending_found: ${result.pending_sessions_found ?? 0}`,
|
|
474
578
|
`distilled: ${Number(result.cluster_result?.distilled_skills_generated ?? 0)}`,
|
|
475
579
|
`indexed: ${Number(result.cluster_result?.skills_indexed ?? 0)}`,
|
|
476
|
-
|
|
580
|
+
result.distillation_skipped
|
|
581
|
+
&& `distillation skipped: ${result.distillation_skip_reason || 'unknown'}`,
|
|
582
|
+
].filter(Boolean).join('\n'));
|
|
477
583
|
}
|
|
478
584
|
return Number(result.sessions_failed ?? 0) > 0 ? 1 : 0;
|
|
479
585
|
}
|
|
586
|
+
if (sub === 'status') {
|
|
587
|
+
const runId = positionals[1] || flags['run-id'] || undefined;
|
|
588
|
+
const { run, events } = await fetchSkillRunStatus(runId);
|
|
589
|
+
if (context.outputMode !== 'human') {
|
|
590
|
+
context.output.writeData({ run, events }, { status: run?.status === 'failed' ? 'error' : 'success' });
|
|
591
|
+
}
|
|
592
|
+
else if (!run) {
|
|
593
|
+
console.log('No skill generation runs found yet. Start one with `moda skills gen`.');
|
|
594
|
+
}
|
|
595
|
+
else {
|
|
596
|
+
console.log(formatRunStatusHuman(run, events));
|
|
597
|
+
}
|
|
598
|
+
return run?.status === 'failed' ? 1 : 0;
|
|
599
|
+
}
|
|
480
600
|
if (sub === 'proposals' && positionals[1] === 'list') {
|
|
481
601
|
const proposals = await fetchSkillProposals(flags, { profile: context.profile });
|
|
482
602
|
if (flags.json === 'true') {
|
|
@@ -533,7 +653,7 @@ export async function runSkillsCommand(input) {
|
|
|
533
653
|
return 0;
|
|
534
654
|
}
|
|
535
655
|
if (sub !== 'pull') {
|
|
536
|
-
throw new CliInputError(sub ? `Unknown skills subcommand '${sub}'` : 'A skills subcommand is required', 'Usage: moda skills gen [--source=all|sdk|coding-agent] [--max-sessions=N] [--start-at=ISO] [--end-at=ISO] [--replay] [--replay-set-id=ID] [--reprocess-segments] [--force-reprocess] [--wait] | moda skills proposals list [--status=ready_for_pr] | moda skills proposal apply <proposal-id> | moda skills pull [--status=approved|proposed|all] | moda skills sync [--dry-run]');
|
|
656
|
+
throw new CliInputError(sub ? `Unknown skills subcommand '${sub}'` : 'A skills subcommand is required', 'Usage: moda skills gen [--source=all|sdk|coding-agent] [--max-sessions=N] [--start-at=ISO] [--end-at=ISO] [--replay] [--replay-set-id=ID] [--reprocess-segments] [--force-reprocess] [--wait] | moda skills status [run-id] | moda skills proposals list [--status=ready_for_pr] | moda skills proposal apply <proposal-id> | moda skills pull [--status=approved|proposed|all] | moda skills sync [--dry-run]');
|
|
537
657
|
}
|
|
538
658
|
const status = (flags.status ?? 'approved').toLowerCase();
|
|
539
659
|
if (!VALID_STATUSES.has(status)) {
|
|
@@ -580,7 +700,7 @@ async function generateSkills(flags) {
|
|
|
580
700
|
replay_set_id: flags['replay-set-id'] || flags['eval-set-id'] || undefined,
|
|
581
701
|
run_clustering: flags['no-clustering'] !== 'true',
|
|
582
702
|
source: normalizeSkillSource(flags.source),
|
|
583
|
-
clustering_timeout_seconds: parsePositiveInt(flags['clustering-timeout-seconds'],
|
|
703
|
+
clustering_timeout_seconds: parsePositiveInt(flags['clustering-timeout-seconds'], 7200),
|
|
584
704
|
clustering_poll_seconds: parsePositiveInt(flags['clustering-poll-seconds'], 15),
|
|
585
705
|
max_cluster_pct: parseOptionalFloat(flags['max-cluster-pct']),
|
|
586
706
|
reprocess_segments: flags['reprocess-segments'] === 'true' || forceReprocess,
|
package/package.json
CHANGED