@moda-ai/cli 1.13.5 → 1.13.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +5 -2
- package/dist/harness-artifact-bodies.d.ts +21 -0
- package/dist/harness-artifact-bodies.js +201 -0
- package/dist/harness-remote.js +14 -2
- package/dist/harness-report.d.ts +7 -0
- package/dist/harness-report.js +50 -1
- package/dist/harness-sync-limits.d.ts +8 -0
- package/dist/harness-sync-limits.js +21 -0
- package/dist/harness.d.ts +6 -0
- package/dist/harness.js +12 -0
- package/dist/prompts-ab.d.ts +18 -0
- package/dist/prompts-ab.js +243 -0
- package/dist/prompts.d.ts +35 -0
- package/dist/prompts.js +14 -2
- package/dist/skills.js +123 -3
- package/package.json +1 -1
- package/skills/integration/index.json +2 -2
package/dist/cli.js
CHANGED
|
@@ -181,9 +181,11 @@ Commands:
|
|
|
181
181
|
prompts sync Upload changed prompt versions
|
|
182
182
|
prompts sync --watch Upload again when prompt files change
|
|
183
183
|
prompts promote <key> Move a dev/staging/prod label
|
|
184
|
+
prompts ab Run a judged prompt A/B replay experiment
|
|
184
185
|
|
|
185
186
|
Skills:
|
|
186
187
|
skills gen Kick off tenant-wide skill generation
|
|
188
|
+
skills status [run-id] Show a generation run's status, outcome, and recent events (default: latest run)
|
|
187
189
|
skills pull Pull tenant-generated skills into .claude/skills (and .cursor/rules)
|
|
188
190
|
skills pull --status=proposed Include not-yet-promoted skills (default: approved)
|
|
189
191
|
skills sync Push local .claude/skills/**/SKILL.md up to Moda
|
|
@@ -309,6 +311,7 @@ Examples:
|
|
|
309
311
|
moda prompts status
|
|
310
312
|
moda prompts sync
|
|
311
313
|
moda prompts promote support.triage --label=prod --version=pver_abc123
|
|
314
|
+
moda prompts ab --baseline=prompts/agent.prompt.md --candidate=prompts/agent-v2.prompt.md --conversations=conv_1,conv_2
|
|
312
315
|
moda skills pull`);
|
|
313
316
|
}
|
|
314
317
|
// --- Command Handlers ---
|
|
@@ -1056,7 +1059,7 @@ export const commandRegistry = createCommandRegistry([
|
|
|
1056
1059
|
legacyCommand({
|
|
1057
1060
|
name: 'prompts',
|
|
1058
1061
|
description: 'Manage prompt versions and labels',
|
|
1059
|
-
examples: ['moda prompts status', 'moda prompts sync'],
|
|
1062
|
+
examples: ['moda prompts status', 'moda prompts sync', 'moda prompts ab --baseline=old.md --candidate=new.md'],
|
|
1060
1063
|
auth: 'api-key',
|
|
1061
1064
|
outputModes: ['human', 'json', 'agent'],
|
|
1062
1065
|
mutability: 'write',
|
|
@@ -1121,7 +1124,7 @@ export const commandRegistry = createCommandRegistry([
|
|
|
1121
1124
|
{
|
|
1122
1125
|
name: 'skills',
|
|
1123
1126
|
description: 'Generate, pull, and sync Moda skills',
|
|
1124
|
-
examples: ['moda skills gen', 'moda skills pull', 'moda skills sync'],
|
|
1127
|
+
examples: ['moda skills gen', 'moda skills status', 'moda skills pull', 'moda skills sync'],
|
|
1125
1128
|
auth: 'none',
|
|
1126
1129
|
outputModes: ['human', 'json', 'agent'],
|
|
1127
1130
|
mutability: 'write',
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { HarnessArtifactType, HarnessEvidence, HarnessGraph } from './harness.js';
|
|
2
|
+
export interface ArtifactBodyAnnotation {
|
|
3
|
+
values: Record<string, string[]>;
|
|
4
|
+
evidence: HarnessEvidence;
|
|
5
|
+
sourceAbsPath: string;
|
|
6
|
+
rootDir: string;
|
|
7
|
+
}
|
|
8
|
+
export interface ArtifactBodyFileItem {
|
|
9
|
+
path: string;
|
|
10
|
+
rootDir: string;
|
|
11
|
+
text?: string;
|
|
12
|
+
}
|
|
13
|
+
export declare function extractToolBodyFromText(text: string, toolName: string): string | undefined;
|
|
14
|
+
export declare function readPromptOrToolBody(type: HarnessArtifactType, sourcePath: string, annotation: ArtifactBodyAnnotation): string | undefined;
|
|
15
|
+
export declare function extractToolBody(file: ArtifactBodyFileItem, tool: {
|
|
16
|
+
name: string;
|
|
17
|
+
reason: string;
|
|
18
|
+
}): string | undefined;
|
|
19
|
+
export declare function hydratePromptAndToolBodies(graph: HarnessGraph, rootDir: string): void;
|
|
20
|
+
export declare function enrichScanCaptureMetadata(graph: HarnessGraph): void;
|
|
21
|
+
//# sourceMappingURL=harness-artifact-bodies.d.ts.map
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
import { existsSync, lstatSync, readFileSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { basename, dirname, isAbsolute, relative, resolve } from 'node:path';
|
|
3
|
+
import { artifactHasBody, firstEvidencePath, requiresArtifactBody, } from './harness-report.js';
|
|
4
|
+
const MAX_SCANNED_FILE_BYTES = 512 * 1024;
|
|
5
|
+
const SNAPSHOT_BUDGET_SKIP_REASONS = new Set([
|
|
6
|
+
'file too large for harness scan',
|
|
7
|
+
'scan text budget exceeded',
|
|
8
|
+
]);
|
|
9
|
+
function first(values) {
|
|
10
|
+
return values?.[0];
|
|
11
|
+
}
|
|
12
|
+
function normalizePath(path) {
|
|
13
|
+
return path.replace(/\\/g, '/');
|
|
14
|
+
}
|
|
15
|
+
function isTextFile(path) {
|
|
16
|
+
return /\.(ts|tsx|js|jsx|mjs|cjs|py|md|json|ya?ml|toml|txt|sh|rb|go|rs|swift|kt|java|cs|php|sql|graphql|gql|proto|css|scss|less|html|vue|svelte)$/i.test(path);
|
|
17
|
+
}
|
|
18
|
+
function isSecretishPath(path) {
|
|
19
|
+
const normalized = normalizePath(path).toLowerCase();
|
|
20
|
+
return normalized.includes('.env') ||
|
|
21
|
+
normalized.endsWith('.pem') ||
|
|
22
|
+
normalized.endsWith('.key') ||
|
|
23
|
+
normalized.includes('/secrets/') ||
|
|
24
|
+
normalized.includes('/credentials/');
|
|
25
|
+
}
|
|
26
|
+
function isOutsideRoot(rel) {
|
|
27
|
+
return rel === '' || rel === '..' || rel.startsWith('../') || isAbsolute(rel);
|
|
28
|
+
}
|
|
29
|
+
function readBoundedRepoText(absPath, rootDir) {
|
|
30
|
+
if (!existsSync(absPath))
|
|
31
|
+
return undefined;
|
|
32
|
+
const rel = normalizePath(relative(rootDir, absPath));
|
|
33
|
+
// Containment: never read files that resolve outside the scan root. Guards against
|
|
34
|
+
// `..` traversal or absolute paths (e.g. via a crafted `path=` annotation / sourcePath /
|
|
35
|
+
// evidence path) being hydrated into synced artifact bodies.
|
|
36
|
+
if (isOutsideRoot(rel))
|
|
37
|
+
return undefined;
|
|
38
|
+
if (isSecretishPath(rel))
|
|
39
|
+
return undefined;
|
|
40
|
+
let stat;
|
|
41
|
+
try {
|
|
42
|
+
stat = lstatSync(absPath);
|
|
43
|
+
}
|
|
44
|
+
catch {
|
|
45
|
+
return undefined;
|
|
46
|
+
}
|
|
47
|
+
if (!stat.isFile() || stat.isSymbolicLink() || stat.size > MAX_SCANNED_FILE_BYTES) {
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
if (!isTextFile(rel))
|
|
51
|
+
return undefined;
|
|
52
|
+
// Realpath containment: the lexical `rel` check is purely textual and `lstatSync` only
|
|
53
|
+
// inspects the final component, so an intermediate directory symlink (e.g.
|
|
54
|
+
// `tools/outside -> /tmp/public`) could still let `readFileSync` follow the link out of
|
|
55
|
+
// the repo. Canonicalize both sides and re-check before reading.
|
|
56
|
+
try {
|
|
57
|
+
if (isOutsideRoot(normalizePath(relative(realpathSync(rootDir), realpathSync(absPath))))) {
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
catch {
|
|
62
|
+
return undefined;
|
|
63
|
+
}
|
|
64
|
+
try {
|
|
65
|
+
return readFileSync(absPath, 'utf8');
|
|
66
|
+
}
|
|
67
|
+
catch {
|
|
68
|
+
return undefined;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
function readRepoRelativeText(rootDir, relPath) {
|
|
72
|
+
return readBoundedRepoText(resolve(rootDir, relPath), rootDir);
|
|
73
|
+
}
|
|
74
|
+
function isDedicatedToolFilePath(path) {
|
|
75
|
+
return /(^|\/)tools?\//i.test(path) && /\.(ts|tsx|js|mjs|py)$/i.test(path);
|
|
76
|
+
}
|
|
77
|
+
function firstEvidenceLineNumber(evidence) {
|
|
78
|
+
if (!Array.isArray(evidence))
|
|
79
|
+
return undefined;
|
|
80
|
+
for (const entry of evidence) {
|
|
81
|
+
if (typeof entry.line === 'number' && Number.isFinite(entry.line) && entry.line > 0) {
|
|
82
|
+
return Math.floor(entry.line);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
function extractSnippetAroundLine(text, line, radius) {
|
|
88
|
+
const lines = text.split(/\r?\n/);
|
|
89
|
+
const start = Math.max(0, line - 1 - radius);
|
|
90
|
+
// `line` is 1-indexed and slice's end is exclusive, so include through
|
|
91
|
+
// (line-1)+radius: end = line + radius. Using line-1+radius dropped the tail line.
|
|
92
|
+
const end = Math.min(lines.length, line + radius);
|
|
93
|
+
return lines.slice(start, end).join('\n');
|
|
94
|
+
}
|
|
95
|
+
export function extractToolBodyFromText(text, toolName) {
|
|
96
|
+
const lines = text.split(/\r?\n/);
|
|
97
|
+
const normalizedName = toolName.replace(/\s+/g, '').toLowerCase();
|
|
98
|
+
for (let index = 0; index < lines.length; index += 1) {
|
|
99
|
+
const line = lines[index];
|
|
100
|
+
const compact = line.replace(/\s+/g, '').toLowerCase();
|
|
101
|
+
const mentionsTool = compact.includes(normalizedName) || compact.includes(`def${normalizedName}`);
|
|
102
|
+
if (!mentionsTool)
|
|
103
|
+
continue;
|
|
104
|
+
if (line.includes('function_tool') ||
|
|
105
|
+
line.includes('tool(') ||
|
|
106
|
+
line.includes('createTool') ||
|
|
107
|
+
line.includes('functionTool') ||
|
|
108
|
+
line.includes('dynamicTool') ||
|
|
109
|
+
line.includes('new Tool') ||
|
|
110
|
+
line.includes('def ')) {
|
|
111
|
+
return extractSnippetAroundLine(text, index + 1, 30);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return undefined;
|
|
115
|
+
}
|
|
116
|
+
function readAnnotatedSourceText(sourcePath, annotation) {
|
|
117
|
+
const explicitSourcePath = normalizePath(sourcePath) !== normalizePath(annotation.evidence.path);
|
|
118
|
+
const candidates = explicitSourcePath ? [
|
|
119
|
+
resolve(annotation.rootDir, sourcePath),
|
|
120
|
+
resolve(dirname(annotation.sourceAbsPath), sourcePath),
|
|
121
|
+
] : [
|
|
122
|
+
resolve(annotation.rootDir, sourcePath),
|
|
123
|
+
resolve(dirname(annotation.sourceAbsPath), sourcePath),
|
|
124
|
+
annotation.sourceAbsPath,
|
|
125
|
+
];
|
|
126
|
+
const seen = new Set();
|
|
127
|
+
for (const candidate of candidates) {
|
|
128
|
+
if (seen.has(candidate))
|
|
129
|
+
continue;
|
|
130
|
+
seen.add(candidate);
|
|
131
|
+
const text = readBoundedRepoText(candidate, annotation.rootDir);
|
|
132
|
+
if (text !== undefined)
|
|
133
|
+
return text;
|
|
134
|
+
}
|
|
135
|
+
return undefined;
|
|
136
|
+
}
|
|
137
|
+
export function readPromptOrToolBody(type, sourcePath, annotation) {
|
|
138
|
+
if (!requiresArtifactBody(type))
|
|
139
|
+
return undefined;
|
|
140
|
+
const text = readAnnotatedSourceText(sourcePath, annotation);
|
|
141
|
+
if (text === undefined)
|
|
142
|
+
return undefined;
|
|
143
|
+
if (type === 'prompt' || isDedicatedToolFilePath(sourcePath))
|
|
144
|
+
return text;
|
|
145
|
+
return extractToolBodyFromText(text, first(annotation.values.name) ?? basename(sourcePath)) ?? text;
|
|
146
|
+
}
|
|
147
|
+
export function extractToolBody(file, tool) {
|
|
148
|
+
const text = file.text ?? readRepoRelativeText(file.rootDir, file.path);
|
|
149
|
+
if (!text)
|
|
150
|
+
return undefined;
|
|
151
|
+
if (isDedicatedToolFilePath(file.path))
|
|
152
|
+
return text;
|
|
153
|
+
return extractToolBodyFromText(text, tool.name);
|
|
154
|
+
}
|
|
155
|
+
function hydrateArtifactBody(artifact, rootDir) {
|
|
156
|
+
if (artifact.sourcePath) {
|
|
157
|
+
const wholeFile = readRepoRelativeText(rootDir, artifact.sourcePath);
|
|
158
|
+
if (wholeFile !== undefined) {
|
|
159
|
+
if (artifact.type === 'prompt' || isDedicatedToolFilePath(artifact.sourcePath)) {
|
|
160
|
+
return wholeFile;
|
|
161
|
+
}
|
|
162
|
+
if (artifact.type === 'tool') {
|
|
163
|
+
return extractToolBodyFromText(wholeFile, artifact.name) ?? wholeFile;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
const evidenceLine = firstEvidenceLineNumber(artifact.evidence);
|
|
168
|
+
const evidencePath = firstEvidencePath(artifact);
|
|
169
|
+
if (evidencePath && evidenceLine !== undefined) {
|
|
170
|
+
const text = readRepoRelativeText(rootDir, evidencePath);
|
|
171
|
+
if (text !== undefined) {
|
|
172
|
+
return extractSnippetAroundLine(text, evidenceLine, artifact.type === 'tool' ? 30 : 80);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return undefined;
|
|
176
|
+
}
|
|
177
|
+
export function hydratePromptAndToolBodies(graph, rootDir) {
|
|
178
|
+
for (const artifact of graph.artifacts) {
|
|
179
|
+
if (!requiresArtifactBody(artifact.type) || artifactHasBody(artifact)) {
|
|
180
|
+
continue;
|
|
181
|
+
}
|
|
182
|
+
const hydrated = hydrateArtifactBody(artifact, rootDir);
|
|
183
|
+
if (hydrated)
|
|
184
|
+
artifact.body = hydrated;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
function countIncompletePromptToolBodies(graph) {
|
|
188
|
+
return graph.artifacts.filter((artifact) => requiresArtifactBody(artifact.type) &&
|
|
189
|
+
!artifactHasBody(artifact)).length;
|
|
190
|
+
}
|
|
191
|
+
export function enrichScanCaptureMetadata(graph) {
|
|
192
|
+
const budgetSkips = graph.scan.skipped
|
|
193
|
+
.map((skip) => skip.reason)
|
|
194
|
+
.filter((reason) => SNAPSHOT_BUDGET_SKIP_REASONS.has(reason));
|
|
195
|
+
const truncationReasons = [...new Set(budgetSkips)];
|
|
196
|
+
graph.scan.truncated = truncationReasons.length > 0;
|
|
197
|
+
graph.scan.truncationReasons = truncationReasons.length > 0 ? truncationReasons : undefined;
|
|
198
|
+
const incomplete = countIncompletePromptToolBodies(graph);
|
|
199
|
+
graph.scan.incompleteArtifactCount = incomplete > 0 ? incomplete : undefined;
|
|
200
|
+
}
|
|
201
|
+
//# sourceMappingURL=harness-artifact-bodies.js.map
|
package/dist/harness-remote.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { resolveApiKey, resolveIngestUrl } from './config.js';
|
|
11
11
|
import { CliInputError } from './errors.js';
|
|
12
|
-
import { normalizeHarnessReport, validateHarnessReport, writeHarnessReportArtifacts, } from './harness-report.js';
|
|
12
|
+
import { normalizeHarnessReport, stampRemoteSnapshotTruncation, validateHarnessReport, writeHarnessReportArtifacts, } from './harness-report.js';
|
|
13
13
|
import { buildSourceSnapshot, emitWrittenArtifacts, numericEnv, numericEnvAllowZero, repairExternalAnalystReport, sleep, summarizeHarness, summarizeInvalidAnalystReport, truncateText, writeHarnessAnalyzeResult, writeHarnessArtifacts, writeHumanProgress, } from './harness.js';
|
|
14
14
|
const DEFAULT_REMOTE_ANALYZE_TIMEOUT_MS = 900_000;
|
|
15
15
|
const DEFAULT_REMOTE_POLL_INITIAL_MS = 3_000;
|
|
@@ -75,6 +75,15 @@ export async function runRemoteHarnessAnalyst(options) {
|
|
|
75
75
|
elapsed_ms: Date.now() - context.startedAt,
|
|
76
76
|
});
|
|
77
77
|
writeHumanProgress(context, `Moda Analyst (remote): uploading ${snapshot.fileCount} file(s) for server-side analysis...`);
|
|
78
|
+
if (snapshot.truncated) {
|
|
79
|
+
context.output.writeEvent({
|
|
80
|
+
event: 'warning',
|
|
81
|
+
phase: 'remote_upload',
|
|
82
|
+
message: 'Source snapshot is truncated; server-side analysis may be incomplete.',
|
|
83
|
+
elapsed_ms: Date.now() - context.startedAt,
|
|
84
|
+
});
|
|
85
|
+
writeHumanProgress(context, 'Warning: source snapshot is truncated — analysis may be incomplete.');
|
|
86
|
+
}
|
|
78
87
|
const accepted = await postRemoteAnalyze(ingestUrl, apiKey, {
|
|
79
88
|
snapshot,
|
|
80
89
|
options: buildRemoteAnalyzeOptions(context.flags),
|
|
@@ -98,7 +107,10 @@ export async function runRemoteHarnessAnalyst(options) {
|
|
|
98
107
|
'Increase MODA_HARNESS_REMOTE_TIMEOUT_MS if large repos need longer.',
|
|
99
108
|
].join('\n'));
|
|
100
109
|
}
|
|
101
|
-
|
|
110
|
+
let report = repairExternalAnalystReport(normalizeHarnessReport(final.report), rootDir);
|
|
111
|
+
if (snapshot.truncated) {
|
|
112
|
+
report = stampRemoteSnapshotTruncation(report);
|
|
113
|
+
}
|
|
102
114
|
const validation = validateHarnessReport(report, rootDir);
|
|
103
115
|
context.output.writeEvent({
|
|
104
116
|
event: 'progress',
|
package/dist/harness-report.d.ts
CHANGED
|
@@ -123,6 +123,13 @@ export declare function loadHarnessReportForSync(rootDir: string, reportPath: st
|
|
|
123
123
|
};
|
|
124
124
|
export declare function readHarnessReport(reportPath: string): HarnessReport;
|
|
125
125
|
export declare function normalizeHarnessReport(value: unknown): HarnessReport;
|
|
126
|
+
/** prompt/tool artifacts must carry full usable content for downstream optimization. */
|
|
127
|
+
export declare function requiresArtifactBody(type: unknown): boolean;
|
|
128
|
+
export declare function artifactHasBody(item: Record<string, unknown>): boolean;
|
|
129
|
+
/** Canonical evidence path for graph items: prefer explicit sourcePath, then evidence. */
|
|
130
|
+
export declare function firstEvidencePath(item: Record<string, unknown>): string | undefined;
|
|
131
|
+
/** Stamp remote snapshot truncation onto a validated report graph scan metadata. */
|
|
132
|
+
export declare function stampRemoteSnapshotTruncation(report: HarnessReport): HarnessReport;
|
|
126
133
|
export declare function resolveHarnessReportPath(rootDir: string, value?: string): string;
|
|
127
134
|
export declare function writeHarnessReportArtifacts(options: {
|
|
128
135
|
rootDir: string;
|
package/dist/harness-report.js
CHANGED
|
@@ -320,6 +320,7 @@ export function validateHarnessReport(report, rootDir) {
|
|
|
320
320
|
check(`graph_${field}_array`, Array.isArray(report.graph[field]), `Graph field ${field} must be an array.`);
|
|
321
321
|
}
|
|
322
322
|
validateGraphItemShapes(report.graph, errors);
|
|
323
|
+
validateCaptureQuality(report.graph, warn);
|
|
323
324
|
}
|
|
324
325
|
if (report?.summary && typeof report.summary === 'object') {
|
|
325
326
|
for (const field of HARNESS_REPORT_SUMMARY_FIELDS) {
|
|
@@ -811,8 +812,18 @@ function validateGraphCollection(graph, errors, field, shape) {
|
|
|
811
812
|
if (field === 'artifacts' && requiresArtifactEvidenceLine(item.type) && !artifactHasRetrievableLocation(item)) {
|
|
812
813
|
errors.push(`Graph ${label} is a ${String(item.type)} artifact and must include a numeric evidence line (or a dedicated whole-file source) so its exact text can be retrieved.`);
|
|
813
814
|
}
|
|
815
|
+
if (field === 'artifacts' && requiresArtifactBody(item.type) && !artifactHasBody(item)) {
|
|
816
|
+
errors.push(`Graph ${label} is a ${String(item.type)} artifact and must include a non-empty string body with its usable content.`);
|
|
817
|
+
}
|
|
814
818
|
}
|
|
815
819
|
}
|
|
820
|
+
/** prompt/tool artifacts must carry full usable content for downstream optimization. */
|
|
821
|
+
export function requiresArtifactBody(type) {
|
|
822
|
+
return type === 'prompt' || type === 'tool';
|
|
823
|
+
}
|
|
824
|
+
export function artifactHasBody(item) {
|
|
825
|
+
return typeof item.body === 'string' && item.body.trim().length > 0;
|
|
826
|
+
}
|
|
816
827
|
/** prompt/tool artifacts must pin an exact source location for downstream retrieval. */
|
|
817
828
|
function requiresArtifactEvidenceLine(type) {
|
|
818
829
|
return type === 'prompt' || type === 'tool';
|
|
@@ -828,6 +839,20 @@ function artifactHasRetrievableLocation(item) {
|
|
|
828
839
|
return true;
|
|
829
840
|
return isWholeFileArtifact(item);
|
|
830
841
|
}
|
|
842
|
+
function validateCaptureQuality(graph, warn) {
|
|
843
|
+
const scan = graph.scan;
|
|
844
|
+
if (!isRecord(scan))
|
|
845
|
+
return;
|
|
846
|
+
if (scan.truncated === true) {
|
|
847
|
+
const reasons = Array.isArray(scan.truncationReasons) && scan.truncationReasons.length > 0
|
|
848
|
+
? scan.truncationReasons.map((reason) => String(reason)).join(', ')
|
|
849
|
+
: 'scan budget or upload cap reached';
|
|
850
|
+
warn('scan_truncated', `Harness scan is partial (truncated): ${reasons}. Re-run with --path=<agent-dir> or review skipped files in the report.`);
|
|
851
|
+
}
|
|
852
|
+
if (typeof scan.incompleteArtifactCount === 'number' && scan.incompleteArtifactCount > 0) {
|
|
853
|
+
warn('artifacts_missing_body', `${scan.incompleteArtifactCount} prompt/tool artifact(s) are still missing captured body content after hydration.`);
|
|
854
|
+
}
|
|
855
|
+
}
|
|
831
856
|
function isWholeFileArtifact(item) {
|
|
832
857
|
if (typeof item.sourcePath !== 'string' || item.sourcePath.length === 0)
|
|
833
858
|
return false;
|
|
@@ -884,7 +909,10 @@ function normalizeGraphItemFieldAliases(collection, item) {
|
|
|
884
909
|
}
|
|
885
910
|
}
|
|
886
911
|
}
|
|
887
|
-
|
|
912
|
+
/** Canonical evidence path for graph items: prefer explicit sourcePath, then evidence. */
|
|
913
|
+
export function firstEvidencePath(item) {
|
|
914
|
+
if (typeof item.sourcePath === 'string' && item.sourcePath.length > 0)
|
|
915
|
+
return item.sourcePath;
|
|
888
916
|
if (!Array.isArray(item.evidence))
|
|
889
917
|
return undefined;
|
|
890
918
|
for (const evidence of item.evidence) {
|
|
@@ -894,6 +922,27 @@ function firstEvidencePath(item) {
|
|
|
894
922
|
}
|
|
895
923
|
return undefined;
|
|
896
924
|
}
|
|
925
|
+
/** Stamp remote snapshot truncation onto a validated report graph scan metadata. */
|
|
926
|
+
export function stampRemoteSnapshotTruncation(report) {
|
|
927
|
+
const scanRecord = isRecord(report.graph.scan) ? report.graph.scan : {};
|
|
928
|
+
const truncationReasons = Array.isArray(scanRecord.truncationReasons)
|
|
929
|
+
? scanRecord.truncationReasons.map(String)
|
|
930
|
+
: [];
|
|
931
|
+
return {
|
|
932
|
+
...report,
|
|
933
|
+
graph: {
|
|
934
|
+
...report.graph,
|
|
935
|
+
scan: {
|
|
936
|
+
...report.graph.scan,
|
|
937
|
+
truncated: true,
|
|
938
|
+
truncationReasons: [
|
|
939
|
+
...truncationReasons,
|
|
940
|
+
'remote source snapshot truncated before upload',
|
|
941
|
+
],
|
|
942
|
+
},
|
|
943
|
+
},
|
|
944
|
+
};
|
|
945
|
+
}
|
|
897
946
|
function requiresGraphItemConfidence(collection) {
|
|
898
947
|
return collection !== 'relationships' && collection !== 'unknowns';
|
|
899
948
|
}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Maximum serialized harness graph size accepted by the backend sync endpoint.
|
|
3
|
+
* Must match `MAX_HARNESS_SYNC_GRAPH_BYTES` in
|
|
4
|
+
* `apps/moda-backend/src/harnesses/harness-sync-limits.ts`.
|
|
5
|
+
*/
|
|
6
|
+
export declare const MAX_HARNESS_SYNC_GRAPH_BYTES = 2000000;
|
|
7
|
+
export declare function assertHarnessGraphSyncSize(graph: unknown): void;
|
|
8
|
+
//# sourceMappingURL=harness-sync-limits.d.ts.map
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { CliInputError } from './errors.js';
|
|
2
|
+
/**
|
|
3
|
+
* Maximum serialized harness graph size accepted by the backend sync endpoint.
|
|
4
|
+
* Must match `MAX_HARNESS_SYNC_GRAPH_BYTES` in
|
|
5
|
+
* `apps/moda-backend/src/harnesses/harness-sync-limits.ts`.
|
|
6
|
+
*/
|
|
7
|
+
export const MAX_HARNESS_SYNC_GRAPH_BYTES = 2_000_000;
|
|
8
|
+
export function assertHarnessGraphSyncSize(graph) {
|
|
9
|
+
if (!graph || typeof graph !== 'object' || Array.isArray(graph)) {
|
|
10
|
+
throw new CliInputError('Harness graph must be an object before sync.');
|
|
11
|
+
}
|
|
12
|
+
const graphBytes = Buffer.byteLength(JSON.stringify(graph), 'utf8');
|
|
13
|
+
if (graphBytes > MAX_HARNESS_SYNC_GRAPH_BYTES) {
|
|
14
|
+
throw new CliInputError(`Harness graph is too large to sync (${graphBytes.toLocaleString()} bytes; limit ${MAX_HARNESS_SYNC_GRAPH_BYTES.toLocaleString()}).`, [
|
|
15
|
+
'Run `moda harness analyze --path=<agent-dir>` to narrow the harness root.',
|
|
16
|
+
'Reduce captured prompt/tool bodies if the graph grew too large.',
|
|
17
|
+
'Split multiple agent roots into separate harness syncs per tenant.',
|
|
18
|
+
].join('\n'));
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=harness-sync-limits.js.map
|
package/dist/harness.d.ts
CHANGED
|
@@ -32,6 +32,12 @@ export interface HarnessScanMetadata {
|
|
|
32
32
|
path: string;
|
|
33
33
|
reason: string;
|
|
34
34
|
}>;
|
|
35
|
+
/** True when the scan dropped files due to size/budget caps or snapshot limits. */
|
|
36
|
+
truncated?: boolean;
|
|
37
|
+
/** Human-readable reasons the scan is partial (budget skips, upload cap, etc.). */
|
|
38
|
+
truncationReasons?: string[];
|
|
39
|
+
/** Prompt/tool artifacts still missing a captured body after hydration. */
|
|
40
|
+
incompleteArtifactCount?: number;
|
|
35
41
|
}
|
|
36
42
|
export interface HarnessScanReadPlan {
|
|
37
43
|
scope: 'broad' | 'narrow';
|
package/dist/harness.js
CHANGED
|
@@ -10,7 +10,9 @@ import { resolveApiKey, resolveTenantId } from './config.js';
|
|
|
10
10
|
import { CliInputError, CliInputRequiredError } from './errors.js';
|
|
11
11
|
import { isRemoteHarnessAnalyze, runRemoteHarnessAnalyst } from './harness-remote.js';
|
|
12
12
|
import { getActiveProfile } from './profiles.js';
|
|
13
|
+
import { enrichScanCaptureMetadata, extractToolBody, hydratePromptAndToolBodies, readPromptOrToolBody, } from './harness-artifact-bodies.js';
|
|
13
14
|
import { approveHarnessReport, createHarnessReport, graphForReport, hashHarnessReport, hashStableJson as hashHarnessReportStableJson, HARNESS_REPORT_PATH, loadHarnessReportForSync, normalizeHarnessReport, readHarnessReport, resolveHarnessReportPath, validateHarnessReport, writeHarnessReportArtifacts, } from './harness-report.js';
|
|
15
|
+
import { assertHarnessGraphSyncSize } from './harness-sync-limits.js';
|
|
14
16
|
import { renderHarnessAnalystChoice, renderHarnessAnalystLaunch, renderHarnessAnalystPreflight, renderHarnessAnalyzeHumanResult, renderHarnessProgress, } from './tui.js';
|
|
15
17
|
const MODA_DIR = '.moda';
|
|
16
18
|
const HARNESS_PATH = join(MODA_DIR, 'harness.json');
|
|
@@ -1607,6 +1609,8 @@ export function scanHarness(options = {}) {
|
|
|
1607
1609
|
applyHeuristics(graph, files);
|
|
1608
1610
|
linkFrameworksAndProviders(graph);
|
|
1609
1611
|
finalizeGraph(graph);
|
|
1612
|
+
hydratePromptAndToolBodies(graph, rootDir);
|
|
1613
|
+
enrichScanCaptureMetadata(graph);
|
|
1610
1614
|
return graph;
|
|
1611
1615
|
}
|
|
1612
1616
|
export function summarizeHarness(graph) {
|
|
@@ -1864,6 +1868,7 @@ function plannedDryRunRemote(plan) {
|
|
|
1864
1868
|
};
|
|
1865
1869
|
}
|
|
1866
1870
|
async function syncHarnessWithRemote(context, body) {
|
|
1871
|
+
assertHarnessGraphSyncSize(body.graph);
|
|
1867
1872
|
const profileOptions = { profile: context.profile, env: context.env };
|
|
1868
1873
|
const hasApiKey = Boolean(resolveApiKey(profileOptions));
|
|
1869
1874
|
if (context.dryRun && !hasApiKey)
|
|
@@ -2436,6 +2441,7 @@ function artifactFromAnnotation(annotation) {
|
|
|
2436
2441
|
const type = artifactType(first(values.kind) ?? first(values.type));
|
|
2437
2442
|
const sourcePath = first(values.path) ?? annotation.evidence.path;
|
|
2438
2443
|
const id = first(values.id) ?? stableId(type, first(values.name) ?? sourcePath);
|
|
2444
|
+
const body = readPromptOrToolBody(type, sourcePath, annotation);
|
|
2439
2445
|
return {
|
|
2440
2446
|
id,
|
|
2441
2447
|
type,
|
|
@@ -2443,6 +2449,7 @@ function artifactFromAnnotation(annotation) {
|
|
|
2443
2449
|
scope: scopeValue(first(values.scope)),
|
|
2444
2450
|
sourcePath,
|
|
2445
2451
|
contentHash: contentHashForAnnotatedSource(sourcePath, annotation),
|
|
2452
|
+
body,
|
|
2446
2453
|
ownedByAgentId: first(values.owner),
|
|
2447
2454
|
usedByAgentIds: values.agent ?? [],
|
|
2448
2455
|
confidence: numberValue(first(values.confidence), 0.85),
|
|
@@ -2811,12 +2818,14 @@ function maybeAddToolArtifact(graph, file) {
|
|
|
2811
2818
|
if (tools.length === 0)
|
|
2812
2819
|
return;
|
|
2813
2820
|
for (const tool of tools) {
|
|
2821
|
+
const body = extractToolBody(file, tool);
|
|
2814
2822
|
upsertArtifact(graph, {
|
|
2815
2823
|
id: stableId('tool', `${file.path}:${tool.name}`),
|
|
2816
2824
|
type: 'tool',
|
|
2817
2825
|
name: titleFromId(tool.name),
|
|
2818
2826
|
scope: 'agent',
|
|
2819
2827
|
sourcePath: file.path,
|
|
2828
|
+
body,
|
|
2820
2829
|
usedByAgentIds: [],
|
|
2821
2830
|
confidence: 0.6,
|
|
2822
2831
|
evidence: [{ path: file.path, reason: `Tool definition detected (${tool.reason}).` }],
|
|
@@ -3216,6 +3225,7 @@ function finalizeGraph(graph) {
|
|
|
3216
3225
|
type: artifactTypeFromId(artifactId),
|
|
3217
3226
|
name: titleFromId(artifactId),
|
|
3218
3227
|
scope: 'agent',
|
|
3228
|
+
sourcePath: agent.evidence[0]?.path,
|
|
3219
3229
|
usedByAgentIds: [agent.id],
|
|
3220
3230
|
confidence: 0.45,
|
|
3221
3231
|
evidence: agent.evidence,
|
|
@@ -3375,6 +3385,8 @@ function upsertArtifact(graph, next) {
|
|
|
3375
3385
|
existing.ownedByAgentId = existing.ownedByAgentId ?? next.ownedByAgentId;
|
|
3376
3386
|
existing.contentHash = existing.contentHash ?? next.contentHash;
|
|
3377
3387
|
existing.sourcePath = existing.sourcePath ?? next.sourcePath;
|
|
3388
|
+
if (!existing.body && next.body)
|
|
3389
|
+
existing.body = next.body;
|
|
3378
3390
|
if (existing.scope !== 'shared')
|
|
3379
3391
|
existing.scope = next.scope;
|
|
3380
3392
|
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { CommandContext } from './command-context.js';
|
|
2
|
+
import type { ProfileIOOptions } from './profiles.js';
|
|
3
|
+
export interface PromptArmPayload {
|
|
4
|
+
content: string;
|
|
5
|
+
prompt_version_id?: string;
|
|
6
|
+
prompt_id?: string;
|
|
7
|
+
prompt_label?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface PromptAbResult {
|
|
10
|
+
replaySetId: string;
|
|
11
|
+
runId: string;
|
|
12
|
+
status: string;
|
|
13
|
+
verdict: string;
|
|
14
|
+
run: Record<string, unknown> | null;
|
|
15
|
+
cases: Array<Record<string, unknown>>;
|
|
16
|
+
}
|
|
17
|
+
export declare function runPromptAb(flags: Record<string, string>, profileOptions: ProfileIOOptions, context: CommandContext): Promise<void>;
|
|
18
|
+
//# sourceMappingURL=prompts-ab.d.ts.map
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { isAbsolute, resolve } from 'node:path';
|
|
3
|
+
import { callControlAPI, callDataAPI } from './api-client.js';
|
|
4
|
+
import { resolveTenantId, validateConfig } from './config.js';
|
|
5
|
+
import { discoverPrompts, parsePromptFileForAb, readPromptLock, runPromptSync, } from './prompts.js';
|
|
6
|
+
const DEFAULT_SEEDS_PER_CASE = 3;
|
|
7
|
+
const DEFAULT_POLL_INTERVAL_MS = 15_000;
|
|
8
|
+
const DEFAULT_WAIT_TIMEOUT_MS = 2 * 60 * 60 * 1000;
|
|
9
|
+
const TERMINAL_RUN_STATUSES = new Set(['completed', 'skipped', 'error']);
|
|
10
|
+
export async function runPromptAb(flags, profileOptions, context) {
|
|
11
|
+
validateConfig();
|
|
12
|
+
const tenantId = flags['tenant-id'] || resolveTenantId(profileOptions);
|
|
13
|
+
if (!tenantId) {
|
|
14
|
+
throw new Error('Missing tenant id. Run `moda init` or pass --tenant-id=<id>.');
|
|
15
|
+
}
|
|
16
|
+
const baselineSource = flags.baseline || flags['baseline-file'] || flags['baseline-key'];
|
|
17
|
+
const candidateSource = flags.candidate || flags['candidate-file'] || flags['candidate-key'];
|
|
18
|
+
if (!baselineSource || !candidateSource) {
|
|
19
|
+
throw new Error('Usage: moda prompts ab --baseline=<file|key> --candidate=<file|key> [--auto-generate|--set-id=ID|--conversations=id1,id2]');
|
|
20
|
+
}
|
|
21
|
+
if (flags.sync === 'true') {
|
|
22
|
+
await runPromptSync({ ...flags, watch: 'false' }, profileOptions);
|
|
23
|
+
}
|
|
24
|
+
const promptArms = {
|
|
25
|
+
prod: resolvePromptArmSpec(baselineSource, 'baseline'),
|
|
26
|
+
proposed: resolvePromptArmSpec(candidateSource, 'candidate'),
|
|
27
|
+
};
|
|
28
|
+
const replaySetId = await ensureReplaySet(flags, tenantId, profileOptions);
|
|
29
|
+
const seedsPerCase = parsePositiveInt(flags.seeds ?? flags['seeds-per-case'], DEFAULT_SEEDS_PER_CASE, 10);
|
|
30
|
+
const assistantModel = (flags.model ?? flags['assistant-model'] ?? '').trim();
|
|
31
|
+
const enqueue = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/run`, {
|
|
32
|
+
method: 'POST',
|
|
33
|
+
body: JSON.stringify({
|
|
34
|
+
promptArms,
|
|
35
|
+
seedsPerCase,
|
|
36
|
+
assistantModel: assistantModel || undefined,
|
|
37
|
+
promotePrimary: flags['no-promote'] !== 'true',
|
|
38
|
+
}),
|
|
39
|
+
}, profileOptions, { timeoutMs: 120_000, retries: 1 });
|
|
40
|
+
const noWait = flags.wait === 'false' || flags['no-wait'] === 'true';
|
|
41
|
+
if (noWait) {
|
|
42
|
+
context.output.writeData({
|
|
43
|
+
replaySetId,
|
|
44
|
+
runId: enqueue.runId,
|
|
45
|
+
status: enqueue.status,
|
|
46
|
+
message: enqueue.message ?? 'Replay comparison queued',
|
|
47
|
+
});
|
|
48
|
+
return;
|
|
49
|
+
}
|
|
50
|
+
const timeoutMs = parsePositiveInt(flags.timeout, DEFAULT_WAIT_TIMEOUT_MS, 24 * 60 * 60 * 1000);
|
|
51
|
+
const pollIntervalMs = parsePositiveInt(flags['poll-interval'], DEFAULT_POLL_INTERVAL_MS, 120_000);
|
|
52
|
+
const deadline = Date.now() + timeoutMs;
|
|
53
|
+
let latest = null;
|
|
54
|
+
while (Date.now() < deadline) {
|
|
55
|
+
latest = await fetchLatestRun(tenantId, replaySetId, enqueue.runId, profileOptions);
|
|
56
|
+
const status = latest.run?.status ?? '';
|
|
57
|
+
const runId = latest.run?.runId ?? '';
|
|
58
|
+
if (TERMINAL_RUN_STATUSES.has(status) && runId === enqueue.runId) {
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
if (context.outputMode === 'human') {
|
|
62
|
+
process.stderr.write(`Waiting for replay run (${status || 'pending'})...\n`);
|
|
63
|
+
}
|
|
64
|
+
await sleep(pollIntervalMs);
|
|
65
|
+
}
|
|
66
|
+
if (!latest?.run
|
|
67
|
+
|| latest.run.runId !== enqueue.runId
|
|
68
|
+
|| !TERMINAL_RUN_STATUSES.has(latest.run.status)) {
|
|
69
|
+
throw new Error(`Timed out after ${timeoutMs}ms waiting for replay run ${enqueue.runId} to finish`);
|
|
70
|
+
}
|
|
71
|
+
const result = {
|
|
72
|
+
replaySetId,
|
|
73
|
+
runId: latest.run.runId,
|
|
74
|
+
status: latest.run.status,
|
|
75
|
+
verdict: formatVerdict(latest),
|
|
76
|
+
run: { ...latest.run },
|
|
77
|
+
cases: latest.cases.map((row) => ({ ...row })),
|
|
78
|
+
};
|
|
79
|
+
if (context.outputMode === 'human') {
|
|
80
|
+
printHumanVerdict(result);
|
|
81
|
+
}
|
|
82
|
+
context.output.writeData(result);
|
|
83
|
+
}
|
|
84
|
+
function resolvePromptArmSpec(source, label) {
|
|
85
|
+
const trimmed = source.trim();
|
|
86
|
+
if (!trimmed) {
|
|
87
|
+
throw new Error(`Empty prompt source for ${label}`);
|
|
88
|
+
}
|
|
89
|
+
const absPath = isAbsolute(trimmed) ? trimmed : resolve(process.cwd(), trimmed);
|
|
90
|
+
let key;
|
|
91
|
+
let content;
|
|
92
|
+
if (existsSync(absPath)) {
|
|
93
|
+
const parsed = parsePromptFileForAb(absPath);
|
|
94
|
+
key = parsed.key;
|
|
95
|
+
content = parsed.content;
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
const discovered = discoverPrompts();
|
|
99
|
+
const match = discovered.find((prompt) => prompt.key === trimmed)
|
|
100
|
+
?? discovered.find((prompt) => prompt.sourcePath === trimmed);
|
|
101
|
+
if (!match) {
|
|
102
|
+
throw new Error(`Could not resolve prompt '${trimmed}' — pass a .prompt.md path or a discovered prompt key`);
|
|
103
|
+
}
|
|
104
|
+
key = match.key;
|
|
105
|
+
content = match.content;
|
|
106
|
+
}
|
|
107
|
+
const locked = readPromptLock()?.prompts[key];
|
|
108
|
+
return {
|
|
109
|
+
content,
|
|
110
|
+
prompt_version_id: locked?.versionId ?? '',
|
|
111
|
+
prompt_id: locked?.promptId ?? '',
|
|
112
|
+
prompt_label: label,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
async function ensureReplaySet(flags, tenantId, profileOptions) {
|
|
116
|
+
const existingSetId = (flags['set-id'] || flags['replay-set-id'] || '').trim();
|
|
117
|
+
if (existingSetId) {
|
|
118
|
+
return existingSetId;
|
|
119
|
+
}
|
|
120
|
+
const conversations = parseCsv(flags.conversations ?? flags['conversation-ids']);
|
|
121
|
+
if (conversations.length) {
|
|
122
|
+
return createSetFromConversations(conversations, flags, tenantId, profileOptions);
|
|
123
|
+
}
|
|
124
|
+
if (flags['auto-generate'] === 'false' && !existingSetId) {
|
|
125
|
+
throw new Error('Provide --set-id=, --conversations=, or allow --auto-generate (default)');
|
|
126
|
+
}
|
|
127
|
+
const caseCount = parsePositiveInt(flags.cases ?? flags['case-count'], 5, 25);
|
|
128
|
+
const lookbackDays = parsePositiveInt(flags['lookback-days'], 30, 365);
|
|
129
|
+
const name = (flags.name ?? flags['set-name'] ?? `Prompt A/B ${new Date().toISOString().slice(0, 10)}`).trim();
|
|
130
|
+
const generated = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/auto-generate`, {
|
|
131
|
+
method: 'POST',
|
|
132
|
+
body: JSON.stringify({ name, caseCount, lookbackDays }),
|
|
133
|
+
}, profileOptions, { timeoutMs: 300_000, retries: 0 });
|
|
134
|
+
if (!generated?.id) {
|
|
135
|
+
throw new Error('Auto-generate did not return a replay set id');
|
|
136
|
+
}
|
|
137
|
+
return generated.id;
|
|
138
|
+
}
|
|
139
|
+
async function createSetFromConversations(conversationIds, flags, tenantId, profileOptions) {
|
|
140
|
+
const name = (flags.name ?? flags['set-name'] ?? `Prompt A/B ${conversationIds.length} conv`).trim();
|
|
141
|
+
const created = await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets`, {
|
|
142
|
+
method: 'POST',
|
|
143
|
+
body: JSON.stringify({ name, description: 'Created by moda prompts ab' }),
|
|
144
|
+
}, profileOptions);
|
|
145
|
+
let position = 0;
|
|
146
|
+
for (const conversationId of conversationIds) {
|
|
147
|
+
const scenario = await loadScenarioFromConversation(conversationId);
|
|
148
|
+
await callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(created.id)}/cases`, {
|
|
149
|
+
method: 'POST',
|
|
150
|
+
body: JSON.stringify({
|
|
151
|
+
title: conversationId,
|
|
152
|
+
scenario,
|
|
153
|
+
sourceConversationId: conversationId,
|
|
154
|
+
successCriteria: [
|
|
155
|
+
'The agent understands the user request',
|
|
156
|
+
'The agent uses tools appropriately when needed',
|
|
157
|
+
'The agent provides a helpful final answer',
|
|
158
|
+
],
|
|
159
|
+
position: position++,
|
|
160
|
+
}),
|
|
161
|
+
}, profileOptions);
|
|
162
|
+
}
|
|
163
|
+
return created.id;
|
|
164
|
+
}
|
|
165
|
+
async function loadScenarioFromConversation(conversationId) {
|
|
166
|
+
try {
|
|
167
|
+
const data = await callDataAPI(`/conversations/${encodeURIComponent(conversationId)}/context?msg_index=0&window=0`);
|
|
168
|
+
const firstUser = (data.messages ?? []).find((message) => message.role === 'user' && String(message.content ?? '').trim());
|
|
169
|
+
const opening = String(firstUser?.content ?? '').trim();
|
|
170
|
+
if (opening) {
|
|
171
|
+
return `The user says: "${opening}"\n\nReplay the conversation faithfully and help them accomplish their goal.`;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
// Fall through to generic scenario.
|
|
176
|
+
}
|
|
177
|
+
return `Replay conversation ${conversationId}: help the user accomplish their original goal using the available tools.`;
|
|
178
|
+
}
|
|
179
|
+
async function fetchLatestRun(tenantId, replaySetId, runId, profileOptions) {
|
|
180
|
+
// Pin the poll to OUR run id — otherwise a concurrent comparison on the same
|
|
181
|
+
// set would make `/runs/latest` return the newer run forever and this command
|
|
182
|
+
// would stall until timeout even after its own run finished.
|
|
183
|
+
return callControlAPI(`/tenants/${encodeURIComponent(tenantId)}/replay-sets/${encodeURIComponent(replaySetId)}/runs/latest?runId=${encodeURIComponent(runId)}`, { method: 'GET' }, profileOptions, { timeoutMs: 60_000, retries: 2 });
|
|
184
|
+
}
|
|
185
|
+
function formatVerdict(payload) {
|
|
186
|
+
const run = payload.run;
|
|
187
|
+
if (!run)
|
|
188
|
+
return 'no run';
|
|
189
|
+
if (run.status === 'skipped')
|
|
190
|
+
return `skipped: ${run.notes || 'no reason'}`;
|
|
191
|
+
if (run.status === 'error')
|
|
192
|
+
return `error: ${run.notes || 'unknown error'}`;
|
|
193
|
+
if (run.status !== 'completed')
|
|
194
|
+
return `status=${run.status}`;
|
|
195
|
+
const delta = run.proposedPassCount - run.prodPassCount;
|
|
196
|
+
const winner = delta > 0 ? 'candidate wins' : delta < 0 ? 'baseline wins' : 'tie on case passes';
|
|
197
|
+
const rates = `baseline ${run.prodPassCount}/${run.caseCount} (${pct(run.prodPassRate)}), candidate ${run.proposedPassCount}/${run.caseCount} (${pct(run.proposedPassRate)})`;
|
|
198
|
+
const notes = run.notes?.trim();
|
|
199
|
+
return `${winner}; ${rates}${notes ? `; ${notes}` : ''}`;
|
|
200
|
+
}
|
|
201
|
+
function printHumanVerdict(result) {
|
|
202
|
+
process.stderr.write('\nPrompt A/B replay verdict\n');
|
|
203
|
+
process.stderr.write('-------------------------\n');
|
|
204
|
+
process.stderr.write(`Replay set: ${result.replaySetId}\n`);
|
|
205
|
+
process.stderr.write(`Run id: ${result.runId}\n`);
|
|
206
|
+
process.stderr.write(`Status: ${result.status}\n`);
|
|
207
|
+
process.stderr.write(`Verdict: ${result.verdict}\n`);
|
|
208
|
+
if (result.cases.length) {
|
|
209
|
+
process.stderr.write('\nPer-case:\n');
|
|
210
|
+
for (const row of result.cases) {
|
|
211
|
+
const caseId = String(row.caseId ?? '');
|
|
212
|
+
const prod = formatPass(row.prodPass);
|
|
213
|
+
const proposed = formatPass(row.proposedPass);
|
|
214
|
+
process.stderr.write(` ${caseId}: baseline=${prod} candidate=${proposed}\n`);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
process.stderr.write('\n');
|
|
218
|
+
}
|
|
219
|
+
function formatPass(value) {
|
|
220
|
+
if (value === null || value === undefined)
|
|
221
|
+
return 'abstained';
|
|
222
|
+
return value ? 'pass' : 'fail';
|
|
223
|
+
}
|
|
224
|
+
function pct(rate) {
|
|
225
|
+
return `${Math.round(Math.max(0, Math.min(1, rate)) * 100)}%`;
|
|
226
|
+
}
|
|
227
|
+
function parsePositiveInt(raw, fallback, max) {
|
|
228
|
+
if (!raw)
|
|
229
|
+
return fallback;
|
|
230
|
+
const parsed = Number.parseInt(raw, 10);
|
|
231
|
+
if (!Number.isFinite(parsed) || parsed < 1)
|
|
232
|
+
return fallback;
|
|
233
|
+
return Math.min(parsed, max);
|
|
234
|
+
}
|
|
235
|
+
function parseCsv(raw) {
|
|
236
|
+
if (!raw?.trim())
|
|
237
|
+
return [];
|
|
238
|
+
return raw.split(',').map((part) => part.trim()).filter(Boolean);
|
|
239
|
+
}
|
|
240
|
+
function sleep(ms) {
|
|
241
|
+
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
|
|
242
|
+
}
|
|
243
|
+
//# sourceMappingURL=prompts-ab.js.map
|
package/dist/prompts.d.ts
CHANGED
|
@@ -1,5 +1,32 @@
|
|
|
1
1
|
import { type CommandContext } from './command-context.js';
|
|
2
2
|
import type { ProfileIOOptions } from './profiles.js';
|
|
3
|
+
interface PromptFile {
|
|
4
|
+
key: string;
|
|
5
|
+
name: string;
|
|
6
|
+
description?: string;
|
|
7
|
+
category?: string;
|
|
8
|
+
sourcePath: string;
|
|
9
|
+
content: string;
|
|
10
|
+
systemPrompt?: string;
|
|
11
|
+
messages?: unknown[];
|
|
12
|
+
variables?: unknown[];
|
|
13
|
+
variablesSchema?: Record<string, unknown>;
|
|
14
|
+
modelConfig?: Record<string, unknown>;
|
|
15
|
+
toolRefs?: unknown[];
|
|
16
|
+
responseSchema?: Record<string, unknown>;
|
|
17
|
+
contentHash: string;
|
|
18
|
+
}
|
|
19
|
+
interface PromptLock {
|
|
20
|
+
version: 1;
|
|
21
|
+
generatedAt: string;
|
|
22
|
+
prompts: Record<string, {
|
|
23
|
+
key: string;
|
|
24
|
+
sourcePath: string;
|
|
25
|
+
contentHash: string;
|
|
26
|
+
versionId?: string;
|
|
27
|
+
promptId?: string;
|
|
28
|
+
}>;
|
|
29
|
+
}
|
|
3
30
|
export declare function runPromptsCommand(input: CommandContext | string[], legacyFlags?: Record<string, string>, legacyProfileOptions?: ProfileIOOptions): Promise<void>;
|
|
4
31
|
export declare function initPrompts(options?: {
|
|
5
32
|
silent?: boolean;
|
|
@@ -20,4 +47,12 @@ export interface PromptSyncResult {
|
|
|
20
47
|
* control their own output. `moda prompts sync` wraps this with JSON output.
|
|
21
48
|
*/
|
|
22
49
|
export declare function runPromptSync(flags: Record<string, string>, profileOptions: ProfileIOOptions): Promise<PromptSyncResult>;
|
|
50
|
+
export declare function discoverPrompts(): PromptFile[];
|
|
51
|
+
export declare function readPromptLock(): PromptLock | null;
|
|
52
|
+
/** Parse a prompt file for A/B runs (body text + registry key). */
|
|
53
|
+
export declare function parsePromptFileForAb(absPath: string): {
|
|
54
|
+
key: string;
|
|
55
|
+
content: string;
|
|
56
|
+
};
|
|
57
|
+
export {};
|
|
23
58
|
//# sourceMappingURL=prompts.d.ts.map
|
package/dist/prompts.js
CHANGED
|
@@ -4,6 +4,7 @@ import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, writeFileS
|
|
|
4
4
|
import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
|
|
5
5
|
import { callControlAPI } from './api-client.js';
|
|
6
6
|
import { createCommandContext } from './command-context.js';
|
|
7
|
+
import { runPromptAb } from './prompts-ab.js';
|
|
7
8
|
const MODA_DIR = '.moda';
|
|
8
9
|
const MANIFEST_PATH = join(MODA_DIR, 'prompts.yml');
|
|
9
10
|
const LOCK_PATH = join(MODA_DIR, 'prompts.lock.json');
|
|
@@ -51,8 +52,11 @@ export async function runPromptsCommand(input, legacyFlags, legacyProfileOptions
|
|
|
51
52
|
case 'promote':
|
|
52
53
|
await promotePrompt(positionals, flags, profileOptions, context);
|
|
53
54
|
break;
|
|
55
|
+
case 'ab':
|
|
56
|
+
await runPromptAb(flags, profileOptions, context);
|
|
57
|
+
break;
|
|
54
58
|
default:
|
|
55
|
-
throw new Error(`Unknown prompts command '${subcommand}'. Use init, status, diff, sync, or
|
|
59
|
+
throw new Error(`Unknown prompts command '${subcommand}'. Use init, status, diff, sync, promote, or ab.`);
|
|
56
60
|
}
|
|
57
61
|
}
|
|
58
62
|
export async function initPrompts(options = {}) {
|
|
@@ -195,7 +199,7 @@ async function getPromptStatus() {
|
|
|
195
199
|
prompts: rows,
|
|
196
200
|
};
|
|
197
201
|
}
|
|
198
|
-
function discoverPrompts() {
|
|
202
|
+
export function discoverPrompts() {
|
|
199
203
|
const manifest = readManifest();
|
|
200
204
|
// A pattern like `src/prompts/**/*.prompt.md` should *only* match files under
|
|
201
205
|
// src/prompts. Extract both the base directory (everything up to the first
|
|
@@ -290,6 +294,14 @@ function readManifest() {
|
|
|
290
294
|
.filter((path) => path.includes('.prompt.'));
|
|
291
295
|
return { promptPaths: promptPaths.length ? promptPaths : DEFAULT_PROMPT_PATHS };
|
|
292
296
|
}
|
|
297
|
+
export function readPromptLock() {
|
|
298
|
+
return readLock();
|
|
299
|
+
}
|
|
300
|
+
/** Parse a prompt file for A/B runs (body text + registry key). */
|
|
301
|
+
export function parsePromptFileForAb(absPath) {
|
|
302
|
+
const parsed = parsePromptFile(absPath);
|
|
303
|
+
return { key: parsed.key, content: parsed.content };
|
|
304
|
+
}
|
|
293
305
|
function readLock() {
|
|
294
306
|
if (!existsSync(LOCK_PATH))
|
|
295
307
|
return null;
|
package/dist/skills.js
CHANGED
|
@@ -74,6 +74,108 @@ async function fetchSkills(status) {
|
|
|
74
74
|
status,
|
|
75
75
|
};
|
|
76
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* Authenticated GET against the ingest worker's `/v1/skills/runs` surface.
|
|
79
|
+
* Returns `{ status, body }` so callers can treat 404 (no such run) as data;
|
|
80
|
+
* every other non-2xx throws.
|
|
81
|
+
*/
|
|
82
|
+
async function skillRunsRequest(url) {
|
|
83
|
+
const apiKey = resolveApiKey();
|
|
84
|
+
if (!apiKey) {
|
|
85
|
+
throw new CliInputError('No Moda API key found. Run `moda init` first (or set MODA_API_KEY).');
|
|
86
|
+
}
|
|
87
|
+
const controller = new AbortController();
|
|
88
|
+
const timeout = setTimeout(() => controller.abort(), SKILLS_REQUEST_TIMEOUT_MS);
|
|
89
|
+
let response;
|
|
90
|
+
try {
|
|
91
|
+
response = await fetch(url, {
|
|
92
|
+
headers: { Authorization: `Bearer ${apiKey}` },
|
|
93
|
+
signal: controller.signal,
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
98
|
+
throw new Error(`Network error: failed to reach ${url} (${message})`);
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
clearTimeout(timeout);
|
|
102
|
+
}
|
|
103
|
+
const text = await response.text().catch(() => '');
|
|
104
|
+
if (!response.ok && response.status !== 404) {
|
|
105
|
+
throw new Error(`Failed to fetch run status (HTTP ${response.status}): ${text.slice(0, 200)}`);
|
|
106
|
+
}
|
|
107
|
+
let body = {};
|
|
108
|
+
try {
|
|
109
|
+
body = text ? JSON.parse(text) : {};
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
body = {};
|
|
113
|
+
}
|
|
114
|
+
return { status: response.status, body };
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Resolve a run's status + recent events. Without a run id, the tenant's
|
|
118
|
+
* latest run (`GET /v1/skills/runs?limit=1`) is used; the detail form
|
|
119
|
+
* (`GET /v1/skills/runs/:id`) always supplies the event breadcrumbs.
|
|
120
|
+
*/
|
|
121
|
+
async function fetchSkillRunStatus(runId) {
|
|
122
|
+
const ingestUrl = resolveIngestUrl().replace(/\/$/, '');
|
|
123
|
+
let resolvedRunId = runId;
|
|
124
|
+
if (!resolvedRunId) {
|
|
125
|
+
const list = await skillRunsRequest(`${ingestUrl}/v1/skills/runs?limit=1`);
|
|
126
|
+
const listBody = list.body;
|
|
127
|
+
const latest = Array.isArray(listBody.runs) ? listBody.runs[0] : undefined;
|
|
128
|
+
if (!latest)
|
|
129
|
+
return { run: null, events: [] };
|
|
130
|
+
resolvedRunId = latest.generation_run_id;
|
|
131
|
+
// Defensive: a row without an id can still be shown, just without events.
|
|
132
|
+
if (!resolvedRunId)
|
|
133
|
+
return { run: latest, events: [] };
|
|
134
|
+
}
|
|
135
|
+
const detail = await skillRunsRequest(`${ingestUrl}/v1/skills/runs/${encodeURIComponent(resolvedRunId)}`);
|
|
136
|
+
if (detail.status === 404) {
|
|
137
|
+
if (runId) {
|
|
138
|
+
throw new CliInputError(`No skill generation run found with id '${runId}'`);
|
|
139
|
+
}
|
|
140
|
+
return { run: null, events: [] };
|
|
141
|
+
}
|
|
142
|
+
const detailBody = detail.body;
|
|
143
|
+
return {
|
|
144
|
+
run: detailBody.run ?? null,
|
|
145
|
+
events: Array.isArray(detailBody.events) ? detailBody.events : [],
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
const STATUS_HUMAN_EVENT_LIMIT = 10;
|
|
149
|
+
/** Human rendering for `moda skills status` (key: value lines, gen-style). */
|
|
150
|
+
function formatRunStatusHuman(run, events) {
|
|
151
|
+
const counters = run.counters ?? {};
|
|
152
|
+
const lines = [
|
|
153
|
+
run.generation_run_id && `run: ${run.generation_run_id}`,
|
|
154
|
+
`status: ${run.status || 'unknown'}`,
|
|
155
|
+
run.current_stage && `stage: ${run.current_stage}`,
|
|
156
|
+
run.outcome && `outcome: ${run.outcome}`,
|
|
157
|
+
`processed: ${Number(counters.sessions_processed ?? 0)}`,
|
|
158
|
+
`failed: ${Number(counters.sessions_failed ?? 0)}`,
|
|
159
|
+
Number(counters.distilled_skills_generated ?? 0) > 0
|
|
160
|
+
&& `distilled: ${Number(counters.distilled_skills_generated ?? 0)}`,
|
|
161
|
+
Number(counters.skills_indexed ?? 0) > 0
|
|
162
|
+
&& `indexed: ${Number(counters.skills_indexed ?? 0)}`,
|
|
163
|
+
(run.generated_skill_ids?.length ?? 0) > 0
|
|
164
|
+
&& `skills: ${(run.generated_skill_ids ?? []).join(', ')}`,
|
|
165
|
+
run.error && `error: ${run.error}`,
|
|
166
|
+
run.started_at && `started: ${run.started_at}`,
|
|
167
|
+
run.updated_at && `updated: ${run.updated_at}`,
|
|
168
|
+
].filter(Boolean);
|
|
169
|
+
if (events.length > 0) {
|
|
170
|
+
lines.push('recent events (newest first):');
|
|
171
|
+
for (const event of events.slice(0, STATUS_HUMAN_EVENT_LIMIT)) {
|
|
172
|
+
lines.push(` ${[event.created_at, event.level, event.stage, event.message]
|
|
173
|
+
.filter(Boolean)
|
|
174
|
+
.join(' ')}`);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
return lines.join('\n');
|
|
178
|
+
}
|
|
77
179
|
/**
|
|
78
180
|
* Write `content` to `filePath`, creating parent dirs. Returns whether the
|
|
79
181
|
* file was created, updated (content differed), or left unchanged.
|
|
@@ -463,6 +565,8 @@ export async function runSkillsCommand(input) {
|
|
|
463
565
|
'accepted: true',
|
|
464
566
|
result.generation_run_id && `generation_run_id: ${result.generation_run_id}`,
|
|
465
567
|
result.message && `message: ${result.message}`,
|
|
568
|
+
result.generation_run_id
|
|
569
|
+
&& `follow progress: moda skills status ${result.generation_run_id}`,
|
|
466
570
|
].filter(Boolean).join('\n'));
|
|
467
571
|
}
|
|
468
572
|
else {
|
|
@@ -473,10 +577,26 @@ export async function runSkillsCommand(input) {
|
|
|
473
577
|
`pending_found: ${result.pending_sessions_found ?? 0}`,
|
|
474
578
|
`distilled: ${Number(result.cluster_result?.distilled_skills_generated ?? 0)}`,
|
|
475
579
|
`indexed: ${Number(result.cluster_result?.skills_indexed ?? 0)}`,
|
|
476
|
-
|
|
580
|
+
result.distillation_skipped
|
|
581
|
+
&& `distillation skipped: ${result.distillation_skip_reason || 'unknown'}`,
|
|
582
|
+
].filter(Boolean).join('\n'));
|
|
477
583
|
}
|
|
478
584
|
return Number(result.sessions_failed ?? 0) > 0 ? 1 : 0;
|
|
479
585
|
}
|
|
586
|
+
if (sub === 'status') {
|
|
587
|
+
const runId = positionals[1] || flags['run-id'] || undefined;
|
|
588
|
+
const { run, events } = await fetchSkillRunStatus(runId);
|
|
589
|
+
if (context.outputMode !== 'human') {
|
|
590
|
+
context.output.writeData({ run, events }, { status: run?.status === 'failed' ? 'error' : 'success' });
|
|
591
|
+
}
|
|
592
|
+
else if (!run) {
|
|
593
|
+
console.log('No skill generation runs found yet. Start one with `moda skills gen`.');
|
|
594
|
+
}
|
|
595
|
+
else {
|
|
596
|
+
console.log(formatRunStatusHuman(run, events));
|
|
597
|
+
}
|
|
598
|
+
return run?.status === 'failed' ? 1 : 0;
|
|
599
|
+
}
|
|
480
600
|
if (sub === 'proposals' && positionals[1] === 'list') {
|
|
481
601
|
const proposals = await fetchSkillProposals(flags, { profile: context.profile });
|
|
482
602
|
if (flags.json === 'true') {
|
|
@@ -533,7 +653,7 @@ export async function runSkillsCommand(input) {
|
|
|
533
653
|
return 0;
|
|
534
654
|
}
|
|
535
655
|
if (sub !== 'pull') {
|
|
536
|
-
throw new CliInputError(sub ? `Unknown skills subcommand '${sub}'` : 'A skills subcommand is required', 'Usage: moda skills gen [--source=all|sdk|coding-agent] [--max-sessions=N] [--start-at=ISO] [--end-at=ISO] [--replay] [--replay-set-id=ID] [--reprocess-segments] [--force-reprocess] [--wait] | moda skills proposals list [--status=ready_for_pr] | moda skills proposal apply <proposal-id> | moda skills pull [--status=approved|proposed|all] | moda skills sync [--dry-run]');
|
|
656
|
+
throw new CliInputError(sub ? `Unknown skills subcommand '${sub}'` : 'A skills subcommand is required', 'Usage: moda skills gen [--source=all|sdk|coding-agent] [--max-sessions=N] [--start-at=ISO] [--end-at=ISO] [--replay] [--replay-set-id=ID] [--reprocess-segments] [--force-reprocess] [--wait] | moda skills status [run-id] | moda skills proposals list [--status=ready_for_pr] | moda skills proposal apply <proposal-id> | moda skills pull [--status=approved|proposed|all] | moda skills sync [--dry-run]');
|
|
537
657
|
}
|
|
538
658
|
const status = (flags.status ?? 'approved').toLowerCase();
|
|
539
659
|
if (!VALID_STATUSES.has(status)) {
|
|
@@ -580,7 +700,7 @@ async function generateSkills(flags) {
|
|
|
580
700
|
replay_set_id: flags['replay-set-id'] || flags['eval-set-id'] || undefined,
|
|
581
701
|
run_clustering: flags['no-clustering'] !== 'true',
|
|
582
702
|
source: normalizeSkillSource(flags.source),
|
|
583
|
-
clustering_timeout_seconds: parsePositiveInt(flags['clustering-timeout-seconds'],
|
|
703
|
+
clustering_timeout_seconds: parsePositiveInt(flags['clustering-timeout-seconds'], 7200),
|
|
584
704
|
clustering_poll_seconds: parsePositiveInt(flags['clustering-poll-seconds'], 15),
|
|
585
705
|
max_cluster_pct: parseOptionalFloat(flags['max-cluster-pct']),
|
|
586
706
|
reprocess_segments: flags['reprocess-segments'] === 'true' || forceReprocess,
|
package/package.json
CHANGED