@mutagent/cli 0.1.12 → 0.1.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +461 -222
- package/dist/bin/cli.js +2249 -499
- package/dist/bin/cli.js.map +20 -12
- package/dist/index.js +127 -93
- package/dist/index.js.map +6 -6
- package/package.json +3 -8
package/dist/bin/cli.js
CHANGED
|
@@ -16,44 +16,24 @@ var __toESM = (mod, isNodeMode, target) => {
|
|
|
16
16
|
});
|
|
17
17
|
return to;
|
|
18
18
|
};
|
|
19
|
+
var __export = (target, all) => {
|
|
20
|
+
for (var name in all)
|
|
21
|
+
__defProp(target, name, {
|
|
22
|
+
get: all[name],
|
|
23
|
+
enumerable: true,
|
|
24
|
+
configurable: true,
|
|
25
|
+
set: (newValue) => all[name] = () => newValue
|
|
26
|
+
});
|
|
27
|
+
};
|
|
28
|
+
var __esm = (fn, res) => () => (fn && (res = fn(fn = 0)), res);
|
|
19
29
|
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
20
30
|
|
|
21
|
-
// src/bin/cli.ts
|
|
22
|
-
import { Command as Command12 } from "commander";
|
|
23
|
-
import chalk13 from "chalk";
|
|
24
|
-
import { readFileSync as readFileSync12 } from "fs";
|
|
25
|
-
import { join as join3, dirname } from "path";
|
|
26
|
-
import { fileURLToPath } from "url";
|
|
27
|
-
|
|
28
|
-
// src/commands/auth.ts
|
|
29
|
-
import { Command } from "commander";
|
|
30
|
-
import inquirer from "inquirer";
|
|
31
|
-
import chalk2 from "chalk";
|
|
32
|
-
import ora from "ora";
|
|
33
|
-
|
|
34
31
|
// src/lib/config.ts
|
|
35
32
|
import { cosmiconfigSync } from "cosmiconfig";
|
|
36
33
|
import { z } from "zod";
|
|
37
34
|
import { homedir } from "os";
|
|
38
35
|
import { join } from "path";
|
|
39
36
|
import { existsSync, readFileSync, writeFileSync, mkdirSync } from "fs";
|
|
40
|
-
var configSchema = z.object({
|
|
41
|
-
apiKey: z.string().optional(),
|
|
42
|
-
endpoint: z.string().default("https://api.mutagent.io"),
|
|
43
|
-
format: z.enum(["table", "json"]).default("table"),
|
|
44
|
-
timeout: z.number().default(30000),
|
|
45
|
-
defaultWorkspace: z.string().optional(),
|
|
46
|
-
defaultOrganization: z.string().optional()
|
|
47
|
-
});
|
|
48
|
-
var credentialsSchema = z.object({
|
|
49
|
-
apiKey: z.string().optional(),
|
|
50
|
-
endpoint: z.string().optional(),
|
|
51
|
-
defaultWorkspace: z.string().optional(),
|
|
52
|
-
defaultOrganization: z.string().optional(),
|
|
53
|
-
expiresAt: z.string().optional()
|
|
54
|
-
}).loose();
|
|
55
|
-
var CREDENTIALS_DIR = join(homedir(), ".config", "mutagent");
|
|
56
|
-
var CREDENTIALS_FILE = join(CREDENTIALS_DIR, "credentials.json");
|
|
57
37
|
function parseJsonSafe(content, schema) {
|
|
58
38
|
try {
|
|
59
39
|
const parsed = JSON.parse(content);
|
|
@@ -161,80 +141,28 @@ function setDefaultOrganization(organizationId) {
|
|
|
161
141
|
};
|
|
162
142
|
writeFileSync(CREDENTIALS_FILE, JSON.stringify(updated, null, 2));
|
|
163
143
|
}
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
144
|
+
var configSchema, credentialsSchema, CREDENTIALS_DIR, CREDENTIALS_FILE;
|
|
145
|
+
var init_config = __esm(() => {
|
|
146
|
+
configSchema = z.object({
|
|
147
|
+
apiKey: z.string().optional(),
|
|
148
|
+
endpoint: z.string().default("https://api.mutagent.io"),
|
|
149
|
+
format: z.enum(["table", "json"]).default("table"),
|
|
150
|
+
timeout: z.number().default(30000),
|
|
151
|
+
defaultWorkspace: z.string().optional(),
|
|
152
|
+
defaultOrganization: z.string().optional()
|
|
153
|
+
});
|
|
154
|
+
credentialsSchema = z.object({
|
|
155
|
+
apiKey: z.string().optional(),
|
|
156
|
+
endpoint: z.string().optional(),
|
|
157
|
+
defaultWorkspace: z.string().optional(),
|
|
158
|
+
defaultOrganization: z.string().optional(),
|
|
159
|
+
expiresAt: z.string().optional()
|
|
160
|
+
}).loose();
|
|
161
|
+
CREDENTIALS_DIR = join(homedir(), ".config", "mutagent");
|
|
162
|
+
CREDENTIALS_FILE = join(CREDENTIALS_DIR, "credentials.json");
|
|
163
|
+
});
|
|
167
164
|
|
|
168
165
|
// src/lib/errors.ts
|
|
169
|
-
class MutagentError extends Error {
|
|
170
|
-
code;
|
|
171
|
-
suggestion;
|
|
172
|
-
exitCode;
|
|
173
|
-
constructor(code, message, suggestion, exitCode = 1) {
|
|
174
|
-
super(message);
|
|
175
|
-
this.code = code;
|
|
176
|
-
this.suggestion = suggestion;
|
|
177
|
-
this.exitCode = exitCode;
|
|
178
|
-
this.name = "MutagentError";
|
|
179
|
-
}
|
|
180
|
-
toJSON() {
|
|
181
|
-
return {
|
|
182
|
-
error: {
|
|
183
|
-
code: this.code,
|
|
184
|
-
message: this.message,
|
|
185
|
-
suggestion: this.suggestion
|
|
186
|
-
}
|
|
187
|
-
};
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
var AUTH_REMEDIATION_MESSAGE = [
|
|
191
|
-
"Authentication required. Options:",
|
|
192
|
-
" Interactive: mutagent auth login --browser",
|
|
193
|
-
" Non-interactive: export MUTAGENT_API_KEY=<your-key>",
|
|
194
|
-
" CI/CD: mutagent auth login --api-key <key>"
|
|
195
|
-
].join(`
|
|
196
|
-
`);
|
|
197
|
-
|
|
198
|
-
class AuthenticationError extends MutagentError {
|
|
199
|
-
suggestions;
|
|
200
|
-
cause;
|
|
201
|
-
constructor(message, options = {}) {
|
|
202
|
-
super("AUTH_REQUIRED", message ?? 'Authentication required. Please run "mutagent auth login"', options.suggestions ? options.suggestions[0] : "Run: mutagent auth login --browser", 2);
|
|
203
|
-
this.suggestions = options.suggestions ?? [
|
|
204
|
-
"mutagent auth login --browser",
|
|
205
|
-
"export MUTAGENT_API_KEY=<your-key>",
|
|
206
|
-
"mutagent auth login --api-key <key>"
|
|
207
|
-
];
|
|
208
|
-
this.suggestion = options.suggestions ? options.suggestions[0] : "Run: mutagent auth login --browser";
|
|
209
|
-
this.cause = options.cause;
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
class ApiError extends MutagentError {
|
|
214
|
-
statusCode;
|
|
215
|
-
constructor(status, message) {
|
|
216
|
-
super(`API_${status}`, message, status === 401 ? "Check your API key with: mutagent auth status" : undefined, 1);
|
|
217
|
-
this.statusCode = status;
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
var WORKSPACE_REMEDIATION_MESSAGE = [
|
|
221
|
-
"Workspace context missing. To fix:",
|
|
222
|
-
" mutagent config set workspace <workspace-id> # Set default workspace",
|
|
223
|
-
" mutagent workspaces list # List available workspaces"
|
|
224
|
-
].join(`
|
|
225
|
-
`);
|
|
226
|
-
|
|
227
|
-
class WorkspaceContextError extends MutagentError {
|
|
228
|
-
constructor(message) {
|
|
229
|
-
super("WORKSPACE_CONTEXT_MISSING", message ?? "Workspace context is required but not configured", "Run: mutagent config set workspace <workspace-id>", 3);
|
|
230
|
-
}
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
class ValidationError extends MutagentError {
|
|
234
|
-
constructor(message) {
|
|
235
|
-
super("VALIDATION_ERROR", message, "Check the command syntax with: mutagent <command> --help", 1);
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
166
|
function handleError(error, isJson) {
|
|
239
167
|
if (error instanceof MutagentError) {
|
|
240
168
|
if (isJson) {
|
|
@@ -313,8 +241,87 @@ function handleError(error, isJson) {
|
|
|
313
241
|
}
|
|
314
242
|
process.exit(1);
|
|
315
243
|
}
|
|
244
|
+
var MutagentError, AUTH_REMEDIATION_MESSAGE, AuthenticationError, ApiError, WORKSPACE_REMEDIATION_MESSAGE, WorkspaceContextError, ValidationError;
|
|
245
|
+
var init_errors = __esm(() => {
|
|
246
|
+
MutagentError = class MutagentError extends Error {
|
|
247
|
+
code;
|
|
248
|
+
suggestion;
|
|
249
|
+
exitCode;
|
|
250
|
+
constructor(code, message, suggestion, exitCode = 1) {
|
|
251
|
+
super(message);
|
|
252
|
+
this.code = code;
|
|
253
|
+
this.suggestion = suggestion;
|
|
254
|
+
this.exitCode = exitCode;
|
|
255
|
+
this.name = "MutagentError";
|
|
256
|
+
}
|
|
257
|
+
toJSON() {
|
|
258
|
+
return {
|
|
259
|
+
error: {
|
|
260
|
+
code: this.code,
|
|
261
|
+
message: this.message,
|
|
262
|
+
suggestion: this.suggestion
|
|
263
|
+
}
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
};
|
|
267
|
+
AUTH_REMEDIATION_MESSAGE = [
|
|
268
|
+
"Authentication required. Options:",
|
|
269
|
+
" Interactive: mutagent auth login --browser",
|
|
270
|
+
" Non-interactive: export MUTAGENT_API_KEY=<your-key>",
|
|
271
|
+
" CI/CD: mutagent auth login --api-key <key>"
|
|
272
|
+
].join(`
|
|
273
|
+
`);
|
|
274
|
+
AuthenticationError = class AuthenticationError extends MutagentError {
|
|
275
|
+
suggestions;
|
|
276
|
+
cause;
|
|
277
|
+
constructor(message, options = {}) {
|
|
278
|
+
super("AUTH_REQUIRED", message ?? 'Authentication required. Please run "mutagent auth login"', options.suggestions ? options.suggestions[0] : "Run: mutagent auth login --browser", 2);
|
|
279
|
+
this.suggestions = options.suggestions ?? [
|
|
280
|
+
"mutagent auth login --browser",
|
|
281
|
+
"export MUTAGENT_API_KEY=<your-key>",
|
|
282
|
+
"mutagent auth login --api-key <key>"
|
|
283
|
+
];
|
|
284
|
+
this.suggestion = options.suggestions ? options.suggestions[0] : "Run: mutagent auth login --browser";
|
|
285
|
+
this.cause = options.cause;
|
|
286
|
+
}
|
|
287
|
+
};
|
|
288
|
+
ApiError = class ApiError extends MutagentError {
|
|
289
|
+
statusCode;
|
|
290
|
+
constructor(status, message) {
|
|
291
|
+
super(`API_${status}`, message, status === 401 ? "Check your API key with: mutagent auth status" : undefined, 1);
|
|
292
|
+
this.statusCode = status;
|
|
293
|
+
}
|
|
294
|
+
};
|
|
295
|
+
WORKSPACE_REMEDIATION_MESSAGE = [
|
|
296
|
+
"Workspace context missing. To fix:",
|
|
297
|
+
" mutagent config set workspace <workspace-id> # Set default workspace",
|
|
298
|
+
" mutagent workspaces list # List available workspaces"
|
|
299
|
+
].join(`
|
|
300
|
+
`);
|
|
301
|
+
WorkspaceContextError = class WorkspaceContextError extends MutagentError {
|
|
302
|
+
constructor(message) {
|
|
303
|
+
super("WORKSPACE_CONTEXT_MISSING", message ?? "Workspace context is required but not configured", "Run: mutagent config set workspace <workspace-id>", 3);
|
|
304
|
+
}
|
|
305
|
+
};
|
|
306
|
+
ValidationError = class ValidationError extends MutagentError {
|
|
307
|
+
constructor(message) {
|
|
308
|
+
super("VALIDATION_ERROR", message, "Check the command syntax with: mutagent <command> --help", 1);
|
|
309
|
+
}
|
|
310
|
+
};
|
|
311
|
+
});
|
|
316
312
|
|
|
317
313
|
// src/lib/sdk-client.ts
|
|
314
|
+
var exports_sdk_client = {};
|
|
315
|
+
__export(exports_sdk_client, {
|
|
316
|
+
validateApiKey: () => validateApiKey,
|
|
317
|
+
resetSDKClient: () => resetSDKClient,
|
|
318
|
+
getSDKClient: () => getSDKClient,
|
|
319
|
+
fetchWorkspaces: () => fetchWorkspaces,
|
|
320
|
+
fetchOrganizations: () => fetchOrganizations,
|
|
321
|
+
MutagentSDK: () => SDKClientWrapper
|
|
322
|
+
});
|
|
323
|
+
import { Mutagent, HTTPClient } from "@mutagent/sdk";
|
|
324
|
+
|
|
318
325
|
class SDKClientWrapper {
|
|
319
326
|
sdk;
|
|
320
327
|
apiKey;
|
|
@@ -439,9 +446,9 @@ class SDKClientWrapper {
|
|
|
439
446
|
systemPrompt: data.systemPrompt ?? undefined,
|
|
440
447
|
humanPrompt: data.humanPrompt ?? undefined,
|
|
441
448
|
rawPrompt: data.rawPrompt ?? undefined,
|
|
442
|
-
inputSchema: data.inputSchema ??
|
|
443
|
-
outputSchema: data.outputSchema,
|
|
444
|
-
metadata: data.metadata,
|
|
449
|
+
inputSchema: data.inputSchema ?? undefined,
|
|
450
|
+
outputSchema: data.outputSchema ?? undefined,
|
|
451
|
+
metadata: data.metadata ?? undefined,
|
|
445
452
|
tags: data.tags ?? undefined
|
|
446
453
|
});
|
|
447
454
|
return response;
|
|
@@ -853,7 +860,6 @@ class SDKClientWrapper {
|
|
|
853
860
|
}
|
|
854
861
|
}
|
|
855
862
|
}
|
|
856
|
-
var sdkClient = null;
|
|
857
863
|
function getSDKClient() {
|
|
858
864
|
if (!sdkClient) {
|
|
859
865
|
const apiKey = getApiKey();
|
|
@@ -870,6 +876,9 @@ function getSDKClient() {
|
|
|
870
876
|
}
|
|
871
877
|
return sdkClient;
|
|
872
878
|
}
|
|
879
|
+
function resetSDKClient() {
|
|
880
|
+
sdkClient = null;
|
|
881
|
+
}
|
|
873
882
|
async function validateApiKey(apiKey, endpoint) {
|
|
874
883
|
try {
|
|
875
884
|
const response = await fetch(`${endpoint}/api/organizations`, {
|
|
@@ -906,6 +915,28 @@ async function fetchWorkspaces(apiKey, endpoint, orgId) {
|
|
|
906
915
|
return [];
|
|
907
916
|
}
|
|
908
917
|
}
|
|
918
|
+
var sdkClient = null;
|
|
919
|
+
var init_sdk_client = __esm(() => {
|
|
920
|
+
init_errors();
|
|
921
|
+
init_config();
|
|
922
|
+
});
|
|
923
|
+
|
|
924
|
+
// src/bin/cli.ts
|
|
925
|
+
import { Command as Command14 } from "commander";
|
|
926
|
+
import chalk18 from "chalk";
|
|
927
|
+
import { readFileSync as readFileSync14 } from "fs";
|
|
928
|
+
import { join as join7, dirname } from "path";
|
|
929
|
+
import { fileURLToPath } from "url";
|
|
930
|
+
|
|
931
|
+
// src/commands/auth.ts
|
|
932
|
+
init_config();
|
|
933
|
+
init_sdk_client();
|
|
934
|
+
import { Command } from "commander";
|
|
935
|
+
import inquirer from "inquirer";
|
|
936
|
+
import chalk3 from "chalk";
|
|
937
|
+
import ora from "ora";
|
|
938
|
+
import { existsSync as existsSync3 } from "fs";
|
|
939
|
+
import { join as join4 } from "path";
|
|
909
940
|
|
|
910
941
|
// src/lib/output.ts
|
|
911
942
|
import chalk from "chalk";
|
|
@@ -1028,6 +1059,9 @@ ${String(data.length)} result(s)`));
|
|
|
1028
1059
|
}
|
|
1029
1060
|
}
|
|
1030
1061
|
|
|
1062
|
+
// src/commands/auth.ts
|
|
1063
|
+
init_errors();
|
|
1064
|
+
|
|
1031
1065
|
// src/lib/browser-auth.ts
|
|
1032
1066
|
import { hostname, platform } from "os";
|
|
1033
1067
|
function generateCliToken() {
|
|
@@ -1165,22 +1199,658 @@ class BrowserAuthError extends Error {
|
|
|
1165
1199
|
}
|
|
1166
1200
|
}
|
|
1167
1201
|
|
|
1202
|
+
// src/lib/mutation-context.ts
|
|
1203
|
+
import { existsSync as existsSync2, readFileSync as readFileSync2, writeFileSync as writeFileSync2, mkdirSync as mkdirSync2 } from "fs";
|
|
1204
|
+
import { join as join2, resolve } from "path";
|
|
1205
|
+
function parseTableRows(lines) {
|
|
1206
|
+
const rows = [];
|
|
1207
|
+
for (const line of lines) {
|
|
1208
|
+
const trimmed = line.trim();
|
|
1209
|
+
if (!trimmed.startsWith("|") || !trimmed.endsWith("|"))
|
|
1210
|
+
continue;
|
|
1211
|
+
if (/^\|[\s-:|]+\|$/.test(trimmed))
|
|
1212
|
+
continue;
|
|
1213
|
+
const cells = trimmed.slice(1, -1).split("|").map((c) => c.trim());
|
|
1214
|
+
rows.push(cells);
|
|
1215
|
+
}
|
|
1216
|
+
return rows;
|
|
1217
|
+
}
|
|
1218
|
+
function emptyToUndefined(value) {
|
|
1219
|
+
return value && value.length > 0 ? value : undefined;
|
|
1220
|
+
}
|
|
1221
|
+
function renderTable(headers, rows) {
|
|
1222
|
+
const separator = headers.map((h) => "-".repeat(Math.max(h.length, 3)));
|
|
1223
|
+
const lines = [
|
|
1224
|
+
`| ${headers.join(" | ")} |`,
|
|
1225
|
+
`| ${separator.join(" | ")} |`,
|
|
1226
|
+
...rows.map((r) => `| ${r.join(" | ")} |`)
|
|
1227
|
+
];
|
|
1228
|
+
return lines.join(`
|
|
1229
|
+
`);
|
|
1230
|
+
}
|
|
1231
|
+
var CONTEXT_DIR = ".mutagent";
|
|
1232
|
+
var CONTEXT_FILE = "mutation-context.md";
|
|
1233
|
+
|
|
1234
|
+
class MutationContext {
|
|
1235
|
+
prompts = [];
|
|
1236
|
+
datasets = [];
|
|
1237
|
+
evaluations = [];
|
|
1238
|
+
integrations = [];
|
|
1239
|
+
filePath;
|
|
1240
|
+
constructor(filePath) {
|
|
1241
|
+
this.filePath = filePath;
|
|
1242
|
+
}
|
|
1243
|
+
static load(projectRoot) {
|
|
1244
|
+
const root = projectRoot ?? process.cwd();
|
|
1245
|
+
const dirPath = join2(resolve(root), CONTEXT_DIR);
|
|
1246
|
+
const filePath = join2(dirPath, CONTEXT_FILE);
|
|
1247
|
+
const ctx = new MutationContext(filePath);
|
|
1248
|
+
if (existsSync2(filePath)) {
|
|
1249
|
+
const content = readFileSync2(filePath, "utf-8");
|
|
1250
|
+
ctx.parse(content);
|
|
1251
|
+
}
|
|
1252
|
+
return ctx;
|
|
1253
|
+
}
|
|
1254
|
+
save() {
|
|
1255
|
+
const dirPath = join2(this.filePath, "..");
|
|
1256
|
+
if (!existsSync2(dirPath)) {
|
|
1257
|
+
mkdirSync2(dirPath, { recursive: true });
|
|
1258
|
+
}
|
|
1259
|
+
writeFileSync2(this.filePath, this.render(), "utf-8");
|
|
1260
|
+
}
|
|
1261
|
+
addDiscoveredPrompt(file, line, preview) {
|
|
1262
|
+
const existing = this.prompts.find((p) => p.file === file && p.line === line);
|
|
1263
|
+
if (existing) {
|
|
1264
|
+
existing.preview = preview;
|
|
1265
|
+
return;
|
|
1266
|
+
}
|
|
1267
|
+
this.prompts.push({ file, line, preview, status: "discovered" });
|
|
1268
|
+
}
|
|
1269
|
+
markPromptUploaded(file, platformId, version) {
|
|
1270
|
+
const entry = this.prompts.find((p) => p.file === file);
|
|
1271
|
+
if (entry) {
|
|
1272
|
+
entry.platformId = platformId;
|
|
1273
|
+
entry.version = version;
|
|
1274
|
+
entry.status = "uploaded";
|
|
1275
|
+
}
|
|
1276
|
+
}
|
|
1277
|
+
getDiscoveredPrompts() {
|
|
1278
|
+
return this.prompts.filter((p) => p.status === "discovered");
|
|
1279
|
+
}
|
|
1280
|
+
getUploadedPrompts() {
|
|
1281
|
+
return this.prompts.filter((p) => p.status === "uploaded");
|
|
1282
|
+
}
|
|
1283
|
+
addDiscoveredDataset(file, name, items) {
|
|
1284
|
+
const existing = this.datasets.find((d) => d.file === file && d.name === name);
|
|
1285
|
+
if (existing) {
|
|
1286
|
+
existing.items = items;
|
|
1287
|
+
return;
|
|
1288
|
+
}
|
|
1289
|
+
this.datasets.push({ file, name, items, status: "discovered" });
|
|
1290
|
+
}
|
|
1291
|
+
markDatasetUploaded(file, platformId, promptId) {
|
|
1292
|
+
const entry = this.datasets.find((d) => d.file === file);
|
|
1293
|
+
if (entry) {
|
|
1294
|
+
entry.platformId = platformId;
|
|
1295
|
+
entry.promptId = promptId;
|
|
1296
|
+
entry.status = "uploaded";
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
addEvaluation(name, criteriaCount, platformId, promptId) {
|
|
1300
|
+
const existing = this.evaluations.find((e) => e.name === name);
|
|
1301
|
+
if (existing) {
|
|
1302
|
+
existing.criteriaCount = criteriaCount;
|
|
1303
|
+
existing.platformId = platformId;
|
|
1304
|
+
existing.promptId = promptId;
|
|
1305
|
+
existing.status = "uploaded";
|
|
1306
|
+
return;
|
|
1307
|
+
}
|
|
1308
|
+
this.evaluations.push({ name, criteriaCount, platformId, promptId, status: "uploaded" });
|
|
1309
|
+
}
|
|
1310
|
+
setIntegrationStatus(framework, pkg, status) {
|
|
1311
|
+
const existing = this.integrations.find((i) => i.framework === framework);
|
|
1312
|
+
if (existing) {
|
|
1313
|
+
existing.pkg = pkg;
|
|
1314
|
+
existing.status = status;
|
|
1315
|
+
return;
|
|
1316
|
+
}
|
|
1317
|
+
this.integrations.push({ framework, pkg, status });
|
|
1318
|
+
}
|
|
1319
|
+
getSummary() {
|
|
1320
|
+
return {
|
|
1321
|
+
prompts: this.prompts.length,
|
|
1322
|
+
datasets: this.datasets.length,
|
|
1323
|
+
evaluations: this.evaluations.length,
|
|
1324
|
+
integrated: this.integrations.map((i) => i.framework)
|
|
1325
|
+
};
|
|
1326
|
+
}
|
|
1327
|
+
render() {
|
|
1328
|
+
const now = new Date().toISOString();
|
|
1329
|
+
const sections = [
|
|
1330
|
+
"# MutagenT Mutation Context",
|
|
1331
|
+
`<!-- MutagenT:AUTO-GENERATED - Do not edit manually -->`,
|
|
1332
|
+
`<!-- Last updated: ${now} -->`,
|
|
1333
|
+
"",
|
|
1334
|
+
"## Discovered Resources",
|
|
1335
|
+
"",
|
|
1336
|
+
"### Prompts",
|
|
1337
|
+
renderTable(["File", "Line", "Content Preview", "Platform ID", "Version", "Status"], this.prompts.map((p) => [
|
|
1338
|
+
p.file,
|
|
1339
|
+
String(p.line),
|
|
1340
|
+
p.preview,
|
|
1341
|
+
p.platformId ?? "",
|
|
1342
|
+
p.version ?? "",
|
|
1343
|
+
p.status
|
|
1344
|
+
])),
|
|
1345
|
+
"",
|
|
1346
|
+
"### Datasets",
|
|
1347
|
+
renderTable(["File", "Name", "Items", "Platform ID", "Prompt ID", "Status"], this.datasets.map((d) => [
|
|
1348
|
+
d.file,
|
|
1349
|
+
d.name,
|
|
1350
|
+
String(d.items),
|
|
1351
|
+
d.platformId ?? "",
|
|
1352
|
+
d.promptId ?? "",
|
|
1353
|
+
d.status
|
|
1354
|
+
])),
|
|
1355
|
+
"",
|
|
1356
|
+
"### Evaluations",
|
|
1357
|
+
renderTable(["Name", "Criteria Count", "Platform ID", "Prompt ID", "Status"], this.evaluations.map((e) => [
|
|
1358
|
+
e.name,
|
|
1359
|
+
String(e.criteriaCount),
|
|
1360
|
+
e.platformId ?? "",
|
|
1361
|
+
e.promptId ?? "",
|
|
1362
|
+
e.status
|
|
1363
|
+
])),
|
|
1364
|
+
"",
|
|
1365
|
+
"## Integration Status",
|
|
1366
|
+
renderTable(["Framework", "Package", "Status"], this.integrations.map((i) => [
|
|
1367
|
+
i.framework,
|
|
1368
|
+
i.pkg,
|
|
1369
|
+
i.status
|
|
1370
|
+
])),
|
|
1371
|
+
""
|
|
1372
|
+
];
|
|
1373
|
+
return sections.join(`
|
|
1374
|
+
`);
|
|
1375
|
+
}
|
|
1376
|
+
parse(content) {
|
|
1377
|
+
const lines = content.split(`
|
|
1378
|
+
`);
|
|
1379
|
+
let currentSection = null;
|
|
1380
|
+
let sectionLines = [];
|
|
1381
|
+
const flushSection = () => {
|
|
1382
|
+
if (!currentSection || sectionLines.length === 0)
|
|
1383
|
+
return;
|
|
1384
|
+
const rows = parseTableRows(sectionLines);
|
|
1385
|
+
const dataRows = rows.slice(1);
|
|
1386
|
+
switch (currentSection) {
|
|
1387
|
+
case "prompts":
|
|
1388
|
+
for (const row of dataRows) {
|
|
1389
|
+
const [file, lineStr, preview, pid, ver, st] = row;
|
|
1390
|
+
if (file !== undefined && lineStr !== undefined && preview !== undefined) {
|
|
1391
|
+
this.prompts.push({
|
|
1392
|
+
file,
|
|
1393
|
+
line: parseInt(lineStr, 10) || 0,
|
|
1394
|
+
preview,
|
|
1395
|
+
platformId: emptyToUndefined(pid),
|
|
1396
|
+
version: emptyToUndefined(ver),
|
|
1397
|
+
status: emptyToUndefined(st) ?? "discovered"
|
|
1398
|
+
});
|
|
1399
|
+
}
|
|
1400
|
+
}
|
|
1401
|
+
break;
|
|
1402
|
+
case "datasets":
|
|
1403
|
+
for (const row of dataRows) {
|
|
1404
|
+
const [file, name, itemsStr, pid, prid, st] = row;
|
|
1405
|
+
if (file !== undefined && name !== undefined && itemsStr !== undefined) {
|
|
1406
|
+
this.datasets.push({
|
|
1407
|
+
file,
|
|
1408
|
+
name,
|
|
1409
|
+
items: parseInt(itemsStr, 10) || 0,
|
|
1410
|
+
platformId: emptyToUndefined(pid),
|
|
1411
|
+
promptId: emptyToUndefined(prid),
|
|
1412
|
+
status: emptyToUndefined(st) ?? "discovered"
|
|
1413
|
+
});
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
break;
|
|
1417
|
+
case "evaluations":
|
|
1418
|
+
for (const row of dataRows) {
|
|
1419
|
+
const [name, ccStr, pid, prid, st] = row;
|
|
1420
|
+
if (name !== undefined && ccStr !== undefined) {
|
|
1421
|
+
this.evaluations.push({
|
|
1422
|
+
name,
|
|
1423
|
+
criteriaCount: parseInt(ccStr, 10) || 0,
|
|
1424
|
+
platformId: emptyToUndefined(pid),
|
|
1425
|
+
promptId: emptyToUndefined(prid),
|
|
1426
|
+
status: emptyToUndefined(st) ?? "discovered"
|
|
1427
|
+
});
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
break;
|
|
1431
|
+
case "integrations":
|
|
1432
|
+
for (const row of dataRows) {
|
|
1433
|
+
const [framework, pkg, status] = row;
|
|
1434
|
+
if (framework !== undefined && pkg !== undefined && status !== undefined) {
|
|
1435
|
+
this.integrations.push({ framework, pkg, status });
|
|
1436
|
+
}
|
|
1437
|
+
}
|
|
1438
|
+
break;
|
|
1439
|
+
}
|
|
1440
|
+
sectionLines = [];
|
|
1441
|
+
};
|
|
1442
|
+
for (const line of lines) {
|
|
1443
|
+
const trimmed = line.trim();
|
|
1444
|
+
if (trimmed === "### Prompts") {
|
|
1445
|
+
flushSection();
|
|
1446
|
+
currentSection = "prompts";
|
|
1447
|
+
continue;
|
|
1448
|
+
}
|
|
1449
|
+
if (trimmed === "### Datasets") {
|
|
1450
|
+
flushSection();
|
|
1451
|
+
currentSection = "datasets";
|
|
1452
|
+
continue;
|
|
1453
|
+
}
|
|
1454
|
+
if (trimmed === "### Evaluations") {
|
|
1455
|
+
flushSection();
|
|
1456
|
+
currentSection = "evaluations";
|
|
1457
|
+
continue;
|
|
1458
|
+
}
|
|
1459
|
+
if (trimmed === "## Integration Status") {
|
|
1460
|
+
flushSection();
|
|
1461
|
+
currentSection = "integrations";
|
|
1462
|
+
continue;
|
|
1463
|
+
}
|
|
1464
|
+
if (trimmed.startsWith("## ") && currentSection !== null && trimmed !== "## Integration Status") {
|
|
1465
|
+
flushSection();
|
|
1466
|
+
currentSection = null;
|
|
1467
|
+
continue;
|
|
1468
|
+
}
|
|
1469
|
+
if (currentSection && trimmed.startsWith("|")) {
|
|
1470
|
+
sectionLines.push(trimmed);
|
|
1471
|
+
}
|
|
1472
|
+
}
|
|
1473
|
+
flushSection();
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
// src/commands/onboarding.ts
|
|
1478
|
+
import chalk2 from "chalk";
|
|
1479
|
+
import { resolve as resolve2 } from "path";
|
|
1480
|
+
|
|
1481
|
+
// src/lib/explorer.ts
|
|
1482
|
+
import { readdirSync, readFileSync as readFileSync3, statSync } from "fs";
|
|
1483
|
+
import { join as join3, relative, extname, basename } from "path";
|
|
1484
|
+
var TEMPLATE_VAR_PATTERN = /\{\{[a-zA-Z_][a-zA-Z0-9_]*\}\}/;
|
|
1485
|
+
var PROMPT_NAME_PATTERN = /(?:const|let|var|export)\s+\w*(?:prompt|system|agent|instruction|template)\w*\s*=/i;
|
|
1486
|
+
var SCHEMA_PATTERN = /["']?(?:inputSchema|outputSchema|properties|required)["']?\s*[=:]/;
|
|
1487
|
+
var ZOD_SCHEMA_PATTERN = /z\.object\s*\(/;
|
|
1488
|
+
var PYDANTIC_PATTERN = /class\s+\w+\s*\(\s*(?:BaseModel|BaseSettings)\s*\)/;
|
|
1489
|
+
var MARKER_START_PATTERN = /MutagenT:START\s+(\w+)(?:\s+id=(\S+))?/;
|
|
1490
|
+
function walkDir(dir, extensions, excludeDirs, maxDepth, currentDepth = 0) {
|
|
1491
|
+
if (currentDepth >= maxDepth)
|
|
1492
|
+
return [];
|
|
1493
|
+
const files = [];
|
|
1494
|
+
let entries;
|
|
1495
|
+
try {
|
|
1496
|
+
entries = readdirSync(dir);
|
|
1497
|
+
} catch {
|
|
1498
|
+
return files;
|
|
1499
|
+
}
|
|
1500
|
+
for (const entry of entries) {
|
|
1501
|
+
const fullPath = join3(dir, entry);
|
|
1502
|
+
let stat;
|
|
1503
|
+
try {
|
|
1504
|
+
stat = statSync(fullPath);
|
|
1505
|
+
} catch {
|
|
1506
|
+
continue;
|
|
1507
|
+
}
|
|
1508
|
+
if (stat.isDirectory()) {
|
|
1509
|
+
if (excludeDirs.includes(entry))
|
|
1510
|
+
continue;
|
|
1511
|
+
files.push(...walkDir(fullPath, extensions, excludeDirs, maxDepth, currentDepth + 1));
|
|
1512
|
+
} else if (stat.isFile()) {
|
|
1513
|
+
const ext = extname(entry).toLowerCase();
|
|
1514
|
+
if (extensions.includes(ext)) {
|
|
1515
|
+
files.push(fullPath);
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1518
|
+
}
|
|
1519
|
+
return files;
|
|
1520
|
+
}
|
|
1521
|
+
function scanForPrompts(filePath, relativePath) {
|
|
1522
|
+
const results = [];
|
|
1523
|
+
let content;
|
|
1524
|
+
try {
|
|
1525
|
+
content = readFileSync3(filePath, "utf-8");
|
|
1526
|
+
} catch {
|
|
1527
|
+
return results;
|
|
1528
|
+
}
|
|
1529
|
+
const lines = content.split(`
|
|
1530
|
+
`);
|
|
1531
|
+
for (let i = 0;i < lines.length; i++) {
|
|
1532
|
+
const line = lines[i];
|
|
1533
|
+
if (!line)
|
|
1534
|
+
continue;
|
|
1535
|
+
if (TEMPLATE_VAR_PATTERN.test(line)) {
|
|
1536
|
+
const preview = line.trim().substring(0, 80);
|
|
1537
|
+
results.push({
|
|
1538
|
+
file: relativePath,
|
|
1539
|
+
line: i + 1,
|
|
1540
|
+
preview,
|
|
1541
|
+
reason: "template-variable"
|
|
1542
|
+
});
|
|
1543
|
+
}
|
|
1544
|
+
if (PROMPT_NAME_PATTERN.test(line)) {
|
|
1545
|
+
const preview = line.trim().substring(0, 80);
|
|
1546
|
+
if (!results.some((r) => r.file === relativePath && r.line === i + 1)) {
|
|
1547
|
+
results.push({
|
|
1548
|
+
file: relativePath,
|
|
1549
|
+
line: i + 1,
|
|
1550
|
+
preview,
|
|
1551
|
+
reason: "prompt-constant"
|
|
1552
|
+
});
|
|
1553
|
+
}
|
|
1554
|
+
}
|
|
1555
|
+
if (ZOD_SCHEMA_PATTERN.test(line)) {
|
|
1556
|
+
const preview = line.trim().substring(0, 80);
|
|
1557
|
+
if (!results.some((r) => r.file === relativePath && r.line === i + 1)) {
|
|
1558
|
+
results.push({
|
|
1559
|
+
file: relativePath,
|
|
1560
|
+
line: i + 1,
|
|
1561
|
+
preview,
|
|
1562
|
+
reason: "zod-schema"
|
|
1563
|
+
});
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
if (PYDANTIC_PATTERN.test(line)) {
|
|
1567
|
+
const preview = line.trim().substring(0, 80);
|
|
1568
|
+
if (!results.some((r) => r.file === relativePath && r.line === i + 1)) {
|
|
1569
|
+
results.push({
|
|
1570
|
+
file: relativePath,
|
|
1571
|
+
line: i + 1,
|
|
1572
|
+
preview,
|
|
1573
|
+
reason: "pydantic-model"
|
|
1574
|
+
});
|
|
1575
|
+
}
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
return results;
|
|
1579
|
+
}
|
|
1580
|
+
function scanForMarkers(filePath, relativePath) {
|
|
1581
|
+
const results = [];
|
|
1582
|
+
let content;
|
|
1583
|
+
try {
|
|
1584
|
+
content = readFileSync3(filePath, "utf-8");
|
|
1585
|
+
} catch {
|
|
1586
|
+
return results;
|
|
1587
|
+
}
|
|
1588
|
+
const lines = content.split(`
|
|
1589
|
+
`);
|
|
1590
|
+
for (let i = 0;i < lines.length; i++) {
|
|
1591
|
+
const line = lines[i];
|
|
1592
|
+
if (!line)
|
|
1593
|
+
continue;
|
|
1594
|
+
const match = MARKER_START_PATTERN.exec(line);
|
|
1595
|
+
if (match) {
|
|
1596
|
+
const rawType = match[1]?.toLowerCase();
|
|
1597
|
+
const platformId = match[2];
|
|
1598
|
+
const validTypes = new Set(["prompt", "dataset", "evaluation"]);
|
|
1599
|
+
if (rawType && validTypes.has(rawType)) {
|
|
1600
|
+
results.push({
|
|
1601
|
+
file: relativePath,
|
|
1602
|
+
line: i + 1,
|
|
1603
|
+
type: rawType,
|
|
1604
|
+
platformId
|
|
1605
|
+
});
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
return results;
|
|
1610
|
+
}
|
|
1611
|
+
function scanJsonForSchemas(filePath, relativePath) {
|
|
1612
|
+
const results = [];
|
|
1613
|
+
const ext = extname(filePath).toLowerCase();
|
|
1614
|
+
if (ext !== ".json")
|
|
1615
|
+
return results;
|
|
1616
|
+
let content;
|
|
1617
|
+
try {
|
|
1618
|
+
content = readFileSync3(filePath, "utf-8");
|
|
1619
|
+
} catch {
|
|
1620
|
+
return results;
|
|
1621
|
+
}
|
|
1622
|
+
if (!SCHEMA_PATTERN.test(content))
|
|
1623
|
+
return results;
|
|
1624
|
+
const preview = `JSON with schema definitions: ${basename(filePath)}`;
|
|
1625
|
+
results.push({
|
|
1626
|
+
file: relativePath,
|
|
1627
|
+
line: 1,
|
|
1628
|
+
preview: preview.substring(0, 80),
|
|
1629
|
+
reason: "json-schema"
|
|
1630
|
+
});
|
|
1631
|
+
return results;
|
|
1632
|
+
}
|
|
1633
|
+
function scanForDatasets(dir, rootPath, excludeDirs, maxDepth) {
|
|
1634
|
+
const results = [];
|
|
1635
|
+
const dataExtensions = [".json", ".jsonl", ".csv"];
|
|
1636
|
+
const files = walkDir(dir, dataExtensions, excludeDirs, maxDepth);
|
|
1637
|
+
for (const filePath of files) {
|
|
1638
|
+
const relativePath = relative(rootPath, filePath);
|
|
1639
|
+
const ext = extname(filePath).toLowerCase();
|
|
1640
|
+
const name = basename(filePath, ext);
|
|
1641
|
+
let content;
|
|
1642
|
+
try {
|
|
1643
|
+
content = readFileSync3(filePath, "utf-8");
|
|
1644
|
+
} catch {
|
|
1645
|
+
continue;
|
|
1646
|
+
}
|
|
1647
|
+
if (ext === ".csv") {
|
|
1648
|
+
const lines = content.trim().split(`
|
|
1649
|
+
`);
|
|
1650
|
+
if (lines.length > 1) {
|
|
1651
|
+
const header = lines[0]?.toLowerCase() ?? "";
|
|
1652
|
+
if (header.includes("input") || header.includes("output") || header.includes("expected")) {
|
|
1653
|
+
results.push({ file: relativePath, name, items: lines.length - 1 });
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
} else if (ext === ".jsonl") {
|
|
1657
|
+
const lines = content.trim().split(`
|
|
1658
|
+
`).filter((l) => l.trim().length > 0);
|
|
1659
|
+
if (lines.length > 0) {
|
|
1660
|
+
try {
|
|
1661
|
+
const first = JSON.parse(lines[0] ?? "{}");
|
|
1662
|
+
if ("input" in first || "expectedOutput" in first) {
|
|
1663
|
+
results.push({ file: relativePath, name, items: lines.length });
|
|
1664
|
+
}
|
|
1665
|
+
} catch {}
|
|
1666
|
+
}
|
|
1667
|
+
} else if (ext === ".json") {
|
|
1668
|
+
try {
|
|
1669
|
+
const parsed = JSON.parse(content);
|
|
1670
|
+
if (Array.isArray(parsed) && parsed.length > 0) {
|
|
1671
|
+
const first = parsed[0];
|
|
1672
|
+
if ("input" in first || "expectedOutput" in first) {
|
|
1673
|
+
results.push({ file: relativePath, name, items: parsed.length });
|
|
1674
|
+
}
|
|
1675
|
+
}
|
|
1676
|
+
} catch {}
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
return results;
|
|
1680
|
+
}
|
|
1681
|
+
function exploreCodebase(options) {
|
|
1682
|
+
const rootPath = options.path;
|
|
1683
|
+
const prompts = [];
|
|
1684
|
+
const datasets = [];
|
|
1685
|
+
const markers = [];
|
|
1686
|
+
const sourceFiles = walkDir(rootPath, options.extensions, options.excludeDirs, options.depth);
|
|
1687
|
+
for (const filePath of sourceFiles) {
|
|
1688
|
+
const relativePath = relative(rootPath, filePath);
|
|
1689
|
+
markers.push(...scanForMarkers(filePath, relativePath));
|
|
1690
|
+
if (!options.markersOnly) {
|
|
1691
|
+
prompts.push(...scanForPrompts(filePath, relativePath));
|
|
1692
|
+
}
|
|
1693
|
+
}
|
|
1694
|
+
if (!options.markersOnly) {
|
|
1695
|
+
const jsonFiles = walkDir(rootPath, [".json"], options.excludeDirs, options.depth);
|
|
1696
|
+
for (const filePath of jsonFiles) {
|
|
1697
|
+
const relativePath = relative(rootPath, filePath);
|
|
1698
|
+
prompts.push(...scanJsonForSchemas(filePath, relativePath));
|
|
1699
|
+
}
|
|
1700
|
+
datasets.push(...scanForDatasets(rootPath, rootPath, options.excludeDirs, options.depth));
|
|
1701
|
+
}
|
|
1702
|
+
return { prompts, datasets, markers };
|
|
1703
|
+
}
|
|
1704
|
+
function parseExtensions(includeGlob) {
|
|
1705
|
+
const braceMatch = /\{([^}]+)\}/.exec(includeGlob);
|
|
1706
|
+
if (braceMatch?.[1]) {
|
|
1707
|
+
return braceMatch[1].split(",").map((ext) => `.${ext.trim()}`);
|
|
1708
|
+
}
|
|
1709
|
+
const extMatch = /\*\.(\w+)$/.exec(includeGlob);
|
|
1710
|
+
if (extMatch?.[1]) {
|
|
1711
|
+
return [`.${extMatch[1]}`];
|
|
1712
|
+
}
|
|
1713
|
+
return [".ts", ".js", ".py", ".tsx", ".jsx"];
|
|
1714
|
+
}
|
|
1715
|
+
function parseExcludeDirs(excludePattern) {
|
|
1716
|
+
return excludePattern.split(",").map((d) => d.trim()).filter(Boolean);
|
|
1717
|
+
}
|
|
1718
|
+
|
|
1719
|
+
// src/commands/onboarding.ts
|
|
1720
|
+
async function runPostOnboarding() {
|
|
1721
|
+
const inquirer = (await import("inquirer")).default;
|
|
1722
|
+
console.log("");
|
|
1723
|
+
console.log(chalk2.bold.cyan(" What would you like to do next?"));
|
|
1724
|
+
console.log("");
|
|
1725
|
+
const { path: selectedPath } = await inquirer.prompt([{
|
|
1726
|
+
type: "list",
|
|
1727
|
+
name: "path",
|
|
1728
|
+
message: "Choose your path:",
|
|
1729
|
+
choices: [
|
|
1730
|
+
{
|
|
1731
|
+
name: `${chalk2.green("A")} Guided Integration — connect your AI framework`,
|
|
1732
|
+
value: "integrate"
|
|
1733
|
+
},
|
|
1734
|
+
{
|
|
1735
|
+
name: `${chalk2.green("B")} Quick Optimization — upload a prompt and optimize it`,
|
|
1736
|
+
value: "optimize"
|
|
1737
|
+
},
|
|
1738
|
+
{
|
|
1739
|
+
name: `${chalk2.green("C")} Exit — explore on your own`,
|
|
1740
|
+
value: "exit"
|
|
1741
|
+
}
|
|
1742
|
+
]
|
|
1743
|
+
}]);
|
|
1744
|
+
if (selectedPath === "exit") {
|
|
1745
|
+
console.log("");
|
|
1746
|
+
console.log(chalk2.dim(" You can run `mutagent --help` anytime to see available commands."));
|
|
1747
|
+
console.log(chalk2.dim(" Try `mutagent explore` to scan your codebase for prompts."));
|
|
1748
|
+
console.log("");
|
|
1749
|
+
return;
|
|
1750
|
+
}
|
|
1751
|
+
console.log("");
|
|
1752
|
+
console.log(chalk2.cyan(" Scanning your codebase..."));
|
|
1753
|
+
console.log("");
|
|
1754
|
+
const scanPath = resolve2(".");
|
|
1755
|
+
const result = exploreCodebase({
|
|
1756
|
+
path: scanPath,
|
|
1757
|
+
depth: 10,
|
|
1758
|
+
extensions: parseExtensions("**/*.{ts,js,py,tsx,jsx}"),
|
|
1759
|
+
excludeDirs: parseExcludeDirs("node_modules,dist,.git,build,.next,__pycache__,venv,.venv"),
|
|
1760
|
+
markersOnly: false
|
|
1761
|
+
});
|
|
1762
|
+
const totalFindings = result.prompts.length + result.datasets.length;
|
|
1763
|
+
if (totalFindings > 0) {
|
|
1764
|
+
console.log(chalk2.green(` Found ${String(result.prompts.length)} prompt(s) and ${String(result.datasets.length)} dataset(s)`));
|
|
1765
|
+
} else {
|
|
1766
|
+
console.log(chalk2.dim(" No prompts or datasets detected yet."));
|
|
1767
|
+
}
|
|
1768
|
+
console.log("");
|
|
1769
|
+
if (selectedPath === "integrate") {
|
|
1770
|
+
await runIntegrationPath();
|
|
1771
|
+
} else {
|
|
1772
|
+
runOptimizationPath(result.prompts.length, result.datasets.length);
|
|
1773
|
+
}
|
|
1774
|
+
}
|
|
1775
|
+
async function runIntegrationPath() {
|
|
1776
|
+
const inquirerMod = (await import("inquirer")).default;
|
|
1777
|
+
const { framework } = await inquirerMod.prompt([{
|
|
1778
|
+
type: "list",
|
|
1779
|
+
name: "framework",
|
|
1780
|
+
message: "Which AI framework are you using?",
|
|
1781
|
+
choices: [
|
|
1782
|
+
{ name: "Mastra", value: "mastra" },
|
|
1783
|
+
{ name: "LangChain", value: "langchain" },
|
|
1784
|
+
{ name: "LangGraph", value: "langgraph" },
|
|
1785
|
+
{ name: "Vercel AI SDK", value: "vercel-ai" },
|
|
1786
|
+
{ name: "Claude Code", value: "claude-code" },
|
|
1787
|
+
{ name: "Other / Generic", value: "generic" }
|
|
1788
|
+
]
|
|
1789
|
+
}]);
|
|
1790
|
+
console.log("");
|
|
1791
|
+
console.log(chalk2.bold(" Next steps:"));
|
|
1792
|
+
console.log("");
|
|
1793
|
+
console.log(` 1. ${chalk2.green(`mutagent integrate ${framework}`)}`);
|
|
1794
|
+
console.log(` Get the integration guide for ${framework}`);
|
|
1795
|
+
console.log("");
|
|
1796
|
+
console.log(` 2. ${chalk2.green(`mutagent integrate ${framework} --verify`)}`);
|
|
1797
|
+
console.log(" Verify your integration is working");
|
|
1798
|
+
console.log("");
|
|
1799
|
+
console.log(` 3. ${chalk2.green("mutagent explore")}`);
|
|
1800
|
+
console.log(" Re-scan to confirm markers are in place");
|
|
1801
|
+
console.log("");
|
|
1802
|
+
}
|
|
1803
|
+
function runOptimizationPath(promptCount, datasetCount) {
|
|
1804
|
+
console.log(chalk2.bold(" Quick Optimization Workflow:"));
|
|
1805
|
+
console.log("");
|
|
1806
|
+
let step = 1;
|
|
1807
|
+
if (promptCount === 0) {
|
|
1808
|
+
console.log(` ${String(step)}. ${chalk2.green('mutagent prompts create --name "my-prompt" --raw "Your prompt with {{variables}}"')}`);
|
|
1809
|
+
console.log(" Create a prompt with template variables");
|
|
1810
|
+
console.log("");
|
|
1811
|
+
step++;
|
|
1812
|
+
} else {
|
|
1813
|
+
console.log(chalk2.dim(` ${chalk2.green("✓")} Prompts detected in codebase (${String(promptCount)} found)`));
|
|
1814
|
+
console.log(chalk2.dim(` Upload one: mutagent prompts create --name "my-prompt" --raw-file <path>`));
|
|
1815
|
+
console.log("");
|
|
1816
|
+
}
|
|
1817
|
+
if (datasetCount === 0) {
|
|
1818
|
+
console.log(` ${String(step)}. ${chalk2.green("mutagent prompts dataset add <prompt-id> --file dataset.json")}`);
|
|
1819
|
+
console.log(" Upload a dataset with input/output pairs");
|
|
1820
|
+
console.log("");
|
|
1821
|
+
step++;
|
|
1822
|
+
} else {
|
|
1823
|
+
console.log(chalk2.dim(` ${chalk2.green("✓")} Datasets detected in codebase (${String(datasetCount)} found)`));
|
|
1824
|
+
console.log(chalk2.dim(` Upload one: mutagent prompts dataset add <prompt-id> --file <path>`));
|
|
1825
|
+
console.log("");
|
|
1826
|
+
}
|
|
1827
|
+
console.log(` ${String(step)}. ${chalk2.green('mutagent prompts evaluation create <prompt-id> --name "My Eval" --file criteria.json')}`);
|
|
1828
|
+
console.log(" Define evaluation criteria");
|
|
1829
|
+
console.log("");
|
|
1830
|
+
step++;
|
|
1831
|
+
console.log(` ${String(step)}. ${chalk2.green("mutagent prompts optimize start <prompt-id> --dataset <dataset-id>")}`);
|
|
1832
|
+
console.log(" Start the optimization loop");
|
|
1833
|
+
console.log("");
|
|
1834
|
+
console.log(chalk2.dim(" Tip: Use `mutagent prompts list` to see your prompt IDs."));
|
|
1835
|
+
console.log("");
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1168
1838
|
// src/commands/auth.ts
|
|
1169
1839
|
function createAuthCommand() {
|
|
1170
1840
|
const auth = new Command("auth").description("Authenticate with MutagenT platform").addHelpText("after", `
|
|
1171
1841
|
Examples:
|
|
1172
|
-
${
|
|
1173
|
-
${
|
|
1174
|
-
${
|
|
1175
|
-
${
|
|
1176
|
-
${
|
|
1842
|
+
${chalk3.dim("$")} mutagent auth login
|
|
1843
|
+
${chalk3.dim("$")} mutagent auth login --browser
|
|
1844
|
+
${chalk3.dim("$")} mutagent auth login --api-key <key>
|
|
1845
|
+
${chalk3.dim("$")} mutagent auth status
|
|
1846
|
+
${chalk3.dim("$")} mutagent auth logout
|
|
1177
1847
|
`);
|
|
1178
1848
|
auth.command("login").description("Authenticate and store API key").option("--api-key <key>", "API key (non-interactive)").option("--browser", "Force browser-based authentication").option("--non-interactive", "Disable interactive prompts (auto-selects browser auth)").option("--endpoint <url>", "API endpoint", "https://api.mutagent.io").addHelpText("after", `
|
|
1179
1849
|
Examples:
|
|
1180
|
-
${
|
|
1181
|
-
${
|
|
1182
|
-
${
|
|
1183
|
-
${
|
|
1850
|
+
${chalk3.dim("$")} mutagent auth login ${chalk3.dim("# Interactive (choose method)")}
|
|
1851
|
+
${chalk3.dim("$")} mutagent auth login --browser ${chalk3.dim("# Browser OAuth flow")}
|
|
1852
|
+
${chalk3.dim("$")} mutagent auth login --api-key mg_live_xxx ${chalk3.dim("# Direct API key (CI/CD)")}
|
|
1853
|
+
${chalk3.dim("$")} mutagent auth login --non-interactive ${chalk3.dim("# Auto browser flow (AI agents)")}
|
|
1184
1854
|
|
|
1185
1855
|
Environment Variables:
|
|
1186
1856
|
MUTAGENT_API_KEY API key (skips login entirely)
|
|
@@ -1191,6 +1861,7 @@ Environment Variables:
|
|
|
1191
1861
|
const isJson = getJsonFlag(auth);
|
|
1192
1862
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
1193
1863
|
try {
|
|
1864
|
+
const wasFirstLogin = !hasCredentials();
|
|
1194
1865
|
let apiKey = options.apiKey ?? process.env.MUTAGENT_API_KEY;
|
|
1195
1866
|
let endpoint = process.env.MUTAGENT_ENDPOINT ?? options.endpoint;
|
|
1196
1867
|
let workspaceName;
|
|
@@ -1228,7 +1899,7 @@ Environment Variables:
|
|
|
1228
1899
|
}
|
|
1229
1900
|
if (!isNonInteractive && !hasCredentials()) {
|
|
1230
1901
|
console.log(`
|
|
1231
|
-
` +
|
|
1902
|
+
` + chalk3.bold.cyan(" Welcome to MutagenT CLI!") + `
|
|
1232
1903
|
`);
|
|
1233
1904
|
console.log(` No credentials found. Please authenticate to continue.
|
|
1234
1905
|
`);
|
|
@@ -1368,6 +2039,9 @@ Environment Variables:
|
|
|
1368
2039
|
output.info("Workspace: " + selectedWsName);
|
|
1369
2040
|
output.info("Endpoint: " + endpoint);
|
|
1370
2041
|
}
|
|
2042
|
+
if (wasFirstLogin && process.stdin.isTTY && !isJson) {
|
|
2043
|
+
await runPostOnboarding();
|
|
2044
|
+
}
|
|
1371
2045
|
} catch (error) {
|
|
1372
2046
|
if (error instanceof MutagentError) {
|
|
1373
2047
|
output.error(error.message);
|
|
@@ -1378,8 +2052,8 @@ Environment Variables:
|
|
|
1378
2052
|
});
|
|
1379
2053
|
auth.command("status").description("Check authentication status").addHelpText("after", `
|
|
1380
2054
|
Examples:
|
|
1381
|
-
${
|
|
1382
|
-
${
|
|
2055
|
+
${chalk3.dim("$")} mutagent auth status
|
|
2056
|
+
${chalk3.dim("$")} mutagent auth status --json
|
|
1383
2057
|
`).action(async () => {
|
|
1384
2058
|
const isJson = getJsonFlag(auth);
|
|
1385
2059
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -1396,13 +2070,26 @@ Examples:
|
|
|
1396
2070
|
process.exit(0);
|
|
1397
2071
|
}
|
|
1398
2072
|
const isValid = await validateApiKey(apiKey, endpoint);
|
|
2073
|
+
const cwd = process.cwd();
|
|
2074
|
+
const hasOnboarding = existsSync3(join4(cwd, ".mutagentrc.json"));
|
|
2075
|
+
const hasContextFile = existsSync3(join4(cwd, ".mutagent", "mutation-context.md"));
|
|
2076
|
+
let contextSummary;
|
|
2077
|
+
if (hasContextFile) {
|
|
2078
|
+
try {
|
|
2079
|
+
const ctx = MutationContext.load(cwd);
|
|
2080
|
+
contextSummary = ctx.getSummary();
|
|
2081
|
+
} catch {}
|
|
2082
|
+
}
|
|
1399
2083
|
if (isJson) {
|
|
1400
2084
|
const statusResult = {
|
|
1401
2085
|
authenticated: isValid,
|
|
1402
2086
|
endpoint,
|
|
1403
2087
|
keyPrefix: `${apiKey.slice(0, 8)}...`,
|
|
1404
2088
|
defaultWorkspace: config.defaultWorkspace,
|
|
1405
|
-
defaultOrganization: config.defaultOrganization
|
|
2089
|
+
defaultOrganization: config.defaultOrganization,
|
|
2090
|
+
onboarding: hasOnboarding,
|
|
2091
|
+
contextFile: hasContextFile,
|
|
2092
|
+
context: contextSummary ?? null
|
|
1406
2093
|
};
|
|
1407
2094
|
if (!isValid) {
|
|
1408
2095
|
statusResult.message = "Authentication invalid — run `mutagent auth login --browser`";
|
|
@@ -1415,7 +2102,7 @@ Examples:
|
|
|
1415
2102
|
output.output(statusResult);
|
|
1416
2103
|
} else {
|
|
1417
2104
|
if (isValid) {
|
|
1418
|
-
output.success("Authenticated");
|
|
2105
|
+
output.success("Authentication: Authenticated");
|
|
1419
2106
|
output.info(`Endpoint: ${endpoint}`);
|
|
1420
2107
|
output.info(`Key: ${apiKey.slice(0, 8)}...`);
|
|
1421
2108
|
if (config.defaultWorkspace) {
|
|
@@ -1425,18 +2112,38 @@ Examples:
|
|
|
1425
2112
|
output.info(`Organization: ${config.defaultOrganization}`);
|
|
1426
2113
|
}
|
|
1427
2114
|
} else {
|
|
1428
|
-
output.error("Authentication
|
|
2115
|
+
output.error("Authentication: Invalid — run `mutagent auth login --browser`");
|
|
1429
2116
|
console.error("");
|
|
1430
2117
|
console.error("Authentication required. Options:");
|
|
1431
2118
|
console.error(" Interactive: mutagent auth login --browser");
|
|
1432
2119
|
console.error(" Non-interactive: export MUTAGENT_API_KEY=<your-key>");
|
|
1433
2120
|
console.error(" CI/CD: mutagent auth login --api-key <key>");
|
|
1434
2121
|
}
|
|
2122
|
+
if (hasOnboarding) {
|
|
2123
|
+
output.success("Onboarding: Complete (.mutagentrc.json found)");
|
|
2124
|
+
} else {
|
|
2125
|
+
output.warn("Onboarding: Not initialized (run `mutagent init`)");
|
|
2126
|
+
}
|
|
2127
|
+
if (hasContextFile && contextSummary) {
|
|
2128
|
+
const parts = [
|
|
2129
|
+
`${String(contextSummary.prompts)} prompts`,
|
|
2130
|
+
`${String(contextSummary.datasets)} datasets`,
|
|
2131
|
+
`${String(contextSummary.evaluations)} evaluations`
|
|
2132
|
+
];
|
|
2133
|
+
output.success(`Context File: .mutagent/mutation-context.md (${parts.join(", ")})`);
|
|
2134
|
+
} else if (hasContextFile) {
|
|
2135
|
+
output.success("Context File: .mutagent/mutation-context.md");
|
|
2136
|
+
} else {
|
|
2137
|
+
output.warn("Context File: Not found (run `mutagent explore`)");
|
|
2138
|
+
}
|
|
2139
|
+
if (contextSummary && contextSummary.integrated.length > 0) {
|
|
2140
|
+
output.success(`Integration: ${contextSummary.integrated.join(", ")} (active)`);
|
|
2141
|
+
}
|
|
1435
2142
|
}
|
|
1436
2143
|
});
|
|
1437
2144
|
auth.command("logout").description("Clear stored credentials").addHelpText("after", `
|
|
1438
2145
|
Examples:
|
|
1439
|
-
${
|
|
2146
|
+
${chalk3.dim("$")} mutagent auth logout
|
|
1440
2147
|
`).action(() => {
|
|
1441
2148
|
const isJson = getJsonFlag(auth);
|
|
1442
2149
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -1447,17 +2154,20 @@ Examples:
|
|
|
1447
2154
|
}
|
|
1448
2155
|
|
|
1449
2156
|
// src/commands/login.ts
|
|
2157
|
+
init_config();
|
|
2158
|
+
init_sdk_client();
|
|
1450
2159
|
import { Command as Command2 } from "commander";
|
|
1451
2160
|
import inquirer2 from "inquirer";
|
|
1452
|
-
import
|
|
2161
|
+
import chalk4 from "chalk";
|
|
1453
2162
|
import ora2 from "ora";
|
|
2163
|
+
init_errors();
|
|
1454
2164
|
function createLoginCommand() {
|
|
1455
2165
|
const login = new Command2("login").description("Login to MutagenT platform").option("--api-key <key>", "API key (non-interactive)").option("--browser", "Force browser-based authentication").option("--non-interactive", "Disable interactive prompts (auto-selects browser auth)").option("--endpoint <url>", "API endpoint", "https://api.mutagent.io").addHelpText("after", `
|
|
1456
2166
|
Examples:
|
|
1457
|
-
${
|
|
1458
|
-
${
|
|
1459
|
-
${
|
|
1460
|
-
${
|
|
2167
|
+
${chalk4.dim("$")} mutagent login
|
|
2168
|
+
${chalk4.dim("$")} mutagent login --browser
|
|
2169
|
+
${chalk4.dim("$")} mutagent login --api-key <key>
|
|
2170
|
+
${chalk4.dim("$")} mutagent login --non-interactive ${chalk4.dim("# Auto browser flow (AI agents)")}
|
|
1461
2171
|
`).action(async (options) => {
|
|
1462
2172
|
const isJson = getJsonFlag(login);
|
|
1463
2173
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -1483,7 +2193,7 @@ Examples:
|
|
|
1483
2193
|
}
|
|
1484
2194
|
if (!isNonInteractive && !hasCredentials()) {
|
|
1485
2195
|
console.log(`
|
|
1486
|
-
` +
|
|
2196
|
+
` + chalk4.bold.cyan(" Welcome to MutagenT CLI!") + `
|
|
1487
2197
|
`);
|
|
1488
2198
|
console.log(` No credentials found. Please authenticate to continue.
|
|
1489
2199
|
`);
|
|
@@ -1590,9 +2300,11 @@ Examples:
|
|
|
1590
2300
|
}
|
|
1591
2301
|
|
|
1592
2302
|
// src/commands/prompts.ts
|
|
2303
|
+
init_sdk_client();
|
|
1593
2304
|
import { Command as Command3 } from "commander";
|
|
1594
|
-
import
|
|
1595
|
-
import { readFileSync as
|
|
2305
|
+
import chalk7 from "chalk";
|
|
2306
|
+
import { readFileSync as readFileSync4, existsSync as existsSync4 } from "fs";
|
|
2307
|
+
init_errors();
|
|
1596
2308
|
|
|
1597
2309
|
// src/lib/ui-links.ts
|
|
1598
2310
|
function getAppBaseUrl() {
|
|
@@ -1610,36 +2322,435 @@ function promptEvaluationsLink(promptId) {
|
|
|
1610
2322
|
function datasetLink(promptId, datasetId) {
|
|
1611
2323
|
return `${getAppBaseUrl()}/prompts/dashboard?prompt=${String(promptId)}&tab=datasets&dataset=${String(datasetId)}`;
|
|
1612
2324
|
}
|
|
1613
|
-
function evaluationLink(promptId, evalId) {
|
|
1614
|
-
return `${getAppBaseUrl()}/prompts/dashboard?prompt=${String(promptId)}&tab=evaluations&eval=${String(evalId)}`;
|
|
2325
|
+
function evaluationLink(promptId, evalId) {
|
|
2326
|
+
return `${getAppBaseUrl()}/prompts/dashboard?prompt=${String(promptId)}&tab=evaluations&eval=${String(evalId)}`;
|
|
2327
|
+
}
|
|
2328
|
+
function traceLink(id) {
|
|
2329
|
+
return `${getAppBaseUrl()}/traces/${id}`;
|
|
2330
|
+
}
|
|
2331
|
+
function optimizerLink(jobId) {
|
|
2332
|
+
const base = `${getAppBaseUrl()}/prompts/optimizer`;
|
|
2333
|
+
return jobId ? `${base}?job=${jobId}` : base;
|
|
2334
|
+
}
|
|
2335
|
+
function playgroundLink() {
|
|
2336
|
+
return `${getAppBaseUrl()}/prompts/playground`;
|
|
2337
|
+
}
|
|
2338
|
+
function providerSettingsLink() {
|
|
2339
|
+
return `${getAppBaseUrl()}/settings/providers`;
|
|
2340
|
+
}
|
|
2341
|
+
function workspaceLink(id) {
|
|
2342
|
+
return `${getAppBaseUrl()}/settings/workspaces/${String(id)}`;
|
|
2343
|
+
}
|
|
2344
|
+
function workspaceLinks(id) {
|
|
2345
|
+
return {
|
|
2346
|
+
settings: workspaceLink(id),
|
|
2347
|
+
api: `/api/workspaces/${String(id)}`
|
|
2348
|
+
};
|
|
2349
|
+
}
|
|
2350
|
+
function providerLink(id) {
|
|
2351
|
+
return `${getAppBaseUrl()}/settings/providers/${String(id)}`;
|
|
2352
|
+
}
|
|
2353
|
+
function providerLinks(id) {
|
|
2354
|
+
return {
|
|
2355
|
+
settings: providerLink(id),
|
|
2356
|
+
api: `/api/providers/${String(id)}`
|
|
2357
|
+
};
|
|
2358
|
+
}
|
|
2359
|
+
function agentLink(id) {
|
|
2360
|
+
return `${getAppBaseUrl()}/agents/${String(id)}`;
|
|
2361
|
+
}
|
|
2362
|
+
function agentLinks(id) {
|
|
2363
|
+
return {
|
|
2364
|
+
dashboard: agentLink(id),
|
|
2365
|
+
api: `/api/agents/${String(id)}`
|
|
2366
|
+
};
|
|
2367
|
+
}
|
|
2368
|
+
function promptLinks(promptId) {
|
|
2369
|
+
return {
|
|
2370
|
+
dashboard: promptLink(promptId),
|
|
2371
|
+
api: `/api/prompts/${String(promptId)}`,
|
|
2372
|
+
datasets: promptDatasetsLink(promptId),
|
|
2373
|
+
evaluations: promptEvaluationsLink(promptId)
|
|
2374
|
+
};
|
|
2375
|
+
}
|
|
2376
|
+
function datasetLinks(promptId, datasetId) {
|
|
2377
|
+
return {
|
|
2378
|
+
dashboard: datasetLink(promptId, datasetId),
|
|
2379
|
+
api: `/api/prompts/${String(promptId)}/datasets/${String(datasetId)}`
|
|
2380
|
+
};
|
|
2381
|
+
}
|
|
2382
|
+
function evaluationLinks(promptId, evalId) {
|
|
2383
|
+
return {
|
|
2384
|
+
dashboard: evaluationLink(promptId, evalId),
|
|
2385
|
+
api: `/api/prompts/${String(promptId)}/evaluations/${String(evalId)}`
|
|
2386
|
+
};
|
|
2387
|
+
}
|
|
2388
|
+
function formatCreationHints(hint) {
|
|
2389
|
+
const lines = [
|
|
2390
|
+
"",
|
|
2391
|
+
` -> View in dashboard: ${hint.dashboardUrl}`,
|
|
2392
|
+
` -> API: GET ${hint.apiPath}`,
|
|
2393
|
+
"",
|
|
2394
|
+
" Tip: Copy the link above to open in your browser or share with your team."
|
|
2395
|
+
];
|
|
2396
|
+
return lines.join(`
|
|
2397
|
+
`);
|
|
2398
|
+
}
|
|
2399
|
+
|
|
2400
|
+
// src/lib/schema-helpers.ts
|
|
2401
|
+
var SUPPORTED_SCHEMA_TYPES = ["string", "number", "boolean", "array", "object"];
|
|
2402
|
+
function isValidJsonSchema(schema) {
|
|
2403
|
+
if (schema === null || schema === undefined)
|
|
2404
|
+
return false;
|
|
2405
|
+
if (typeof schema !== "object")
|
|
2406
|
+
return false;
|
|
2407
|
+
if (Array.isArray(schema))
|
|
2408
|
+
return false;
|
|
2409
|
+
const obj = schema;
|
|
2410
|
+
if (!("type" in obj) || typeof obj.type !== "string")
|
|
2411
|
+
return false;
|
|
2412
|
+
if (obj.type === "object") {
|
|
2413
|
+
if (!("properties" in obj) || typeof obj.properties !== "object" || obj.properties === null) {
|
|
2414
|
+
return false;
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
return true;
|
|
2418
|
+
}
|
|
2419
|
+
function buildSchemaFromVariables(variables) {
|
|
2420
|
+
const properties = {};
|
|
2421
|
+
for (const variable of variables) {
|
|
2422
|
+
const prop = { type: variable.type };
|
|
2423
|
+
if (variable.description) {
|
|
2424
|
+
prop.description = variable.description;
|
|
2425
|
+
}
|
|
2426
|
+
properties[variable.name] = prop;
|
|
2427
|
+
}
|
|
2428
|
+
return {
|
|
2429
|
+
type: "object",
|
|
2430
|
+
properties
|
|
2431
|
+
};
|
|
2432
|
+
}
|
|
2433
|
+
function formatSchemaWarning(fieldName) {
|
|
2434
|
+
return [
|
|
2435
|
+
`${fieldName} doesn't appear to be a valid JSON Schema.`,
|
|
2436
|
+
"",
|
|
2437
|
+
" Expected format:",
|
|
2438
|
+
" {",
|
|
2439
|
+
' "type": "object",',
|
|
2440
|
+
' "properties": {',
|
|
2441
|
+
' "variable_name": { "type": "string", "description": "Description here" }',
|
|
2442
|
+
" }",
|
|
2443
|
+
" }",
|
|
2444
|
+
"",
|
|
2445
|
+
" You can also provide a schema file: mutagent prompts create --file schema.json"
|
|
2446
|
+
].join(`
|
|
2447
|
+
`);
|
|
2448
|
+
}
|
|
2449
|
+
|
|
2450
|
+
// src/lib/eval-creator.ts
|
|
2451
|
+
import chalk5 from "chalk";
|
|
2452
|
+
var RUBRIC_TEMPLATES = {
|
|
2453
|
+
"Exact Match": "Score 1.0 if the output exactly matches the expected value, 0.0 otherwise.",
|
|
2454
|
+
"Semantic Similarity": "Score 0.0-1.0 based on semantic similarity to the expected output. 1.0 = identical meaning, 0.5 = partially related, 0.0 = unrelated.",
|
|
2455
|
+
"Contains Key Info": "Score 1.0 if all key information from expected output is present, 0.5 if partially present, 0.0 if missing.",
|
|
2456
|
+
"Format Compliance": "Score 1.0 if the output follows the expected format/structure, 0.5 for minor deviations, 0.0 for wrong format.",
|
|
2457
|
+
"Factual Accuracy": "Score 1.0 if all facts are correct, 0.5 if mostly correct with minor errors, 0.0 if incorrect."
|
|
2458
|
+
};
|
|
2459
|
+
var EVAL_TYPE_DESCRIPTIONS = {
|
|
2460
|
+
accuracy: "Check if outputs match expected results (exact or semantic)",
|
|
2461
|
+
quality: "Assess output quality, clarity, and helpfulness",
|
|
2462
|
+
custom: "Define your own evaluation criteria"
|
|
2463
|
+
};
|
|
2464
|
+
function extractSchemaFields(schema, prefix) {
|
|
2465
|
+
if (!schema || typeof schema !== "object")
|
|
2466
|
+
return [];
|
|
2467
|
+
const obj = schema;
|
|
2468
|
+
const properties = obj.properties;
|
|
2469
|
+
if (properties) {
|
|
2470
|
+
return Object.keys(properties).map((key) => `${prefix}.${key}`);
|
|
2471
|
+
}
|
|
2472
|
+
const keys = Object.keys(obj).filter((k) => k !== "type" && k !== "required" && k !== "description");
|
|
2473
|
+
if (keys.length > 0) {
|
|
2474
|
+
return keys.map((key) => `${prefix}.${key}`);
|
|
2475
|
+
}
|
|
2476
|
+
return [];
|
|
2477
|
+
}
|
|
2478
|
+
async function runGuidedEvalCreator(promptId) {
|
|
2479
|
+
const inquirer3 = (await import("inquirer")).default;
|
|
2480
|
+
console.log("");
|
|
2481
|
+
console.log(chalk5.cyan(" Fetching prompt details..."));
|
|
2482
|
+
let inputFields = [];
|
|
2483
|
+
let outputFields = [];
|
|
2484
|
+
try {
|
|
2485
|
+
const { getSDKClient: getSDKClient2 } = await Promise.resolve().then(() => (init_sdk_client(), exports_sdk_client));
|
|
2486
|
+
const client = getSDKClient2();
|
|
2487
|
+
const prompt = await client.getPrompt(promptId);
|
|
2488
|
+
inputFields = extractSchemaFields(prompt.inputSchema, "input");
|
|
2489
|
+
outputFields = extractSchemaFields(prompt.outputSchema, "output");
|
|
2490
|
+
if (inputFields.length > 0 || outputFields.length > 0) {
|
|
2491
|
+
console.log(chalk5.green(` Found ${String(inputFields.length)} input field(s) and ${String(outputFields.length)} output field(s)`));
|
|
2492
|
+
} else {
|
|
2493
|
+
console.log(chalk5.yellow(" No schema fields detected. You can still define criteria manually."));
|
|
2494
|
+
}
|
|
2495
|
+
} catch {
|
|
2496
|
+
console.log(chalk5.yellow(" Could not fetch prompt. You can still define criteria manually."));
|
|
2497
|
+
}
|
|
2498
|
+
console.log("");
|
|
2499
|
+
const { evalType } = await inquirer3.prompt([{
|
|
2500
|
+
type: "list",
|
|
2501
|
+
name: "evalType",
|
|
2502
|
+
message: "What type of evaluation?",
|
|
2503
|
+
choices: Object.entries(EVAL_TYPE_DESCRIPTIONS).map(([value, description]) => ({
|
|
2504
|
+
name: `${value} — ${description}`,
|
|
2505
|
+
value
|
|
2506
|
+
}))
|
|
2507
|
+
}]);
|
|
2508
|
+
const defaultName = evalType.charAt(0).toUpperCase() + evalType.slice(1) + " Evaluation";
|
|
2509
|
+
const { evalName } = await inquirer3.prompt([{
|
|
2510
|
+
type: "input",
|
|
2511
|
+
name: "evalName",
|
|
2512
|
+
message: "Evaluation name:",
|
|
2513
|
+
default: defaultName,
|
|
2514
|
+
validate: (input) => {
|
|
2515
|
+
if (!input.trim())
|
|
2516
|
+
return "Evaluation name is required";
|
|
2517
|
+
return true;
|
|
2518
|
+
}
|
|
2519
|
+
}]);
|
|
2520
|
+
const allFields = [...inputFields, ...outputFields];
|
|
2521
|
+
const criteria = [];
|
|
2522
|
+
let addMore = true;
|
|
2523
|
+
while (addMore) {
|
|
2524
|
+
console.log("");
|
|
2525
|
+
console.log(chalk5.bold(` Criterion #${String(criteria.length + 1)}`));
|
|
2526
|
+
const { criterionName } = await inquirer3.prompt([{
|
|
2527
|
+
type: "input",
|
|
2528
|
+
name: "criterionName",
|
|
2529
|
+
message: "Name this criterion:",
|
|
2530
|
+
validate: (input) => {
|
|
2531
|
+
if (!input.trim())
|
|
2532
|
+
return "Criterion name is required";
|
|
2533
|
+
return true;
|
|
2534
|
+
}
|
|
2535
|
+
}]);
|
|
2536
|
+
let targetField;
|
|
2537
|
+
if (allFields.length > 0) {
|
|
2538
|
+
const fieldChoices = [
|
|
2539
|
+
...allFields.map((f) => ({ name: f, value: f })),
|
|
2540
|
+
{ name: "(custom field name)", value: "__custom__" }
|
|
2541
|
+
];
|
|
2542
|
+
const { field } = await inquirer3.prompt([{
|
|
2543
|
+
type: "list",
|
|
2544
|
+
name: "field",
|
|
2545
|
+
message: "Target field:",
|
|
2546
|
+
choices: fieldChoices
|
|
2547
|
+
}]);
|
|
2548
|
+
if (field === "__custom__") {
|
|
2549
|
+
const { customField } = await inquirer3.prompt([{
|
|
2550
|
+
type: "input",
|
|
2551
|
+
name: "customField",
|
|
2552
|
+
message: "Custom field name (e.g., output.summary):"
|
|
2553
|
+
}]);
|
|
2554
|
+
targetField = customField.trim();
|
|
2555
|
+
} else {
|
|
2556
|
+
targetField = field;
|
|
2557
|
+
}
|
|
2558
|
+
} else {
|
|
2559
|
+
const { customField } = await inquirer3.prompt([{
|
|
2560
|
+
type: "input",
|
|
2561
|
+
name: "customField",
|
|
2562
|
+
message: "Target field (e.g., output.result):",
|
|
2563
|
+
default: "output"
|
|
2564
|
+
}]);
|
|
2565
|
+
targetField = customField.trim();
|
|
2566
|
+
}
|
|
2567
|
+
const rubricChoices = [
|
|
2568
|
+
...Object.entries(RUBRIC_TEMPLATES).map(([name, value]) => ({
|
|
2569
|
+
name: `${name} — ${chalk5.dim(value.substring(0, 50))}...`,
|
|
2570
|
+
value
|
|
2571
|
+
})),
|
|
2572
|
+
{ name: "(write custom rubric)", value: "__custom__" }
|
|
2573
|
+
];
|
|
2574
|
+
const { rubric } = await inquirer3.prompt([{
|
|
2575
|
+
type: "list",
|
|
2576
|
+
name: "rubric",
|
|
2577
|
+
message: "Scoring rubric:",
|
|
2578
|
+
choices: rubricChoices
|
|
2579
|
+
}]);
|
|
2580
|
+
let scoringRubric;
|
|
2581
|
+
if (rubric === "__custom__") {
|
|
2582
|
+
const { customRubric } = await inquirer3.prompt([{
|
|
2583
|
+
type: "input",
|
|
2584
|
+
name: "customRubric",
|
|
2585
|
+
message: "Describe the scoring rubric:",
|
|
2586
|
+
validate: (input) => {
|
|
2587
|
+
if (!input.trim())
|
|
2588
|
+
return "Rubric is required";
|
|
2589
|
+
return true;
|
|
2590
|
+
}
|
|
2591
|
+
}]);
|
|
2592
|
+
scoringRubric = customRubric.trim();
|
|
2593
|
+
} else {
|
|
2594
|
+
scoringRubric = rubric;
|
|
2595
|
+
}
|
|
2596
|
+
criteria.push({
|
|
2597
|
+
name: criterionName.trim(),
|
|
2598
|
+
targetField,
|
|
2599
|
+
scoringRubric,
|
|
2600
|
+
weight: 1
|
|
2601
|
+
});
|
|
2602
|
+
const { continueAdding } = await inquirer3.prompt([{
|
|
2603
|
+
type: "confirm",
|
|
2604
|
+
name: "continueAdding",
|
|
2605
|
+
message: "Add another criterion?",
|
|
2606
|
+
default: false
|
|
2607
|
+
}]);
|
|
2608
|
+
addMore = continueAdding;
|
|
2609
|
+
}
|
|
2610
|
+
if (criteria.length === 0) {
|
|
2611
|
+
console.log(chalk5.yellow(`
|
|
2612
|
+
No criteria defined. Aborting guided eval creation.`));
|
|
2613
|
+
return;
|
|
2614
|
+
}
|
|
2615
|
+
console.log("");
|
|
2616
|
+
console.log(chalk5.bold(" Evaluation Summary:"));
|
|
2617
|
+
console.log(` Name: ${chalk5.green(evalName)}`);
|
|
2618
|
+
console.log(` Type: ${evalType}`);
|
|
2619
|
+
console.log(` Criteria: ${String(criteria.length)}`);
|
|
2620
|
+
for (const c of criteria) {
|
|
2621
|
+
console.log(` - ${chalk5.cyan(c.name)} → ${c.targetField}`);
|
|
2622
|
+
}
|
|
2623
|
+
console.log("");
|
|
2624
|
+
const { confirmed } = await inquirer3.prompt([{
|
|
2625
|
+
type: "confirm",
|
|
2626
|
+
name: "confirmed",
|
|
2627
|
+
message: "Create this evaluation?",
|
|
2628
|
+
default: true
|
|
2629
|
+
}]);
|
|
2630
|
+
if (!confirmed) {
|
|
2631
|
+
console.log(chalk5.dim(`
|
|
2632
|
+
Cancelled.`));
|
|
2633
|
+
return;
|
|
2634
|
+
}
|
|
2635
|
+
return {
|
|
2636
|
+
name: evalName.trim(),
|
|
2637
|
+
description: `${evalType} evaluation with ${String(criteria.length)} criteria`,
|
|
2638
|
+
evalConfig: { criteria }
|
|
2639
|
+
};
|
|
2640
|
+
}
|
|
2641
|
+
|
|
2642
|
+
// src/lib/scorecard.ts
|
|
2643
|
+
import chalk6 from "chalk";
|
|
2644
|
+
function truncateText(text, maxLen) {
|
|
2645
|
+
if (text.length <= maxLen)
|
|
2646
|
+
return text;
|
|
2647
|
+
return text.substring(0, maxLen - 3) + "...";
|
|
1615
2648
|
}
|
|
1616
|
-
function
|
|
1617
|
-
|
|
2649
|
+
function formatScoreChange(before, after) {
|
|
2650
|
+
if (before === undefined || after === undefined)
|
|
2651
|
+
return "";
|
|
2652
|
+
const diff = after - before;
|
|
2653
|
+
const pct = before > 0 ? Math.round(diff / before * 100) : 0;
|
|
2654
|
+
if (diff > 0)
|
|
2655
|
+
return chalk6.green(` (+${String(pct)}%)`);
|
|
2656
|
+
if (diff < 0)
|
|
2657
|
+
return chalk6.red(` (${String(pct)}%)`);
|
|
2658
|
+
return chalk6.dim(" (no change)");
|
|
1618
2659
|
}
|
|
1619
|
-
function
|
|
1620
|
-
|
|
1621
|
-
|
|
2660
|
+
function formatScore(score) {
|
|
2661
|
+
if (score === undefined)
|
|
2662
|
+
return chalk6.dim("N/A");
|
|
2663
|
+
return score >= 0.8 ? chalk6.green(score.toFixed(2)) : score >= 0.5 ? chalk6.yellow(score.toFixed(2)) : chalk6.red(score.toFixed(2));
|
|
1622
2664
|
}
|
|
1623
|
-
function
|
|
1624
|
-
|
|
2665
|
+
function renderScorecard(data) {
|
|
2666
|
+
const { job, prompt } = data;
|
|
2667
|
+
const boxWidth = 55;
|
|
2668
|
+
const topBorder = `┌${"─".repeat(boxWidth)}┐`;
|
|
2669
|
+
const bottomBorder = `└${"─".repeat(boxWidth)}┘`;
|
|
2670
|
+
const separator = `│ ${"─".repeat(boxWidth - 2)} │`;
|
|
2671
|
+
const pad = (text, width) => {
|
|
2672
|
+
const stripped = text.replace(/\u001B\[[0-9;]*m/g, "");
|
|
2673
|
+
const remaining = width - stripped.length;
|
|
2674
|
+
return remaining > 0 ? text + " ".repeat(remaining) : text;
|
|
2675
|
+
};
|
|
2676
|
+
const line = (text) => `│ ${pad(text, boxWidth - 2)} │`;
|
|
2677
|
+
const originalScore = data.originalScore;
|
|
2678
|
+
const bestScore = data.bestScore;
|
|
2679
|
+
const iterations = data.iterationsCompleted ?? job.config?.maxIterations ?? 0;
|
|
2680
|
+
const originalText = data.originalPrompt?.systemPrompt ?? data.originalPrompt?.rawPrompt ?? "(original prompt)";
|
|
2681
|
+
const optimizedText = prompt.systemPrompt ?? prompt.rawPrompt ?? prompt.humanPrompt ?? "(optimized prompt)";
|
|
2682
|
+
console.log("");
|
|
2683
|
+
console.log(topBorder);
|
|
2684
|
+
console.log(line(chalk6.bold("Optimization Results")));
|
|
2685
|
+
console.log(separator);
|
|
2686
|
+
console.log(line(chalk6.dim("BEFORE")));
|
|
2687
|
+
console.log(line(` Prompt: ${chalk6.dim(truncateText(originalText, 38))}`));
|
|
2688
|
+
console.log(line(` Score: ${formatScore(originalScore)}`));
|
|
2689
|
+
console.log(line(""));
|
|
2690
|
+
console.log(line(chalk6.bold("AFTER")));
|
|
2691
|
+
console.log(line(` Prompt: ${chalk6.cyan(truncateText(optimizedText, 38))}`));
|
|
2692
|
+
console.log(line(` Score: ${formatScore(bestScore)}${formatScoreChange(originalScore, bestScore)}`));
|
|
2693
|
+
console.log(separator);
|
|
2694
|
+
const statusStr = job.status === "completed" ? chalk6.green("completed") : chalk6.yellow(job.status);
|
|
2695
|
+
console.log(line(`Status: ${statusStr} | Iterations: ${String(iterations)}`));
|
|
2696
|
+
if (job.config?.model) {
|
|
2697
|
+
console.log(line(`Model: ${chalk6.dim(job.config.model)}`));
|
|
2698
|
+
}
|
|
2699
|
+
if (data.scoreProgression && data.scoreProgression.length > 0) {
|
|
2700
|
+
console.log(line(""));
|
|
2701
|
+
console.log(line(chalk6.dim("Score Progression:")));
|
|
2702
|
+
const progression = data.scoreProgression.map((s, i) => `#${String(i + 1)}: ${s.toFixed(2)}`).join(" ");
|
|
2703
|
+
if (progression.length > boxWidth - 4) {
|
|
2704
|
+
const mid = Math.ceil(data.scoreProgression.length / 2);
|
|
2705
|
+
const line1 = data.scoreProgression.slice(0, mid).map((s, i) => `#${String(i + 1)}: ${s.toFixed(2)}`).join(" ");
|
|
2706
|
+
const line2 = data.scoreProgression.slice(mid).map((s, i) => `#${String(i + mid + 1)}: ${s.toFixed(2)}`).join(" ");
|
|
2707
|
+
console.log(line(chalk6.dim(line1)));
|
|
2708
|
+
console.log(line(chalk6.dim(line2)));
|
|
2709
|
+
} else {
|
|
2710
|
+
console.log(line(chalk6.dim(progression)));
|
|
2711
|
+
}
|
|
2712
|
+
}
|
|
2713
|
+
console.log(separator);
|
|
2714
|
+
console.log(line(`Dashboard: ${chalk6.underline(optimizerLink(job.jobId))}`));
|
|
2715
|
+
console.log(bottomBorder);
|
|
2716
|
+
console.log("");
|
|
1625
2717
|
}
|
|
1626
|
-
function
|
|
1627
|
-
|
|
2718
|
+
async function promptScorecardAction() {
|
|
2719
|
+
const inquirer3 = (await import("inquirer")).default;
|
|
2720
|
+
const { action } = await inquirer3.prompt([{
|
|
2721
|
+
type: "list",
|
|
2722
|
+
name: "action",
|
|
2723
|
+
message: "What would you like to do?",
|
|
2724
|
+
choices: [
|
|
2725
|
+
{ name: chalk6.green("Apply") + " — Update prompt to optimized version", value: "apply" },
|
|
2726
|
+
{ name: chalk6.red("Reject") + " — Keep original prompt", value: "reject" },
|
|
2727
|
+
{ name: chalk6.dim("View Details") + " — Show full diff", value: "details" }
|
|
2728
|
+
]
|
|
2729
|
+
}]);
|
|
2730
|
+
return { action };
|
|
1628
2731
|
}
|
|
1629
|
-
function
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
2732
|
+
function showPromptDiff(original, optimized) {
|
|
2733
|
+
console.log("");
|
|
2734
|
+
console.log(chalk6.bold(" Prompt Diff:"));
|
|
2735
|
+
console.log("");
|
|
2736
|
+
console.log(chalk6.red(" - " + (original ?? "(empty)")));
|
|
2737
|
+
console.log(chalk6.green(" + " + (optimized ?? "(empty)")));
|
|
2738
|
+
console.log("");
|
|
1635
2739
|
}
|
|
1636
2740
|
|
|
1637
2741
|
// src/commands/prompts.ts
|
|
2742
|
+
function updateMutationContext(updater) {
|
|
2743
|
+
try {
|
|
2744
|
+
const ctx = MutationContext.load();
|
|
2745
|
+
updater(ctx);
|
|
2746
|
+
ctx.save();
|
|
2747
|
+
} catch {}
|
|
2748
|
+
}
|
|
1638
2749
|
var PREREQUISITES_TEXT = `
|
|
1639
|
-
${
|
|
1640
|
-
1. Evaluation criteria defined ${
|
|
1641
|
-
2. Dataset uploaded ${
|
|
1642
|
-
${
|
|
2750
|
+
${chalk7.yellow("Prerequisites:")}
|
|
2751
|
+
1. Evaluation criteria defined ${chalk7.dim("(via dashboard or evaluation create)")}
|
|
2752
|
+
2. Dataset uploaded ${chalk7.dim("mutagent prompts dataset list <prompt-id>")}
|
|
2753
|
+
${chalk7.dim("Note: LLM provider config is only required when the server uses external providers (USE_EXT_PROVIDERS=true)")}`;
|
|
1643
2754
|
function parseValidationErrors(error) {
|
|
1644
2755
|
if (error instanceof ApiError) {
|
|
1645
2756
|
try {
|
|
@@ -1659,6 +2770,117 @@ function parseValidationErrors(error) {
|
|
|
1659
2770
|
}
|
|
1660
2771
|
return ["An unknown error occurred"];
|
|
1661
2772
|
}
|
|
2773
|
+
function isSchemaEmpty(schema) {
|
|
2774
|
+
if (schema === undefined || schema === null)
|
|
2775
|
+
return true;
|
|
2776
|
+
if (typeof schema !== "object")
|
|
2777
|
+
return false;
|
|
2778
|
+
const obj = schema;
|
|
2779
|
+
if (Object.keys(obj).length === 0)
|
|
2780
|
+
return true;
|
|
2781
|
+
return false;
|
|
2782
|
+
}
|
|
2783
|
+
function warnMissingSchemas(data, output) {
|
|
2784
|
+
if (isSchemaEmpty(data.inputSchema)) {
|
|
2785
|
+
output.warn("No inputSchema provided. Optimization requires inputSchema with defined variables.");
|
|
2786
|
+
} else if (!isValidJsonSchema(data.inputSchema)) {
|
|
2787
|
+
output.warn(formatSchemaWarning("inputSchema"));
|
|
2788
|
+
}
|
|
2789
|
+
if (isSchemaEmpty(data.outputSchema)) {
|
|
2790
|
+
output.warn("No outputSchema provided. This may limit optimization effectiveness.");
|
|
2791
|
+
} else if (!isValidJsonSchema(data.outputSchema)) {
|
|
2792
|
+
output.warn(formatSchemaWarning("outputSchema"));
|
|
2793
|
+
}
|
|
2794
|
+
}
|
|
2795
|
+
async function collectSchemaInteractively() {
|
|
2796
|
+
const inquirer3 = (await import("inquirer")).default;
|
|
2797
|
+
const variables = [];
|
|
2798
|
+
let addMore = true;
|
|
2799
|
+
while (addMore) {
|
|
2800
|
+
const answers = await inquirer3.prompt([
|
|
2801
|
+
{
|
|
2802
|
+
type: "input",
|
|
2803
|
+
name: "name",
|
|
2804
|
+
message: "Variable name:",
|
|
2805
|
+
validate: (input) => {
|
|
2806
|
+
if (!input.trim())
|
|
2807
|
+
return "Variable name is required";
|
|
2808
|
+
if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(input.trim())) {
|
|
2809
|
+
return "Variable name must start with a letter/underscore and contain only alphanumeric/underscore";
|
|
2810
|
+
}
|
|
2811
|
+
return true;
|
|
2812
|
+
}
|
|
2813
|
+
},
|
|
2814
|
+
{
|
|
2815
|
+
type: "list",
|
|
2816
|
+
name: "type",
|
|
2817
|
+
message: "Variable type:",
|
|
2818
|
+
choices: [...SUPPORTED_SCHEMA_TYPES]
|
|
2819
|
+
},
|
|
2820
|
+
{
|
|
2821
|
+
type: "input",
|
|
2822
|
+
name: "description",
|
|
2823
|
+
message: "Description (optional):"
|
|
2824
|
+
}
|
|
2825
|
+
]);
|
|
2826
|
+
variables.push({
|
|
2827
|
+
name: answers.name.trim(),
|
|
2828
|
+
type: answers.type,
|
|
2829
|
+
description: answers.description.trim() || undefined
|
|
2830
|
+
});
|
|
2831
|
+
const continueAnswer = await inquirer3.prompt([{
|
|
2832
|
+
type: "confirm",
|
|
2833
|
+
name: "addAnother",
|
|
2834
|
+
message: "Add another variable?",
|
|
2835
|
+
default: false
|
|
2836
|
+
}]);
|
|
2837
|
+
addMore = continueAnswer.addAnother;
|
|
2838
|
+
}
|
|
2839
|
+
return buildSchemaFromVariables(variables);
|
|
2840
|
+
}
|
|
2841
|
+
async function promptForInputSchema() {
|
|
2842
|
+
const inquirer3 = (await import("inquirer")).default;
|
|
2843
|
+
const answer = await inquirer3.prompt([{
|
|
2844
|
+
type: "list",
|
|
2845
|
+
name: "choice",
|
|
2846
|
+
message: "Would you like to define input variables for this prompt?",
|
|
2847
|
+
choices: [
|
|
2848
|
+
{ name: "Yes, define variables interactively", value: "interactive" },
|
|
2849
|
+
{ name: "Provide a JSON Schema file path", value: "file" },
|
|
2850
|
+
{ name: "Skip (not recommended for optimization)", value: "skip" }
|
|
2851
|
+
]
|
|
2852
|
+
}]);
|
|
2853
|
+
if (answer.choice === "interactive") {
|
|
2854
|
+
return collectSchemaInteractively();
|
|
2855
|
+
}
|
|
2856
|
+
if (answer.choice === "file") {
|
|
2857
|
+
const fileAnswer = await inquirer3.prompt([{
|
|
2858
|
+
type: "input",
|
|
2859
|
+
name: "path",
|
|
2860
|
+
message: "JSON Schema file path:",
|
|
2861
|
+
validate: (input) => {
|
|
2862
|
+
if (!input.trim())
|
|
2863
|
+
return "File path is required";
|
|
2864
|
+
if (!existsSync4(input.trim()))
|
|
2865
|
+
return `File not found: ${input.trim()}`;
|
|
2866
|
+
return true;
|
|
2867
|
+
}
|
|
2868
|
+
}]);
|
|
2869
|
+
const content = readFileSync4(fileAnswer.path.trim(), "utf-8");
|
|
2870
|
+
try {
|
|
2871
|
+
const parsed = JSON.parse(content);
|
|
2872
|
+
if (!isValidJsonSchema(parsed)) {
|
|
2873
|
+
console.log(chalk7.yellow(`
|
|
2874
|
+
Warning: ${formatSchemaWarning("inputSchema")}
|
|
2875
|
+
`));
|
|
2876
|
+
}
|
|
2877
|
+
return parsed;
|
|
2878
|
+
} catch {
|
|
2879
|
+
throw new MutagentError("INVALID_JSON", `Failed to parse JSON Schema from ${fileAnswer.path.trim()}`, "Ensure the file contains valid JSON");
|
|
2880
|
+
}
|
|
2881
|
+
}
|
|
2882
|
+
return;
|
|
2883
|
+
}
|
|
1662
2884
|
function warnSingleBraceVariables(content, output) {
|
|
1663
2885
|
const singleBracePattern = /(?<!\{)\{([a-zA-Z_][a-zA-Z0-9_]*)\}(?!\})/g;
|
|
1664
2886
|
const matches = [...content.matchAll(singleBracePattern)];
|
|
@@ -1706,12 +2928,12 @@ function parseDatasetFile(rawContent, filePath) {
|
|
|
1706
2928
|
function createPromptsCommand() {
|
|
1707
2929
|
const prompts = new Command3("prompts").description("Manage prompts, datasets, evaluations, and optimizations").addHelpText("after", `
|
|
1708
2930
|
Examples:
|
|
1709
|
-
${
|
|
1710
|
-
${
|
|
1711
|
-
${
|
|
1712
|
-
${
|
|
1713
|
-
${
|
|
1714
|
-
${
|
|
2931
|
+
${chalk7.dim("$")} mutagent prompts list
|
|
2932
|
+
${chalk7.dim("$")} mutagent prompts get <prompt-id>
|
|
2933
|
+
${chalk7.dim("$")} mutagent prompts create --file prompt.json
|
|
2934
|
+
${chalk7.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
2935
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
2936
|
+
${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
|
|
1715
2937
|
|
|
1716
2938
|
Subcommands:
|
|
1717
2939
|
list, get, create, update, delete
|
|
@@ -1721,11 +2943,11 @@ Subcommands:
|
|
|
1721
2943
|
`);
|
|
1722
2944
|
prompts.command("list").description("List all prompts").option("-l, --limit <n>", "Limit results", "50").addHelpText("after", `
|
|
1723
2945
|
Examples:
|
|
1724
|
-
${
|
|
1725
|
-
${
|
|
1726
|
-
${
|
|
2946
|
+
${chalk7.dim("$")} mutagent prompts list
|
|
2947
|
+
${chalk7.dim("$")} mutagent prompts list --limit 10
|
|
2948
|
+
${chalk7.dim("$")} mutagent prompts list --json
|
|
1727
2949
|
|
|
1728
|
-
${
|
|
2950
|
+
${chalk7.dim("Tip: Use --json for machine-readable output (AI agents, CI pipelines).")}
|
|
1729
2951
|
`).action(async (options) => {
|
|
1730
2952
|
const isJson = getJsonFlag(prompts);
|
|
1731
2953
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -1756,11 +2978,11 @@ ${chalk4.dim("Tip: Use --json for machine-readable output (AI agents, CI pipelin
|
|
|
1756
2978
|
});
|
|
1757
2979
|
prompts.command("get").description("Get prompt details with nested data").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("--with-datasets", "Include datasets").option("--with-evals", "Include evaluations").addHelpText("after", `
|
|
1758
2980
|
Examples:
|
|
1759
|
-
${
|
|
1760
|
-
${
|
|
1761
|
-
${
|
|
2981
|
+
${chalk7.dim("$")} mutagent prompts get <id>
|
|
2982
|
+
${chalk7.dim("$")} mutagent prompts get <id> --with-datasets --with-evals
|
|
2983
|
+
${chalk7.dim("$")} mutagent prompts get <id> --json
|
|
1762
2984
|
|
|
1763
|
-
${
|
|
2985
|
+
${chalk7.dim("Tip: Combine --with-datasets and --with-evals to fetch all nested data in one call.")}
|
|
1764
2986
|
`).action(async (id, options) => {
|
|
1765
2987
|
const isJson = getJsonFlag(prompts);
|
|
1766
2988
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -1784,35 +3006,46 @@ ${chalk4.dim("Tip: Combine --with-datasets and --with-evals to fetch all nested
|
|
|
1784
3006
|
handleError(error, isJson);
|
|
1785
3007
|
}
|
|
1786
3008
|
});
|
|
1787
|
-
prompts.command("create").description("Create a new prompt").option("-f, --file <path>", "Create from JSON file").option("--raw-file <path>", "Create from plain text file (used as rawPrompt)").option("-n, --name <name>", "Prompt name").option("-c, --content <content>", "Prompt content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).addHelpText("after", `
|
|
3009
|
+
prompts.command("create").description("Create a new prompt").option("-d, --data <json>", "Prompt as JSON string (recommended — curl-style inline)").option("-f, --file <path>", "Create from JSON file").option("--raw-file <path>", "Create from plain text file (used as rawPrompt)").option("-n, --name <name>", "Prompt name").option("-c, --content <content>", "Prompt content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).option("--output-schema <json>", "Output schema as JSON string (required for optimization)").addHelpText("after", `
|
|
1788
3010
|
Examples:
|
|
1789
|
-
${
|
|
1790
|
-
${
|
|
1791
|
-
${
|
|
1792
|
-
${
|
|
1793
|
-
${chalk4.dim("$")} mutagent prompts create --name "from-file" --raw-file prompt.txt
|
|
3011
|
+
${chalk7.dim("$")} mutagent prompts create -d '{"name":"summarizer","systemPrompt":"Summarize","humanPrompt":"{{text}}","outputSchema":{"type":"object","properties":{"summary":{"type":"string"}}}}'
|
|
3012
|
+
${chalk7.dim("$")} mutagent prompts create --name "my-prompt" --system "You are helpful" --human "Hello" --output-schema '{"type":"object","properties":{"result":{"type":"string"}}}'
|
|
3013
|
+
${chalk7.dim("$")} mutagent prompts create --file prompt.json
|
|
3014
|
+
${chalk7.dim("$")} mutagent prompts create --name "raw-prompt" --raw "Summarize: {{text}}"
|
|
1794
3015
|
|
|
1795
|
-
Prompt Input Methods (pick one):
|
|
3016
|
+
Prompt Input Methods (pick one, priority order):
|
|
3017
|
+
-d, --data Inline JSON object (recommended for CI/scripts)
|
|
3018
|
+
--file Load from JSON file (full prompt object)
|
|
3019
|
+
--raw-file Load plain text file as raw prompt
|
|
1796
3020
|
--system/--human Structured system + user message pair
|
|
1797
3021
|
--raw Single raw prompt text with {{variables}}
|
|
1798
|
-
--raw-file Load plain text file as raw prompt
|
|
1799
3022
|
--messages Full messages array as JSON
|
|
1800
|
-
--file Load from JSON file (full prompt object)
|
|
1801
3023
|
|
|
1802
|
-
${
|
|
3024
|
+
${chalk7.dim("Note: --data and --file are mutually exclusive. outputSchema is required (include in --data/--file or use --output-schema).")}
|
|
1803
3025
|
`).action(async (options) => {
|
|
1804
3026
|
const isJson = getJsonFlag(prompts);
|
|
1805
3027
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
1806
3028
|
try {
|
|
1807
3029
|
let data;
|
|
1808
|
-
if (options.file) {
|
|
1809
|
-
|
|
3030
|
+
if (options.data && options.file) {
|
|
3031
|
+
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
|
|
3032
|
+
}
|
|
3033
|
+
if (options.data) {
|
|
3034
|
+
try {
|
|
3035
|
+
data = JSON.parse(options.data);
|
|
3036
|
+
} catch {
|
|
3037
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"name":"my-prompt","systemPrompt":"You are helpful"}'`);
|
|
3038
|
+
}
|
|
3039
|
+
if (options.name)
|
|
3040
|
+
data.name = options.name;
|
|
3041
|
+
} else if (options.file) {
|
|
3042
|
+
const content = readFileSync4(options.file, "utf-8");
|
|
1810
3043
|
data = JSON.parse(content);
|
|
1811
3044
|
} else if (options.rawFile) {
|
|
1812
|
-
if (!
|
|
3045
|
+
if (!existsSync4(options.rawFile)) {
|
|
1813
3046
|
throw new MutagentError("FILE_NOT_FOUND", `File not found: ${options.rawFile}`, "Check the file path and try again");
|
|
1814
3047
|
}
|
|
1815
|
-
const textContent =
|
|
3048
|
+
const textContent = readFileSync4(options.rawFile, "utf-8");
|
|
1816
3049
|
data = {
|
|
1817
3050
|
name: options.name ?? options.rawFile.replace(/^.*[\\/]/, "").replace(/\.[^.]+$/, ""),
|
|
1818
3051
|
rawPrompt: textContent
|
|
@@ -1841,48 +3074,117 @@ ${chalk4.dim("Hint: Use --file with JSON or --raw-file for plain text. Get promp
|
|
|
1841
3074
|
} else if (options.content) {
|
|
1842
3075
|
data.rawPrompt = options.content;
|
|
1843
3076
|
} else {
|
|
1844
|
-
throw new MutagentError("MISSING_ARGUMENTS", "Prompt content required", "Use --raw, --raw-file, --system/--human, or --messages to specify prompt content");
|
|
3077
|
+
throw new MutagentError("MISSING_ARGUMENTS", "Prompt content required", "Use --data, --raw, --raw-file, --system/--human, or --messages to specify prompt content");
|
|
1845
3078
|
}
|
|
1846
3079
|
} else {
|
|
1847
|
-
throw new MutagentError("MISSING_ARGUMENTS", "Either --file, --raw-file, or --name with prompt content is required", "Use --file to load from JSON, --raw-file for plain text, or provide --name with --raw, --system/--human, or --messages");
|
|
3080
|
+
throw new MutagentError("MISSING_ARGUMENTS", "Either --data, --file, --raw-file, or --name with prompt content is required", "Use --data for inline JSON, --file to load from JSON, --raw-file for plain text, or provide --name with --raw, --system/--human, or --messages");
|
|
1848
3081
|
}
|
|
1849
3082
|
const promptContent = data.rawPrompt ?? data.systemPrompt ?? data.humanPrompt ?? "";
|
|
1850
3083
|
if (promptContent && !isJson) {
|
|
1851
3084
|
warnSingleBraceVariables(promptContent, output);
|
|
1852
3085
|
}
|
|
3086
|
+
if (options.outputSchema) {
|
|
3087
|
+
try {
|
|
3088
|
+
data.outputSchema = JSON.parse(options.outputSchema);
|
|
3089
|
+
} catch {
|
|
3090
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --output-schema flag", `Provide a valid JSON Schema, e.g., '{"type":"object","properties":{"result":{"type":"string"}}}'`);
|
|
3091
|
+
}
|
|
3092
|
+
}
|
|
3093
|
+
if (isSchemaEmpty(data.outputSchema)) {
|
|
3094
|
+
if (!isJson && process.stdin.isTTY) {
|
|
3095
|
+
const inquirer3 = (await import("inquirer")).default;
|
|
3096
|
+
output.warn("outputSchema is required for optimization. Define the expected output structure.");
|
|
3097
|
+
const schemaAnswer = await inquirer3.prompt([{
|
|
3098
|
+
type: "input",
|
|
3099
|
+
name: "schema",
|
|
3100
|
+
message: "Output schema (JSON):",
|
|
3101
|
+
validate: (input) => {
|
|
3102
|
+
if (!input.trim())
|
|
3103
|
+
return "outputSchema is required";
|
|
3104
|
+
try {
|
|
3105
|
+
JSON.parse(input);
|
|
3106
|
+
return true;
|
|
3107
|
+
} catch {
|
|
3108
|
+
return 'Invalid JSON. Example: {"type":"object","properties":{"result":{"type":"string"}}}';
|
|
3109
|
+
}
|
|
3110
|
+
}
|
|
3111
|
+
}]);
|
|
3112
|
+
data.outputSchema = JSON.parse(schemaAnswer.schema);
|
|
3113
|
+
} else {
|
|
3114
|
+
throw new MutagentError("MISSING_ARGUMENTS", "outputSchema is required for prompt creation", `Use --output-schema '{"type":"object","properties":{...}}' or include outputSchema in --data/--file`);
|
|
3115
|
+
}
|
|
3116
|
+
}
|
|
3117
|
+
if (!isJson && isSchemaEmpty(data.inputSchema)) {
|
|
3118
|
+
if (process.stdin.isTTY) {
|
|
3119
|
+
const schema = await promptForInputSchema();
|
|
3120
|
+
if (schema) {
|
|
3121
|
+
data.inputSchema = schema;
|
|
3122
|
+
}
|
|
3123
|
+
} else {
|
|
3124
|
+
output.warn("No inputSchema provided. Optimization requires inputSchema with defined variables.");
|
|
3125
|
+
}
|
|
3126
|
+
}
|
|
3127
|
+
warnMissingSchemas(data, output);
|
|
1853
3128
|
const client = getSDKClient();
|
|
1854
3129
|
const prompt = await client.createPrompt(data);
|
|
1855
3130
|
if (isJson) {
|
|
1856
3131
|
output.output({ ...prompt, _links: promptLinks(prompt.id) });
|
|
1857
3132
|
} else {
|
|
1858
|
-
output.success(`Created prompt: ${prompt.name}`);
|
|
1859
|
-
|
|
1860
|
-
|
|
3133
|
+
output.success(`Created prompt: ${prompt.name} (id: ${String(prompt.id)})`);
|
|
3134
|
+
const hints = formatCreationHints({
|
|
3135
|
+
resourceType: "Prompt",
|
|
3136
|
+
id: prompt.id,
|
|
3137
|
+
name: prompt.name,
|
|
3138
|
+
dashboardUrl: promptLink(prompt.id),
|
|
3139
|
+
apiPath: `/api/prompts/${String(prompt.id)}`
|
|
3140
|
+
});
|
|
3141
|
+
console.log(hints);
|
|
3142
|
+
}
|
|
3143
|
+
const sourceFile = options.file ?? options.rawFile;
|
|
3144
|
+
if (sourceFile) {
|
|
3145
|
+
updateMutationContext((ctx) => {
|
|
3146
|
+
const preview = (data.rawPrompt ?? data.systemPrompt ?? data.name ?? "").slice(0, 50);
|
|
3147
|
+
ctx.addDiscoveredPrompt(sourceFile, 1, preview);
|
|
3148
|
+
ctx.markPromptUploaded(sourceFile, String(prompt.id), prompt.version);
|
|
3149
|
+
});
|
|
1861
3150
|
}
|
|
1862
3151
|
} catch (error) {
|
|
1863
3152
|
handleError(error, isJson);
|
|
1864
3153
|
}
|
|
1865
3154
|
});
|
|
1866
|
-
prompts.command("update").description("Update a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("-f, --file <path>", "Update from JSON file").option("--raw-file <path>", "Update from plain text file (used as rawPrompt)").option("-n, --name <name>", "New name").option("-c, --content <content>", "New content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).addHelpText("after", `
|
|
3155
|
+
prompts.command("update").description("Update a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Update fields as JSON string (recommended — curl-style inline)").option("-f, --file <path>", "Update from JSON file").option("--raw-file <path>", "Update from plain text file (used as rawPrompt)").option("-n, --name <name>", "New name").option("-c, --content <content>", "New content (rawPrompt) [DEPRECATED: use --raw]").option("-r, --raw <text>", "Raw prompt text (single prompt)").option("--system <text>", "System prompt (use with --human)").option("--human <text>", "Human prompt (use with --system)").option("--messages <json>", `Messages array as JSON (e.g., '[{"role":"system","content":"..."}]')`).addHelpText("after", `
|
|
1867
3156
|
Examples:
|
|
1868
|
-
${
|
|
1869
|
-
${
|
|
1870
|
-
${
|
|
1871
|
-
${
|
|
1872
|
-
${
|
|
3157
|
+
${chalk7.dim("$")} mutagent prompts update <id> -d '{"name":"new-name","systemPrompt":"Updated prompt"}'
|
|
3158
|
+
${chalk7.dim("$")} mutagent prompts update <id> --name "new-name"
|
|
3159
|
+
${chalk7.dim("$")} mutagent prompts update <id> --system "Updated system prompt"
|
|
3160
|
+
${chalk7.dim("$")} mutagent prompts update <id> --raw-file updated-prompt.txt
|
|
3161
|
+
${chalk7.dim("$")} mutagent prompts update <id> --file updated-prompt.json
|
|
3162
|
+
|
|
3163
|
+
${chalk7.dim("Note: --data and --file are mutually exclusive. CLI flags (--name) override --data fields.")}
|
|
1873
3164
|
`).action(async (id, options) => {
|
|
1874
3165
|
const isJson = getJsonFlag(prompts);
|
|
1875
3166
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
1876
3167
|
try {
|
|
1877
3168
|
let data = {};
|
|
1878
|
-
if (options.file) {
|
|
1879
|
-
|
|
3169
|
+
if (options.data && options.file) {
|
|
3170
|
+
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
|
|
3171
|
+
}
|
|
3172
|
+
if (options.data) {
|
|
3173
|
+
try {
|
|
3174
|
+
data = JSON.parse(options.data);
|
|
3175
|
+
} catch {
|
|
3176
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"name":"new-name","systemPrompt":"Updated prompt"}'`);
|
|
3177
|
+
}
|
|
3178
|
+
if (options.name)
|
|
3179
|
+
data.name = options.name;
|
|
3180
|
+
} else if (options.file) {
|
|
3181
|
+
const content = readFileSync4(options.file, "utf-8");
|
|
1880
3182
|
data = JSON.parse(content);
|
|
1881
3183
|
} else if (options.rawFile) {
|
|
1882
|
-
if (!
|
|
3184
|
+
if (!existsSync4(options.rawFile)) {
|
|
1883
3185
|
throw new MutagentError("FILE_NOT_FOUND", `File not found: ${options.rawFile}`, "Check the file path and try again");
|
|
1884
3186
|
}
|
|
1885
|
-
const textContent =
|
|
3187
|
+
const textContent = readFileSync4(options.rawFile, "utf-8");
|
|
1886
3188
|
data.rawPrompt = textContent;
|
|
1887
3189
|
if (options.name)
|
|
1888
3190
|
data.name = options.name;
|
|
@@ -1913,7 +3215,7 @@ Examples:
|
|
|
1913
3215
|
}
|
|
1914
3216
|
}
|
|
1915
3217
|
if (Object.keys(data).length === 0) {
|
|
1916
|
-
throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --file, --raw-file, --name, --raw, --system/--human, or --messages");
|
|
3218
|
+
throw new MutagentError("MISSING_ARGUMENTS", "No update data provided", "Use --data, --file, --raw-file, --name, --raw, --system/--human, or --messages");
|
|
1917
3219
|
}
|
|
1918
3220
|
const promptContent = data.rawPrompt ?? data.systemPrompt ?? data.humanPrompt ?? "";
|
|
1919
3221
|
if (promptContent && !isJson) {
|
|
@@ -1924,8 +3226,21 @@ Examples:
|
|
|
1924
3226
|
if (isJson) {
|
|
1925
3227
|
output.output({ ...prompt, _links: promptLinks(prompt.id) });
|
|
1926
3228
|
} else {
|
|
1927
|
-
output.success(`Updated prompt: ${prompt.name}`);
|
|
1928
|
-
|
|
3229
|
+
output.success(`Updated prompt: ${prompt.name} (id: ${String(prompt.id)})`);
|
|
3230
|
+
const hints = formatCreationHints({
|
|
3231
|
+
resourceType: "Prompt",
|
|
3232
|
+
id: prompt.id,
|
|
3233
|
+
name: prompt.name,
|
|
3234
|
+
dashboardUrl: promptLink(prompt.id),
|
|
3235
|
+
apiPath: `/api/prompts/${String(prompt.id)}`
|
|
3236
|
+
});
|
|
3237
|
+
console.log(hints);
|
|
3238
|
+
}
|
|
3239
|
+
const sourceFile = options.file ?? options.rawFile;
|
|
3240
|
+
if (sourceFile) {
|
|
3241
|
+
updateMutationContext((ctx) => {
|
|
3242
|
+
ctx.markPromptUploaded(sourceFile, String(prompt.id), prompt.version);
|
|
3243
|
+
});
|
|
1929
3244
|
}
|
|
1930
3245
|
} catch (error) {
|
|
1931
3246
|
handleError(error, isJson);
|
|
@@ -1933,11 +3248,11 @@ Examples:
|
|
|
1933
3248
|
});
|
|
1934
3249
|
prompts.command("delete").description("Delete a prompt").argument("<id>", "Prompt ID (from: mutagent prompts list)").option("--force", "Skip confirmation").addHelpText("after", `
|
|
1935
3250
|
Examples:
|
|
1936
|
-
${
|
|
1937
|
-
${
|
|
1938
|
-
${
|
|
3251
|
+
${chalk7.dim("$")} mutagent prompts delete <id>
|
|
3252
|
+
${chalk7.dim("$")} mutagent prompts delete <id> --force
|
|
3253
|
+
${chalk7.dim("$")} mutagent prompts delete <id> --force --json
|
|
1939
3254
|
|
|
1940
|
-
${
|
|
3255
|
+
${chalk7.dim("Tip: Use --force to skip confirmation (required for non-interactive/CI usage).")}
|
|
1941
3256
|
`).action(async (id, options) => {
|
|
1942
3257
|
const isJson = getJsonFlag(prompts);
|
|
1943
3258
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -1964,14 +3279,14 @@ ${chalk4.dim("Tip: Use --force to skip confirmation (required for non-interactiv
|
|
|
1964
3279
|
});
|
|
1965
3280
|
const dataset = prompts.command("dataset").description("Manage datasets for prompts").addHelpText("after", `
|
|
1966
3281
|
Examples:
|
|
1967
|
-
${
|
|
1968
|
-
${
|
|
1969
|
-
${
|
|
3282
|
+
${chalk7.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
3283
|
+
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl
|
|
3284
|
+
${chalk7.dim("$")} mutagent prompts dataset remove <prompt-id> <dataset-id>
|
|
1970
3285
|
`);
|
|
1971
3286
|
dataset.command("list").description("List datasets for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
|
|
1972
3287
|
Examples:
|
|
1973
|
-
${
|
|
1974
|
-
${
|
|
3288
|
+
${chalk7.dim("$")} mutagent prompts dataset list <prompt-id>
|
|
3289
|
+
${chalk7.dim("$")} mutagent prompts dataset list <prompt-id> --json
|
|
1975
3290
|
`).action(async (promptId) => {
|
|
1976
3291
|
const isJson = getJsonFlag(prompts);
|
|
1977
3292
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -1981,7 +3296,7 @@ Examples:
|
|
|
1981
3296
|
if (isJson) {
|
|
1982
3297
|
const withLinks = datasets.map((d) => ({
|
|
1983
3298
|
...d,
|
|
1984
|
-
_links:
|
|
3299
|
+
_links: datasetLinks(promptId, d.id)
|
|
1985
3300
|
}));
|
|
1986
3301
|
output.output(withLinks);
|
|
1987
3302
|
} else {
|
|
@@ -1997,21 +3312,21 @@ Examples:
|
|
|
1997
3312
|
});
|
|
1998
3313
|
dataset.command("add").description("Add dataset to a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-f, --file <path>", "Dataset file (JSON array, JSONL, or CSV)").option("-d, --data <json>", "Inline JSON array of dataset items").option("-n, --name <name>", "Dataset name (default: timestamped)").addHelpText("after", `
|
|
1999
3314
|
Examples:
|
|
2000
|
-
${
|
|
2001
|
-
${
|
|
2002
|
-
${
|
|
2003
|
-
${
|
|
3315
|
+
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.json
|
|
3316
|
+
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.jsonl --name "My Dataset"
|
|
3317
|
+
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> --file dataset.csv
|
|
3318
|
+
${chalk7.dim("$")} mutagent prompts dataset add <prompt-id> -d '[{"input":{"text":"hello"},"expectedOutput":{"result":"world"}}]'
|
|
2004
3319
|
|
|
2005
3320
|
Supported file formats:
|
|
2006
|
-
${
|
|
2007
|
-
${
|
|
2008
|
-
${
|
|
3321
|
+
${chalk7.dim(".json")} JSON array of objects: [{"input": {...}, "expectedOutput": {...}}, ...]
|
|
3322
|
+
${chalk7.dim(".jsonl")} One JSON object per line (newline-delimited JSON)
|
|
3323
|
+
${chalk7.dim(".csv")} Comma-separated values with header row
|
|
2009
3324
|
|
|
2010
3325
|
Inline data format (-d):
|
|
2011
3326
|
JSON array of objects, e.g.:
|
|
2012
|
-
${
|
|
3327
|
+
${chalk7.dim('[{"input": {"text": "hello"}, "expectedOutput": {"result": "world"}}]')}
|
|
2013
3328
|
|
|
2014
|
-
${
|
|
3329
|
+
${chalk7.dim("Note: --file and -d are mutually exclusive. Provide one or the other.")}
|
|
2015
3330
|
`).action(async (promptId, options) => {
|
|
2016
3331
|
const isJson = getJsonFlag(prompts);
|
|
2017
3332
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -2037,26 +3352,62 @@ ${chalk4.dim("Note: --file and -d are mutually exclusive. Provide one or the oth
|
|
|
2037
3352
|
}
|
|
2038
3353
|
} else {
|
|
2039
3354
|
const filePath = options.file;
|
|
2040
|
-
const rawContent =
|
|
3355
|
+
const rawContent = readFileSync4(filePath, "utf-8");
|
|
2041
3356
|
content = parseDatasetFile(rawContent, filePath);
|
|
2042
3357
|
}
|
|
3358
|
+
let datasetName = options.name;
|
|
3359
|
+
if (!datasetName && !isJson && process.stdin.isTTY) {
|
|
3360
|
+
const inquirer3 = (await import("inquirer")).default;
|
|
3361
|
+
const defaultName = options.file ? options.file.replace(/^.*[\\/]/, "").replace(/\.[^.]+$/, "") : undefined;
|
|
3362
|
+
const nameAnswer = await inquirer3.prompt([{
|
|
3363
|
+
type: "input",
|
|
3364
|
+
name: "name",
|
|
3365
|
+
message: "Name this dataset:",
|
|
3366
|
+
default: defaultName,
|
|
3367
|
+
validate: (input) => {
|
|
3368
|
+
if (!input.trim())
|
|
3369
|
+
return "Dataset name is required";
|
|
3370
|
+
if (input.trim().length < 2)
|
|
3371
|
+
return "Dataset name must be at least 2 characters";
|
|
3372
|
+
return true;
|
|
3373
|
+
}
|
|
3374
|
+
}]);
|
|
3375
|
+
datasetName = nameAnswer.name.trim();
|
|
3376
|
+
}
|
|
2043
3377
|
const client = getSDKClient();
|
|
2044
|
-
const datasetResult = await client.addDataset(promptId, content,
|
|
2045
|
-
if (
|
|
2046
|
-
output.
|
|
3378
|
+
const datasetResult = await client.addDataset(promptId, content, datasetName);
|
|
3379
|
+
if (isJson) {
|
|
3380
|
+
output.output({
|
|
3381
|
+
...datasetResult,
|
|
3382
|
+
_links: datasetLinks(promptId, datasetResult.id)
|
|
3383
|
+
});
|
|
3384
|
+
} else {
|
|
3385
|
+
output.success(`Added dataset "${datasetResult.name}" to prompt: ${promptId} (id: ${String(datasetResult.id)})`);
|
|
2047
3386
|
if (datasetResult.itemCount !== undefined && datasetResult.itemCount > 0) {
|
|
2048
3387
|
output.info(`Items uploaded: ${String(datasetResult.itemCount)}`);
|
|
2049
3388
|
}
|
|
3389
|
+
const hints = formatCreationHints({
|
|
3390
|
+
resourceType: "Dataset",
|
|
3391
|
+
id: datasetResult.id,
|
|
3392
|
+
name: datasetResult.name,
|
|
3393
|
+
dashboardUrl: datasetLink(promptId, datasetResult.id),
|
|
3394
|
+
apiPath: `/api/prompts/${promptId}/datasets/${String(datasetResult.id)}`
|
|
3395
|
+
});
|
|
3396
|
+
console.log(hints);
|
|
2050
3397
|
}
|
|
2051
|
-
|
|
3398
|
+
const datasetFile = options.file ?? "inline-data";
|
|
3399
|
+
updateMutationContext((ctx) => {
|
|
3400
|
+
ctx.addDiscoveredDataset(datasetFile, datasetResult.name, datasetResult.itemCount ?? 0);
|
|
3401
|
+
ctx.markDatasetUploaded(datasetFile, String(datasetResult.id), promptId);
|
|
3402
|
+
});
|
|
2052
3403
|
} catch (error) {
|
|
2053
3404
|
handleError(error, isJson);
|
|
2054
3405
|
}
|
|
2055
3406
|
});
|
|
2056
3407
|
dataset.command("remove").description("Remove dataset from a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").argument("<dataset-id>", "Dataset ID (from: mutagent prompts dataset list <prompt-id>)").addHelpText("after", `
|
|
2057
3408
|
Examples:
|
|
2058
|
-
${
|
|
2059
|
-
${
|
|
3409
|
+
${chalk7.dim("$")} mutagent prompts dataset remove <prompt-id> <dataset-id>
|
|
3410
|
+
${chalk7.dim("$")} mutagent prompts dataset remove <prompt-id> <dataset-id> --json
|
|
2060
3411
|
`).action(async (promptId, datasetId) => {
|
|
2061
3412
|
const isJson = getJsonFlag(prompts);
|
|
2062
3413
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -2070,14 +3421,14 @@ Examples:
|
|
|
2070
3421
|
});
|
|
2071
3422
|
const evaluation = prompts.command("evaluation").description("Manage evaluations for prompts").addHelpText("after", `
|
|
2072
3423
|
Examples:
|
|
2073
|
-
${
|
|
2074
|
-
${
|
|
2075
|
-
${
|
|
3424
|
+
${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
3425
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "My Eval"
|
|
3426
|
+
${chalk7.dim("$")} mutagent prompts evaluation results <run-id>
|
|
2076
3427
|
`);
|
|
2077
3428
|
evaluation.command("list").description("List evaluations for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").addHelpText("after", `
|
|
2078
3429
|
Examples:
|
|
2079
|
-
${
|
|
2080
|
-
${
|
|
3430
|
+
${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id>
|
|
3431
|
+
${chalk7.dim("$")} mutagent prompts evaluation list <prompt-id> --json
|
|
2081
3432
|
`).action(async (promptId) => {
|
|
2082
3433
|
const isJson = getJsonFlag(prompts);
|
|
2083
3434
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -2087,7 +3438,7 @@ Examples:
|
|
|
2087
3438
|
if (isJson) {
|
|
2088
3439
|
const withLinks = evals.map((e) => ({
|
|
2089
3440
|
...e,
|
|
2090
|
-
_links:
|
|
3441
|
+
_links: evaluationLinks(promptId, e.id)
|
|
2091
3442
|
}));
|
|
2092
3443
|
output.output(withLinks);
|
|
2093
3444
|
} else {
|
|
@@ -2101,30 +3452,85 @@ Examples:
|
|
|
2101
3452
|
handleError(error, isJson);
|
|
2102
3453
|
}
|
|
2103
3454
|
});
|
|
2104
|
-
evaluation.command("create").description("Create an evaluation configuration for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").
|
|
3455
|
+
evaluation.command("create").description("Create an evaluation configuration for a prompt").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").option("-d, --data <json>", "Evaluation as JSON string (recommended — curl-style inline)").option("-n, --name <name>", "Evaluation name (required unless --guided)").option("-f, --file <path>", "Load evaluation criteria from JSON file").option("--description <text>", "Evaluation description").option("--dataset <id>", "Dataset ID to associate (from: mutagent prompts dataset list)").option("--guided", "Interactive guided mode — build criteria step by step").addHelpText("after", `
|
|
2105
3456
|
Examples:
|
|
2106
|
-
${
|
|
2107
|
-
${
|
|
2108
|
-
${
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
${
|
|
2114
|
-
|
|
3457
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --guided
|
|
3458
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --data '{"name":"Accuracy","evalConfig":{"criteria":[...]}}' --name "Accuracy"
|
|
3459
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Accuracy Check"
|
|
3460
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "Full Eval" --file criteria.json --dataset <dataset-id>
|
|
3461
|
+
${chalk7.dim("$")} mutagent prompts evaluation create <prompt-id> --name "QA" --description "Quality assurance eval" --json
|
|
3462
|
+
|
|
3463
|
+
Criteria file format (--file or --data):
|
|
3464
|
+
${chalk7.dim('{ "name": "...", "evalConfig": { "criteria": [...] }, "llmConfig": { "model": "gpt-4" } }')}
|
|
3465
|
+
|
|
3466
|
+
${chalk7.dim("Note: --data and --file are mutually exclusive. CLI flags (--name, --description) override --data fields.")}
|
|
3467
|
+
${chalk7.dim("Tip: Use --guided for an interactive walkthrough of criteria creation.")}
|
|
3468
|
+
${chalk7.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent prompts dataset list <prompt-id>")}
|
|
2115
3469
|
`).action(async (promptId, options) => {
|
|
2116
3470
|
const isJson = getJsonFlag(prompts);
|
|
2117
3471
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
2118
3472
|
try {
|
|
3473
|
+
if (options.guided) {
|
|
3474
|
+
if (!process.stdin.isTTY || isJson) {
|
|
3475
|
+
throw new MutagentError("INTERACTIVE_REQUIRED", "--guided requires an interactive terminal", "Remove --guided for non-interactive usage, or use --data/--file to provide criteria");
|
|
3476
|
+
}
|
|
3477
|
+
const guidedResult = await runGuidedEvalCreator(promptId);
|
|
3478
|
+
if (!guidedResult) {
|
|
3479
|
+
return;
|
|
3480
|
+
}
|
|
3481
|
+
const client2 = getSDKClient();
|
|
3482
|
+
const evalResult2 = await client2.createEvaluation(promptId, {
|
|
3483
|
+
name: guidedResult.name,
|
|
3484
|
+
description: guidedResult.description,
|
|
3485
|
+
evalConfig: guidedResult.evalConfig,
|
|
3486
|
+
datasetId: options.dataset ? parseInt(options.dataset, 10) : undefined
|
|
3487
|
+
});
|
|
3488
|
+
output.success(`Created evaluation: ${evalResult2.name} (id: ${String(evalResult2.id)})`);
|
|
3489
|
+
const hints = formatCreationHints({
|
|
3490
|
+
resourceType: "Evaluation",
|
|
3491
|
+
id: evalResult2.id,
|
|
3492
|
+
name: evalResult2.name,
|
|
3493
|
+
dashboardUrl: evaluationLink(promptId, evalResult2.id),
|
|
3494
|
+
apiPath: `/api/prompts/${promptId}/evaluations/${String(evalResult2.id)}`
|
|
3495
|
+
});
|
|
3496
|
+
console.log(hints);
|
|
3497
|
+
const criteriaCount2 = guidedResult.evalConfig.criteria.length;
|
|
3498
|
+
updateMutationContext((ctx) => {
|
|
3499
|
+
ctx.addEvaluation(evalResult2.name, criteriaCount2, String(evalResult2.id), promptId);
|
|
3500
|
+
});
|
|
3501
|
+
return;
|
|
3502
|
+
}
|
|
3503
|
+
if (!options.name) {
|
|
3504
|
+
throw new MutagentError("MISSING_ARGUMENTS", "Evaluation name is required", "Use --name <name> or --guided for interactive mode");
|
|
3505
|
+
}
|
|
3506
|
+
if (options.data && options.file) {
|
|
3507
|
+
throw new MutagentError("INVALID_ARGUMENTS", "Cannot use --data and --file together", "Use --data for inline JSON or --file for file-based input, not both");
|
|
3508
|
+
}
|
|
2119
3509
|
const evalData = {
|
|
2120
3510
|
name: options.name,
|
|
2121
3511
|
description: options.description
|
|
2122
3512
|
};
|
|
2123
|
-
if (options.
|
|
2124
|
-
|
|
3513
|
+
if (options.data) {
|
|
3514
|
+
try {
|
|
3515
|
+
const parsed = JSON.parse(options.data);
|
|
3516
|
+
if (parsed.evalConfig)
|
|
3517
|
+
evalData.evalConfig = parsed.evalConfig;
|
|
3518
|
+
if (parsed.llmConfig)
|
|
3519
|
+
evalData.llmConfig = parsed.llmConfig;
|
|
3520
|
+
if (parsed.tags)
|
|
3521
|
+
evalData.tags = parsed.tags;
|
|
3522
|
+
if (parsed.metadata)
|
|
3523
|
+
evalData.metadata = parsed.metadata;
|
|
3524
|
+
if (parsed.datasetId)
|
|
3525
|
+
evalData.datasetId = parsed.datasetId;
|
|
3526
|
+
} catch {
|
|
3527
|
+
throw new MutagentError("INVALID_JSON", "Invalid JSON in --data flag", `Provide a valid JSON object, e.g., '{"evalConfig":{"criteria":[...]},"llmConfig":{"model":"gpt-4"}}'`);
|
|
3528
|
+
}
|
|
3529
|
+
} else if (options.file) {
|
|
3530
|
+
if (!existsSync4(options.file)) {
|
|
2125
3531
|
throw new MutagentError("FILE_NOT_FOUND", `File not found: ${options.file}`, "Check the file path and try again");
|
|
2126
3532
|
}
|
|
2127
|
-
const fileContent =
|
|
3533
|
+
const fileContent = readFileSync4(options.file, "utf-8");
|
|
2128
3534
|
try {
|
|
2129
3535
|
const parsed = JSON.parse(fileContent);
|
|
2130
3536
|
if (parsed.evalConfig)
|
|
@@ -2139,6 +3545,10 @@ ${chalk4.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent
|
|
|
2139
3545
|
throw new MutagentError("INVALID_JSON", `Failed to parse criteria file: ${options.file}`, "Ensure the file contains valid JSON with evalConfig and/or llmConfig");
|
|
2140
3546
|
}
|
|
2141
3547
|
}
|
|
3548
|
+
if (options.name)
|
|
3549
|
+
evalData.name = options.name;
|
|
3550
|
+
if (options.description)
|
|
3551
|
+
evalData.description = options.description;
|
|
2142
3552
|
if (options.dataset) {
|
|
2143
3553
|
evalData.datasetId = parseInt(options.dataset, 10);
|
|
2144
3554
|
if (isNaN(evalData.datasetId)) {
|
|
@@ -2150,21 +3560,31 @@ ${chalk4.dim("Get prompt IDs: mutagent prompts list | Get dataset IDs: mutagent
|
|
|
2150
3560
|
if (isJson) {
|
|
2151
3561
|
output.output({
|
|
2152
3562
|
...evalResult,
|
|
2153
|
-
_links:
|
|
3563
|
+
_links: evaluationLinks(promptId, evalResult.id)
|
|
2154
3564
|
});
|
|
2155
3565
|
} else {
|
|
2156
|
-
output.success(`Created evaluation: ${evalResult.name}`);
|
|
2157
|
-
|
|
2158
|
-
|
|
3566
|
+
output.success(`Created evaluation: ${evalResult.name} (id: ${String(evalResult.id)})`);
|
|
3567
|
+
const hints = formatCreationHints({
|
|
3568
|
+
resourceType: "Evaluation",
|
|
3569
|
+
id: evalResult.id,
|
|
3570
|
+
name: evalResult.name,
|
|
3571
|
+
dashboardUrl: evaluationLink(promptId, evalResult.id),
|
|
3572
|
+
apiPath: `/api/prompts/${promptId}/evaluations/${String(evalResult.id)}`
|
|
3573
|
+
});
|
|
3574
|
+
console.log(hints);
|
|
2159
3575
|
}
|
|
3576
|
+
const criteriaCount = evalData.evalConfig?.criteria?.length ?? 0;
|
|
3577
|
+
updateMutationContext((ctx) => {
|
|
3578
|
+
ctx.addEvaluation(evalResult.name, criteriaCount, String(evalResult.id), promptId);
|
|
3579
|
+
});
|
|
2160
3580
|
} catch (error) {
|
|
2161
3581
|
handleError(error, isJson);
|
|
2162
3582
|
}
|
|
2163
3583
|
});
|
|
2164
3584
|
evaluation.command("results").description("Get evaluation results").argument("<run-id>", "Evaluation run ID (from: mutagent prompts evaluation list <prompt-id>)").addHelpText("after", `
|
|
2165
3585
|
Examples:
|
|
2166
|
-
${
|
|
2167
|
-
${
|
|
3586
|
+
${chalk7.dim("$")} mutagent prompts evaluation results <run-id>
|
|
3587
|
+
${chalk7.dim("$")} mutagent prompts evaluation results <run-id> --json
|
|
2168
3588
|
`).action(async (runId) => {
|
|
2169
3589
|
const isJson = getJsonFlag(prompts);
|
|
2170
3590
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -2178,21 +3598,21 @@ Examples:
|
|
|
2178
3598
|
});
|
|
2179
3599
|
const optimize = prompts.command("optimize").description("Manage prompt optimization jobs").addHelpText("after", `
|
|
2180
3600
|
Examples:
|
|
2181
|
-
${
|
|
2182
|
-
${
|
|
2183
|
-
${
|
|
3601
|
+
${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
|
|
3602
|
+
${chalk7.dim("$")} mutagent prompts optimize status <job-id>
|
|
3603
|
+
${chalk7.dim("$")} mutagent prompts optimize results <job-id>
|
|
2184
3604
|
|
|
2185
3605
|
Workflow: start -> status (poll) -> results
|
|
2186
3606
|
`);
|
|
2187
3607
|
optimize.command("start").description("Start prompt optimization").argument("<prompt-id>", "Prompt ID (from: mutagent prompts list)").requiredOption("-d, --dataset <id>", "Dataset ID for optimization (from: mutagent prompts dataset list <prompt-id>)").option("--max-iterations <n>", "Max optimization iterations (default: 3)").option("--target-score <n>", "Target accuracy 0-1 (e.g., 0.9)").option("--patience <n>", "Iterations without improvement before stopping").option("--model <model-id>", 'Target LLM model (e.g., "claude-sonnet-4-5-20250929")').option("--eval-model <model-id>", "Evaluation model (defaults to target model)").addHelpText("after", `
|
|
2188
3608
|
Examples:
|
|
2189
|
-
${
|
|
2190
|
-
${
|
|
2191
|
-
${
|
|
2192
|
-
${
|
|
3609
|
+
${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id>
|
|
3610
|
+
${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --max-iterations 5
|
|
3611
|
+
${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --target-score 0.95 --model claude-sonnet-4-5-20250929
|
|
3612
|
+
${chalk7.dim("$")} mutagent prompts optimize start <prompt-id> --dataset <dataset-id> --json
|
|
2193
3613
|
${PREREQUISITES_TEXT}
|
|
2194
3614
|
|
|
2195
|
-
${
|
|
3615
|
+
${chalk7.dim("Monitor progress with: mutagent prompts optimize status <job-id>")}
|
|
2196
3616
|
`).action(async (promptId, options) => {
|
|
2197
3617
|
const isJson = getJsonFlag(prompts);
|
|
2198
3618
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -2206,19 +3626,30 @@ ${chalk4.dim("Monitor progress with: mutagent prompts optimize status <job-id>")
|
|
|
2206
3626
|
evalModel: options.evalModel
|
|
2207
3627
|
});
|
|
2208
3628
|
if (isJson) {
|
|
2209
|
-
output.output({
|
|
3629
|
+
output.output({
|
|
3630
|
+
...job,
|
|
3631
|
+
_links: {
|
|
3632
|
+
dashboard: optimizerLink(job.jobId),
|
|
3633
|
+
api: `/api/prompts/${promptId}/optimizations/${job.jobId}`
|
|
3634
|
+
}
|
|
3635
|
+
});
|
|
2210
3636
|
} else {
|
|
2211
3637
|
output.success(`Started optimization job: ${job.jobId}`);
|
|
2212
|
-
|
|
2213
|
-
|
|
3638
|
+
const hints = formatCreationHints({
|
|
3639
|
+
resourceType: "Optimization",
|
|
3640
|
+
id: job.jobId,
|
|
3641
|
+
dashboardUrl: optimizerLink(job.jobId),
|
|
3642
|
+
apiPath: `/api/prompts/${promptId}/optimizations/${job.jobId}`
|
|
3643
|
+
});
|
|
3644
|
+
console.log(hints);
|
|
2214
3645
|
}
|
|
2215
3646
|
} catch (error) {
|
|
2216
3647
|
if (error instanceof ApiError) {
|
|
2217
3648
|
const messages = parseValidationErrors(error);
|
|
2218
3649
|
if (!isJson) {
|
|
2219
|
-
console.error(
|
|
3650
|
+
console.error(chalk7.red("Optimization failed. Missing requirements:"));
|
|
2220
3651
|
for (const msg of messages) {
|
|
2221
|
-
console.error(
|
|
3652
|
+
console.error(chalk7.red(` - ${msg}`));
|
|
2222
3653
|
}
|
|
2223
3654
|
console.error("");
|
|
2224
3655
|
console.error(PREREQUISITES_TEXT);
|
|
@@ -2229,8 +3660,8 @@ ${chalk4.dim("Monitor progress with: mutagent prompts optimize status <job-id>")
|
|
|
2229
3660
|
});
|
|
2230
3661
|
optimize.command("status").description("Check optimization status").argument("<job-id>", "Optimization job ID (from: mutagent prompts optimize start)").addHelpText("after", `
|
|
2231
3662
|
Examples:
|
|
2232
|
-
${
|
|
2233
|
-
${
|
|
3663
|
+
${chalk7.dim("$")} mutagent prompts optimize status <job-id>
|
|
3664
|
+
${chalk7.dim("$")} mutagent prompts optimize status <job-id> --json
|
|
2234
3665
|
`).action(async (jobId) => {
|
|
2235
3666
|
const isJson = getJsonFlag(prompts);
|
|
2236
3667
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -2253,19 +3684,59 @@ Examples:
|
|
|
2253
3684
|
});
|
|
2254
3685
|
optimize.command("results").description("Get optimization results").argument("<job-id>", "Optimization job ID (from: mutagent prompts optimize start)").addHelpText("after", `
|
|
2255
3686
|
Examples:
|
|
2256
|
-
${
|
|
2257
|
-
${
|
|
3687
|
+
${chalk7.dim("$")} mutagent prompts optimize results <job-id>
|
|
3688
|
+
${chalk7.dim("$")} mutagent prompts optimize results <job-id> --json
|
|
2258
3689
|
`).action(async (jobId) => {
|
|
2259
3690
|
const isJson = getJsonFlag(prompts);
|
|
2260
3691
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
2261
3692
|
try {
|
|
2262
3693
|
const client = getSDKClient();
|
|
2263
3694
|
const results = await client.getOptimizationResults(jobId);
|
|
3695
|
+
const resultData = results;
|
|
2264
3696
|
if (isJson) {
|
|
2265
|
-
output.output({ ...
|
|
3697
|
+
output.output({ ...resultData, _links: { optimizer: optimizerLink(jobId) } });
|
|
2266
3698
|
} else {
|
|
2267
|
-
|
|
2268
|
-
|
|
3699
|
+
renderScorecard(resultData);
|
|
3700
|
+
const jobData = resultData.job;
|
|
3701
|
+
const isCompleted = jobData?.status === "completed";
|
|
3702
|
+
if (isCompleted && process.stdin.isTTY) {
|
|
3703
|
+
let showDetails = true;
|
|
3704
|
+
while (showDetails) {
|
|
3705
|
+
const { action } = await promptScorecardAction();
|
|
3706
|
+
if (action === "details") {
|
|
3707
|
+
const original = resultData.originalPrompt;
|
|
3708
|
+
const optimized = resultData.prompt;
|
|
3709
|
+
showPromptDiff(original?.systemPrompt ?? original?.rawPrompt ?? null, optimized?.systemPrompt ?? optimized?.rawPrompt ?? null);
|
|
3710
|
+
continue;
|
|
3711
|
+
}
|
|
3712
|
+
if (action === "apply") {
|
|
3713
|
+
const optimizedPrompt = resultData.prompt;
|
|
3714
|
+
const promptIdStr = String(jobData.promptId);
|
|
3715
|
+
if (optimizedPrompt && promptIdStr) {
|
|
3716
|
+
output.info("Applying optimized prompt...");
|
|
3717
|
+
const updateData = {};
|
|
3718
|
+
if (optimizedPrompt.systemPrompt)
|
|
3719
|
+
updateData.systemPrompt = optimizedPrompt.systemPrompt;
|
|
3720
|
+
if (optimizedPrompt.humanPrompt)
|
|
3721
|
+
updateData.humanPrompt = optimizedPrompt.humanPrompt;
|
|
3722
|
+
if (optimizedPrompt.rawPrompt)
|
|
3723
|
+
updateData.rawPrompt = optimizedPrompt.rawPrompt;
|
|
3724
|
+
await client.updatePrompt(promptIdStr, updateData);
|
|
3725
|
+
output.success("Optimized prompt applied as new version!");
|
|
3726
|
+
updateMutationContext((ctx) => {
|
|
3727
|
+
ctx.markPromptUploaded(`optimization:${jobId}`, promptIdStr, "optimized");
|
|
3728
|
+
});
|
|
3729
|
+
}
|
|
3730
|
+
} else {
|
|
3731
|
+
output.info("Kept original prompt. No changes applied.");
|
|
3732
|
+
}
|
|
3733
|
+
showDetails = false;
|
|
3734
|
+
}
|
|
3735
|
+
} else if (!isCompleted) {
|
|
3736
|
+
const statusStr = jobData?.status ?? "unknown";
|
|
3737
|
+
output.info(`Job status: ${statusStr}. Results may be partial.`);
|
|
3738
|
+
output.info(`View: ${optimizerLink(jobId)}`);
|
|
3739
|
+
}
|
|
2269
3740
|
}
|
|
2270
3741
|
} catch (error) {
|
|
2271
3742
|
handleError(error, isJson);
|
|
@@ -2275,26 +3746,28 @@ Examples:
|
|
|
2275
3746
|
}
|
|
2276
3747
|
|
|
2277
3748
|
// src/commands/traces.ts
|
|
3749
|
+
init_sdk_client();
|
|
2278
3750
|
import { Command as Command4 } from "commander";
|
|
2279
|
-
import
|
|
3751
|
+
import chalk8 from "chalk";
|
|
3752
|
+
init_errors();
|
|
2280
3753
|
function createTracesCommand() {
|
|
2281
3754
|
const traces = new Command4("traces").description("View and analyze traces (replaces Langfuse)").addHelpText("after", `
|
|
2282
3755
|
Examples:
|
|
2283
|
-
${
|
|
2284
|
-
${
|
|
2285
|
-
${
|
|
2286
|
-
${
|
|
2287
|
-
${
|
|
3756
|
+
${chalk8.dim("$")} mutagent traces list
|
|
3757
|
+
${chalk8.dim("$")} mutagent traces list --prompt <prompt-id>
|
|
3758
|
+
${chalk8.dim("$")} mutagent traces get <trace-id>
|
|
3759
|
+
${chalk8.dim("$")} mutagent traces analyze <prompt-id>
|
|
3760
|
+
${chalk8.dim("$")} mutagent traces export --format json --output traces.json
|
|
2288
3761
|
|
|
2289
3762
|
Note: MutagenT traces replace Langfuse for observability.
|
|
2290
3763
|
`);
|
|
2291
3764
|
traces.command("list").description("List traces").option("-p, --prompt <id>", "Filter by prompt ID").option("-l, --limit <n>", "Limit results", "50").addHelpText("after", `
|
|
2292
3765
|
Examples:
|
|
2293
|
-
${
|
|
2294
|
-
${
|
|
2295
|
-
${
|
|
3766
|
+
${chalk8.dim("$")} mutagent traces list
|
|
3767
|
+
${chalk8.dim("$")} mutagent traces list --prompt <prompt-id>
|
|
3768
|
+
${chalk8.dim("$")} mutagent traces list --limit 10 --json
|
|
2296
3769
|
|
|
2297
|
-
${
|
|
3770
|
+
${chalk8.dim("Tip: Filter by prompt to see traces for a specific prompt version.")}
|
|
2298
3771
|
`).action(async (options) => {
|
|
2299
3772
|
const isJson = getJsonFlag(traces);
|
|
2300
3773
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -2328,10 +3801,10 @@ ${chalk5.dim("Tip: Filter by prompt to see traces for a specific prompt version.
|
|
|
2328
3801
|
});
|
|
2329
3802
|
traces.command("get").description("Get trace details").argument("<id>", "Trace ID").addHelpText("after", `
|
|
2330
3803
|
Examples:
|
|
2331
|
-
${
|
|
2332
|
-
${
|
|
3804
|
+
${chalk8.dim("$")} mutagent traces get <trace-id>
|
|
3805
|
+
${chalk8.dim("$")} mutagent traces get <trace-id> --json
|
|
2333
3806
|
|
|
2334
|
-
${
|
|
3807
|
+
${chalk8.dim("Returns full trace details including spans, tokens, and latency.")}
|
|
2335
3808
|
`).action(async (id) => {
|
|
2336
3809
|
const isJson = getJsonFlag(traces);
|
|
2337
3810
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -2350,10 +3823,10 @@ ${chalk5.dim("Returns full trace details including spans, tokens, and latency.")
|
|
|
2350
3823
|
});
|
|
2351
3824
|
traces.command("analyze").description("Analyze traces for a prompt").argument("<prompt-id>", "Prompt ID").addHelpText("after", `
|
|
2352
3825
|
Examples:
|
|
2353
|
-
${
|
|
2354
|
-
${
|
|
3826
|
+
${chalk8.dim("$")} mutagent traces analyze <prompt-id>
|
|
3827
|
+
${chalk8.dim("$")} mutagent traces analyze <prompt-id> --json
|
|
2355
3828
|
|
|
2356
|
-
${
|
|
3829
|
+
${chalk8.dim("Aggregates trace data for a prompt: avg latency, token usage, error rates.")}
|
|
2357
3830
|
`).action(async (promptId) => {
|
|
2358
3831
|
const isJson = getJsonFlag(traces);
|
|
2359
3832
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -2371,12 +3844,12 @@ ${chalk5.dim("Aggregates trace data for a prompt: avg latency, token usage, erro
|
|
|
2371
3844
|
});
|
|
2372
3845
|
traces.command("export").description("Export traces").option("-p, --prompt <id>", "Filter by prompt ID").option("-f, --format <format>", "Export format (json, csv)", "json").option("-o, --output <path>", "Output file path").addHelpText("after", `
|
|
2373
3846
|
Examples:
|
|
2374
|
-
${
|
|
2375
|
-
${
|
|
2376
|
-
${
|
|
2377
|
-
${
|
|
3847
|
+
${chalk8.dim("$")} mutagent traces export
|
|
3848
|
+
${chalk8.dim("$")} mutagent traces export --format json --output traces.json
|
|
3849
|
+
${chalk8.dim("$")} mutagent traces export --format csv --output traces.csv
|
|
3850
|
+
${chalk8.dim("$")} mutagent traces export --prompt <prompt-id> --format json
|
|
2378
3851
|
|
|
2379
|
-
${
|
|
3852
|
+
${chalk8.dim("Exports to stdout by default. Use --output to save to a file.")}
|
|
2380
3853
|
`).action(async (options) => {
|
|
2381
3854
|
const isJson = getJsonFlag(traces);
|
|
2382
3855
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -2404,8 +3877,8 @@ ${chalk5.dim("Exports to stdout by default. Use --output to save to a file.")}
|
|
|
2404
3877
|
throw new Error(`Unsupported format: ${options.format}`);
|
|
2405
3878
|
}
|
|
2406
3879
|
if (options.output) {
|
|
2407
|
-
const { writeFileSync:
|
|
2408
|
-
|
|
3880
|
+
const { writeFileSync: writeFileSync3 } = await import("fs");
|
|
3881
|
+
writeFileSync3(options.output, content);
|
|
2409
3882
|
output.success(`Exported ${String(tracesList.length)} traces to ${options.output}`);
|
|
2410
3883
|
} else {
|
|
2411
3884
|
process.stdout.write(content);
|
|
@@ -2418,13 +3891,15 @@ ${chalk5.dim("Exports to stdout by default. Use --output to save to a file.")}
|
|
|
2418
3891
|
}
|
|
2419
3892
|
|
|
2420
3893
|
// src/commands/integrate.ts
|
|
3894
|
+
init_config();
|
|
2421
3895
|
import { Command as Command5 } from "commander";
|
|
2422
|
-
import
|
|
2423
|
-
import { writeFileSync as
|
|
3896
|
+
import chalk9 from "chalk";
|
|
3897
|
+
import { writeFileSync as writeFileSync3, existsSync as existsSync11 } from "fs";
|
|
2424
3898
|
import { execSync } from "child_process";
|
|
3899
|
+
init_errors();
|
|
2425
3900
|
|
|
2426
3901
|
// src/lib/integrations/mastra.ts
|
|
2427
|
-
import { readFileSync as
|
|
3902
|
+
import { readFileSync as readFileSync5, existsSync as existsSync5 } from "fs";
|
|
2428
3903
|
function renderPrerequisites(config) {
|
|
2429
3904
|
const keyPreview = config.apiKey.slice(0, 8) + "..." + config.apiKey.slice(-4);
|
|
2430
3905
|
return `✓ MUTAGENT_API_KEY: ${keyPreview}
|
|
@@ -2438,14 +3913,14 @@ var mastraIntegration = {
|
|
|
2438
3913
|
const files = ["mastra.config.ts", "mastra.config.js"];
|
|
2439
3914
|
const found = [];
|
|
2440
3915
|
for (const file of files) {
|
|
2441
|
-
if (
|
|
3916
|
+
if (existsSync5(file)) {
|
|
2442
3917
|
found.push(file);
|
|
2443
3918
|
}
|
|
2444
3919
|
}
|
|
2445
3920
|
let hasMastraDep = false;
|
|
2446
|
-
if (
|
|
3921
|
+
if (existsSync5("package.json")) {
|
|
2447
3922
|
try {
|
|
2448
|
-
const pkg = JSON.parse(
|
|
3923
|
+
const pkg = JSON.parse(readFileSync5("package.json", "utf-8"));
|
|
2449
3924
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
2450
3925
|
hasMastraDep = "@mastra/core" in deps;
|
|
2451
3926
|
} catch {}
|
|
@@ -2620,15 +4095,15 @@ mutagent traces analyze <prompt-id>
|
|
|
2620
4095
|
};
|
|
2621
4096
|
|
|
2622
4097
|
// src/lib/integrations/langchain.ts
|
|
2623
|
-
import { readFileSync as
|
|
4098
|
+
import { readFileSync as readFileSync6, existsSync as existsSync6 } from "fs";
|
|
2624
4099
|
var langchainIntegration = {
|
|
2625
4100
|
name: "langchain",
|
|
2626
4101
|
description: "LangChain framework",
|
|
2627
4102
|
async detect() {
|
|
2628
4103
|
let hasLangchain = false;
|
|
2629
|
-
if (
|
|
4104
|
+
if (existsSync6("package.json")) {
|
|
2630
4105
|
try {
|
|
2631
|
-
const pkg = JSON.parse(
|
|
4106
|
+
const pkg = JSON.parse(readFileSync6("package.json", "utf-8"));
|
|
2632
4107
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
2633
4108
|
hasLangchain = "langchain" in deps || "@langchain/core" in deps;
|
|
2634
4109
|
} catch {}
|
|
@@ -2760,15 +4235,15 @@ mutagent traces analyze <prompt-id>
|
|
|
2760
4235
|
};
|
|
2761
4236
|
|
|
2762
4237
|
// src/lib/integrations/langgraph.ts
|
|
2763
|
-
import { readFileSync as
|
|
4238
|
+
import { readFileSync as readFileSync7, existsSync as existsSync7 } from "fs";
|
|
2764
4239
|
var langgraphIntegration = {
|
|
2765
4240
|
name: "langgraph",
|
|
2766
4241
|
description: "LangGraph agent workflow framework",
|
|
2767
4242
|
async detect() {
|
|
2768
4243
|
let hasLanggraph = false;
|
|
2769
|
-
if (
|
|
4244
|
+
if (existsSync7("package.json")) {
|
|
2770
4245
|
try {
|
|
2771
|
-
const pkg = JSON.parse(
|
|
4246
|
+
const pkg = JSON.parse(readFileSync7("package.json", "utf-8"));
|
|
2772
4247
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
2773
4248
|
hasLanggraph = "@langchain/langgraph" in deps;
|
|
2774
4249
|
} catch {}
|
|
@@ -2886,15 +4361,15 @@ mutagent traces analyze <node-prompt-id>
|
|
|
2886
4361
|
};
|
|
2887
4362
|
|
|
2888
4363
|
// src/lib/integrations/vercel-ai.ts
|
|
2889
|
-
import { readFileSync as
|
|
4364
|
+
import { readFileSync as readFileSync8, existsSync as existsSync8 } from "fs";
|
|
2890
4365
|
var vercelAiIntegration = {
|
|
2891
4366
|
name: "vercel-ai",
|
|
2892
4367
|
description: "Vercel AI SDK",
|
|
2893
4368
|
async detect() {
|
|
2894
4369
|
let hasAiSdk = false;
|
|
2895
|
-
if (
|
|
4370
|
+
if (existsSync8("package.json")) {
|
|
2896
4371
|
try {
|
|
2897
|
-
const pkg = JSON.parse(
|
|
4372
|
+
const pkg = JSON.parse(readFileSync8("package.json", "utf-8"));
|
|
2898
4373
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
2899
4374
|
hasAiSdk = "ai" in deps;
|
|
2900
4375
|
} catch {}
|
|
@@ -3007,15 +4482,15 @@ mutagent traces export --prompt <prompt-id> --format json
|
|
|
3007
4482
|
};
|
|
3008
4483
|
|
|
3009
4484
|
// src/lib/integrations/openai.ts
|
|
3010
|
-
import { readFileSync as
|
|
4485
|
+
import { readFileSync as readFileSync9, existsSync as existsSync9 } from "fs";
|
|
3011
4486
|
var openaiIntegration = {
|
|
3012
4487
|
name: "openai",
|
|
3013
4488
|
description: "OpenAI SDK integration with automatic tracing",
|
|
3014
4489
|
async detect() {
|
|
3015
4490
|
let hasOpenAI = false;
|
|
3016
|
-
if (
|
|
4491
|
+
if (existsSync9("package.json")) {
|
|
3017
4492
|
try {
|
|
3018
|
-
const pkg = JSON.parse(
|
|
4493
|
+
const pkg = JSON.parse(readFileSync9("package.json", "utf-8"));
|
|
3019
4494
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
3020
4495
|
hasOpenAI = "openai" in deps;
|
|
3021
4496
|
} catch {}
|
|
@@ -3145,15 +4620,15 @@ mutagent traces export --format json
|
|
|
3145
4620
|
};
|
|
3146
4621
|
|
|
3147
4622
|
// src/lib/integrations/claude-code.ts
|
|
3148
|
-
import { readFileSync as
|
|
4623
|
+
import { readFileSync as readFileSync10, existsSync as existsSync10 } from "fs";
|
|
3149
4624
|
var claudeCodeIntegration = {
|
|
3150
4625
|
name: "claude-code",
|
|
3151
4626
|
description: "Claude Code SDK",
|
|
3152
4627
|
async detect() {
|
|
3153
4628
|
let hasClaudeCode = false;
|
|
3154
|
-
if (
|
|
4629
|
+
if (existsSync10("package.json")) {
|
|
3155
4630
|
try {
|
|
3156
|
-
const pkg = JSON.parse(
|
|
4631
|
+
const pkg = JSON.parse(readFileSync10("package.json", "utf-8"));
|
|
3157
4632
|
const deps = { ...pkg.dependencies, ...pkg.devDependencies };
|
|
3158
4633
|
hasClaudeCode = "@anthropic-ai/claude-code" in deps || "@anthropic-ai/sdk" in deps;
|
|
3159
4634
|
} catch {}
|
|
@@ -3465,16 +4940,16 @@ function getFrameworkMetadata(name) {
|
|
|
3465
4940
|
|
|
3466
4941
|
// src/commands/integrate.ts
|
|
3467
4942
|
function detectPackageManager() {
|
|
3468
|
-
if (
|
|
4943
|
+
if (existsSync11("bun.lockb") || existsSync11("bun.lock")) {
|
|
3469
4944
|
return "bun";
|
|
3470
4945
|
}
|
|
3471
|
-
if (
|
|
4946
|
+
if (existsSync11("pnpm-lock.yaml")) {
|
|
3472
4947
|
return "pnpm";
|
|
3473
4948
|
}
|
|
3474
|
-
if (
|
|
4949
|
+
if (existsSync11("yarn.lock")) {
|
|
3475
4950
|
return "yarn";
|
|
3476
4951
|
}
|
|
3477
|
-
if (
|
|
4952
|
+
if (existsSync11("package-lock.json")) {
|
|
3478
4953
|
return "npm";
|
|
3479
4954
|
}
|
|
3480
4955
|
try {
|
|
@@ -3524,12 +4999,12 @@ AI AGENT: Please determine the framework by:
|
|
|
3524
4999
|
function createIntegrateCommand() {
|
|
3525
5000
|
const integrate = new Command5("integrate").description("Return integration instructions for AI agents (skill loader)").addHelpText("after", `
|
|
3526
5001
|
Examples:
|
|
3527
|
-
${
|
|
3528
|
-
${
|
|
3529
|
-
${
|
|
3530
|
-
${
|
|
3531
|
-
${
|
|
3532
|
-
${
|
|
5002
|
+
${chalk9.dim("$")} mutagent integrate # Return exploration instructions for AI agents
|
|
5003
|
+
${chalk9.dim("$")} mutagent integrate langchain # Return @mutagent/langchain integration steps
|
|
5004
|
+
${chalk9.dim("$")} mutagent integrate openai # Return @mutagent/openai integration steps
|
|
5005
|
+
${chalk9.dim("$")} mutagent integrate vercel-ai --verify # Verify integration
|
|
5006
|
+
${chalk9.dim("$")} mutagent integrate langchain --output ./INTEGRATION.md
|
|
5007
|
+
${chalk9.dim("$")} mutagent integrate --list # List available frameworks
|
|
3533
5008
|
|
|
3534
5009
|
Frameworks: mastra, langchain, langgraph, vercel-ai, openai, claude-code, generic
|
|
3535
5010
|
|
|
@@ -3548,7 +5023,7 @@ It returns INSTRUCTIONS that AI agents execute - it does NOT auto-install packag
|
|
|
3548
5023
|
if (!frameworkArg) {
|
|
3549
5024
|
const explorationInstructions = generateExplorationInstructions();
|
|
3550
5025
|
if (options.output) {
|
|
3551
|
-
|
|
5026
|
+
writeFileSync3(options.output, explorationInstructions);
|
|
3552
5027
|
output.success(`Exploration instructions saved to ${options.output}`);
|
|
3553
5028
|
} else if (options.raw === true || isJson) {
|
|
3554
5029
|
process.stdout.write(explorationInstructions);
|
|
@@ -3596,7 +5071,7 @@ export MUTAGENT_API_KEY=${apiKey.substring(0, 12)}...
|
|
|
3596
5071
|
AI AGENT: Execute these steps using Bash for install, Write/Edit for code changes.
|
|
3597
5072
|
`;
|
|
3598
5073
|
if (options.output) {
|
|
3599
|
-
|
|
5074
|
+
writeFileSync3(options.output, instructions);
|
|
3600
5075
|
output.success(`Integration instructions saved to ${options.output}`);
|
|
3601
5076
|
} else if (options.raw === true || isJson) {
|
|
3602
5077
|
process.stdout.write(instructions);
|
|
@@ -3631,23 +5106,25 @@ AI AGENT: Execute these steps using Bash for install, Write/Edit for code change
|
|
|
3631
5106
|
}
|
|
3632
5107
|
|
|
3633
5108
|
// src/commands/agents.ts
|
|
5109
|
+
init_sdk_client();
|
|
3634
5110
|
import { Command as Command6 } from "commander";
|
|
3635
|
-
import
|
|
3636
|
-
import { readFileSync as
|
|
5111
|
+
import chalk10 from "chalk";
|
|
5112
|
+
import { readFileSync as readFileSync11 } from "fs";
|
|
5113
|
+
init_errors();
|
|
3637
5114
|
function createAgentsCommand() {
|
|
3638
5115
|
const agents = new Command6("agents").description("Manage AI agents").addHelpText("after", `
|
|
3639
5116
|
Examples:
|
|
3640
|
-
${
|
|
3641
|
-
${
|
|
3642
|
-
${
|
|
3643
|
-
${
|
|
3644
|
-
${
|
|
3645
|
-
${
|
|
3646
|
-
${
|
|
3647
|
-
${
|
|
3648
|
-
${
|
|
3649
|
-
${
|
|
3650
|
-
${
|
|
5117
|
+
${chalk10.dim("$")} mutagent agents list
|
|
5118
|
+
${chalk10.dim("$")} mutagent agents get <agent-id>
|
|
5119
|
+
${chalk10.dim("$")} mutagent agents create --file agent.json
|
|
5120
|
+
${chalk10.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "You are a code reviewer..."
|
|
5121
|
+
${chalk10.dim("$")} mutagent agents update <agent-id> --name "Updated Name"
|
|
5122
|
+
${chalk10.dim("$")} mutagent agents delete <agent-id> --force
|
|
5123
|
+
${chalk10.dim("$")} mutagent agents conversations list <agent-id>
|
|
5124
|
+
${chalk10.dim("$")} mutagent agents conversations get <agent-id> <conversation-id>
|
|
5125
|
+
${chalk10.dim("$")} mutagent agents conversations create <agent-id>
|
|
5126
|
+
${chalk10.dim("$")} mutagent agents conversations delete <agent-id> <conversation-id>
|
|
5127
|
+
${chalk10.dim("$")} mutagent agents conversations messages <agent-id> <conversation-id>
|
|
3651
5128
|
|
|
3652
5129
|
Subcommands:
|
|
3653
5130
|
list, get, create, update, delete
|
|
@@ -3655,9 +5132,9 @@ Subcommands:
|
|
|
3655
5132
|
`);
|
|
3656
5133
|
agents.command("list").description("List all agents").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").option("-n, --name <name>", "Filter by name").option("-s, --status <status>", "Filter by status (active, paused, archived)").addHelpText("after", `
|
|
3657
5134
|
Examples:
|
|
3658
|
-
${
|
|
3659
|
-
${
|
|
3660
|
-
${
|
|
5135
|
+
${chalk10.dim("$")} mutagent agents list
|
|
5136
|
+
${chalk10.dim("$")} mutagent agents list --status active
|
|
5137
|
+
${chalk10.dim("$")} mutagent agents list --name "reviewer" --json
|
|
3661
5138
|
`).action(async (options) => {
|
|
3662
5139
|
const isJson = getJsonFlag(agents);
|
|
3663
5140
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3677,7 +5154,11 @@ Examples:
|
|
|
3677
5154
|
}
|
|
3678
5155
|
const result = await client.listAgents(filters);
|
|
3679
5156
|
if (isJson) {
|
|
3680
|
-
|
|
5157
|
+
const withLinks = result.data.map((a) => ({
|
|
5158
|
+
...a,
|
|
5159
|
+
_links: agentLinks(a.id)
|
|
5160
|
+
}));
|
|
5161
|
+
output.output({ ...result, data: withLinks });
|
|
3681
5162
|
} else {
|
|
3682
5163
|
const formatted = result.data.map((a) => ({
|
|
3683
5164
|
id: a.id,
|
|
@@ -3686,7 +5167,8 @@ Examples:
|
|
|
3686
5167
|
model: a.model,
|
|
3687
5168
|
status: a.status,
|
|
3688
5169
|
isPublic: a.isPublic ? "Yes" : "No",
|
|
3689
|
-
updated: a.updatedAt ? new Date(a.updatedAt).toLocaleDateString() : "N/A"
|
|
5170
|
+
updated: a.updatedAt ? new Date(a.updatedAt).toLocaleDateString() : "N/A",
|
|
5171
|
+
url: agentLink(a.id)
|
|
3690
5172
|
}));
|
|
3691
5173
|
output.output(formatted);
|
|
3692
5174
|
}
|
|
@@ -3696,8 +5178,8 @@ Examples:
|
|
|
3696
5178
|
});
|
|
3697
5179
|
agents.command("get").description("Get agent details").argument("<id>", "Agent ID").addHelpText("after", `
|
|
3698
5180
|
Examples:
|
|
3699
|
-
${
|
|
3700
|
-
${
|
|
5181
|
+
${chalk10.dim("$")} mutagent agents get <agent-id>
|
|
5182
|
+
${chalk10.dim("$")} mutagent agents get <agent-id> --json
|
|
3701
5183
|
`).action(async (id) => {
|
|
3702
5184
|
const isJson = getJsonFlag(agents);
|
|
3703
5185
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3705,7 +5187,7 @@ Examples:
|
|
|
3705
5187
|
const client = getSDKClient();
|
|
3706
5188
|
const agent = await client.getAgent(id);
|
|
3707
5189
|
if (isJson) {
|
|
3708
|
-
output.output(agent);
|
|
5190
|
+
output.output({ ...agent, _links: agentLinks(agent.id) });
|
|
3709
5191
|
} else {
|
|
3710
5192
|
const formatted = {
|
|
3711
5193
|
id: agent.id,
|
|
@@ -3722,15 +5204,16 @@ Examples:
|
|
|
3722
5204
|
tags: agent.tags?.join(", ") ?? "None",
|
|
3723
5205
|
createdBy: agent.createdBy ?? "N/A",
|
|
3724
5206
|
createdAt: agent.createdAt ? new Date(agent.createdAt).toLocaleString() : "N/A",
|
|
3725
|
-
updatedAt: agent.updatedAt ? new Date(agent.updatedAt).toLocaleString() : "N/A"
|
|
5207
|
+
updatedAt: agent.updatedAt ? new Date(agent.updatedAt).toLocaleString() : "N/A",
|
|
5208
|
+
url: agentLink(agent.id)
|
|
3726
5209
|
};
|
|
3727
5210
|
output.output(formatted);
|
|
3728
5211
|
if (agent.systemPrompt) {
|
|
3729
|
-
console.log(
|
|
5212
|
+
console.log(chalk10.bold(`
|
|
3730
5213
|
System Prompt:`));
|
|
3731
|
-
console.log(
|
|
5214
|
+
console.log(chalk10.gray("─".repeat(60)));
|
|
3732
5215
|
console.log(agent.systemPrompt);
|
|
3733
|
-
console.log(
|
|
5216
|
+
console.log(chalk10.gray("─".repeat(60)));
|
|
3734
5217
|
}
|
|
3735
5218
|
}
|
|
3736
5219
|
} catch (error) {
|
|
@@ -3739,18 +5222,18 @@ System Prompt:`));
|
|
|
3739
5222
|
});
|
|
3740
5223
|
agents.command("create").description("Create a new agent").option("-f, --file <path>", "Create from JSON file").option("-n, --name <name>", "Agent name").option("-s, --slug <slug>", "Agent slug (URL-friendly identifier)").option("-p, --system-prompt <prompt>", "System prompt").option("-m, --model <model>", "Model (claude-sonnet-4-5, claude-opus-4-5, claude-haiku-4-5)").option("-d, --description <description>", "Agent description").addHelpText("after", `
|
|
3741
5224
|
Examples:
|
|
3742
|
-
${
|
|
3743
|
-
${
|
|
3744
|
-
${
|
|
5225
|
+
${chalk10.dim("$")} mutagent agents create --name "Code Reviewer" --slug code-reviewer --system-prompt "Review code for bugs"
|
|
5226
|
+
${chalk10.dim("$")} mutagent agents create --file agent.json
|
|
5227
|
+
${chalk10.dim("$")} mutagent agents create --name "Helper" --slug helper --system-prompt "..." --model claude-sonnet-4-5 --json
|
|
3745
5228
|
|
|
3746
|
-
${
|
|
5229
|
+
${chalk10.dim("Required: --name, --slug, and --system-prompt (or --file).")}
|
|
3747
5230
|
`).action(async (options) => {
|
|
3748
5231
|
const isJson = getJsonFlag(agents);
|
|
3749
5232
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3750
5233
|
try {
|
|
3751
5234
|
let data;
|
|
3752
5235
|
if (options.file) {
|
|
3753
|
-
const content =
|
|
5236
|
+
const content = readFileSync11(options.file, "utf-8");
|
|
3754
5237
|
data = JSON.parse(content);
|
|
3755
5238
|
} else if (options.name && options.slug && options.systemPrompt) {
|
|
3756
5239
|
data = {
|
|
@@ -3775,16 +5258,16 @@ ${chalk7.dim("Required: --name, --slug, and --system-prompt (or --file).")}
|
|
|
3775
5258
|
});
|
|
3776
5259
|
agents.command("update").description("Update an agent").argument("<id>", "Agent ID").option("-f, --file <path>", "Update from JSON file").option("-n, --name <name>", "New name").option("-p, --system-prompt <prompt>", "New system prompt").option("-m, --model <model>", "New model").option("-d, --description <description>", "New description").option("-s, --status <status>", "New status (active, paused, archived)").addHelpText("after", `
|
|
3777
5260
|
Examples:
|
|
3778
|
-
${
|
|
3779
|
-
${
|
|
3780
|
-
${
|
|
5261
|
+
${chalk10.dim("$")} mutagent agents update <id> --name "New Name"
|
|
5262
|
+
${chalk10.dim("$")} mutagent agents update <id> --status paused
|
|
5263
|
+
${chalk10.dim("$")} mutagent agents update <id> --file updated-agent.json --json
|
|
3781
5264
|
`).action(async (id, options) => {
|
|
3782
5265
|
const isJson = getJsonFlag(agents);
|
|
3783
5266
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
3784
5267
|
try {
|
|
3785
5268
|
let data = {};
|
|
3786
5269
|
if (options.file) {
|
|
3787
|
-
const content =
|
|
5270
|
+
const content = readFileSync11(options.file, "utf-8");
|
|
3788
5271
|
data = JSON.parse(content);
|
|
3789
5272
|
} else {
|
|
3790
5273
|
if (options.name)
|
|
@@ -3811,11 +5294,11 @@ Examples:
|
|
|
3811
5294
|
});
|
|
3812
5295
|
agents.command("delete").description("Delete an agent").argument("<id>", "Agent ID").option("--force", "Skip confirmation").addHelpText("after", `
|
|
3813
5296
|
Examples:
|
|
3814
|
-
${
|
|
3815
|
-
${
|
|
3816
|
-
${
|
|
5297
|
+
${chalk10.dim("$")} mutagent agents delete <id>
|
|
5298
|
+
${chalk10.dim("$")} mutagent agents delete <id> --force
|
|
5299
|
+
${chalk10.dim("$")} mutagent agents delete <id> --force --json
|
|
3817
5300
|
|
|
3818
|
-
${
|
|
5301
|
+
${chalk10.dim("Tip: Use --force to skip confirmation (required for non-interactive/CI usage).")}
|
|
3819
5302
|
`).action(async (id, options) => {
|
|
3820
5303
|
const isJson = getJsonFlag(agents);
|
|
3821
5304
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3842,14 +5325,14 @@ ${chalk7.dim("Tip: Use --force to skip confirmation (required for non-interactiv
|
|
|
3842
5325
|
});
|
|
3843
5326
|
const conversations = agents.command("conversations").description("Manage conversations for agents").addHelpText("after", `
|
|
3844
5327
|
Examples:
|
|
3845
|
-
${
|
|
3846
|
-
${
|
|
3847
|
-
${
|
|
5328
|
+
${chalk10.dim("$")} mutagent agents conversations list <agent-id>
|
|
5329
|
+
${chalk10.dim("$")} mutagent agents conversations create <agent-id>
|
|
5330
|
+
${chalk10.dim("$")} mutagent agents conversations messages <agent-id> <conversation-id>
|
|
3848
5331
|
`);
|
|
3849
5332
|
conversations.command("list").description("List conversations for an agent").argument("<agent-id>", "Agent ID").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").addHelpText("after", `
|
|
3850
5333
|
Examples:
|
|
3851
|
-
${
|
|
3852
|
-
${
|
|
5334
|
+
${chalk10.dim("$")} mutagent agents conversations list <agent-id>
|
|
5335
|
+
${chalk10.dim("$")} mutagent agents conversations list <agent-id> --limit 10 --json
|
|
3853
5336
|
`).action(async (agentId, options) => {
|
|
3854
5337
|
const isJson = getJsonFlag(agents);
|
|
3855
5338
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3881,8 +5364,8 @@ Examples:
|
|
|
3881
5364
|
});
|
|
3882
5365
|
conversations.command("get").description("Get conversation details").argument("<agent-id>", "Agent ID").argument("<conversation-id>", "Conversation ID").addHelpText("after", `
|
|
3883
5366
|
Examples:
|
|
3884
|
-
${
|
|
3885
|
-
${
|
|
5367
|
+
${chalk10.dim("$")} mutagent agents conversations get <agent-id> <conversation-id>
|
|
5368
|
+
${chalk10.dim("$")} mutagent agents conversations get <agent-id> <conversation-id> --json
|
|
3886
5369
|
`).action(async (agentId, conversationId) => {
|
|
3887
5370
|
const isJson = getJsonFlag(agents);
|
|
3888
5371
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3909,8 +5392,8 @@ Examples:
|
|
|
3909
5392
|
});
|
|
3910
5393
|
conversations.command("create").description("Start a new conversation with an agent").argument("<agent-id>", "Agent ID").option("-t, --title <title>", "Conversation title").addHelpText("after", `
|
|
3911
5394
|
Examples:
|
|
3912
|
-
${
|
|
3913
|
-
${
|
|
5395
|
+
${chalk10.dim("$")} mutagent agents conversations create <agent-id>
|
|
5396
|
+
${chalk10.dim("$")} mutagent agents conversations create <agent-id> --title "Debug session" --json
|
|
3914
5397
|
`).action(async (agentId, options) => {
|
|
3915
5398
|
const isJson = getJsonFlag(agents);
|
|
3916
5399
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3929,8 +5412,8 @@ Examples:
|
|
|
3929
5412
|
});
|
|
3930
5413
|
conversations.command("delete").description("Delete a conversation").argument("<agent-id>", "Agent ID").argument("<conversation-id>", "Conversation ID").option("--force", "Skip confirmation").addHelpText("after", `
|
|
3931
5414
|
Examples:
|
|
3932
|
-
${
|
|
3933
|
-
${
|
|
5415
|
+
${chalk10.dim("$")} mutagent agents conversations delete <agent-id> <conversation-id>
|
|
5416
|
+
${chalk10.dim("$")} mutagent agents conversations delete <agent-id> <conversation-id> --force --json
|
|
3934
5417
|
`).action(async (agentId, conversationId, options) => {
|
|
3935
5418
|
const isJson = getJsonFlag(agents);
|
|
3936
5419
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3957,8 +5440,8 @@ Examples:
|
|
|
3957
5440
|
});
|
|
3958
5441
|
conversations.command("messages").description("List messages in a conversation").argument("<agent-id>", "Agent ID").argument("<conversation-id>", "Conversation ID").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").addHelpText("after", `
|
|
3959
5442
|
Examples:
|
|
3960
|
-
${
|
|
3961
|
-
${
|
|
5443
|
+
${chalk10.dim("$")} mutagent agents conversations messages <agent-id> <conversation-id>
|
|
5444
|
+
${chalk10.dim("$")} mutagent agents conversations messages <agent-id> <conversation-id> --limit 20 --json
|
|
3962
5445
|
`).action(async (agentId, conversationId, options) => {
|
|
3963
5446
|
const isJson = getJsonFlag(agents);
|
|
3964
5447
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -3991,20 +5474,21 @@ Examples:
|
|
|
3991
5474
|
}
|
|
3992
5475
|
|
|
3993
5476
|
// src/commands/config.ts
|
|
5477
|
+
init_config();
|
|
3994
5478
|
import { Command as Command7 } from "commander";
|
|
3995
|
-
import
|
|
5479
|
+
import chalk11 from "chalk";
|
|
3996
5480
|
function createConfigCommand() {
|
|
3997
5481
|
const config = new Command7("config").description("Manage CLI configuration").addHelpText("after", `
|
|
3998
5482
|
Examples:
|
|
3999
|
-
${
|
|
4000
|
-
${
|
|
4001
|
-
${
|
|
4002
|
-
${
|
|
5483
|
+
${chalk11.dim("$")} mutagent config list
|
|
5484
|
+
${chalk11.dim("$")} mutagent config get endpoint
|
|
5485
|
+
${chalk11.dim("$")} mutagent config set workspace <workspace-id>
|
|
5486
|
+
${chalk11.dim("$")} mutagent config set org <org-id>
|
|
4003
5487
|
`);
|
|
4004
5488
|
config.command("list").description("List all configuration").addHelpText("after", `
|
|
4005
5489
|
Examples:
|
|
4006
|
-
${
|
|
4007
|
-
${
|
|
5490
|
+
${chalk11.dim("$")} mutagent config list
|
|
5491
|
+
${chalk11.dim("$")} mutagent config list --json
|
|
4008
5492
|
`).action(() => {
|
|
4009
5493
|
const isJson = getJsonFlag(config);
|
|
4010
5494
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4017,11 +5501,11 @@ Examples:
|
|
|
4017
5501
|
});
|
|
4018
5502
|
config.command("get").description("Get configuration value").argument("<key>", "Configuration key (apiKey, endpoint, format, timeout, defaultWorkspace, defaultOrganization)").addHelpText("after", `
|
|
4019
5503
|
Examples:
|
|
4020
|
-
${
|
|
4021
|
-
${
|
|
4022
|
-
${
|
|
5504
|
+
${chalk11.dim("$")} mutagent config get endpoint
|
|
5505
|
+
${chalk11.dim("$")} mutagent config get defaultWorkspace
|
|
5506
|
+
${chalk11.dim("$")} mutagent config get apiKey --json
|
|
4023
5507
|
|
|
4024
|
-
${
|
|
5508
|
+
${chalk11.dim("Keys: apiKey, endpoint, format, timeout, defaultWorkspace, defaultOrganization")}
|
|
4025
5509
|
`).action((key) => {
|
|
4026
5510
|
const isJson = getJsonFlag(config);
|
|
4027
5511
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4042,9 +5526,9 @@ ${chalk8.dim("Keys: apiKey, endpoint, format, timeout, defaultWorkspace, default
|
|
|
4042
5526
|
const set = new Command7("set").description("Set configuration value");
|
|
4043
5527
|
set.command("workspace").description("Set default workspace ID").argument("<id>", "Workspace ID to set as default").addHelpText("after", `
|
|
4044
5528
|
Examples:
|
|
4045
|
-
${
|
|
5529
|
+
${chalk11.dim("$")} mutagent config set workspace <workspace-id>
|
|
4046
5530
|
|
|
4047
|
-
${
|
|
5531
|
+
${chalk11.dim("Persists workspace ID so you don't need to pass headers on every request.")}
|
|
4048
5532
|
`).action((id) => {
|
|
4049
5533
|
const isJson = getJsonFlag(config);
|
|
4050
5534
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4057,9 +5541,9 @@ ${chalk8.dim("Persists workspace ID so you don't need to pass headers on every r
|
|
|
4057
5541
|
});
|
|
4058
5542
|
set.command("org").description("Set default organization ID").argument("<id>", "Organization ID to set as default").addHelpText("after", `
|
|
4059
5543
|
Examples:
|
|
4060
|
-
${
|
|
5544
|
+
${chalk11.dim("$")} mutagent config set org <org-id>
|
|
4061
5545
|
|
|
4062
|
-
${
|
|
5546
|
+
${chalk11.dim("Persists organization ID for org-scoped API keys.")}
|
|
4063
5547
|
`).action((id) => {
|
|
4064
5548
|
const isJson = getJsonFlag(config);
|
|
4065
5549
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4075,9 +5559,11 @@ ${chalk8.dim("Persists organization ID for org-scoped API keys.")}
|
|
|
4075
5559
|
}
|
|
4076
5560
|
|
|
4077
5561
|
// src/commands/playground.ts
|
|
5562
|
+
init_sdk_client();
|
|
4078
5563
|
import { Command as Command8 } from "commander";
|
|
4079
|
-
import
|
|
4080
|
-
import { readFileSync as
|
|
5564
|
+
import chalk12 from "chalk";
|
|
5565
|
+
import { readFileSync as readFileSync12 } from "fs";
|
|
5566
|
+
init_errors();
|
|
4081
5567
|
function parseSSELine(line) {
|
|
4082
5568
|
if (!line || line.startsWith(":")) {
|
|
4083
5569
|
return null;
|
|
@@ -4103,12 +5589,12 @@ function parsePromptStreamEvent(data) {
|
|
|
4103
5589
|
function createPlaygroundCommand() {
|
|
4104
5590
|
const playground = new Command8("playground").description("Execute and test prompts interactively").addHelpText("after", `
|
|
4105
5591
|
Examples:
|
|
4106
|
-
${
|
|
4107
|
-
${
|
|
4108
|
-
${
|
|
4109
|
-
${
|
|
4110
|
-
${
|
|
4111
|
-
${
|
|
5592
|
+
${chalk12.dim("$")} mutagent playground run <prompt-id> --input '{"name": "John"}'
|
|
5593
|
+
${chalk12.dim("$")} mutagent playground run <prompt-id> --file input.json
|
|
5594
|
+
${chalk12.dim("$")} mutagent playground run <prompt-id> --input '{}' --stream
|
|
5595
|
+
${chalk12.dim("$")} mutagent playground run <prompt-id> -i '{}' --model gpt-4-turbo
|
|
5596
|
+
${chalk12.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
|
|
5597
|
+
${chalk12.dim("$")} mutagent playground run <prompt-id> --messages '[{"role":"user","content":"Hi"}]'
|
|
4112
5598
|
|
|
4113
5599
|
Input Format:
|
|
4114
5600
|
The input must be a valid JSON object matching the prompt's input schema.
|
|
@@ -4124,10 +5610,10 @@ Streaming:
|
|
|
4124
5610
|
`);
|
|
4125
5611
|
playground.command("run").description("Execute a prompt with input variables").argument("<prompt-id>", "Prompt ID to execute (from: mutagent prompts list)").option("-i, --input <json>", "Input variables as JSON").option("-f, --file <path>", "Input from JSON file").option("-s, --stream", "Stream the response").option("-m, --model <model>", "Override model").option("--system <text>", "Set system prompt text").option("--human <text>", "Set human/user message text").option("--messages <json>", "Pass full messages array as JSON string").addHelpText("after", `
|
|
4126
5612
|
Examples:
|
|
4127
|
-
${
|
|
4128
|
-
${
|
|
4129
|
-
${
|
|
4130
|
-
${
|
|
5613
|
+
${chalk12.dim("$")} mutagent playground run <prompt-id> --input '{"name": "John"}'
|
|
5614
|
+
${chalk12.dim("$")} mutagent playground run <prompt-id> --file input.json --stream
|
|
5615
|
+
${chalk12.dim("$")} mutagent playground run <prompt-id> --system "You are helpful" --human "Hello"
|
|
5616
|
+
${chalk12.dim("$")} mutagent playground run <prompt-id> --input '{}' --model gpt-4-turbo --json
|
|
4131
5617
|
|
|
4132
5618
|
Input Methods (pick one):
|
|
4133
5619
|
--input '{"key":"value"}' Inline JSON variables
|
|
@@ -4135,7 +5621,7 @@ Input Methods (pick one):
|
|
|
4135
5621
|
--system/--human Quick system + user message
|
|
4136
5622
|
--messages '[...]' Full messages array
|
|
4137
5623
|
|
|
4138
|
-
${
|
|
5624
|
+
${chalk12.dim(`Hint: Test before evaluating: mutagent playground run <id> --input '{"key":"value"}'`)}
|
|
4139
5625
|
`).action(async (promptId, options) => {
|
|
4140
5626
|
const isJson = getJsonFlag(playground);
|
|
4141
5627
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4157,21 +5643,21 @@ ${chalk9.dim(`Hint: Test before evaluating: mutagent playground run <id> --input
|
|
|
4157
5643
|
}
|
|
4158
5644
|
});
|
|
4159
5645
|
} else {
|
|
4160
|
-
console.log(
|
|
5646
|
+
console.log(chalk12.bold(`
|
|
4161
5647
|
Execution Result:`));
|
|
4162
|
-
console.log(
|
|
4163
|
-
console.log(
|
|
5648
|
+
console.log(chalk12.gray("─".repeat(50)));
|
|
5649
|
+
console.log(chalk12.cyan("Output:"));
|
|
4164
5650
|
console.log(result.output);
|
|
4165
|
-
console.log(
|
|
4166
|
-
console.log(
|
|
4167
|
-
console.log(
|
|
5651
|
+
console.log(chalk12.gray("─".repeat(50)));
|
|
5652
|
+
console.log(chalk12.dim(`Model: ${result.model}`));
|
|
5653
|
+
console.log(chalk12.dim(`Execution Time: ${String(result.executionTimeMs)}ms`));
|
|
4168
5654
|
if (result.tokens) {
|
|
4169
|
-
console.log(
|
|
5655
|
+
console.log(chalk12.dim(`Tokens: ${String(result.tokens.prompt)} prompt + ${String(result.tokens.completion)} completion = ${String(result.tokens.total)} total`));
|
|
4170
5656
|
}
|
|
4171
5657
|
if (result.cost !== undefined) {
|
|
4172
|
-
console.log(
|
|
5658
|
+
console.log(chalk12.dim(`Cost: $${result.cost.toFixed(6)}`));
|
|
4173
5659
|
}
|
|
4174
|
-
console.log(
|
|
5660
|
+
console.log(chalk12.dim(`Playground: ${playgroundLink()}`));
|
|
4175
5661
|
console.log();
|
|
4176
5662
|
}
|
|
4177
5663
|
}
|
|
@@ -4238,7 +5724,7 @@ function parsePromptStyleInput(options) {
|
|
|
4238
5724
|
function getInputJson(options) {
|
|
4239
5725
|
if (options.file) {
|
|
4240
5726
|
try {
|
|
4241
|
-
return
|
|
5727
|
+
return readFileSync12(options.file, "utf-8");
|
|
4242
5728
|
} catch (err) {
|
|
4243
5729
|
throw new MutagentError("FILE_READ_ERROR", `Failed to read input file: ${options.file}`, err instanceof Error ? err.message : "Check the file path and permissions");
|
|
4244
5730
|
}
|
|
@@ -4271,9 +5757,9 @@ async function executeStreaming(client, promptId, input, model, isJson, output)
|
|
|
4271
5757
|
const decoder = new TextDecoder;
|
|
4272
5758
|
let buffer = "";
|
|
4273
5759
|
if (!isJson) {
|
|
4274
|
-
console.log(
|
|
5760
|
+
console.log(chalk12.bold(`
|
|
4275
5761
|
Streaming Output:`));
|
|
4276
|
-
console.log(
|
|
5762
|
+
console.log(chalk12.gray("─".repeat(50)));
|
|
4277
5763
|
}
|
|
4278
5764
|
try {
|
|
4279
5765
|
for (;; ) {
|
|
@@ -4312,15 +5798,15 @@ Streaming Output:`));
|
|
|
4312
5798
|
console.log(JSON.stringify({ type: "complete", result: event.result }));
|
|
4313
5799
|
} else {
|
|
4314
5800
|
console.log();
|
|
4315
|
-
console.log(
|
|
5801
|
+
console.log(chalk12.gray("─".repeat(50)));
|
|
4316
5802
|
if (event.result) {
|
|
4317
|
-
console.log(
|
|
4318
|
-
console.log(
|
|
5803
|
+
console.log(chalk12.dim(`Model: ${event.result.model}`));
|
|
5804
|
+
console.log(chalk12.dim(`Execution Time: ${String(event.result.executionTimeMs)}ms`));
|
|
4319
5805
|
if (event.result.tokens) {
|
|
4320
|
-
console.log(
|
|
5806
|
+
console.log(chalk12.dim(`Tokens: ${String(event.result.tokens.prompt)} prompt + ${String(event.result.tokens.completion)} completion = ${String(event.result.tokens.total)} total`));
|
|
4321
5807
|
}
|
|
4322
5808
|
if (event.result.cost !== undefined) {
|
|
4323
|
-
console.log(
|
|
5809
|
+
console.log(chalk12.dim(`Cost: $${event.result.cost.toFixed(6)}`));
|
|
4324
5810
|
}
|
|
4325
5811
|
}
|
|
4326
5812
|
console.log();
|
|
@@ -4343,13 +5829,15 @@ Streaming Output:`));
|
|
|
4343
5829
|
}
|
|
4344
5830
|
|
|
4345
5831
|
// src/commands/workspaces.ts
|
|
5832
|
+
init_sdk_client();
|
|
4346
5833
|
import { Command as Command9 } from "commander";
|
|
4347
|
-
import
|
|
5834
|
+
import chalk13 from "chalk";
|
|
5835
|
+
init_errors();
|
|
4348
5836
|
function createWorkspacesCommand() {
|
|
4349
5837
|
const workspaces = new Command9("workspaces").description("View workspaces (read-only)").addHelpText("after", `
|
|
4350
5838
|
Examples:
|
|
4351
|
-
${
|
|
4352
|
-
${
|
|
5839
|
+
${chalk13.dim("$")} mutagent workspaces list
|
|
5840
|
+
${chalk13.dim("$")} mutagent workspaces get <workspace-id>
|
|
4353
5841
|
|
|
4354
5842
|
Subcommands:
|
|
4355
5843
|
list, get
|
|
@@ -4358,8 +5846,8 @@ Note: Workspace management (create, update, delete) is available in the Admin Pa
|
|
|
4358
5846
|
`);
|
|
4359
5847
|
workspaces.command("list").description("List all workspaces").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").addHelpText("after", `
|
|
4360
5848
|
Examples:
|
|
4361
|
-
${
|
|
4362
|
-
${
|
|
5849
|
+
${chalk13.dim("$")} mutagent workspaces list
|
|
5850
|
+
${chalk13.dim("$")} mutagent workspaces list --limit 10 --json
|
|
4363
5851
|
`).action(async (options) => {
|
|
4364
5852
|
const isJson = getJsonFlag(workspaces);
|
|
4365
5853
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4373,14 +5861,19 @@ Examples:
|
|
|
4373
5861
|
}
|
|
4374
5862
|
const result = await client.listWorkspaces(filters);
|
|
4375
5863
|
if (isJson) {
|
|
4376
|
-
|
|
5864
|
+
const withLinks = result.data.map((w) => ({
|
|
5865
|
+
...w,
|
|
5866
|
+
_links: workspaceLinks(w.id)
|
|
5867
|
+
}));
|
|
5868
|
+
output.output({ ...result, data: withLinks });
|
|
4377
5869
|
} else {
|
|
4378
5870
|
const formatted = result.data.map((w) => ({
|
|
4379
5871
|
id: w.id,
|
|
4380
5872
|
name: w.name,
|
|
4381
5873
|
slug: w.slug,
|
|
4382
5874
|
isDefault: w.isDefault ? "Yes" : "No",
|
|
4383
|
-
updated: w.updatedAt ? new Date(w.updatedAt).toLocaleDateString() : "N/A"
|
|
5875
|
+
updated: w.updatedAt ? new Date(w.updatedAt).toLocaleDateString() : "N/A",
|
|
5876
|
+
url: workspaceLink(w.id)
|
|
4384
5877
|
}));
|
|
4385
5878
|
output.output(formatted);
|
|
4386
5879
|
}
|
|
@@ -4390,8 +5883,8 @@ Examples:
|
|
|
4390
5883
|
});
|
|
4391
5884
|
workspaces.command("get").description("Get workspace details").argument("<id>", "Workspace ID").addHelpText("after", `
|
|
4392
5885
|
Examples:
|
|
4393
|
-
${
|
|
4394
|
-
${
|
|
5886
|
+
${chalk13.dim("$")} mutagent workspaces get <workspace-id>
|
|
5887
|
+
${chalk13.dim("$")} mutagent workspaces get <workspace-id> --json
|
|
4395
5888
|
`).action(async (id) => {
|
|
4396
5889
|
const isJson = getJsonFlag(workspaces);
|
|
4397
5890
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4399,7 +5892,7 @@ Examples:
|
|
|
4399
5892
|
const client = getSDKClient();
|
|
4400
5893
|
const workspace = await client.getWorkspace(id);
|
|
4401
5894
|
if (isJson) {
|
|
4402
|
-
output.output(workspace);
|
|
5895
|
+
output.output({ ...workspace, _links: workspaceLinks(workspace.id) });
|
|
4403
5896
|
} else {
|
|
4404
5897
|
const formatted = {
|
|
4405
5898
|
id: workspace.id,
|
|
@@ -4409,7 +5902,8 @@ Examples:
|
|
|
4409
5902
|
isDefault: workspace.isDefault ? "Yes" : "No",
|
|
4410
5903
|
createdBy: workspace.createdBy ?? "N/A",
|
|
4411
5904
|
createdAt: workspace.createdAt ? new Date(workspace.createdAt).toLocaleString() : "N/A",
|
|
4412
|
-
updatedAt: workspace.updatedAt ? new Date(workspace.updatedAt).toLocaleString() : "N/A"
|
|
5905
|
+
updatedAt: workspace.updatedAt ? new Date(workspace.updatedAt).toLocaleString() : "N/A",
|
|
5906
|
+
url: workspaceLink(workspace.id)
|
|
4413
5907
|
};
|
|
4414
5908
|
output.output(formatted);
|
|
4415
5909
|
}
|
|
@@ -4421,8 +5915,10 @@ Examples:
|
|
|
4421
5915
|
}
|
|
4422
5916
|
|
|
4423
5917
|
// src/commands/providers.ts
|
|
5918
|
+
init_sdk_client();
|
|
4424
5919
|
import { Command as Command10 } from "commander";
|
|
4425
|
-
import
|
|
5920
|
+
import chalk14 from "chalk";
|
|
5921
|
+
init_errors();
|
|
4426
5922
|
var VALID_PROVIDER_TYPES = [
|
|
4427
5923
|
"openai",
|
|
4428
5924
|
"anthropic",
|
|
@@ -4445,9 +5941,9 @@ function validateProviderType(type) {
|
|
|
4445
5941
|
function createProvidersCommand() {
|
|
4446
5942
|
const providers = new Command10("providers").description("View LLM providers (read-only)").addHelpText("after", `
|
|
4447
5943
|
Examples:
|
|
4448
|
-
${
|
|
4449
|
-
${
|
|
4450
|
-
${
|
|
5944
|
+
${chalk14.dim("$")} mutagent providers list
|
|
5945
|
+
${chalk14.dim("$")} mutagent providers get <provider-id>
|
|
5946
|
+
${chalk14.dim("$")} mutagent providers test <provider-id>
|
|
4451
5947
|
|
|
4452
5948
|
Provider Types:
|
|
4453
5949
|
openai, anthropic, google, azure, bedrock, cohere, mistral, groq, together, replicate, custom
|
|
@@ -4459,9 +5955,9 @@ Note: Provider management (create, update, delete) is available in the Admin Pan
|
|
|
4459
5955
|
`);
|
|
4460
5956
|
providers.command("list").description("List all providers").option("-l, --limit <n>", "Limit results", "50").option("-o, --offset <n>", "Offset for pagination").option("-t, --type <type>", "Filter by provider type").addHelpText("after", `
|
|
4461
5957
|
Examples:
|
|
4462
|
-
${
|
|
4463
|
-
${
|
|
4464
|
-
${
|
|
5958
|
+
${chalk14.dim("$")} mutagent providers list
|
|
5959
|
+
${chalk14.dim("$")} mutagent providers list --type openai
|
|
5960
|
+
${chalk14.dim("$")} mutagent providers list --json
|
|
4465
5961
|
`).action(async (options) => {
|
|
4466
5962
|
const isJson = getJsonFlag(providers);
|
|
4467
5963
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4478,7 +5974,11 @@ Examples:
|
|
|
4478
5974
|
}
|
|
4479
5975
|
const result = await client.listProviders(filters);
|
|
4480
5976
|
if (isJson) {
|
|
4481
|
-
|
|
5977
|
+
const withLinks = result.data.map((p) => ({
|
|
5978
|
+
...p,
|
|
5979
|
+
_links: providerLinks(p.id)
|
|
5980
|
+
}));
|
|
5981
|
+
output.output({ ...result, data: withLinks });
|
|
4482
5982
|
} else {
|
|
4483
5983
|
if (result.data.length === 0) {
|
|
4484
5984
|
output.info("No providers configured.");
|
|
@@ -4490,7 +5990,8 @@ Examples:
|
|
|
4490
5990
|
type: p.type,
|
|
4491
5991
|
baseUrl: p.baseUrl ?? "default",
|
|
4492
5992
|
active: p.isActive ? "Yes" : "No",
|
|
4493
|
-
updated: p.updatedAt ? new Date(p.updatedAt).toLocaleDateString() : "N/A"
|
|
5993
|
+
updated: p.updatedAt ? new Date(p.updatedAt).toLocaleDateString() : "N/A",
|
|
5994
|
+
url: providerLink(p.id)
|
|
4494
5995
|
}));
|
|
4495
5996
|
output.output(formatted);
|
|
4496
5997
|
}
|
|
@@ -4501,8 +6002,8 @@ Examples:
|
|
|
4501
6002
|
});
|
|
4502
6003
|
providers.command("get").description("Get provider details").argument("<id>", "Provider ID (from: mutagent providers list)").addHelpText("after", `
|
|
4503
6004
|
Examples:
|
|
4504
|
-
${
|
|
4505
|
-
${
|
|
6005
|
+
${chalk14.dim("$")} mutagent providers get <provider-id>
|
|
6006
|
+
${chalk14.dim("$")} mutagent providers get <provider-id> --json
|
|
4506
6007
|
`).action(async (id) => {
|
|
4507
6008
|
const isJson = getJsonFlag(providers);
|
|
4508
6009
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4510,7 +6011,7 @@ Examples:
|
|
|
4510
6011
|
const client = getSDKClient();
|
|
4511
6012
|
const provider = await client.getProvider(id);
|
|
4512
6013
|
if (isJson) {
|
|
4513
|
-
output.output(provider);
|
|
6014
|
+
output.output({ ...provider, _links: providerLinks(provider.id) });
|
|
4514
6015
|
} else {
|
|
4515
6016
|
const formatted = {
|
|
4516
6017
|
id: provider.id,
|
|
@@ -4520,7 +6021,8 @@ Examples:
|
|
|
4520
6021
|
isActive: provider.isActive ? "Yes" : "No",
|
|
4521
6022
|
createdBy: provider.createdBy ?? "N/A",
|
|
4522
6023
|
createdAt: provider.createdAt ? new Date(provider.createdAt).toLocaleString() : "N/A",
|
|
4523
|
-
updatedAt: provider.updatedAt ? new Date(provider.updatedAt).toLocaleString() : "N/A"
|
|
6024
|
+
updatedAt: provider.updatedAt ? new Date(provider.updatedAt).toLocaleString() : "N/A",
|
|
6025
|
+
url: providerLink(provider.id)
|
|
4524
6026
|
};
|
|
4525
6027
|
output.output(formatted);
|
|
4526
6028
|
}
|
|
@@ -4530,10 +6032,10 @@ Examples:
|
|
|
4530
6032
|
});
|
|
4531
6033
|
providers.command("test").description("Test provider connectivity").argument("<id>", "Provider ID (from: mutagent providers list)").addHelpText("after", `
|
|
4532
6034
|
Examples:
|
|
4533
|
-
${
|
|
4534
|
-
${
|
|
6035
|
+
${chalk14.dim("$")} mutagent providers test <provider-id>
|
|
6036
|
+
${chalk14.dim("$")} mutagent providers test <provider-id> --json
|
|
4535
6037
|
|
|
4536
|
-
${
|
|
6038
|
+
${chalk14.dim("Tests connectivity and lists available models for the provider.")}
|
|
4537
6039
|
`).action(async (id) => {
|
|
4538
6040
|
const isJson = getJsonFlag(providers);
|
|
4539
6041
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4544,13 +6046,13 @@ ${chalk11.dim("Tests connectivity and lists available models for the provider.")
|
|
|
4544
6046
|
}
|
|
4545
6047
|
const result = await client.testProvider(id);
|
|
4546
6048
|
if (isJson) {
|
|
4547
|
-
output.output(result);
|
|
6049
|
+
output.output({ ...result, _links: providerLinks(id) });
|
|
4548
6050
|
} else {
|
|
4549
6051
|
if (result.success) {
|
|
4550
6052
|
output.success(`Provider test passed (${String(result.responseTimeMs)}ms)`);
|
|
4551
|
-
console.log(
|
|
6053
|
+
console.log(chalk14.green(`Message: ${result.message}`));
|
|
4552
6054
|
if (result.availableModels && result.availableModels.length > 0) {
|
|
4553
|
-
console.log(
|
|
6055
|
+
console.log(chalk14.bold(`
|
|
4554
6056
|
Available Models:`));
|
|
4555
6057
|
result.availableModels.forEach((model) => {
|
|
4556
6058
|
console.log(` - ${model}`);
|
|
@@ -4559,7 +6061,7 @@ Available Models:`));
|
|
|
4559
6061
|
} else {
|
|
4560
6062
|
output.error(`Provider test failed: ${result.message}`);
|
|
4561
6063
|
if (result.error) {
|
|
4562
|
-
console.log(
|
|
6064
|
+
console.log(chalk14.red(`Error: ${result.error}`));
|
|
4563
6065
|
}
|
|
4564
6066
|
}
|
|
4565
6067
|
}
|
|
@@ -4571,12 +6073,14 @@ Available Models:`));
|
|
|
4571
6073
|
}
|
|
4572
6074
|
|
|
4573
6075
|
// src/commands/init.ts
|
|
6076
|
+
init_config();
|
|
4574
6077
|
import { Command as Command11 } from "commander";
|
|
4575
6078
|
import inquirer3 from "inquirer";
|
|
4576
|
-
import
|
|
4577
|
-
import { existsSync as
|
|
6079
|
+
import chalk15 from "chalk";
|
|
6080
|
+
import { existsSync as existsSync12, readFileSync as readFileSync13, writeFileSync as writeFileSync4 } from "fs";
|
|
4578
6081
|
import { execSync as execSync2 } from "child_process";
|
|
4579
|
-
import { join as
|
|
6082
|
+
import { join as join5 } from "path";
|
|
6083
|
+
init_errors();
|
|
4580
6084
|
var FRAMEWORK_DETECTION_MAP = {
|
|
4581
6085
|
"@mastra/core": {
|
|
4582
6086
|
name: "mastra",
|
|
@@ -4626,16 +6130,16 @@ var FRAMEWORK_DETECTION_MAP = {
|
|
|
4626
6130
|
}
|
|
4627
6131
|
};
|
|
4628
6132
|
function detectPackageManager2(cwd = process.cwd()) {
|
|
4629
|
-
if (
|
|
6133
|
+
if (existsSync12(join5(cwd, "bun.lockb")) || existsSync12(join5(cwd, "bun.lock"))) {
|
|
4630
6134
|
return "bun";
|
|
4631
6135
|
}
|
|
4632
|
-
if (
|
|
6136
|
+
if (existsSync12(join5(cwd, "pnpm-lock.yaml"))) {
|
|
4633
6137
|
return "pnpm";
|
|
4634
6138
|
}
|
|
4635
|
-
if (
|
|
6139
|
+
if (existsSync12(join5(cwd, "yarn.lock"))) {
|
|
4636
6140
|
return "yarn";
|
|
4637
6141
|
}
|
|
4638
|
-
if (
|
|
6142
|
+
if (existsSync12(join5(cwd, "package-lock.json"))) {
|
|
4639
6143
|
return "npm";
|
|
4640
6144
|
}
|
|
4641
6145
|
try {
|
|
@@ -4656,13 +6160,13 @@ function getInstallCommand2(pm, packages) {
|
|
|
4656
6160
|
return commands[pm];
|
|
4657
6161
|
}
|
|
4658
6162
|
function detectFrameworkFromPackageJson(cwd = process.cwd()) {
|
|
4659
|
-
const pkgPath =
|
|
4660
|
-
if (!
|
|
6163
|
+
const pkgPath = join5(cwd, "package.json");
|
|
6164
|
+
if (!existsSync12(pkgPath)) {
|
|
4661
6165
|
return null;
|
|
4662
6166
|
}
|
|
4663
6167
|
let pkg;
|
|
4664
6168
|
try {
|
|
4665
|
-
pkg = JSON.parse(
|
|
6169
|
+
pkg = JSON.parse(readFileSync13(pkgPath, "utf-8"));
|
|
4666
6170
|
} catch {
|
|
4667
6171
|
return null;
|
|
4668
6172
|
}
|
|
@@ -4678,23 +6182,23 @@ function detectFrameworkFromPackageJson(cwd = process.cwd()) {
|
|
|
4678
6182
|
return null;
|
|
4679
6183
|
}
|
|
4680
6184
|
function hasRcConfig(cwd = process.cwd()) {
|
|
4681
|
-
return
|
|
6185
|
+
return existsSync12(join5(cwd, ".mutagentrc.json"));
|
|
4682
6186
|
}
|
|
4683
6187
|
function writeRcConfig(config, cwd = process.cwd()) {
|
|
4684
|
-
const rcPath =
|
|
4685
|
-
|
|
6188
|
+
const rcPath = join5(cwd, ".mutagentrc.json");
|
|
6189
|
+
writeFileSync4(rcPath, JSON.stringify(config, null, 2) + `
|
|
4686
6190
|
`);
|
|
4687
6191
|
}
|
|
4688
6192
|
function createInitCommand() {
|
|
4689
6193
|
const init = new Command11("init").description("Initialize MutagenT in your project").option("--non-interactive", "Skip interactive prompts (defaults to CLI-only mode)").addHelpText("after", `
|
|
4690
6194
|
Examples:
|
|
4691
|
-
${
|
|
4692
|
-
${
|
|
6195
|
+
${chalk15.dim("$")} mutagent init # Interactive setup wizard
|
|
6196
|
+
${chalk15.dim("$")} mutagent init --non-interactive # CLI-only mode (no prompts)
|
|
4693
6197
|
|
|
4694
6198
|
Modes:
|
|
4695
|
-
${
|
|
4696
|
-
${
|
|
4697
|
-
${
|
|
6199
|
+
${chalk15.bold("Full scaffold")} Install SDK + integration package, create config, setup tracing
|
|
6200
|
+
${chalk15.bold("CLI-only")} Verify auth + create .mutagentrc.json with workspace/endpoint
|
|
6201
|
+
${chalk15.bold("Skip")} Exit without changes
|
|
4698
6202
|
`).action(async (options) => {
|
|
4699
6203
|
const isJson = getJsonFlag(init);
|
|
4700
6204
|
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
@@ -4863,15 +6367,259 @@ Modes:
|
|
|
4863
6367
|
return init;
|
|
4864
6368
|
}
|
|
4865
6369
|
|
|
6370
|
+
// src/commands/explore.ts
|
|
6371
|
+
import { Command as Command12 } from "commander";
|
|
6372
|
+
import chalk16 from "chalk";
|
|
6373
|
+
import { resolve as resolve3 } from "path";
|
|
6374
|
+
init_errors();
|
|
6375
|
+
function createExploreCommand() {
|
|
6376
|
+
const explore = new Command12("explore").description("Scan codebase for prompts, datasets, and MutagenT markers").option("-p, --path <dir>", "Directory to scan", ".").option("--depth <n>", "Max directory depth", "10").option("--include <glob>", "Include file pattern", "**/*.{ts,js,py,tsx,jsx}").option("--exclude <dirs>", "Comma-separated directories to exclude", "node_modules,dist,.git,build,.next,__pycache__,venv,.venv").option("--markers-only", "Only find existing MutagenT markers").addHelpText("after", `
|
|
6377
|
+
Examples:
|
|
6378
|
+
${chalk16.dim("$")} mutagent explore
|
|
6379
|
+
${chalk16.dim("$")} mutagent explore --path ./src
|
|
6380
|
+
${chalk16.dim("$")} mutagent explore --include "**/*.{ts,py}" --depth 5
|
|
6381
|
+
${chalk16.dim("$")} mutagent explore --markers-only
|
|
6382
|
+
${chalk16.dim("$")} mutagent explore --json
|
|
6383
|
+
|
|
6384
|
+
Detection modes:
|
|
6385
|
+
${chalk16.dim("Heuristic")} Template variables ({{var}}), prompt constants, schema definitions
|
|
6386
|
+
${chalk16.dim("Marker")} MutagenT:START/END comment markers from previous uploads
|
|
6387
|
+
|
|
6388
|
+
${chalk16.dim("Results are saved to .mutagent/mutation-context.md for use by other commands.")}
|
|
6389
|
+
`).action((options) => {
|
|
6390
|
+
const isJson = getJsonFlag(explore);
|
|
6391
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6392
|
+
try {
|
|
6393
|
+
const scanPath = resolve3(options.path ?? ".");
|
|
6394
|
+
const depth = parseInt(options.depth ?? "10", 10);
|
|
6395
|
+
const extensions = parseExtensions(options.include ?? "**/*.{ts,js,py,tsx,jsx}");
|
|
6396
|
+
const excludeDirs = parseExcludeDirs(options.exclude ?? "node_modules,dist,.git,build,.next,__pycache__,venv,.venv");
|
|
6397
|
+
const markersOnly = options.markersOnly ?? false;
|
|
6398
|
+
const exploreOpts = {
|
|
6399
|
+
path: scanPath,
|
|
6400
|
+
depth,
|
|
6401
|
+
extensions,
|
|
6402
|
+
excludeDirs,
|
|
6403
|
+
markersOnly
|
|
6404
|
+
};
|
|
6405
|
+
if (!isJson) {
|
|
6406
|
+
console.log(chalk16.cyan(`
|
|
6407
|
+
Scanning ${scanPath}...
|
|
6408
|
+
`));
|
|
6409
|
+
}
|
|
6410
|
+
const result = exploreCodebase(exploreOpts);
|
|
6411
|
+
const ctx = MutationContext.load(scanPath);
|
|
6412
|
+
for (const prompt of result.prompts) {
|
|
6413
|
+
ctx.addDiscoveredPrompt(prompt.file, prompt.line, prompt.preview);
|
|
6414
|
+
}
|
|
6415
|
+
for (const dataset of result.datasets) {
|
|
6416
|
+
ctx.addDiscoveredDataset(dataset.file, dataset.name, dataset.items);
|
|
6417
|
+
}
|
|
6418
|
+
ctx.save();
|
|
6419
|
+
if (isJson) {
|
|
6420
|
+
output.output({
|
|
6421
|
+
prompts: result.prompts,
|
|
6422
|
+
datasets: result.datasets,
|
|
6423
|
+
markers: result.markers,
|
|
6424
|
+
summary: {
|
|
6425
|
+
totalPrompts: result.prompts.length,
|
|
6426
|
+
totalDatasets: result.datasets.length,
|
|
6427
|
+
totalMarkers: result.markers.length
|
|
6428
|
+
},
|
|
6429
|
+
contextFile: ".mutagent/mutation-context.md"
|
|
6430
|
+
});
|
|
6431
|
+
} else {
|
|
6432
|
+
const totalFindings = result.prompts.length + result.datasets.length + result.markers.length;
|
|
6433
|
+
if (totalFindings === 0) {
|
|
6434
|
+
output.info("No prompts, datasets, or markers found.");
|
|
6435
|
+
console.log(chalk16.dim(`
|
|
6436
|
+
Tip: Create a prompt with template variables like {{input}} to get started.`));
|
|
6437
|
+
return;
|
|
6438
|
+
}
|
|
6439
|
+
if (result.prompts.length > 0) {
|
|
6440
|
+
console.log(chalk16.bold(` Prompts Found (${String(result.prompts.length)}):`));
|
|
6441
|
+
console.log();
|
|
6442
|
+
for (const p of result.prompts) {
|
|
6443
|
+
const reasonTag = chalk16.dim(`[${p.reason}]`);
|
|
6444
|
+
console.log(` ${chalk16.green(p.file)}:${chalk16.yellow(String(p.line))} ${reasonTag}`);
|
|
6445
|
+
console.log(` ${chalk16.dim(p.preview)}`);
|
|
6446
|
+
}
|
|
6447
|
+
console.log();
|
|
6448
|
+
}
|
|
6449
|
+
if (result.datasets.length > 0) {
|
|
6450
|
+
console.log(chalk16.bold(` Datasets Found (${String(result.datasets.length)}):`));
|
|
6451
|
+
console.log();
|
|
6452
|
+
for (const d of result.datasets) {
|
|
6453
|
+
console.log(` ${chalk16.green(d.file)} ${chalk16.dim(`(${String(d.items)} items)`)}`);
|
|
6454
|
+
}
|
|
6455
|
+
console.log();
|
|
6456
|
+
}
|
|
6457
|
+
if (result.markers.length > 0) {
|
|
6458
|
+
console.log(chalk16.bold(` MutagenT Markers (${String(result.markers.length)}):`));
|
|
6459
|
+
console.log();
|
|
6460
|
+
for (const m of result.markers) {
|
|
6461
|
+
const idPart = m.platformId ? chalk16.cyan(` id=${m.platformId}`) : "";
|
|
6462
|
+
console.log(` ${chalk16.green(m.file)}:${chalk16.yellow(String(m.line))} ${chalk16.magenta(m.type)}${idPart}`);
|
|
6463
|
+
}
|
|
6464
|
+
console.log();
|
|
6465
|
+
}
|
|
6466
|
+
console.log(chalk16.dim(" ─────────────────────────────────"));
|
|
6467
|
+
console.log(` ${chalk16.bold("Summary:")} ${String(result.prompts.length)} prompts, ${String(result.datasets.length)} datasets, ${String(result.markers.length)} markers`);
|
|
6468
|
+
console.log(chalk16.dim(` Saved to .mutagent/mutation-context.md`));
|
|
6469
|
+
console.log();
|
|
6470
|
+
}
|
|
6471
|
+
} catch (error) {
|
|
6472
|
+
handleError(error, isJson);
|
|
6473
|
+
}
|
|
6474
|
+
});
|
|
6475
|
+
return explore;
|
|
6476
|
+
}
|
|
6477
|
+
|
|
6478
|
+
// src/commands/skills.ts
|
|
6479
|
+
import { Command as Command13 } from "commander";
|
|
6480
|
+
import chalk17 from "chalk";
|
|
6481
|
+
import { existsSync as existsSync13, mkdirSync as mkdirSync3, writeFileSync as writeFileSync5 } from "fs";
|
|
6482
|
+
import { join as join6 } from "path";
|
|
6483
|
+
var SKILL_FRONTMATTER = `---
|
|
6484
|
+
name: mutagent-cli
|
|
6485
|
+
description: |
|
|
6486
|
+
MutagenT CLI - AI Prompt Optimization Platform CLI.
|
|
6487
|
+
Guides coding agents through prompt upload, evaluation creation,
|
|
6488
|
+
dataset curation, optimization, and framework integration.
|
|
6489
|
+
Triggers: "mutagent", "optimize prompt", "upload prompt", "integrate tracing",
|
|
6490
|
+
"create evaluation", "upload dataset", "mutagent cli".
|
|
6491
|
+
---`;
|
|
6492
|
+
var SKILL_BODY = `# MutagenT CLI Skill
|
|
6493
|
+
|
|
6494
|
+
## Quick Reference
|
|
6495
|
+
Run \`mutagent --help\` for full command list and examples.
|
|
6496
|
+
|
|
6497
|
+
## Post-Onboarding Decision Tree
|
|
6498
|
+
|
|
6499
|
+
After \`mutagent auth login\`, the user lands in one of 3 paths:
|
|
6500
|
+
|
|
6501
|
+
### Path A: Guided Integration (Tracing)
|
|
6502
|
+
1. \`mutagent explore\` — Map codebase, discover prompts/agents
|
|
6503
|
+
2. User selects framework from detected or asks
|
|
6504
|
+
3. \`mutagent integrate <framework>\` — Get integration instructions
|
|
6505
|
+
4. Apply tracing code to user's codebase
|
|
6506
|
+
5. Verify with test run
|
|
6507
|
+
|
|
6508
|
+
### Path B: Free-flow Optimization
|
|
6509
|
+
1. \`mutagent explore\` — Map codebase, discover prompts
|
|
6510
|
+
2. User selects prompts to upload
|
|
6511
|
+
3. \`mutagent prompts create --data '{...}'\` — Upload with REQUIRED outputSchema
|
|
6512
|
+
4. Guided eval creator OR \`mutagent prompts evaluation create\`
|
|
6513
|
+
5. \`mutagent prompts dataset add\` — Upload/curate datasets (named)
|
|
6514
|
+
6. \`mutagent prompts optimize start\` — Run optimization
|
|
6515
|
+
7. Review scorecard → Apply/Reject optimized prompt
|
|
6516
|
+
|
|
6517
|
+
### Path C: Manual
|
|
6518
|
+
User uses CLI commands directly. Run \`mutagent --help\`.
|
|
6519
|
+
|
|
6520
|
+
## State Tracking
|
|
6521
|
+
- \`.mutagent/mutation-context.md\` — Local context file tracking all uploads
|
|
6522
|
+
- \`mutagent auth status\` — Shows onboarding completion + integration state
|
|
6523
|
+
- Comment markers (\`// MutagenT:START ... // MutagenT:END\`) in source files
|
|
6524
|
+
|
|
6525
|
+
## Wireframe Templates
|
|
6526
|
+
|
|
6527
|
+
### Active Operation Status Card
|
|
6528
|
+
\`\`\`
|
|
6529
|
+
┌─────────────────────────────────────────┐
|
|
6530
|
+
│ ⚡ MutagenT: Optimizing │
|
|
6531
|
+
│ ─────────────────────────────────────── │
|
|
6532
|
+
│ Prompt: "email-summarizer" (ID: 42) │
|
|
6533
|
+
│ Dataset: "customer-emails" (150 items) │
|
|
6534
|
+
│ Status: Running — Iteration 3/10 │
|
|
6535
|
+
│ Score: 0.72 → 0.85 (+18%) │
|
|
6536
|
+
│ ████████████░░░░░░░░ 60% │
|
|
6537
|
+
│ │
|
|
6538
|
+
│ \uD83D\uDD17 View: https://app.mutagent.io/... │
|
|
6539
|
+
└─────────────────────────────────────────┘
|
|
6540
|
+
\`\`\`
|
|
6541
|
+
|
|
6542
|
+
### Optimization Scorecard
|
|
6543
|
+
\`\`\`
|
|
6544
|
+
┌─────────────────────────────────────────┐
|
|
6545
|
+
│ \uD83D\uDCCA Optimization Results │
|
|
6546
|
+
│ ─────────────────────────────────────── │
|
|
6547
|
+
│ BEFORE │
|
|
6548
|
+
│ System: "You are a helpful..." │
|
|
6549
|
+
│ Score: 0.62 │
|
|
6550
|
+
│ │
|
|
6551
|
+
│ AFTER │
|
|
6552
|
+
│ System: "You are an expert..." │
|
|
6553
|
+
│ Score: 0.91 (+47%) │
|
|
6554
|
+
│ │
|
|
6555
|
+
│ Iterations: 5 | Best: #4 │
|
|
6556
|
+
│ ─────────────────────────────────────── │
|
|
6557
|
+
│ Score Progression: │
|
|
6558
|
+
│ #1: 0.62 #2: 0.71 #3: 0.78 │
|
|
6559
|
+
│ #4: 0.91 #5: 0.89 │
|
|
6560
|
+
│ │
|
|
6561
|
+
│ [Apply] [Reject] [View Details] │
|
|
6562
|
+
│ │
|
|
6563
|
+
│ \uD83D\uDD17 Dashboard: https://app.mutagent... │
|
|
6564
|
+
│ \uD83D\uDD17 Optimizer: https://app.mutagent... │
|
|
6565
|
+
└─────────────────────────────────────────┘
|
|
6566
|
+
\`\`\`
|
|
6567
|
+
|
|
6568
|
+
## Evaluation Criteria Reminder
|
|
6569
|
+
Every evaluation MUST specify criteria targeting either:
|
|
6570
|
+
- Input variable fields (from inputSchema)
|
|
6571
|
+
- Output fields (from outputSchema / structured output)`;
|
|
6572
|
+
var SKILL_DIR = ".claude/skills/mutagent-cli";
|
|
6573
|
+
var SKILL_FILE = "SKILL.md";
|
|
6574
|
+
function createSkillsCommand() {
|
|
6575
|
+
const skills = new Command13("skills").description("Manage MutagenT CLI skills for coding agents");
|
|
6576
|
+
skills.command("install").description("Install MutagenT CLI skill for Claude Code").addHelpText("after", `
|
|
6577
|
+
Examples:
|
|
6578
|
+
${chalk17.dim("$")} mutagent skills install
|
|
6579
|
+
|
|
6580
|
+
This creates a Claude Code skill at .claude/skills/mutagent-cli/SKILL.md
|
|
6581
|
+
that teaches coding agents how to use the MutagenT CLI effectively.
|
|
6582
|
+
`).action((_options, cmd) => {
|
|
6583
|
+
const parentCmd = cmd.parent?.parent;
|
|
6584
|
+
const isJson = parentCmd ? getJsonFlag(parentCmd) : false;
|
|
6585
|
+
const output = new OutputFormatter(isJson ? "json" : "table");
|
|
6586
|
+
const skillDir = join6(process.cwd(), SKILL_DIR);
|
|
6587
|
+
const skillPath = join6(skillDir, SKILL_FILE);
|
|
6588
|
+
if (!existsSync13(skillDir)) {
|
|
6589
|
+
mkdirSync3(skillDir, { recursive: true });
|
|
6590
|
+
}
|
|
6591
|
+
const content = `${SKILL_FRONTMATTER}
|
|
6592
|
+
|
|
6593
|
+
${SKILL_BODY}
|
|
6594
|
+
`;
|
|
6595
|
+
writeFileSync5(skillPath, content, "utf-8");
|
|
6596
|
+
if (isJson) {
|
|
6597
|
+
output.output({
|
|
6598
|
+
installed: true,
|
|
6599
|
+
path: skillPath,
|
|
6600
|
+
name: "mutagent-cli"
|
|
6601
|
+
});
|
|
6602
|
+
} else {
|
|
6603
|
+
output.success(`Installed MutagenT CLI skill`);
|
|
6604
|
+
console.log(` ${chalk17.dim("Path:")} ${skillPath}`);
|
|
6605
|
+
console.log("");
|
|
6606
|
+
console.log(` ${chalk17.dim("This skill teaches coding agents how to use the MutagenT CLI.")}`);
|
|
6607
|
+
console.log(` ${chalk17.dim("It will be automatically loaded by Claude Code when relevant triggers match.")}`);
|
|
6608
|
+
}
|
|
6609
|
+
});
|
|
6610
|
+
return skills;
|
|
6611
|
+
}
|
|
6612
|
+
|
|
4866
6613
|
// src/bin/cli.ts
|
|
6614
|
+
init_config();
|
|
4867
6615
|
var cliVersion = "0.1.1";
|
|
4868
6616
|
try {
|
|
4869
6617
|
const __dirname2 = dirname(fileURLToPath(import.meta.url));
|
|
4870
|
-
const pkgPath =
|
|
4871
|
-
const pkg = JSON.parse(
|
|
6618
|
+
const pkgPath = join7(__dirname2, "..", "..", "package.json");
|
|
6619
|
+
const pkg = JSON.parse(readFileSync14(pkgPath, "utf-8"));
|
|
4872
6620
|
cliVersion = pkg.version ?? cliVersion;
|
|
4873
6621
|
} catch {}
|
|
4874
|
-
var program = new
|
|
6622
|
+
var program = new Command14;
|
|
4875
6623
|
program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimization platform
|
|
4876
6624
|
|
|
4877
6625
|
Documentation: https://docs.mutagent.io/cli
|
|
@@ -4880,44 +6628,44 @@ program.name("mutagent").description(`MutagenT CLI - AI-native prompt optimizati
|
|
|
4880
6628
|
showGlobalOptions: true
|
|
4881
6629
|
});
|
|
4882
6630
|
program.addHelpText("after", `
|
|
4883
|
-
${
|
|
4884
|
-
Export your API key: ${
|
|
4885
|
-
Or pass inline: ${
|
|
4886
|
-
Machine-readable output: ${
|
|
4887
|
-
Disable prompts: ${
|
|
4888
|
-
Set default workspace: ${
|
|
4889
|
-
Set default org: ${
|
|
6631
|
+
${chalk18.yellow("Non-Interactive Mode (CI/CD & Coding Agents):")}
|
|
6632
|
+
Export your API key: ${chalk18.green("export MUTAGENT_API_KEY=mt_your_key_here")}
|
|
6633
|
+
Or pass inline: ${chalk18.green("mutagent prompts list --api-key mt_your_key")}
|
|
6634
|
+
Machine-readable output: ${chalk18.green("mutagent prompts list --json")}
|
|
6635
|
+
Disable prompts: ${chalk18.green("mutagent prompts list --non-interactive")}
|
|
6636
|
+
Set default workspace: ${chalk18.green("mutagent config set workspace <workspace-id>")}
|
|
6637
|
+
Set default org: ${chalk18.green("mutagent config set org <org-id>")}
|
|
4890
6638
|
`);
|
|
4891
6639
|
program.addHelpText("after", `
|
|
4892
|
-
${
|
|
4893
|
-
${
|
|
6640
|
+
${chalk18.yellow("Workflows:")}
|
|
6641
|
+
${chalk18.bold("Evaluate → Optimize Loop:")}
|
|
4894
6642
|
1. mutagent prompts create --name "..." --raw-file prompt.txt
|
|
4895
6643
|
2. mutagent prompts dataset add <prompt-id> --name "..." --file data.json
|
|
4896
6644
|
3. mutagent prompts evaluation create <prompt-id> --name "..." --file criteria.json
|
|
4897
6645
|
4. mutagent prompts optimize start <prompt-id> --dataset <id> --max-iterations 3
|
|
4898
6646
|
|
|
4899
|
-
${
|
|
6647
|
+
${chalk18.bold("Quick Test:")}
|
|
4900
6648
|
mutagent playground run <prompt-id> --input '{"key":"value"}'
|
|
4901
6649
|
|
|
4902
|
-
${
|
|
4903
|
-
${
|
|
4904
|
-
${
|
|
4905
|
-
${
|
|
4906
|
-
${
|
|
6650
|
+
${chalk18.bold("Prerequisites for Optimization:")}
|
|
6651
|
+
${chalk18.green("✓")} Prompt with input/output parameters
|
|
6652
|
+
${chalk18.green("✓")} Dataset with items (input + expectedOutput pairs)
|
|
6653
|
+
${chalk18.green("✓")} Evaluation with criteria (field-level, input/output focused)
|
|
6654
|
+
${chalk18.dim("•")} LLM provider ${chalk18.dim("(only when server uses external providers)")}
|
|
4907
6655
|
`);
|
|
4908
6656
|
program.addHelpText("after", `
|
|
4909
|
-
${
|
|
4910
|
-
${
|
|
4911
|
-
${
|
|
4912
|
-
${
|
|
4913
|
-
${
|
|
4914
|
-
${
|
|
4915
|
-
${
|
|
4916
|
-
${
|
|
4917
|
-
${
|
|
4918
|
-
${!hasCredentials() ?
|
|
6657
|
+
${chalk18.cyan("┌─ AI AGENT INTEGRATION HINT ────────────────────────────────────────────────┐")}
|
|
6658
|
+
${chalk18.cyan("│")} ${chalk18.cyan("│")}
|
|
6659
|
+
${chalk18.cyan("│")} Frameworks: mastra, langchain, langgraph, vercel-ai, claude-code, generic ${chalk18.cyan("│")}
|
|
6660
|
+
${chalk18.cyan("│")} ${chalk18.cyan("│")}
|
|
6661
|
+
${chalk18.cyan("│")} Get integration guide: mutagent integrate <framework> ${chalk18.cyan("│")}
|
|
6662
|
+
${chalk18.cyan("│")} Verify setup: mutagent integrate <framework> --verify ${chalk18.cyan("│")}
|
|
6663
|
+
${chalk18.cyan("│")} Use --json for AI parsing: mutagent <command> --json ${chalk18.cyan("│")}
|
|
6664
|
+
${chalk18.cyan("│")} ${chalk18.cyan("│")}
|
|
6665
|
+
${chalk18.cyan("└────────────────────────────────────────────────────────────────────────────┘")}
|
|
6666
|
+
${!hasCredentials() ? chalk18.yellow(`
|
|
4919
6667
|
Warning: Not authenticated. Run: mutagent auth login --browser
|
|
4920
|
-
`) : ""}${!hasRcConfig() ?
|
|
6668
|
+
`) : ""}${!hasRcConfig() ? chalk18.green(`
|
|
4921
6669
|
Get started: mutagent init
|
|
4922
6670
|
`) : ""}`);
|
|
4923
6671
|
program.hook("preAction", (thisCommand) => {
|
|
@@ -4943,7 +6691,9 @@ program.addCommand(createAgentsCommand());
|
|
|
4943
6691
|
program.addCommand(createPlaygroundCommand());
|
|
4944
6692
|
program.addCommand(createWorkspacesCommand());
|
|
4945
6693
|
program.addCommand(createProvidersCommand());
|
|
6694
|
+
program.addCommand(createExploreCommand());
|
|
6695
|
+
program.addCommand(createSkillsCommand());
|
|
4946
6696
|
program.parse();
|
|
4947
6697
|
|
|
4948
|
-
//# debugId=
|
|
6698
|
+
//# debugId=769CE1E3092A705664756E2164756E21
|
|
4949
6699
|
//# sourceMappingURL=cli.js.map
|