@mutagent/cli 0.1.196 → 0.1.197
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cli.js +13 -602
- package/dist/bin/cli.js.map +9 -10
- package/dist/index.js +1 -566
- package/dist/index.js.map +5 -6
- package/package.json +1 -1
package/dist/bin/cli.js
CHANGED
|
@@ -317,125 +317,6 @@ var init_errors = __esm(() => {
|
|
|
317
317
|
};
|
|
318
318
|
});
|
|
319
319
|
|
|
320
|
-
// src/lib/scorecard-extraction.ts
|
|
321
|
-
function extractScorecardDetails(rawState, iterCtx) {
|
|
322
|
-
if (!iterCtx)
|
|
323
|
-
return {};
|
|
324
|
-
const gc = rawState.globalContext;
|
|
325
|
-
const gcCtx = gc?.context;
|
|
326
|
-
const beforeExec = gcCtx?.executions;
|
|
327
|
-
const beforeResults = beforeExec?.results ?? [];
|
|
328
|
-
const beforeById = new Map;
|
|
329
|
-
for (const r of beforeResults) {
|
|
330
|
-
const id = r.id;
|
|
331
|
-
if (id)
|
|
332
|
-
beforeById.set(id, r);
|
|
333
|
-
}
|
|
334
|
-
let originalScore;
|
|
335
|
-
if (beforeResults.length > 0) {
|
|
336
|
-
const sum = beforeResults.reduce((acc, r) => {
|
|
337
|
-
const eval_ = r.evaluation;
|
|
338
|
-
return acc + (eval_?.score ?? 0);
|
|
339
|
-
}, 0);
|
|
340
|
-
originalScore = sum / beforeResults.length;
|
|
341
|
-
}
|
|
342
|
-
const afterExec = iterCtx.executionResults;
|
|
343
|
-
const afterResults = afterExec?.executions ?? [];
|
|
344
|
-
const datasetResults = afterResults.length > 0 ? afterResults.map((r) => {
|
|
345
|
-
const id = r.id || "unknown";
|
|
346
|
-
const afterEval = r.evaluation;
|
|
347
|
-
const afterScore = afterEval?.score ?? 0;
|
|
348
|
-
const beforeResult = beforeById.get(id);
|
|
349
|
-
const beforeEval = beforeResult?.evaluation;
|
|
350
|
-
const beforeScore = beforeEval?.score;
|
|
351
|
-
return { id, beforeScore, afterScore };
|
|
352
|
-
}) : undefined;
|
|
353
|
-
const criteriaScores = extractCriteriaScores(beforeResults, afterResults);
|
|
354
|
-
const rawFailureModes = iterCtx.failureModes;
|
|
355
|
-
const failureModes = rawFailureModes?.categories && rawFailureModes.failures ? rawFailureModes.categories.map((category) => ({
|
|
356
|
-
category,
|
|
357
|
-
failures: (rawFailureModes.failures?.[category] ?? []).map((f) => ({
|
|
358
|
-
description: f.description ?? f.label,
|
|
359
|
-
summary: f.summary
|
|
360
|
-
}))
|
|
361
|
-
})) : undefined;
|
|
362
|
-
const rawMutations = iterCtx.mutations;
|
|
363
|
-
const mutations = rawMutations && rawMutations.length > 0 ? rawMutations.map((m) => ({
|
|
364
|
-
label: m.label ?? "Unknown mutation",
|
|
365
|
-
status: m.status ?? "pending",
|
|
366
|
-
priority: m.priority,
|
|
367
|
-
rationale: m.target?.rationale
|
|
368
|
-
})) : undefined;
|
|
369
|
-
const evaluationDetails = afterResults.length > 0 ? afterResults.map((r) => {
|
|
370
|
-
const id = r.id || "unknown";
|
|
371
|
-
const eval_ = r.evaluation;
|
|
372
|
-
const score = eval_?.score ?? 0;
|
|
373
|
-
const success = eval_?.success ?? false;
|
|
374
|
-
const metrics = eval_?.evaluations?.map((metric) => {
|
|
375
|
-
const criteria = metric.evaluationChecklist?.items?.map((item) => ({
|
|
376
|
-
name: item.evaluationParameter ?? item.criteria ?? "unknown",
|
|
377
|
-
score: item.llmScore ?? 0,
|
|
378
|
-
success: item.success ?? false
|
|
379
|
-
}));
|
|
380
|
-
return {
|
|
381
|
-
name: metric.name ?? "unknown",
|
|
382
|
-
score: metric.score ?? 0,
|
|
383
|
-
success: metric.success ?? false,
|
|
384
|
-
failureMode: metric.failureMode,
|
|
385
|
-
reasoning: metric.reasoning,
|
|
386
|
-
criteria: criteria && criteria.length > 0 ? criteria : undefined
|
|
387
|
-
};
|
|
388
|
-
});
|
|
389
|
-
return {
|
|
390
|
-
itemId: id,
|
|
391
|
-
score,
|
|
392
|
-
success,
|
|
393
|
-
metrics: metrics && metrics.length > 0 ? metrics : undefined
|
|
394
|
-
};
|
|
395
|
-
}) : undefined;
|
|
396
|
-
return {
|
|
397
|
-
originalScore,
|
|
398
|
-
criteriaScores,
|
|
399
|
-
datasetResults,
|
|
400
|
-
failureModes,
|
|
401
|
-
mutations,
|
|
402
|
-
evaluationDetails
|
|
403
|
-
};
|
|
404
|
-
}
|
|
405
|
-
function extractCriteriaScores(beforeResults, afterResults) {
|
|
406
|
-
const metricNames = new Set;
|
|
407
|
-
const beforeScores = new Map;
|
|
408
|
-
const afterScores = new Map;
|
|
409
|
-
for (const r of beforeResults) {
|
|
410
|
-
const eval_ = r.evaluation;
|
|
411
|
-
for (const m of eval_?.evaluations ?? []) {
|
|
412
|
-
const name = m.name ?? "unknown";
|
|
413
|
-
metricNames.add(name);
|
|
414
|
-
const existing = beforeScores.get(name) ?? [];
|
|
415
|
-
existing.push(m.score ?? 0);
|
|
416
|
-
beforeScores.set(name, existing);
|
|
417
|
-
}
|
|
418
|
-
}
|
|
419
|
-
for (const r of afterResults) {
|
|
420
|
-
const eval_ = r.evaluation;
|
|
421
|
-
for (const m of eval_?.evaluations ?? []) {
|
|
422
|
-
const name = m.name ?? "unknown";
|
|
423
|
-
metricNames.add(name);
|
|
424
|
-
const existing = afterScores.get(name) ?? [];
|
|
425
|
-
existing.push(m.score ?? 0);
|
|
426
|
-
afterScores.set(name, existing);
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
if (metricNames.size === 0)
|
|
430
|
-
return;
|
|
431
|
-
const avg = (arr) => arr.reduce((a, b) => a + b, 0) / arr.length;
|
|
432
|
-
return Array.from(metricNames).map((name) => ({
|
|
433
|
-
name,
|
|
434
|
-
before: beforeScores.has(name) ? avg(beforeScores.get(name) ?? []) : undefined,
|
|
435
|
-
after: afterScores.has(name) ? avg(afterScores.get(name) ?? []) : undefined
|
|
436
|
-
}));
|
|
437
|
-
}
|
|
438
|
-
|
|
439
320
|
// src/lib/sdk-client.ts
|
|
440
321
|
var exports_sdk_client = {};
|
|
441
322
|
__export(exports_sdk_client, {
|
|
@@ -546,416 +427,6 @@ class SDKClientWrapper {
|
|
|
546
427
|
}
|
|
547
428
|
return response.json();
|
|
548
429
|
}
|
|
549
|
-
async requestRaw(path, options) {
|
|
550
|
-
const optHeaders = options?.headers;
|
|
551
|
-
let extraHeaders = {};
|
|
552
|
-
if (optHeaders instanceof Headers) {
|
|
553
|
-
optHeaders.forEach((value, key) => {
|
|
554
|
-
extraHeaders[key] = value;
|
|
555
|
-
});
|
|
556
|
-
} else if (optHeaders) {
|
|
557
|
-
extraHeaders = optHeaders;
|
|
558
|
-
}
|
|
559
|
-
const tenancyHeaders = {};
|
|
560
|
-
if (this.workspaceId)
|
|
561
|
-
tenancyHeaders["x-workspace-id"] = this.workspaceId;
|
|
562
|
-
if (this.organizationId)
|
|
563
|
-
tenancyHeaders["x-organization-id"] = this.organizationId;
|
|
564
|
-
return fetch(`${this.endpoint}${path}`, {
|
|
565
|
-
...options,
|
|
566
|
-
headers: {
|
|
567
|
-
"x-api-key": this.apiKey,
|
|
568
|
-
"Content-Type": "application/json",
|
|
569
|
-
...tenancyHeaders,
|
|
570
|
-
...extraHeaders
|
|
571
|
-
}
|
|
572
|
-
});
|
|
573
|
-
}
|
|
574
|
-
async listPrompts() {
|
|
575
|
-
try {
|
|
576
|
-
const response = await this.sdk.prompt.listPrompts();
|
|
577
|
-
return response.result.data;
|
|
578
|
-
} catch (error) {
|
|
579
|
-
this.handleError(error);
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
async getPrompt(id) {
|
|
583
|
-
try {
|
|
584
|
-
const response = await this.sdk.prompt.getPrompt({ id: parseInt(id, 10) });
|
|
585
|
-
return response;
|
|
586
|
-
} catch (error) {
|
|
587
|
-
this.handleError(error);
|
|
588
|
-
}
|
|
589
|
-
}
|
|
590
|
-
async createPrompt(data) {
|
|
591
|
-
try {
|
|
592
|
-
const response = await this.sdk.prompt.createPrompt({
|
|
593
|
-
name: data.name ?? "",
|
|
594
|
-
description: data.description ?? undefined,
|
|
595
|
-
version: data.version ?? undefined,
|
|
596
|
-
isLatest: data.isLatest ?? undefined,
|
|
597
|
-
systemPrompt: data.systemPrompt ?? undefined,
|
|
598
|
-
humanPrompt: data.humanPrompt ?? undefined,
|
|
599
|
-
rawPrompt: data.rawPrompt ?? undefined,
|
|
600
|
-
inputSchema: data.inputSchema ?? undefined,
|
|
601
|
-
outputSchema: data.outputSchema ?? undefined,
|
|
602
|
-
metadata: data.metadata ?? undefined,
|
|
603
|
-
tags: data.tags ?? undefined
|
|
604
|
-
});
|
|
605
|
-
return response;
|
|
606
|
-
} catch (error) {
|
|
607
|
-
this.handleError(error);
|
|
608
|
-
}
|
|
609
|
-
}
|
|
610
|
-
async updatePrompt(id, data) {
|
|
611
|
-
try {
|
|
612
|
-
const body = {
|
|
613
|
-
description: data.description ?? undefined,
|
|
614
|
-
isLatest: data.isLatest ?? undefined,
|
|
615
|
-
systemPrompt: data.systemPrompt ?? undefined,
|
|
616
|
-
humanPrompt: data.humanPrompt ?? undefined,
|
|
617
|
-
rawPrompt: data.rawPrompt ?? undefined,
|
|
618
|
-
inputSchema: data.inputSchema,
|
|
619
|
-
outputSchema: data.outputSchema,
|
|
620
|
-
metadata: data.metadata,
|
|
621
|
-
tags: data.tags ?? undefined
|
|
622
|
-
};
|
|
623
|
-
if (data.name !== undefined) {
|
|
624
|
-
body.name = data.name;
|
|
625
|
-
}
|
|
626
|
-
const updateArgs = { id: parseInt(id, 10), body };
|
|
627
|
-
const response = await this.sdk.prompt.updatePrompt(updateArgs);
|
|
628
|
-
return response;
|
|
629
|
-
} catch (error) {
|
|
630
|
-
this.handleError(error);
|
|
631
|
-
}
|
|
632
|
-
}
|
|
633
|
-
async deletePrompt(id, options) {
|
|
634
|
-
try {
|
|
635
|
-
if (options?.force) {
|
|
636
|
-
await this.request(`/api/prompt/${id}?force=true`, { method: "DELETE" });
|
|
637
|
-
} else {
|
|
638
|
-
await this.sdk.prompt.deletePrompt({ id: parseInt(id, 10) });
|
|
639
|
-
}
|
|
640
|
-
} catch (error) {
|
|
641
|
-
this.handleError(error);
|
|
642
|
-
}
|
|
643
|
-
}
|
|
644
|
-
async getDataset(datasetId) {
|
|
645
|
-
try {
|
|
646
|
-
return await this.request(`/api/prompts/datasets/${String(datasetId)}`);
|
|
647
|
-
} catch (error) {
|
|
648
|
-
this.handleError(error);
|
|
649
|
-
}
|
|
650
|
-
}
|
|
651
|
-
async getDatasetItem(itemId) {
|
|
652
|
-
try {
|
|
653
|
-
return await this.request(`/api/prompts/dataset-items/${String(itemId)}`);
|
|
654
|
-
} catch (error) {
|
|
655
|
-
this.handleError(error);
|
|
656
|
-
}
|
|
657
|
-
}
|
|
658
|
-
async listDatasets(promptId) {
|
|
659
|
-
try {
|
|
660
|
-
const response = await this.request(`/api/prompt/${promptId}/datasets`);
|
|
661
|
-
return Array.isArray(response) ? response : response.data;
|
|
662
|
-
} catch (error) {
|
|
663
|
-
this.handleError(error);
|
|
664
|
-
}
|
|
665
|
-
}
|
|
666
|
-
async addDataset(promptId, fileContent, name) {
|
|
667
|
-
try {
|
|
668
|
-
let parsed;
|
|
669
|
-
try {
|
|
670
|
-
parsed = JSON.parse(fileContent);
|
|
671
|
-
} catch {
|
|
672
|
-
parsed = null;
|
|
673
|
-
}
|
|
674
|
-
let datasetName = name ?? `Dataset ${new Date().toISOString().slice(0, 16)}`;
|
|
675
|
-
let items;
|
|
676
|
-
if (parsed !== null && !Array.isArray(parsed) && typeof parsed === "object" && "name" in parsed) {
|
|
677
|
-
const meta = parsed;
|
|
678
|
-
datasetName = meta.name ?? datasetName;
|
|
679
|
-
if (Array.isArray(meta.items)) {
|
|
680
|
-
items = meta.items;
|
|
681
|
-
}
|
|
682
|
-
} else if (Array.isArray(parsed)) {
|
|
683
|
-
items = parsed;
|
|
684
|
-
}
|
|
685
|
-
const dataset = await this.request(`/api/prompt/${promptId}/datasets`, {
|
|
686
|
-
method: "POST",
|
|
687
|
-
body: JSON.stringify({
|
|
688
|
-
name: datasetName
|
|
689
|
-
})
|
|
690
|
-
});
|
|
691
|
-
let itemCount = 0;
|
|
692
|
-
if (items && items.length > 0) {
|
|
693
|
-
const mappedItems = items.map((item, index) => ({
|
|
694
|
-
input: item.input ?? item,
|
|
695
|
-
expectedOutput: item.expectedOutput ?? item.expected_output ?? undefined,
|
|
696
|
-
name: item.name ?? `Item ${String(index + 1)}`,
|
|
697
|
-
userFeedback: item.userFeedback,
|
|
698
|
-
systemFeedback: item.systemFeedback,
|
|
699
|
-
labels: item.labels,
|
|
700
|
-
metadata: item.metadata
|
|
701
|
-
}));
|
|
702
|
-
const missingExpectedOutputCount = mappedItems.filter((item) => item.expectedOutput === undefined).length;
|
|
703
|
-
const bulkResult = await this.requestRaw(`/api/prompts/datasets/${String(dataset.id)}/items/bulk`, {
|
|
704
|
-
method: "POST",
|
|
705
|
-
body: JSON.stringify({ items: mappedItems })
|
|
706
|
-
});
|
|
707
|
-
if (!bulkResult.ok) {
|
|
708
|
-
const bodyText = await bulkResult.text();
|
|
709
|
-
let parsed2;
|
|
710
|
-
try {
|
|
711
|
-
parsed2 = JSON.parse(bodyText);
|
|
712
|
-
} catch {}
|
|
713
|
-
const typed = parsed2;
|
|
714
|
-
if (typed && typed.kind === "bulk_insert_rollback") {
|
|
715
|
-
const constraint = typeof typed.failingConstraint === "string" ? typed.failingConstraint : "unknown";
|
|
716
|
-
const rowIndex = typeof typed.failingRowIndex === "number" ? typed.failingRowIndex : null;
|
|
717
|
-
throw new ApiError(bulkResult.status, `Bulk insert rolled back — constraint: ${constraint}` + (rowIndex !== null ? `, first failing row: ${String(rowIndex)}` : ""));
|
|
718
|
-
}
|
|
719
|
-
throw new ApiError(bulkResult.status, bodyText || "Bulk insert failed");
|
|
720
|
-
}
|
|
721
|
-
itemCount = mappedItems.length;
|
|
722
|
-
return { ...dataset, itemCount, missingExpectedOutputCount };
|
|
723
|
-
}
|
|
724
|
-
return { ...dataset, itemCount };
|
|
725
|
-
} catch (error) {
|
|
726
|
-
this.handleError(error);
|
|
727
|
-
}
|
|
728
|
-
}
|
|
729
|
-
async listDatasetItems(datasetId) {
|
|
730
|
-
try {
|
|
731
|
-
const response = await this.request(`/api/prompts/datasets/${datasetId}/items`);
|
|
732
|
-
return Array.isArray(response) ? response : response.data;
|
|
733
|
-
} catch (error) {
|
|
734
|
-
this.handleError(error);
|
|
735
|
-
}
|
|
736
|
-
}
|
|
737
|
-
async deleteDataset(_promptId, datasetId, options) {
|
|
738
|
-
try {
|
|
739
|
-
const query = options?.force ? "?force=true" : "";
|
|
740
|
-
await this.request(`/api/prompts/datasets/${datasetId}${query}`, { method: "DELETE" });
|
|
741
|
-
} catch (error) {
|
|
742
|
-
this.handleError(error);
|
|
743
|
-
}
|
|
744
|
-
}
|
|
745
|
-
async listEvaluations(promptId) {
|
|
746
|
-
try {
|
|
747
|
-
const response = await this.request(`/api/prompts/evaluations?promptId=${promptId}`);
|
|
748
|
-
return response.data;
|
|
749
|
-
} catch (error) {
|
|
750
|
-
this.handleError(error);
|
|
751
|
-
}
|
|
752
|
-
}
|
|
753
|
-
async getEvaluation(evaluationId) {
|
|
754
|
-
try {
|
|
755
|
-
return await this.request(`/api/prompts/evaluations/${evaluationId}`);
|
|
756
|
-
} catch (error) {
|
|
757
|
-
this.handleError(error);
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
async createEvaluation(promptId, data) {
|
|
761
|
-
try {
|
|
762
|
-
return await this.request("/api/prompts/evaluations", {
|
|
763
|
-
method: "POST",
|
|
764
|
-
body: JSON.stringify({
|
|
765
|
-
promptId: parseInt(promptId, 10),
|
|
766
|
-
name: data.name,
|
|
767
|
-
description: data.description,
|
|
768
|
-
evalConfig: data.evalConfig,
|
|
769
|
-
llmConfig: data.llmConfig,
|
|
770
|
-
tags: data.tags,
|
|
771
|
-
metadata: data.metadata
|
|
772
|
-
})
|
|
773
|
-
});
|
|
774
|
-
} catch (error) {
|
|
775
|
-
this.handleError(error);
|
|
776
|
-
}
|
|
777
|
-
}
|
|
778
|
-
async deleteEvaluation(evaluationId) {
|
|
779
|
-
try {
|
|
780
|
-
await this.request(`/api/prompts/evaluations/${evaluationId}`, { method: "DELETE" });
|
|
781
|
-
} catch (error) {
|
|
782
|
-
this.handleError(error);
|
|
783
|
-
}
|
|
784
|
-
}
|
|
785
|
-
async listOptimizations() {
|
|
786
|
-
try {
|
|
787
|
-
const response = await this.request("/api/optimizations");
|
|
788
|
-
if (Array.isArray(response))
|
|
789
|
-
return response;
|
|
790
|
-
const r = response;
|
|
791
|
-
if (Array.isArray(r.data))
|
|
792
|
-
return r.data;
|
|
793
|
-
return [];
|
|
794
|
-
} catch (error) {
|
|
795
|
-
this.handleError(error);
|
|
796
|
-
}
|
|
797
|
-
}
|
|
798
|
-
async listExperiments(filters) {
|
|
799
|
-
try {
|
|
800
|
-
const params = new URLSearchParams;
|
|
801
|
-
if (filters?.promptId !== undefined)
|
|
802
|
-
params.set("promptId", String(filters.promptId));
|
|
803
|
-
if (filters?.status)
|
|
804
|
-
params.set("status", filters.status);
|
|
805
|
-
if (filters?.limit !== undefined)
|
|
806
|
-
params.set("limit", String(filters.limit));
|
|
807
|
-
if (filters?.offset !== undefined)
|
|
808
|
-
params.set("offset", String(filters.offset));
|
|
809
|
-
const qs = params.toString();
|
|
810
|
-
const response = await this.request(`/api/prompts/experiments${qs ? `?${qs}` : ""}`);
|
|
811
|
-
if (Array.isArray(response))
|
|
812
|
-
return { experiments: response, total: response.length };
|
|
813
|
-
const r = response;
|
|
814
|
-
const experiments = r.experiments ?? r.data ?? [];
|
|
815
|
-
return { experiments, total: r.total ?? experiments.length };
|
|
816
|
-
} catch (error) {
|
|
817
|
-
this.handleError(error);
|
|
818
|
-
}
|
|
819
|
-
}
|
|
820
|
-
async createExperiment(data) {
|
|
821
|
-
try {
|
|
822
|
-
return await this.request("/api/prompts/experiments", {
|
|
823
|
-
method: "POST",
|
|
824
|
-
body: JSON.stringify(data)
|
|
825
|
-
});
|
|
826
|
-
} catch (error) {
|
|
827
|
-
this.handleError(error);
|
|
828
|
-
}
|
|
829
|
-
}
|
|
830
|
-
async getExperiment(id) {
|
|
831
|
-
try {
|
|
832
|
-
return await this.request(`/api/prompts/experiments/${id}`);
|
|
833
|
-
} catch (error) {
|
|
834
|
-
this.handleError(error);
|
|
835
|
-
}
|
|
836
|
-
}
|
|
837
|
-
async deleteExperiment(id) {
|
|
838
|
-
try {
|
|
839
|
-
await this.request(`/api/prompts/experiments/${id}`, { method: "DELETE" });
|
|
840
|
-
} catch (error) {
|
|
841
|
-
this.handleError(error);
|
|
842
|
-
}
|
|
843
|
-
}
|
|
844
|
-
async executeExperiment(id) {
|
|
845
|
-
try {
|
|
846
|
-
return await this.request(`/api/prompts/experiments/${id}/execute`, { method: "POST", body: JSON.stringify({}) });
|
|
847
|
-
} catch (error) {
|
|
848
|
-
this.handleError(error);
|
|
849
|
-
}
|
|
850
|
-
}
|
|
851
|
-
async playgroundEval(promptId, data) {
|
|
852
|
-
try {
|
|
853
|
-
return await this.request(`/api/prompt/${promptId}/playground/eval`, {
|
|
854
|
-
method: "POST",
|
|
855
|
-
body: JSON.stringify(data)
|
|
856
|
-
});
|
|
857
|
-
} catch (error) {
|
|
858
|
-
this.handleError(error);
|
|
859
|
-
}
|
|
860
|
-
}
|
|
861
|
-
async cancelOptimization(jobId) {
|
|
862
|
-
try {
|
|
863
|
-
return await this.request(`/api/optimization/${jobId}/cancel`, {
|
|
864
|
-
method: "POST"
|
|
865
|
-
});
|
|
866
|
-
} catch (error) {
|
|
867
|
-
this.handleError(error);
|
|
868
|
-
}
|
|
869
|
-
}
|
|
870
|
-
async startOptimization(promptId, datasetId, evaluationId, config) {
|
|
871
|
-
try {
|
|
872
|
-
return await this.request(`/api/prompt/${promptId}/optimize`, {
|
|
873
|
-
method: "POST",
|
|
874
|
-
body: JSON.stringify({
|
|
875
|
-
datasetId: parseInt(datasetId, 10),
|
|
876
|
-
evaluationId: parseInt(evaluationId, 10),
|
|
877
|
-
config: {
|
|
878
|
-
maxIterations: config?.maxIterations ?? 1,
|
|
879
|
-
targetScore: config?.targetScore ?? 0.8,
|
|
880
|
-
patience: config?.patience,
|
|
881
|
-
...config?.executionModel ? { executionModel: config.executionModel } : {},
|
|
882
|
-
...config?.evaluationModel ? { evaluationModel: config.evaluationModel } : {},
|
|
883
|
-
...config?.optimizationModel ? { optimizationModel: config.optimizationModel } : {},
|
|
884
|
-
...config?.providerId ? { executionProviderId: config.providerId } : {},
|
|
885
|
-
...config?.evalProviderId ? { evaluationProviderId: config.evalProviderId } : {},
|
|
886
|
-
...config?.optProviderId ? { optimizationProviderId: config.optProviderId } : {}
|
|
887
|
-
},
|
|
888
|
-
executionMode: "worker_loop"
|
|
889
|
-
})
|
|
890
|
-
});
|
|
891
|
-
} catch (error) {
|
|
892
|
-
this.handleError(error);
|
|
893
|
-
}
|
|
894
|
-
}
|
|
895
|
-
async getOptimizationStatus(jobId) {
|
|
896
|
-
try {
|
|
897
|
-
const raw = await this.request(`/api/optimization/${jobId}`);
|
|
898
|
-
const id = raw.id;
|
|
899
|
-
return {
|
|
900
|
-
...raw,
|
|
901
|
-
jobId: id ?? jobId,
|
|
902
|
-
progressPercent: typeof raw.progress === "number" ? raw.progress : 0
|
|
903
|
-
};
|
|
904
|
-
} catch (error) {
|
|
905
|
-
this.handleError(error);
|
|
906
|
-
}
|
|
907
|
-
}
|
|
908
|
-
async getOptimizationScorecard(jobId) {
|
|
909
|
-
try {
|
|
910
|
-
const res = await this.request(`/api/optimization/${jobId}/results`);
|
|
911
|
-
if (res.scorecard?.rendered) {
|
|
912
|
-
return { rendered: res.scorecard.rendered, data: res.scorecard.data ?? [] };
|
|
913
|
-
}
|
|
914
|
-
return null;
|
|
915
|
-
} catch {
|
|
916
|
-
return null;
|
|
917
|
-
}
|
|
918
|
-
}
|
|
919
|
-
async getOptimizationResults(jobId) {
|
|
920
|
-
try {
|
|
921
|
-
const job = await this.request(`/api/optimization/${jobId}`);
|
|
922
|
-
const progress = await this.request(`/api/optimization/${jobId}/progress`);
|
|
923
|
-
const prompt = await this.getPrompt(String(job.promptId ?? ""));
|
|
924
|
-
const statesRes = await this.request(`/api/optimization/${jobId}/states`).catch(() => ({ states: [] }));
|
|
925
|
-
const scorecardDigest = await this.getOptimizationScorecard(jobId);
|
|
926
|
-
const latestState = statesRes.states[statesRes.states.length - 1];
|
|
927
|
-
const rawState = latestState?.state ?? {};
|
|
928
|
-
const iterCtx = rawState.iterationContext ?? rawState.current?.context;
|
|
929
|
-
const basePromptObj = iterCtx?.basePrompt;
|
|
930
|
-
const currentPromptObj = iterCtx?.currentPrompt;
|
|
931
|
-
const mutatedPromptText = typeof currentPromptObj?.prompt === "string" ? currentPromptObj.prompt : undefined;
|
|
932
|
-
const originalPromptText = typeof basePromptObj?.prompt === "string" ? basePromptObj.prompt : undefined;
|
|
933
|
-
const extracted = extractScorecardDetails(rawState, iterCtx);
|
|
934
|
-
return {
|
|
935
|
-
job: {
|
|
936
|
-
id: job.id ?? jobId,
|
|
937
|
-
promptId: job.promptId ?? 0,
|
|
938
|
-
status: job.status ?? "unknown",
|
|
939
|
-
config: job.config
|
|
940
|
-
},
|
|
941
|
-
prompt,
|
|
942
|
-
bestScore: job.bestScore,
|
|
943
|
-
originalScore: extracted.originalScore,
|
|
944
|
-
iterationsCompleted: job.currentIteration,
|
|
945
|
-
scoreProgression: Array.isArray(progress.progression) ? progress.progression.map((p) => typeof p.score === "number" ? p.score : 0) : undefined,
|
|
946
|
-
mutatedPromptText,
|
|
947
|
-
originalPromptText,
|
|
948
|
-
criteriaScores: extracted.criteriaScores,
|
|
949
|
-
datasetResults: extracted.datasetResults,
|
|
950
|
-
failureModes: extracted.failureModes,
|
|
951
|
-
mutations: extracted.mutations,
|
|
952
|
-
evaluationDetails: extracted.evaluationDetails,
|
|
953
|
-
...scorecardDigest ? { scorecard: scorecardDigest } : {}
|
|
954
|
-
};
|
|
955
|
-
} catch (error) {
|
|
956
|
-
this.handleError(error);
|
|
957
|
-
}
|
|
958
|
-
}
|
|
959
430
|
async listAgents(filters) {
|
|
960
431
|
try {
|
|
961
432
|
const response = await this.sdk.agents.listAgents({
|
|
@@ -1063,42 +534,6 @@ class SDKClientWrapper {
|
|
|
1063
534
|
this.handleError(error);
|
|
1064
535
|
}
|
|
1065
536
|
}
|
|
1066
|
-
async executePrompt(promptId, input, options) {
|
|
1067
|
-
try {
|
|
1068
|
-
return await this.request(`/api/prompt/${promptId}/playground/call`, {
|
|
1069
|
-
method: "POST",
|
|
1070
|
-
body: JSON.stringify({ input, model: options?.model })
|
|
1071
|
-
});
|
|
1072
|
-
} catch (error) {
|
|
1073
|
-
this.handleError(error);
|
|
1074
|
-
}
|
|
1075
|
-
}
|
|
1076
|
-
async executePromptStream(promptId, input, options) {
|
|
1077
|
-
const response = await fetch(`${this.endpoint}/api/prompt/${promptId}/execute/stream`, {
|
|
1078
|
-
method: "POST",
|
|
1079
|
-
headers: {
|
|
1080
|
-
"x-api-key": this.apiKey,
|
|
1081
|
-
"Content-Type": "application/json",
|
|
1082
|
-
Accept: "text/event-stream"
|
|
1083
|
-
},
|
|
1084
|
-
body: JSON.stringify({ input, ...options })
|
|
1085
|
-
});
|
|
1086
|
-
if (!response.ok) {
|
|
1087
|
-
if (response.status === 401) {
|
|
1088
|
-
throw new AuthenticationError;
|
|
1089
|
-
}
|
|
1090
|
-
if (response.status === 403) {
|
|
1091
|
-
throw new ApiError(403, await response.text() || "Access denied");
|
|
1092
|
-
}
|
|
1093
|
-
const error = await response.text();
|
|
1094
|
-
const errorLower = error.toLowerCase();
|
|
1095
|
-
if (errorLower.includes("workspace") && (errorLower.includes("missing") || errorLower.includes("required")) || errorLower.includes("x-workspace-id")) {
|
|
1096
|
-
throw new WorkspaceContextError(error);
|
|
1097
|
-
}
|
|
1098
|
-
throw new ApiError(response.status, error);
|
|
1099
|
-
}
|
|
1100
|
-
return response;
|
|
1101
|
-
}
|
|
1102
537
|
async listProviders(filters) {
|
|
1103
538
|
try {
|
|
1104
539
|
const response = await this.sdk.providerConfigs.listProviders({
|
|
@@ -4011,7 +3446,7 @@ init_errors();
|
|
|
4011
3446
|
init_sdk_client();
|
|
4012
3447
|
var PROVIDERS_URL = "https://app.mutagent.io/settings/providers";
|
|
4013
3448
|
function createUsageCommand() {
|
|
4014
|
-
const usage = new Command8("usage").description("Show resource counts (
|
|
3449
|
+
const usage = new Command8("usage").description("Show account resource counts (providers, workspaces) and provider status").addHelpText("after", `
|
|
4015
3450
|
Examples:
|
|
4016
3451
|
${chalk14.dim("$")} mutagent usage
|
|
4017
3452
|
${chalk14.dim("$")} mutagent usage --json
|
|
@@ -4026,41 +3461,20 @@ Examples:
|
|
|
4026
3461
|
}
|
|
4027
3462
|
const spinner = await createSpinner("Fetching usage data...", isJson).start();
|
|
4028
3463
|
const client = await getSDKClient();
|
|
4029
|
-
const
|
|
4030
|
-
|
|
4031
|
-
|
|
4032
|
-
let evaluationCount = 0;
|
|
4033
|
-
if (promptCount > 0) {
|
|
4034
|
-
const results = await Promise.all(prompts.map(async (prompt) => {
|
|
4035
|
-
const id = String(prompt.id);
|
|
4036
|
-
const [datasets, evaluations] = await Promise.all([
|
|
4037
|
-
client.listDatasets(id).catch(() => []),
|
|
4038
|
-
client.listEvaluations(id).catch(() => [])
|
|
4039
|
-
]);
|
|
4040
|
-
return { datasets: datasets.length, evaluations: evaluations.length };
|
|
4041
|
-
}));
|
|
4042
|
-
for (const r of results) {
|
|
4043
|
-
datasetCount += r.datasets;
|
|
4044
|
-
evaluationCount += r.evaluations;
|
|
4045
|
-
}
|
|
4046
|
-
}
|
|
4047
|
-
const [optimizations, experiments] = await Promise.all([
|
|
4048
|
-
client.listOptimizations().catch(() => []),
|
|
4049
|
-
client.listExperiments().catch(() => [])
|
|
3464
|
+
const [providers, workspaces] = await Promise.all([
|
|
3465
|
+
client.listProviders().catch(() => ({ data: [], total: 0 })),
|
|
3466
|
+
client.listWorkspaces().catch(() => ({ data: [], total: 0 }))
|
|
4050
3467
|
]);
|
|
4051
|
-
const
|
|
4052
|
-
const
|
|
3468
|
+
const providerCount = providers.total || providers.data.length;
|
|
3469
|
+
const workspaceCount = workspaces.total || workspaces.data.length;
|
|
4053
3470
|
if (spinner && typeof spinner.stop === "function") {
|
|
4054
3471
|
spinner.stop();
|
|
4055
3472
|
}
|
|
4056
3473
|
if (isJson) {
|
|
4057
3474
|
output.output({
|
|
4058
3475
|
resources: {
|
|
4059
|
-
|
|
4060
|
-
|
|
4061
|
-
evaluations: evaluationCount,
|
|
4062
|
-
optimizations: optimizationCount,
|
|
4063
|
-
experiments: experimentCount
|
|
3476
|
+
providers: providerCount,
|
|
3477
|
+
workspaces: workspaceCount
|
|
4064
3478
|
},
|
|
4065
3479
|
_links: {
|
|
4066
3480
|
providers: PROVIDERS_URL,
|
|
@@ -4073,13 +3487,10 @@ Examples:
|
|
|
4073
3487
|
console.log(chalk14.dim("─".repeat(45)));
|
|
4074
3488
|
console.log("");
|
|
4075
3489
|
console.log(chalk14.bold("Resources:"));
|
|
4076
|
-
console.log(`
|
|
4077
|
-
console.log(`
|
|
4078
|
-
console.log(` Evaluations: ${chalk14.cyan(String(evaluationCount))}`);
|
|
4079
|
-
console.log(` Optimizations: ${chalk14.cyan(String(optimizationCount))}`);
|
|
4080
|
-
console.log(` Experiments: ${chalk14.cyan(String(experimentCount))}`);
|
|
3490
|
+
console.log(` Providers: ${chalk14.cyan(String(providerCount))}`);
|
|
3491
|
+
console.log(` Workspaces: ${chalk14.cyan(String(workspaceCount))}`);
|
|
4081
3492
|
console.log("");
|
|
4082
|
-
console.log(`
|
|
3493
|
+
console.log(` Manage providers: ${chalk14.underline(PROVIDERS_URL)}`);
|
|
4083
3494
|
console.log("");
|
|
4084
3495
|
}
|
|
4085
3496
|
} catch (error) {
|
|
@@ -5443,7 +4854,7 @@ if (process.env.CLI_VERSION) {
|
|
|
5443
4854
|
}
|
|
5444
4855
|
setCliVersion(cliVersion);
|
|
5445
4856
|
var program = new Command12;
|
|
5446
|
-
program.name("mutagent").description(`MutagenT CLI -
|
|
4857
|
+
program.name("mutagent").description(`MutagenT CLI - command-line client for the MutagenT platform
|
|
5447
4858
|
|
|
5448
4859
|
Documentation: https://docs.mutagent.io/cli
|
|
5449
4860
|
Dashboard: https://app.mutagent.io`).option("-v, --version", "Display the version number").option("--json", "Output results as JSON (for AI agents)").option("--api-key <key>", "MutagenT API key").option("--endpoint <url>", "MutagenT server endpoint").option("--non-interactive", "Disable interactive prompts (for CI/AI agents)").configureHelp({
|
|
@@ -5561,5 +4972,5 @@ if (isInteractive && !isSkillCommand) {
|
|
|
5561
4972
|
}
|
|
5562
4973
|
program.parse();
|
|
5563
4974
|
|
|
5564
|
-
//# debugId=
|
|
4975
|
+
//# debugId=9788EBFA942A533664756E2164756E21
|
|
5565
4976
|
//# sourceMappingURL=cli.js.map
|