@plumloom/cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +2 -0
- package/README.md +71 -0
- package/dist/actions/configured-runs.d.ts +25 -0
- package/dist/actions/configured-runs.d.ts.map +1 -0
- package/dist/actions/configured-runs.js +35 -0
- package/dist/actions/configured-runs.js.map +1 -0
- package/dist/actions/context.d.ts +5 -0
- package/dist/actions/context.d.ts.map +1 -0
- package/dist/actions/context.js +2 -0
- package/dist/actions/context.js.map +1 -0
- package/dist/actions/doctor.d.ts +46 -0
- package/dist/actions/doctor.d.ts.map +1 -0
- package/dist/actions/doctor.js +130 -0
- package/dist/actions/doctor.js.map +1 -0
- package/dist/actions/evaluations.d.ts +87 -0
- package/dist/actions/evaluations.d.ts.map +1 -0
- package/dist/actions/evaluations.js +307 -0
- package/dist/actions/evaluations.js.map +1 -0
- package/dist/actions/gate.d.ts +63 -0
- package/dist/actions/gate.d.ts.map +1 -0
- package/dist/actions/gate.js +272 -0
- package/dist/actions/gate.js.map +1 -0
- package/dist/actions/identity.d.ts +4 -0
- package/dist/actions/identity.d.ts.map +1 -0
- package/dist/actions/identity.js +4 -0
- package/dist/actions/identity.js.map +1 -0
- package/dist/actions/models.d.ts +4 -0
- package/dist/actions/models.d.ts.map +1 -0
- package/dist/actions/models.js +4 -0
- package/dist/actions/models.js.map +1 -0
- package/dist/actions/quality-standards.d.ts +30 -0
- package/dist/actions/quality-standards.d.ts.map +1 -0
- package/dist/actions/quality-standards.js +86 -0
- package/dist/actions/quality-standards.js.map +1 -0
- package/dist/actions/results.d.ts +12 -0
- package/dist/actions/results.d.ts.map +1 -0
- package/dist/actions/results.js +65 -0
- package/dist/actions/results.js.map +1 -0
- package/dist/actions/runs.d.ts +22 -0
- package/dist/actions/runs.d.ts.map +1 -0
- package/dist/actions/runs.js +59 -0
- package/dist/actions/runs.js.map +1 -0
- package/dist/actions/suite-gate.d.ts +24 -0
- package/dist/actions/suite-gate.d.ts.map +1 -0
- package/dist/actions/suite-gate.js +40 -0
- package/dist/actions/suite-gate.js.map +1 -0
- package/dist/actions/suite.d.ts +49 -0
- package/dist/actions/suite.d.ts.map +1 -0
- package/dist/actions/suite.js +68 -0
- package/dist/actions/suite.js.map +1 -0
- package/dist/actions/validation.d.ts +3 -0
- package/dist/actions/validation.d.ts.map +1 -0
- package/dist/actions/validation.js +26 -0
- package/dist/actions/validation.js.map +1 -0
- package/dist/actions/workspaces.d.ts +13 -0
- package/dist/actions/workspaces.d.ts.map +1 -0
- package/dist/actions/workspaces.js +18 -0
- package/dist/actions/workspaces.js.map +1 -0
- package/dist/api/api.d.ts +237 -0
- package/dist/api/api.d.ts.map +1 -0
- package/dist/api/api.js +396 -0
- package/dist/api/api.js.map +1 -0
- package/dist/api/client.d.ts +38 -0
- package/dist/api/client.d.ts.map +1 -0
- package/dist/api/client.js +295 -0
- package/dist/api/client.js.map +1 -0
- package/dist/api/endpoints.d.ts +31 -0
- package/dist/api/endpoints.d.ts.map +1 -0
- package/dist/api/endpoints.js +32 -0
- package/dist/api/endpoints.js.map +1 -0
- package/dist/api/errors.d.ts +22 -0
- package/dist/api/errors.d.ts.map +1 -0
- package/dist/api/errors.js +84 -0
- package/dist/api/errors.js.map +1 -0
- package/dist/api/schemas.d.ts +169 -0
- package/dist/api/schemas.d.ts.map +1 -0
- package/dist/api/schemas.js +194 -0
- package/dist/api/schemas.js.map +1 -0
- package/dist/auth/credential-store.d.ts +9 -0
- package/dist/auth/credential-store.d.ts.map +1 -0
- package/dist/auth/credential-store.js +2 -0
- package/dist/auth/credential-store.js.map +1 -0
- package/dist/auth/credentials.d.ts +27 -0
- package/dist/auth/credentials.d.ts.map +1 -0
- package/dist/auth/credentials.js +81 -0
- package/dist/auth/credentials.js.map +1 -0
- package/dist/auth/keyring-store.d.ts +9 -0
- package/dist/auth/keyring-store.d.ts.map +1 -0
- package/dist/auth/keyring-store.js +57 -0
- package/dist/auth/keyring-store.js.map +1 -0
- package/dist/auth/login.d.ts +28 -0
- package/dist/auth/login.d.ts.map +1 -0
- package/dist/auth/login.js +27 -0
- package/dist/auth/login.js.map +1 -0
- package/dist/auth/password-prompt.d.ts +5 -0
- package/dist/auth/password-prompt.d.ts.map +1 -0
- package/dist/auth/password-prompt.js +10 -0
- package/dist/auth/password-prompt.js.map +1 -0
- package/dist/auth/redact.d.ts +3 -0
- package/dist/auth/redact.d.ts.map +1 -0
- package/dist/auth/redact.js +69 -0
- package/dist/auth/redact.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +4 -0
- package/dist/cli.js.map +1 -0
- package/dist/commands/executor.d.ts +36 -0
- package/dist/commands/executor.d.ts.map +1 -0
- package/dist/commands/executor.js +1218 -0
- package/dist/commands/executor.js.map +1 -0
- package/dist/commands/program.d.ts +21 -0
- package/dist/commands/program.d.ts.map +1 -0
- package/dist/commands/program.js +588 -0
- package/dist/commands/program.js.map +1 -0
- package/dist/commands/types.d.ts +139 -0
- package/dist/commands/types.d.ts.map +1 -0
- package/dist/commands/types.js +2 -0
- package/dist/commands/types.js.map +1 -0
- package/dist/config.d.ts +12 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +57 -0
- package/dist/config.js.map +1 -0
- package/dist/configured-run/file-inputs.d.ts +8 -0
- package/dist/configured-run/file-inputs.d.ts.map +1 -0
- package/dist/configured-run/file-inputs.js +160 -0
- package/dist/configured-run/file-inputs.js.map +1 -0
- package/dist/configured-run/validation.d.ts +103 -0
- package/dist/configured-run/validation.d.ts.map +1 -0
- package/dist/configured-run/validation.js +327 -0
- package/dist/configured-run/validation.js.map +1 -0
- package/dist/domain/common.d.ts +6 -0
- package/dist/domain/common.d.ts.map +1 -0
- package/dist/domain/common.js +10 -0
- package/dist/domain/common.js.map +1 -0
- package/dist/domain/types.d.ts +117 -0
- package/dist/domain/types.d.ts.map +1 -0
- package/dist/domain/types.js +2 -0
- package/dist/domain/types.js.map +1 -0
- package/dist/errors/autoeval-error.d.ts +13 -0
- package/dist/errors/autoeval-error.d.ts.map +1 -0
- package/dist/errors/autoeval-error.js +36 -0
- package/dist/errors/autoeval-error.js.map +1 -0
- package/dist/errors/hints.d.ts +3 -0
- package/dist/errors/hints.d.ts.map +1 -0
- package/dist/errors/hints.js +43 -0
- package/dist/errors/hints.js.map +1 -0
- package/dist/gate/decision.d.ts +97 -0
- package/dist/gate/decision.d.ts.map +1 -0
- package/dist/gate/decision.js +436 -0
- package/dist/gate/decision.js.map +1 -0
- package/dist/gate/policy.d.ts +31 -0
- package/dist/gate/policy.d.ts.map +1 -0
- package/dist/gate/policy.js +26 -0
- package/dist/gate/policy.js.map +1 -0
- package/dist/guide/eval-creation-backend.d.ts +20 -0
- package/dist/guide/eval-creation-backend.d.ts.map +1 -0
- package/dist/guide/eval-creation-backend.js +16 -0
- package/dist/guide/eval-creation-backend.js.map +1 -0
- package/dist/guide/eval-guide.d.ts +24 -0
- package/dist/guide/eval-guide.d.ts.map +1 -0
- package/dist/guide/eval-guide.js +219 -0
- package/dist/guide/eval-guide.js.map +1 -0
- package/dist/mcp/context.d.ts +21 -0
- package/dist/mcp/context.d.ts.map +1 -0
- package/dist/mcp/context.js +19 -0
- package/dist/mcp/context.js.map +1 -0
- package/dist/mcp/schemas.d.ts +206 -0
- package/dist/mcp/schemas.d.ts.map +1 -0
- package/dist/mcp/schemas.js +80 -0
- package/dist/mcp/schemas.js.map +1 -0
- package/dist/mcp/server.d.ts +5 -0
- package/dist/mcp/server.d.ts.map +1 -0
- package/dist/mcp/server.js +10 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp/tools.d.ts +42 -0
- package/dist/mcp/tools.d.ts.map +1 -0
- package/dist/mcp/tools.js +287 -0
- package/dist/mcp/tools.js.map +1 -0
- package/dist/mcp.d.ts +3 -0
- package/dist/mcp.d.ts.map +1 -0
- package/dist/mcp.js +32 -0
- package/dist/mcp.js.map +1 -0
- package/dist/output/big-digits.d.ts +7 -0
- package/dist/output/big-digits.d.ts.map +1 -0
- package/dist/output/big-digits.js +43 -0
- package/dist/output/big-digits.js.map +1 -0
- package/dist/output/clipboard.d.ts +29 -0
- package/dist/output/clipboard.d.ts.map +1 -0
- package/dist/output/clipboard.js +82 -0
- package/dist/output/clipboard.js.map +1 -0
- package/dist/output/colors.d.ts +21 -0
- package/dist/output/colors.d.ts.map +1 -0
- package/dist/output/colors.js +21 -0
- package/dist/output/colors.js.map +1 -0
- package/dist/output/digest.d.ts +165 -0
- package/dist/output/digest.d.ts.map +1 -0
- package/dist/output/digest.js +771 -0
- package/dist/output/digest.js.map +1 -0
- package/dist/output/human.d.ts +95 -0
- package/dist/output/human.d.ts.map +1 -0
- package/dist/output/human.js +643 -0
- package/dist/output/human.js.map +1 -0
- package/dist/output/json.d.ts +2 -0
- package/dist/output/json.d.ts.map +1 -0
- package/dist/output/json.js +5 -0
- package/dist/output/json.js.map +1 -0
- package/dist/output/layout.d.ts +18 -0
- package/dist/output/layout.d.ts.map +1 -0
- package/dist/output/layout.js +64 -0
- package/dist/output/layout.js.map +1 -0
- package/dist/output/markdown.d.ts +16 -0
- package/dist/output/markdown.d.ts.map +1 -0
- package/dist/output/markdown.js +38 -0
- package/dist/output/markdown.js.map +1 -0
- package/dist/output/progress.d.ts +29 -0
- package/dist/output/progress.d.ts.map +1 -0
- package/dist/output/progress.js +136 -0
- package/dist/output/progress.js.map +1 -0
- package/dist/output/safe-text.d.ts +11 -0
- package/dist/output/safe-text.d.ts.map +1 -0
- package/dist/output/safe-text.js +44 -0
- package/dist/output/safe-text.js.map +1 -0
- package/dist/output/scorecard.d.ts +23 -0
- package/dist/output/scorecard.d.ts.map +1 -0
- package/dist/output/scorecard.js +506 -0
- package/dist/output/scorecard.js.map +1 -0
- package/dist/output/section.d.ts +15 -0
- package/dist/output/section.d.ts.map +1 -0
- package/dist/output/section.js +44 -0
- package/dist/output/section.js.map +1 -0
- package/dist/output/spinner.d.ts +24 -0
- package/dist/output/spinner.d.ts.map +1 -0
- package/dist/output/spinner.js +34 -0
- package/dist/output/spinner.js.map +1 -0
- package/dist/output/splash.d.ts +37 -0
- package/dist/output/splash.d.ts.map +1 -0
- package/dist/output/splash.js +169 -0
- package/dist/output/splash.js.map +1 -0
- package/dist/output/table.d.ts +33 -0
- package/dist/output/table.d.ts.map +1 -0
- package/dist/output/table.js +105 -0
- package/dist/output/table.js.map +1 -0
- package/dist/output/trend.d.ts +19 -0
- package/dist/output/trend.d.ts.map +1 -0
- package/dist/output/trend.js +38 -0
- package/dist/output/trend.js.map +1 -0
- package/dist/output/writer.d.ts +14 -0
- package/dist/output/writer.d.ts.map +1 -0
- package/dist/output/writer.js +33 -0
- package/dist/output/writer.js.map +1 -0
- package/dist/polling/clock.d.ts +9 -0
- package/dist/polling/clock.d.ts.map +1 -0
- package/dist/polling/clock.js +23 -0
- package/dist/polling/clock.js.map +1 -0
- package/dist/polling/wait-for-result.d.ts +16 -0
- package/dist/polling/wait-for-result.d.ts.map +1 -0
- package/dist/polling/wait-for-result.js +29 -0
- package/dist/polling/wait-for-result.js.map +1 -0
- package/dist/polling/wait-for-run.d.ts +14 -0
- package/dist/polling/wait-for-run.d.ts.map +1 -0
- package/dist/polling/wait-for-run.js +36 -0
- package/dist/polling/wait-for-run.js.map +1 -0
- package/dist/quickstart/prompt.d.ts +18 -0
- package/dist/quickstart/prompt.d.ts.map +1 -0
- package/dist/quickstart/prompt.js +37 -0
- package/dist/quickstart/prompt.js.map +1 -0
- package/dist/quickstart/sample.d.ts +28 -0
- package/dist/quickstart/sample.d.ts.map +1 -0
- package/dist/quickstart/sample.js +235 -0
- package/dist/quickstart/sample.js.map +1 -0
- package/dist/quickstart/selection.d.ts +53 -0
- package/dist/quickstart/selection.d.ts.map +1 -0
- package/dist/quickstart/selection.js +97 -0
- package/dist/quickstart/selection.js.map +1 -0
- package/dist/run.d.ts +28 -0
- package/dist/run.d.ts.map +1 -0
- package/dist/run.js +134 -0
- package/dist/run.js.map +1 -0
- package/dist/runtime/action-context.d.ts +19 -0
- package/dist/runtime/action-context.d.ts.map +1 -0
- package/dist/runtime/action-context.js +40 -0
- package/dist/runtime/action-context.js.map +1 -0
- package/dist/suite/execution.d.ts +46 -0
- package/dist/suite/execution.d.ts.map +1 -0
- package/dist/suite/execution.js +97 -0
- package/dist/suite/execution.js.map +1 -0
- package/dist/suite/manifest.d.ts +16 -0
- package/dist/suite/manifest.d.ts.map +1 -0
- package/dist/suite/manifest.js +92 -0
- package/dist/suite/manifest.js.map +1 -0
- package/dist/suite/reporting.d.ts +35 -0
- package/dist/suite/reporting.d.ts.map +1 -0
- package/dist/suite/reporting.js +131 -0
- package/dist/suite/reporting.js.map +1 -0
- package/dist/trace/deepseek-harness.d.ts +31 -0
- package/dist/trace/deepseek-harness.d.ts.map +1 -0
- package/dist/trace/deepseek-harness.js +412 -0
- package/dist/trace/deepseek-harness.js.map +1 -0
- package/dist/trace/import.d.ts +9 -0
- package/dist/trace/import.d.ts.map +1 -0
- package/dist/trace/import.js +45 -0
- package/dist/trace/import.js.map +1 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +3 -0
- package/dist/version.js.map +1 -0
- package/package.json +82 -0
|
@@ -0,0 +1,643 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { paint } from './colors.js';
|
|
3
|
+
import { summarizeEvaluationConfiguration, summarizeScenarioResults } from './digest.js';
|
|
4
|
+
import { renderCallout, renderHeading, renderMetadata } from './layout.js';
|
|
5
|
+
import { renderResultScorecard } from './scorecard.js';
|
|
6
|
+
import { safeMultilineTerminalText, safeTerminalText } from './safe-text.js';
|
|
7
|
+
import { formatTable } from './table.js';
|
|
8
|
+
function clean(value) {
|
|
9
|
+
return safeTerminalText(value);
|
|
10
|
+
}
|
|
11
|
+
const sessionScorecardSchema = z
|
|
12
|
+
.object({
|
|
13
|
+
outcome: z
|
|
14
|
+
.object({
|
|
15
|
+
achieved: z.boolean(),
|
|
16
|
+
score: z.number().nullable(),
|
|
17
|
+
has_data: z.boolean(),
|
|
18
|
+
})
|
|
19
|
+
.loose(),
|
|
20
|
+
overall: z.number().nullable(),
|
|
21
|
+
per_metric: z.record(z.string(), z
|
|
22
|
+
.object({
|
|
23
|
+
score: z.number().nullable(),
|
|
24
|
+
has_data: z.boolean(),
|
|
25
|
+
})
|
|
26
|
+
.loose()),
|
|
27
|
+
judge_agreement: z
|
|
28
|
+
.object({
|
|
29
|
+
pass: z.number().int().nonnegative().nullable(),
|
|
30
|
+
total: z.number().int().nonnegative().nullable(),
|
|
31
|
+
})
|
|
32
|
+
.loose()
|
|
33
|
+
.optional(),
|
|
34
|
+
})
|
|
35
|
+
.loose();
|
|
36
|
+
const trajectoryScorecardSchema = z
|
|
37
|
+
.object({
|
|
38
|
+
trajectory_score: z.number().nullable(),
|
|
39
|
+
judge_agreement: z
|
|
40
|
+
.object({
|
|
41
|
+
agreed: z.number().int().nonnegative().nullable(),
|
|
42
|
+
total: z.number().int().nonnegative().nullable(),
|
|
43
|
+
})
|
|
44
|
+
.loose(),
|
|
45
|
+
dimensions: z.array(z
|
|
46
|
+
.object({
|
|
47
|
+
key: z.string(),
|
|
48
|
+
label: z.string().min(1),
|
|
49
|
+
score: z.number().nullable(),
|
|
50
|
+
})
|
|
51
|
+
.loose()),
|
|
52
|
+
})
|
|
53
|
+
.loose();
|
|
54
|
+
function formatScore(score) {
|
|
55
|
+
return score === null ? 'Not available' : String(score);
|
|
56
|
+
}
|
|
57
|
+
function formatAgreement(agreement) {
|
|
58
|
+
const pass = agreement?.pass;
|
|
59
|
+
const total = agreement?.total;
|
|
60
|
+
if (pass === undefined || pass === null || total === undefined || total === null) {
|
|
61
|
+
return 'Not available';
|
|
62
|
+
}
|
|
63
|
+
return `${pass}/${total}`;
|
|
64
|
+
}
|
|
65
|
+
function formatSessionScorecard(input, indent = '') {
|
|
66
|
+
const parsed = sessionScorecardSchema.safeParse(input);
|
|
67
|
+
if (!parsed.success)
|
|
68
|
+
return [`${indent}Score summary: Not available`];
|
|
69
|
+
const scorecard = parsed.data;
|
|
70
|
+
const lines = [
|
|
71
|
+
`${indent}Outcome: ${scorecard.outcome.has_data ? (scorecard.outcome.achieved ? 'Pass' : 'Fail') : 'Not available'}`,
|
|
72
|
+
`${indent}Overall score: ${formatScore(scorecard.overall)}`,
|
|
73
|
+
`${indent}Judge agreement: ${formatAgreement(scorecard.judge_agreement)}`,
|
|
74
|
+
];
|
|
75
|
+
const metrics = Object.entries(scorecard.per_metric).sort(([left], [right]) => left.localeCompare(right));
|
|
76
|
+
if (metrics.length > 0) {
|
|
77
|
+
lines.push(`${indent}Metric scores:`);
|
|
78
|
+
for (const [name, metric] of metrics) {
|
|
79
|
+
lines.push(`${indent} ${clean(name)}: ${metric.has_data ? formatScore(metric.score) : 'Not available'}`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return lines;
|
|
83
|
+
}
|
|
84
|
+
function formatTrajectoryScorecard(input, indent = '') {
|
|
85
|
+
const parsed = trajectoryScorecardSchema.safeParse(input);
|
|
86
|
+
if (!parsed.success)
|
|
87
|
+
return [`${indent}Trajectory: Not available`];
|
|
88
|
+
const scorecard = parsed.data;
|
|
89
|
+
const agreement = scorecard.judge_agreement.agreed === null || scorecard.judge_agreement.total === null
|
|
90
|
+
? 'Not available'
|
|
91
|
+
: `${scorecard.judge_agreement.agreed}/${scorecard.judge_agreement.total}`;
|
|
92
|
+
const lines = [
|
|
93
|
+
`${indent}Trajectory score: ${formatScore(scorecard.trajectory_score)}`,
|
|
94
|
+
`${indent}Judge agreement: ${agreement}`,
|
|
95
|
+
];
|
|
96
|
+
if (scorecard.dimensions.length > 0) {
|
|
97
|
+
lines.push(`${indent}Trajectory dimensions:`);
|
|
98
|
+
for (const dimension of scorecard.dimensions) {
|
|
99
|
+
lines.push(`${indent} ${clean(dimension.label)}: ${formatScore(dimension.score)}`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return lines;
|
|
103
|
+
}
|
|
104
|
+
export function renderIdentity(identity) {
|
|
105
|
+
return `Authenticated as ${clean(identity.email)}${identity.name ? ` (${clean(identity.name)})` : ''}\n`;
|
|
106
|
+
}
|
|
107
|
+
export function renderLogin(input) {
|
|
108
|
+
const persistence = input.wasPersisted
|
|
109
|
+
? 'The CLI key was saved in the OS credential store'
|
|
110
|
+
: input.source === 'environment'
|
|
111
|
+
? 'The environment-provided CLI key was not persisted'
|
|
112
|
+
: 'Using the CLI key already stored in the OS credential store';
|
|
113
|
+
const replaced = input.replacedInvalidStoredKey
|
|
114
|
+
? 'Removed an invalid stored CLI key before signing in\n'
|
|
115
|
+
: '';
|
|
116
|
+
return `${replaced}${renderIdentity(input.identity)}${persistence}\n`;
|
|
117
|
+
}
|
|
118
|
+
export function renderLoginGuidance() {
|
|
119
|
+
return [
|
|
120
|
+
'No valid Autoeval CLI key was found.',
|
|
121
|
+
'',
|
|
122
|
+
' 1. Go to https://app.plumloom.ai',
|
|
123
|
+
' 2. Create a CLI key',
|
|
124
|
+
' 3. Copy the key starting with pl_sk_',
|
|
125
|
+
'',
|
|
126
|
+
'Paste it below — it will be verified and saved to your OS credential store.',
|
|
127
|
+
'',
|
|
128
|
+
].join('\n');
|
|
129
|
+
}
|
|
130
|
+
export function renderLogout(wasRemoved, hasEnvironmentCredential = false) {
|
|
131
|
+
if (wasRemoved) {
|
|
132
|
+
return 'Removed the locally stored CLI key. Environment credentials were not changed\n';
|
|
133
|
+
}
|
|
134
|
+
if (hasEnvironmentCredential) {
|
|
135
|
+
return [
|
|
136
|
+
'No locally stored CLI key was found',
|
|
137
|
+
'AUTOEVAL_API_KEY is currently set in your environment, so authentication still works',
|
|
138
|
+
'Unset AUTOEVAL_API_KEY in your shell to fully log out of this terminal session',
|
|
139
|
+
'',
|
|
140
|
+
].join('\n');
|
|
141
|
+
}
|
|
142
|
+
return 'No locally stored CLI key was found. Environment credentials were not changed\n';
|
|
143
|
+
}
|
|
144
|
+
export function renderWorkspaces(page) {
|
|
145
|
+
if (page.items.length === 0) {
|
|
146
|
+
return 'No workspaces found\nCreate one with `autoeval workspace create --name "My workspace"`\n';
|
|
147
|
+
}
|
|
148
|
+
const table = formatTable([
|
|
149
|
+
{ header: 'NAME', minWidth: 20, maxWidth: 44 },
|
|
150
|
+
{ header: 'WORKSPACE ID', atomic: true, tone: 'meta' },
|
|
151
|
+
{ header: 'EVALUATIONS', align: 'right' },
|
|
152
|
+
], page.items.map((workspace) => [
|
|
153
|
+
workspace.name,
|
|
154
|
+
workspace.id,
|
|
155
|
+
String(workspace.evaluationCount),
|
|
156
|
+
]));
|
|
157
|
+
return `${table}\n\nShowing ${page.items.length} of ${page.total} workspaces\nList evaluations with \`autoeval eval list --workspace <workspace-id>\`\n`;
|
|
158
|
+
}
|
|
159
|
+
export function renderWorkspaceSummaryList(page) {
|
|
160
|
+
if (page.items.length === 0)
|
|
161
|
+
return 'No workspaces found\n';
|
|
162
|
+
return `${formatTable([
|
|
163
|
+
{ header: 'NAME', minWidth: 20, maxWidth: 44 },
|
|
164
|
+
{ header: 'ID', atomic: true, tone: 'meta' },
|
|
165
|
+
{ header: 'EVALUATIONS', align: 'right' },
|
|
166
|
+
], page.items.map((workspace) => [
|
|
167
|
+
workspace.name,
|
|
168
|
+
workspace.id,
|
|
169
|
+
String(workspace.evaluationCount),
|
|
170
|
+
]))}\n`;
|
|
171
|
+
}
|
|
172
|
+
export function renderCreatedWorkspace(workspace) {
|
|
173
|
+
return `Created workspace ${clean(workspace.name)} (${clean(workspace.id)})\n`;
|
|
174
|
+
}
|
|
175
|
+
export function renderCreatedEvaluation(evaluation) {
|
|
176
|
+
return `Created evaluation ${clean(evaluation.evaluationName)} (${clean(evaluation.evaluationId)}) in workspace ${clean(evaluation.workspaceId)}\n`;
|
|
177
|
+
}
|
|
178
|
+
export function renderEvaluations(page) {
|
|
179
|
+
if (page.items.length === 0) {
|
|
180
|
+
return 'No evaluations found in this workspace\nCreate one with `autoeval eval create --workspace <workspace-id> --name "My evaluation"`\n';
|
|
181
|
+
}
|
|
182
|
+
const table = formatTable([
|
|
183
|
+
{ header: 'NAME', minWidth: 20, maxWidth: 52 },
|
|
184
|
+
{ header: 'EVALUATION ID', atomic: true, tone: 'meta' },
|
|
185
|
+
], page.items.map((evaluation) => [evaluation.name, evaluation.id]));
|
|
186
|
+
return `${table}\n\nShowing ${page.items.length} of ${page.total} evaluations\nInspect one with \`autoeval eval show <evaluation-id>\`\n`;
|
|
187
|
+
}
|
|
188
|
+
export function renderEvaluationVersions(history) {
|
|
189
|
+
if (history.versions.length === 0) {
|
|
190
|
+
return `No versions found for evaluation ${clean(history.evaluationId)}\n`;
|
|
191
|
+
}
|
|
192
|
+
const table = formatTable([
|
|
193
|
+
{ header: 'VERSION', atomic: true },
|
|
194
|
+
{ header: 'CONFIG ID', atomic: true, tone: 'meta' },
|
|
195
|
+
{ header: 'CREATED', atomic: true },
|
|
196
|
+
{ header: 'STATUS', atomic: true },
|
|
197
|
+
], history.versions.map((version) => [
|
|
198
|
+
String(version.version),
|
|
199
|
+
version.configVersionId,
|
|
200
|
+
version.createdAt ?? '-',
|
|
201
|
+
version.isCurrent ? 'current' : '',
|
|
202
|
+
]));
|
|
203
|
+
return `${table}\n\nShow one with \`autoeval eval show ${clean(history.evaluationId)} --version <number>\`\n`;
|
|
204
|
+
}
|
|
205
|
+
export function renderEvaluationSummaryList(page) {
|
|
206
|
+
if (page.items.length === 0)
|
|
207
|
+
return 'No evaluations found in this workspace\n';
|
|
208
|
+
return `${formatTable([
|
|
209
|
+
{ header: 'NAME', minWidth: 20, maxWidth: 52 },
|
|
210
|
+
{ header: 'ID', atomic: true, tone: 'meta' },
|
|
211
|
+
], page.items.map((evaluation) => [evaluation.name, evaluation.id]))}\n`;
|
|
212
|
+
}
|
|
213
|
+
export function renderEvaluation(evaluation) {
|
|
214
|
+
const evaluationName = [evaluation.raw, evaluation.configuration]
|
|
215
|
+
.map((candidate) => ['evaluation_name', 'evaluationName', 'eval_name', 'name']
|
|
216
|
+
.map((key) => candidate[key])
|
|
217
|
+
.find((value) => typeof value === 'string' && value.trim().length > 0))
|
|
218
|
+
.find((value) => value !== undefined);
|
|
219
|
+
return [
|
|
220
|
+
`Evaluation: ${evaluationName ? `${clean(evaluationName)} (${clean(evaluation.evaluationId)})` : clean(evaluation.evaluationId)}`,
|
|
221
|
+
`Context: ${clean(evaluation.contextType)}`,
|
|
222
|
+
'',
|
|
223
|
+
...(evaluation.version === undefined
|
|
224
|
+
? []
|
|
225
|
+
: [`Evaluation version: ${evaluation.version}${evaluation.isCurrent ? ' (current)' : ''}`]),
|
|
226
|
+
` Methodology version ID: ${paint.meta(clean(evaluation.methodologyVersionId))}`,
|
|
227
|
+
` Config version ID: ${paint.meta(clean(evaluation.configVersionId))}`,
|
|
228
|
+
'',
|
|
229
|
+
summarizeEvaluationConfiguration(evaluation),
|
|
230
|
+
'',
|
|
231
|
+
'Use --json to view the validated configuration payload',
|
|
232
|
+
'',
|
|
233
|
+
].join('\n');
|
|
234
|
+
}
|
|
235
|
+
export function renderModels(models) {
|
|
236
|
+
if (models.length === 0)
|
|
237
|
+
return 'No supported models were returned\n';
|
|
238
|
+
const sorted = [...models].sort((left, right) => `${left.provider}/${left.displayName}`.localeCompare(`${right.provider}/${right.displayName}`));
|
|
239
|
+
const table = formatTable([
|
|
240
|
+
{ header: 'NAME' },
|
|
241
|
+
{ header: 'PROVIDER' },
|
|
242
|
+
{ header: 'MODEL ID', atomic: true, tone: 'meta' },
|
|
243
|
+
{ header: 'FLAGS', tone: 'warn' },
|
|
244
|
+
], sorted.map((model) => [
|
|
245
|
+
model.displayName,
|
|
246
|
+
model.provider,
|
|
247
|
+
model.id,
|
|
248
|
+
[model.isLocked ? 'locked' : '', model.isDeprecated ? 'deprecated' : '']
|
|
249
|
+
.filter(Boolean)
|
|
250
|
+
.join(', '),
|
|
251
|
+
]));
|
|
252
|
+
return `${table}\n\nShowing ${sorted.length} models\n`;
|
|
253
|
+
}
|
|
254
|
+
export function renderRunStatus(status) {
|
|
255
|
+
const rows = [
|
|
256
|
+
['Run ID', clean(status.runId)],
|
|
257
|
+
['Evaluation', clean(status.evaluationId)],
|
|
258
|
+
['Status', clean(status.state)],
|
|
259
|
+
];
|
|
260
|
+
if (status.progress) {
|
|
261
|
+
const completed = status.progress.runsCompleted ?? '?';
|
|
262
|
+
const total = status.progress.totalRuns ?? '?';
|
|
263
|
+
const percentage = status.progress.percentage ?? '?';
|
|
264
|
+
rows.push(['Progress', `${completed}/${total} (${percentage}%)`]);
|
|
265
|
+
}
|
|
266
|
+
if (status.failureCode)
|
|
267
|
+
rows.push(['Failure code', clean(status.failureCode)]);
|
|
268
|
+
const metadata = renderMetadata(rows.map(([label, value]) => [label ?? '', value ?? '']));
|
|
269
|
+
const next = [
|
|
270
|
+
'',
|
|
271
|
+
'Use the Run ID above with status and results:',
|
|
272
|
+
` autoeval status ${clean(status.evaluationId)} ${clean(status.runId)}`,
|
|
273
|
+
` autoeval results ${clean(status.evaluationId)} ${clean(status.runId)}`,
|
|
274
|
+
'Run with --json for full reliability detail',
|
|
275
|
+
'',
|
|
276
|
+
].join('\n');
|
|
277
|
+
return `${metadata}${next}`;
|
|
278
|
+
}
|
|
279
|
+
export function renderRunExecution(execution) {
|
|
280
|
+
const seconds = Math.round(execution.outcome.elapsedMs / 1_000);
|
|
281
|
+
return `${renderRunStatus(execution.outcome.status)}Finished in ${seconds}s\n`;
|
|
282
|
+
}
|
|
283
|
+
export function renderQuickstartPlan(plan) {
|
|
284
|
+
return renderMetadata([
|
|
285
|
+
['Workspace', `${clean(plan.workspace.name)} (${clean(plan.workspace.id)})`],
|
|
286
|
+
['Judge', `${clean(plan.judge.displayName)} (${clean(plan.judge.id)})`],
|
|
287
|
+
...(plan.primary === undefined
|
|
288
|
+
? []
|
|
289
|
+
: [['Under test', `${clean(plan.primary.displayName)} (${clean(plan.primary.id)})`]]),
|
|
290
|
+
['Sample', clean(plan.sample)],
|
|
291
|
+
['Evaluation', `${clean(plan.evaluationName)} (${clean(plan.evaluationId)})`],
|
|
292
|
+
]);
|
|
293
|
+
}
|
|
294
|
+
export function renderQuickstartNextSteps(evaluationId, runId) {
|
|
295
|
+
return [
|
|
296
|
+
'',
|
|
297
|
+
'Re-read this run any time:',
|
|
298
|
+
` autoeval results ${clean(evaluationId)} ${clean(runId)}`,
|
|
299
|
+
'Run with --json for full reliability detail',
|
|
300
|
+
'',
|
|
301
|
+
].join('\n');
|
|
302
|
+
}
|
|
303
|
+
export function renderTraceImport(report) {
|
|
304
|
+
const rows = [
|
|
305
|
+
['Wrote', clean(report.outputFile)],
|
|
306
|
+
['Harness schema', clean(report.schema)],
|
|
307
|
+
['Events read', String(report.events)],
|
|
308
|
+
['Spans written', `${report.spans} (${report.llmSpans} LLM, ${report.toolSpans} tool)`],
|
|
309
|
+
];
|
|
310
|
+
const notes = [];
|
|
311
|
+
if (report.unpairedToolCalls > 0) {
|
|
312
|
+
notes.push(`${report.unpairedToolCalls} tool call(s) had no matching result.`);
|
|
313
|
+
}
|
|
314
|
+
if (report.skippedSubagentSessions > 0) {
|
|
315
|
+
notes.push(`${report.skippedSubagentSessions} subagent event(s) skipped; subagents log to their own sessions.`);
|
|
316
|
+
}
|
|
317
|
+
if (report.syntheticTimestamps) {
|
|
318
|
+
notes.push('The log carried no timestamps, so span times are synthetic and ordering-only.');
|
|
319
|
+
}
|
|
320
|
+
const metadata = renderMetadata(rows.map(([label, value]) => [label, value]));
|
|
321
|
+
return notes.length === 0 ? metadata : `${metadata}\n${notes.join('\n')}\n`;
|
|
322
|
+
}
|
|
323
|
+
function renderModelReference(model) {
|
|
324
|
+
return `${clean(model.displayName)} (${clean(model.id)})`;
|
|
325
|
+
}
|
|
326
|
+
export function renderConfiguredRunValidation(result) {
|
|
327
|
+
const artifactLabel = result.contextType === 'scenario'
|
|
328
|
+
? 'Scenarios'
|
|
329
|
+
: result.contextType === 'conversation'
|
|
330
|
+
? 'Messages'
|
|
331
|
+
: 'Resource spans';
|
|
332
|
+
const lines = [
|
|
333
|
+
`Valid ${clean(result.contextType)} evaluation input`,
|
|
334
|
+
`Judge: ${renderModelReference(result.models.judge)}`,
|
|
335
|
+
];
|
|
336
|
+
if (result.models.primary) {
|
|
337
|
+
lines.push(`Primary: ${renderModelReference(result.models.primary)}`);
|
|
338
|
+
}
|
|
339
|
+
if (result.models.comparisons.length > 0) {
|
|
340
|
+
lines.push(`Comparisons: ${result.models.comparisons.map(renderModelReference).join(', ')}`);
|
|
341
|
+
}
|
|
342
|
+
lines.push(`${artifactLabel}: ${result.artifactCount}`);
|
|
343
|
+
return `${lines.join('\n')}\n`;
|
|
344
|
+
}
|
|
345
|
+
export function renderQualityStandard(qualityStandard, fallbackId) {
|
|
346
|
+
const idCandidate = qualityStandard.id ??
|
|
347
|
+
qualityStandard.quality_standard_id ??
|
|
348
|
+
qualityStandard.qualityStandardId ??
|
|
349
|
+
qualityStandard.qs_id;
|
|
350
|
+
const qualityStandardId = typeof idCandidate === 'string' && idCandidate.trim() ? idCandidate : fallbackId;
|
|
351
|
+
const nameCandidate = qualityStandard.name ?? qualityStandard.qs_name;
|
|
352
|
+
const name = typeof nameCandidate === 'string' && nameCandidate.trim() ? nameCandidate : 'Not available';
|
|
353
|
+
const judgeModel = typeof qualityStandard.judge_model === 'string' && qualityStandard.judge_model.trim()
|
|
354
|
+
? qualityStandard.judge_model
|
|
355
|
+
: 'Not available';
|
|
356
|
+
const rubric = typeof qualityStandard.rubric === 'string' && qualityStandard.rubric.trim()
|
|
357
|
+
? qualityStandard.rubric
|
|
358
|
+
: 'Not available';
|
|
359
|
+
const anchors = Array.isArray(qualityStandard.anchors)
|
|
360
|
+
? qualityStandard.anchors.filter((anchor) => typeof anchor === 'object' && anchor !== null && !Array.isArray(anchor))
|
|
361
|
+
: [];
|
|
362
|
+
const indent = (value, spaces) => {
|
|
363
|
+
const text = typeof value === 'string' && value.trim() ? value : 'Not available';
|
|
364
|
+
const padding = ' '.repeat(spaces);
|
|
365
|
+
return safeMultilineTerminalText(text)
|
|
366
|
+
.split(/\r?\n/u)
|
|
367
|
+
.map((line) => `${padding}${line}`)
|
|
368
|
+
.join('\n');
|
|
369
|
+
};
|
|
370
|
+
const lines = [
|
|
371
|
+
'Quality Standard',
|
|
372
|
+
'',
|
|
373
|
+
`Name: ${clean(name)}`,
|
|
374
|
+
`QS ID: ${clean(qualityStandardId)}`,
|
|
375
|
+
`Judge model: ${clean(judgeModel)}`,
|
|
376
|
+
'',
|
|
377
|
+
'Rubric',
|
|
378
|
+
indent(rubric, 2),
|
|
379
|
+
'',
|
|
380
|
+
`Anchors (${anchors.length})`,
|
|
381
|
+
];
|
|
382
|
+
for (const [index, anchor] of anchors.entries()) {
|
|
383
|
+
const rating = anchor.score === 5 ? 'Accept' : anchor.score === 1 ? 'Reject' : 'Unknown';
|
|
384
|
+
const score = typeof anchor.score === 'number' ? String(anchor.score) : 'Not available';
|
|
385
|
+
lines.push('', `${index + 1}. ${rating} (${score})`, ' Input', indent(anchor.input, 5), ' Response', indent(anchor.response, 5), ' Reference', indent(anchor.reference, 5), ' Reasoning', indent(anchor.reasoning, 5));
|
|
386
|
+
}
|
|
387
|
+
if (anchors.length === 0) {
|
|
388
|
+
lines.push('', 'No anchors were returned.');
|
|
389
|
+
}
|
|
390
|
+
return `${lines.join('\n')}\n`;
|
|
391
|
+
}
|
|
392
|
+
export function renderWorkspaceQualityStandard(qualityStandard, workspaceId) {
|
|
393
|
+
if (qualityStandard === null) {
|
|
394
|
+
return 'No Quality Standard assigned to this workspace.\n';
|
|
395
|
+
}
|
|
396
|
+
const idCandidate = qualityStandard.quality_standard_id ??
|
|
397
|
+
qualityStandard.qualityStandardId ??
|
|
398
|
+
qualityStandard.id ??
|
|
399
|
+
qualityStandard.qs_id;
|
|
400
|
+
const qualityStandardId = typeof idCandidate === 'string' && idCandidate.trim() ? idCandidate : undefined;
|
|
401
|
+
const nameCandidate = qualityStandard.name ?? qualityStandard.qs_name;
|
|
402
|
+
const name = typeof nameCandidate === 'string' && nameCandidate.trim() ? nameCandidate : undefined;
|
|
403
|
+
if (name && qualityStandardId) {
|
|
404
|
+
return `Workspace ${clean(workspaceId)} is assigned quality standard ${clean(name)} (${clean(qualityStandardId)})\n`;
|
|
405
|
+
}
|
|
406
|
+
if (qualityStandardId) {
|
|
407
|
+
return `Workspace ${clean(workspaceId)} is assigned quality standard ${clean(qualityStandardId)}\n`;
|
|
408
|
+
}
|
|
409
|
+
return `Retrieved workspace quality standard assignment for ${clean(workspaceId)}\n`;
|
|
410
|
+
}
|
|
411
|
+
/**
|
|
412
|
+
* Plain-text result summary. It carries no colour or table layout because it is
|
|
413
|
+
* also the text handed to downstream consumers as data.
|
|
414
|
+
*/
|
|
415
|
+
export function renderResultsSummary(results) {
|
|
416
|
+
if (results.contextType === 'scenario') {
|
|
417
|
+
return `${summarizeScenarioResults(results)}\n`;
|
|
418
|
+
}
|
|
419
|
+
if (results.contextType === 'conversation') {
|
|
420
|
+
return ['Context: conversation', ...formatSessionScorecard(results.conversation), ''].join('\n');
|
|
421
|
+
}
|
|
422
|
+
const lines = [
|
|
423
|
+
'Context: agent_trace',
|
|
424
|
+
'Session evaluation:',
|
|
425
|
+
...formatSessionScorecard(results.agentTrace, ' '),
|
|
426
|
+
'Trajectory evaluation:',
|
|
427
|
+
];
|
|
428
|
+
if (results.trajectory?.has_trajectory === false) {
|
|
429
|
+
lines.push(' Trajectory: Not available');
|
|
430
|
+
}
|
|
431
|
+
else if (results.trajectory) {
|
|
432
|
+
lines.push(...formatTrajectoryScorecard(results.trajectory, ' '));
|
|
433
|
+
}
|
|
434
|
+
else {
|
|
435
|
+
lines.push(' Trajectory: Not available');
|
|
436
|
+
}
|
|
437
|
+
lines.push('');
|
|
438
|
+
return lines.join('\n');
|
|
439
|
+
}
|
|
440
|
+
/**
|
|
441
|
+
* Result display: a readable scorecard over the payloads the backend returned.
|
|
442
|
+
*
|
|
443
|
+
* Scores are read from the payload, never derived beyond the arithmetic the
|
|
444
|
+
* scorecard documents, and the untouched payloads stay one flag away via
|
|
445
|
+
* `--json` for anyone who needs the raw shape.
|
|
446
|
+
*/
|
|
447
|
+
export function renderResults(results, options = {}) {
|
|
448
|
+
const scorecard = renderResultScorecard(results, undefined, {
|
|
449
|
+
showOutputs: options.showOutputs === true,
|
|
450
|
+
...(options.scenarioStatus === undefined ? {} : { scenarioStatus: options.scenarioStatus }),
|
|
451
|
+
}).replace(/\n+$/u, '');
|
|
452
|
+
const hint = results.contextType === 'scenario'
|
|
453
|
+
? '--show-outputs for full responses · --json for the raw payload'
|
|
454
|
+
: 'Use --json for full reliability detail and the untouched backend payload';
|
|
455
|
+
return `${scorecard}\n\n${hint}\n`;
|
|
456
|
+
}
|
|
457
|
+
/** Render the release-gate verdict as an aligned, scannable report. */
|
|
458
|
+
export function renderGateReport(summary) {
|
|
459
|
+
const { report } = summary;
|
|
460
|
+
const verdict = report.decision;
|
|
461
|
+
const tone = report.decision === 'PASS' ? 'pass' : report.decision === 'FAIL' ? 'fail' : 'warn';
|
|
462
|
+
const lines = [
|
|
463
|
+
renderCallout(tone, `Gate: ${verdict}`).replace(/\n$/u, ''),
|
|
464
|
+
renderMetadata([
|
|
465
|
+
['Evaluation', clean(summary.run.evaluationId)],
|
|
466
|
+
['Run ID', clean(summary.run.runId)],
|
|
467
|
+
['Context', clean(report.contextType)],
|
|
468
|
+
['Finished in', `${Math.round(summary.elapsedMs / 1_000)}s`],
|
|
469
|
+
]).replace(/\n$/u, ''),
|
|
470
|
+
'',
|
|
471
|
+
];
|
|
472
|
+
if (report.checks.length === 0) {
|
|
473
|
+
lines.push(report.contextType === 'scenario'
|
|
474
|
+
? 'No thresholds were configured, so the gate did not evaluate any metric.'
|
|
475
|
+
: 'No metric thresholds were configured, so this gate is INCONCLUSIVE.', '');
|
|
476
|
+
return lines.join('\n');
|
|
477
|
+
}
|
|
478
|
+
lines.push(formatTable([
|
|
479
|
+
{ header: 'METRIC', minWidth: 22 },
|
|
480
|
+
{ header: 'THRESHOLD', minWidth: 12 },
|
|
481
|
+
{ header: 'ACTUAL', minWidth: 16 },
|
|
482
|
+
{ header: 'STATUS' },
|
|
483
|
+
], report.checks.map((check) => [
|
|
484
|
+
check.detail ? `${check.label} (${check.detail})` : check.label,
|
|
485
|
+
check.threshold,
|
|
486
|
+
check.actual,
|
|
487
|
+
check.decision === 'PASS' ? 'pass' : check.decision,
|
|
488
|
+
])));
|
|
489
|
+
lines.push('');
|
|
490
|
+
return lines.join('\n');
|
|
491
|
+
}
|
|
492
|
+
/** Render execution and result-fetch status for every eval in a suite. */
|
|
493
|
+
export function renderSuiteSummary(summary, clusters = []) {
|
|
494
|
+
const blocks = [...summary.evals]
|
|
495
|
+
.sort((left, right) => left.index - right.index)
|
|
496
|
+
.map((entry) => {
|
|
497
|
+
const lines = [];
|
|
498
|
+
const execution = entry.execution;
|
|
499
|
+
const label = execution ? clean(execution.evaluationName) : clean(entry.inputFile);
|
|
500
|
+
lines.push(`Eval ${entry.index + 1}: ${label}`);
|
|
501
|
+
lines.push(` Source: ${clean(entry.inputFile)}`);
|
|
502
|
+
if (execution) {
|
|
503
|
+
lines.push(` Evaluation ID: ${clean(execution.evaluationId)}`);
|
|
504
|
+
lines.push(` Run ID: ${clean(execution.runId)}`);
|
|
505
|
+
lines.push(` Context: ${clean(execution.contextType)}`);
|
|
506
|
+
lines.push(` Execution: ${clean(execution.state)} in ${Math.round(execution.elapsedMs / 1_000)}s`);
|
|
507
|
+
}
|
|
508
|
+
lines.push(` Status: ${clean(entry.status)}`);
|
|
509
|
+
if (entry.error !== undefined)
|
|
510
|
+
lines.push(` Error: ${clean(entry.error)}`);
|
|
511
|
+
if (entry.results) {
|
|
512
|
+
lines.push(renderResults(entry.results, {
|
|
513
|
+
...(entry.execution?.statusRaw === undefined
|
|
514
|
+
? {}
|
|
515
|
+
: { scenarioStatus: entry.execution.statusRaw }),
|
|
516
|
+
})
|
|
517
|
+
.split('\n')
|
|
518
|
+
.map((line) => (line === '' ? line : ` ${line}`))
|
|
519
|
+
.join('\n'));
|
|
520
|
+
}
|
|
521
|
+
return `${lines.join('\n')}\n`;
|
|
522
|
+
});
|
|
523
|
+
return [
|
|
524
|
+
renderHeading('Suite summary').replace(/^\n+|\n+$/gu, ''),
|
|
525
|
+
`Workspace: ${clean(summary.workspaceId)}`,
|
|
526
|
+
`Evals: ${summary.total} total, ${summary.completed} with results, ${summary.failed} failed`,
|
|
527
|
+
'',
|
|
528
|
+
...blocks,
|
|
529
|
+
renderFailureClusters(clusters),
|
|
530
|
+
].join('\n');
|
|
531
|
+
}
|
|
532
|
+
/** Render the suite verdict, counts, and evidence for every blocking eval. */
|
|
533
|
+
export function renderSuiteGateReport(result, clusters = []) {
|
|
534
|
+
const tone = result.suiteDecision === 'PASS' ? 'pass' : result.suiteDecision === 'FAIL' ? 'fail' : 'warn';
|
|
535
|
+
const lines = [
|
|
536
|
+
renderCallout(tone, `Suite release gate: ${result.suiteDecision}`).replace(/\n$/u, ''),
|
|
537
|
+
renderMetadata([
|
|
538
|
+
['Workspace', clean(result.workspaceId)],
|
|
539
|
+
['PASS', String(result.counts.PASS)],
|
|
540
|
+
['FAIL', String(result.counts.FAIL)],
|
|
541
|
+
['INCONCLUSIVE', String(result.counts.INCONCLUSIVE)],
|
|
542
|
+
['ERROR', String(result.counts.ERROR)],
|
|
543
|
+
]).replace(/\n$/u, ''),
|
|
544
|
+
'',
|
|
545
|
+
];
|
|
546
|
+
const notPassing = result.perEvalDecisions.filter((entry) => entry.decision !== 'PASS');
|
|
547
|
+
if (notPassing.length === 0) {
|
|
548
|
+
lines.push('Every eval in the suite met its configured release criteria.', '');
|
|
549
|
+
return lines.join('\n');
|
|
550
|
+
}
|
|
551
|
+
lines.push(renderHeading('Blocking evals').replace(/^\n+|\n+$/gu, ''));
|
|
552
|
+
for (const entry of notPassing) {
|
|
553
|
+
const label = entry.evaluationName ?? entry.inputFile;
|
|
554
|
+
lines.push(`- ${clean(label)} [${entry.decision}]`);
|
|
555
|
+
if (entry.contextType)
|
|
556
|
+
lines.push(` context: ${clean(entry.contextType)}`);
|
|
557
|
+
if (entry.runId)
|
|
558
|
+
lines.push(` run: ${clean(entry.runId)}`);
|
|
559
|
+
const convergence = entry.evidence.convergence;
|
|
560
|
+
if (convergence)
|
|
561
|
+
lines.push(` convergence: ${clean(convergence.status)}`);
|
|
562
|
+
for (const check of entry.checks) {
|
|
563
|
+
if (check.decision === 'PASS')
|
|
564
|
+
continue;
|
|
565
|
+
lines.push(` ${clean(check.label)}: ${clean(check.actual)} required ${clean(check.threshold)} -> ${check.decision}`);
|
|
566
|
+
lines.push(` reason: ${clean(check.reason)}`);
|
|
567
|
+
}
|
|
568
|
+
lines.push(` reason: ${clean(entry.reason)}`);
|
|
569
|
+
lines.push('');
|
|
570
|
+
}
|
|
571
|
+
lines.push(renderFailureClusters(clusters));
|
|
572
|
+
return lines.join('\n');
|
|
573
|
+
}
|
|
574
|
+
/** Render distinct root causes behind a blocked suite, most frequent first. */
|
|
575
|
+
export function renderFailureClusters(clusters) {
|
|
576
|
+
if (clusters.length === 0)
|
|
577
|
+
return '';
|
|
578
|
+
const total = clusters.reduce((sum, cluster) => sum + cluster.count, 0);
|
|
579
|
+
const lines = [
|
|
580
|
+
renderHeading('Failure clusters').replace(/^\n+|\n+$/gu, ''),
|
|
581
|
+
`${total} failure(s) grouped into ${clusters.length} root cause(s).`,
|
|
582
|
+
'',
|
|
583
|
+
formatTable([
|
|
584
|
+
{ header: 'ROOT CAUSE', minWidth: 30 },
|
|
585
|
+
{ header: 'CATEGORY', minWidth: 12 },
|
|
586
|
+
{ header: 'COUNT', align: 'right', minWidth: 5 },
|
|
587
|
+
{ header: 'EVALS', minWidth: 20 },
|
|
588
|
+
], clusters.map((cluster) => [
|
|
589
|
+
clean(cluster.label),
|
|
590
|
+
cluster.category,
|
|
591
|
+
String(cluster.count),
|
|
592
|
+
clean(cluster.evals.join(', ')),
|
|
593
|
+
])),
|
|
594
|
+
'',
|
|
595
|
+
];
|
|
596
|
+
for (const cluster of clusters) {
|
|
597
|
+
lines.push(`- ${clean(cluster.label)}: ${clean(cluster.exemplar)}`);
|
|
598
|
+
}
|
|
599
|
+
lines.push('');
|
|
600
|
+
return lines.join('\n');
|
|
601
|
+
}
|
|
602
|
+
function doctorCheckStatusLabel(check) {
|
|
603
|
+
return check.status === 'pass' ? 'pass' : check.status === 'fail' ? 'FAIL' : 'skipped';
|
|
604
|
+
}
|
|
605
|
+
/** Pre-flight report: one row per check, then the remediation for each failure. */
|
|
606
|
+
export function renderDoctorReport(report) {
|
|
607
|
+
const tone = report.status === 'ready' ? 'pass' : 'fail';
|
|
608
|
+
const lines = [
|
|
609
|
+
renderCallout(tone, report.status === 'ready'
|
|
610
|
+
? 'Pre-flight checks passed. This configuration is ready to run.'
|
|
611
|
+
: 'Pre-flight checks failed. Fix the items below before running the suite.').replace(/\n$/u, ''),
|
|
612
|
+
renderMetadata([
|
|
613
|
+
['Passed', String(report.counts.pass)],
|
|
614
|
+
['Failed', String(report.counts.fail)],
|
|
615
|
+
['Skipped', String(report.counts.skipped)],
|
|
616
|
+
]).replace(/\n$/u, ''),
|
|
617
|
+
'',
|
|
618
|
+
formatTable([
|
|
619
|
+
{ header: 'CHECK', minWidth: 24 },
|
|
620
|
+
{
|
|
621
|
+
header: 'STATUS',
|
|
622
|
+
minWidth: 8,
|
|
623
|
+
toneOf: (value) => (value === 'pass' ? 'pass' : value === 'FAIL' ? 'fail' : 'meta'),
|
|
624
|
+
},
|
|
625
|
+
{ header: 'DETAIL', minWidth: 30 },
|
|
626
|
+
], report.checks.map((check) => [
|
|
627
|
+
clean(check.label),
|
|
628
|
+
doctorCheckStatusLabel(check),
|
|
629
|
+
clean(check.detail),
|
|
630
|
+
])),
|
|
631
|
+
'',
|
|
632
|
+
];
|
|
633
|
+
const failures = report.checks.filter((check) => check.status === 'fail');
|
|
634
|
+
if (failures.length > 0) {
|
|
635
|
+
lines.push(renderHeading('How to fix').replace(/^\n+|\n+$/gu, ''));
|
|
636
|
+
for (const check of failures) {
|
|
637
|
+
lines.push(`- ${clean(check.label)}: ${clean(check.hint ?? check.detail)}`);
|
|
638
|
+
}
|
|
639
|
+
lines.push('');
|
|
640
|
+
}
|
|
641
|
+
return lines.join('\n');
|
|
642
|
+
}
|
|
643
|
+
//# sourceMappingURL=human.js.map
|