niceeval 0.13.4-canary.49 → 0.14.0

package/dist/show/render.cjs

@@ -26,6 +26,15 @@ const metric = (value, formatValue = fmt) => {
         : `${rendered} (${value.state})`;
 };
 const passRate = (value) => metric(value, (rate) => `${fmt(rate * 100)}%`);
+const duration = (value) => metric(value, (milliseconds) => {
+    if (milliseconds < 1000)
+        return `${fmt(milliseconds)} ms`;
+    if (milliseconds < 60000)
+        return `${fmt(milliseconds / 1000)} s`;
+    if (milliseconds < 3600000)
+        return `${fmt(milliseconds / 60000)} min`;
+    return `${fmt(milliseconds / 3600000)} h`;
+});
 const score = (value) => value.state === "not-scored"
     ? value.state
     : `${fmt(value.earned)}/${fmt(value.possible)} (${value.state}${value.state === "unavailable"
@@ -42,8 +51,10 @@ const aggregateEntries = (value) => ({
                 : `${value.passed} passed; ${value.failed} failed; ${value.errored} errored; ${value.skipped} skipped`,
         },
         { key: "Pass rate", value: passRate(value.passRate) },
-        { key: "Score", value: metric(value.score) },
-        { key: "Duration", value: metric(value.durationMs, (value) => `${fmt(value)} ms`) },
+        ...(value.evaluationKind === "pass"
+            ? []
+            : [{ key: "Score", value: metric(value.score) }]),
+        { key: "Duration", value: duration(value.durationMs) },
         { key: "Tokens", value: metric(value.tokens) },
     ],
 });
@@ -65,35 +76,114 @@ function groupExperiments(value) {
     }
     return [...groups].map(([name, experiments]) => ({ name, experiments }));
 }
-const attemptTable = (cells, group) => ({
-    kind: "table",
-    columns: [
-        { header: "Eval", maxWidth: 55 },
-        { header: "Attempt", maxWidth: 14 },
-        { header: "Score" },
-    ],
-    overflow: "wrap",
-    rows: cells.flatMap((cell) => cell.members.length === 0
-        ? [[
-                relativeToGroup(cell.evalId, group),
-                "not-recorded",
-                metric(cell.aggregate.score),
-            ]]
-        : cell.members.map((member) => [
-            relativeToGroup(cell.evalId, group),
-            member.publication.state === "published"
-                ? member.publication.attemptLocator
-                : member.publication.state === "absent"
-                    ? `absent (${member.publication.reason})`
-                    : "pending",
-            member.publication.state === "published"
-                ? metric(member.publication.score)
-                : member.publication.state,
-        ])),
-});
+const attemptBlocks = (cells, group, all) => {
+    let shownErrors = 0;
+    return cells.flatMap((cell) => {
+        const showScore = cell.aggregate.evaluationKind !== "pass";
+        const members = all
+            ? cell.members
+            : cell.members.filter((member) => {
+                if (member.publication.state !== "published")
+                    return true;
+                if (member.publication.verdict === "passed" ||
+                    member.publication.verdict === "failed")
+                    return false;
+                if (member.publication.verdict !== "errored")
+                    return true;
+                shownErrors += 1;
+                return shownErrors <= 5;
+            });
+        if (cell.members.length > 0 && members.length === 0)
+            return [];
+        return [
+            {
+                kind: "divider",
+                title: `Eval ${relativeToGroup(cell.evalId, group)}`,
+                attachNext: true,
+            },
+            {
+                kind: "table",
+                columns: [
+                    { header: "Attempt", maxWidth: 14 },
+                    { header: "Verdict" },
+                    { header: "Duration" },
+                    ...(showScore ? [{ header: "Score" }] : []),
+                ],
+                overflow: "wrap",
+                rows: members.length === 0
+                    ? [[
+                            "not-recorded",
+                            "not-recorded",
+                            duration(cell.aggregate.durationMs),
+                            ...(showScore ? [metric(cell.aggregate.score)] : []),
+                        ]]
+                    : members.map((member) => [
+                        member.publication.state === "published"
+                            ? member.publication.attemptLocator
+                            : member.publication.state === "absent"
+                                ? `absent (${member.publication.reason})`
+                                : "pending",
+                        member.publication.state === "published"
+                            ? member.publication.verdict ?? "not-recorded"
+                            : member.publication.state,
+                        member.publication.state === "published"
+                            ? duration(member.publication.durationMs)
+                            : member.publication.state,
+                        ...(showScore
+                            ? [member.publication.state === "published"
+                                    ? metric(member.publication.score)
+                                    : member.publication.state]
+                            : []),
+                    ]),
+            },
+        ];
+    });
+};
+const hiddenAttemptSummary = (cells) => {
+    const counts = { passed: 0, failed: 0, errored: 0 };
+    for (const cell of cells) {
+        for (const member of cell.members) {
+            if (member.publication.state !== "published")
+                continue;
+            const verdict = member.publication.verdict;
+            if (verdict === "passed" ||
+                verdict === "failed" ||
+                verdict === "errored")
+                counts[verdict] += 1;
+        }
+    }
+    const hidden = [
+        ...(counts.passed > 0 ? [`${counts.passed} passed`] : []),
+        ...(counts.failed > 0 ? [`${counts.failed} failed`] : []),
+        ...(counts.errored > 5 ? [`${counts.errored - 5} errored`] : []),
+    ];
+    return hidden.length === 0 ? null : `${hidden.join("; ")} Attempts hidden`;
+};
+const compactContinuation = (cells, experimentId, all) => {
+    if (all)
+        return [];
+    const hidden = hiddenAttemptSummary(cells);
+    return hidden === null
+        ? []
+        : [
+            {
+                kind: "divider",
+                title: hidden,
+                attachNext: true,
+            },
+            {
+                kind: "keyValue",
+                entries: [{
+                        key: "See more",
+                        value: `niceeval show --experiment ${experimentId}`,
+                    }],
+            },
+        ];
+};
 const stateCount = (state, count, noun) => `${state}; ${count} ${noun}s`;
 const textOrNotRecorded = (value) => value ?? "not-recorded";
-function renderOverview(value) {
+function renderOverview(value, options = {}) {
+    const all = options.all === true;
     const blocks = [
         {
             kind: "panel",
@@ -109,30 +199,33 @@ function renderOverview(value) {
         blocks.push({
             kind: "panel",
             title: "Experiments",
-            blocks: groups.flatMap((group) => [
-                ...(group.name === null
-                    ? []
-                    : [{ kind: "divider", title: group.name }]),
-                {
-                    kind: "table",
-                    columns: [
-                        { header: "Experiment" },
-                        { header: "Observed" },
-                        { header: "Agent" },
-                        { header: "Model" },
-                        { header: "Pass rate" },
-                        { header: "Score" },
-                    ],
-                    rows: group.experiments.map((experiment) => [
-                        relativeToGroup(experiment.experimentId, group.name),
-                        `${experiment.aggregate.observed}/${experiment.aggregate.expected}`,
-                        executionValue(experiment.agent),
-                        executionValue(experiment.model),
-                        passRate(experiment.aggregate.passRate),
-                        metric(experiment.aggregate.score),
-                    ]),
-                },
-            ]),
+            blocks: groups.flatMap((group) => {
+                const showScore = group.experiments.some(({ aggregate }) => aggregate.evaluationKind !== "pass");
+                return [
+                    ...(group.name === null
+                        ? []
+                        : [{ kind: "divider", title: group.name }]),
+                    {
+                        kind: "table",
+                        columns: [
+                            { header: "Experiment" },
+                            { header: "Observed" },
+                            { header: "Agent" },
+                            { header: "Model" },
+                            { header: "Pass rate" },
+                            ...(showScore ? [{ header: "Score" }] : []),
+                        ],
+                        rows: group.experiments.map((experiment) => [
+                            relativeToGroup(experiment.experimentId, group.name),
+                            `${experiment.aggregate.observed}/${experiment.aggregate.expected}`,
+                            executionValue(experiment.agent),
+                            executionValue(experiment.model),
+                            passRate(experiment.aggregate.passRate),
+                            ...(showScore ? [metric(experiment.aggregate.score)] : []),
+                        ]),
+                    },
+                ];
+            }),
         });
     }
     if (value.cells.length > 0) {
@@ -148,9 +241,9 @@ function renderOverview(value) {
                         {
                             kind: "divider",
                             title: `Experiment ${experiment.experimentId}`,
-                            attachNext: true,
                         },
-                        attemptTable(experimentCells, group.name),
+                        ...attemptBlocks(experimentCells, group.name, all),
+                        ...compactContinuation(experimentCells, experiment.experimentId, all),
                     ];
                 }),
             });
@@ -167,32 +260,7 @@ function renderExperiment(value) {
                 { kind: "divider", title: "Summary" },
                 aggregateEntries(value.aggregate),
                 { kind: "divider", title: "Attempts" },
-                {
-                    kind: "table",
-                    columns: [
-                        { header: "Eval", maxWidth: 55 },
-                        { header: "Attempt", maxWidth: 14 },
-                        { header: "Score" },
-                    ],
-                    overflow: "wrap",
-                    rows: value.cells.flatMap((cell) => cell.members.length === 0
-                        ? [[
-                                cell.evalId,
-                                "not-recorded",
-                                metric(cell.aggregate.score),
-                            ]]
-                        : cell.members.map((member) => [
-                            cell.evalId,
-                            member.publication.state === "published"
-                                ? member.publication.attemptLocator
-                                : member.publication.state === "absent"
-                                    ? `absent (${member.publication.reason})`
-                                    : "pending",
-                            member.publication.state === "published"
-                                ? metric(member.publication.score)
-                                : member.publication.state,
-                        ])),
-                },
+                ...attemptBlocks(value.cells, null, true),
             ],
         },
     ]);