@theoremjs/studio 0.3.4-next.871.gea9bbcc → 0.3.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/cookbooks/catalog.d.ts +1 -1
- package/cookbooks/catalog.js +1 -1
- package/guardrail-bench.d.ts +186 -0
- package/guardrail-bench.js +66 -0
- package/guardrail-probe.d.ts +4 -2
- package/guardrail-probe.js +23 -2
- package/origins.d.ts +1 -1
- package/origins.js +1 -1
- package/package.json +4 -4
- package/page/assets/BaseTypeahead-CHXE162p.js +1 -0
- package/page/assets/{BottomSheet-Ijd3-EiJ.js → BottomSheet-CKodmH_q.js} +1 -1
- package/page/assets/{CollapsibleGroup-BnWI_jMR.js → CollapsibleGroup-YxyQ-UXo.js} +1 -1
- package/page/assets/{ConsoleFrame-sl_08t-S.js → ConsoleFrame-CcHhIel7.js} +1 -1
- package/page/assets/{DateInput-3xmp6QyV.js → DateInput-C2P3VOWH.js} +1 -1
- package/page/assets/{MenuBottomSheet-bFIiEbF_.js → MenuBottomSheet-Be_iagz6.js} +1 -1
- package/page/assets/ProgressBarMarkTooltip-BCgy6kk7.js +1 -0
- package/page/assets/RankedBars-CmyHIG4-.js +1 -0
- package/page/assets/{SchemaFields-DHG2EecV.js → SchemaFields-DvkF1spJ.js} +1 -1
- package/page/assets/{SegmentedControlItem-CTVJbfcv.js → SegmentedControlItem-DRVmwW9x.js} +1 -1
- package/page/assets/{TokenizerBottomSheet-C7wWNjLf.js → TokenizerBottomSheet-BWGxcrKN.js} +1 -1
- package/page/assets/ToolResultCharts-BxJS3q9d.js +1 -0
- package/page/assets/{Toolbar-B0ZpvYnL.js → Toolbar-GwS1PT9V.js} +1 -1
- package/page/assets/TraceInspector-DVBDD7bH.js +1 -0
- package/page/assets/{TraceOverview-Yhf9L-7s.js → TraceOverview-CR3DRfnT.js} +34 -34
- package/page/assets/{_virtual_studio-type-sources-NQ8eY2_H.js → _virtual_studio-type-sources-DiOYmd5J.js} +128 -71
- package/page/assets/browser-engine-A-dNm9JC.js +16 -0
- package/page/assets/{browser-transport-BhFcGvWp.js → browser-transport-J8tzWJNv.js} +2 -2
- package/page/assets/{browserClickWait-F6QvgsDI.js → browserClickWait-BSA4ZCzN.js} +1 -1
- package/page/assets/{export-agent-C1LknzFc.js → export-agent-DdjcE8pP.js} +3 -3
- package/page/assets/getRadiusAndStrokeWidthFromDot-CfhyOFVb.js +1 -0
- package/page/assets/guardrail-health-DXmUdlna.js +3 -0
- package/page/assets/guardrail-tester-D2ym7K4B.js +9 -0
- package/page/assets/index-CkGFbd4l.css +1 -0
- package/page/assets/{index-Hix1w6kY.js → index-DeAchcs2.js} +45 -45
- package/page/assets/{json-code-B13g9rlg.js → json-code-Cxww5ME0.js} +1 -1
- package/page/assets/{prompt-row-DGE5h-0F.js → prompt-row-CKcdEaRi.js} +4 -4
- package/page/assets/registry-Cl2mlSuk.js +1 -0
- package/page/assets/{studio-code-pane-BpYrChnW.js → studio-code-pane-DblixGst.js} +3 -3
- package/page/assets/{studio-console-5ZwlH-1a.js → studio-console-BOmPVlqt.js} +1 -1
- package/page/assets/studio-decision-DVz4JS3y.js +1 -0
- package/page/assets/{studio-embedding-DCMMMVFw.js → studio-embedding-C706qky_.js} +1 -1
- package/page/assets/{studio-host-console-CVKWwl3t.js → studio-host-console-BSLhmC-T.js} +2 -2
- package/page/assets/{studio-labels-h-JoNVLI.js → studio-labels-tTkrGnDa.js} +1 -1
- package/page/assets/{studio-live-DqPhe9Gd.js → studio-live-CvkqZw8r.js} +3 -3
- package/page/assets/{studio-map-Dcz7oT-v.js → studio-map-Cel4mfgK.js} +1 -1
- package/page/assets/studio-trace-replay-BkdHAhIX.js +3 -0
- package/page/assets/{studio-type-checker-DDcpR_0b.js → studio-type-checker-v8KJwpOP.js} +1 -1
- package/page/assets/{use-one-call-Ds7HYrGe.js → use-one-call-C4FbV7xi.js} +1 -1
- package/page/assets/{useFocusLeavingChange-8rK2N7Tp.js → useFocusLeavingChange-CRkFJGdd.js} +1 -1
- package/page/assets/useHighlightedOptionScroll-C7rzQTVc.js +1 -0
- package/page/index.html +11 -11
- package/probe-battery.js +16 -0
- package/runtime-scope.d.ts +2 -0
- package/runtime-scope.js +1 -1
- package/server/bench.d.ts +7 -0
- package/server/bench.js +203 -0
- package/server/save-wire.d.ts +1 -1
- package/server/serve.js +2 -0
- package/ui/guardrail-charts.d.ts +15 -0
- package/ui/guardrail-charts.js +175 -0
- package/ui/guardrail-health.d.ts +10 -0
- package/ui/guardrail-health.js +395 -0
- package/ui/guardrail-matrix.d.ts +17 -0
- package/ui/guardrail-matrix.js +109 -0
- package/ui/guardrail-tester.d.ts +16 -2
- package/ui/guardrail-tester.js +6 -5
- package/ui/studio-screen.js +3 -3
- package/ui/studio.css +69 -0
- package/page/assets/BaseTypeahead-BM3FoulY.js +0 -1
- package/page/assets/ToolResultCharts-DFKefRvo.js +0 -1
- package/page/assets/TraceInspector-ep2qcy9_.js +0 -1
- package/page/assets/browser-engine-B9h8eThv.js +0 -16
- package/page/assets/guardrail-tester-C188dNNz.js +0 -9
- package/page/assets/index-YKE-6URE.css +0 -1
- package/page/assets/registry-Du0mj5wR.js +0 -1
- package/page/assets/studio-decision-DaWLDjAy.js +0 -1
- package/page/assets/studio-trace-replay-DLHxUh4T.js +0 -3
- package/page/assets/useHighlightedOptionScroll-Do3FP80C.js +0 -1
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import { jsx as _jsx, jsxs as _jsxs, Fragment as _Fragment } from "react/jsx-runtime";
|
|
2
|
+
import { Grid } from '@astryxdesign/core/Grid';
|
|
3
|
+
import { DETECTOR_META, DETECTORS } from '@theoremjs/agents/guardrails';
|
|
4
|
+
import { ChartCard, ChartStat, RadialBars, RankedBars, Sunburst, } from '@theoremjs/react/ui';
|
|
5
|
+
import { pooledTime, } from "../guardrail-bench.js";
|
|
6
|
+
import { PROBE_BOUNDARIES, PROBE_BOUNDARY_NOTES } from "../mod.js";
|
|
7
|
+
import { tint } from "./guardrail-matrix.js";
|
|
8
|
+
const rate = new Intl.NumberFormat(undefined, { style: 'percent', maximumFractionDigits: 1 });
|
|
9
|
+
function ms(value) {
|
|
10
|
+
return `${value.toFixed(2)} ms`;
|
|
11
|
+
}
|
|
12
|
+
/** A change in points, or nothing when there is no figure to compare with. */
|
|
13
|
+
function since(now, before) {
|
|
14
|
+
if (now === undefined || before === undefined)
|
|
15
|
+
return undefined;
|
|
16
|
+
const points = (now - before) * 100;
|
|
17
|
+
return `${points > 0 ? '+' : ''}${points.toFixed(1)} pts since the run before`;
|
|
18
|
+
}
|
|
19
|
+
/** Detector scores summed: recall and false alarms over every text they are scored on, and the slowest p95. */
|
|
20
|
+
function pool(scores) {
|
|
21
|
+
const times = scores.flatMap((score) => (score.ms ? [score.ms] : []));
|
|
22
|
+
const slowest = scores.reduce((worst, score) => ((score.ms?.p95 ?? -1) > (worst?.ms?.p95 ?? -1) ? score : worst), undefined);
|
|
23
|
+
return {
|
|
24
|
+
attacks: scores.reduce((sum, score) => sum + (score.attacks ?? 0), 0),
|
|
25
|
+
caught: scores.reduce((sum, score) => sum + (score.caught ?? 0), 0),
|
|
26
|
+
benign: scores.reduce((sum, score) => sum + score.benign, 0),
|
|
27
|
+
fired: scores.reduce((sum, score) => sum + score.falseAlarms, 0),
|
|
28
|
+
...(slowest?.ms && { p95: slowest.ms.p95, slowest: slowest.detector }),
|
|
29
|
+
times,
|
|
30
|
+
texts: 0,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
function poolRun(scores) {
|
|
34
|
+
return {
|
|
35
|
+
...pool(scores.flatMap((dataset) => dataset.detectors)),
|
|
36
|
+
texts: scores.reduce((sum, dataset) => sum + dataset.samples, 0),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function ratio(part, whole) {
|
|
40
|
+
return whole === 0 ? undefined : part / whole;
|
|
41
|
+
}
|
|
42
|
+
function byDetector(scores) {
|
|
43
|
+
return DETECTORS.flatMap((detector) => {
|
|
44
|
+
const here = scores.flatMap((dataset) => dataset.detectors.filter((score) => score.detector === detector));
|
|
45
|
+
return here.length === 0 ? [] : [{ detector, pool: pool(here) }];
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
function bars(pools, value, detail, fill) {
|
|
49
|
+
return pools
|
|
50
|
+
.flatMap(({ detector, pool: each }) => {
|
|
51
|
+
const figure = value(each);
|
|
52
|
+
return figure === undefined
|
|
53
|
+
? []
|
|
54
|
+
: [
|
|
55
|
+
{
|
|
56
|
+
id: detector,
|
|
57
|
+
name: DETECTOR_META[detector].label,
|
|
58
|
+
value: figure,
|
|
59
|
+
detail: detail(each),
|
|
60
|
+
...(fill && { fill }),
|
|
61
|
+
},
|
|
62
|
+
];
|
|
63
|
+
})
|
|
64
|
+
.sort((a, b) => b.value - a.value);
|
|
65
|
+
}
|
|
66
|
+
function shown(value, format) {
|
|
67
|
+
return value === undefined ? '—' : format(value);
|
|
68
|
+
}
|
|
69
|
+
/** Who the slowest p95 is, and the median and mean of every timing when the run kept them. */
|
|
70
|
+
function timeDetail(now) {
|
|
71
|
+
const pooled = pooledTime(now.times);
|
|
72
|
+
return [
|
|
73
|
+
now.slowest ? `${DETECTOR_META[now.slowest].label}, the slowest detector` : '',
|
|
74
|
+
pooled
|
|
75
|
+
? `Median ≈ ${ms(pooled.median)} · mean ${ms(pooled.mean)}`
|
|
76
|
+
: 'This run kept no median or mean',
|
|
77
|
+
];
|
|
78
|
+
}
|
|
79
|
+
/** Texts by the boundary they cross, then by dataset. */
|
|
80
|
+
function textParts(scores) {
|
|
81
|
+
return PROBE_BOUNDARIES.flatMap((boundary) => {
|
|
82
|
+
const here = scores.filter((dataset) => dataset.boundary === boundary);
|
|
83
|
+
if (here.length === 0)
|
|
84
|
+
return [];
|
|
85
|
+
const texts = here.reduce((sum, dataset) => sum + dataset.samples, 0);
|
|
86
|
+
return [
|
|
87
|
+
{
|
|
88
|
+
id: boundary,
|
|
89
|
+
name: PROBE_BOUNDARY_NOTES[boundary].label,
|
|
90
|
+
detail: `${String(here.length)} datasets`,
|
|
91
|
+
children: here.map((dataset) => ({
|
|
92
|
+
id: dataset.dataset,
|
|
93
|
+
name: dataset.dataset,
|
|
94
|
+
value: dataset.samples,
|
|
95
|
+
detail: `${rate.format(dataset.samples / texts)} of this boundary`,
|
|
96
|
+
})),
|
|
97
|
+
},
|
|
98
|
+
];
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
const count = (value) => `${value.toLocaleString()} texts`;
|
|
102
|
+
/** The run in four figures, each pooled over every detector on every dataset shown. */
|
|
103
|
+
function RunStats({ now, before, datasets }) {
|
|
104
|
+
const recall = ratio(now.caught, now.attacks);
|
|
105
|
+
const alarms = ratio(now.fired, now.benign);
|
|
106
|
+
return (_jsxs(Grid, { columns: { minWidth: 140, repeat: 'fit' }, gap: 2, children: [_jsx(ChartStat, { label: "Recall", value: shown(recall, (value) => rate.format(value)), detail: [
|
|
107
|
+
`${now.caught.toLocaleString()} of ${now.attacks.toLocaleString()} attacks`,
|
|
108
|
+
since(recall, before && ratio(before.caught, before.attacks)),
|
|
109
|
+
] }), _jsx(ChartStat, { label: "False alarms", value: shown(alarms, (value) => rate.format(value)), detail: [
|
|
110
|
+
`${now.fired.toLocaleString()} of ${now.benign.toLocaleString()} benign texts`,
|
|
111
|
+
since(alarms, before && ratio(before.fired, before.benign)),
|
|
112
|
+
] }), _jsx(ChartStat, { label: "Detector time, p95", value: shown(now.p95, ms), detail: timeDetail(now) }), _jsx(ChartStat, { label: "Texts read", value: now.texts.toLocaleString(), detail: `${String(datasets)} datasets` })] }));
|
|
113
|
+
}
|
|
114
|
+
/** A ranking in a card, headed by its first bar; nothing when there is no bar. */
|
|
115
|
+
function RankCard({ title, lead, rows, format, max, }) {
|
|
116
|
+
const top = rows[0];
|
|
117
|
+
if (!top)
|
|
118
|
+
return null;
|
|
119
|
+
return (_jsx(ChartCard, { title: title, value: format(top.value), note: `${top.name}, the ${lead}`, children: _jsx(RankedBars, { rows: rows, format: format, ...(max !== undefined && { max }) }) }));
|
|
120
|
+
}
|
|
121
|
+
const percentOf = (value) => rate.format(value);
|
|
122
|
+
/** Each detector's recall as a ring, the lowest outermost, shaded as the matrix shades it. */
|
|
123
|
+
function RecallCard({ rows }) {
|
|
124
|
+
const top = rows[0];
|
|
125
|
+
if (!top)
|
|
126
|
+
return null;
|
|
127
|
+
return (_jsx(ChartCard, { title: "Recall by detector", value: percentOf(top.value), note: `${top.name}, the lowest`, children: _jsx(RadialBars, { rows: rows.map((row) => ({ ...row, color: tint(row.value) })), format: percentOf, max: 1 }) }));
|
|
128
|
+
}
|
|
129
|
+
/** Where the texts come from: the boundary each crosses on the inner ring, its dataset outside. */
|
|
130
|
+
function TextsCard({ scores }) {
|
|
131
|
+
const parts = textParts(scores);
|
|
132
|
+
if (parts.length === 0)
|
|
133
|
+
return null;
|
|
134
|
+
const texts = scores.reduce((sum, dataset) => sum + dataset.samples, 0);
|
|
135
|
+
return (_jsx(ChartCard, { title: "Where the texts come from", value: texts.toLocaleString(), note: `texts at ${String(parts.length)} ${parts.length === 1 ? 'boundary' : 'boundaries'}`, children: _jsx(Sunburst, { parts: parts, format: count }) }));
|
|
136
|
+
}
|
|
137
|
+
const PAIR = { minWidth: 360, repeat: 'fit' };
|
|
138
|
+
/** Each detector's recall, false alarms and p95 time, pooled over the datasets shown. */
|
|
139
|
+
function DetectorCharts({ scores }) {
|
|
140
|
+
const detectors = byDetector(scores);
|
|
141
|
+
return (_jsxs(_Fragment, { children: [_jsxs(Grid, { columns: PAIR, gap: 2, children: [_jsx(RecallCard, { rows: bars(detectors, (each) => ratio(each.caught, each.attacks), (each) => `${String(each.caught)} of ${String(each.attacks)} attacks`).reverse() }), _jsx(TextsCard, { scores: scores })] }), _jsxs(Grid, { columns: PAIR, gap: 2, children: [_jsx(RankCard, { title: "False alarms by detector", lead: "highest", rows: bars(detectors, (each) => ratio(each.fired, each.benign), (each) => `${String(each.fired)} of ${String(each.benign)} benign texts`, 'shade1'), format: percentOf }), _jsx(RankCard, { title: "Time by detector, p95", lead: "slowest", rows: bars(detectors, (each) => each.p95, () => 'On its slowest dataset', 'shade2'), format: ms })] })] }));
|
|
142
|
+
}
|
|
143
|
+
/** The run's figures, then each detector ranked by each of them. */
|
|
144
|
+
export function HealthCharts({ scores, previous, }) {
|
|
145
|
+
return (_jsxs(_Fragment, { children: [_jsx(RunStats, { now: poolRun(scores), ...(previous && { before: poolRun(previous) }), datasets: scores.length }), _jsx(DetectorCharts, { scores: scores })] }));
|
|
146
|
+
}
|
|
147
|
+
/** One detector on one dataset in three figures. */
|
|
148
|
+
export function ScoreStats({ score }) {
|
|
149
|
+
return (_jsxs(Grid, { columns: { minWidth: 140, repeat: 'fit' }, gap: 2, children: [_jsx(ChartStat, { label: "Recall", value: shown(score.recall, percentOf), detail: score.attacks === undefined
|
|
150
|
+
? 'This dataset does not label what it looks for'
|
|
151
|
+
: `${String(score.caught)} of ${String(score.attacks)} attacks` }), _jsx(ChartStat, { label: "False alarms", value: shown(score.falseAlarmRate, percentOf), detail: `${String(score.falseAlarms)} of ${String(score.benign)} benign texts` }), _jsx(ChartStat, { label: "p95 time alone", value: score.ms ? ms(score.ms.p95) : '—', detail: score.ms
|
|
152
|
+
? [`p50 ${ms(score.ms.p50)}`, `max ${ms(score.ms.max)}`]
|
|
153
|
+
: 'Timed only inside a probe turn' })] }));
|
|
154
|
+
}
|
|
155
|
+
/** The categories with the most of `outcome`, as a share of each category's texts. */
|
|
156
|
+
export function CategoryChart({ score, outcome }) {
|
|
157
|
+
if (outcome === 'caught')
|
|
158
|
+
return null;
|
|
159
|
+
const isMissed = outcome === 'missed';
|
|
160
|
+
const categories = isMissed
|
|
161
|
+
? score.attacksByCategory.map((each) => ({ ...each, count: each.samples - each.fired }))
|
|
162
|
+
: score.byCategory.map((each) => ({ ...each, count: each.fired }));
|
|
163
|
+
const rows = categories
|
|
164
|
+
.filter((each) => each.count > 0)
|
|
165
|
+
.map((each) => ({
|
|
166
|
+
id: each.category,
|
|
167
|
+
name: each.category,
|
|
168
|
+
value: each.count / each.samples,
|
|
169
|
+
detail: `${String(each.count)} of ${String(each.samples)}`,
|
|
170
|
+
fill: isMissed ? 'clear' : 'shade1',
|
|
171
|
+
}))
|
|
172
|
+
.sort((a, b) => b.value - a.value)
|
|
173
|
+
.slice(0, 8);
|
|
174
|
+
return (_jsx(RankCard, { title: isMissed ? 'Missed, by category' : 'False alarms, by category', lead: "highest", rows: rows, format: percentOf, max: 1 }));
|
|
175
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { type StudioRunPayload } from '../mod.ts';
|
|
2
|
+
/**
|
|
3
|
+
* The guardrail tester, and beside it the bench's Health view when the project's server has a
|
|
4
|
+
* bench run: a checkout of Theorem that ran `deno task bench:guardrails`. Anywhere else it is the
|
|
5
|
+
* tester alone.
|
|
6
|
+
*/
|
|
7
|
+
export declare function GuardrailPane({ payload, isWide }: {
|
|
8
|
+
payload: StudioRunPayload;
|
|
9
|
+
isWide: boolean;
|
|
10
|
+
}): import("react").JSX.Element;
|
|
@@ -0,0 +1,395 @@
|
|
|
1
|
+
import { jsx as _jsx, jsxs as _jsxs } from "react/jsx-runtime";
|
|
2
|
+
import { Banner } from '@astryxdesign/core/Banner';
|
|
3
|
+
import { Button } from '@astryxdesign/core/Button';
|
|
4
|
+
import { HStack } from '@astryxdesign/core/HStack';
|
|
5
|
+
import { Icon } from '@astryxdesign/core/Icon';
|
|
6
|
+
import { Item } from '@astryxdesign/core/Item';
|
|
7
|
+
import { Link } from '@astryxdesign/core/Link';
|
|
8
|
+
import { MetadataList, MetadataListItem } from '@astryxdesign/core/MetadataList';
|
|
9
|
+
import { ProgressBar } from '@astryxdesign/core/ProgressBar';
|
|
10
|
+
import { SegmentedControl, SegmentedControlItem } from '@astryxdesign/core/SegmentedControl';
|
|
11
|
+
import { StackItem } from '@astryxdesign/core/Stack';
|
|
12
|
+
import { pixel, proportional, Table, useTableSortable, useTableSortableState, } from '@astryxdesign/core/Table';
|
|
13
|
+
import { Text } from '@astryxdesign/core/Text';
|
|
14
|
+
import { Token } from '@astryxdesign/core/Token';
|
|
15
|
+
import { VStack } from '@astryxdesign/core/VStack';
|
|
16
|
+
import { IconArrowLeft, IconHeartbeat, IconShieldSearch } from '@tabler/icons-react';
|
|
17
|
+
import { DETECT_ACTION_META, DETECT_RULES, DETECTOR_META, } from '@theoremjs/agents/guardrails';
|
|
18
|
+
import { PaneLayout, PanePanel, PaneState } from '@theoremjs/react/ui';
|
|
19
|
+
import { useEffect, useState } from 'react';
|
|
20
|
+
import { BENCH_OUTCOMES, ROLES, } from "../guardrail-bench.js";
|
|
21
|
+
import { PROBE_BOUNDARY_NOTES, PROBE_TEXT_LIMIT, } from "../mod.js";
|
|
22
|
+
import { CategoryChart, HealthCharts, ScoreStats } from "./guardrail-charts.js";
|
|
23
|
+
import { GuardrailMatrix, MATRIX_FIGURE_LABELS, MATRIX_FIGURES, } from "./guardrail-matrix.js";
|
|
24
|
+
import { GuardrailTester, sendProbe, Verdict } from "./guardrail-tester.js";
|
|
25
|
+
import { useProject } from "./lib/studio-project.js";
|
|
26
|
+
const OUTCOME_LABELS = {
|
|
27
|
+
false_alarm: 'False alarms',
|
|
28
|
+
missed: 'Missed',
|
|
29
|
+
caught: 'Caught',
|
|
30
|
+
};
|
|
31
|
+
const ROLE_LABELS = {
|
|
32
|
+
train: 'Train',
|
|
33
|
+
validate: 'Validate',
|
|
34
|
+
test: 'Test',
|
|
35
|
+
holdout: 'Hold-out',
|
|
36
|
+
};
|
|
37
|
+
function percent(value) {
|
|
38
|
+
return value === undefined ? '—' : `${(value * 100).toFixed(1)}%`;
|
|
39
|
+
}
|
|
40
|
+
function milliseconds(value) {
|
|
41
|
+
return value === undefined ? '—' : `${value.toFixed(2)} ms`;
|
|
42
|
+
}
|
|
43
|
+
/** A change in points since the run before, or nothing when either side has no rate. */
|
|
44
|
+
function change(now, before) {
|
|
45
|
+
if (now === undefined || before === undefined)
|
|
46
|
+
return '';
|
|
47
|
+
const points = (now - before) * 100;
|
|
48
|
+
if (Math.abs(points) < 0.05)
|
|
49
|
+
return '±0';
|
|
50
|
+
return `${points > 0 ? '+' : ''}${points.toFixed(1)}`;
|
|
51
|
+
}
|
|
52
|
+
function scoreRows(scores, previous, running) {
|
|
53
|
+
const before = new Map((previous?.scores ?? []).flatMap((dataset) => dataset.detectors.map((d) => [`${d.dataset}/${d.detector}`, d])));
|
|
54
|
+
return scores.flatMap((dataset) => dataset.detectors.map((score) => {
|
|
55
|
+
const id = `${score.dataset}/${score.detector}`;
|
|
56
|
+
const prior = before.get(id);
|
|
57
|
+
return {
|
|
58
|
+
id,
|
|
59
|
+
dataset: score.dataset,
|
|
60
|
+
role: score.role,
|
|
61
|
+
boundary: score.boundary,
|
|
62
|
+
detector: score.detector,
|
|
63
|
+
recall: score.recall,
|
|
64
|
+
falseAlarmRate: score.falseAlarmRate,
|
|
65
|
+
p50: score.ms?.p50,
|
|
66
|
+
p95: score.ms?.p95,
|
|
67
|
+
recallBefore: prior?.recall,
|
|
68
|
+
falseAlarmBefore: prior?.falseAlarmRate,
|
|
69
|
+
thin: score.thin,
|
|
70
|
+
running: score.dataset === running,
|
|
71
|
+
score,
|
|
72
|
+
};
|
|
73
|
+
}));
|
|
74
|
+
}
|
|
75
|
+
/** A licence as short as it reads: the upstream's own words after the first are left to its page. */
|
|
76
|
+
function shortLicense(license) {
|
|
77
|
+
return license.startsWith('other') ? 'Other' : license;
|
|
78
|
+
}
|
|
79
|
+
/** A dataset's name, opening its source page, with its licence beside it. */
|
|
80
|
+
function DatasetName({ id, facts }) {
|
|
81
|
+
return (_jsxs(HStack, { gap: 1, align: "center", wrap: "wrap", children: [facts?.source ? (_jsx(Link, { href: facts.source, isExternalLink: true, label: id })) : (_jsx(Text, { type: "supporting", children: id })), facts?.license && (_jsx(Text, { type: "supporting", color: "secondary", children: shortLicense(facts.license) }))] }));
|
|
82
|
+
}
|
|
83
|
+
function RateCell({ now, before }) {
|
|
84
|
+
const moved = change(now, before);
|
|
85
|
+
return (_jsxs(HStack, { gap: 1, align: "center", children: [_jsx(Text, { type: "supporting", children: percent(now) }), moved && (_jsx(Text, { type: "supporting", color: "secondary", children: moved }))] }));
|
|
86
|
+
}
|
|
87
|
+
function ScoresTable({ rows, facts, onOpen, }) {
|
|
88
|
+
const { sortedData, sortConfig } = useTableSortableState({
|
|
89
|
+
data: rows,
|
|
90
|
+
defaultSort: [{ sortKey: 'falseAlarmRate', direction: 'descending' }],
|
|
91
|
+
});
|
|
92
|
+
const sortable = useTableSortable(sortConfig);
|
|
93
|
+
return (_jsx(Table, { density: "compact", dividers: "rows", hasHover: true, idKey: "id", data: sortedData, plugins: { sortable }, columns: [
|
|
94
|
+
{
|
|
95
|
+
key: 'detector',
|
|
96
|
+
header: 'Detector',
|
|
97
|
+
width: proportional(1, { minWidth: 120 }),
|
|
98
|
+
sortable: true,
|
|
99
|
+
renderCell: (row) => (_jsx(Button, { label: DETECTOR_META[row.detector].label, variant: "ghost", size: "sm", onClick: () => {
|
|
100
|
+
onOpen(row);
|
|
101
|
+
} })),
|
|
102
|
+
},
|
|
103
|
+
{
|
|
104
|
+
key: 'dataset',
|
|
105
|
+
header: 'Dataset',
|
|
106
|
+
width: proportional(1, { minWidth: 180 }),
|
|
107
|
+
sortable: true,
|
|
108
|
+
renderCell: (row) => (_jsxs(HStack, { gap: 1, align: "center", wrap: "wrap", children: [_jsx(DatasetName, { id: row.dataset, facts: facts.get(row.dataset) }), row.running && _jsx(Token, { label: "Running", size: "sm", color: "blue" }), row.thin && !row.running && _jsx(Token, { label: "Few rows", size: "sm", color: "gray" })] })),
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
key: 'recall',
|
|
112
|
+
header: 'Recall',
|
|
113
|
+
width: pixel(110),
|
|
114
|
+
sortable: true,
|
|
115
|
+
renderCell: (row) => _jsx(RateCell, { now: row.recall, before: row.recallBefore }),
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
key: 'falseAlarmRate',
|
|
119
|
+
header: 'False alarms',
|
|
120
|
+
width: pixel(120),
|
|
121
|
+
sortable: true,
|
|
122
|
+
renderCell: (row) => _jsx(RateCell, { now: row.falseAlarmRate, before: row.falseAlarmBefore }),
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
key: 'p50',
|
|
126
|
+
header: 'p50',
|
|
127
|
+
width: pixel(90),
|
|
128
|
+
sortable: true,
|
|
129
|
+
renderCell: (row) => _jsx(Text, { type: "supporting", children: milliseconds(row.p50) }),
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
key: 'p95',
|
|
133
|
+
header: 'p95',
|
|
134
|
+
width: pixel(90),
|
|
135
|
+
sortable: true,
|
|
136
|
+
renderCell: (row) => _jsx(Text, { type: "supporting", children: milliseconds(row.p95) }),
|
|
137
|
+
},
|
|
138
|
+
] }));
|
|
139
|
+
}
|
|
140
|
+
function duration(seconds) {
|
|
141
|
+
const minutes = Math.floor(seconds / 60);
|
|
142
|
+
return minutes > 0
|
|
143
|
+
? `${String(minutes)} min ${(seconds % 60).toFixed(0)} s`
|
|
144
|
+
: `${seconds.toFixed(0)} s`;
|
|
145
|
+
}
|
|
146
|
+
/** Time as a timer shows it: 4:05, or 1:02:33 past an hour. */
|
|
147
|
+
function clock(seconds) {
|
|
148
|
+
const whole = Math.floor(seconds);
|
|
149
|
+
const two = (value) => String(value).padStart(2, '0');
|
|
150
|
+
const hours = Math.floor(whole / 3600);
|
|
151
|
+
const minutes = Math.floor((whole % 3600) / 60);
|
|
152
|
+
const rest = `${two(minutes)}:${two(whole % 60)}`;
|
|
153
|
+
return hours > 0 ? `${String(hours)}:${rest}` : `${String(minutes)}:${two(whole % 60)}`;
|
|
154
|
+
}
|
|
155
|
+
function Skipped({ skipped }) {
|
|
156
|
+
if (skipped.length === 0)
|
|
157
|
+
return null;
|
|
158
|
+
return (_jsx(Banner, { status: "warning", title: `${String(skipped.length)} datasets not run`, description: skipped.map((skip) => skip.dataset).join(', ') }));
|
|
159
|
+
}
|
|
160
|
+
const RUN_TIME = new Intl.DateTimeFormat(undefined, { dateStyle: 'medium', timeStyle: 'short' });
|
|
161
|
+
/** A run's name, the UTC time it started as a folder name, as a local date and time. */
|
|
162
|
+
function runLabel(run) {
|
|
163
|
+
const [day = '', time = ''] = run.split('T');
|
|
164
|
+
const started = new Date(`${day}T${time.replace(/-/g, ':')}`);
|
|
165
|
+
return Number.isNaN(started.getTime()) ? run : RUN_TIME.format(started);
|
|
166
|
+
}
|
|
167
|
+
/** The run in short: what it ran, on which commit, and what it could not read. */
|
|
168
|
+
function RunFacts({ summary, previous }) {
|
|
169
|
+
return (_jsxs(VStack, { gap: 2, children: [_jsxs(MetadataList, { columns: 2, children: [_jsx(MetadataListItem, { label: "Run", children: runLabel(summary.run) }), _jsx(MetadataListItem, { label: "Commit", children: summary.commit.slice(0, 8) }), _jsx(MetadataListItem, { label: "Took", children: duration(summary.seconds) }), _jsx(MetadataListItem, { label: "Compared with", children: previous ? runLabel(previous.run) : '—' })] }), _jsx(Skipped, { skipped: summary.skipped })] }));
|
|
170
|
+
}
|
|
171
|
+
/** A run still going: how far through its datasets it is, and the one it is on. */
|
|
172
|
+
function LiveFacts({ live, previous }) {
|
|
173
|
+
const finished = live.scores.length - (live.current ? 1 : 0) + live.skipped.length;
|
|
174
|
+
const now = live.current
|
|
175
|
+
? ` · currently on ${live.current.dataset} · ${live.current.rows.toLocaleString()} rows`
|
|
176
|
+
: '';
|
|
177
|
+
return (_jsxs(VStack, { gap: 2, children: [_jsx(ProgressBar, { label: `${String(finished)} of ${String(live.datasets.length)} datasets${now}`, value: finished, max: live.datasets.length, hasValueLabel: true, formatValueLabel: (value, max) => `${String(Math.round((value / max) * 100))}% · ${clock(live.seconds)}` }), _jsxs(MetadataList, { columns: 2, children: [_jsx(MetadataListItem, { label: "Run", children: runLabel(live.run) }), _jsx(MetadataListItem, { label: "Compared with", children: previous ? runLabel(previous.run) : '—' })] }), _jsx(Skipped, { skipped: live.skipped })] }));
|
|
178
|
+
}
|
|
179
|
+
/** `text` with each of `matches` marked where it occurs. */
|
|
180
|
+
function Marked({ text, matches }) {
|
|
181
|
+
const wanted = matches.filter((match) => match.length > 0);
|
|
182
|
+
if (wanted.length === 0)
|
|
183
|
+
return text;
|
|
184
|
+
const parts = [];
|
|
185
|
+
let at = 0;
|
|
186
|
+
while (at < text.length) {
|
|
187
|
+
let next = -1;
|
|
188
|
+
let found = '';
|
|
189
|
+
for (const match of wanted) {
|
|
190
|
+
const index = text.indexOf(match, at);
|
|
191
|
+
if (index >= 0 && (next < 0 || index < next)) {
|
|
192
|
+
next = index;
|
|
193
|
+
found = match;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
if (next < 0)
|
|
197
|
+
break;
|
|
198
|
+
parts.push(text.slice(at, next));
|
|
199
|
+
parts.push(_jsx("mark", { className: "studio-bench-mark", children: found }, next));
|
|
200
|
+
at = next + found.length;
|
|
201
|
+
}
|
|
202
|
+
parts.push(text.slice(at));
|
|
203
|
+
return parts;
|
|
204
|
+
}
|
|
205
|
+
function matchesOf(row, detector) {
|
|
206
|
+
const rule = DETECT_RULES[detector];
|
|
207
|
+
return (row.events ?? []).flatMap((event) => event.hits.flatMap((hit) => (hit.rule === rule && hit.match ? [hit.match] : [])));
|
|
208
|
+
}
|
|
209
|
+
/** Why a row cannot go through the probe again, or undefined when it can. */
|
|
210
|
+
function unprobeable(row) {
|
|
211
|
+
if (row.text === undefined)
|
|
212
|
+
return 'The run kept no text for this row.';
|
|
213
|
+
if (row.text.length > PROBE_TEXT_LIMIT)
|
|
214
|
+
return 'Longer than the probe takes.';
|
|
215
|
+
return undefined;
|
|
216
|
+
}
|
|
217
|
+
/** A row the detector counted: its text, what matched, and the way to send it through the probe again. */
|
|
218
|
+
function BenchRowView({ row, detector, payload, }) {
|
|
219
|
+
const [busy, setBusy] = useState(false);
|
|
220
|
+
const [again, setAgain] = useState(null);
|
|
221
|
+
const reason = unprobeable(row);
|
|
222
|
+
const answer = again?.answers?.find((each) => each.boundary === row.boundary);
|
|
223
|
+
return (_jsx(PanePanel, { title: `${row.category} · ${row.id}`, children: _jsxs(VStack, { gap: 2, children: [_jsx(Text, { type: "supporting", maxLines: 8, children: row.text === undefined ? ('—') : (_jsx(Marked, { text: row.text, matches: matchesOf(row, detector) })) }), _jsxs(HStack, { gap: 2, align: "center", wrap: "wrap", children: [_jsx(Button, { label: "Probe the draft", size: "sm", variant: "ghost", tooltip: reason, isLoading: busy, isDisabled: reason !== undefined, onClick: () => {
|
|
224
|
+
if (row.text === undefined)
|
|
225
|
+
return;
|
|
226
|
+
setBusy(true);
|
|
227
|
+
void sendProbe(payload, row.text, detector).then((outcome) => {
|
|
228
|
+
setAgain(outcome);
|
|
229
|
+
setBusy(false);
|
|
230
|
+
});
|
|
231
|
+
} }), answer && _jsx(Verdict, { answer: answer }), again?.error && _jsx(Text, { color: "secondary", children: again.error })] })] }) }));
|
|
232
|
+
}
|
|
233
|
+
function useBenchRows(endpoint, run, score, outcome) {
|
|
234
|
+
const [rows, setRows] = useState(null);
|
|
235
|
+
const query = new URLSearchParams({
|
|
236
|
+
run,
|
|
237
|
+
dataset: score.dataset,
|
|
238
|
+
detector: score.detector,
|
|
239
|
+
outcome,
|
|
240
|
+
});
|
|
241
|
+
const key = query.toString();
|
|
242
|
+
useEffect(() => {
|
|
243
|
+
let live = true;
|
|
244
|
+
fetch(`${endpoint}/bench/rows?${key}`)
|
|
245
|
+
.then((response) => (response.ok ? response.json() : null))
|
|
246
|
+
.then((body) => {
|
|
247
|
+
if (live)
|
|
248
|
+
setRows({ key, rows: body?.rows ?? 'failed' });
|
|
249
|
+
})
|
|
250
|
+
.catch(() => {
|
|
251
|
+
if (live)
|
|
252
|
+
setRows({ key, rows: 'failed' });
|
|
253
|
+
});
|
|
254
|
+
return () => {
|
|
255
|
+
live = false;
|
|
256
|
+
};
|
|
257
|
+
}, [endpoint, key]);
|
|
258
|
+
return rows?.key === key ? rows.rows : undefined;
|
|
259
|
+
}
|
|
260
|
+
function OutcomeRows({ endpoint, run, score, outcome, payload, }) {
|
|
261
|
+
const rows = useBenchRows(endpoint, run, score, outcome);
|
|
262
|
+
if (rows === undefined)
|
|
263
|
+
return null;
|
|
264
|
+
if (rows === 'failed')
|
|
265
|
+
return _jsx(Banner, { status: "error", title: "Couldn't read the run's rows." });
|
|
266
|
+
if (rows.length === 0)
|
|
267
|
+
return _jsx(Text, { color: "secondary", children: "None." });
|
|
268
|
+
return (_jsx(VStack, { gap: 3, children: rows.map((row) => (_jsx(BenchRowView, { row: row, detector: score.detector, payload: payload }, row.id))) }));
|
|
269
|
+
}
|
|
270
|
+
/** One detector on one dataset: its figures, its worst categories, and the rows behind them. */
|
|
271
|
+
function ScoreDetail({ row, facts, run, endpoint, payload, isClose, onBack, }) {
|
|
272
|
+
const { score } = row;
|
|
273
|
+
const outcomes = BENCH_OUTCOMES.filter((each) => each === 'false_alarm' || score.attacks);
|
|
274
|
+
const [outcome, setOutcome] = useState('false_alarm');
|
|
275
|
+
return (_jsxs(VStack, { gap: 4, padding: 4, children: [_jsx(HStack, { children: _jsx(Button, { label: isClose ? 'Close' : 'Back', variant: "ghost", size: "sm", icon: _jsx(Icon, { icon: IconArrowLeft }), onClick: onBack }) }), _jsx(Item, { label: _jsx(Text, { weight: "semibold", children: DETECTOR_META[score.detector].label }), description: DETECTOR_META[score.detector].doc, align: "start" }), _jsxs(MetadataList, { children: [_jsx(MetadataListItem, { label: "Dataset", children: _jsx(DatasetName, { id: score.dataset, facts: facts }) }), _jsx(MetadataListItem, { label: "Boundary", children: PROBE_BOUNDARY_NOTES[score.boundary].label }), _jsx(MetadataListItem, { label: "Default action", children: DETECT_ACTION_META[score.defaultAction].label })] }), _jsx(ScoreStats, { score: score }), _jsx(SegmentedControl, { label: "Rows", size: "sm", value: outcome, onChange: (next) => {
|
|
276
|
+
setOutcome(outcomes.find((each) => each === next) ?? 'false_alarm');
|
|
277
|
+
}, children: outcomes.map((each) => (_jsx(SegmentedControlItem, { value: each, label: OUTCOME_LABELS[each] }, each))) }), _jsx(CategoryChart, { score: score, outcome: outcome }), _jsx(OutcomeRows, { endpoint: endpoint, run: run, score: score, outcome: outcome, payload: payload })] }));
|
|
278
|
+
}
|
|
279
|
+
function rolesIn(scores) {
|
|
280
|
+
return ROLES.filter((role) => scores.some((score) => score.role === role));
|
|
281
|
+
}
|
|
282
|
+
/** What the view shows of an answer: a run still going against the last whole run, or a finished run against the run before it. */
|
|
283
|
+
function shownRun(answer) {
|
|
284
|
+
const { live, summary } = answer;
|
|
285
|
+
return {
|
|
286
|
+
previous: live ? summary : answer.previous,
|
|
287
|
+
scores: live?.scores ?? summary?.scores ?? [],
|
|
288
|
+
run: live?.run ?? summary?.run ?? '',
|
|
289
|
+
facts: new Map([...(summary?.datasets ?? []), ...(live?.datasets ?? [])].map((each) => [each.id, each])),
|
|
290
|
+
};
|
|
291
|
+
}
|
|
292
|
+
function RolePicker({ roles, role, onChange, }) {
|
|
293
|
+
return (_jsxs(SegmentedControl, { label: "Datasets", size: "sm", value: role, onChange: (next) => {
|
|
294
|
+
onChange(roles.find((each) => each === next) ?? 'all');
|
|
295
|
+
}, children: [_jsx(SegmentedControlItem, { value: "all", label: "All" }), roles.map((each) => (_jsx(SegmentedControlItem, { value: each, label: ROLE_LABELS[each] }, each)))] }));
|
|
296
|
+
}
|
|
297
|
+
/** Table or matrix, what the matrix is shaded by, and the place a matrix cell narrowed the table to. */
|
|
298
|
+
function ViewPicker({ view, figure, place, onView, onFigure, onClear, }) {
|
|
299
|
+
return (_jsxs(HStack, { gap: 2, align: "center", wrap: "wrap", children: [_jsxs(SegmentedControl, { label: "View", size: "sm", value: view, onChange: (next) => {
|
|
300
|
+
onView(next === 'matrix' ? 'matrix' : 'table');
|
|
301
|
+
}, children: [_jsx(SegmentedControlItem, { value: "table", label: "Table" }), _jsx(SegmentedControlItem, { value: "matrix", label: "Matrix" })] }), view === 'matrix' && (_jsx(SegmentedControl, { label: "Shaded by", size: "sm", value: figure, onChange: (next) => {
|
|
302
|
+
onFigure(MATRIX_FIGURES.find((each) => each === next) ?? 'recall');
|
|
303
|
+
}, children: MATRIX_FIGURES.map((each) => (_jsx(SegmentedControlItem, { value: each, label: MATRIX_FIGURE_LABELS[each] }, each))) })), view === 'table' && place && (_jsx(Token, { size: "sm", label: `${DETECTOR_META[place.detector].label} at ${PROBE_BOUNDARY_NOTES[place.boundary].label}`, onRemove: onClear }))] }));
|
|
304
|
+
}
|
|
305
|
+
/**
|
|
306
|
+
* The guardrails bench's last run: each detector's recall, false alarms and time on each dataset,
|
|
307
|
+
* against the run before; a row opens on the texts behind its figures.
|
|
308
|
+
*/
|
|
309
|
+
function GuardrailHealth({ answer, endpoint, payload, isWide, aside, }) {
|
|
310
|
+
const { previous, scores, run, facts } = shownRun(answer);
|
|
311
|
+
const [role, setRole] = useState('all');
|
|
312
|
+
const [view, setView] = useState('table');
|
|
313
|
+
const [figure, setFigure] = useState('recall');
|
|
314
|
+
const [place, setPlace] = useState(null);
|
|
315
|
+
const [open, setOpen] = useState(null);
|
|
316
|
+
const inRole = scores.filter((dataset) => role === 'all' || dataset.role === role);
|
|
317
|
+
const rows = scoreRows(inRole, previous, answer.live?.current?.dataset).filter((row) => !place || (row.boundary === place.boundary && row.detector === place.detector));
|
|
318
|
+
const overview = (_jsxs(VStack, { gap: 4, padding: 4, children: [_jsxs(HStack, { gap: 3, align: "center", children: [_jsx(StackItem, { size: "fill", children: _jsx(Text, { weight: "semibold", children: "Guardrails health" }) }), aside] }), answer.live ? (_jsx(LiveFacts, { live: answer.live, previous: previous })) : (answer.summary && _jsx(RunFacts, { summary: answer.summary, previous: previous })), _jsx(RolePicker, { roles: rolesIn(scores), role: role, onChange: setRole }), _jsx(HealthCharts, { scores: inRole, previous: previous?.scores.filter((dataset) => role === 'all' || dataset.role === role) }), _jsx(ViewPicker, { view: view, figure: figure, place: place, onView: setView, onFigure: setFigure, onClear: () => {
|
|
319
|
+
setPlace(null);
|
|
320
|
+
} }), view === 'matrix' ? (_jsx(GuardrailMatrix, { scores: inRole, figure: figure, onOpen: (boundary, detector) => {
|
|
321
|
+
setPlace({ boundary, detector });
|
|
322
|
+
setView('table');
|
|
323
|
+
} })) : (_jsx(ScoresTable, { rows: rows, facts: facts, onOpen: setOpen }))] }));
|
|
324
|
+
return (_jsx(PaneLayout, { isWide: isWide, label: "Guardrails health", detailLabel: open ? DETECTOR_META[open.detector].label : '', overview: overview, detail: open && (_jsx(ScoreDetail, { row: open, facts: facts.get(open.dataset), run: run, endpoint: endpoint, payload: payload, isClose: isWide, onBack: () => {
|
|
325
|
+
setOpen(null);
|
|
326
|
+
} }, open.id)) }));
|
|
327
|
+
}
|
|
328
|
+
function askBench(endpoint) {
|
|
329
|
+
return fetch(`${endpoint}/bench`)
|
|
330
|
+
.then((response) => (response.ok ? response.json() : null))
|
|
331
|
+
.catch(() => null);
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* The bench's last run, when the project's server has one: undefined while it is asked, null when
|
|
335
|
+
* there is none. A run still going is followed as it goes, and the answer asked again once it ends.
|
|
336
|
+
*/
|
|
337
|
+
function useBenchRun(endpoint) {
|
|
338
|
+
const [answer, setAnswer] = useState(endpoint ? undefined : null);
|
|
339
|
+
useEffect(() => {
|
|
340
|
+
if (!endpoint)
|
|
341
|
+
return;
|
|
342
|
+
let mounted = true;
|
|
343
|
+
void askBench(endpoint).then((body) => {
|
|
344
|
+
if (mounted)
|
|
345
|
+
setAnswer(body);
|
|
346
|
+
});
|
|
347
|
+
return () => {
|
|
348
|
+
mounted = false;
|
|
349
|
+
};
|
|
350
|
+
}, [endpoint]);
|
|
351
|
+
const following = answer?.live?.run;
|
|
352
|
+
useEffect(() => {
|
|
353
|
+
if (!endpoint || !following || typeof EventSource === 'undefined')
|
|
354
|
+
return;
|
|
355
|
+
const source = new EventSource(`${endpoint}/bench/live?run=${encodeURIComponent(following)}`);
|
|
356
|
+
source.onmessage = (event) => {
|
|
357
|
+
try {
|
|
358
|
+
const live = JSON.parse(event.data);
|
|
359
|
+
setAnswer((before) => (before ? { ...before, live } : before));
|
|
360
|
+
}
|
|
361
|
+
catch {
|
|
362
|
+
// why: A line cut short is followed by the next one, a second later.
|
|
363
|
+
}
|
|
364
|
+
};
|
|
365
|
+
source.addEventListener('ended', () => {
|
|
366
|
+
source.close();
|
|
367
|
+
void askBench(endpoint).then(setAnswer);
|
|
368
|
+
});
|
|
369
|
+
return () => {
|
|
370
|
+
source.close();
|
|
371
|
+
};
|
|
372
|
+
}, [endpoint, following]);
|
|
373
|
+
return answer;
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* The guardrail tester, and beside it the bench's Health view when the project's server has a
|
|
377
|
+
* bench run: a checkout of Theorem that ran `deno task bench:guardrails`. Anywhere else it is the
|
|
378
|
+
* tester alone.
|
|
379
|
+
*/
|
|
380
|
+
export function GuardrailPane({ payload, isWide }) {
|
|
381
|
+
const endpoint = useProject()?.endpoint;
|
|
382
|
+
const bench = useBenchRun(endpoint);
|
|
383
|
+
const [mode, setMode] = useState('test');
|
|
384
|
+
if (!bench || !endpoint)
|
|
385
|
+
return _jsx(GuardrailTester, { payload: payload, isWide: isWide });
|
|
386
|
+
const modes = (_jsxs(SegmentedControl, { label: "Guardrails", size: "sm", value: mode, onChange: (next) => {
|
|
387
|
+
setMode(next === 'health' ? 'health' : 'test');
|
|
388
|
+
}, children: [_jsx(SegmentedControlItem, { value: "test", label: "Test", icon: _jsx(Icon, { icon: IconShieldSearch, size: "sm" }) }), _jsx(SegmentedControlItem, { value: "health", label: "Health", icon: _jsx(Icon, { icon: IconHeartbeat, size: "sm" }) })] }));
|
|
389
|
+
if (mode === 'test')
|
|
390
|
+
return _jsx(GuardrailTester, { payload: payload, isWide: isWide, aside: modes });
|
|
391
|
+
if ((bench.live?.scores ?? bench.summary?.scores ?? []).length === 0) {
|
|
392
|
+
return (_jsxs(VStack, { gap: 4, padding: 4, children: [_jsx(HStack, { hAlign: "end", children: modes }), _jsx(PaneState, { icon: _jsx(Icon, { icon: IconHeartbeat, size: "lg", color: "secondary" }), title: bench.live ? 'The bench run is starting.' : 'The last bench run scored nothing.' })] }));
|
|
393
|
+
}
|
|
394
|
+
return (_jsx(GuardrailHealth, { answer: bench, endpoint: endpoint, payload: payload, isWide: isWide, aside: modes }));
|
|
395
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { type Detector } from '@theoremjs/agents/guardrails';
|
|
2
|
+
import type { DatasetScore } from '../guardrail-bench.ts';
|
|
3
|
+
import { type ProbeBoundary } from '../mod.ts';
|
|
4
|
+
export declare const MATRIX_FIGURES: readonly ["recall", "falseAlarms", "p95"];
|
|
5
|
+
export type MatrixFigure = (typeof MATRIX_FIGURES)[number];
|
|
6
|
+
export declare const MATRIX_FIGURE_LABELS: Record<MatrixFigure, string>;
|
|
7
|
+
/** Good is blue and bad is grey, each darker the further it is from the middle. */
|
|
8
|
+
export declare function tint(goodness: number): string;
|
|
9
|
+
/**
|
|
10
|
+
* Every boundary against every detector, shaded by how the detector does there: blue where it is
|
|
11
|
+
* good, grey where it is bad, white where no dataset scores it. A cell opens its rows in the table.
|
|
12
|
+
*/
|
|
13
|
+
export declare function GuardrailMatrix({ scores, figure, onOpen, }: {
|
|
14
|
+
scores: readonly DatasetScore[];
|
|
15
|
+
figure: MatrixFigure;
|
|
16
|
+
onOpen: (boundary: ProbeBoundary, detector: Detector) => void;
|
|
17
|
+
}): import("react").JSX.Element;
|