@ggui-ai/ui-gen 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{adapter-Cg_rkpGq.d.ts → adapter-BJwn74rx.d.ts} +49 -1
- package/dist/adapters/generation-dispatch.d.ts +8 -5
- package/dist/adapters/generation-dispatch.js +1118 -193
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +27 -6
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +10 -9
- package/dist/advanced/index.js +1126 -202
- package/dist/advanced/index.js.map +1 -1
- package/dist/blueprint-validator.js +416 -92
- package/dist/blueprint-validator.js.map +1 -1
- package/dist/boilerplate.d.ts +2 -2
- package/dist/boilerplate.js +10 -3
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +3 -2
- package/dist/check/index.js +9 -2
- package/dist/check/index.js.map +1 -1
- package/dist/{create-ui-generator-DFnnArrv.d.ts → create-ui-generator-N4gRXpcL.d.ts} +3 -4
- package/dist/{design-mode-DLufhrNp.d.ts → design-mode-DGWN_Gkm.d.ts} +1 -1
- package/dist/{dispatch-CStl9eAw.d.ts → dispatch-BUoXRj_r.d.ts} +2 -2
- package/dist/evaluation/axis-checks/index.d.ts +5 -4
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +4 -3
- package/dist/evaluation/index.d.ts +107 -539
- package/dist/evaluation/index.js +552 -55
- package/dist/evaluation/index.js.map +1 -1
- package/dist/evaluation/types.d.ts +17 -0
- package/dist/evaluation/types.js.map +1 -1
- package/dist/{generate-9IeNT4ZY.d.ts → generate-tSO_shQ0.d.ts} +1 -1
- package/dist/harness/check/runtime-render/index.d.ts +5 -5
- package/dist/harness/check/runtime-render/index.js +416 -92
- package/dist/harness/check/runtime-render/index.js.map +1 -1
- package/dist/harness/check/runtime-render/render-check-worker.js +393 -88
- package/dist/harness/check/runtime-render/render-check-worker.js.map +1 -1
- package/dist/harness/index.d.ts +4 -5
- package/dist/harness/index.js +36 -5
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/prompts.d.ts +2 -2
- package/dist/harness/prompts.js +10 -1
- package/dist/harness/prompts.js.map +1 -1
- package/dist/harness/result-types.d.ts +17 -3
- package/dist/harness/result-types.js.map +1 -1
- package/dist/harness/runtime.d.ts +14 -6
- package/dist/harness/runtime.js +11 -4
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +4 -2
- package/dist/harness/types-public.js +10 -3
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +4 -3
- package/dist/index.d.ts +46 -11
- package/dist/index.js +1128 -200
- package/dist/index.js.map +1 -1
- package/dist/patch.d.ts +3 -3
- package/dist/{prompts-YU9piOgs.d.ts → prompts-BhNUuRoZ.d.ts} +1 -1
- package/dist/tools/render-check-worker.js +8 -1
- package/dist/tools/render-check-worker.js.map +1 -1
- package/dist/{types-public-BJTLpzWn.d.ts → types-public-B7wn4ce5.d.ts} +384 -3
- package/dist/validation/index.d.ts +1 -1
- package/dist/validation/index.js +1 -1
- package/dist/validation/index.js.map +1 -1
- package/dist/visual-evaluator-C9L-o6ap.d.ts +639 -0
- package/dist/workflows.d.ts +3 -3
- package/package.json +12 -12
- package/src/adapters/generation-dispatch.ts +2 -0
- package/src/boilerplate/free-design-prompt.ts +3 -3
- package/src/boilerplate/hard-sections.ts +1 -1
- package/src/boilerplate/templates/layouts/chat-mobile.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/chat-universal.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/fullscreen-desktop.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/fullscreen-universal.tsx.tmpl +3 -1
- package/src/check/run-tier0.ts +6 -3
- package/src/contract-context.ts +1 -1
- package/src/evaluation/cost-tracker.ts +54 -6
- package/src/evaluation/criteria/edge-probe.ts +286 -0
- package/src/evaluation/criteria/edge.ts +202 -0
- package/src/evaluation/criteria/resolve.ts +55 -7
- package/src/evaluation/index.ts +5 -1
- package/src/evaluation/judge-spend.ts +48 -0
- package/src/evaluation/types-public.ts +91 -0
- package/src/evaluation/types.ts +10 -0
- package/src/evaluation/visual-evaluator.ts +236 -40
- package/src/harness/check/runtime-render/adapter.ts +18 -0
- package/src/harness/check/runtime-render/probe-pending.ts +111 -0
- package/src/harness/check/runtime-render/render-check-host.ts +10 -6
- package/src/harness/check/runtime-render/render-check-worker.ts +6 -3
- package/src/harness/check/runtime-render/render-check.ts +417 -133
- package/src/harness/check/runtime-render/walk.ts +200 -0
- package/src/harness/coding/assemble-result.ts +2 -0
- package/src/harness/coding/generate-task-runner.ts +27 -0
- package/src/harness/coding/init-session.ts +5 -3
- package/src/harness/coding/run-coding-turn.ts +5 -1
- package/src/harness/coding/run-eval-round.ts +63 -3
- package/src/harness/in-loop-evaluation.ts +39 -0
- package/src/harness/llm-router.ts +58 -1
- package/src/harness/result-types.ts +9 -0
- package/src/harness/runtime.ts +9 -0
- package/src/harness/types-public.ts +10 -1
- package/src/index.ts +5 -0
- package/src/internal/verdict-line.ts +33 -0
- package/src/taught-tokens.ts +32 -0
- package/src/tools/render-check-worker.ts +2 -1
- package/src/tools/render-check.ts +8 -7
- package/src/validation/primitives.ts +1 -1
- package/dist/llm-router-nfO7wPec.d.ts +0 -267
- package/dist/{bank-CT4eEkta.d.ts → bank-Ctdp3FUA.d.ts} +3 -3
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `space.edge` as an instrument (#1663): the alignment rule applied to measured blocks, never to a model's estimate.
|
|
3
|
+
*
|
|
4
|
+
* The rule (docs/design/direction/visual-criteria.md, "The edge rule, measurable"): in each column of a card, the things
|
|
5
|
+
* that stack start on one edge. A block may start in only two other places — inside a box, on the box's inner edge,
|
|
6
|
+
* which everything inside the box then shares; or next to a box or a band, on the box's own outer edge or on the edge
|
|
7
|
+
* of the text inside it. Same edge means within 2 px. A card fails if any captured width fails; this function judges one
|
|
8
|
+
* width.
|
|
9
|
+
*
|
|
10
|
+
* Levels that are allowed: a hanging marker (a block with two edges, the marker's and its text's); a list inside
|
|
11
|
+
* running text (list items share an edge of their own); a table (measured at its rule, its cells not re-checked);
|
|
12
|
+
* columns side by side (each with its own edge); a centred group (one centre axis). A frame that shows only loading
|
|
13
|
+
* placeholders is not evaluated.
|
|
14
|
+
*
|
|
15
|
+
* Input is the measured block list (a DOM probe at capture produces it); output is the verdict plus every offset in
|
|
16
|
+
* px, so a near miss reads apart from a split. This module measures nothing itself. *
|
|
17
|
+
* **Validated instrument (#1663, 2026-10-02):** this file as of `cec465319` (with `7bb4da13e`) passed a fresh sealed
|
|
18
|
+
* blind draw at the registered bar (27/30, κ 0.76), stricter than the design lead's reading on its disagreements. Any
|
|
19
|
+
* later change to what this file measures or decides is a new instrument until another fresh sealed draw.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/** The rule's tolerance: two edges within this many px are the same edge. */
|
|
23
|
+
export const EDGE_TOLERANCE_PX = 2;
|
|
24
|
+
|
|
25
|
+
/** One block of a captured card, at one canvas width. All x in CSS px from the card's left. */
|
|
26
|
+
export interface EdgeBlock {
|
|
27
|
+
/** A stable label for evidence (a DOM path in production, a name in tests). */
|
|
28
|
+
readonly id: string;
|
|
29
|
+
/** A text block, a control group (measured at its first control), or a surface (a box: border or its own fill). */
|
|
30
|
+
readonly kind: 'text' | 'controls' | 'surface';
|
|
31
|
+
/**
|
|
32
|
+
* The block's edge: a text's first-character origin, a bounded control's or a surface's border box, an unbounded
|
|
33
|
+
* control's text. For a hanging marker, the marker's edge.
|
|
34
|
+
*/
|
|
35
|
+
readonly edge: number;
|
|
36
|
+
/** Horizontal extent, for telling side-by-side columns apart. */
|
|
37
|
+
readonly left: number;
|
|
38
|
+
readonly right: number;
|
|
39
|
+
/** Vertical position, for finding a surface's first text. */
|
|
40
|
+
readonly top: number;
|
|
41
|
+
/** The surface this block sits inside, by id; absent at the card's own level. */
|
|
42
|
+
readonly within?: string;
|
|
43
|
+
/** A hanging marker's text edge: what stacks with the block may line up with this or with `edge`. */
|
|
44
|
+
readonly textEdge?: number;
|
|
45
|
+
/** Structure the DOM names: a list item, a table (measured at its rule), a member of a centred group. */
|
|
46
|
+
readonly role?: 'list-item' | 'table' | 'centred' | 'placeholder';
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface EdgeOffset {
|
|
50
|
+
/** The container checked: the card, or a surface's id. */
|
|
51
|
+
readonly container: string;
|
|
52
|
+
/** The block that sits off every allowed edge. */
|
|
53
|
+
readonly block: string;
|
|
54
|
+
/** Its edge, and the nearest edge the rule allowed it, in px. */
|
|
55
|
+
readonly at: number;
|
|
56
|
+
readonly nearest: number;
|
|
57
|
+
readonly offPx: number;
|
|
58
|
+
/** Why the block is off when it is not plain distance (a centred block among edge-aligned ones). */
|
|
59
|
+
readonly reason?: string;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export interface EdgeVerdict {
|
|
63
|
+
readonly verdict: 'pass' | 'fail' | 'n/a';
|
|
64
|
+
/** Every block that sits off the allowed edges, with its offset; empty on a pass. */
|
|
65
|
+
readonly offsets: readonly EdgeOffset[];
|
|
66
|
+
readonly evidence: string;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const CARD = 'card';
|
|
70
|
+
const same = (a: number, b: number): boolean => Math.abs(a - b) <= EDGE_TOLERANCE_PX;
|
|
71
|
+
const overlaps = (a: EdgeBlock, b: EdgeBlock): boolean => a.left < b.right && b.left < a.right;
|
|
72
|
+
const axis = (b: EdgeBlock): number => (b.left + b.right) / 2;
|
|
73
|
+
|
|
74
|
+
/** A surface's content edge: where its first text starts (its topmost text block's edge), or undefined if it has none. */
|
|
75
|
+
function contentEdge(surface: EdgeBlock, blocks: readonly EdgeBlock[]): number | undefined {
|
|
76
|
+
const texts = blocks.filter((b) => b.within === surface.id && b.kind !== 'surface' && b.role !== 'centred');
|
|
77
|
+
if (texts.length === 0) return undefined;
|
|
78
|
+
const first = texts.reduce((a, b) => (b.top < a.top ? b : a));
|
|
79
|
+
return first.textEdge ?? first.edge;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Columns among a container's direct blocks: non-surface blocks that overlap horizontally, transitively. */
|
|
83
|
+
function columns(direct: readonly EdgeBlock[]): EdgeBlock[][] {
|
|
84
|
+
const all = direct.filter((b) => b.kind !== 'surface');
|
|
85
|
+
// A block that spans two columns side by side (it overlaps two blocks that do not overlap each other) belongs to
|
|
86
|
+
// their parent column, never to either (decided 2026-10-02): it must not chain them into one.
|
|
87
|
+
const bridges = (b: EdgeBlock): boolean => {
|
|
88
|
+
const under = all.filter((x) => x !== b && overlaps(x, b));
|
|
89
|
+
return under.some((x, i) => under.slice(i + 1).some((y) => !overlaps(x, y)));
|
|
90
|
+
};
|
|
91
|
+
const flow = all.filter((b) => !bridges(b));
|
|
92
|
+
const parentPool: EdgeBlock[] = all.filter((b) => bridges(b));
|
|
93
|
+
const parent = flow.map((_, i) => i);
|
|
94
|
+
const find = (i: number): number => (parent[i] === i ? i : (parent[i] = find(parent[i]!)));
|
|
95
|
+
for (let i = 0; i < flow.length; i += 1) for (let j = i + 1; j < flow.length; j += 1) if (overlaps(flow[i]!, flow[j]!)) parent[find(i)] = find(j);
|
|
96
|
+
const groups = new Map<number, EdgeBlock[]>();
|
|
97
|
+
flow.forEach((b, i) => groups.set(find(i), [...(groups.get(find(i)) ?? []), b]));
|
|
98
|
+
const cols = [...groups.values()];
|
|
99
|
+
// A surface joins the one column whose text or controls it overlaps. One that overlaps two or more columns spans
|
|
100
|
+
// them and goes to their parent, like a spanning text block; one that overlaps none stands with the other
|
|
101
|
+
// unclaimed blocks. Only flow blocks decide, so a surface never chains another surface into a column.
|
|
102
|
+
for (const s of direct.filter((b) => b.kind === 'surface')) {
|
|
103
|
+
const hit = cols.filter((c) => c.some((b) => b.kind !== 'surface' && overlaps(b, s)));
|
|
104
|
+
if (hit.length === 1) hit[0]!.push(s);
|
|
105
|
+
else parentPool.push(s);
|
|
106
|
+
}
|
|
107
|
+
// The parent's blocks and the unclaimed surfaces form columns among themselves: blocks that overlap stack.
|
|
108
|
+
const lp = parentPool.map((_, i) => i);
|
|
109
|
+
const lfind = (i: number): number => (lp[i] === i ? i : (lp[i] = lfind(lp[i]!)));
|
|
110
|
+
for (let i = 0; i < parentPool.length; i += 1) for (let j = i + 1; j < parentPool.length; j += 1) if (overlaps(parentPool[i]!, parentPool[j]!)) lp[lfind(i)] = lfind(j);
|
|
111
|
+
const lgroups = new Map<number, EdgeBlock[]>();
|
|
112
|
+
parentPool.forEach((b, i) => lgroups.set(lfind(i), [...(lgroups.get(lfind(i)) ?? []), b]));
|
|
113
|
+
cols.push(...lgroups.values());
|
|
114
|
+
return cols;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** One column's check: one edge, or a stacked surface's content edge (the surface is then a ground). */
|
|
118
|
+
function checkColumn(container: string, col: readonly EdgeBlock[], blocks: readonly EdgeBlock[]): EdgeOffset[] {
|
|
119
|
+
const surfaces = col.filter((b) => b.kind === 'surface');
|
|
120
|
+
const grounds = surfaces.flatMap((s) => {
|
|
121
|
+
const c = s.role === 'table' ? undefined : contentEdge(s, blocks);
|
|
122
|
+
return c === undefined ? [] : [c];
|
|
123
|
+
});
|
|
124
|
+
const flow = col.filter((b) => b.kind !== 'surface' && b.role !== 'centred' && b.role !== 'list-item');
|
|
125
|
+
const listItems = col.filter((b) => b.role === 'list-item');
|
|
126
|
+
// Every edge a flow block or a surface offers; the common edge E must sit on one of them.
|
|
127
|
+
const candidates = [...flow.flatMap((b) => [b.edge, ...(b.textEdge !== undefined ? [b.textEdge] : [])]), ...surfaces.flatMap((s) => [s.edge]), ...grounds];
|
|
128
|
+
const blockEdges = (b: EdgeBlock): number[] => [b.edge, ...(b.textEdge !== undefined ? [b.textEdge] : [])];
|
|
129
|
+
// The end edges another block of the column offers: its boxes' and its other control sets'.
|
|
130
|
+
const endEdgesBesides = (b: EdgeBlock): number[] => [...surfaces.map((s) => s.right), ...flow.filter((x) => x !== b && x.kind === 'controls').map((x) => x.right)];
|
|
131
|
+
// A column of only list items or only a centred group has no common edge to find; their own checks below still run.
|
|
132
|
+
let best: EdgeOffset[] | null = candidates.length === 0 ? [] : null;
|
|
133
|
+
for (const E of candidates) {
|
|
134
|
+
const allowed = [E, ...grounds];
|
|
135
|
+
const off: EdgeOffset[] = [];
|
|
136
|
+
for (const b of flow) {
|
|
137
|
+
if (blockEdges(b).some((x) => allowed.some((a) => same(x, a)))) continue;
|
|
138
|
+
// A control set alone on its line may END on the end edge its column's boxes or other controls share.
|
|
139
|
+
if (b.kind === 'controls' && endEdgesBesides(b).some((r) => same(b.right, r))) continue;
|
|
140
|
+
const nearest = allowed.reduce((n, a) => (Math.abs(a - b.edge) < Math.abs(n - b.edge) ? a : n));
|
|
141
|
+
off.push({ container, block: b.id, at: b.edge, nearest, offPx: Math.abs(b.edge - nearest) });
|
|
142
|
+
}
|
|
143
|
+
for (const s of surfaces) {
|
|
144
|
+
const c = s.role === 'table' ? undefined : contentEdge(s, blocks);
|
|
145
|
+
if (same(s.edge, E) || (c !== undefined && same(c, E))) continue;
|
|
146
|
+
const nearest = c !== undefined && Math.abs(c - E) < Math.abs(s.edge - E) ? c : s.edge;
|
|
147
|
+
off.push({ container, block: s.id, at: nearest, nearest: E, offPx: Math.abs(nearest - E) });
|
|
148
|
+
}
|
|
149
|
+
if (best === null || off.length < best.length || (off.length === best.length && sum(off) < sum(best))) best = off;
|
|
150
|
+
if (best.length === 0) break;
|
|
151
|
+
}
|
|
152
|
+
const offsets = best ?? [];
|
|
153
|
+
// A list inside running text: its items share an edge of their own.
|
|
154
|
+
if (listItems.length > 1) {
|
|
155
|
+
const e0 = listItems[0]!.edge;
|
|
156
|
+
for (const li of listItems.slice(1)) if (!same(li.edge, e0)) offsets.push({ container, block: li.id, at: li.edge, nearest: e0, offPx: Math.abs(li.edge - e0) });
|
|
157
|
+
}
|
|
158
|
+
// A centred group: its members share one centre axis, and it must be the whole content of its column. A column is
|
|
159
|
+
// edge-aligned or centred, never both (decided 2026-10-02).
|
|
160
|
+
const centred = col.filter((b) => b.role === 'centred');
|
|
161
|
+
// A centred group of two or more may follow a label or heading when nothing edge-aligned follows it (an empty state
|
|
162
|
+
// under its section label). One centred block among edge-aligned ones, or a column that returns to an edge after
|
|
163
|
+
// the group, fails (decided 2026-10-02).
|
|
164
|
+
const edgeAligned = col.filter((b) => b.role !== 'centred');
|
|
165
|
+
const groupTop = centred.length > 0 ? Math.min(...centred.map((c) => c.top)) : 0;
|
|
166
|
+
const groupEndsColumn = centred.length > 1 && edgeAligned.every((b) => b.top < groupTop);
|
|
167
|
+
if (centred.length > 0 && edgeAligned.some((b) => b.kind !== 'surface') && !groupEndsColumn) {
|
|
168
|
+
const E = flow[0]?.edge ?? edgeAligned[0]!.edge;
|
|
169
|
+
for (const c of centred) offsets.push({ container, block: c.id, at: c.edge, nearest: E, offPx: Math.abs(c.edge - E), reason: 'a centred block in an edge-aligned column' });
|
|
170
|
+
} else if (centred.length > 1) {
|
|
171
|
+
const a0 = axis(centred[0]!);
|
|
172
|
+
for (const c of centred.slice(1)) if (!same(axis(c), a0)) offsets.push({ container, block: c.id, at: axis(c), nearest: a0, offPx: Math.abs(axis(c) - a0) });
|
|
173
|
+
}
|
|
174
|
+
return offsets;
|
|
175
|
+
}
|
|
176
|
+
const sum = (o: readonly EdgeOffset[]): number => o.reduce((s, x) => s + x.offPx, 0);
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Judge one captured width. `placeholdersOnly`: the frame shows only loading placeholders, so the row is not evaluated.
|
|
180
|
+
*/
|
|
181
|
+
export function judgeEdges(measured: readonly EdgeBlock[], opts: { readonly placeholdersOnly?: boolean } = {}): EdgeVerdict {
|
|
182
|
+
if (opts.placeholdersOnly === true) return { verdict: 'n/a', offsets: [], evidence: 'only loading placeholders are shown; not evaluated' };
|
|
183
|
+
let blocks = measured;
|
|
184
|
+
// A placeholder (a box with nothing in it, shown while content loads) is not a block, and the rule needs two blocks to
|
|
185
|
+
// compare: with fewer than two once placeholders are set aside, the frame is not evaluated (decided 2026-10-02).
|
|
186
|
+
// The two blocks are content, text or controls: a box holds them and is not one of them for this count.
|
|
187
|
+
const kept = blocks.filter((b) => b.role !== 'placeholder');
|
|
188
|
+
const content = kept.filter((b) => b.kind !== 'surface').length;
|
|
189
|
+
if (content < 2) return { verdict: 'n/a', offsets: [], evidence: kept.length === 0 ? 'no blocks were measured' : 'fewer than two blocks once placeholders are set aside; not evaluated' };
|
|
190
|
+
blocks = kept;
|
|
191
|
+
const offsets: EdgeOffset[] = [];
|
|
192
|
+
const containers = [CARD, ...blocks.filter((b) => b.kind === 'surface' && b.role !== 'table').map((b) => b.id)];
|
|
193
|
+
for (const container of containers) {
|
|
194
|
+
const direct = blocks.filter((b) => (b.within ?? CARD) === container);
|
|
195
|
+
for (const col of columns(direct)) offsets.push(...checkColumn(container, col, blocks));
|
|
196
|
+
}
|
|
197
|
+
if (offsets.length === 0) return { verdict: 'pass', offsets, evidence: 'every column shares its edge, or sits on a stacked surface’s content edge, within 2 px' };
|
|
198
|
+
const worst = offsets.reduce((a, b) => (b.offPx > a.offPx ? b : a));
|
|
199
|
+
const lines = offsets.map((o) => `${o.block} at x${round(o.at)} is ${round(o.offPx)} px from x${round(o.nearest)} (in ${o.container}${o.reason !== undefined ? `; ${o.reason}` : ''})`);
|
|
200
|
+
return { verdict: 'fail', offsets, evidence: `${offsets.length} block(s) off the allowed edges, largest ${round(worst.offPx)} px: ${lines.join('; ')}` };
|
|
201
|
+
}
|
|
202
|
+
const round = (n: number): string => (Math.round(n * 10) / 10).toString();
|
|
@@ -9,9 +9,11 @@
|
|
|
9
9
|
* a gate later reads the set.
|
|
10
10
|
*/
|
|
11
11
|
import type { CriteriaAnswer } from '../types.js';
|
|
12
|
-
import type { CriteriaBlock, CriteriaContext, CriteriaVerdict, CriterionVerdict } from '../types-public.js';
|
|
12
|
+
import type { CriteriaAnswerCounts, CriteriaBlock, CriteriaContext, CriteriaVerdict, CriterionVerdict } from '../types-public.js';
|
|
13
|
+
import type { VisionFinishReason } from '../../harness/llm-router.js';
|
|
13
14
|
import { bankRows, type BankRow, type CriteriaBank } from './bank.js';
|
|
14
15
|
import { CRITERIA_SELECTOR_VERSION, type CriteriaSelectionResult } from './select.js';
|
|
16
|
+
import type { EdgeVerdict } from './edge.js';
|
|
15
17
|
|
|
16
18
|
/** The instruments the capture runs today, by the bank ids they answer (v1 binding; a bank field may replace it). */
|
|
17
19
|
export const INSTRUMENT_BY_ID: Readonly<Record<string, 'fit' | 'fill'>> = {
|
|
@@ -20,18 +22,37 @@ export const INSTRUMENT_BY_ID: Readonly<Record<string, 'fit' | 'fill'>> = {
|
|
|
20
22
|
'finish.canvas.inhabited': 'fill',
|
|
21
23
|
};
|
|
22
24
|
|
|
25
|
+
/**
|
|
26
|
+
* ggui#1663 — instruments that read BESIDE a judge row, report-only: the row keeps the judge's verdict and gains the
|
|
27
|
+
* measurement as `instrument`, so the two can be compared before any verdict moves.
|
|
28
|
+
*/
|
|
29
|
+
export const INSTRUMENT_BESIDE_JUDGE: Readonly<Record<string, 'edge'>> = {
|
|
30
|
+
'space.edge': 'edge',
|
|
31
|
+
};
|
|
32
|
+
|
|
23
33
|
export interface CriteriaMeasurements {
|
|
24
34
|
readonly overflow: boolean;
|
|
25
35
|
readonly contentHeight: number | null;
|
|
26
36
|
readonly viewportHeight: number;
|
|
27
37
|
readonly inkRatio: number | null;
|
|
38
|
+
/**
|
|
39
|
+
* ggui#1663 — the edge rule on the capture's DOM: absent when this capture had no page to read (a stored frame),
|
|
40
|
+
* `null` when the probe ran and failed.
|
|
41
|
+
*/
|
|
42
|
+
readonly edge?: EdgeVerdict | null;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function besideRead(kind: 'edge', m: CriteriaMeasurements): Read {
|
|
46
|
+
if (m.edge === undefined) return { verdict: 'n/a', evidence: 'not measured: this capture had no page to read' };
|
|
47
|
+
if (m.edge === null) return { verdict: 'n/a', evidence: 'edge probe failed on this capture' };
|
|
48
|
+
return { verdict: m.edge.verdict, evidence: m.edge.evidence };
|
|
28
49
|
}
|
|
29
50
|
|
|
30
51
|
type Read = { verdict: CriteriaVerdict; evidence: string };
|
|
31
52
|
|
|
32
|
-
function majority(id: string, answers: readonly CriteriaAnswer[][]): Read {
|
|
53
|
+
function majority(id: string, answers: readonly CriteriaAnswer[][], unansweredReason: string | undefined): Read {
|
|
33
54
|
const votes = answers.flatMap((a) => a.filter((x) => x.id === id));
|
|
34
|
-
if (votes.length === 0) return { verdict: 'n/a', evidence: 'not answered' };
|
|
55
|
+
if (votes.length === 0) return { verdict: 'n/a', evidence: unansweredReason ?? 'not answered' };
|
|
35
56
|
const counts = new Map<CriteriaVerdict, number>();
|
|
36
57
|
for (const v of votes) counts.set(v.verdict, (counts.get(v.verdict) ?? 0) + 1);
|
|
37
58
|
const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]);
|
|
@@ -51,8 +72,8 @@ function instrumentRead(kind: 'fit' | 'fill', m: CriteriaMeasurements): Read {
|
|
|
51
72
|
return { verdict: 'n/a', evidence: `fill ${m.inkRatio.toFixed(3)} reported; no floor ruled` };
|
|
52
73
|
}
|
|
53
74
|
|
|
54
|
-
function readRow(row: BankRow, answers: readonly CriteriaAnswer[][], m: CriteriaMeasurements): Read {
|
|
55
|
-
if (row.evaluation === 'judge') return majority(row.id, answers);
|
|
75
|
+
function readRow(row: BankRow, answers: readonly CriteriaAnswer[][], m: CriteriaMeasurements, unansweredReason: string | undefined): Read {
|
|
76
|
+
if (row.evaluation === 'judge') return majority(row.id, answers, unansweredReason);
|
|
56
77
|
if (row.evaluation === 'human') return { verdict: 'n/a', evidence: `reader's column (status ${row.status})` };
|
|
57
78
|
if (row.status !== 'live') return { verdict: 'n/a', evidence: `not implemented (status ${row.status})` };
|
|
58
79
|
const kind = INSTRUMENT_BY_ID[row.id];
|
|
@@ -65,14 +86,31 @@ export function resolveCriteriaBlock(args: {
|
|
|
65
86
|
readonly selected: CriteriaSelectionResult;
|
|
66
87
|
readonly answers: readonly CriteriaAnswer[][];
|
|
67
88
|
readonly measurements: CriteriaMeasurements;
|
|
89
|
+
/**
|
|
90
|
+
* ggui#1127 — why a judge row has no answer, when the caller knows: a salvaged criteria answer keeps no criteria,
|
|
91
|
+
* so its rows say so instead of reading as a clean `n/a`. Absent ⇒ "not answered".
|
|
92
|
+
*/
|
|
93
|
+
readonly unansweredReason?: string;
|
|
94
|
+
/** ggui#1687 — the call that answered, as it came back: its answer's counts and its stop reason. */
|
|
95
|
+
readonly call?: { readonly answer: CriteriaAnswerCounts; readonly finishReason?: VisionFinishReason };
|
|
68
96
|
}): CriteriaBlock {
|
|
69
97
|
const byId = new Map<string, BankRow>(bankRows(args.bank).map((r) => [r.id, r]));
|
|
70
98
|
const verdicts: CriterionVerdict[] = [];
|
|
71
99
|
for (const s of args.selected.selection) {
|
|
72
100
|
const row = byId.get(s.id);
|
|
73
101
|
if (row === undefined) continue;
|
|
74
|
-
const read = readRow(row, args.answers, args.measurements);
|
|
75
|
-
|
|
102
|
+
const read = readRow(row, args.answers, args.measurements, args.unansweredReason);
|
|
103
|
+
const beside = INSTRUMENT_BESIDE_JUDGE[row.id];
|
|
104
|
+
verdicts.push({
|
|
105
|
+
id: row.id,
|
|
106
|
+
severity: row.severity,
|
|
107
|
+
method: row.evaluation,
|
|
108
|
+
status: row.status,
|
|
109
|
+
source: s.source,
|
|
110
|
+
verdict: read.verdict,
|
|
111
|
+
evidence: read.evidence,
|
|
112
|
+
...(beside !== undefined && row.evaluation === 'judge' ? { instrument: besideRead(beside, args.measurements) } : {}),
|
|
113
|
+
});
|
|
76
114
|
}
|
|
77
115
|
return {
|
|
78
116
|
criteriaSetId: args.selected.criteriaSetId,
|
|
@@ -81,6 +119,16 @@ export function resolveCriteriaBlock(args: {
|
|
|
81
119
|
context: args.context,
|
|
82
120
|
selection: args.selected.selection,
|
|
83
121
|
verdicts,
|
|
122
|
+
...(args.call !== undefined
|
|
123
|
+
? {
|
|
124
|
+
call: {
|
|
125
|
+
answer: args.call.answer,
|
|
126
|
+
// Kept answers whose id names no row of the bank: the judge answered something nobody asked.
|
|
127
|
+
unknownIds: args.answers.flat().filter((a) => !byId.has(a.id)).length,
|
|
128
|
+
...(args.call.finishReason !== undefined ? { finishReason: args.call.finishReason } : {}),
|
|
129
|
+
},
|
|
130
|
+
}
|
|
131
|
+
: {})
|
|
84
132
|
};
|
|
85
133
|
}
|
|
86
134
|
|
package/src/evaluation/index.ts
CHANGED
|
@@ -129,6 +129,7 @@ export type {
|
|
|
129
129
|
ScreenshotAttempt,
|
|
130
130
|
CanvasVisualResult,
|
|
131
131
|
CriteriaCallTokens,
|
|
132
|
+
VisualLegTokens,
|
|
132
133
|
CanvasFitPolicy,
|
|
133
134
|
CaptureMode,
|
|
134
135
|
ScreenshotDeps,
|
|
@@ -168,6 +169,9 @@ export { designTreeSha256, judgeDesignIdentity, resetJudgeDesignIdentityCache, t
|
|
|
168
169
|
export { bankRows, criteriaBankSchema, parseCriteriaBank, type BankRow, type CriteriaBank, type CriteriaScope } from './criteria/bank.js';
|
|
169
170
|
export { CRITERIA_SELECTOR_VERSION, canonicalCriteriaContext, criteriaSetIdFor, selectCriteria, type CriteriaSelectionResult } from './criteria/select.js';
|
|
170
171
|
export { chromaOfCssTokens, criteriaContextFor, type CriteriaContextInput, type CriteriaContextSources } from './criteria/context.js';
|
|
171
|
-
export { INSTRUMENT_BY_ID, buildCriteriaJudgeBlock, resolveCriteriaBlock, type CriteriaMeasurements } from './criteria/resolve.js';
|
|
172
|
+
export { INSTRUMENT_BESIDE_JUDGE, INSTRUMENT_BY_ID, buildCriteriaJudgeBlock, resolveCriteriaBlock, type CriteriaMeasurements } from './criteria/resolve.js';
|
|
173
|
+
export { EDGE_TOLERANCE_PX, judgeEdges, type EdgeBlock, type EdgeOffset, type EdgeVerdict } from './criteria/edge.js';
|
|
174
|
+
export { EDGE_BLOCKS_EXPRESSION, parseEdgeProbe, type EdgeProbe } from './criteria/edge-probe.js';
|
|
172
175
|
export { judgeStoredCapture, type StoredCaptureInput, type StoredCaptureOutcome, type StoredCaptureVerdict } from './visual-evaluator.js';
|
|
176
|
+
export { judgedFrameTokens, type JudgeSpend, type JudgedTokens } from './judge-spend.js';
|
|
173
177
|
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ggui#1640 — one judged frame's (or one visual evaluation's) spend, summed over EVERY leg the judge reports.
|
|
3
|
+
*
|
|
4
|
+
* The judge reports its legs apart: the scoring calls' `inputTokens` / `outputTokens`, and, since ggui#1436, the
|
|
5
|
+
* report-only criteria call's `criteriaTokens`. A caller that sums only the first leg under-counts, and a dollar
|
|
6
|
+
* ceiling built on that sum stops late. {@link judgedFrameTokens} is the one sum. The two `satisfies` tables below list
|
|
7
|
+
* every `*Tokens` member of the results it reads, so a leg added to either result type fails the build here until the
|
|
8
|
+
* sum counts it.
|
|
9
|
+
*/
|
|
10
|
+
import type { CriteriaCallTokens, StoredCaptureVerdict, VisualEvaluationResult } from './visual-evaluator.js';
|
|
11
|
+
|
|
12
|
+
/** A spend in tokens: input and output, over every leg. */
|
|
13
|
+
export interface JudgeSpend {
|
|
14
|
+
readonly inputTokens: number;
|
|
15
|
+
readonly outputTokens: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** The members of `T` whose names end in `Tokens`. */
|
|
19
|
+
type TokenMember<T> = { [K in keyof T]-?: K extends `${string}Tokens` ? K : never }[keyof T];
|
|
20
|
+
|
|
21
|
+
/** Which leg each `*Tokens` member belongs to. A member missing here, or listed and gone, is a compile error. */
|
|
22
|
+
const STORED_VERDICT_LEGS = {
|
|
23
|
+
inputTokens: 'scoring',
|
|
24
|
+
outputTokens: 'scoring',
|
|
25
|
+
criteriaTokens: 'criteria',
|
|
26
|
+
} as const satisfies Record<TokenMember<StoredCaptureVerdict>, 'scoring' | 'criteria'>;
|
|
27
|
+
const VISUAL_RESULT_LEGS = {
|
|
28
|
+
inputTokens: 'scoring',
|
|
29
|
+
outputTokens: 'scoring',
|
|
30
|
+
criteriaTokens: 'criteria',
|
|
31
|
+
} as const satisfies Record<TokenMember<VisualEvaluationResult>, 'scoring' | 'criteria'>;
|
|
32
|
+
void STORED_VERDICT_LEGS;
|
|
33
|
+
void VISUAL_RESULT_LEGS;
|
|
34
|
+
|
|
35
|
+
/** The legs {@link judgedFrameTokens} reads, as both result types carry them. */
|
|
36
|
+
export interface JudgedTokens {
|
|
37
|
+
readonly inputTokens?: number;
|
|
38
|
+
readonly outputTokens?: number;
|
|
39
|
+
readonly criteriaTokens?: CriteriaCallTokens;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** A judged frame's spend over every leg: the scoring calls plus the report-only criteria call, when one ran. */
|
|
43
|
+
export function judgedFrameTokens(result: JudgedTokens): JudgeSpend {
|
|
44
|
+
return {
|
|
45
|
+
inputTokens: (result.inputTokens ?? 0) + (result.criteriaTokens?.inputTokens ?? 0),
|
|
46
|
+
outputTokens: (result.outputTokens ?? 0) + (result.criteriaTokens?.outputTokens ?? 0),
|
|
47
|
+
};
|
|
48
|
+
}
|
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
|
|
21
21
|
export type Priority = "P0" | "P1" | "P2";
|
|
22
22
|
|
|
23
|
+
import type { VisionFinishReason } from '../harness/llm-router.js';
|
|
23
24
|
import type { GenerationRuntimeProbeOutcome, GenerationRuntimeProbeStatus } from "@ggui-ai/mcp-server-core";
|
|
24
25
|
import type { CanvasClass, DesignMode } from "../design-mode.js";
|
|
25
26
|
|
|
@@ -61,6 +62,11 @@ export interface EvalIssue {
|
|
|
61
62
|
description: string;
|
|
62
63
|
fix: string;
|
|
63
64
|
line?: number;
|
|
65
|
+
/**
|
|
66
|
+
* The visual leg's provenance (ggui#1545): `judge` (the model's finding) or `instrument` (a deterministic
|
|
67
|
+
* measurement). Absent on every other leg's issues. `visualEvaluation.actOn` acts on `judge` only.
|
|
68
|
+
*/
|
|
69
|
+
origin?: "judge" | "instrument";
|
|
64
70
|
}
|
|
65
71
|
|
|
66
72
|
/**
|
|
@@ -100,6 +106,23 @@ export interface ProbeHostLoad {
|
|
|
100
106
|
readonly cores: number;
|
|
101
107
|
}
|
|
102
108
|
|
|
109
|
+
/**
|
|
110
|
+
* ggui#1398 — whether the controls the probe clicked showed they were
|
|
111
|
+
* working once their action was pending. Report-only: it is read by the
|
|
112
|
+
* benchmark and never counted into the probe's warnings or fed back to
|
|
113
|
+
* the coding loop.
|
|
114
|
+
*/
|
|
115
|
+
export interface ProbePendingAffordance {
|
|
116
|
+
/** Controls the pending phase marked pending: those whose click dispatched their action (click or submit wiring) and that were still in the page. `visible + missing.length === dispatched`. */
|
|
117
|
+
readonly dispatched: number;
|
|
118
|
+
/** Of those, how many visibly changed while pending (disabled, busy, text, or replaced). */
|
|
119
|
+
readonly visible: number;
|
|
120
|
+
/** The actions whose control did not visibly change, including one whose card threw while pending. */
|
|
121
|
+
readonly missing: readonly string[];
|
|
122
|
+
/** Actions whose control was no longer in the page when its turn came (the card replaced it, or blanked after a throw): never marked, not counted as dispatched. */
|
|
123
|
+
readonly gone: readonly string[];
|
|
124
|
+
}
|
|
125
|
+
|
|
103
126
|
/**
|
|
104
127
|
* The engine's own fields on a probe's meta, beside the outcome a
|
|
105
128
|
* generation's metadata reports: why a probe did not finish, its render
|
|
@@ -116,6 +139,11 @@ export interface RuntimeProbeMetaDetail {
|
|
|
116
139
|
readonly hostLoad?: ProbeHostLoad;
|
|
117
140
|
/** Time the check waited for a probe slot before it started, ms; present only when > 0 (ggui#1380 C2b). */
|
|
118
141
|
readonly queuedMs?: number;
|
|
142
|
+
/**
|
|
143
|
+
* `ran` only: the pending-affordance walk (ggui#1398). Absent when no
|
|
144
|
+
* action walk ran, so "no data" and "0 of 0" read differently.
|
|
145
|
+
*/
|
|
146
|
+
readonly pendingAffordance?: ProbePendingAffordance;
|
|
119
147
|
}
|
|
120
148
|
|
|
121
149
|
/**
|
|
@@ -334,6 +362,54 @@ export interface CanvasJudgeRecord {
|
|
|
334
362
|
readonly sigma: number;
|
|
335
363
|
/** The judge's per-canvas critique per sample — the words beside a swing. */
|
|
336
364
|
readonly notes: string[];
|
|
365
|
+
/**
|
|
366
|
+
* ggui#1127 — the cause of each scoring sample whose answer failed to parse and was SALVAGED: its four dimensions
|
|
367
|
+
* were recovered from the closed prefix, its issues kept only as far as they closed, its critique lost. One entry
|
|
368
|
+
* per salvaged sample; absent when every sample parsed. The salvaged sample's score still counts.
|
|
369
|
+
*/
|
|
370
|
+
readonly salvaged?: readonly JudgeSalvageCause[];
|
|
371
|
+
/**
|
|
372
|
+
* ggui#1127 — the criteria call's answer was salvaged. A salvage keeps no criteria, so that frame's judge rows read
|
|
373
|
+
* "not answered" with this cause rather than a clean `n/a`. Absent when the call parsed or was not asked.
|
|
374
|
+
*/
|
|
375
|
+
readonly criteriaSalvaged?: JudgeSalvageCause;
|
|
376
|
+
/**
|
|
377
|
+
* ggui#1687 — the model asked for, and every model the provider said answered this canvas's calls (scoring and
|
|
378
|
+
* criteria), deduplicated. `served` is empty when no provider named one: unknown, never "the requested model".
|
|
379
|
+
*/
|
|
380
|
+
readonly models?: { readonly requested: string; readonly served: readonly string[] };
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
/**
|
|
384
|
+
* ggui#1687 — what the criteria call's answer held, before resolution. A row the judge left unanswered reads `n/a`
|
|
385
|
+
* either way; this says whether the call came back empty, cut, or with entries the parser had to drop.
|
|
386
|
+
*/
|
|
387
|
+
export interface CriteriaAnswerCounts {
|
|
388
|
+
/** The answer carried a `criteria` array. `false`: absent or not an array, so no row was read from it. */
|
|
389
|
+
readonly array: boolean;
|
|
390
|
+
/** Entries in the array. */
|
|
391
|
+
readonly received: number;
|
|
392
|
+
/** Entries kept: an object with a string id and a verdict of pass, fail or n/a. */
|
|
393
|
+
readonly kept: number;
|
|
394
|
+
/** Entries dropped, by why. */
|
|
395
|
+
readonly dropped: { readonly notObject: number; readonly badId: number; readonly badVerdict: number };
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* ggui#1127 — why a judge answer that failed to parse had to be salvaged. `output-cap`: the provider reported it
|
|
400
|
+
* stopped at the output cap. `malformed`: the provider reported a normal stop, so the answer ended but did not
|
|
401
|
+
* parse. `stopped`: the provider stopped it for another reason it named (a safety stop, or one it reports as
|
|
402
|
+
* other), so it is cut, not malformed. `unknown`: the provider or route reported no stop reason. Never inferred
|
|
403
|
+
* from the answer's length.
|
|
404
|
+
*/
|
|
405
|
+
export type JudgeSalvageCause = "output-cap" | "malformed" | "stopped" | "unknown";
|
|
406
|
+
|
|
407
|
+
/** ggui#1127 — salvaged judge answers across a canvas run, per leg, as N of M. */
|
|
408
|
+
export interface JudgeSalvageCounts {
|
|
409
|
+
/** Scoring answers: `answers` = samples that parsed or were salvaged, `salvaged` = those salvaged. */
|
|
410
|
+
readonly scoring: { readonly salvaged: number; readonly answers: number };
|
|
411
|
+
/** Criteria calls: `answered` = calls that parsed or were salvaged, `salvaged` = those salvaged (criteria lost). */
|
|
412
|
+
readonly criteria: { readonly salvaged: number; readonly answered: number };
|
|
337
413
|
}
|
|
338
414
|
|
|
339
415
|
// ─── Visual criteria (ggui#1436) ──────────────────────────────────────────
|
|
@@ -393,6 +469,19 @@ export interface CriterionVerdict {
|
|
|
393
469
|
source: CriteriaSource;
|
|
394
470
|
verdict: CriteriaVerdict;
|
|
395
471
|
evidence: string;
|
|
472
|
+
/**
|
|
473
|
+
* ggui#1663 — a measured reading beside a judge row, REPORT-ONLY: the row's `verdict` stays the judge's, so this
|
|
474
|
+
* changes no verdict, score or bar. Present only on rows an instrument answers beside the judge (`space.edge`).
|
|
475
|
+
*/
|
|
476
|
+
instrument?: { verdict: CriteriaVerdict; evidence: string };
|
|
477
|
+
}
|
|
478
|
+
/** ggui#1687 — the criteria call as it came back: its answer's counts, ids it answered outside the bank, its stop. */
|
|
479
|
+
export interface CriteriaCallRecord {
|
|
480
|
+
readonly answer: CriteriaAnswerCounts;
|
|
481
|
+
/** Kept entries whose id names no row of the bank. */
|
|
482
|
+
readonly unknownIds: number;
|
|
483
|
+
/** The provider's normalized stop reason (`length` = the output cap); absent when it reported none. */
|
|
484
|
+
readonly finishReason?: VisionFinishReason;
|
|
396
485
|
}
|
|
397
486
|
export interface CriteriaBlock {
|
|
398
487
|
/** sha256(bank version | selector version | canonical context), 16 hex — the block's own digest. */
|
|
@@ -402,6 +491,8 @@ export interface CriteriaBlock {
|
|
|
402
491
|
context: CriteriaContext;
|
|
403
492
|
selection: CriteriaSelection[];
|
|
404
493
|
verdicts: CriterionVerdict[];
|
|
494
|
+
/** ggui#1687 — the call that answered this block, when one did. Absent for a block resolved with no call. */
|
|
495
|
+
call?: CriteriaCallRecord;
|
|
405
496
|
}
|
|
406
497
|
/** The roll-up across canvases; a `must` at `n/a` is listed, never counted as a pass. */
|
|
407
498
|
export interface CriteriaRollup {
|
package/src/evaluation/types.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { CriteriaAnswerCounts } from './types-public.js';
|
|
1
2
|
// packages/ui-gen/src/evaluation/types.ts
|
|
2
3
|
|
|
3
4
|
/**
|
|
@@ -28,6 +29,13 @@ export interface EvaluationIssue {
|
|
|
28
29
|
severity: 'critical' | 'major' | 'minor';
|
|
29
30
|
/** Suggested fix for the agent to apply */
|
|
30
31
|
fix: string;
|
|
32
|
+
/**
|
|
33
|
+
* Who produced it, on the visual leg (ggui#1545): `judge` for the model's own findings, `instrument` for a
|
|
34
|
+
* deterministic measurement (fit, overflow, blank). Absent on issues from other evaluators. The loop's
|
|
35
|
+
* `visualEvaluation.actOn` acts on `judge` findings only, so a new instrument check never joins the treatment
|
|
36
|
+
* by accident: it has to say it is a judge finding, and absence reads as "not acted on".
|
|
37
|
+
*/
|
|
38
|
+
origin?: 'judge' | 'instrument';
|
|
31
39
|
}
|
|
32
40
|
|
|
33
41
|
/**
|
|
@@ -57,6 +65,8 @@ export interface EvaluationResult {
|
|
|
57
65
|
outputTokens?: number;
|
|
58
66
|
/** ggui#1436 — the judge's criteria answers, when the prompt asked for them; never part of the score. */
|
|
59
67
|
criteriaAnswers?: CriteriaAnswer[];
|
|
68
|
+
/** ggui#1687 — what the answer's criteria array held and what the parser dropped; set whenever the answer parsed. */
|
|
69
|
+
criteriaCounts?: CriteriaAnswerCounts;
|
|
60
70
|
}
|
|
61
71
|
|
|
62
72
|
/**
|