@ggui-ai/ui-gen 0.25.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/{adapter-Cg_rkpGq.d.ts → adapter-h4_V69LI.d.ts} +49 -1
- package/dist/adapters/base.d.ts +6 -1
- package/dist/adapters/base.js.map +1 -1
- package/dist/adapters/generation-dispatch.d.ts +8 -5
- package/dist/adapters/generation-dispatch.js +1174 -198
- package/dist/adapters/generation-dispatch.js.map +1 -1
- package/dist/adapters/index.js +59 -16
- package/dist/adapters/index.js.map +1 -1
- package/dist/advanced/index.d.ts +10 -9
- package/dist/advanced/index.js +1215 -216
- package/dist/advanced/index.js.map +1 -1
- package/dist/blueprint-validator.js +416 -92
- package/dist/blueprint-validator.js.map +1 -1
- package/dist/boilerplate.d.ts +2 -2
- package/dist/boilerplate.js +10 -5
- package/dist/boilerplate.js.map +1 -1
- package/dist/check/index.d.ts +3 -8
- package/dist/check/index.js +55 -2
- package/dist/check/index.js.map +1 -1
- package/dist/{create-ui-generator-DFnnArrv.d.ts → create-ui-generator-BrZHF0cf.d.ts} +3 -4
- package/dist/{design-mode-DLufhrNp.d.ts → design-mode-DGWN_Gkm.d.ts} +1 -1
- package/dist/{dispatch-CStl9eAw.d.ts → dispatch-B07TI30d.d.ts} +2 -2
- package/dist/evaluation/axis-checks/index.d.ts +5 -4
- package/dist/evaluation/axis-checks/index.js.map +1 -1
- package/dist/evaluation/axis-checks/registry.d.ts +4 -3
- package/dist/evaluation/index.d.ts +124 -546
- package/dist/evaluation/index.js +588 -65
- package/dist/evaluation/index.js.map +1 -1
- package/dist/evaluation/types.d.ts +17 -0
- package/dist/evaluation/types.js.map +1 -1
- package/dist/{generate-9IeNT4ZY.d.ts → generate-tSO_shQ0.d.ts} +1 -1
- package/dist/harness/check/runtime-render/index.d.ts +5 -5
- package/dist/harness/check/runtime-render/index.js +416 -92
- package/dist/harness/check/runtime-render/index.js.map +1 -1
- package/dist/harness/check/runtime-render/render-check-worker.js +393 -88
- package/dist/harness/check/runtime-render/render-check-worker.js.map +1 -1
- package/dist/harness/index.d.ts +4 -5
- package/dist/harness/index.js +36 -7
- package/dist/harness/index.js.map +1 -1
- package/dist/harness/prompts.d.ts +2 -2
- package/dist/harness/prompts.js +7 -1
- package/dist/harness/prompts.js.map +1 -1
- package/dist/harness/result-types.d.ts +17 -3
- package/dist/harness/result-types.js.map +1 -1
- package/dist/harness/runtime.d.ts +14 -6
- package/dist/harness/runtime.js +13 -7
- package/dist/harness/runtime.js.map +1 -1
- package/dist/harness/types-public.d.ts +4 -2
- package/dist/harness/types-public.js +10 -5
- package/dist/harness/types-public.js.map +1 -1
- package/dist/harness/validator-trace-sink.d.ts +4 -3
- package/dist/index.d.ts +46 -11
- package/dist/index.js +1217 -214
- package/dist/index.js.map +1 -1
- package/dist/patch.d.ts +3 -3
- package/dist/{prompts-YU9piOgs.d.ts → prompts-BhNUuRoZ.d.ts} +1 -1
- package/dist/tools/render-check-worker.js +8 -1
- package/dist/tools/render-check-worker.js.map +1 -1
- package/dist/{types-public-BJTLpzWn.d.ts → types-public-h8IMnYuh.d.ts} +388 -3
- package/dist/validation/index.d.ts +1 -1
- package/dist/validation/index.js +1 -1
- package/dist/validation/index.js.map +1 -1
- package/dist/visual-evaluator-C2strxqD.d.ts +639 -0
- package/dist/workflows.d.ts +3 -3
- package/package.json +12 -12
- package/src/adapters/base.ts +6 -1
- package/src/adapters/claude/message-parser.ts +5 -6
- package/src/adapters/claude/sdk-tool-pins.ts +44 -0
- package/src/adapters/claude/sdk.ts +3 -3
- package/src/adapters/claude/source-capture.ts +20 -0
- package/src/adapters/generation-dispatch.ts +2 -0
- package/src/boilerplate/free-design-prompt.ts +3 -3
- package/src/boilerplate/hard-sections.ts +1 -1
- package/src/boilerplate/system-prompt.ts +3 -2
- package/src/boilerplate/templates/layouts/chat-mobile.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/chat-universal.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/fullscreen-desktop.tsx.tmpl +3 -1
- package/src/boilerplate/templates/layouts/fullscreen-universal.tsx.tmpl +3 -1
- package/src/check/run-tier0.ts +67 -3
- package/src/contract-context.ts +1 -1
- package/src/design-system-docs.ts +2 -1
- package/src/evaluation/cost-tracker.ts +54 -6
- package/src/evaluation/criteria/edge-probe.ts +286 -0
- package/src/evaluation/criteria/edge.ts +202 -0
- package/src/evaluation/criteria/resolve.ts +74 -8
- package/src/evaluation/index.ts +5 -1
- package/src/evaluation/judge-spend.ts +48 -0
- package/src/evaluation/loop.ts +25 -4
- package/src/evaluation/message-parsing.ts +11 -12
- package/src/evaluation/types-public.ts +95 -0
- package/src/evaluation/types.ts +10 -0
- package/src/evaluation/visual-evaluator.ts +236 -40
- package/src/harness/check/runtime-render/adapter.ts +18 -0
- package/src/harness/check/runtime-render/probe-pending.ts +111 -0
- package/src/harness/check/runtime-render/render-check-host.ts +10 -6
- package/src/harness/check/runtime-render/render-check-worker.ts +6 -3
- package/src/harness/check/runtime-render/render-check.ts +417 -133
- package/src/harness/check/runtime-render/walk.ts +200 -0
- package/src/harness/coding/assemble-result.ts +2 -0
- package/src/harness/coding/generate-task-runner.ts +27 -0
- package/src/harness/coding/init-session.ts +5 -3
- package/src/harness/coding/run-coding-turn.ts +5 -1
- package/src/harness/coding/run-eval-round.ts +63 -3
- package/src/harness/in-loop-evaluation.ts +39 -0
- package/src/harness/llm-router.ts +58 -1
- package/src/harness/result-types.ts +9 -0
- package/src/harness/runtime.ts +9 -0
- package/src/harness/types-public.ts +10 -1
- package/src/index.ts +5 -0
- package/src/internal/verdict-line.ts +33 -0
- package/src/taught-tokens.ts +31 -0
- package/src/tools/render-check-worker.ts +2 -1
- package/src/tools/render-check.ts +8 -7
- package/src/validation/primitives.ts +1 -1
- package/dist/llm-router-nfO7wPec.d.ts +0 -267
- package/dist/{bank-CT4eEkta.d.ts → bank-Ctdp3FUA.d.ts} +3 -3
package/dist/evaluation/index.js
CHANGED
|
@@ -415,6 +415,13 @@ function buildCodingCriteriaSummary(designMode = "constrained") {
|
|
|
415
415
|
}
|
|
416
416
|
return lines.join("\n");
|
|
417
417
|
}
|
|
418
|
+
|
|
419
|
+
// src/adapters/claude/sdk-tool-pins.ts
|
|
420
|
+
function sdkToolPins(input) {
|
|
421
|
+
const builtins = [...input.builtinTools ?? []];
|
|
422
|
+
const approved = input.allowedTools !== void 0 ? [...input.allowedTools] : Object.keys(input.mcpServers ?? {}).map((name) => `mcp__${name}`);
|
|
423
|
+
return { tools: builtins, settingSources: [], allowedTools: [...approved, ...builtins] };
|
|
424
|
+
}
|
|
418
425
|
function createAnthropicClient(rawKey) {
|
|
419
426
|
if (rawKey === void 0) {
|
|
420
427
|
return new Anthropic({
|
|
@@ -816,6 +823,26 @@ async function collectLoginTurn(stream, opts) {
|
|
|
816
823
|
}
|
|
817
824
|
|
|
818
825
|
// src/harness/llm-router.ts
|
|
826
|
+
function servedModelOf(name) {
|
|
827
|
+
return typeof name === "string" && name.length > 0 ? { servedModel: name } : {};
|
|
828
|
+
}
|
|
829
|
+
function anthropicFinishReason(stopReason) {
|
|
830
|
+
if (stopReason === null || stopReason === void 0) return void 0;
|
|
831
|
+
if (stopReason === "end_turn" || stopReason === "stop_sequence") return "stop";
|
|
832
|
+
if (stopReason === "max_tokens") return "length";
|
|
833
|
+
if (stopReason === "refusal") return "content-filter";
|
|
834
|
+
return "other";
|
|
835
|
+
}
|
|
836
|
+
function googleFinishReason(finishReason) {
|
|
837
|
+
if (finishReason === null || finishReason === void 0) return void 0;
|
|
838
|
+
if (finishReason === "STOP") return "stop";
|
|
839
|
+
if (finishReason === "MAX_TOKENS") return "length";
|
|
840
|
+
if (["SAFETY", "RECITATION", "BLOCKLIST", "PROHIBITED_CONTENT", "SPII", "IMAGE_SAFETY"].includes(finishReason)) return "content-filter";
|
|
841
|
+
return "other";
|
|
842
|
+
}
|
|
843
|
+
function visionFinish(finishReason) {
|
|
844
|
+
return finishReason !== void 0 ? { finishReason } : {};
|
|
845
|
+
}
|
|
819
846
|
function addStrictSchemaConstraints(schema) {
|
|
820
847
|
const result = { ...schema };
|
|
821
848
|
if (result.type === "object" && result.properties) {
|
|
@@ -1156,7 +1183,9 @@ var AnthropicAgent = class extends LLMAgent {
|
|
|
1156
1183
|
return {
|
|
1157
1184
|
text,
|
|
1158
1185
|
inputTokens: response.usage.input_tokens,
|
|
1159
|
-
outputTokens: response.usage.output_tokens
|
|
1186
|
+
outputTokens: response.usage.output_tokens,
|
|
1187
|
+
...visionFinish(anthropicFinishReason(response.stop_reason)),
|
|
1188
|
+
...servedModelOf(response.model)
|
|
1160
1189
|
};
|
|
1161
1190
|
} catch (e) {
|
|
1162
1191
|
emitLlmTraceEvent({
|
|
@@ -1767,7 +1796,9 @@ var GoogleAgent = class extends LLMAgent {
|
|
|
1767
1796
|
text,
|
|
1768
1797
|
// ggui#1186 (third site): `promptTokenCount` includes `cachedContentTokenCount` — report the non-cached input.
|
|
1769
1798
|
inputTokens: splitCacheInclusiveUsage(usage?.promptTokenCount ?? 0, usage?.candidatesTokenCount ?? 0, usage?.cachedContentTokenCount ?? 0).tokens.input,
|
|
1770
|
-
outputTokens: usage?.candidatesTokenCount ?? 0
|
|
1799
|
+
outputTokens: usage?.candidatesTokenCount ?? 0,
|
|
1800
|
+
...visionFinish(googleFinishReason(response.candidates?.[0]?.finishReason)),
|
|
1801
|
+
...servedModelOf(response.modelVersion)
|
|
1771
1802
|
};
|
|
1772
1803
|
const endedAt = Date.now();
|
|
1773
1804
|
emitLlmTraceEvent({
|
|
@@ -2323,6 +2354,16 @@ function getDefaultEvalModel(provider) {
|
|
|
2323
2354
|
}
|
|
2324
2355
|
}
|
|
2325
2356
|
|
|
2357
|
+
// src/adapters/claude/source-capture.ts
|
|
2358
|
+
var COMPILE_TOOL_SUFFIX = "__compile_component";
|
|
2359
|
+
function sourceCodeFromToolUse(name, input) {
|
|
2360
|
+
if (typeof name !== "string") return void 0;
|
|
2361
|
+
const field2 = name === "Write" ? "content" : name.endsWith(COMPILE_TOOL_SUFFIX) ? "code" : void 0;
|
|
2362
|
+
if (field2 === void 0 || typeof input !== "object" || input === null) return void 0;
|
|
2363
|
+
const value = Reflect.get(input, field2);
|
|
2364
|
+
return typeof value === "string" && value.length > 0 ? value : void 0;
|
|
2365
|
+
}
|
|
2366
|
+
|
|
2326
2367
|
// src/evaluation/message-parsing.ts
|
|
2327
2368
|
function extractToolResultTexts(message) {
|
|
2328
2369
|
if (message.type !== "user") return [];
|
|
@@ -2392,12 +2433,9 @@ function extractSourceCodeFromMessage(message) {
|
|
|
2392
2433
|
const content = betaMessage?.content;
|
|
2393
2434
|
if (!Array.isArray(content)) return void 0;
|
|
2394
2435
|
for (const block of content) {
|
|
2395
|
-
if (block.type
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
sourceCode = input.content;
|
|
2399
|
-
}
|
|
2400
|
-
}
|
|
2436
|
+
if (block.type !== "tool_use") continue;
|
|
2437
|
+
const value = sourceCodeFromToolUse(block.name, block.input);
|
|
2438
|
+
if (value !== void 0) sourceCode = value;
|
|
2401
2439
|
}
|
|
2402
2440
|
return sourceCode;
|
|
2403
2441
|
}
|
|
@@ -2514,12 +2552,18 @@ async function runEvaluationLoop(options) {
|
|
|
2514
2552
|
const evaluationResults = [];
|
|
2515
2553
|
let currentCode = context.compiledCode;
|
|
2516
2554
|
let currentSourceCode = context.sourceCode;
|
|
2517
|
-
let
|
|
2555
|
+
let round2 = 0;
|
|
2518
2556
|
const maxRounds = Math.min(config.maxRounds ?? 3, MAX_EVAL_ROUNDS_HARD_LIMIT);
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2557
|
+
const offersATool = Object.keys(generatorOptions?.mcpServers ?? {}).length > 0 || (generatorOptions?.builtinTools?.length ?? 0) > 0;
|
|
2558
|
+
if (maxRounds > 1 && !offersATool) {
|
|
2559
|
+
throw new Error(
|
|
2560
|
+
"runEvaluationLoop: a fix round needs a tool that can return code \u2014 pass generatorOptions.mcpServers (e.g. a server with compile_component) or opt into a built-in with generatorOptions.builtinTools (e.g. ['Write'])."
|
|
2561
|
+
);
|
|
2562
|
+
}
|
|
2563
|
+
while (round2 < maxRounds) {
|
|
2564
|
+
round2++;
|
|
2565
|
+
onProgress?.({ type: "evaluating", round: round2 });
|
|
2566
|
+
console.log(`[eval] Round ${round2}: evaluating...`);
|
|
2523
2567
|
const evalContext = {
|
|
2524
2568
|
...context,
|
|
2525
2569
|
sourceCode: currentSourceCode || context.sourceCode
|
|
@@ -2527,13 +2571,13 @@ async function runEvaluationLoop(options) {
|
|
|
2527
2571
|
const evalResult = await runEvaluation(evalContext, config);
|
|
2528
2572
|
evaluationResults.push(evalResult);
|
|
2529
2573
|
console.log(
|
|
2530
|
-
`[eval] Round ${
|
|
2574
|
+
`[eval] Round ${round2}: score=${evalResult.finalScore}, passed=${evalResult.passed}, issues=${evalResult.issues.length}`
|
|
2531
2575
|
);
|
|
2532
|
-
if (evalResult.passed ||
|
|
2576
|
+
if (evalResult.passed || round2 >= maxRounds) {
|
|
2533
2577
|
break;
|
|
2534
2578
|
}
|
|
2535
|
-
onProgress?.({ type: "fixing", round });
|
|
2536
|
-
console.log(`[eval] Round ${
|
|
2579
|
+
onProgress?.({ type: "fixing", round: round2 });
|
|
2580
|
+
console.log(`[eval] Round ${round2}: fixing...`);
|
|
2537
2581
|
const fixPrompt = buildFixPrompt(evalResult, context.originalPrompt);
|
|
2538
2582
|
const cliPath = process.env.CLAUDE_WRAPPER_PATH;
|
|
2539
2583
|
const env = {};
|
|
@@ -2555,12 +2599,15 @@ async function runEvaluationLoop(options) {
|
|
|
2555
2599
|
maxTurns: 15,
|
|
2556
2600
|
maxBudgetUsd: config.maxBudgetPerFix,
|
|
2557
2601
|
env,
|
|
2558
|
-
permissionMode: "bypassPermissions",
|
|
2559
|
-
allowDangerouslySkipPermissions: true,
|
|
2560
2602
|
...cliPath && { pathToClaudeCodeExecutable: cliPath },
|
|
2561
2603
|
...generatorOptions?.cwd && { cwd: generatorOptions.cwd },
|
|
2562
2604
|
...generatorOptions?.mcpServers && { mcpServers: generatorOptions.mcpServers },
|
|
2563
|
-
|
|
2605
|
+
// ggui#1714 — no CLI built-ins unless asked for, no host settings, no permission bypass.
|
|
2606
|
+
...sdkToolPins({
|
|
2607
|
+
mcpServers: generatorOptions?.mcpServers,
|
|
2608
|
+
allowedTools: generatorOptions?.allowedTools,
|
|
2609
|
+
builtinTools: generatorOptions?.builtinTools
|
|
2610
|
+
}),
|
|
2564
2611
|
...generatorOptions?.model && { model: generatorOptions.model },
|
|
2565
2612
|
...generatorOptions?.stderr && { stderr: generatorOptions.stderr }
|
|
2566
2613
|
}
|
|
@@ -2573,7 +2620,7 @@ async function runEvaluationLoop(options) {
|
|
|
2573
2620
|
}
|
|
2574
2621
|
if (fixedCompiledCode) {
|
|
2575
2622
|
currentCode = fixedCompiledCode;
|
|
2576
|
-
console.log(`[eval] Round ${
|
|
2623
|
+
console.log(`[eval] Round ${round2}: fixed code captured (${currentCode.length} bytes)`);
|
|
2577
2624
|
}
|
|
2578
2625
|
if (fixedSourceCode) {
|
|
2579
2626
|
currentSourceCode = fixedSourceCode;
|
|
@@ -4692,9 +4739,19 @@ var INSTRUMENT_BY_ID = {
|
|
|
4692
4739
|
"floor.fit": "fit",
|
|
4693
4740
|
"finish.canvas.inhabited": "fill"
|
|
4694
4741
|
};
|
|
4695
|
-
|
|
4742
|
+
var INSTRUMENT_BESIDE_JUDGE = {
|
|
4743
|
+
"space.edge": "edge"
|
|
4744
|
+
};
|
|
4745
|
+
var REQUIRES_POST_ACTION_CAPTURE = /* @__PURE__ */ new Set(["state.feedback"]);
|
|
4746
|
+
var NO_POST_ACTION = "not evaluated: no post-action capture";
|
|
4747
|
+
function besideRead(kind, m) {
|
|
4748
|
+
if (m.edge === void 0) return { verdict: "n/a", evidence: "not measured: this capture had no page to read" };
|
|
4749
|
+
if (m.edge === null) return { verdict: "n/a", evidence: "edge probe failed on this capture" };
|
|
4750
|
+
return { verdict: m.edge.verdict, evidence: m.edge.evidence };
|
|
4751
|
+
}
|
|
4752
|
+
function majority(id, answers, unansweredReason) {
|
|
4696
4753
|
const votes = answers.flatMap((a) => a.filter((x) => x.id === id));
|
|
4697
|
-
if (votes.length === 0) return { verdict: "n/a", evidence: "not answered" };
|
|
4754
|
+
if (votes.length === 0) return { verdict: "n/a", evidence: unansweredReason ?? "not answered" };
|
|
4698
4755
|
const counts = /* @__PURE__ */ new Map();
|
|
4699
4756
|
for (const v of votes) counts.set(v.verdict, (counts.get(v.verdict) ?? 0) + 1);
|
|
4700
4757
|
const ranked = [...counts.entries()].sort((a, b) => b[1] - a[1]);
|
|
@@ -4711,8 +4768,9 @@ function instrumentRead(kind, m) {
|
|
|
4711
4768
|
if (m.inkRatio === null) return { verdict: "n/a", evidence: "capture unreadable" };
|
|
4712
4769
|
return { verdict: "n/a", evidence: `fill ${m.inkRatio.toFixed(3)} reported; no floor ruled` };
|
|
4713
4770
|
}
|
|
4714
|
-
function readRow(row, answers, m) {
|
|
4715
|
-
if (row.evaluation === "judge"
|
|
4771
|
+
function readRow(row, answers, m, unansweredReason) {
|
|
4772
|
+
if (row.evaluation === "judge" && REQUIRES_POST_ACTION_CAPTURE.has(row.id) && m.postActionCapture !== true) return { verdict: "n/a", evidence: NO_POST_ACTION };
|
|
4773
|
+
if (row.evaluation === "judge") return majority(row.id, answers, unansweredReason);
|
|
4716
4774
|
if (row.evaluation === "human") return { verdict: "n/a", evidence: `reader's column (status ${row.status})` };
|
|
4717
4775
|
if (row.status !== "live") return { verdict: "n/a", evidence: `not implemented (status ${row.status})` };
|
|
4718
4776
|
const kind = INSTRUMENT_BY_ID[row.id];
|
|
@@ -4724,8 +4782,18 @@ function resolveCriteriaBlock(args) {
|
|
|
4724
4782
|
for (const s of args.selected.selection) {
|
|
4725
4783
|
const row = byId.get(s.id);
|
|
4726
4784
|
if (row === void 0) continue;
|
|
4727
|
-
const read = readRow(row, args.answers, args.measurements);
|
|
4728
|
-
|
|
4785
|
+
const read = readRow(row, args.answers, args.measurements, args.unansweredReason);
|
|
4786
|
+
const beside = INSTRUMENT_BESIDE_JUDGE[row.id];
|
|
4787
|
+
verdicts.push({
|
|
4788
|
+
id: row.id,
|
|
4789
|
+
severity: row.severity,
|
|
4790
|
+
method: row.evaluation,
|
|
4791
|
+
status: row.status,
|
|
4792
|
+
source: s.source,
|
|
4793
|
+
verdict: read.verdict,
|
|
4794
|
+
evidence: read.evidence,
|
|
4795
|
+
...beside !== void 0 && row.evaluation === "judge" ? { instrument: besideRead(beside, args.measurements) } : {}
|
|
4796
|
+
});
|
|
4729
4797
|
}
|
|
4730
4798
|
return {
|
|
4731
4799
|
criteriaSetId: args.selected.criteriaSetId,
|
|
@@ -4733,7 +4801,15 @@ function resolveCriteriaBlock(args) {
|
|
|
4733
4801
|
selectorVersion: CRITERIA_SELECTOR_VERSION,
|
|
4734
4802
|
context: args.context,
|
|
4735
4803
|
selection: args.selected.selection,
|
|
4736
|
-
verdicts
|
|
4804
|
+
verdicts,
|
|
4805
|
+
...args.call !== void 0 ? {
|
|
4806
|
+
call: {
|
|
4807
|
+
answer: args.call.answer,
|
|
4808
|
+
// Kept answers whose id names no row of the bank: the judge answered something nobody asked.
|
|
4809
|
+
unknownIds: args.answers.flat().filter((a) => !byId.has(a.id)).length,
|
|
4810
|
+
...args.call.finishReason !== void 0 ? { finishReason: args.call.finishReason } : {}
|
|
4811
|
+
}
|
|
4812
|
+
} : {}
|
|
4737
4813
|
};
|
|
4738
4814
|
}
|
|
4739
4815
|
var CRITERIA_PROPS_MAX_CHARS = 2e3;
|
|
@@ -4753,7 +4829,7 @@ ${text}
|
|
|
4753
4829
|
}
|
|
4754
4830
|
function buildCriteriaJudgeBlock(bank, selected, inputs = {}) {
|
|
4755
4831
|
const byId = new Map(bankRows(bank).map((r) => [r.id, r]));
|
|
4756
|
-
const asked = selected.selection.map((s) => byId.get(s.id)).filter((r) => r !== void 0 && r.evaluation === "judge");
|
|
4832
|
+
const asked = selected.selection.map((s) => byId.get(s.id)).filter((r) => r !== void 0 && r.evaluation === "judge").filter((r) => inputs.postActionCapture === true || !REQUIRES_POST_ACTION_CAPTURE.has(r.id));
|
|
4757
4833
|
if (asked.length === 0) return "";
|
|
4758
4834
|
const lines = asked.map((r) => `- ${r.id} (${r.severity}): ${r.text} Evidence must name: ${r.evidence}`);
|
|
4759
4835
|
return "## Criteria \u2014 answer each one from the FRAME only\nRead these off the screenshot. Never infer them from the request, the styling profile or any direction text above.\n" + (inputs.frame !== void 0 ? `${criteriaFrameLine(inputs.frame)}
|
|
@@ -4938,6 +5014,356 @@ function judgeDesignIdentity(src) {
|
|
|
4938
5014
|
function resetJudgeDesignIdentityCache() {
|
|
4939
5015
|
identityCache.clear();
|
|
4940
5016
|
}
|
|
5017
|
+
|
|
5018
|
+
// src/evaluation/criteria/edge.ts
|
|
5019
|
+
var EDGE_TOLERANCE_PX = 2;
|
|
5020
|
+
var CARD = "card";
|
|
5021
|
+
var same = (a, b) => Math.abs(a - b) <= EDGE_TOLERANCE_PX;
|
|
5022
|
+
var overlaps = (a, b) => a.left < b.right && b.left < a.right;
|
|
5023
|
+
var axis = (b) => (b.left + b.right) / 2;
|
|
5024
|
+
function contentEdge(surface, blocks) {
|
|
5025
|
+
const texts = blocks.filter((b) => b.within === surface.id && b.kind !== "surface" && b.role !== "centred");
|
|
5026
|
+
if (texts.length === 0) return void 0;
|
|
5027
|
+
const first = texts.reduce((a, b) => b.top < a.top ? b : a);
|
|
5028
|
+
return first.textEdge ?? first.edge;
|
|
5029
|
+
}
|
|
5030
|
+
function columns(direct) {
|
|
5031
|
+
const all = direct.filter((b) => b.kind !== "surface");
|
|
5032
|
+
const bridges = (b) => {
|
|
5033
|
+
const under = all.filter((x) => x !== b && overlaps(x, b));
|
|
5034
|
+
return under.some((x, i) => under.slice(i + 1).some((y) => !overlaps(x, y)));
|
|
5035
|
+
};
|
|
5036
|
+
const flow = all.filter((b) => !bridges(b));
|
|
5037
|
+
const parentPool = all.filter((b) => bridges(b));
|
|
5038
|
+
const parent = flow.map((_, i) => i);
|
|
5039
|
+
const find = (i) => parent[i] === i ? i : parent[i] = find(parent[i]);
|
|
5040
|
+
for (let i = 0; i < flow.length; i += 1) for (let j = i + 1; j < flow.length; j += 1) if (overlaps(flow[i], flow[j])) parent[find(i)] = find(j);
|
|
5041
|
+
const groups = /* @__PURE__ */ new Map();
|
|
5042
|
+
flow.forEach((b, i) => groups.set(find(i), [...groups.get(find(i)) ?? [], b]));
|
|
5043
|
+
const cols = [...groups.values()];
|
|
5044
|
+
for (const s of direct.filter((b) => b.kind === "surface")) {
|
|
5045
|
+
const hit = cols.filter((c) => c.some((b) => b.kind !== "surface" && overlaps(b, s)));
|
|
5046
|
+
if (hit.length === 1) hit[0].push(s);
|
|
5047
|
+
else parentPool.push(s);
|
|
5048
|
+
}
|
|
5049
|
+
const lp = parentPool.map((_, i) => i);
|
|
5050
|
+
const lfind = (i) => lp[i] === i ? i : lp[i] = lfind(lp[i]);
|
|
5051
|
+
for (let i = 0; i < parentPool.length; i += 1) for (let j = i + 1; j < parentPool.length; j += 1) if (overlaps(parentPool[i], parentPool[j])) lp[lfind(i)] = lfind(j);
|
|
5052
|
+
const lgroups = /* @__PURE__ */ new Map();
|
|
5053
|
+
parentPool.forEach((b, i) => lgroups.set(lfind(i), [...lgroups.get(lfind(i)) ?? [], b]));
|
|
5054
|
+
cols.push(...lgroups.values());
|
|
5055
|
+
return cols;
|
|
5056
|
+
}
|
|
5057
|
+
function checkColumn(container, col, blocks) {
|
|
5058
|
+
const surfaces = col.filter((b) => b.kind === "surface");
|
|
5059
|
+
const grounds = surfaces.flatMap((s) => {
|
|
5060
|
+
const c = s.role === "table" ? void 0 : contentEdge(s, blocks);
|
|
5061
|
+
return c === void 0 ? [] : [c];
|
|
5062
|
+
});
|
|
5063
|
+
const flow = col.filter((b) => b.kind !== "surface" && b.role !== "centred" && b.role !== "list-item");
|
|
5064
|
+
const listItems = col.filter((b) => b.role === "list-item");
|
|
5065
|
+
const candidates = [...flow.flatMap((b) => [b.edge, ...b.textEdge !== void 0 ? [b.textEdge] : []]), ...surfaces.flatMap((s) => [s.edge]), ...grounds];
|
|
5066
|
+
const blockEdges = (b) => [b.edge, ...b.textEdge !== void 0 ? [b.textEdge] : []];
|
|
5067
|
+
const endEdgesBesides = (b) => [...surfaces.map((s) => s.right), ...flow.filter((x) => x !== b && x.kind === "controls").map((x) => x.right)];
|
|
5068
|
+
let best = candidates.length === 0 ? [] : null;
|
|
5069
|
+
for (const E of candidates) {
|
|
5070
|
+
const allowed = [E, ...grounds];
|
|
5071
|
+
const off = [];
|
|
5072
|
+
for (const b of flow) {
|
|
5073
|
+
if (blockEdges(b).some((x) => allowed.some((a) => same(x, a)))) continue;
|
|
5074
|
+
if (b.kind === "controls" && endEdgesBesides(b).some((r) => same(b.right, r))) continue;
|
|
5075
|
+
const nearest = allowed.reduce((n, a) => Math.abs(a - b.edge) < Math.abs(n - b.edge) ? a : n);
|
|
5076
|
+
off.push({ container, block: b.id, at: b.edge, nearest, offPx: Math.abs(b.edge - nearest) });
|
|
5077
|
+
}
|
|
5078
|
+
for (const s of surfaces) {
|
|
5079
|
+
const c = s.role === "table" ? void 0 : contentEdge(s, blocks);
|
|
5080
|
+
if (same(s.edge, E) || c !== void 0 && same(c, E)) continue;
|
|
5081
|
+
const nearest = c !== void 0 && Math.abs(c - E) < Math.abs(s.edge - E) ? c : s.edge;
|
|
5082
|
+
off.push({ container, block: s.id, at: nearest, nearest: E, offPx: Math.abs(nearest - E) });
|
|
5083
|
+
}
|
|
5084
|
+
if (best === null || off.length < best.length || off.length === best.length && sum(off) < sum(best)) best = off;
|
|
5085
|
+
if (best.length === 0) break;
|
|
5086
|
+
}
|
|
5087
|
+
const offsets = best ?? [];
|
|
5088
|
+
if (listItems.length > 1) {
|
|
5089
|
+
const e0 = listItems[0].edge;
|
|
5090
|
+
for (const li of listItems.slice(1)) if (!same(li.edge, e0)) offsets.push({ container, block: li.id, at: li.edge, nearest: e0, offPx: Math.abs(li.edge - e0) });
|
|
5091
|
+
}
|
|
5092
|
+
const centred = col.filter((b) => b.role === "centred");
|
|
5093
|
+
const edgeAligned = col.filter((b) => b.role !== "centred");
|
|
5094
|
+
const groupTop = centred.length > 0 ? Math.min(...centred.map((c) => c.top)) : 0;
|
|
5095
|
+
const groupEndsColumn = centred.length > 1 && edgeAligned.every((b) => b.top < groupTop);
|
|
5096
|
+
if (centred.length > 0 && edgeAligned.some((b) => b.kind !== "surface") && !groupEndsColumn) {
|
|
5097
|
+
const E = flow[0]?.edge ?? edgeAligned[0].edge;
|
|
5098
|
+
for (const c of centred) offsets.push({ container, block: c.id, at: c.edge, nearest: E, offPx: Math.abs(c.edge - E), reason: "a centred block in an edge-aligned column" });
|
|
5099
|
+
} else if (centred.length > 1) {
|
|
5100
|
+
const a0 = axis(centred[0]);
|
|
5101
|
+
for (const c of centred.slice(1)) if (!same(axis(c), a0)) offsets.push({ container, block: c.id, at: axis(c), nearest: a0, offPx: Math.abs(axis(c) - a0) });
|
|
5102
|
+
}
|
|
5103
|
+
return offsets;
|
|
5104
|
+
}
|
|
5105
|
+
var sum = (o) => o.reduce((s, x) => s + x.offPx, 0);
|
|
5106
|
+
function judgeEdges(measured, opts = {}) {
|
|
5107
|
+
if (opts.placeholdersOnly === true) return { verdict: "n/a", offsets: [], evidence: "only loading placeholders are shown; not evaluated" };
|
|
5108
|
+
let blocks = measured;
|
|
5109
|
+
const kept = blocks.filter((b) => b.role !== "placeholder");
|
|
5110
|
+
const content = kept.filter((b) => b.kind !== "surface").length;
|
|
5111
|
+
if (content < 2) return { verdict: "n/a", offsets: [], evidence: kept.length === 0 ? "no blocks were measured" : "fewer than two blocks once placeholders are set aside; not evaluated" };
|
|
5112
|
+
blocks = kept;
|
|
5113
|
+
const offsets = [];
|
|
5114
|
+
const containers = [CARD, ...blocks.filter((b) => b.kind === "surface" && b.role !== "table").map((b) => b.id)];
|
|
5115
|
+
for (const container of containers) {
|
|
5116
|
+
const direct = blocks.filter((b) => (b.within ?? CARD) === container);
|
|
5117
|
+
for (const col of columns(direct)) offsets.push(...checkColumn(container, col, blocks));
|
|
5118
|
+
}
|
|
5119
|
+
if (offsets.length === 0) return { verdict: "pass", offsets, evidence: "every column shares its edge, or sits on a stacked surface\u2019s content edge, within 2 px" };
|
|
5120
|
+
const worst = offsets.reduce((a, b) => b.offPx > a.offPx ? b : a);
|
|
5121
|
+
const lines = offsets.map((o) => `${o.block} at x${round(o.at)} is ${round(o.offPx)} px from x${round(o.nearest)} (in ${o.container}${o.reason !== void 0 ? `; ${o.reason}` : ""})`);
|
|
5122
|
+
return { verdict: "fail", offsets, evidence: `${offsets.length} block(s) off the allowed edges, largest ${round(worst.offPx)} px: ${lines.join("; ")}` };
|
|
5123
|
+
}
|
|
5124
|
+
var round = (n) => (Math.round(n * 10) / 10).toString();
|
|
5125
|
+
|
|
5126
|
+
// src/evaluation/criteria/edge-probe.ts
|
|
5127
|
+
var EDGE_BLOCKS_EXPRESSION = String.raw`(() => {
|
|
5128
|
+
const root = document.getElementById('root');
|
|
5129
|
+
if (!root) return null;
|
|
5130
|
+
const rootBox = root.getBoundingClientRect();
|
|
5131
|
+
const x0 = rootBox.left;
|
|
5132
|
+
const px = (v) => Math.round((v - x0) * 2) / 2;
|
|
5133
|
+
const CONTROL = 'button,input,select,textarea,[role="button"],[role="tab"],[role="switch"],[role="checkbox"],[role="radio"],[role="option"],[role="link"],a[href]';
|
|
5134
|
+
const ALNUM = /[\p{L}\p{N}]/u;
|
|
5135
|
+
const css = (el) => getComputedStyle(el);
|
|
5136
|
+
const clear = (c) => !c || c === 'transparent' || /rgba\([^)]*,\s*0(\.0+)?\)$/.test(c);
|
|
5137
|
+
// Visible through every ancestor (an ancestor at opacity 0 or visibility hidden hides it), and not degenerate.
|
|
5138
|
+
const shown = (el) => {
|
|
5139
|
+
const s = css(el);
|
|
5140
|
+
if (s.display === 'none' || s.visibility === 'hidden' || Number(s.opacity) === 0) return false;
|
|
5141
|
+
if (typeof el.checkVisibility === 'function' && !el.checkVisibility({ opacityProperty: true, visibilityProperty: true })) return false;
|
|
5142
|
+
const r = el.getBoundingClientRect();
|
|
5143
|
+
return r.width > 1 && r.height > 1;
|
|
5144
|
+
};
|
|
5145
|
+
const inline = (s) => /^inline/.test(s.display) && s.display !== 'inline-block' && s.display !== 'inline-flex' && s.display !== 'inline-grid';
|
|
5146
|
+
const groundOf = (el) => {
|
|
5147
|
+
for (let p = el.parentElement; p; p = p.parentElement) { const b = css(p).backgroundColor; if (!clear(b)) return b; }
|
|
5148
|
+
return 'rgb(255, 255, 255)';
|
|
5149
|
+
};
|
|
5150
|
+
const sides = (s) => ['Top', 'Right', 'Bottom', 'Left'].filter((k) => parseFloat(s['border' + k + 'Width']) > 0 && s['border' + k + 'Style'] !== 'none' && !clear(s['border' + k + 'Color'])).length;
|
|
5151
|
+
const bounded = (el) => { const s = css(el); return sides(s) >= 1 || (!clear(s.backgroundColor) && s.backgroundColor !== groundOf(el)); };
|
|
5152
|
+
// A control: a native or ARIA one, or a bounded box the visitor clicks (a pointer cursor or a tab stop) that holds no
|
|
5153
|
+
// control of its own and is not the whole card. Its contents are not blocks of the column (decided 2026-10-02).
|
|
5154
|
+
const isControl = (el) => {
|
|
5155
|
+
const s = css(el);
|
|
5156
|
+
if (inline(s)) return false;
|
|
5157
|
+
if (el.matches(CONTROL)) return true;
|
|
5158
|
+
if (el === root) return false;
|
|
5159
|
+
const clickable = s.cursor === 'pointer' || (el.hasAttribute('tabindex') && el.tabIndex >= 0);
|
|
5160
|
+
if (!clickable || !bounded(el) || el.querySelector(CONTROL)) return false;
|
|
5161
|
+
const r = el.getBoundingClientRect();
|
|
5162
|
+
return r.width * r.height < 0.6 * rootBox.width * rootBox.height;
|
|
5163
|
+
};
|
|
5164
|
+
const isTable = (el) => el.matches('table,[role="table"],[role="grid"]');
|
|
5165
|
+
const isSurface = (el) => {
|
|
5166
|
+
if (el === root || isControl(el)) return false;
|
|
5167
|
+
const s = css(el);
|
|
5168
|
+
if (/^inline/.test(s.display) || s.display === 'contents') return false;
|
|
5169
|
+
if (isTable(el)) return true;
|
|
5170
|
+
return sides(s) >= 3 || (!clear(s.backgroundColor) && s.backgroundColor !== groundOf(el)) || (s.boxShadow && s.boxShadow !== 'none');
|
|
5171
|
+
};
|
|
5172
|
+
const path = (el) => {
|
|
5173
|
+
const parts = [];
|
|
5174
|
+
for (let e = el; e && e !== root && parts.length < 4; e = e.parentElement) {
|
|
5175
|
+
const i = e.parentElement ? Array.prototype.indexOf.call(e.parentElement.children, e) + 1 : 1;
|
|
5176
|
+
parts.unshift(e.tagName.toLowerCase() + ':' + i);
|
|
5177
|
+
}
|
|
5178
|
+
return parts.join('>');
|
|
5179
|
+
};
|
|
5180
|
+
const label = (el) => { const t = (el.innerText || '').replace(/\s+/g, ' ').trim(); return path(el) + (t ? ' "' + t.slice(0, 24) + '"' : ''); };
|
|
5181
|
+
const surfaces = [];
|
|
5182
|
+
const surfaceOf = (el, self) => { for (let p = self ? el : el.parentElement; p && p !== root; p = p.parentElement) if (surfaces.indexOf(p) >= 0) return p; return null; };
|
|
5183
|
+
const inTable = (el) => { for (let p = el.parentElement; p && p !== root; p = p.parentElement) if (isTable(p)) return true; return false; };
|
|
5184
|
+
const inControl = (el) => { for (let p = el; p && p !== root; p = p.parentElement) if (isControl(p)) return p; return null; };
|
|
5185
|
+
const all = Array.prototype.slice.call(root.querySelectorAll('*')).filter(shown);
|
|
5186
|
+
for (const el of all) if (isSurface(el) && !inTable(el) && !inControl(el)) surfaces.push(el);
|
|
5187
|
+
const ids = new Map();
|
|
5188
|
+
const idOf = (el) => { if (!ids.has(el)) ids.set(el, label(el)); return ids.get(el); };
|
|
5189
|
+
const blocks = [];
|
|
5190
|
+
// Per block: the element it stands for and its first line's box, for rows.
|
|
5191
|
+
const owners = [];
|
|
5192
|
+
const lines = [];
|
|
5193
|
+
// Per block: its raw line boxes and left, so a row reads the line two items share, not a wrapped item's widest line.
|
|
5194
|
+
const rects = [];
|
|
5195
|
+
const rawLeft = [];
|
|
5196
|
+
const words = [];
|
|
5197
|
+
const within = (el, self) => { const s = surfaceOf(el, self); return s ? { within: idOf(s) } : {}; };
|
|
5198
|
+
for (const s of surfaces) {
|
|
5199
|
+
const r = s.getBoundingClientRect();
|
|
5200
|
+
owners.push(s); lines.push({ top: r.top, bottom: r.bottom }); rects.push([r]); rawLeft.push(r.left); words.push('');
|
|
5201
|
+
// A placeholder: a box with no text, no control and no image or icon inside it (a loading bar). Not a block.
|
|
5202
|
+
const holds = Array.prototype.some.call(s.querySelectorAll('*'), (el) => shown(el) && (el.matches(CONTROL) || el.tagName === 'IMG' || el.tagName.toLowerCase() === 'svg')) || ALNUM.test(s.innerText || '');
|
|
5203
|
+
blocks.push(Object.assign({ id: idOf(s), kind: 'surface', edge: px(r.left), left: px(r.left), right: px(r.right), top: Math.round(r.top) }, within(s, false), isTable(s) ? { role: 'table' } : holds ? {} : { role: 'placeholder' }));
|
|
5204
|
+
}
|
|
5205
|
+
const blockOf = (node) => { for (let p = node.parentElement; p && p !== root; p = p.parentElement) if (!inline(css(p))) return p; return root; };
|
|
5206
|
+
const groups = new Map();
|
|
5207
|
+
const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT);
|
|
5208
|
+
for (let n = walker.nextNode(); n; n = walker.nextNode()) {
|
|
5209
|
+
if (!/\S/.test(n.data) || !n.parentElement || n.parentElement.closest('style,script,noscript,template') || !shown(n.parentElement) || inControl(n.parentElement) || inTable(n.parentElement)) continue;
|
|
5210
|
+
const b = blockOf(n);
|
|
5211
|
+
if (!groups.has(b)) groups.set(b, []);
|
|
5212
|
+
groups.get(b).push(n);
|
|
5213
|
+
}
|
|
5214
|
+
// Ink: text whose own colour is not transparent. The edge is the first visible character in ink (decided 2026-10-02):
|
|
5215
|
+
// leading invisible text, private-use icon-font glyphs and formatting characters are not characters of the block,
|
|
5216
|
+
// while a currency sign or a quotation mark is.
|
|
5217
|
+
const inked = (n) => !clear(css(n.parentElement).color) && ALNUM.test(n.data);
|
|
5218
|
+
const textRects = (nodes) => {
|
|
5219
|
+
const rs = [];
|
|
5220
|
+
for (const n of nodes) { if (!inked(n)) continue; const range = document.createRange(); range.selectNodeContents(n); for (const r of range.getClientRects()) if (r.width > 0 && r.height > 0) rs.push(r); }
|
|
5221
|
+
return rs;
|
|
5222
|
+
};
|
|
5223
|
+
const glyphAt = (n, i) => { const range = document.createRange(); range.setStart(n, i); range.setEnd(n, i + 1); const r = range.getClientRects()[0]; return r || null; };
|
|
5224
|
+
const firstGlyph = (nodes) => {
|
|
5225
|
+
for (const n of nodes) { if (!inked(n)) continue; const i = n.data.search(/[^\s\p{Co}\p{Cf}]/u); if (i < 0) continue; const r = glyphAt(n, i); if (r) return r; }
|
|
5226
|
+
return null;
|
|
5227
|
+
};
|
|
5228
|
+
// A marker paints: a loaded image, an SVG with a filled or stroked shape, or an element with its own fill, border
|
|
5229
|
+
// or visible text. An element that paints nothing is not a marker.
|
|
5230
|
+
const painted = (m) => {
|
|
5231
|
+
if (!shown(m)) return false;
|
|
5232
|
+
if (m.tagName === 'IMG') return m.complete && m.naturalWidth > 0;
|
|
5233
|
+
if (m.tagName.toLowerCase() === 'svg' || m.closest('svg')) {
|
|
5234
|
+
const shapes = m.querySelectorAll('path,circle,rect,line,polyline,polygon,ellipse,use,text');
|
|
5235
|
+
return Array.prototype.some.call(shapes, (sh) => { const st = css(sh); return (st.fill !== 'none' && !clear(st.fill)) || (st.stroke !== 'none' && !clear(st.stroke)); });
|
|
5236
|
+
}
|
|
5237
|
+
const s = css(m);
|
|
5238
|
+
if (!clear(s.backgroundColor) || sides(s) >= 1 || ((m.textContent || '').trim().length > 0 && !clear(s.color))) return true;
|
|
5239
|
+
// A wrapper that paints nothing itself is a marker when an image or icon inside it paints.
|
|
5240
|
+
return Array.prototype.some.call(m.querySelectorAll('svg,img'), (k) => painted(k));
|
|
5241
|
+
};
|
|
5242
|
+
const markerBefore = (b, textLeft, lineTop, lineBottom) => {
|
|
5243
|
+
const cands = [];
|
|
5244
|
+
const consider = (m) => { if (!painted(m)) return; const r = m.getBoundingClientRect(); if (r.left < textLeft - 2 && r.bottom > lineTop && r.top < lineBottom) cands.push(r.left); };
|
|
5245
|
+
for (const m of b.querySelectorAll('svg,img,[aria-hidden="true"],i')) if ((m.textContent || '').trim().length <= 2) consider(m);
|
|
5246
|
+
const p = b.parentElement;
|
|
5247
|
+
if (p && p !== root && /flex/.test(css(p).display) && !/column/.test(css(p).flexDirection)) for (const sib of p.children) { if (sib === b) break; if (sib.matches('svg,img,i,[aria-hidden="true"]') || !(sib.textContent || '').trim()) consider(sib); }
|
|
5248
|
+
return cands.length ? Math.min.apply(null, cands) : null;
|
|
5249
|
+
};
|
|
5250
|
+
let texts = 0;
|
|
5251
|
+
let ornaments = 0;
|
|
5252
|
+
for (const [b, nodes] of groups) {
|
|
5253
|
+
if (!nodes.some((n) => ALNUM.test(n.data))) { ornaments += 1; continue; }
|
|
5254
|
+
const rs = textRects(nodes);
|
|
5255
|
+
const g = firstGlyph(nodes);
|
|
5256
|
+
if (!rs.length || g === null) continue;
|
|
5257
|
+
texts += 1;
|
|
5258
|
+
const first = g.left;
|
|
5259
|
+
const left = Math.min.apply(null, rs.map((r) => r.left));
|
|
5260
|
+
const right = Math.max.apply(null, rs.map((r) => r.right));
|
|
5261
|
+
const top = Math.min.apply(null, rs.map((r) => r.top));
|
|
5262
|
+
const marker = markerBefore(b, first, g.top, g.bottom);
|
|
5263
|
+
const s = css(b);
|
|
5264
|
+
const box = b.getBoundingClientRect();
|
|
5265
|
+
const parent = b.parentElement ? css(b.parentElement) : null;
|
|
5266
|
+
const centredText = s.textAlign === 'center' && right - left < box.width * 0.9;
|
|
5267
|
+
const centredBox = parent && /flex/.test(parent.display) && /column/.test(parent.flexDirection) && parent.alignItems === 'center';
|
|
5268
|
+
const list = b.matches('li') || s.display === 'list-item' ? b.closest('ul,ol') : null;
|
|
5269
|
+
const inProse = list && list.previousElementSibling && list.previousElementSibling.matches('p');
|
|
5270
|
+
const block = { id: idOf(b), kind: 'text', edge: px(marker !== null ? marker : first), left: px(Math.min(left, marker !== null ? marker : left)), right: px(right), top: Math.round(top) };
|
|
5271
|
+
if (marker !== null) block.textEdge = px(first);
|
|
5272
|
+
if (centredText || centredBox) block.role = 'centred';
|
|
5273
|
+
else if (inProse) block.role = 'list-item';
|
|
5274
|
+
owners.push(b); lines.push({ top: g.top, bottom: g.bottom }); rects.push(rs); rawLeft.push(Math.min(left, marker !== null ? marker : left)); words.push(nodes.map((n) => n.data).join('').trim());
|
|
5275
|
+
blocks.push(Object.assign(block, within(b, true)));
|
|
5276
|
+
}
|
|
5277
|
+
const ctlGroups = new Map();
|
|
5278
|
+
for (const el of all) { if (!isControl(el) || inTable(el) || (el.parentElement && inControl(el.parentElement))) continue; const p = el.parentElement || root; if (!ctlGroups.has(p)) ctlGroups.set(p, []); ctlGroups.get(p).push(el); }
|
|
5279
|
+
let controls = 0;
|
|
5280
|
+
for (const [p, els] of ctlGroups) {
|
|
5281
|
+
const rs = els.map((e) => e.getBoundingClientRect());
|
|
5282
|
+
const topRow = Math.min.apply(null, rs.map((r) => r.top));
|
|
5283
|
+
let firstEl = null; let firstR = null;
|
|
5284
|
+
els.forEach((e, i) => { const r = rs[i]; if (r.top - topRow < r.height / 2 && (!firstR || r.left < firstR.left)) { firstEl = e; firstR = r; } });
|
|
5285
|
+
let edge = firstR.left;
|
|
5286
|
+
if (!bounded(firstEl)) { const tw = document.createTreeWalker(firstEl, NodeFilter.SHOW_TEXT); const ns = []; for (let n = tw.nextNode(); n; n = tw.nextNode()) ns.push(n); const g = firstGlyph(ns); if (g) edge = g.left; }
|
|
5287
|
+
controls += 1;
|
|
5288
|
+
const ps = css(p);
|
|
5289
|
+
const centred = ps.justifyContent === 'center' || (ps.textAlign === 'center' && !/flex|grid/.test(ps.display));
|
|
5290
|
+
const block = { id: idOf(p) + ' [controls]', kind: 'controls', edge: px(edge), left: px(Math.min.apply(null, rs.map((r) => r.left))), right: px(Math.max.apply(null, rs.map((r) => r.right))), top: Math.round(topRow) };
|
|
5291
|
+
if (centred) block.role = 'centred';
|
|
5292
|
+
owners.push(firstEl); lines.push({ top: firstR.top, bottom: firstR.bottom }); rects.push(rs); rawLeft.push(Math.min.apply(null, rs.map((r) => r.left))); words.push('');
|
|
5293
|
+
blocks.push(Object.assign(block, within(firstEl, true)));
|
|
5294
|
+
}
|
|
5295
|
+
// Rows (decided 2026-10-01, read by geometry 2026-10-02): items set on one line, however the CSS sets them. An item
|
|
5296
|
+
// after another measured item on its line that holds one block (a text, a control, or a box holding at most one
|
|
5297
|
+
// text) belongs to the row and is not measured; an item with a stack inside it is a column.
|
|
5298
|
+
const dropped = new Set();
|
|
5299
|
+
const sameLine = (a, b) => Math.min(a.bottom, b.bottom) - Math.max(a.top, b.top) > Math.min(a.bottom - a.top, b.bottom - b.top) / 2;
|
|
5300
|
+
const contains = (o, el) => o === el || o.contains(el);
|
|
5301
|
+
const innerTexts = (i) => owners.map((el, k) => k).filter((k) => k !== i && blocks[k].kind !== 'surface' && contains(owners[i], owners[k]));
|
|
5302
|
+
const isPill = (i) => blocks[i].kind === 'surface' && innerTexts(i).length <= 1 && !owners.some((el, k) => k !== i && blocks[k].kind === 'surface' && contains(owners[i], el));
|
|
5303
|
+
const itemOf = (el, other) => { let e = el; while (e.parentElement && e.parentElement !== root && !e.parentElement.contains(other)) e = e.parentElement; return e; };
|
|
5304
|
+
for (let i = 0; i < blocks.length; i += 1) {
|
|
5305
|
+
if (blocks[i].kind === 'surface' && !isPill(i)) continue;
|
|
5306
|
+
const before = blocks.findIndex((bj, j) => j !== i && bj.within === blocks[i].within && !(bj.kind === 'surface' && !isPill(j)) && !contains(owners[j], owners[i]) && !contains(owners[i], owners[j]) && rects[j].some((r) => sameLine(r, lines[i]) && r.right <= rawLeft[i] + 1));
|
|
5307
|
+
if (before < 0) continue;
|
|
5308
|
+
// A number or bullet in front of a text on its line is a hanging marker, not a row: the pair is one block with two
|
|
5309
|
+
// edges, the marker's and the text's.
|
|
5310
|
+
if (blocks[i].kind === 'text' && blocks[before].kind === 'text' && /^[\p{N}\p{P}\p{S}]{1,3}$/u.test(words[before])) {
|
|
5311
|
+
blocks[i] = Object.assign({}, blocks[i], { edge: blocks[before].edge, left: Math.min(blocks[i].left, blocks[before].left), textEdge: blocks[i].edge });
|
|
5312
|
+
delete blocks[i].role;
|
|
5313
|
+
dropped.add(before);
|
|
5314
|
+
continue;
|
|
5315
|
+
}
|
|
5316
|
+
const item = itemOf(owners[i], owners[before]);
|
|
5317
|
+
const inside = owners.map((el, k) => k).filter((k) => contains(item, owners[k]) && blocks[k].kind !== 'surface');
|
|
5318
|
+
const boxes = owners.map((el, k) => k).filter((k) => contains(item, owners[k]) && blocks[k].kind === 'surface' && k !== i);
|
|
5319
|
+
if (blocks[i].kind === 'surface') { if (boxes.length === 0) { dropped.add(i); for (const k of innerTexts(i)) dropped.add(k); } continue; }
|
|
5320
|
+
if (inside.length === 1 && boxes.length === 0) dropped.add(i);
|
|
5321
|
+
}
|
|
5322
|
+
const kept = blocks.filter((_, i) => !dropped.has(i));
|
|
5323
|
+
// Placeholders: a frame with no text in ink shows nothing to evaluate, whatever controls or boxes it draws.
|
|
5324
|
+
return { blocks: kept, placeholdersOnly: texts === 0, skipped: { rowItems: dropped.size, ornaments } };
|
|
5325
|
+
})()`;
|
|
5326
|
+
var KINDS = ["text", "controls", "surface"];
|
|
5327
|
+
var ROLES = ["list-item", "table", "centred", "placeholder"];
|
|
5328
|
+
var isKind = (v) => KINDS.some((k) => k === v);
|
|
5329
|
+
var isRole = (v) => ROLES.some((r) => r === v);
|
|
5330
|
+
var num = (v) => typeof v === "number" && Number.isFinite(v);
|
|
5331
|
+
var field = (o, key) => Reflect.get(o, key);
|
|
5332
|
+
function parseEdgeProbe(value) {
|
|
5333
|
+
if (typeof value !== "object" || value === null) return null;
|
|
5334
|
+
const raw = field(value, "blocks");
|
|
5335
|
+
const placeholdersOnly = field(value, "placeholdersOnly");
|
|
5336
|
+
if (!Array.isArray(raw) || typeof placeholdersOnly !== "boolean") return null;
|
|
5337
|
+
const skippedRaw = field(value, "skipped");
|
|
5338
|
+
if (typeof skippedRaw !== "object" || skippedRaw === null) return null;
|
|
5339
|
+
const rowItems = field(skippedRaw, "rowItems");
|
|
5340
|
+
const ornaments = field(skippedRaw, "ornaments");
|
|
5341
|
+
if (!num(rowItems) || !num(ornaments)) return null;
|
|
5342
|
+
const blocks = [];
|
|
5343
|
+
for (const entry of raw) {
|
|
5344
|
+
if (typeof entry !== "object" || entry === null) return null;
|
|
5345
|
+
const [id, kind, edge, left, right, top] = ["id", "kind", "edge", "left", "right", "top"].map((k) => field(entry, k));
|
|
5346
|
+
const [within, textEdge, role] = ["within", "textEdge", "role"].map((k) => field(entry, k));
|
|
5347
|
+
if (typeof id !== "string" || !isKind(kind) || !num(edge) || !num(left) || !num(right) || !num(top)) return null;
|
|
5348
|
+
if (within !== void 0 && typeof within !== "string") return null;
|
|
5349
|
+
if (textEdge !== void 0 && !num(textEdge)) return null;
|
|
5350
|
+
if (role !== void 0 && !isRole(role)) return null;
|
|
5351
|
+
blocks.push({
|
|
5352
|
+
id,
|
|
5353
|
+
kind,
|
|
5354
|
+
edge,
|
|
5355
|
+
left,
|
|
5356
|
+
right,
|
|
5357
|
+
top,
|
|
5358
|
+
...within !== void 0 ? { within } : {},
|
|
5359
|
+
...textEdge !== void 0 ? { textEdge } : {},
|
|
5360
|
+
...role !== void 0 ? { role } : {}
|
|
5361
|
+
});
|
|
5362
|
+
}
|
|
5363
|
+
return { blocks, placeholdersOnly, skipped: { rowItems, ornaments } };
|
|
5364
|
+
}
|
|
5365
|
+
|
|
5366
|
+
// src/evaluation/visual-evaluator.ts
|
|
4941
5367
|
function canvasFitPolicy(canvas) {
|
|
4942
5368
|
switch (canvas) {
|
|
4943
5369
|
case "xs-chat-card":
|
|
@@ -5021,6 +5447,7 @@ function canvasOverflowIssue(canvas, viewport, contentHeight, verdict) {
|
|
|
5021
5447
|
const box = isDeclaredViewport(canvas, viewport) ? `declared ${viewport.width}\xD7${viewport.height}; class box ${classBox.width}\xD7${classBox.height}` : `${viewport.width}\xD7${viewport.height}`;
|
|
5022
5448
|
return {
|
|
5023
5449
|
dimension: "canvas-overflow",
|
|
5450
|
+
origin: "instrument",
|
|
5024
5451
|
severity: verdict === "fail" ? "critical" : "major",
|
|
5025
5452
|
description: `Rendered content is ${contentHeight}px tall on the ${canvas} canvas (${box}) \u2014 ${hidden}px ${verdict === "fail" ? "is cut off: the inline card does not scroll" : "sits below the first screen"}.`,
|
|
5026
5453
|
fix: "Fit the composition to the canvas: fewer and shorter sections, one compact row of chips, no hero taller than a third of the box, no fixed min-heights or tall paddings \u2014 measure against the canvas, not the page."
|
|
@@ -5029,6 +5456,7 @@ function canvasOverflowIssue(canvas, viewport, contentHeight, verdict) {
|
|
|
5029
5456
|
function canvasOverflowXIssue(canvas, viewport, hiddenPx, verdict) {
|
|
5030
5457
|
return {
|
|
5031
5458
|
dimension: "canvas-overflow-x",
|
|
5459
|
+
origin: "instrument",
|
|
5032
5460
|
severity: verdict === "fail" ? "critical" : "major",
|
|
5033
5461
|
description: `Content runs ${hiddenPx}px past the ${canvas} card's ${viewport.width}px width \u2014 the card's frame clips it, so a visitor sees a cut-off row or a sideways scroll.`,
|
|
5034
5462
|
fix: "Keep every row within the card: wrap long text (overflow-wrap: anywhere), let a wide table or code block scroll inside its own container, and give no child a fixed width wider than the card."
|
|
@@ -5039,6 +5467,7 @@ function canvasBlankIssue(canvas, viewport) {
|
|
|
5039
5467
|
const box = isDeclaredViewport(canvas, viewport) ? `declared ${viewport.width}\xD7${viewport.height}; class box ${classBox.width}\xD7${classBox.height}` : `${viewport.width}\xD7${viewport.height}`;
|
|
5040
5468
|
return {
|
|
5041
5469
|
dimension: "canvas-blank",
|
|
5470
|
+
origin: "instrument",
|
|
5042
5471
|
severity: "critical",
|
|
5043
5472
|
description: `Nothing painted on the ${canvas} canvas (${box}): the capture is one flat colour \u2014 the component mounted and rendered no pixels.`,
|
|
5044
5473
|
fix: "Return and paint the component tree: a root that renders nothing is a blank, not an empty state. Check that the default export returns its JSX, that no early return yields null on the sample props, and that the first paint shows visible content."
|
|
@@ -5260,6 +5689,19 @@ async function measureContentHeight(page, expression = CONTENT_HEIGHT_EXPRESSION
|
|
|
5260
5689
|
return null;
|
|
5261
5690
|
}
|
|
5262
5691
|
}
|
|
5692
|
+
async function measureEdges(page) {
|
|
5693
|
+
try {
|
|
5694
|
+
const probe = parseEdgeProbe(await page.evaluate(EDGE_BLOCKS_EXPRESSION));
|
|
5695
|
+
if (probe === null) {
|
|
5696
|
+
console.warn("[visual-eval] edge probe unavailable: the page returned no readable blocks");
|
|
5697
|
+
return null;
|
|
5698
|
+
}
|
|
5699
|
+
return judgeEdges(probe.blocks, { placeholdersOnly: probe.placeholdersOnly });
|
|
5700
|
+
} catch (e) {
|
|
5701
|
+
console.warn(`[visual-eval] edge probe unavailable: ${e instanceof Error ? e.message : String(e)}`);
|
|
5702
|
+
return null;
|
|
5703
|
+
}
|
|
5704
|
+
}
|
|
5263
5705
|
async function captureScreenshotDetailed(html, viewport = DEFAULT_SCREENSHOT_VIEWPORT, deps = {}, capture = "full-page", natural = { padPx: JUDGE_INLINE_PAD_PX }) {
|
|
5264
5706
|
try {
|
|
5265
5707
|
const launchOptions = await resolveLaunchOptions(viewport, deps);
|
|
@@ -5281,12 +5723,14 @@ async function captureScreenshotDetailed(html, viewport = DEFAULT_SCREENSHOT_VIE
|
|
|
5281
5723
|
}
|
|
5282
5724
|
const framed = cardHeightPx === null ? viewport.height - 2 * natural.padPx : floorPx !== void 0 ? Math.max(cardHeightPx, floorPx) : cardHeightPx;
|
|
5283
5725
|
const clip = naturalClip(viewport, framed, natural.padPx);
|
|
5726
|
+
const edge2 = await measureEdges(page);
|
|
5284
5727
|
const screenshot2 = await page.screenshot({ type: "png", fullPage: false, clip });
|
|
5285
|
-
return { png: Buffer.from(screenshot2), contentHeight: cardHeightPx, overflowX };
|
|
5728
|
+
return { png: Buffer.from(screenshot2), contentHeight: cardHeightPx, overflowX, edge: edge2 };
|
|
5286
5729
|
}
|
|
5287
5730
|
const contentHeight = await measureContentHeight(page);
|
|
5731
|
+
const edge = await measureEdges(page);
|
|
5288
5732
|
const screenshot = await page.screenshot({ type: "png", fullPage: capture === "full-page" });
|
|
5289
|
-
return { png: Buffer.from(screenshot), contentHeight };
|
|
5733
|
+
return { png: Buffer.from(screenshot), contentHeight, edge };
|
|
5290
5734
|
} finally {
|
|
5291
5735
|
await browser.close();
|
|
5292
5736
|
}
|
|
@@ -5415,6 +5859,43 @@ function overflowXVerdict(canvas, frame) {
|
|
|
5415
5859
|
if (px === void 0 || px === null || px <= OVERFLOW_X_ROUNDING_PX || frame.policy.overflow === "none") return null;
|
|
5416
5860
|
return canvasOverflowXIssue(canvas, frame.viewport, px, frame.policy.overflow);
|
|
5417
5861
|
}
|
|
5862
|
+
function judgeSalvageOf(error, text, finishReason) {
|
|
5863
|
+
const cause = finishReason === void 0 ? "unknown" : finishReason === "length" ? "output-cap" : finishReason === "stop" ? "malformed" : "stopped";
|
|
5864
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
5865
|
+
const position = /position (\d+)/.exec(message);
|
|
5866
|
+
const start = text.indexOf("{");
|
|
5867
|
+
const offset = position !== null ? (start > 0 ? start : 0) + Number(position[1]) : void 0;
|
|
5868
|
+
const parseError = message.startsWith("Visual evaluator did not return valid JSON") ? "no complete JSON object in the answer" : message.replace(/"(?:[^"\\]|\\.)*"/g, '"\u2026"').slice(0, 160);
|
|
5869
|
+
return { cause, ...finishReason !== void 0 ? { finishReason } : {}, parseError, ...offset !== void 0 ? { offset } : {}, chars: text.length };
|
|
5870
|
+
}
|
|
5871
|
+
function judgeRecordOf(k, parsed, samples, criteria, requestedModel) {
|
|
5872
|
+
const salvaged = parsed.flatMap((a) => a.salvage !== void 0 ? [a.salvage.cause] : []);
|
|
5873
|
+
const answered = [...parsed, ...criteria?.kind === "ok" ? [criteria] : []];
|
|
5874
|
+
const served = [...new Set(answered.flatMap((a) => a.servedModel !== void 0 ? [a.servedModel] : []))];
|
|
5875
|
+
const criteriaSalvaged = criteria?.kind === "ok" ? criteria.salvage?.cause : void 0;
|
|
5876
|
+
return {
|
|
5877
|
+
k,
|
|
5878
|
+
rule: "median",
|
|
5879
|
+
samples,
|
|
5880
|
+
sigma: populationSigma(samples),
|
|
5881
|
+
notes: parsed.map((a) => a.result.critique ?? ""),
|
|
5882
|
+
...salvaged.length > 0 ? { salvaged } : {},
|
|
5883
|
+
...criteriaSalvaged !== void 0 ? { criteriaSalvaged } : {},
|
|
5884
|
+
models: { requested: requestedModel, served }
|
|
5885
|
+
};
|
|
5886
|
+
}
|
|
5887
|
+
function callFacts(response) {
|
|
5888
|
+
return {
|
|
5889
|
+
...response.finishReason !== void 0 ? { finishReason: response.finishReason } : {},
|
|
5890
|
+
...response.servedModel !== void 0 ? { servedModel: response.servedModel } : {}
|
|
5891
|
+
};
|
|
5892
|
+
}
|
|
5893
|
+
var NO_CRITERIA_ARRAY = { array: false, received: 0, kept: 0, dropped: { notObject: 0, badId: 0, badVerdict: 0 } };
|
|
5894
|
+
function criteriaCallOf(criteria) {
|
|
5895
|
+
if (criteria?.kind !== "ok") return {};
|
|
5896
|
+
const answer = criteria.result.criteriaCounts ?? NO_CRITERIA_ARRAY;
|
|
5897
|
+
return { call: { answer, ...criteria.finishReason !== void 0 ? { finishReason: criteria.finishReason } : {} } };
|
|
5898
|
+
}
|
|
5418
5899
|
async function judgeFrame(judge, config, model, png, originalPrompt, profileBlock, canvas, k, criteriaBlock) {
|
|
5419
5900
|
const [scoring, criteria] = await Promise.all([
|
|
5420
5901
|
Promise.all(Array.from({ length: k }, () => judgeAndParse(judge, config, model, png, originalPrompt, profileBlock, canvas))),
|
|
@@ -5426,7 +5907,10 @@ function criteriaCallTokens(criteria) {
|
|
|
5426
5907
|
return criteria?.kind === "ok" ? { inputTokens: criteria.inputTokens, outputTokens: criteria.outputTokens } : void 0;
|
|
5427
5908
|
}
|
|
5428
5909
|
function criteriaReads(criteria) {
|
|
5429
|
-
return criteria?.kind === "ok" ? [criteria.result.criteriaAnswers ?? []] : [];
|
|
5910
|
+
return criteria?.kind === "ok" && criteria.salvage === void 0 ? [criteria.result.criteriaAnswers ?? []] : [];
|
|
5911
|
+
}
|
|
5912
|
+
function criteriaUnansweredReason(criteria) {
|
|
5913
|
+
return criteria?.kind === "ok" && criteria.salvage !== void 0 ? { unansweredReason: `not answered: the criteria answer was salvaged (${criteria.salvage.cause})` } : {};
|
|
5430
5914
|
}
|
|
5431
5915
|
function judgeCountFor(config, canvas) {
|
|
5432
5916
|
const k = Math.max(1, Math.floor(config.judgeK ?? 1));
|
|
@@ -5449,12 +5933,16 @@ async function judgeAndParse(judge, config, model, screenshot, originalPrompt, p
|
|
|
5449
5933
|
const response = await judge(config, model, VISUAL_EVAL_PROMPT, screenshot, originalPrompt, profileBlock, criteriaBlock);
|
|
5450
5934
|
try {
|
|
5451
5935
|
const result = parseVisualResponse(response.text, config.passThreshold);
|
|
5452
|
-
return { kind: "ok", result, inputTokens: response.inputTokens, outputTokens: response.outputTokens };
|
|
5936
|
+
return { kind: "ok", result, inputTokens: response.inputTokens, outputTokens: response.outputTokens, ...callFacts(response) };
|
|
5453
5937
|
} catch (e) {
|
|
5454
5938
|
const salvaged = salvageTruncatedVisualAnswer(response.text, config.passThreshold);
|
|
5455
5939
|
if (salvaged !== null) {
|
|
5456
|
-
|
|
5457
|
-
|
|
5940
|
+
const salvage = judgeSalvageOf(e, response.text, response.finishReason);
|
|
5941
|
+
const token = criteriaBlock.length > 0 ? "visual_criteria_salvaged" : "visual_judge_salvaged";
|
|
5942
|
+
console.warn(
|
|
5943
|
+
`[visual-eval] ${token}${canvas ? ` canvas=${canvas}` : ""} cause=${salvage.cause} finish=${salvage.finishReason ?? "unreported"} at=${salvage.offset ?? "?"}/${salvage.chars} (${salvage.parseError}): score recovered from the closed prefix, issues list partial`
|
|
5944
|
+
);
|
|
5945
|
+
return { kind: "ok", result: salvaged, inputTokens: response.inputTokens, outputTokens: response.outputTokens, salvage, ...callFacts(response) };
|
|
5458
5946
|
}
|
|
5459
5947
|
const reason = `judge answer unparsable: ${e instanceof Error ? e.message : String(e)}`;
|
|
5460
5948
|
if (attempt === 1) {
|
|
@@ -5533,17 +6021,11 @@ async function runVisualEvaluationDetailed(context, config, deps = {}) {
|
|
|
5533
6021
|
const result2 = representative.result;
|
|
5534
6022
|
result2.finalScore = median;
|
|
5535
6023
|
result2.passed = median >= config.passThreshold;
|
|
5536
|
-
result2.inputTokens = parsed.reduce((
|
|
5537
|
-
result2.outputTokens = parsed.reduce((
|
|
6024
|
+
result2.inputTokens = parsed.reduce((sum2, a) => sum2 + a.inputTokens, 0);
|
|
6025
|
+
result2.outputTokens = parsed.reduce((sum2, a) => sum2 + a.outputTokens, 0);
|
|
5538
6026
|
const response2 = { inputTokens: result2.inputTokens, outputTokens: result2.outputTokens };
|
|
5539
6027
|
const criteriaTokens = criteriaCallTokens(criteriaAnswer);
|
|
5540
|
-
const judgeRecord =
|
|
5541
|
-
k,
|
|
5542
|
-
rule: "median",
|
|
5543
|
-
samples,
|
|
5544
|
-
sigma: populationSigma(samples),
|
|
5545
|
-
notes: parsed.map((a) => a.result.critique ?? "")
|
|
5546
|
-
};
|
|
6028
|
+
const judgeRecord = judgeRecordOf(k, parsed, samples, criteriaAnswer, model);
|
|
5547
6029
|
const contentHeight = attempt2.contentHeight;
|
|
5548
6030
|
const overflow = contentHeight !== null && contentHeight > viewport.height;
|
|
5549
6031
|
const fitIssue = fitVerdict(canvas, frame);
|
|
@@ -5567,7 +6049,9 @@ async function runVisualEvaluationDetailed(context, config, deps = {}) {
|
|
|
5567
6049
|
context: criteriaContext,
|
|
5568
6050
|
selected: criteriaSelected,
|
|
5569
6051
|
answers: criteriaReads(criteriaAnswer),
|
|
5570
|
-
measurements: { overflow, contentHeight, viewportHeight: viewport.height, inkRatio }
|
|
6052
|
+
measurements: { overflow, contentHeight, viewportHeight: viewport.height, inkRatio, edge: attempt2.edge ?? null },
|
|
6053
|
+
...criteriaUnansweredReason(criteriaAnswer),
|
|
6054
|
+
...criteriaCallOf(criteriaAnswer)
|
|
5571
6055
|
}) : void 0;
|
|
5572
6056
|
perCanvasResults.push(result2);
|
|
5573
6057
|
perCanvas.push({
|
|
@@ -5629,7 +6113,7 @@ function canvasScopedIssue(canvas, issue) {
|
|
|
5629
6113
|
}
|
|
5630
6114
|
function aggregateCanvasResults(perCanvas, results) {
|
|
5631
6115
|
const n = results.length;
|
|
5632
|
-
const mean = (pick) => Math.round(results.reduce((
|
|
6116
|
+
const mean = (pick) => Math.round(results.reduce((sum2, r) => sum2 + pick(r), 0) / n);
|
|
5633
6117
|
const dimensions = {
|
|
5634
6118
|
completeness: mean((r) => r.dimensions.completeness),
|
|
5635
6119
|
visualPolish: mean((r) => r.dimensions.visualPolish),
|
|
@@ -5649,7 +6133,18 @@ function aggregateCanvasResults(perCanvas, results) {
|
|
|
5649
6133
|
outputTokens += r.outputTokens ?? 0;
|
|
5650
6134
|
});
|
|
5651
6135
|
const criteriaSpent = perCanvas.flatMap((c) => c.criteriaTokens !== void 0 ? [c.criteriaTokens] : []);
|
|
6136
|
+
const judgeSalvage = {
|
|
6137
|
+
scoring: {
|
|
6138
|
+
salvaged: perCanvas.reduce((sum2, c) => sum2 + (c.judge.salvaged?.length ?? 0), 0),
|
|
6139
|
+
answers: perCanvas.reduce((sum2, c) => sum2 + c.judge.samples.length, 0)
|
|
6140
|
+
},
|
|
6141
|
+
criteria: {
|
|
6142
|
+
salvaged: perCanvas.filter((c) => c.judge.criteriaSalvaged !== void 0).length,
|
|
6143
|
+
answered: perCanvas.filter((c) => c.criteriaTokens !== void 0).length
|
|
6144
|
+
}
|
|
6145
|
+
};
|
|
5652
6146
|
return {
|
|
6147
|
+
judgeSalvage,
|
|
5653
6148
|
passed: perCanvas.every((c) => c.passed),
|
|
5654
6149
|
finalScore: mean((r) => r.finalScore),
|
|
5655
6150
|
dimensions,
|
|
@@ -5659,8 +6154,8 @@ function aggregateCanvasResults(perCanvas, results) {
|
|
|
5659
6154
|
outputTokens,
|
|
5660
6155
|
...criteriaSpent.length > 0 ? {
|
|
5661
6156
|
criteriaTokens: {
|
|
5662
|
-
inputTokens: criteriaSpent.reduce((
|
|
5663
|
-
outputTokens: criteriaSpent.reduce((
|
|
6157
|
+
inputTokens: criteriaSpent.reduce((sum2, t) => sum2 + t.inputTokens, 0),
|
|
6158
|
+
outputTokens: criteriaSpent.reduce((sum2, t) => sum2 + t.outputTokens, 0)
|
|
5664
6159
|
}
|
|
5665
6160
|
} : {},
|
|
5666
6161
|
canvases: [...perCanvas]
|
|
@@ -5746,7 +6241,9 @@ async function judgeStoredCapture(input, config, deps = {}) {
|
|
|
5746
6241
|
context: criteriaContext,
|
|
5747
6242
|
selected: criteriaSelected,
|
|
5748
6243
|
answers: criteriaReads(criteriaAnswer),
|
|
5749
|
-
measurements: { overflow, contentHeight, viewportHeight: viewport.height, inkRatio }
|
|
6244
|
+
measurements: { overflow, contentHeight, viewportHeight: viewport.height, inkRatio },
|
|
6245
|
+
...criteriaUnansweredReason(criteriaAnswer),
|
|
6246
|
+
...criteriaCallOf(criteriaAnswer)
|
|
5750
6247
|
}) : void 0;
|
|
5751
6248
|
const criteriaTokens = criteriaCallTokens(criteriaAnswer);
|
|
5752
6249
|
return {
|
|
@@ -5757,13 +6254,13 @@ async function judgeStoredCapture(input, config, deps = {}) {
|
|
|
5757
6254
|
score: median,
|
|
5758
6255
|
passed,
|
|
5759
6256
|
issues,
|
|
5760
|
-
judge:
|
|
6257
|
+
judge: judgeRecordOf(k, parsed, samples, criteriaAnswer, model),
|
|
5761
6258
|
contentHeight,
|
|
5762
6259
|
overflow,
|
|
5763
6260
|
inkRatio,
|
|
5764
6261
|
...criteria !== void 0 ? { criteria } : {},
|
|
5765
|
-
inputTokens: parsed.reduce((
|
|
5766
|
-
outputTokens: parsed.reduce((
|
|
6262
|
+
inputTokens: parsed.reduce((sum2, a) => sum2 + a.inputTokens, 0),
|
|
6263
|
+
outputTokens: parsed.reduce((sum2, a) => sum2 + a.outputTokens, 0),
|
|
5767
6264
|
...criteriaTokens !== void 0 ? { criteriaTokens } : {}
|
|
5768
6265
|
}
|
|
5769
6266
|
};
|
|
@@ -5802,7 +6299,7 @@ function salvageTruncatedVisualAnswer(text, passThreshold) {
|
|
|
5802
6299
|
hierarchy,
|
|
5803
6300
|
aesthetics,
|
|
5804
6301
|
issues,
|
|
5805
|
-
critique: `[
|
|
6302
|
+
critique: `[salvaged judge answer \u2014 it did not parse; score recovered from the closed prefix; ${issues.length} issue(s) kept, the rest and the critique lost]`
|
|
5806
6303
|
});
|
|
5807
6304
|
return parseVisualResponse(rebuilt, passThreshold);
|
|
5808
6305
|
}
|
|
@@ -5831,31 +6328,43 @@ function parseVisualResponse(text, passThreshold) {
|
|
|
5831
6328
|
};
|
|
5832
6329
|
const issues = (raw.issues || []).map((i) => ({
|
|
5833
6330
|
dimension: i.dimension || "visual",
|
|
6331
|
+
origin: "judge",
|
|
5834
6332
|
severity: i.severity || "minor",
|
|
5835
6333
|
description: i.description || "",
|
|
5836
6334
|
fix: i.fix || ""
|
|
5837
6335
|
}));
|
|
5838
|
-
const
|
|
6336
|
+
const criteria = parseCriteriaAnswers(raw.criteria);
|
|
5839
6337
|
return {
|
|
5840
6338
|
passed: finalScore >= passThreshold,
|
|
5841
6339
|
finalScore,
|
|
5842
6340
|
dimensions,
|
|
5843
6341
|
issues,
|
|
5844
6342
|
critique: raw.critique,
|
|
5845
|
-
...
|
|
6343
|
+
...criteria.answers !== void 0 ? { criteriaAnswers: criteria.answers } : {},
|
|
6344
|
+
criteriaCounts: criteria.counts
|
|
5846
6345
|
};
|
|
5847
6346
|
}
|
|
5848
6347
|
function parseCriteriaAnswers(raw) {
|
|
5849
|
-
if (!Array.isArray(raw)) return
|
|
5850
|
-
const
|
|
6348
|
+
if (!Array.isArray(raw)) return { counts: NO_CRITERIA_ARRAY };
|
|
6349
|
+
const answers = [];
|
|
6350
|
+
const dropped = { notObject: 0, badId: 0, badVerdict: 0 };
|
|
5851
6351
|
for (const entry of raw) {
|
|
5852
|
-
if (typeof entry !== "object" || entry === null)
|
|
6352
|
+
if (typeof entry !== "object" || entry === null) {
|
|
6353
|
+
dropped.notObject += 1;
|
|
6354
|
+
continue;
|
|
6355
|
+
}
|
|
5853
6356
|
const o = entry;
|
|
5854
|
-
if (typeof o.id !== "string")
|
|
5855
|
-
|
|
5856
|
-
|
|
6357
|
+
if (typeof o.id !== "string") {
|
|
6358
|
+
dropped.badId += 1;
|
|
6359
|
+
continue;
|
|
6360
|
+
}
|
|
6361
|
+
if (o.verdict !== "pass" && o.verdict !== "fail" && o.verdict !== "n/a") {
|
|
6362
|
+
dropped.badVerdict += 1;
|
|
6363
|
+
continue;
|
|
6364
|
+
}
|
|
6365
|
+
answers.push({ id: o.id, verdict: o.verdict, evidence: typeof o.evidence === "string" ? o.evidence : "" });
|
|
5857
6366
|
}
|
|
5858
|
-
return
|
|
6367
|
+
return { answers, counts: { array: true, received: raw.length, kept: answers.length, dropped } };
|
|
5859
6368
|
}
|
|
5860
6369
|
function getDefaultVisualModel(provider) {
|
|
5861
6370
|
switch (provider) {
|
|
@@ -5875,7 +6384,12 @@ async function runVisualEval(context, config, deps = {}) {
|
|
|
5875
6384
|
const issues = (result.issues || []).map(toEvalIssue);
|
|
5876
6385
|
const summary = summarizeVisualResult(result);
|
|
5877
6386
|
const coverage = { status: "ran" };
|
|
5878
|
-
|
|
6387
|
+
const tokens = result.inputTokens !== void 0 && result.outputTokens !== void 0 ? {
|
|
6388
|
+
inputTokens: result.inputTokens,
|
|
6389
|
+
outputTokens: result.outputTokens,
|
|
6390
|
+
...result.criteriaTokens !== void 0 ? { criteria: result.criteriaTokens } : {}
|
|
6391
|
+
} : void 0;
|
|
6392
|
+
return { issues, ...summary !== void 0 ? { summary } : {}, coverage, ...tokens !== void 0 ? { tokens } : {} };
|
|
5879
6393
|
}
|
|
5880
6394
|
function toEvalIssue(issue) {
|
|
5881
6395
|
return {
|
|
@@ -5885,7 +6399,8 @@ function toEvalIssue(issue) {
|
|
|
5885
6399
|
subcategory: issue.dimension,
|
|
5886
6400
|
severity: issue.severity === "critical" ? "critical" : "major",
|
|
5887
6401
|
description: issue.description,
|
|
5888
|
-
fix: issue.fix || ""
|
|
6402
|
+
fix: issue.fix || "",
|
|
6403
|
+
...issue.origin !== void 0 ? { origin: issue.origin } : {}
|
|
5889
6404
|
};
|
|
5890
6405
|
}
|
|
5891
6406
|
function cssTokensForAppTheme(theme, mode = "light", themeId) {
|
|
@@ -5925,6 +6440,14 @@ function criteriaContextFor(src) {
|
|
|
5925
6440
|
};
|
|
5926
6441
|
}
|
|
5927
6442
|
|
|
5928
|
-
|
|
6443
|
+
// src/evaluation/judge-spend.ts
|
|
6444
|
+
function judgedFrameTokens(result) {
|
|
6445
|
+
return {
|
|
6446
|
+
inputTokens: (result.inputTokens ?? 0) + (result.criteriaTokens?.inputTokens ?? 0),
|
|
6447
|
+
outputTokens: (result.outputTokens ?? 0) + (result.criteriaTokens?.outputTokens ?? 0)
|
|
6448
|
+
};
|
|
6449
|
+
}
|
|
6450
|
+
|
|
6451
|
+
export { CANVAS_BREAKPOINTS, CANVAS_CLASSES, CANVAS_DESCRIPTORS, CANVAS_VIEWPORTS, CARD_HEIGHT_EXPRESSION, CONTENT_HEIGHT_EXPRESSION, CRITERIA, CRITERIA_SELECTOR_VERSION, DEFAULT_EVAL_MAX_ROUNDS, DEFAULT_QUALITY_CONFIG, EDGE_BLOCKS_EXPRESSION, EDGE_TOLERANCE_PX, HOST_COLOUR_PATTERN, HOST_FRAME_BOUNDS, INK_TOLERANCE, INSTRUMENT_BESIDE_JUDGE, INSTRUMENT_BY_ID, JUDGE_GROUND_MARGIN_PX, JUDGE_INK_INSET_PX, LLM_EVAL_STATIC_CRITERIA, MAX_EVAL_ROUNDS_HARD_LIMIT, VISUAL_EVAL_PROMPT, VISUAL_JUDGE_MAX_OUTPUT_TOKENS, VISUAL_JUDGE_PROMPT_DIGEST, VISUAL_JUDGE_PROMPT_VERSION, bankRows, buildCodingCriteriaSummary, buildCriteriaJudgeBlock, buildFixPrompt, callMultimodalLLM, canonicalCriteriaContext, canvasBlankIssue, canvasDisplayModeForShell, canvasFitPolicy, canvasForRendering, canvasForViewportWidth, canvasOverflowIssue, captureScreenshot, captureScreenshotDetailed, chromaOfCssTokens, computeEvaluationScore, createEvaluationToolsServer, criteriaBankSchema, criteriaContextFor, criteriaFor, criteriaSetIdFor, cssTokensForAppTheme, decodePng, describeCanvas, designTreeSha256, extractCompiledCode, extractCompiledCodeFromMessage, extractEvalResult, extractSourceCode, extractSourceCodeFromMessage, extractToolResultTexts, getActionableIssues, getCriteriaByPriority, getCriterionById, getEvaluatorSystemPrompt, getLLMCriteria, isBlocked, judgeDesignIdentity, judgeEdges, judgeStoredCapture, judgedFrameTokens, matches, notApplicableCoverage, parseCriteriaBank, parseEdgeProbe, priorityForIssue, readInkExtent, resetJudgeDesignIdentityCache, resolveCriteriaBlock, resolveDesignPackageDir, resolveLaunchOptions, runAxisChecks, runEvaluation, runEvaluationLoop, runVisualEval, runVisualEvaluation, runVisualEvaluationDetailed, selectCriteria, summarizeVisualResult };
|
|
5929
6452
|
//# sourceMappingURL=index.js.map
|
|
5930
6453
|
//# sourceMappingURL=index.js.map
|