@polyxd/verifier 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +177 -0
- package/README.md +29 -0
- package/dist/agent.d.ts +47 -0
- package/dist/agent.js +82 -0
- package/dist/agent.js.map +1 -0
- package/dist/browser.d.ts +22 -0
- package/dist/browser.js +41 -0
- package/dist/browser.js.map +1 -0
- package/dist/checkset.d.ts +7 -0
- package/dist/checkset.js +25 -0
- package/dist/checkset.js.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +74 -0
- package/dist/cli.js.map +1 -0
- package/dist/consistency.d.ts +26 -0
- package/dist/consistency.js +71 -0
- package/dist/consistency.js.map +1 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/rendered.d.ts +16 -0
- package/dist/rendered.js +85 -0
- package/dist/rendered.js.map +1 -0
- package/dist/reward.d.ts +68 -0
- package/dist/reward.js +188 -0
- package/dist/reward.js.map +1 -0
- package/dist/static.d.ts +15 -0
- package/dist/static.js +235 -0
- package/dist/static.js.map +1 -0
- package/dist/verify.d.ts +43 -0
- package/dist/verify.js +81 -0
- package/dist/verify.js.map +1 -0
- package/harness-dist/assets/index-C1BzlRVN.js +49 -0
- package/harness-dist/assets/index-D6-rG5Hn.css +1 -0
- package/harness-dist/index.html +14 -0
- package/package.json +54 -0
package/dist/cli.d.ts
ADDED
package/dist/cli.js
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* polyxd-verify <ui-document.json ...> [--themes material3,carbon] [--modes light,dark] [--widths 390,1100]
|
|
4
|
+
* [--registry capabilities.json] [--tasks tasks.json] [--json report.json]
|
|
5
|
+
*/
|
|
6
|
+
import { readFile, writeFile } from "node:fs/promises";
|
|
7
|
+
import { basename } from "node:path";
|
|
8
|
+
import { parseArgs } from "node:util";
|
|
9
|
+
import { DEFAULTS, launch, verifyDocument } from "./index.js";
|
|
10
|
+
const { values, positionals } = parseArgs({
|
|
11
|
+
allowPositionals: true,
|
|
12
|
+
options: {
|
|
13
|
+
themes: { type: "string" },
|
|
14
|
+
modes: { type: "string" },
|
|
15
|
+
widths: { type: "string" },
|
|
16
|
+
registry: { type: "string" },
|
|
17
|
+
tasks: { type: "string" },
|
|
18
|
+
json: { type: "string" },
|
|
19
|
+
quiet: { type: "boolean", short: "q" },
|
|
20
|
+
},
|
|
21
|
+
});
|
|
22
|
+
if (!positionals.length) {
|
|
23
|
+
console.error("Usage: polyxd-verify <ui-document.json ...> [--themes a,b] [--modes light,dark] [--widths 390,1100] [--registry file] [--tasks file] [--json out]");
|
|
24
|
+
process.exit(2);
|
|
25
|
+
}
|
|
26
|
+
const list = (v) => v?.split(",").map((s) => s.trim()).filter(Boolean);
|
|
27
|
+
const readJson = async (p) => JSON.parse(await readFile(p, "utf8"));
|
|
28
|
+
const registry = values.registry ? await readJson(values.registry) : undefined;
|
|
29
|
+
const allTasks = values.tasks ? (await readJson(values.tasks)).tasks : [];
|
|
30
|
+
const options = {
|
|
31
|
+
themes: list(values.themes) ?? DEFAULTS.themes,
|
|
32
|
+
modes: list(values.modes) ?? DEFAULTS.modes,
|
|
33
|
+
widths: list(values.widths)?.map(Number) ?? DEFAULTS.widths,
|
|
34
|
+
};
|
|
35
|
+
const browser = await launch();
|
|
36
|
+
const reports = [];
|
|
37
|
+
try {
|
|
38
|
+
for (const file of positionals) {
|
|
39
|
+
const name = basename(file, ".json");
|
|
40
|
+
const report = await verifyDocument(await readJson(file), { ...options, browser, registry, tasks: allTasks.filter((t) => t.document === name) });
|
|
41
|
+
reports.push({ file, ...report });
|
|
42
|
+
const agent = report.agentRuns ? ` agent ${report.agentSuccess}/${report.agentRuns}` : "";
|
|
43
|
+
console.log(`${String(report.score).padStart(3)} ${name} (${report.errors} errors, ${report.warnings} warnings${agent})`);
|
|
44
|
+
if (!values.quiet) {
|
|
45
|
+
const seen = new Set();
|
|
46
|
+
const lines = [
|
|
47
|
+
...report.static.map((f) => ({ f, where: "document" })),
|
|
48
|
+
...report.targets.flatMap((t) => [
|
|
49
|
+
...t.findings.map((f) => ({ f, where: `${t.theme}/${t.mode}/${t.width}` })),
|
|
50
|
+
...t.agent.filter((a) => !a.success).map((a) => ({ f: { severity: "error", check: `agent:${a.task}`, message: a.error ?? "failed" }, where: `${t.theme}/${t.mode}/${t.width}` })),
|
|
51
|
+
]),
|
|
52
|
+
];
|
|
53
|
+
for (const { f, where } of lines) {
|
|
54
|
+
const key = `${f.check}|${f.message}`;
|
|
55
|
+
if (seen.has(key))
|
|
56
|
+
continue;
|
|
57
|
+
seen.add(key);
|
|
58
|
+
console.log(` ${f.severity === "error" ? "error" : "warn "} ${f.check} ${f.message} [${where}]`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
finally {
|
|
64
|
+
await browser.close();
|
|
65
|
+
}
|
|
66
|
+
const mean = reports.reduce((s, r) => s + r.score, 0) / reports.length;
|
|
67
|
+
const runs = reports.reduce((s, r) => s + r.agentRuns, 0);
|
|
68
|
+
const ok = reports.reduce((s, r) => s + r.agentSuccess, 0);
|
|
69
|
+
const renders = reports.reduce((s, r) => s + r.targets.length, 0);
|
|
70
|
+
console.log(`\n${reports.length} documents · ${renders} renders · mean score ${mean.toFixed(1)}${runs ? ` · agent tasks ${ok}/${runs}` : ""}`);
|
|
71
|
+
if (values.json)
|
|
72
|
+
await writeFile(values.json, JSON.stringify({ options, reports }, null, 2));
|
|
73
|
+
process.exit(reports.some((r) => r.errors > 0 || r.agentSuccess < r.agentRuns) ? 1 : 0);
|
|
74
|
+
//# sourceMappingURL=cli.js.map
|
package/dist/cli.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA;;;GAGG;AACH,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAe,MAAM,YAAY,CAAC;AAE3E,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;IACxC,gBAAgB,EAAE,IAAI;IACtB,OAAO,EAAE;QACP,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC1B,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACzB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC1B,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QAC5B,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACzB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;QACxB,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;KACvC;CACF,CAAC,CAAC;AAEH,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;IACxB,OAAO,CAAC,KAAK,CAAC,mJAAmJ,CAAC,CAAC;IACnK,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,MAAM,IAAI,GAAG,CAAC,CAAqB,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAC3F,MAAM,QAAQ,GAAG,KAAK,EAAE,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;AAC5E,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC/E,MAAM,QAAQ,GAAU,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;AACjF,MAAM,OAAO,GAAG;IACd,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM;IAC9C,KAAK,EAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAA0B,IAAI,QAAQ,CAAC,KAAK;IACrE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM;CAC5D,CAAC;AAEF,MAAM,OAAO,GAAG,MAAM,MAAM,EAAE,CAAC;AAC/B,MAAM,OAAO,GAAkC,EAAE,CAAC;AAClD,IAAI,CAAC;IACH,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,EAAE,CAAC,CAAC;QACjJ,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;QAClC,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1F,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,IAAI,MAAM,MAAM,CAAC,MAAM,YAAY,MAAM,CAAC,QAAQ,YAAY,KAAK,GAAG,CAAC,CAAC;QAC5H,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAClB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,MAAM,KAAK,GAAG;gBACZ,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC;gBACvD,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;oBAC/B,GAAG,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBAC3E,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,KAAK,IAAI,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;iBAClL,CAAC;aACH,CAAC;YACF,KAAK,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,KAAK,EAAE,CAAC;gBACjC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;gBACtC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,OAAO,MAAM,KAAK,GAAG,CAAC,CAAC;YAC3G,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;QAAS,CAAC;IACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;AACvE,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;AAC1D,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;AAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;AAClE,OAAO,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,MAAM,gBAAgB,OAAO,yBAAyB,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,kBAAkB,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/I,IAAI,MAAM,CAAC,IAAI;IAAE,MAAM,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AAC7F,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Consistency between two generations of the same intent: does the thing the user saw last time
|
|
3
|
+
* look and sit the same way now? Components are matched by their semantic `key`, which the
|
|
4
|
+
* generator keeps stable and interface memory records (see Maru, docs/decisions/0001).
|
|
5
|
+
*/
|
|
6
|
+
export interface Consistency {
|
|
7
|
+
/** 0 (nothing recognisable) to 1 (same structure, order and labels) */
|
|
8
|
+
score: number;
|
|
9
|
+
parts: {
|
|
10
|
+
pattern: number;
|
|
11
|
+
coverage: number;
|
|
12
|
+
components: number;
|
|
13
|
+
order: number;
|
|
14
|
+
labels: number;
|
|
15
|
+
};
|
|
16
|
+
differences: string[];
|
|
17
|
+
}
|
|
18
|
+
interface Keyed {
|
|
19
|
+
component: string;
|
|
20
|
+
label?: string;
|
|
21
|
+
index: number;
|
|
22
|
+
}
|
|
23
|
+
/** Keyed things in reading order: components with a key, plus keyed items inside them (detail rows, columns, steps…). */
|
|
24
|
+
export declare function signature(doc: any): Map<string, Keyed>;
|
|
25
|
+
export declare function compare(previous: any, current: any): Consistency;
|
|
26
|
+
export {};
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { readingOrder } from "@polyxd/spec/checks";
|
|
2
|
+
const WEIGHTS = { pattern: 0.15, coverage: 0.15, components: 0.3, order: 0.2, labels: 0.2 };
|
|
3
|
+
/** Keyed things in reading order: components with a key, plus keyed items inside them (detail rows, columns, steps…). */
|
|
4
|
+
export function signature(doc) {
|
|
5
|
+
const out = new Map();
|
|
6
|
+
let index = 0;
|
|
7
|
+
const add = (key, component, label) => {
|
|
8
|
+
if (!key || out.has(key))
|
|
9
|
+
return;
|
|
10
|
+
out.set(key, { component, label: typeof label === "string" ? label : undefined, index: index++ });
|
|
11
|
+
};
|
|
12
|
+
for (const c of readingOrder(doc)) {
|
|
13
|
+
add(c.key, c.component, c.label ?? c.title ?? c.summary);
|
|
14
|
+
for (const list of [c.items, c.columns, c.attributes, c.steps, c.views, c.series]) {
|
|
15
|
+
if (Array.isArray(list))
|
|
16
|
+
for (const item of list)
|
|
17
|
+
add(item?.key, `${c.component}.item`, item?.label ?? item?.title);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
return out;
|
|
21
|
+
}
|
|
22
|
+
/** Fraction of ordered pairs that keep their relative order (1 = identical order). */
|
|
23
|
+
function orderAgreement(a, b) {
|
|
24
|
+
const pos = new Map(b.map((k, i) => [k, i]));
|
|
25
|
+
let agree = 0;
|
|
26
|
+
let total = 0;
|
|
27
|
+
for (let i = 0; i < a.length; i++) {
|
|
28
|
+
for (let j = i + 1; j < a.length; j++) {
|
|
29
|
+
total++;
|
|
30
|
+
if (pos.get(a[i]) < pos.get(a[j]))
|
|
31
|
+
agree++;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return total ? agree / total : 1;
|
|
35
|
+
}
|
|
36
|
+
export function compare(previous, current) {
|
|
37
|
+
const a = signature(previous);
|
|
38
|
+
const b = signature(current);
|
|
39
|
+
const shared = [...a.keys()].filter((k) => b.has(k));
|
|
40
|
+
const union = new Set([...a.keys(), ...b.keys()]);
|
|
41
|
+
const differences = [];
|
|
42
|
+
const pattern = previous.surface?.pattern === current.surface?.pattern ? 1 : 0;
|
|
43
|
+
if (!pattern)
|
|
44
|
+
differences.push(`pattern changed: ${previous.surface?.pattern ?? "none"} → ${current.surface?.pattern ?? "none"}`);
|
|
45
|
+
const coverage = union.size ? shared.length / union.size : 1;
|
|
46
|
+
for (const k of a.keys())
|
|
47
|
+
if (!b.has(k))
|
|
48
|
+
differences.push(`"${k}" disappeared`);
|
|
49
|
+
for (const k of b.keys())
|
|
50
|
+
if (!a.has(k))
|
|
51
|
+
differences.push(`"${k}" is new`);
|
|
52
|
+
const sameComponent = shared.filter((k) => a.get(k).component === b.get(k).component);
|
|
53
|
+
for (const k of shared)
|
|
54
|
+
if (a.get(k).component !== b.get(k).component)
|
|
55
|
+
differences.push(`"${k}" changed from ${a.get(k).component} to ${b.get(k).component}`);
|
|
56
|
+
const components = shared.length ? sameComponent.length / shared.length : 1;
|
|
57
|
+
const orderA = shared.sort((x, y) => a.get(x).index - a.get(y).index);
|
|
58
|
+
const order = orderAgreement(orderA, [...orderA].sort((x, y) => b.get(x).index - b.get(y).index));
|
|
59
|
+
if (order < 1)
|
|
60
|
+
differences.push(`order changed (${Math.round(order * 100)}% of pairs kept their order)`);
|
|
61
|
+
const labelled = shared.filter((k) => a.get(k).label !== undefined || b.get(k).label !== undefined);
|
|
62
|
+
const sameLabel = labelled.filter((k) => a.get(k).label === b.get(k).label);
|
|
63
|
+
for (const k of labelled)
|
|
64
|
+
if (a.get(k).label !== b.get(k).label)
|
|
65
|
+
differences.push(`"${k}" relabelled: "${a.get(k).label}" → "${b.get(k).label}"`);
|
|
66
|
+
const labels = labelled.length ? sameLabel.length / labelled.length : 1;
|
|
67
|
+
const parts = { pattern, coverage, components, order, labels };
|
|
68
|
+
const score = Object.entries(WEIGHTS).reduce((s, [k, w]) => s + w * parts[k], 0);
|
|
69
|
+
return { score: Math.round(score * 1000) / 1000, parts, differences };
|
|
70
|
+
}
|
|
71
|
+
//# sourceMappingURL=consistency.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"consistency.js","sourceRoot":"","sources":["../src/consistency.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAoBnD,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;AAE5F,yHAAyH;AACzH,MAAM,UAAU,SAAS,CAAC,GAAQ;IAChC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAiB,CAAC;IACrC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,GAAG,GAAG,CAAC,GAAuB,EAAE,SAAiB,EAAE,KAAc,EAAE,EAAE;QACzE,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO;QACjC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;IACpG,CAAC,CAAC;IACF,KAAK,MAAM,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC;QACzD,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YAClF,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;gBAAE,KAAK,MAAM,IAAI,IAAI,IAAI;oBAAE,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,SAAS,OAAO,EAAE,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,CAAC,CAAC;QACtH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,sFAAsF;AACtF,SAAS,cAAc,CAAC,CAAW,EAAE,CAAW;IAC9C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,KAAK,EAAE,CAAC;YACR,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE;gBAAE,KAAK,EAAE,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,QAAa,EAAE,OAAY;IACjD,MAAM,CAAC,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,CAAC,GAAG,SAAS,CAAC,OAAO,CAAC,CAAC;IAC7B,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClD,MAAM,WAAW,GAAa,EAAE,CAAC;IAEjC,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,EAAE,OAAO,KAAK,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,IAAI,CAAC,OAAO;QAAE,WAAW,CAAC,IAAI,CAAC,oBAAoB,QAAQ,CAAC,OAAO,EAAE,OAAO,IAAI,MAAM,MAAM,OAAO,CAAC,OAAO,EAAE,OAAO,IAAI,MAAM,EAAE,CAAC,CAAC;IAElI,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;QAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAChF,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE;QAAE,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAE3E,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,SAAS,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,SAAS,CAAC,CAAC;IACxF,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,SAAS,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,SAAS;YAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,SAAS,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,SAAS,EAAE,CAAC,CAAC;IAClK,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5E,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC;IACxE,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,CAAC;IACpG,IAAI,KAAK,GAAG,CAAC;QAAE,WAAW,CAAC,IAAI,CAAC,kBAAkB,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC,8BAA8B,CAAC,CAAC;IAEzG,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IACtG,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC;IAC9E,KAAK,MAAM,CAAC,IAAI,QAAQ;QAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,KAAK;YAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,KAAK,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,KAAK,GAAG,CAAC,CAAC;IACtJ,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAExE,MAAM,KAAK,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAC/D,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAuB,CAAC,EAAE,CAAC,CAAC,CAAC;IACvG,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;AACxE,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { verifyDocument, score, DEFAULTS, type Report, type TargetReport, type VerifyOptions } from "./verify.ts";
|
|
2
|
+
export { staticAudit, type StaticOptions } from "./static.ts";
|
|
3
|
+
export { axeAudit, layoutAudit, type Finding } from "./rendered.ts";
|
|
4
|
+
export { runTask, type Task, type Step, type AgentResult } from "./agent.ts";
|
|
5
|
+
export { compare, signature, type Consistency } from "./consistency.ts";
|
|
6
|
+
export { launch, renderPage, type RenderTarget } from "./browser.ts";
|
|
7
|
+
export { rewardFor, wiredCapabilities, probeActionable, coverageOf, type Reward, type RequestExpectation } from "./reward.ts";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { verifyDocument, score, DEFAULTS } from "./verify.js";
|
|
2
|
+
export { staticAudit } from "./static.js";
|
|
3
|
+
export { axeAudit, layoutAudit } from "./rendered.js";
|
|
4
|
+
export { runTask } from "./agent.js";
|
|
5
|
+
export { compare, signature } from "./consistency.js";
|
|
6
|
+
export { launch, renderPage } from "./browser.js";
|
|
7
|
+
export { rewardFor, wiredCapabilities, probeActionable, coverageOf } from "./reward.js";
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,QAAQ,EAAsD,MAAM,aAAa,CAAC;AAClH,OAAO,EAAE,WAAW,EAAsB,MAAM,aAAa,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAgB,MAAM,eAAe,CAAC;AACpE,OAAO,EAAE,OAAO,EAA0C,MAAM,YAAY,CAAC;AAC7E,OAAO,EAAE,OAAO,EAAE,SAAS,EAAoB,MAAM,kBAAkB,CAAC;AACxE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAqB,MAAM,cAAc,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,eAAe,EAAE,UAAU,EAAwC,MAAM,aAAa,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Page } from "playwright";
|
|
2
|
+
export interface Finding {
|
|
3
|
+
severity: "error" | "warning";
|
|
4
|
+
check: string;
|
|
5
|
+
message: string;
|
|
6
|
+
/** How many elements are affected */
|
|
7
|
+
count?: number;
|
|
8
|
+
}
|
|
9
|
+
/** axe-core WCAG 2.2 AA audit of the surface (including dialogs, which portal inside it). */
|
|
10
|
+
export declare function axeAudit(page: Page): Promise<Finding[]>;
|
|
11
|
+
/**
|
|
12
|
+
* Layout checks axe doesn't cover:
|
|
13
|
+
* - no horizontal overflow at this width
|
|
14
|
+
* - interactive targets meet WCAG 2.2 2.5.8 (24px, or 24px of clear spacing) and the pack's own minimum
|
|
15
|
+
*/
|
|
16
|
+
export declare function layoutAudit(page: Page): Promise<Finding[]>;
|
package/dist/rendered.js
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
const WCAG_TAGS = ["wcag2a", "wcag2aa", "wcag21a", "wcag21aa", "wcag22aa"];
|
|
2
|
+
/** axe-core WCAG 2.2 AA audit of the surface (including dialogs, which portal inside it). */
|
|
3
|
+
export async function axeAudit(page) {
|
|
4
|
+
const violations = await page.evaluate(async (tags) => {
|
|
5
|
+
const axe = window.axe;
|
|
6
|
+
const result = await axe.run(document.querySelector(".pxd-surface"), { runOnly: { type: "tag", values: tags }, resultTypes: ["violations"] });
|
|
7
|
+
return result.violations.map((v) => ({ id: v.id, impact: v.impact, help: v.help, count: v.nodes.length, sample: v.nodes[0]?.target?.join(" ") }));
|
|
8
|
+
}, WCAG_TAGS);
|
|
9
|
+
return violations.map((v) => ({
|
|
10
|
+
severity: v.impact === "critical" || v.impact === "serious" ? "error" : "warning",
|
|
11
|
+
check: `axe:${v.id}`,
|
|
12
|
+
message: `${v.help} (${v.count}× e.g. ${v.sample})`,
|
|
13
|
+
count: v.count,
|
|
14
|
+
}));
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Layout checks axe doesn't cover:
|
|
18
|
+
* - no horizontal overflow at this width
|
|
19
|
+
* - interactive targets meet WCAG 2.2 2.5.8 (24px, or 24px of clear spacing) and the pack's own minimum
|
|
20
|
+
*/
|
|
21
|
+
export async function layoutAudit(page) {
|
|
22
|
+
const r = await page.evaluate(() => {
|
|
23
|
+
const surface = document.querySelector(".pxd-surface");
|
|
24
|
+
const overflow = surface.scrollWidth - surface.clientWidth;
|
|
25
|
+
const packMin = parseFloat(getComputedStyle(surface).getPropertyValue("--pxd-size-target-min")) || 24;
|
|
26
|
+
const selector = "button, input:not([type=hidden]), textarea, select, a[href], [role=radio], [role=checkbox], [role=switch], [role=tab], [role=slider]";
|
|
27
|
+
const els = [...surface.querySelectorAll(selector)].filter((el) => {
|
|
28
|
+
const rect = el.getBoundingClientRect();
|
|
29
|
+
const style = getComputedStyle(el);
|
|
30
|
+
return rect.width > 0 && rect.height > 0 && style.visibility !== "hidden" && !el.closest("[aria-hidden=true]");
|
|
31
|
+
});
|
|
32
|
+
// Radix renders a visually hidden native input next to its radios/checkboxes; skip those.
|
|
33
|
+
const targets = els.filter((el) => !(el instanceof HTMLInputElement && el.getAttribute("aria-hidden") === "true"));
|
|
34
|
+
// A stretched link (e.g. an actionable card) is as big as the element it covers.
|
|
35
|
+
const rects = targets.map((el) => (el.closest(".pxd-card-actionable") && el.classList.contains("pxd-card-link") ? el.closest(".pxd-card-actionable") : el).getBoundingClientRect());
|
|
36
|
+
const name = (el) => el.getAttribute("aria-label") || el.textContent?.trim().slice(0, 30) || el.id || el.tagName.toLowerCase();
|
|
37
|
+
const tooSmall = [];
|
|
38
|
+
const belowPack = [];
|
|
39
|
+
targets.forEach((el, i) => {
|
|
40
|
+
const { width, height } = rects[i];
|
|
41
|
+
if (el.classList.contains("pxd-card-link"))
|
|
42
|
+
return;
|
|
43
|
+
const small = Math.min(width, height);
|
|
44
|
+
// An element whose visible label is also clickable gets the label's area too.
|
|
45
|
+
const label = el.id ? surface.querySelector(`label[for="${CSS.escape(el.id)}"]`) : null;
|
|
46
|
+
const labelRect = label?.getBoundingClientRect();
|
|
47
|
+
const effective = labelRect && labelRect.width > 0 ? Math.max(small, Math.min(labelRect.height, 1000)) : small;
|
|
48
|
+
if (effective < 24) {
|
|
49
|
+
// Spacing exception: a 24px circle centred on the target must not intersect another target.
|
|
50
|
+
const cx = rects[i].x + width / 2;
|
|
51
|
+
const cy = rects[i].y + height / 2;
|
|
52
|
+
const crowded = rects.some((o, j) => {
|
|
53
|
+
if (j === i)
|
|
54
|
+
return false;
|
|
55
|
+
const dx = Math.max(o.x - cx, 0, cx - (o.x + o.width));
|
|
56
|
+
const dy = Math.max(o.y - cy, 0, cy - (o.y + o.height));
|
|
57
|
+
return Math.hypot(dx, dy) < 12;
|
|
58
|
+
});
|
|
59
|
+
if (crowded)
|
|
60
|
+
tooSmall.push(`${name(el)} (${Math.round(width)}×${Math.round(height)})`);
|
|
61
|
+
}
|
|
62
|
+
else if (effective < packMin - 0.5 && el.tagName === "BUTTON" && !el.closest("tr, .pxd-row-item, .pxd-breadcrumbs")) {
|
|
63
|
+
// Rows and breadcrumb trails are dense by design: inside one, WCAG's 24px minimum applies,
|
|
64
|
+
// not the pack's comfortable size — a 44px-tall breadcrumb is not what any of these systems draws.
|
|
65
|
+
belowPack.push(`${name(el)} (${Math.round(width)}×${Math.round(height)})`);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
// What happens must be the last thing read before the button that does it.
|
|
69
|
+
const consequence = surface.querySelector(".pxd-dialog-consequence");
|
|
70
|
+
const bar = consequence?.closest(".pxd-dialog")?.querySelector(".pxd-action-bar");
|
|
71
|
+
const consequenceMisplaced = !!consequence && !!bar && consequence.nextElementSibling !== bar;
|
|
72
|
+
return { overflow, tooSmall, belowPack, packMin, consequenceMisplaced };
|
|
73
|
+
});
|
|
74
|
+
const out = [];
|
|
75
|
+
if (r.consequenceMisplaced)
|
|
76
|
+
out.push({ severity: "error", check: "layout:consequence-placement", message: "the consequence must sit directly above the buttons it warns about" });
|
|
77
|
+
if (r.overflow > 1)
|
|
78
|
+
out.push({ severity: "error", check: "layout:overflow", message: `content is ${r.overflow}px wider than the surface (horizontal scrolling)` });
|
|
79
|
+
if (r.tooSmall.length)
|
|
80
|
+
out.push({ severity: "error", check: "layout:target-size", message: `targets under 24px without spacing (WCAG 2.5.8): ${r.tooSmall.slice(0, 4).join(", ")}`, count: r.tooSmall.length });
|
|
81
|
+
if (r.belowPack.length)
|
|
82
|
+
out.push({ severity: "warning", check: "layout:target-size-pack", message: `buttons below the pack's ${r.packMin}px target: ${r.belowPack.slice(0, 4).join(", ")}`, count: r.belowPack.length });
|
|
83
|
+
return out;
|
|
84
|
+
}
|
|
85
|
+
//# sourceMappingURL=rendered.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rendered.js","sourceRoot":"","sources":["../src/rendered.ts"],"names":[],"mappings":"AAUA,MAAM,SAAS,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;AAE3E,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAU;IACvC,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QACpD,MAAM,GAAG,GAAI,MAAc,CAAC,GAAG,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QAC9I,OAAO,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACzJ,CAAC,EAAE,SAAS,CAAC,CAAC;IACd,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC;QACjC,QAAQ,EAAE,CAAC,CAAC,MAAM,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;QACjF,KAAK,EAAE,OAAO,CAAC,CAAC,EAAE,EAAE;QACpB,OAAO,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,MAAM,GAAG;QACnD,KAAK,EAAE,CAAC,CAAC,KAAK;KACf,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAU;IAC1C,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE;QACjC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,cAAc,CAAgB,CAAC;QACtE,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QAC3D,MAAM,OAAO,GAAG,UAAU,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC,IAAI,EAAE,CAAC;QACtG,MAAM,QAAQ,GAAG,sIAAsI,CAAC;QACxJ,MAAM,GAAG,GAAG,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAc,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;YAC7E,MAAM,IAAI,GAAG,EAAE,CAAC,qBAAqB,EAAE,CAAC;YACxC,MAAM,KAAK,GAAG,gBAAgB,CAAC,EAAE,CAAC,CAAC;YACnC,OAAO,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,UAAU,KAAK,QAAQ,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;QACjH,CAAC,CAAC,CAAC;QACH,0FAA0F;QAC1F,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,gBAAgB,IAAI,EAAE,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;QACnH,iFAAiF;QACjF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,sBAAsB,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,sBAAsB,CAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,qBAAqB,EAAE,CAAC,CAAC;QACrL,MAAM,IAAI,GAAG,CAAC,EAAe,EAAE,EAAE,CAAC,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;QAC5I,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE;YACxB,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACnC,IAAI,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,eAAe,CAAC;gBAAE,OAAO;YACnD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YACtC,8EAA8E;YAC9E,MAAM,KAAK,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,aAAa,CAAc,cAAc,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACrG,MAAM,SAAS,GAAG,KAAK,EAAE,qBAAqB,EAAE,CAAC;YACjD,MAAM,SAAS,GAAG,SAAS,IAAI,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;YAC/G,IAAI,SAAS,GAAG,EAAE,EAAE,CAAC;gBACnB,4FAA4F;gBAC5F,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;gBAClC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC;gBACnC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;oBAClC,IAAI,CAAC,KAAK,CAAC;wBAAE,OAAO,KAAK,CAAC;oBAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;oBACvD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;oBACxD,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC;gBACjC,CAAC,CAAC,CAAC;gBACH,IAAI,OAAO;oBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACzF,CAAC;iBAAM,IAAI,SAAS,GAAG,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,qCAAqC,CAAC,EAAE,CAAC;gBACtH,2FAA2F;gBAC3F,mGAAmG;gBACnG,SAAS,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7E,CAAC;QACH,CAAC,CAAC,CAAC;QACH,2EAA2E;QAC3E,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,yBAAyB,CAAC,CAAC;QACrE,MAAM,GAAG,GAAG,WAAW,EAAE,OAAO,CAAC,aAAa,CAAC,EAAE,aAAa,CAAC,iBAAiB,CAAC,CAAC;QAClF,MAAM,oBAAoB,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,IAAI,WAAW,CAAC,kBAAkB,KAAK,GAAG,CAAC;QAC9F,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC;IAC1E,CAAC,CAAC,CAAC;IACH,MAAM,GAAG,GAAc,EAAE,CAAC;IAC1B,IAAI,CAAC,CAAC,oBAAoB;QAAE,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,8BAA8B,EAAE,OAAO,EAAE,oEAAoE,EAAE,CAAC,CAAC;IAClL,IAAI,CAAC,CAAC,QAAQ,GAAG,CAAC;QAAE,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,iBAAiB,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC,QAAQ,kDAAkD,EAAE,CAAC,CAAC;IACnK,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM;QAAE,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,oBAAoB,EAAE,OAAO,EAAE,oDAAoD,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAChN,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM;QAAE,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,yBAAyB,EAAE,OAAO,EAAE,4BAA4B,CAAC,CAAC,OAAO,cAAc,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IACzN,OAAO,GAAG,CAAC;AACb,CAAC"}
|
package/dist/reward.d.ts
ADDED
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import type { Browser, Page } from "playwright";
|
|
2
|
+
import { type VerifyOptions } from "./verify.ts";
|
|
3
|
+
/**
|
|
4
|
+
* The training reward for one candidate surface (decision 0003).
|
|
5
|
+
*
|
|
6
|
+
* The verifier's score is the floor: valid, safe, accessible, laid out. On its own it rewards
|
|
7
|
+
* timid interfaces — a summary with nothing to do scores well and does nothing (research log,
|
|
8
|
+
* Phase 5 run 1). So the reward also asks whether the surface does the job: does it offer what the
|
|
9
|
+
* request needed (coverage), and can an agent actually set it in motion?
|
|
10
|
+
*
|
|
11
|
+
* Coverage replaced a weaker question — "was any offered capability attached to anything" — after a
|
|
12
|
+
* designer ranked the model's own output and the verifier disagreed with all of it. On "find 30
|
|
13
|
+
* minutes with Tom and Priya", the candidate that attempted nothing scored highest and was ranked
|
|
14
|
+
* last: a document that doesn't try can't make mistakes.
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* What the bench says a request needs. Every request in bench/requests*.json already declares it:
|
|
18
|
+
* the capability the answer has to wire, the controls it has to offer by name, the components it
|
|
19
|
+
* should be built from, and the pattern it should follow. `mustNot` is prose for a reader, so it
|
|
20
|
+
* isn't read here.
|
|
21
|
+
*/
|
|
22
|
+
export interface RequestExpectation {
|
|
23
|
+
pattern?: string;
|
|
24
|
+
components?: string[];
|
|
25
|
+
capability?: string;
|
|
26
|
+
names?: string[];
|
|
27
|
+
}
|
|
28
|
+
export interface Reward {
|
|
29
|
+
/** 0–100, the verifier's score */
|
|
30
|
+
score: number;
|
|
31
|
+
/** Offered capabilities attached to something operable, over offered */
|
|
32
|
+
wiring: number;
|
|
33
|
+
/** How much of the job the request asked for the document actually does, 0–1 */
|
|
34
|
+
coverage: number;
|
|
35
|
+
/** How closely it matches the components and pattern the bench expected, 0–1. Reported, not scored. */
|
|
36
|
+
form: number;
|
|
37
|
+
/** The parts of the request's expectation the document doesn't meet */
|
|
38
|
+
missing: string[];
|
|
39
|
+
/** Whether a scripted agent could trigger one of the offered capabilities */
|
|
40
|
+
actionable: boolean;
|
|
41
|
+
/** Components in the document, used for the length penalty */
|
|
42
|
+
size: number;
|
|
43
|
+
/** The combined reward, 0–100-ish */
|
|
44
|
+
reward: number;
|
|
45
|
+
findings: string[];
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* How much of what the request asked for the document actually offers.
|
|
49
|
+
*
|
|
50
|
+
* The verifier's score says whether a document is correct; it has no term for whether it does the
|
|
51
|
+
* job. This is that term, and it needs no new annotation: the bench already declares, per request,
|
|
52
|
+
* the capability the answer has to wire and the control it has to offer by name. A surface that
|
|
53
|
+
* attempts nothing scores well and covers nothing.
|
|
54
|
+
*/
|
|
55
|
+
export declare function coverageOf(flat: any, expect?: RequestExpectation): {
|
|
56
|
+
coverage: number;
|
|
57
|
+
missing: string[];
|
|
58
|
+
};
|
|
59
|
+
/** Capabilities attached to a control a person or an agent can operate. */
|
|
60
|
+
export declare function wiredCapabilities(flat: any): Set<string>;
|
|
61
|
+
/**
|
|
62
|
+
* Can an agent set the surface's main action in motion? Presses the most prominent action it can
|
|
63
|
+
* find — the primary button, else the only button — and reports whether the host heard about it.
|
|
64
|
+
* A surface nobody can act on is decoration, however well it scores.
|
|
65
|
+
*/
|
|
66
|
+
export declare function probeActionable(page: Page, offered: string[]): Promise<boolean>;
|
|
67
|
+
/** Scores one candidate for training: the verifier, plus whether the surface can be used. */
|
|
68
|
+
export declare function rewardFor(doc: any, browser: Browser, offered: string[], opts?: VerifyOptions, medianSize?: number, expect?: RequestExpectation): Promise<Reward>;
|
package/dist/reward.js
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { renderPage, sentActions } from "./browser.js";
|
|
2
|
+
import { verifyDocument } from "./verify.js";
|
|
3
|
+
/** Every label in the document a person could read off a control. */
|
|
4
|
+
function labelsIn(flat) {
|
|
5
|
+
const out = [];
|
|
6
|
+
const add = (v) => typeof v === "string" && out.push(v);
|
|
7
|
+
for (const c of flat.components ?? []) {
|
|
8
|
+
add(c.label);
|
|
9
|
+
add(c.title);
|
|
10
|
+
for (const p of ["submit", "confirm", "cancel", "choose", "clear", "finish", "action"])
|
|
11
|
+
add(c[p]?.label);
|
|
12
|
+
for (const a of c.actions ?? [])
|
|
13
|
+
add(a.label);
|
|
14
|
+
for (const a of c.rowActions ?? [])
|
|
15
|
+
add(a.label);
|
|
16
|
+
for (const a of c.bulkActions ?? [])
|
|
17
|
+
add(a.label);
|
|
18
|
+
}
|
|
19
|
+
return out;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* How much of what the request asked for the document actually offers.
|
|
23
|
+
*
|
|
24
|
+
* The verifier's score says whether a document is correct; it has no term for whether it does the
|
|
25
|
+
* job. This is that term, and it needs no new annotation: the bench already declares, per request,
|
|
26
|
+
* the capability the answer has to wire and the control it has to offer by name. A surface that
|
|
27
|
+
* attempts nothing scores well and covers nothing.
|
|
28
|
+
*/
|
|
29
|
+
export function coverageOf(flat, expect) {
|
|
30
|
+
if (!expect)
|
|
31
|
+
return { coverage: 1, missing: [] };
|
|
32
|
+
const parts = [];
|
|
33
|
+
const missing = [];
|
|
34
|
+
if (expect.capability) {
|
|
35
|
+
const wired = wiredCapabilities(flat).has(expect.capability);
|
|
36
|
+
parts.push(wired ? 1 : 0);
|
|
37
|
+
if (!wired)
|
|
38
|
+
missing.push(`capability ${expect.capability}`);
|
|
39
|
+
}
|
|
40
|
+
if (expect.names?.length) {
|
|
41
|
+
const labels = labelsIn(flat).map((l) => l.toLowerCase());
|
|
42
|
+
const hit = expect.names.filter((n) => labels.some((l) => l.includes(n.toLowerCase())));
|
|
43
|
+
parts.push(hit.length / expect.names.length);
|
|
44
|
+
for (const n of expect.names)
|
|
45
|
+
if (!hit.includes(n))
|
|
46
|
+
missing.push(`control "${n}"`);
|
|
47
|
+
}
|
|
48
|
+
if (expect.components?.length) {
|
|
49
|
+
const present = new Set((flat.components ?? []).map((c) => c.component));
|
|
50
|
+
const hit = expect.components.filter((c) => present.has(c));
|
|
51
|
+
parts.push(hit.length / expect.components.length);
|
|
52
|
+
for (const c of expect.components)
|
|
53
|
+
if (!present.has(c))
|
|
54
|
+
missing.push(`component ${c}`);
|
|
55
|
+
}
|
|
56
|
+
if (expect.pattern) {
|
|
57
|
+
const same = flat.surface?.pattern === expect.pattern;
|
|
58
|
+
parts.push(same ? 1 : 0);
|
|
59
|
+
if (!same)
|
|
60
|
+
missing.push(`pattern ${expect.pattern}`);
|
|
61
|
+
}
|
|
62
|
+
return { coverage: parts.length ? parts.reduce((a, b) => a + b, 0) / parts.length : 1, missing };
|
|
63
|
+
}
|
|
64
|
+
/** Capabilities attached to a control a person or an agent can operate. */
|
|
65
|
+
export function wiredCapabilities(flat) {
|
|
66
|
+
const out = new Set();
|
|
67
|
+
const add = (a) => typeof a?.event?.name === "string" && out.add(a.event.name);
|
|
68
|
+
for (const c of flat.components ?? []) {
|
|
69
|
+
if (c.component === "Action" || c.component === "Card" || c.component === "Toggle")
|
|
70
|
+
add(c.action);
|
|
71
|
+
if (c.component === "Table")
|
|
72
|
+
add(c.rowAction);
|
|
73
|
+
if (Array.isArray(c.views))
|
|
74
|
+
for (const v of c.views)
|
|
75
|
+
add(v.action);
|
|
76
|
+
for (const p of ["submit", "confirm", "finish", "choose", "clear"])
|
|
77
|
+
add(c[p]?.action ?? c[p]);
|
|
78
|
+
for (const b of c.breadcrumbs ?? [])
|
|
79
|
+
add(b.action);
|
|
80
|
+
}
|
|
81
|
+
return out;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Can an agent set the surface's main action in motion? Presses the most prominent action it can
|
|
85
|
+
* find — the primary button, else the only button — and reports whether the host heard about it.
|
|
86
|
+
* A surface nobody can act on is decoration, however well it scores.
|
|
87
|
+
*/
|
|
88
|
+
export async function probeActionable(page, offered) {
|
|
89
|
+
if (!offered.length)
|
|
90
|
+
return true;
|
|
91
|
+
await fillRequired(page);
|
|
92
|
+
const before = (await sentActions(page)).length;
|
|
93
|
+
const candidates = [
|
|
94
|
+
page.locator(".pxd-surface .pxd-button-primary"),
|
|
95
|
+
page.locator('.pxd-surface button[type="submit"]'),
|
|
96
|
+
page.locator(".pxd-surface .pxd-card-link"),
|
|
97
|
+
page.locator(".pxd-surface .pxd-button"),
|
|
98
|
+
];
|
|
99
|
+
for (const locator of candidates) {
|
|
100
|
+
if ((await locator.count()) === 0)
|
|
101
|
+
continue;
|
|
102
|
+
try {
|
|
103
|
+
await locator.first().click({ timeout: 2000 });
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
const sent = await sentActions(page);
|
|
109
|
+
if (sent.length > before && sent.slice(before).some((a) => offered.includes(a.name)))
|
|
110
|
+
return true;
|
|
111
|
+
}
|
|
112
|
+
return false;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Fills what the surface insists on before pressing anything: an agent completing a task would,
|
|
116
|
+
* and a form that can't be submitted empty isn't the same as a surface nobody can act on.
|
|
117
|
+
*/
|
|
118
|
+
async function fillRequired(page) {
|
|
119
|
+
const fields = page.locator(".pxd-surface input[required], .pxd-surface textarea[required]");
|
|
120
|
+
for (let i = 0; i < (await fields.count()); i++) {
|
|
121
|
+
const field = fields.nth(i);
|
|
122
|
+
try {
|
|
123
|
+
if (await field.inputValue())
|
|
124
|
+
continue;
|
|
125
|
+
const type = await field.getAttribute("type");
|
|
126
|
+
await field.fill(type === "number" ? "10" : type === "email" ? "someone@example.com" : type === "date" ? "2026-10-01" : "Test");
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
// A control that can't be typed into (a radio, a checkbox) is handled below.
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
for (const group of [".pxd-surface [role=radiogroup]", ".pxd-surface .pxd-people-list", ".pxd-surface .pxd-chips"]) {
|
|
133
|
+
const first = page.locator(`${group} [role=radio], ${group} [role=checkbox]`).first();
|
|
134
|
+
if ((await first.count()) && (await first.getAttribute("aria-checked")) !== "true") {
|
|
135
|
+
await first.click({ timeout: 1500 }).catch(() => { });
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
/** Scores one candidate for training: the verifier, plus whether the surface can be used. */
|
|
140
|
+
export async function rewardFor(doc, browser, offered, opts = {}, medianSize = 0, expect) {
|
|
141
|
+
const report = await verifyDocument(doc, { browser, themes: ["material3"], modes: ["light"], widths: [390], ...opts });
|
|
142
|
+
const wiredSet = wiredCapabilities(doc);
|
|
143
|
+
const wiring = offered.length ? offered.filter((c) => wiredSet.has(c)).length / offered.length : 1;
|
|
144
|
+
// What the request asked for, when the bench says; otherwise fall back to wiring, which asks the
|
|
145
|
+
// weaker question of whether any offered capability was attached to anything.
|
|
146
|
+
// Two different questions, kept apart. Coverage is *did it do the job* — the capability the
|
|
147
|
+
// request needs, and a control named the way the task presses it. Form is *did it look the way
|
|
148
|
+
// the bench expected* — components and pattern — and it is reported but not scored: a designer
|
|
149
|
+
// ranking the model's own output was happy with surfaces the bench expected a Table from.
|
|
150
|
+
const { coverage, missing } = expect ? coverageOf(doc, { capability: expect.capability, names: expect.names }) : { coverage: wiring, missing: [] };
|
|
151
|
+
const form = expect ? coverageOf(doc, { components: expect.components, pattern: expect.pattern }).coverage : 1;
|
|
152
|
+
let actionable = false;
|
|
153
|
+
if (report.score > 0) {
|
|
154
|
+
const { page } = await renderPage(browser, doc, { theme: "material3", mode: "light", width: 390 });
|
|
155
|
+
try {
|
|
156
|
+
actionable = await probeActionable(page, offered);
|
|
157
|
+
}
|
|
158
|
+
finally {
|
|
159
|
+
await page.close();
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
const size = doc.components?.length ?? 0;
|
|
163
|
+
// There is no length penalty any more. It was there so padding a document to satisfy checks
|
|
164
|
+
// wouldn't pay, but it pushed the same way as every other term: on "stop emailing me" the
|
|
165
|
+
// designer ranked the four-toggle version first and the one-toggle version last, and the reward
|
|
166
|
+
// was already biased towards the small one. `medianSize` is kept in the signature so callers
|
|
167
|
+
// don't change; it is no longer read.
|
|
168
|
+
const padding = 0;
|
|
169
|
+
void medianSize;
|
|
170
|
+
// Coverage multiplies rather than adds. A surface that doesn't do what was asked isn't a good
|
|
171
|
+
// surface with a deduction; it isn't an answer, and adding a bonus for coverage leaves it ahead
|
|
172
|
+
// of a surface that tried and made two mistakes — which is the ordering a designer ranked last.
|
|
173
|
+
// The 0.2 floor keeps a partly-covering document from collapsing to nothing, because the bench's
|
|
174
|
+
// expectations are themselves fallible.
|
|
175
|
+
const reward = Math.max(0, report.score * (0.2 + 0.8 * coverage) + (actionable ? 20 : 0) - padding);
|
|
176
|
+
return {
|
|
177
|
+
score: report.score,
|
|
178
|
+
wiring,
|
|
179
|
+
coverage,
|
|
180
|
+
form,
|
|
181
|
+
missing,
|
|
182
|
+
actionable,
|
|
183
|
+
size,
|
|
184
|
+
reward,
|
|
185
|
+
findings: [...report.static, ...report.targets.flatMap((t) => t.findings)].filter((f) => f.severity === "error").map((f) => f.check),
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
//# sourceMappingURL=reward.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reward.js","sourceRoot":"","sources":["../src/reward.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,cAAc,EAAsB,MAAM,aAAa,CAAC;AAiDjE,qEAAqE;AACrE,SAAS,QAAQ,CAAC,IAAS;IACzB,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,CAAC,CAAU,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC;QACtC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACb,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC;YAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QACzG,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,OAAO,IAAI,EAAE;YAAE,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAC9C,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,UAAU,IAAI,EAAE;YAAE,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,IAAI,EAAE;YAAE,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,UAAU,CAAC,IAAS,EAAE,MAA2B;IAC/D,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACjD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QACtB,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC7D,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,CAAC,KAAK;YAAE,OAAO,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC;QACxF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK;YAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QAC9E,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAClD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU;YAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IACzF,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,KAAK,MAAM,CAAC,OAAO,CAAC;QACtD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;AACnG,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,iBAAiB,CAAC,IAAS;IACzC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9B,MAAM,GAAG,GAAG,CAAC,CAAM,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACpF,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,UAAU,IAAI,EAAE,EAAE,CAAC;QACtC,IAAI,CAAC,CAAC,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC,SAAS,KAAK,MAAM,IAAI,CAAC,CAAC,SAAS,KAAK,QAAQ;YAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAClG,IAAI,CAAC,CAAC,SAAS,KAAK,OAAO;YAAE,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;YAAE,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK;gBAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACnE,KAAK,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC;YAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9F,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,IAAI,EAAE;YAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAU,EAAE,OAAiB;IACjE,IAAI,CAAC,OAAO,CAAC,MAAM;QAAE,OAAO,IAAI,CAAC;IACjC,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IACzB,MAAM,MAAM,GAAG,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAChD,MAAM,UAAU,GAAG;QACjB,IAAI,CAAC,OAAO,CAAC,kCAAkC,CAAC;QAChD,IAAI,CAAC,OAAO,CAAC,oCAAoC,CAAC;QAClD,IAAI,CAAC,OAAO,CAAC,6BAA6B,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,0BAA0B,CAAC;KACzC,CAAC;IACF,KAAK,MAAM,OAAO,IAAI,UAAU,EAAE,CAAC;QACjC,IAAI,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC;YAAE,SAAS;QAC5C,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;IACpG,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,YAAY,CAAC,IAAU;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,+DAA+D,CAAC,CAAC;IAC7F,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAChD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,CAAC;YACH,IAAI,MAAM,KAAK,CAAC,UAAU,EAAE;gBAAE,SAAS;YACvC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAC9C,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAClI,CAAC;QAAC,MAAM,CAAC;YACP,6EAA6E;QAC/E,CAAC;IACH,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,CAAC,gCAAgC,EAAE,+BAA+B,EAAE,yBAAyB,CAAC,EAAE,CAAC;QACnH,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,KAAK,kBAAkB,KAAK,kBAAkB,CAAC,CAAC,KAAK,EAAE,CAAC;QACtF,IAAI,CAAC,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,YAAY,CAAC,cAAc,CAAC,CAAC,KAAK,MAAM,EAAE,CAAC;YACnF,MAAM,KAAK,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;AACH,CAAC;AAED,6FAA6F;AAC7F,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,GAAQ,EACR,OAAgB,EAChB,OAAiB,EACjB,IAAI,GAAkB,EAAE,EACxB,UAAU,GAAG,CAAC,EACd,MAA2B;IAE3B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,KAAK,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC,CAAC;IACvH,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACnG,iGAAiG;IACjG,8EAA8E;IAC9E,4FAA4F;IAC5F,+FAA+F;IAC/F,+FAA+F;IAC/F,0FAA0F;IAC1F,MAAM,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACnJ,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAE/G,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,UAAU,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QACnG,IAAI,CAAC;YACH,UAAU,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACpD,CAAC;gBAAS,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACrB,CAAC;IACH,CAAC;IAED,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;IACzC,4FAA4F;IAC5F,0FAA0F;IAC1F,gGAAgG;IAChG,6FAA6F;IAC7F,sCAAsC;IACtC,MAAM,OAAO,GAAG,CAAC,CAAC;IAClB,KAAK,UAAU,CAAC;IAChB,8FAA8F;IAC9F,gGAAgG;IAChG,gGAAgG;IAChG,iGAAiG;IACjG,wCAAwC;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,GAAG,CAAC,GAAG,GAAG,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC;IACpG,OAAO;QACL,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM;QACN,QAAQ;QACR,IAAI;QACJ,OAAO;QACP,UAAU;QACV,IAAI;QACJ,MAAM;QACN,QAAQ,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;KACrI,CAAC;AACJ,CAAC"}
|
package/dist/static.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { type Rule } from "@polyxd/spec/patterns";
|
|
2
|
+
import { type CapabilityRegistry } from "@polyxd/spec/capabilities";
|
|
3
|
+
import type { Finding } from "./rendered.ts";
|
|
4
|
+
export interface StaticOptions {
|
|
5
|
+
registry?: CapabilityRegistry;
|
|
6
|
+
/** Design Direction or acceptance rules to hold the document to */
|
|
7
|
+
rules?: Rule[];
|
|
8
|
+
/** Primary actions allowed in one view (Design Direction's profile.emphasisBudget) */
|
|
9
|
+
emphasisBudget?: number;
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Document-level checks that need no rendering: the spec validator, the declared pattern,
|
|
13
|
+
* capabilities, direction/acceptance rules, plus agent-readiness heuristics.
|
|
14
|
+
*/
|
|
15
|
+
export declare function staticAudit(doc: any, opts?: StaticOptions): Finding[];
|