@visual-layer/grounding 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/compare.d.ts +34 -0
- package/dist/compare.d.ts.map +1 -0
- package/dist/compare.js +112 -0
- package/dist/compare.js.map +1 -0
- package/dist/eval.d.ts +36 -0
- package/dist/eval.d.ts.map +1 -0
- package/dist/eval.js +56 -0
- package/dist/eval.js.map +1 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/index.js.map +1 -0
- package/dist/score.d.ts +28 -0
- package/dist/score.d.ts.map +1 -0
- package/dist/score.js +117 -0
- package/dist/score.js.map +1 -0
- package/package.json +33 -0
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* compare_graphs — deterministic structural comparison of two ScreenGraphs.
|
|
3
|
+
* MVP supports only `mode=fixture-v0`: match interactables by normalized text,
|
|
4
|
+
* report missing/extra and bbox deltas. Production-scale modes are post-MVP.
|
|
5
|
+
*/
|
|
6
|
+
import { type ScreenGraph, type Warning, type Transform } from '@visual-layer/core';
|
|
7
|
+
export type CompareMode = 'fixture-v0';
|
|
8
|
+
export interface BboxDelta {
|
|
9
|
+
interactableId: string;
|
|
10
|
+
dx: number;
|
|
11
|
+
dy: number;
|
|
12
|
+
dw: number;
|
|
13
|
+
dh: number;
|
|
14
|
+
}
|
|
15
|
+
export interface CompareOptions {
|
|
16
|
+
mode?: CompareMode;
|
|
17
|
+
/** Required when graphs live in different coordinate spaces. */
|
|
18
|
+
transform?: Transform;
|
|
19
|
+
/** Cap on candidate pairs scored; emit compare_truncated when exceeded. */
|
|
20
|
+
maxPairs?: number;
|
|
21
|
+
/** Minimum text similarity to consider two interactables a match. */
|
|
22
|
+
matchThreshold?: number;
|
|
23
|
+
}
|
|
24
|
+
export interface CompareResult {
|
|
25
|
+
missingFromActual: string[];
|
|
26
|
+
extraInActual: string[];
|
|
27
|
+
bboxDeltas: BboxDelta[];
|
|
28
|
+
warnings: Warning[];
|
|
29
|
+
}
|
|
30
|
+
export declare function compareGraphs(expected: ScreenGraph, actual: ScreenGraph, opts?: CompareOptions): CompareResult;
|
|
31
|
+
declare function unsupported(detail: string): Error;
|
|
32
|
+
declare function transformMissing(detail: string): Error;
|
|
33
|
+
export { unsupported, transformMissing };
|
|
34
|
+
//# sourceMappingURL=compare.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../src/compare.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EACL,KAAK,WAAW,EAEhB,KAAK,OAAO,EAEZ,KAAK,SAAS,EAMf,MAAM,oBAAoB,CAAC;AAE5B,MAAM,MAAM,WAAW,GAAG,YAAY,CAAC;AAEvC,MAAM,WAAW,SAAS;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;IACX,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,gEAAgE;IAChE,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qEAAqE;IACrE,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAKD,wBAAgB,aAAa,CAC3B,QAAQ,EAAE,WAAW,EACrB,MAAM,EAAE,WAAW,EACnB,IAAI,GAAE,cAAmB,GACxB,aAAa,CAqEf;AAwCD,iBAAS,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAI1C;AAED,iBAAS,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,KAAK,CAI/C;AAMD,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC"}
|
package/dist/compare.js
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* compare_graphs — deterministic structural comparison of two ScreenGraphs.
|
|
3
|
+
* MVP supports only `mode=fixture-v0`: match interactables by normalized text,
|
|
4
|
+
* report missing/extra and bbox deltas. Production-scale modes are post-MVP.
|
|
5
|
+
*/
|
|
6
|
+
import { compareText, normalizeText, convertBbox, warn, } from '@visual-layer/core';
|
|
7
|
+
const DEFAULT_MAX_PAIRS = 5000;
|
|
8
|
+
const DEFAULT_MATCH = 0.6;
|
|
9
|
+
export function compareGraphs(expected, actual, opts = {}) {
|
|
10
|
+
const mode = opts.mode ?? 'fixture-v0';
|
|
11
|
+
if (mode !== 'fixture-v0') {
|
|
12
|
+
throw unsupported(`mode "${mode}" not supported in MVP; use fixture-v0`);
|
|
13
|
+
}
|
|
14
|
+
const warnings = [];
|
|
15
|
+
const expSpace = dominantSpace(expected);
|
|
16
|
+
const actSpace = dominantSpace(actual);
|
|
17
|
+
if (expSpace && actSpace && expSpace !== actSpace && !opts.transform) {
|
|
18
|
+
throw transformMissing(`expected space "${expSpace}" != actual space "${actSpace}"; supply a transform`);
|
|
19
|
+
}
|
|
20
|
+
// Validate a supplied transform maps actual -> expected space (prevents silently
|
|
21
|
+
// applying a factor in the wrong direction).
|
|
22
|
+
if (opts.transform && expSpace && actSpace) {
|
|
23
|
+
const t = opts.transform;
|
|
24
|
+
if (t.from !== actSpace || t.to !== expSpace) {
|
|
25
|
+
throw transformMissing(`transform ${t.from}->${t.to} does not match actual(${actSpace})->expected(${expSpace})`);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
const exInts = expected.interactables;
|
|
29
|
+
const acInts = actual.interactables;
|
|
30
|
+
const maxPairs = opts.maxPairs ?? DEFAULT_MAX_PAIRS;
|
|
31
|
+
const matchThreshold = opts.matchThreshold ?? DEFAULT_MATCH;
|
|
32
|
+
const matchedActual = new Set();
|
|
33
|
+
const bboxDeltas = [];
|
|
34
|
+
const compared = new Set();
|
|
35
|
+
// Honest truncation: stop scoring once the pair budget is exhausted, and report
|
|
36
|
+
// how many expected interactables were skipped (skipped are NOT counted as missing).
|
|
37
|
+
let pairsBudget = maxPairs;
|
|
38
|
+
let skipped = 0;
|
|
39
|
+
for (const ex of exInts) {
|
|
40
|
+
if (pairsBudget <= 0) {
|
|
41
|
+
skipped++;
|
|
42
|
+
continue;
|
|
43
|
+
}
|
|
44
|
+
compared.add(ex.id);
|
|
45
|
+
const best = bestMatch(ex, acInts, matchThreshold, matchedActual);
|
|
46
|
+
pairsBudget -= acInts.length;
|
|
47
|
+
if (!best)
|
|
48
|
+
continue;
|
|
49
|
+
matchedActual.add(best.match.id);
|
|
50
|
+
const actualBbox = opts.transform ? convertBbox(best.match.bbox, opts.transform) : best.match.bbox;
|
|
51
|
+
bboxDeltas.push({
|
|
52
|
+
interactableId: ex.id,
|
|
53
|
+
dx: round1(ex.bbox[0] - actualBbox[0]),
|
|
54
|
+
dy: round1(ex.bbox[1] - actualBbox[1]),
|
|
55
|
+
dw: round1(ex.bbox[2] - actualBbox[2]),
|
|
56
|
+
dh: round1(ex.bbox[3] - actualBbox[3]),
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
if (skipped > 0) {
|
|
60
|
+
warnings.push(warn('compare_truncated', `${skipped} expected interactables skipped (pairs cap ${maxPairs})`, undefined));
|
|
61
|
+
}
|
|
62
|
+
// missing = compared expected that found no match; un-compared (skipped) excluded
|
|
63
|
+
const missingFromActual = exInts
|
|
64
|
+
.filter((e) => compared.has(e.id) && !bboxDeltas.some((d) => d.interactableId === e.id))
|
|
65
|
+
.map((e) => e.id);
|
|
66
|
+
const extraInActual = acInts.filter((a) => !matchedActual.has(a.id)).map((a) => a.id);
|
|
67
|
+
return { missingFromActual, extraInActual, bboxDeltas, warnings };
|
|
68
|
+
}
|
|
69
|
+
function bestMatch(ex, candidates, threshold, used) {
|
|
70
|
+
let best = null;
|
|
71
|
+
for (const c of candidates) {
|
|
72
|
+
if (used.has(c.id))
|
|
73
|
+
continue;
|
|
74
|
+
const score = compareText(labelFor(ex), labelFor(c));
|
|
75
|
+
// role agreement is a hard precondition — different role = not the same element
|
|
76
|
+
if (ex.role && c.role && ex.role.toLowerCase() !== c.role.toLowerCase())
|
|
77
|
+
continue;
|
|
78
|
+
if (score >= threshold && (!best || score > best.score)) {
|
|
79
|
+
best = { match: c, score };
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return best;
|
|
83
|
+
}
|
|
84
|
+
function labelFor(i) {
|
|
85
|
+
const raw = i.text ?? i.name ?? '';
|
|
86
|
+
const n = normalizeText(raw);
|
|
87
|
+
// fall back to a role-only key when there's no text — keeps role-only elements matchable
|
|
88
|
+
return n.length > 0 ? n : (i.role ?? '');
|
|
89
|
+
}
|
|
90
|
+
function dominantSpace(g) {
|
|
91
|
+
if (g.interactables.length === 0)
|
|
92
|
+
return undefined; // empty graph imposes no space constraint
|
|
93
|
+
const spaces = new Set(g.interactables.map((i) => i.coordinateSpace));
|
|
94
|
+
if (spaces.size === 1)
|
|
95
|
+
return [...spaces][0];
|
|
96
|
+
return 'mixed';
|
|
97
|
+
}
|
|
98
|
+
function unsupported(detail) {
|
|
99
|
+
const e = new Error(`unsupported_compare_mode: ${detail}`);
|
|
100
|
+
e.code = 'unsupported_compare_mode';
|
|
101
|
+
return e;
|
|
102
|
+
}
|
|
103
|
+
function transformMissing(detail) {
|
|
104
|
+
const e = new Error(`coordinate_transform_missing: ${detail}`);
|
|
105
|
+
e.code = 'coordinate_transform_missing';
|
|
106
|
+
return e;
|
|
107
|
+
}
|
|
108
|
+
function round1(n) {
|
|
109
|
+
return Math.round(n * 10) / 10;
|
|
110
|
+
}
|
|
111
|
+
export { unsupported, transformMissing };
|
|
112
|
+
//# sourceMappingURL=compare.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compare.js","sourceRoot":"","sources":["../src/compare.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAOL,WAAW,EACX,aAAa,EACb,WAAW,EACX,IAAI,GACL,MAAM,oBAAoB,CAAC;AA6B5B,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAC/B,MAAM,aAAa,GAAG,GAAG,CAAC;AAE1B,MAAM,UAAU,aAAa,CAC3B,QAAqB,EACrB,MAAmB,EACnB,OAAuB,EAAE;IAEzB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,YAAY,CAAC;IACvC,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;QAC1B,MAAM,WAAW,CAAC,SAAS,IAAI,wCAAwC,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAEvC,IAAI,QAAQ,IAAI,QAAQ,IAAI,QAAQ,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QACrE,MAAM,gBAAgB,CACpB,mBAAmB,QAAQ,sBAAsB,QAAQ,uBAAuB,CACjF,CAAC;IACJ,CAAC;IACD,iFAAiF;IACjF,6CAA6C;IAC7C,IAAI,IAAI,CAAC,SAAS,IAAI,QAAQ,IAAI,QAAQ,EAAE,CAAC;QAC3C,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC;QACzB,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YAC7C,MAAM,gBAAgB,CACpB,aAAa,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,EAAE,0BAA0B,QAAQ,eAAe,QAAQ,GAAG,CACzF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,aAAa,CAAC;IACtC,MAAM,MAAM,GAAG,MAAM,CAAC,aAAa,CAAC;IACpC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IAEpD,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,aAAa,CAAC;IAC5D,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAC;IACxC,MAAM,UAAU,GAAgB,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,gFAAgF;IAChF,qFAAqF;IACrF,IAAI,WAAW,GAAG,QAAQ,CAAC;IAC3B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,EAAE,IAAI,MAAM,EAAE,CAAC;QACxB,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;YACrB,OAAO,EAAE,CAAC;YACV,SAAS;QACX,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACpB,MAAM,IAAI,GAAG,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,aAAa,CAAC,CAAC;QAClE,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;QACnG,UAAU,CAAC,IAAI,CAAC;YACd,cAAc,EAAE,EAAE,CAAC,EAAE;YACrB,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YACtC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YACtC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YACtC,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;SACvC,CAAC,CAAC;IACL,CAAC;IACD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QAChB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,OAAO,8CAA8C,QAAQ,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC;IAC3H,CAAC;IAED,kFAAkF;IAClF,MAAM,iBAAiB,GAAG,MAAM;SAC7B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACvF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACpB,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAEtF,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AACpE,CAAC;AAOD,SAAS,SAAS,CAChB,EAAgB,EAChB,UAAmC,EACnC,SAAiB,EACjB,IAAiB;IAEjB,IAAI,IAAI,GAAqB,IAAI,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAAE,SAAS;QAC7B,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,gFAAgF;QAChF,IAAI,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE;YAAE,SAAS;QAClF,IAAI,KAAK,IAAI,SAAS,IAAI,CAAC,CAAC,IAAI,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACxD,IAAI,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,QAAQ,CAAC,CAAe;IAC/B,MAAM,GAAG,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;IACnC,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAC;IAC7B,yFAAyF;IACzF,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;AAC3C,CAAC;AAED,SAAS,aAAa,CAAC,CAAc;IACnC,IAAI,CAAC,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC,CAAC,0CAA0C;IAC9F,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC;IACtE,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAE,CAAC;IAC9C,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,WAAW,CAAC,MAAc;IACjC,MAAM,CAAC,GAAG,IAAI,KAAK,CAAC,6BAA6B,MAAM,EAAE,CAA8B,CAAC;IACxF,CAAC,CAAC,IAAI,GAAG,0BAA0B,CAAC;IACpC,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,gBAAgB,CAAC,MAAc;IACtC,MAAM,CAAC,GAAG,IAAI,KAAK,CAAC,iCAAiC,MAAM,EAAE,CAA8B,CAAC;IAC5F,CAAC,CAAC,IAAI,GAAG,8BAA8B,CAAC;IACxC,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,MAAM,CAAC,CAAS;IACvB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,EAAE,CAAC;AACjC,CAAC;AAED,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,CAAC"}
|
package/dist/eval.d.ts
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixture-only grounding eval harness (regression block alpha, §18.1).
|
|
3
|
+
* No live capture — drives groundInstruction over golden fixtures and reports
|
|
4
|
+
* top-K accuracy. A regression in fixture accuracy blocks alpha release.
|
|
5
|
+
*/
|
|
6
|
+
import { type ScreenGraph, type Warning } from '@visual-layer/core';
|
|
7
|
+
import { type GroundOptions } from './score.js';
|
|
8
|
+
export interface EvalFixture {
|
|
9
|
+
id: string;
|
|
10
|
+
graph: ScreenGraph;
|
|
11
|
+
instruction: string;
|
|
12
|
+
/** The interactable id that SHOULD rank first / within topK. */
|
|
13
|
+
expectedInteractableId: string;
|
|
14
|
+
topK?: number;
|
|
15
|
+
}
|
|
16
|
+
export interface EvalFailure {
|
|
17
|
+
fixtureId: string;
|
|
18
|
+
expectedInteractableId: string;
|
|
19
|
+
returnedIds: string[];
|
|
20
|
+
}
|
|
21
|
+
export interface EvalOptions extends GroundOptions {
|
|
22
|
+
topK?: number;
|
|
23
|
+
/** Block threshold for top-1 accuracy (warning when below). Default 1.0 for happy-path fixtures. */
|
|
24
|
+
minTop1Accuracy?: number;
|
|
25
|
+
/** Minimum confidence for a rank-0 match to count as a top-1 hit. Default 0.5. */
|
|
26
|
+
hitConfidence?: number;
|
|
27
|
+
}
|
|
28
|
+
export interface EvalResult {
|
|
29
|
+
cases: number;
|
|
30
|
+
top1Accuracy: number;
|
|
31
|
+
topKAccuracy: number;
|
|
32
|
+
failures: EvalFailure[];
|
|
33
|
+
warnings: Warning[];
|
|
34
|
+
}
|
|
35
|
+
export declare function evalGrounding(fixtures: readonly EvalFixture[], opts?: EvalOptions): EvalResult;
|
|
36
|
+
//# sourceMappingURL=eval.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval.d.ts","sourceRoot":"","sources":["../src/eval.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,KAAK,WAAW,EAAE,KAAK,OAAO,EAAQ,MAAM,oBAAoB,CAAC;AAC1E,OAAO,EAAqB,KAAK,aAAa,EAAE,MAAM,YAAY,CAAC;AAEnE,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,WAAW,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,gEAAgE;IAChE,sBAAsB,EAAE,MAAM,CAAC;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,WAAY,SAAQ,aAAa;IAChD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oGAAoG;IACpG,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,kFAAkF;IAClF,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAED,wBAAgB,aAAa,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,EAAE,IAAI,GAAE,WAAgB,GAAG,UAAU,CAiDlG"}
|
package/dist/eval.js
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fixture-only grounding eval harness (regression block alpha, §18.1).
|
|
3
|
+
* No live capture — drives groundInstruction over golden fixtures and reports
|
|
4
|
+
* top-K accuracy. A regression in fixture accuracy blocks alpha release.
|
|
5
|
+
*/
|
|
6
|
+
import { warn } from '@visual-layer/core';
|
|
7
|
+
import { groundInstruction } from './score.js';
|
|
8
|
+
export function evalGrounding(fixtures, opts = {}) {
|
|
9
|
+
const topK = opts.topK ?? 1;
|
|
10
|
+
const minTop1 = opts.minTop1Accuracy ?? 1.0;
|
|
11
|
+
const hitConfidence = opts.hitConfidence ?? 0.5;
|
|
12
|
+
const warnings = [];
|
|
13
|
+
const failures = [];
|
|
14
|
+
let top1Hits = 0;
|
|
15
|
+
let topKHits = 0;
|
|
16
|
+
let valid = 0;
|
|
17
|
+
for (const f of fixtures) {
|
|
18
|
+
if (!f.graph || !f.instruction || !f.expectedInteractableId) {
|
|
19
|
+
throw invalid(`fixture "${f.id}" missing graph/instruction/expectedInteractableId`);
|
|
20
|
+
}
|
|
21
|
+
const ids = new Set(f.graph.interactables.map((i) => i.id));
|
|
22
|
+
if (!ids.has(f.expectedInteractableId)) {
|
|
23
|
+
throw invalid(`fixture "${f.id}" expectedInteractableId not present in graph`);
|
|
24
|
+
}
|
|
25
|
+
const k = f.topK ?? topK;
|
|
26
|
+
const { candidates } = groundInstruction(f.graph, f.instruction, { ...opts, topK: k });
|
|
27
|
+
const returned = candidates.map((c) => c.interactableId);
|
|
28
|
+
const top = candidates[0];
|
|
29
|
+
const matched = candidates.find((c) => c.interactableId === f.expectedInteractableId);
|
|
30
|
+
const matchedConf = matched?.confidence ?? 0;
|
|
31
|
+
valid++;
|
|
32
|
+
// top-1 hit requires correct id at rank 0 AND sufficient confidence — weak matches do not count
|
|
33
|
+
const isTop1Hit = !!top && top.interactableId === f.expectedInteractableId && top.confidence >= hitConfidence;
|
|
34
|
+
if (isTop1Hit)
|
|
35
|
+
top1Hits++;
|
|
36
|
+
// top-K hit requires the expected id within top-K AND the expected's own confidence >= gate
|
|
37
|
+
if (matched && returned.slice(0, k).includes(f.expectedInteractableId) && matchedConf >= hitConfidence)
|
|
38
|
+
topKHits++;
|
|
39
|
+
if (!isTop1Hit) {
|
|
40
|
+
failures.push({ fixtureId: f.id, expectedInteractableId: f.expectedInteractableId, returnedIds: returned });
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
const cases = valid;
|
|
44
|
+
const top1Accuracy = cases === 0 ? 0 : top1Hits / cases;
|
|
45
|
+
const topKAccuracy = cases === 0 ? 0 : topKHits / cases;
|
|
46
|
+
if (cases > 0 && top1Accuracy < minTop1) {
|
|
47
|
+
warnings.push(warn('eval_below_threshold', `top-1 accuracy ${(top1Accuracy * 100).toFixed(1)}% < ${(minTop1 * 100).toFixed(1)}% threshold`, undefined));
|
|
48
|
+
}
|
|
49
|
+
return { cases, top1Accuracy, topKAccuracy, failures, warnings };
|
|
50
|
+
}
|
|
51
|
+
function invalid(detail) {
|
|
52
|
+
const e = new Error(`fixture_invalid: ${detail}`);
|
|
53
|
+
e.code = 'fixture_invalid';
|
|
54
|
+
return e;
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=eval.js.map
|
package/dist/eval.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"eval.js","sourceRoot":"","sources":["../src/eval.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAkC,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAC1E,OAAO,EAAE,iBAAiB,EAAsB,MAAM,YAAY,CAAC;AAiCnE,MAAM,UAAU,aAAa,CAAC,QAAgC,EAAE,OAAoB,EAAE;IACpF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,eAAe,IAAI,GAAG,CAAC;IAC5C,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,GAAG,CAAC;IAChD,MAAM,QAAQ,GAAc,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAkB,EAAE,CAAC;IAEnC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,sBAAsB,EAAE,CAAC;YAC5D,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC,EAAE,oDAAoD,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,sBAAsB,CAAC,EAAE,CAAC;YACvC,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC,EAAE,+CAA+C,CAAC,CAAC;QACjF,CAAC;QAED,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC;QACzB,MAAM,EAAE,UAAU,EAAE,GAAG,iBAAiB,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,WAAW,EAAE,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;QACvF,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;QACzD,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,CAAC,CAAC,sBAAsB,CAAC,CAAC;QACtF,MAAM,WAAW,GAAG,OAAO,EAAE,UAAU,IAAI,CAAC,CAAC;QAE7C,KAAK,EAAE,CAAC;QACR,gGAAgG;QAChG,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,cAAc,KAAK,CAAC,CAAC,sBAAsB,IAAI,GAAG,CAAC,UAAU,IAAI,aAAa,CAAC;QAC9G,IAAI,SAAS;YAAE,QAAQ,EAAE,CAAC;QAC1B,4FAA4F;QAC5F,IAAI,OAAO,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,sBAAsB,CAAC,IAAI,WAAW,IAAI,aAAa;YAAE,QAAQ,EAAE,CAAC;QACnH,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,sBAAsB,EAAE,CAAC,CAAC,sBAAsB,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC9G,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC;IACpB,MAAM,YAAY,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC;IACxD,MAAM,YAAY,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,CAAC;IAExD,IAAI,KAAK,GAAG,CAAC,IAAI,YAAY,GAAG,OAAO,EAAE,CAAC;QACxC,QAAQ,CAAC,IAAI,CACX,IAAI,CAAC,sBAAsB,EAAE,kBAAkB,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,EAAE,SAAS,CAAC,CACzI,CAAC;IACJ,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC;AACnE,CAAC;AAED,SAAS,OAAO,CAAC,MAAc;IAC7B,MAAM,CAAC,GAAG,IAAI,KAAK,CAAC,oBAAoB,MAAM,EAAE,CAA8B,CAAC;IAC/E,CAAC,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAC3B,OAAO,CAAC,CAAC;AACX,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAE1B,eAAO,MAAM,iBAAiB,UAAU,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAE1B,MAAM,CAAC,MAAM,iBAAiB,GAAG,OAAO,CAAC"}
|
package/dist/score.d.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic grounding scorer. Ranks interactables against a natural-language
|
|
3
|
+
* instruction using structure-first signals: text similarity, role match, bbox
|
|
4
|
+
* visibility, source confidence. No model calls (§5 grounding responsibilities).
|
|
5
|
+
*/
|
|
6
|
+
import { type ScreenGraph, type Interactable, type Warning, type Bbox, type ScoreBreakdown } from '@visual-layer/core';
|
|
7
|
+
export interface GroundCandidate {
|
|
8
|
+
interactableId: string;
|
|
9
|
+
role?: string;
|
|
10
|
+
name?: string;
|
|
11
|
+
bbox: Bbox;
|
|
12
|
+
confidence: number;
|
|
13
|
+
scoreBreakdown: ScoreBreakdown;
|
|
14
|
+
sources: Interactable['source'];
|
|
15
|
+
}
|
|
16
|
+
export interface GroundOptions {
|
|
17
|
+
topK?: number;
|
|
18
|
+
/** Minimum composite score to be returned. Default 0.1. */
|
|
19
|
+
minScore?: number;
|
|
20
|
+
/** Weights for the composite blend. Need not sum to 1. */
|
|
21
|
+
weights?: Partial<Record<keyof ScoreBreakdown, number>>;
|
|
22
|
+
}
|
|
23
|
+
export interface GroundResult {
|
|
24
|
+
candidates: GroundCandidate[];
|
|
25
|
+
warnings: Warning[];
|
|
26
|
+
}
|
|
27
|
+
export declare function groundInstruction(graph: ScreenGraph, instruction: string, opts?: GroundOptions): GroundResult;
|
|
28
|
+
//# sourceMappingURL=score.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../src/score.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,OAAO,EACZ,KAAK,IAAI,EACT,KAAK,cAAc,EAMpB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,WAAW,eAAe;IAC9B,cAAc,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,cAAc,CAAC;IAC/B,OAAO,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0DAA0D;IAC1D,OAAO,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,cAAc,EAAE,MAAM,CAAC,CAAC,CAAC;CACzD;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAaD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,GAAE,aAAkB,GAAG,YAAY,CAqEjH"}
|
package/dist/score.js
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic grounding scorer. Ranks interactables against a natural-language
|
|
3
|
+
* instruction using structure-first signals: text similarity, role match, bbox
|
|
4
|
+
* visibility, source confidence. No model calls (§5 grounding responsibilities).
|
|
5
|
+
*/
|
|
6
|
+
import { compareText, contains as bboxContains, hasArea, blendConfidence, warn, } from '@visual-layer/core';
|
|
7
|
+
const DEFAULT_WEIGHTS = { text: 0.6, role: 0.15, bbox: 0.1, source: 0.15 };
|
|
8
|
+
const ROLE_HINTS = [
|
|
9
|
+
{ re: /\b(click|press|tap|hit)\b/i, roles: ['button', 'menuitem', 'menuitemcheckbox', 'menuitemradio', 'tab', 'treeitem'] },
|
|
10
|
+
{ re: /\b(open|follow|visit|go to)\b/i, roles: ['link'] },
|
|
11
|
+
{ re: /\b(type|enter|input|fill|fill in|email|password|write)\b/i, roles: ['textbox', 'searchbox', 'spinbutton'] },
|
|
12
|
+
{ re: /\b(search)\b/i, roles: ['searchbox', 'textbox'] },
|
|
13
|
+
{ re: /\b(select|choose|pick|dropdown)\b/i, roles: ['combobox', 'listbox', 'option'] },
|
|
14
|
+
{ re: /\b(check|uncheck|toggle|checkbox)\b/i, roles: ['checkbox', 'switch', 'menuitemcheckbox'] },
|
|
15
|
+
];
|
|
16
|
+
export function groundInstruction(graph, instruction, opts = {}) {
|
|
17
|
+
const topK = opts.topK ?? 5;
|
|
18
|
+
const minScore = opts.minScore ?? 0.1;
|
|
19
|
+
const weights = { ...DEFAULT_WEIGHTS, ...(opts.weights ?? {}) };
|
|
20
|
+
const warnings = [];
|
|
21
|
+
if (graph.interactables.length === 0) {
|
|
22
|
+
return {
|
|
23
|
+
candidates: [],
|
|
24
|
+
warnings: [warn('no_candidates', 'graph has no interactables', undefined)],
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
const hintedRoles = roleHintsFor(instruction);
|
|
28
|
+
const scored = graph.interactables.map((i) => {
|
|
29
|
+
const label = labelFor(i);
|
|
30
|
+
const text = compareText(instruction, label);
|
|
31
|
+
const role = roleScore(i, hintedRoles);
|
|
32
|
+
const bbox = bboxScore(i, graph.viewport);
|
|
33
|
+
const source = sourceScore(i);
|
|
34
|
+
const confidence = blendConfidence([
|
|
35
|
+
{ value: text, weight: weights.text ?? 0 },
|
|
36
|
+
{ value: role, weight: weights.role ?? 0 },
|
|
37
|
+
{ value: bbox, weight: weights.bbox ?? 0 },
|
|
38
|
+
{ value: source, weight: weights.source ?? 0 },
|
|
39
|
+
]);
|
|
40
|
+
return { inter: i, confidence, breakdown: { text, role, bbox, source } };
|
|
41
|
+
});
|
|
42
|
+
scored.sort((a, b) => b.confidence - a.confidence);
|
|
43
|
+
// duplicate-label detection among high scorers
|
|
44
|
+
const labelCounts = new Map();
|
|
45
|
+
for (const s of scored) {
|
|
46
|
+
if (s.confidence >= minScore) {
|
|
47
|
+
const l = labelFor(s.inter);
|
|
48
|
+
if (l)
|
|
49
|
+
labelCounts.set(l, (labelCounts.get(l) ?? 0) + 1);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
for (const [label, count] of labelCounts) {
|
|
53
|
+
if (count > 1)
|
|
54
|
+
warnings.push(warn('duplicate_label', `"${label}" matched ${count}x`, undefined));
|
|
55
|
+
}
|
|
56
|
+
const candidates = scored
|
|
57
|
+
.filter((s) => s.confidence >= minScore)
|
|
58
|
+
.slice(0, topK)
|
|
59
|
+
.map((s) => ({
|
|
60
|
+
interactableId: s.inter.id,
|
|
61
|
+
role: s.inter.role,
|
|
62
|
+
name: s.inter.name,
|
|
63
|
+
bbox: s.inter.bbox,
|
|
64
|
+
confidence: round3(s.confidence),
|
|
65
|
+
scoreBreakdown: {
|
|
66
|
+
text: round3(s.breakdown.text),
|
|
67
|
+
role: round3(s.breakdown.role),
|
|
68
|
+
bbox: round3(s.breakdown.bbox),
|
|
69
|
+
source: round3(s.breakdown.source),
|
|
70
|
+
},
|
|
71
|
+
sources: s.inter.source,
|
|
72
|
+
}));
|
|
73
|
+
if (candidates.length === 0) {
|
|
74
|
+
warnings.push(warn('no_candidates', `no interactable matched "${instruction}"`, undefined));
|
|
75
|
+
}
|
|
76
|
+
else if (candidates[0].confidence < 0.5) {
|
|
77
|
+
warnings.push(warn('low_confidence', `top candidate confidence ${candidates[0].confidence.toFixed(2)} < 0.5`, undefined));
|
|
78
|
+
}
|
|
79
|
+
return { candidates, warnings };
|
|
80
|
+
}
|
|
81
|
+
function labelFor(i) {
|
|
82
|
+
return (i.text ?? i.name ?? '').toString();
|
|
83
|
+
}
|
|
84
|
+
function roleHintsFor(instruction) {
|
|
85
|
+
for (const h of ROLE_HINTS) {
|
|
86
|
+
if (h.re.test(instruction))
|
|
87
|
+
return h.roles;
|
|
88
|
+
}
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
function roleScore(i, hinted) {
|
|
92
|
+
if (!i.role)
|
|
93
|
+
return 0.5;
|
|
94
|
+
if (hinted === null)
|
|
95
|
+
return 0.7; // neutral — no verb hint in instruction
|
|
96
|
+
return hinted.includes(i.role.toLowerCase()) ? 1.0 : 0.1;
|
|
97
|
+
}
|
|
98
|
+
function bboxScore(i, viewport) {
|
|
99
|
+
if (!hasArea(i.bbox))
|
|
100
|
+
return 0;
|
|
101
|
+
if (!viewport)
|
|
102
|
+
return 0.8; // no viewport to test against
|
|
103
|
+
const vpBbox = [0, 0, viewport.width, viewport.height];
|
|
104
|
+
return bboxContains(vpBbox, i.bbox) ? 1.0 : 0.4;
|
|
105
|
+
}
|
|
106
|
+
function sourceScore(i) {
|
|
107
|
+
// more sources = more corroboration
|
|
108
|
+
if (i.source.includes('dom') && i.source.includes('ax'))
|
|
109
|
+
return 1.0;
|
|
110
|
+
if (i.source.length >= 2)
|
|
111
|
+
return 0.9;
|
|
112
|
+
return 0.8;
|
|
113
|
+
}
|
|
114
|
+
function round3(n) {
|
|
115
|
+
return Math.round(n * 1000) / 1000;
|
|
116
|
+
}
|
|
117
|
+
//# sourceMappingURL=score.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"score.js","sourceRoot":"","sources":["../src/score.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAML,WAAW,EACX,QAAQ,IAAI,YAAY,EACxB,OAAO,EACP,eAAe,EACf,IAAI,GACL,MAAM,oBAAoB,CAAC;AAyB5B,MAAM,eAAe,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAW,CAAC;AAEpF,MAAM,UAAU,GAAmD;IACjE,EAAE,EAAE,EAAE,4BAA4B,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,UAAU,EAAE,kBAAkB,EAAE,eAAe,EAAE,KAAK,EAAE,UAAU,CAAC,EAAE;IAC3H,EAAE,EAAE,EAAE,gCAAgC,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,EAAE;IACzD,EAAE,EAAE,EAAE,2DAA2D,EAAE,KAAK,EAAE,CAAC,SAAS,EAAE,WAAW,EAAE,YAAY,CAAC,EAAE;IAClH,EAAE,EAAE,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,WAAW,EAAE,SAAS,CAAC,EAAE;IACxD,EAAE,EAAE,EAAE,oCAAoC,EAAE,KAAK,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE;IACtF,EAAE,EAAE,EAAE,sCAAsC,EAAE,KAAK,EAAE,CAAC,UAAU,EAAE,QAAQ,EAAE,kBAAkB,CAAC,EAAE;CAClG,CAAC;AAEF,MAAM,UAAU,iBAAiB,CAAC,KAAkB,EAAE,WAAmB,EAAE,OAAsB,EAAE;IACjG,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;IAC5B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC;IACtC,MAAM,OAAO,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;IAChE,MAAM,QAAQ,GAAc,EAAE,CAAC;IAE/B,IAAI,KAAK,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrC,OAAO;YACL,UAAU,EAAE,EAAE;YACd,QAAQ,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,4BAA4B,EAAE,SAAS,CAAC,CAAC;SAC3E,CAAC;IACJ,CAAC;IAED,MAAM,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC,CAAC;IAE9C,MAAM,MAAM,GAAG,KAAK,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC3C,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,IAAI,GAAG,WAAW,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC;QACvC,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC1C,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,UAAU,GAAG,eAAe,CAAC;YACjC,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE;YAC1C,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE;YAC1C,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC,EAAE;YAC1C,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE;SAC/C,CAAC,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAA2B,EAAE,CAAC;IACpG,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IAEnD,+CAA+C;IAC/C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,CAAC,CAAC,UAAU,IAAI,QAAQ,EAAE,CAAC;YAC7B,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC5B,IAAI,CAAC;gBAAE,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IACD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;QACzC,IAAI,KAAK,GAAG,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,EAAE,IAAI,KAAK,aAAa,KAAK,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC;IACnG,CAAC;IAED,MAAM,UAAU,GAAsB,MAAM;SACzC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,QAAQ,CAAC;SACvC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC;SACd,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QACX,cAAc,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE;QAC1B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI;QAClB,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI;QAClB,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI;QAClB,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC;QAChC,cAAc,EAAE;YACd,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;YAC9B,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;YAC9B,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;YAC9B,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC;SACnC;QACD,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM;KACxB,CAAC,CAAC,CAAC;IAEN,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,4BAA4B,WAAW,GAAG,EAAE,SAAS,CAAC,CAAC,CAAC;IAC9F,CAAC;SAAM,IAAI,UAAU,CAAC,CAAC,CAAE,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;QAC3C,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,4BAA4B,UAAU,CAAC,CAAC,CAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;IAC7H,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,QAAQ,CAAC,CAAe;IAC/B,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;AAC7C,CAAC;AAED,SAAS,YAAY,CAAC,WAAmB;IACvC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3B,IAAI,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC;YAAE,OAAO,CAAC,CAAC,KAAK,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,SAAS,CAAC,CAAe,EAAE,MAAuB;IACzD,IAAI,CAAC,CAAC,CAAC,IAAI;QAAE,OAAO,GAAG,CAAC;IACxB,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,GAAG,CAAC,CAAC,wCAAwC;IACzE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;AAC3D,CAAC;AAED,SAAS,SAAS,CAAC,CAAe,EAAE,QAAiC;IACnE,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC/B,IAAI,CAAC,QAAQ;QAAE,OAAO,GAAG,CAAC,CAAC,8BAA8B;IACzD,MAAM,MAAM,GAAS,CAAC,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC7D,OAAO,YAAY,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;AAClD,CAAC;AAED,SAAS,WAAW,CAAC,CAAe;IAClC,oCAAoC;IACpC,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC;IACpE,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,GAAG,CAAC;IACrC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,MAAM,CAAC,CAAS;IACvB,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;AACrC,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@visual-layer/grounding",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Deterministic grounding scorer, compare_graphs, eval_grounding",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"repository": {
|
|
8
|
+
"type": "git",
|
|
9
|
+
"url": "https://gitlab.com/locotine/vision-layer.git",
|
|
10
|
+
"directory": "packages/grounding"
|
|
11
|
+
},
|
|
12
|
+
"publishConfig": {
|
|
13
|
+
"access": "public"
|
|
14
|
+
},
|
|
15
|
+
"main": "./dist/index.js",
|
|
16
|
+
"types": "./dist/index.d.ts",
|
|
17
|
+
"exports": {
|
|
18
|
+
".": {
|
|
19
|
+
"types": "./dist/index.d.ts",
|
|
20
|
+
"import": "./dist/index.js"
|
|
21
|
+
}
|
|
22
|
+
},
|
|
23
|
+
"files": [
|
|
24
|
+
"dist"
|
|
25
|
+
],
|
|
26
|
+
"dependencies": {
|
|
27
|
+
"@visual-layer/core": "1.0.0"
|
|
28
|
+
},
|
|
29
|
+
"scripts": {
|
|
30
|
+
"build": "tsc -b",
|
|
31
|
+
"typecheck": "tsc -b"
|
|
32
|
+
}
|
|
33
|
+
}
|