cbrowser 18.78.0 → 18.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analysis/accessibility-empathy.d.ts.map +1 -1
- package/dist/analysis/accessibility-empathy.js +395 -66
- package/dist/analysis/accessibility-empathy.js.map +1 -1
- package/dist/analysis/agent-ready-audit.d.ts.map +1 -1
- package/dist/analysis/agent-ready-audit.js +78 -0
- package/dist/analysis/agent-ready-audit.js.map +1 -1
- package/dist/analysis/page-understanding.d.ts +139 -1
- package/dist/analysis/page-understanding.d.ts.map +1 -1
- package/dist/analysis/page-understanding.js +179 -12
- package/dist/analysis/page-understanding.js.map +1 -1
- package/dist/artifact-store.d.ts +19 -5
- package/dist/artifact-store.d.ts.map +1 -1
- package/dist/artifact-store.js +41 -11
- package/dist/artifact-store.js.map +1 -1
- package/dist/browser.d.ts +9 -0
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js +81 -5
- package/dist/browser.js.map +1 -1
- package/dist/cli.js +55 -10
- package/dist/cli.js.map +1 -1
- package/dist/cognitive/index.d.ts.map +1 -1
- package/dist/cognitive/index.js +3 -2
- package/dist/cognitive/index.js.map +1 -1
- package/dist/config.d.ts +57 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +234 -7
- package/dist/config.js.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/mcp-server-remote.d.ts.map +1 -1
- package/dist/mcp-server-remote.js +108 -4
- package/dist/mcp-server-remote.js.map +1 -1
- package/dist/mcp-server.d.ts.map +1 -1
- package/dist/mcp-server.js +62 -938
- package/dist/mcp-server.js.map +1 -1
- package/dist/mcp-tools/account-personas.d.ts +41 -0
- package/dist/mcp-tools/account-personas.d.ts.map +1 -0
- package/dist/mcp-tools/account-personas.js +98 -0
- package/dist/mcp-tools/account-personas.js.map +1 -0
- package/dist/mcp-tools/base/analysis-tools.d.ts.map +1 -1
- package/dist/mcp-tools/base/analysis-tools.js +12 -2
- package/dist/mcp-tools/base/analysis-tools.js.map +1 -1
- package/dist/mcp-tools/base/artifact-tools.d.ts +3 -0
- package/dist/mcp-tools/base/artifact-tools.d.ts.map +1 -0
- package/dist/mcp-tools/base/artifact-tools.js +78 -0
- package/dist/mcp-tools/base/artifact-tools.js.map +1 -0
- package/dist/mcp-tools/base/audit-tools.d.ts +11 -0
- package/dist/mcp-tools/base/audit-tools.d.ts.map +1 -1
- package/dist/mcp-tools/base/audit-tools.js +595 -340
- package/dist/mcp-tools/base/audit-tools.js.map +1 -1
- package/dist/mcp-tools/base/browser-management-tools.d.ts.map +1 -1
- package/dist/mcp-tools/base/browser-management-tools.js +18 -0
- package/dist/mcp-tools/base/browser-management-tools.js.map +1 -1
- package/dist/mcp-tools/base/bug-analysis-tools.d.ts.map +1 -1
- package/dist/mcp-tools/base/bug-analysis-tools.js +59 -3
- package/dist/mcp-tools/base/bug-analysis-tools.js.map +1 -1
- package/dist/mcp-tools/base/capture-tools.d.ts +37 -0
- package/dist/mcp-tools/base/capture-tools.d.ts.map +1 -1
- package/dist/mcp-tools/base/capture-tools.js +267 -6
- package/dist/mcp-tools/base/capture-tools.js.map +1 -1
- package/dist/mcp-tools/base/cognitive-tools.d.ts +12 -0
- package/dist/mcp-tools/base/cognitive-tools.d.ts.map +1 -1
- package/dist/mcp-tools/base/cognitive-tools.js +63 -15
- package/dist/mcp-tools/base/cognitive-tools.js.map +1 -1
- package/dist/mcp-tools/base/extraction-tools.d.ts.map +1 -1
- package/dist/mcp-tools/base/extraction-tools.js +33 -2
- package/dist/mcp-tools/base/extraction-tools.js.map +1 -1
- package/dist/mcp-tools/base/index.d.ts +2 -0
- package/dist/mcp-tools/base/index.d.ts.map +1 -1
- package/dist/mcp-tools/base/index.js +9 -0
- package/dist/mcp-tools/base/index.js.map +1 -1
- package/dist/mcp-tools/base/marketing-tools.js +3 -3
- package/dist/mcp-tools/base/marketing-tools.js.map +1 -1
- package/dist/mcp-tools/base/persona-comparison-tools.d.ts.map +1 -1
- package/dist/mcp-tools/base/persona-comparison-tools.js +529 -40
- package/dist/mcp-tools/base/persona-comparison-tools.js.map +1 -1
- package/dist/mcp-tools/base/site-knowledge-tools.d.ts.map +1 -1
- package/dist/mcp-tools/base/site-knowledge-tools.js +36 -3
- package/dist/mcp-tools/base/site-knowledge-tools.js.map +1 -1
- package/dist/mcp-tools/base/values-tools.d.ts +113 -0
- package/dist/mcp-tools/base/values-tools.d.ts.map +1 -1
- package/dist/mcp-tools/base/values-tools.js +868 -78
- package/dist/mcp-tools/base/values-tools.js.map +1 -1
- package/dist/mcp-tools/base/visual-testing-tools.d.ts.map +1 -1
- package/dist/mcp-tools/base/visual-testing-tools.js +130 -10
- package/dist/mcp-tools/base/visual-testing-tools.js.map +1 -1
- package/dist/mcp-tools/index.d.ts +3 -1
- package/dist/mcp-tools/index.d.ts.map +1 -1
- package/dist/mcp-tools/index.js +3 -28
- package/dist/mcp-tools/index.js.map +1 -1
- package/dist/mcp-tools/persona-creation-tools.d.ts.map +1 -1
- package/dist/mcp-tools/persona-creation-tools.js +363 -20
- package/dist/mcp-tools/persona-creation-tools.js.map +1 -1
- package/dist/mcp-tools/persona-lifecycle-tools.d.ts +3 -0
- package/dist/mcp-tools/persona-lifecycle-tools.d.ts.map +1 -0
- package/dist/mcp-tools/persona-lifecycle-tools.js +406 -0
- package/dist/mcp-tools/persona-lifecycle-tools.js.map +1 -0
- package/dist/mcp-tools/screenshot-utils.d.ts.map +1 -1
- package/dist/mcp-tools/screenshot-utils.js +56 -10
- package/dist/mcp-tools/screenshot-utils.js.map +1 -1
- package/dist/mcp-tools/tier-gate.js +2 -2
- package/dist/mcp-tools/tier-gate.js.map +1 -1
- package/dist/mcp-tools/tool-categories.d.ts.map +1 -1
- package/dist/mcp-tools/tool-categories.js +4 -1
- package/dist/mcp-tools/tool-categories.js.map +1 -1
- package/dist/mcp-tools/ui-resources.d.ts +167 -0
- package/dist/mcp-tools/ui-resources.d.ts.map +1 -0
- package/dist/mcp-tools/ui-resources.js +707 -0
- package/dist/mcp-tools/ui-resources.js.map +1 -0
- package/dist/mcp-tools/widget-kit.d.ts +299 -0
- package/dist/mcp-tools/widget-kit.d.ts.map +1 -0
- package/dist/mcp-tools/widget-kit.js +2034 -0
- package/dist/mcp-tools/widget-kit.js.map +1 -0
- package/dist/mcp-ui-resources.d.ts.map +1 -1
- package/dist/mcp-ui-resources.js +14 -4
- package/dist/mcp-ui-resources.js.map +1 -1
- package/dist/persona-questionnaire.d.ts +69 -1
- package/dist/persona-questionnaire.d.ts.map +1 -1
- package/dist/persona-questionnaire.js +177 -76
- package/dist/persona-questionnaire.js.map +1 -1
- package/dist/persona-scope.d.ts +20 -0
- package/dist/persona-scope.d.ts.map +1 -0
- package/dist/persona-scope.js +59 -0
- package/dist/persona-scope.js.map +1 -0
- package/dist/personas.d.ts +51 -17
- package/dist/personas.d.ts.map +1 -1
- package/dist/personas.js +947 -286
- package/dist/personas.js.map +1 -1
- package/dist/recording/capture-player.d.ts +68 -0
- package/dist/recording/capture-player.d.ts.map +1 -0
- package/dist/recording/capture-player.js +298 -0
- package/dist/recording/capture-player.js.map +1 -0
- package/dist/recording/engine.d.ts +52 -0
- package/dist/recording/engine.d.ts.map +1 -1
- package/dist/recording/engine.js +241 -3
- package/dist/recording/engine.js.map +1 -1
- package/dist/recording/inline-player.d.ts +39 -0
- package/dist/recording/inline-player.d.ts.map +1 -0
- package/dist/recording/inline-player.js +272 -0
- package/dist/recording/inline-player.js.map +1 -0
- package/dist/recording/saliency-overlay.d.ts +133 -0
- package/dist/recording/saliency-overlay.d.ts.map +1 -0
- package/dist/recording/saliency-overlay.js +360 -0
- package/dist/recording/saliency-overlay.js.map +1 -0
- package/dist/recording/types.d.ts +32 -0
- package/dist/recording/types.d.ts.map +1 -1
- package/dist/recording/types.js +46 -0
- package/dist/recording/types.js.map +1 -1
- package/dist/site-model/manager.d.ts +20 -0
- package/dist/site-model/manager.d.ts.map +1 -1
- package/dist/site-model/manager.js +52 -0
- package/dist/site-model/manager.js.map +1 -1
- package/dist/trait-reference.d.ts +26 -2
- package/dist/trait-reference.d.ts.map +1 -1
- package/dist/trait-reference.js +129 -19
- package/dist/trait-reference.js.map +1 -1
- package/dist/types.d.ts +105 -18
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/dist/url-resolution.d.ts +58 -0
- package/dist/url-resolution.d.ts.map +1 -0
- package/dist/url-resolution.js +143 -0
- package/dist/url-resolution.js.map +1 -0
- package/dist/values/big-five-inference.d.ts +121 -0
- package/dist/values/big-five-inference.d.ts.map +1 -0
- package/dist/values/big-five-inference.js +241 -0
- package/dist/values/big-five-inference.js.map +1 -0
- package/dist/values/big-five-values.d.ts +76 -0
- package/dist/values/big-five-values.d.ts.map +1 -0
- package/dist/values/big-five-values.js +152 -0
- package/dist/values/big-five-values.js.map +1 -0
- package/dist/values/index.d.ts +1 -1
- package/dist/values/index.d.ts.map +1 -1
- package/dist/values/index.js +1 -1
- package/dist/values/index.js.map +1 -1
- package/dist/values/persona-completeness.d.ts +91 -0
- package/dist/values/persona-completeness.d.ts.map +1 -0
- package/dist/values/persona-completeness.js +114 -0
- package/dist/values/persona-completeness.js.map +1 -0
- package/dist/values/persona-diff.d.ts +50 -0
- package/dist/values/persona-diff.d.ts.map +1 -0
- package/dist/values/persona-diff.js +126 -0
- package/dist/values/persona-diff.js.map +1 -0
- package/dist/values/persona-values.d.ts +16 -0
- package/dist/values/persona-values.d.ts.map +1 -1
- package/dist/values/persona-values.js +33 -0
- package/dist/values/persona-values.js.map +1 -1
- package/dist/values/schwartz-values.d.ts +9 -1
- package/dist/values/schwartz-values.d.ts.map +1 -1
- package/dist/values/schwartz-values.js +10 -2
- package/dist/values/schwartz-values.js.map +1 -1
- package/dist/values/value-mappings.d.ts +41 -2
- package/dist/values/value-mappings.d.ts.map +1 -1
- package/dist/values/value-mappings.js +65 -8
- package/dist/values/value-mappings.js.map +1 -1
- package/dist/visual/attention-quality.d.ts +27 -0
- package/dist/visual/attention-quality.d.ts.map +1 -1
- package/dist/visual/attention-quality.js +164 -2
- package/dist/visual/attention-quality.js.map +1 -1
- package/dist/visual/attention-transport.d.ts +41 -2
- package/dist/visual/attention-transport.d.ts.map +1 -1
- package/dist/visual/attention-transport.js +98 -26
- package/dist/visual/attention-transport.js.map +1 -1
- package/dist/visual/barrier-overlay-html.d.ts +10 -0
- package/dist/visual/barrier-overlay-html.d.ts.map +1 -1
- package/dist/visual/barrier-overlay-html.js.map +1 -1
- package/dist/visual/cognitive-models.d.ts +109 -13
- package/dist/visual/cognitive-models.d.ts.map +1 -1
- package/dist/visual/cognitive-models.js +166 -3
- package/dist/visual/cognitive-models.js.map +1 -1
- package/dist/visual/cognitive-transport-chain.d.ts +149 -5
- package/dist/visual/cognitive-transport-chain.d.ts.map +1 -1
- package/dist/visual/cognitive-transport-chain.js +558 -30
- package/dist/visual/cognitive-transport-chain.js.map +1 -1
- package/dist/visual/cognitive-transport.d.ts +45 -3
- package/dist/visual/cognitive-transport.d.ts.map +1 -1
- package/dist/visual/cognitive-transport.js +126 -12
- package/dist/visual/cognitive-transport.js.map +1 -1
- package/dist/visual/fixation-corpus.d.ts +52 -0
- package/dist/visual/fixation-corpus.d.ts.map +1 -1
- package/dist/visual/fixation-corpus.js +95 -0
- package/dist/visual/fixation-corpus.js.map +1 -1
- package/dist/visual/llm-relevance.d.ts +193 -0
- package/dist/visual/llm-relevance.d.ts.map +1 -0
- package/dist/visual/llm-relevance.js +480 -0
- package/dist/visual/llm-relevance.js.map +1 -0
- package/dist/visual/perceptual-transport.d.ts +60 -0
- package/dist/visual/perceptual-transport.d.ts.map +1 -1
- package/dist/visual/perceptual-transport.js +228 -19
- package/dist/visual/perceptual-transport.js.map +1 -1
- package/dist/visual/persona-context.d.ts +23 -0
- package/dist/visual/persona-context.d.ts.map +1 -0
- package/dist/visual/persona-context.js +44 -0
- package/dist/visual/persona-context.js.map +1 -0
- package/package.json +2 -1
|
@@ -45,6 +45,17 @@ export interface CorpusStimulus {
|
|
|
45
45
|
imagePath: string;
|
|
46
46
|
/** Ground-truth fixation density map per horizon, when the corpus has them. */
|
|
47
47
|
fixationMaps: Partial<Record<FixationHorizon, string>>;
|
|
48
|
+
/**
|
|
49
|
+
* Per-population maps, when the corpus separates groups.
|
|
50
|
+
*
|
|
51
|
+
* This is the field that makes a persona model testable. Without it every
|
|
52
|
+
* persona is scored against one undifferentiated ground truth and is
|
|
53
|
+
* penalised for deviating from the average viewer — which is what a persona
|
|
54
|
+
* model is FOR.
|
|
55
|
+
*/
|
|
56
|
+
groupMaps?: Record<string, string>;
|
|
57
|
+
/** Per-population raw scanpath files, when available. */
|
|
58
|
+
scanpaths?: Record<string, string>;
|
|
48
59
|
}
|
|
49
60
|
export interface CorpusInfo {
|
|
50
61
|
name: string;
|
|
@@ -54,6 +65,8 @@ export interface CorpusInfo {
|
|
|
54
65
|
/** Absent when the corpus ships only density maps and no raw logs. */
|
|
55
66
|
fixationLogDir?: string;
|
|
56
67
|
citation: string;
|
|
68
|
+
/** Set when the corpus's licence is unresolved. Blocks shipping derived results. */
|
|
69
|
+
licenceHold?: string;
|
|
57
70
|
}
|
|
58
71
|
/**
|
|
59
72
|
* One participant fixation as recorded by the tracker.
|
|
@@ -116,6 +129,45 @@ export declare function toGridFixations(fixations: RawFixation[], gridWidth: num
|
|
|
116
129
|
* grid that lands near 1.
|
|
117
130
|
*/
|
|
118
131
|
export declare function fixationsToGrid(fixations: Fixation[], width: number, height: number, sigma?: number): SaliencyGrid;
|
|
132
|
+
/**
|
|
133
|
+
* Load a Saliency4ASD-layout corpus (Duan et al. MMSys 2019 / Gutierrez et al.
|
|
134
|
+
* SPIC 2021): 300 natural-scene images from MIT1003, with fixation maps
|
|
135
|
+
* collected SEPARATELY from 14 children with ASD and 14 typically-developing
|
|
136
|
+
* controls.
|
|
137
|
+
*
|
|
138
|
+
* Separate per-group maps are what make this corpus different in kind. Every
|
|
139
|
+
* other corpus here gives one aggregate ground truth, so a persona model can
|
|
140
|
+
* only be scored on how well it predicts an undifferentiated population — which
|
|
141
|
+
* penalises it for the deviation that is its entire purpose. Two groups permit
|
|
142
|
+
* the crossover question instead: does the ASD model beat the neurotypical model
|
|
143
|
+
* on ASD viewers AND lose to it on TD viewers? Only that interaction
|
|
144
|
+
* distinguishes "models this population" from "is simply a worse model".
|
|
145
|
+
*
|
|
146
|
+
* DATA IMBALANCE, from the dataset README: for some images fewer than 14 ASD
|
|
147
|
+
* subjects contributed, because some children looked away from the screen
|
|
148
|
+
* entirely. Those maps are built from fewer fixations and are correspondingly
|
|
149
|
+
* noisier, so `asdSubjectCount` is surfaced per stimulus — an unweighted mean
|
|
150
|
+
* across images silently treats a 3-subject map as equal evidence to a
|
|
151
|
+
* 14-subject one.
|
|
152
|
+
*
|
|
153
|
+
* LICENCE, unresolved as of 2026-07-29: the Zenodo record says CC BY 4.0 while
|
|
154
|
+
* the deposit's own UsageAgreement.txt forbids commercial use. Held outside the
|
|
155
|
+
* repository pending written clarification from the depositors. Nothing derived
|
|
156
|
+
* from it may ship until that is answered.
|
|
157
|
+
*/
|
|
158
|
+
export declare function loadSaliency4ASDCorpus(root: string): CorpusInfo | null;
|
|
159
|
+
/**
|
|
160
|
+
* Parse a Saliency4ASD scanpath file: `Idx, x, y, duration`, pixel coordinates
|
|
161
|
+
* in the stimulus's own space, concatenated across subjects.
|
|
162
|
+
*
|
|
163
|
+
* Subject boundaries are marked by Idx resetting to 0, which is the only signal
|
|
164
|
+
* the format gives. Counting subjects matters because of the imbalance the
|
|
165
|
+
* README warns about.
|
|
166
|
+
*/
|
|
167
|
+
export declare function parseSaliency4ASDScanpath(filePath: string): {
|
|
168
|
+
fixations: Fixation[];
|
|
169
|
+
subjectCount: number;
|
|
170
|
+
};
|
|
119
171
|
/**
|
|
120
172
|
* Deterministic train/test split.
|
|
121
173
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fixation-corpus.d.ts","sourceRoot":"","sources":["../../src/visual/fixation-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAIH,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAEpE,qEAAqE;AACrE,MAAM,MAAM,eAAe,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AAEjD,MAAM,WAAW,cAAc;IAC7B,wDAAwD;IACxD,EAAE,EAAE,MAAM,CAAC;IACX,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"fixation-corpus.d.ts","sourceRoot":"","sources":["../../src/visual/fixation-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAIH,OAAO,KAAK,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAEpE,qEAAqE;AACrE,MAAM,MAAM,eAAe,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AAEjD,MAAM,WAAW,cAAc;IAC7B,wDAAwD;IACxD,EAAE,EAAE,MAAM,CAAC;IACX,2CAA2C;IAC3C,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,CAAC,CAAC;IACvD;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,sEAAsE;IACtE,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,oFAAoF;IACpF,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,WAAW;IAC1B,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;IACd,wBAAwB;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,oFAAoF;IACpF,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAgC/D;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,GAAG,WAAW,EAAE,CA+ChE;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,OAAO,EAAE,MAAM,GAAG,WAAW,EAAE,CAExF;AAED,4DAA4D;AAC5D,wBAAgB,eAAe,CAC7B,SAAS,EAAE,WAAW,EAAE,EACxB,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,QAAQ,EAAE,CAMZ;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,SAAS,EAAE,QAAQ,EAAE,EACrB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,KAAK,SAAM,GACV,YAAY,CAoBd;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CA0CtE;AAED;;;;;;;GAOG;AACH,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,MAAM,GACf;IAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;IAAC,YAAY,EAAE,MAAM,CAAA;CAAE,CAcjD;AAED;;;;;;;;GAQG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,UAAU,EAClB,YAAY,SAAM,GACjB;IAAE,KAAK,EAAE,cAAc,EAAE,CAAC;IAAC,IAAI,EAAE,cAAc,EAAE,CAAA;CAAE,CAarD"}
|
|
@@ -185,6 +185,101 @@ export function fixationsToGrid(fixations, width, height, sigma = 1.0) {
|
|
|
185
185
|
}
|
|
186
186
|
return { values, width, height };
|
|
187
187
|
}
|
|
188
|
+
/**
|
|
189
|
+
* Load a Saliency4ASD-layout corpus (Duan et al. MMSys 2019 / Gutierrez et al.
|
|
190
|
+
* SPIC 2021): 300 natural-scene images from MIT1003, with fixation maps
|
|
191
|
+
* collected SEPARATELY from 14 children with ASD and 14 typically-developing
|
|
192
|
+
* controls.
|
|
193
|
+
*
|
|
194
|
+
* Separate per-group maps are what make this corpus different in kind. Every
|
|
195
|
+
* other corpus here gives one aggregate ground truth, so a persona model can
|
|
196
|
+
* only be scored on how well it predicts an undifferentiated population — which
|
|
197
|
+
* penalises it for the deviation that is its entire purpose. Two groups permit
|
|
198
|
+
* the crossover question instead: does the ASD model beat the neurotypical model
|
|
199
|
+
* on ASD viewers AND lose to it on TD viewers? Only that interaction
|
|
200
|
+
* distinguishes "models this population" from "is simply a worse model".
|
|
201
|
+
*
|
|
202
|
+
* DATA IMBALANCE, from the dataset README: for some images fewer than 14 ASD
|
|
203
|
+
* subjects contributed, because some children looked away from the screen
|
|
204
|
+
* entirely. Those maps are built from fewer fixations and are correspondingly
|
|
205
|
+
* noisier, so `asdSubjectCount` is surfaced per stimulus — an unweighted mean
|
|
206
|
+
* across images silently treats a 3-subject map as equal evidence to a
|
|
207
|
+
* 14-subject one.
|
|
208
|
+
*
|
|
209
|
+
* LICENCE, unresolved as of 2026-07-29: the Zenodo record says CC BY 4.0 while
|
|
210
|
+
* the deposit's own UsageAgreement.txt forbids commercial use. Held outside the
|
|
211
|
+
* repository pending written clarification from the depositors. Nothing derived
|
|
212
|
+
* from it may ship until that is answered.
|
|
213
|
+
*/
|
|
214
|
+
export function loadSaliency4ASDCorpus(root) {
|
|
215
|
+
const imagesDir = join(root, "Images");
|
|
216
|
+
if (!existsSync(imagesDir))
|
|
217
|
+
return null;
|
|
218
|
+
const stimuli = [];
|
|
219
|
+
for (const file of readdirSync(imagesDir)) {
|
|
220
|
+
const ext = extname(file).toLowerCase();
|
|
221
|
+
if (ext !== ".png" && ext !== ".jpg" && ext !== ".jpeg")
|
|
222
|
+
continue;
|
|
223
|
+
const id = basename(file, ext);
|
|
224
|
+
// Fixmaps are named "<id>_s.png" regardless of the stimulus extension.
|
|
225
|
+
const asdMap = join(root, "ASD_FixMaps", `${id}_s.png`);
|
|
226
|
+
const tdMap = join(root, "TD_FixMaps", `${id}_s.png`);
|
|
227
|
+
const fixationMaps = {};
|
|
228
|
+
// This corpus has no multi-duration split; the single map is recorded under
|
|
229
|
+
// the longest horizon rather than inventing 1s/3s data it does not contain.
|
|
230
|
+
if (existsSync(tdMap))
|
|
231
|
+
fixationMaps["7s"] = tdMap;
|
|
232
|
+
stimuli.push({
|
|
233
|
+
id,
|
|
234
|
+
imagePath: join(imagesDir, file),
|
|
235
|
+
fixationMaps,
|
|
236
|
+
groupMaps: {
|
|
237
|
+
...(existsSync(asdMap) ? { asd: asdMap } : {}),
|
|
238
|
+
...(existsSync(tdMap) ? { td: tdMap } : {}),
|
|
239
|
+
},
|
|
240
|
+
scanpaths: {
|
|
241
|
+
asd: join(root, "ASD", `ASD_scanpath_${id}.txt`),
|
|
242
|
+
td: join(root, "TD", `TD_scanpath_${id}.txt`),
|
|
243
|
+
},
|
|
244
|
+
});
|
|
245
|
+
}
|
|
246
|
+
return {
|
|
247
|
+
name: "Saliency4ASD",
|
|
248
|
+
root,
|
|
249
|
+
stimuli: stimuli.sort((a, b) => a.id.localeCompare(b.id, undefined, { numeric: true })),
|
|
250
|
+
horizons: ["7s"],
|
|
251
|
+
citation: "Duan et al., A Dataset of Eye Movements for the Children with Autism Spectrum Disorder. ACM MMSys 2019; " +
|
|
252
|
+
"Gutierrez et al., Signal Processing: Image Communication 92:116092, 2021. Images from Judd et al., ICCV 2009 (MIT1003).",
|
|
253
|
+
licenceHold: "Zenodo record says CC BY 4.0; the deposit's UsageAgreement.txt forbids commercial use. Unresolved — do not ship derived results.",
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
257
|
+
* Parse a Saliency4ASD scanpath file: `Idx, x, y, duration`, pixel coordinates
|
|
258
|
+
* in the stimulus's own space, concatenated across subjects.
|
|
259
|
+
*
|
|
260
|
+
* Subject boundaries are marked by Idx resetting to 0, which is the only signal
|
|
261
|
+
* the format gives. Counting subjects matters because of the imbalance the
|
|
262
|
+
* README warns about.
|
|
263
|
+
*/
|
|
264
|
+
export function parseSaliency4ASDScanpath(filePath) {
|
|
265
|
+
if (!existsSync(filePath))
|
|
266
|
+
return { fixations: [], subjectCount: 0 };
|
|
267
|
+
const lines = readFileSync(filePath, "utf8").split("\n").map((l) => l.trim()).filter(Boolean);
|
|
268
|
+
const fixations = [];
|
|
269
|
+
let subjectCount = 0;
|
|
270
|
+
for (const line of lines) {
|
|
271
|
+
if (/^idx/i.test(line))
|
|
272
|
+
continue;
|
|
273
|
+
const parts = line.split(",").map((v) => Number(v.trim()));
|
|
274
|
+
if (parts.length < 4 || parts.some((v) => !Number.isFinite(v)))
|
|
275
|
+
continue;
|
|
276
|
+
const [idx, x, y, duration] = parts;
|
|
277
|
+
if (idx === 0)
|
|
278
|
+
subjectCount++;
|
|
279
|
+
fixations.push({ x, y, duration });
|
|
280
|
+
}
|
|
281
|
+
return { fixations, subjectCount };
|
|
282
|
+
}
|
|
188
283
|
/**
|
|
189
284
|
* Deterministic train/test split.
|
|
190
285
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fixation-corpus.js","sourceRoot":"","sources":["../../src/visual/fixation-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"fixation-corpus.js","sourceRoot":"","sources":["../../src/visual/fixation-corpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,IAAI,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AA4D/C;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACvC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAC;IAExC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAsB,EAAE,CAAC;QACxD,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;YAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACxC,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,OAAO;YAAE,SAAS;QAClE,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC/B,MAAM,YAAY,GAA6C,EAAE,CAAC;QAClE,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,eAAe,EAAE,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YACpE,IAAI,UAAU,CAAC,SAAS,CAAC;gBAAE,YAAY,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;QACzD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC;IACvE,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,iBAAiB,CAAC,CAAC;IAC7C,OAAO;QACL,IAAI,EAAE,OAAO;QACb,IAAI;QACJ,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACzD,QAAQ;QACR,cAAc,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACvD,QAAQ,EACN,2FAA2F;KAC9F,CAAC;AACJ,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAe;IAC/C,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAClE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAEhC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACxD,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;IAC9F,MAAM,MAAM,GAAG,GAAG,CAAC,YAAY,CAAC,CAAC;IACjC,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IACxB,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IACxB,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IACxB,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IACxB,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,CAAC;IACxB,MAAM,GAAG,GAAG,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1B,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAEhC,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;IACzE,MAAM,GAAG,GAAkB,EAAE,CAAC;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,EAAE,IAAI,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,GAAG;YAAE,SAAS,CAAC,uBAAuB;QAC3E,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAChC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,SAAS;QACjE,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC;YAAE,SAAS,CAAC,eAAe;QAE/E,4EAA4E;QAC5E,2EAA2E;QAC3E,2EAA2E;QAC3E,MAAM,KAAK,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACxD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,GAAG,KAAK,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;QAC1C,IAAI,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QACrC,IAAI,KAAK;YAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEzB,GAAG,CAAC,IAAI,CAAC;YACP,KAAK;YACL,KAAK;YACL,QAAQ,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC9C,SAAS,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/C,aAAa;YACb,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE;SACzD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAAC,SAAwB,EAAE,OAAe;IACvE,OAAO,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC;AACzD,CAAC;AAED,4DAA4D;AAC5D,MAAM,UAAU,eAAe,CAC7B,SAAwB,EACxB,SAAiB,EACjB,UAAkB;IAElB,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3B,CAAC,EAAE,CAAC,CAAC,KAAK,GAAG,SAAS;QACtB,CAAC,EAAE,CAAC,CAAC,KAAK,GAAG,UAAU;QACvB,QAAQ,EAAE,CAAC,CAAC,QAAQ,GAAG,IAAI;KAC5B,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAC7B,SAAqB,EACrB,KAAa,EACb,MAAc,EACd,KAAK,GAAG,GAAG;IAEX,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC;IAErC,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;QAC1B,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QACf,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;QACf,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC;QAChD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC;QACvD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC;QAChD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC;QACxD,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,KAAK,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC9B,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;gBACrD,MAAM,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AACnC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IACjD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACvC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC;QAAE,OAAO,IAAI,CAAC;IAExC,MAAM,OAAO,GAAqB,EAAE,CAAC;IACrC,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1C,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACxC,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,OAAO;YAAE,SAAS;QAClE,MAAM,EAAE,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC/B,uEAAuE;QACvE,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,YAAY,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;QACtD,MAAM,YAAY,GAA6C,EAAE,CAAC;QAClE,4EAA4E;QAC5E,4EAA4E;QAC5E,IAAI,UAAU,CAAC,KAAK,CAAC;YAAE,YAAY,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QAClD,OAAO,CAAC,IAAI,CAAC;YACX,EAAE;YACF,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC;YAChC,YAAY;YACZ,SAAS,EAAE;gBACT,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC9C,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC5C;YACD,SAAS,EAAE;gBACT,GAAG,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,gBAAgB,EAAE,MAAM,CAAC;gBAChD,EAAE,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,CAAC;aAC9C;SACF,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,IAAI,EAAE,cAAc;QACpB,IAAI;QACJ,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QACvF,QAAQ,EAAE,CAAC,IAAI,CAAC;QAChB,QAAQ,EACN,0GAA0G;YAC1G,yHAAyH;QAC3H,WAAW,EACT,kIAAkI;KACrI,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,yBAAyB,CACvC,QAAgB;IAEhB,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;IACrE,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC9F,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3D,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,SAAS;QACzE,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,GAAG,KAAK,CAAC;QACpC,IAAI,GAAG,KAAK,CAAC;YAAE,YAAY,EAAE,CAAC;QAC9B,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,CAAC;AACrC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,WAAW,CACzB,MAAkB,EAClB,YAAY,GAAG,GAAG;IAElB,MAAM,IAAI,GAAG,CAAC,CAAS,EAAU,EAAE;QACjC,IAAI,CAAC,GAAG,UAAU,CAAC;QACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACrB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;IAChC,CAAC,CAAC;IACF,MAAM,KAAK,GAAqB,EAAE,CAAC;IACnC,MAAM,IAAI,GAAqB,EAAE,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO;QAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnF,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACzB,CAAC"}
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM-judged element relevance for the semantic attention layer.
|
|
3
|
+
*
|
|
4
|
+
* WHAT THIS REPLACES. `computeGoalRelevance` scores an element by counting how
|
|
5
|
+
* many goal keywords appear as substrings of its text, then scaling by 2.0. It
|
|
6
|
+
* has no stemming, no synonyms and no semantics, so for the goal "buy a
|
|
7
|
+
* subscription" the button "Checkout" scores ZERO, "Get Started" scores ZERO and
|
|
8
|
+
* "Upgrade to Pro" scores ZERO — while a footer link reading "Subscription
|
|
9
|
+
* terms" takes the full boost. Substring matching also fires on fragments:
|
|
10
|
+
* "car rental" boosts "careers" and "shopping cart".
|
|
11
|
+
*
|
|
12
|
+
* That layer is supposed to model task-driven attention — Yarbus 1967, one of
|
|
13
|
+
* the most replicated findings in eye-tracking — and it systematically misses
|
|
14
|
+
* exactly the elements a goal-directed user goes for, because real calls to
|
|
15
|
+
* action are rarely worded like the goal.
|
|
16
|
+
*
|
|
17
|
+
* WHAT IT DOES INSTEAD. One LLM call per (persona, goal, element-set) judges
|
|
18
|
+
* every element at once. The model gets the persona's traits, motivational
|
|
19
|
+
* values and description, not just its name, so relevance is judged the way the
|
|
20
|
+
* persona would weigh it: an impatient power-user and an anxious first-timer
|
|
21
|
+
* looking for the same thing do not attend to the same elements, and a semantic
|
|
22
|
+
* vocabulary can express that where six element-type multipliers cannot.
|
|
23
|
+
*
|
|
24
|
+
* ONE CALL PER PAGE, NOT PER ELEMENT. A page carries tens of elements and the
|
|
25
|
+
* overlay runs per frame; per-element or per-frame calls would make this
|
|
26
|
+
* unusable. The cache key covers everything that can change the answer, so a
|
|
27
|
+
* repeated run is free and deterministic — an LLM layer is only usable in a
|
|
28
|
+
* regression context if repeating a run repeats the answer.
|
|
29
|
+
*
|
|
30
|
+
* FAILURE IS A DOWNGRADE, NEVER AN ERROR. No API key, a refused call, or
|
|
31
|
+
* unparseable output falls back to the keyword scorer and reports `source`
|
|
32
|
+
* accordingly. A relevance score whose provenance is unknown is worse than a
|
|
33
|
+
* weak one: the caller must be able to say which method produced the number.
|
|
34
|
+
*
|
|
35
|
+
* @copyright 2026 Alexandria Eden alexandria.shai.eden@gmail.com https://cbrowser.ai
|
|
36
|
+
* @license MIT
|
|
37
|
+
*/
|
|
38
|
+
/**
|
|
39
|
+
* Element as the relevance judge sees it.
|
|
40
|
+
*
|
|
41
|
+
* Carries geometry and colour as well as text, because "does this deserve
|
|
42
|
+
* attention" is not answerable from words alone: a primary action rendered as
|
|
43
|
+
* 11px grey-on-white in a footer and the same words on a filled 200x56 button
|
|
44
|
+
* are different elements to a viewer, and a low-vision or impatient persona
|
|
45
|
+
* weighs that difference heavily.
|
|
46
|
+
*/
|
|
47
|
+
export interface RelevanceElement {
|
|
48
|
+
/** Stable index the model scores against, so scores map back unambiguously. */
|
|
49
|
+
index: number;
|
|
50
|
+
type: string;
|
|
51
|
+
text: string;
|
|
52
|
+
/** Viewport-relative position. Above/below the fold changes attention. */
|
|
53
|
+
x?: number;
|
|
54
|
+
y?: number;
|
|
55
|
+
width?: number;
|
|
56
|
+
height?: number;
|
|
57
|
+
/** Computed foreground colour. */
|
|
58
|
+
color?: string;
|
|
59
|
+
/** Nearest non-transparent background colour. */
|
|
60
|
+
backgroundColor?: string;
|
|
61
|
+
fontSize?: number;
|
|
62
|
+
fontWeight?: number;
|
|
63
|
+
}
|
|
64
|
+
export interface RelevanceContext {
|
|
65
|
+
goal?: string;
|
|
66
|
+
/**
|
|
67
|
+
* Whether the caller is entitled to spend a model call.
|
|
68
|
+
*
|
|
69
|
+
* This layer costs money per page, so entitlement is checked HERE rather than
|
|
70
|
+
* only at the tool boundary: `screen_capture` is a free-tier tool whose
|
|
71
|
+
* `attention_overlay` option would otherwise let any free key spend calls.
|
|
72
|
+
* Absent means allowed, so non-MCP callers (CLI, tests, research harness) are
|
|
73
|
+
* unaffected.
|
|
74
|
+
*/
|
|
75
|
+
entitled?: boolean;
|
|
76
|
+
/**
|
|
77
|
+
* Screenshot bytes. Supplying them lets the judge see the rendered page —
|
|
78
|
+
* visual hierarchy, whether an element actually stands out, what competes
|
|
79
|
+
* with it — rather than inferring layout from numbers.
|
|
80
|
+
*/
|
|
81
|
+
screenshot?: Buffer;
|
|
82
|
+
/** Media type for the screenshot. Defaults to png. */
|
|
83
|
+
screenshotMediaType?: "image/png" | "image/jpeg";
|
|
84
|
+
/** Viewport, so the judge can reason about the fold. */
|
|
85
|
+
viewport?: {
|
|
86
|
+
width: number;
|
|
87
|
+
height: number;
|
|
88
|
+
};
|
|
89
|
+
personaName: string;
|
|
90
|
+
/** Free-text persona description, when the persona carries one. */
|
|
91
|
+
personaDescription?: string;
|
|
92
|
+
/** Cognitive traits, 0-1. Patience, comprehension, risk tolerance and so on. */
|
|
93
|
+
traits?: Record<string, number>;
|
|
94
|
+
/** Schwartz motivational values, 0-1. */
|
|
95
|
+
values?: Record<string, number>;
|
|
96
|
+
}
|
|
97
|
+
export interface RelevanceResult {
|
|
98
|
+
/** True when the judge was given the rendered screenshot. */
|
|
99
|
+
sawScreenshot?: boolean;
|
|
100
|
+
/**
|
|
101
|
+
* Why these scores, in the persona's terms. Two sentences.
|
|
102
|
+
*
|
|
103
|
+
* Requested from the model alongside the scores rather than derived after,
|
|
104
|
+
* because a number with no stated reason cannot be argued with — and the
|
|
105
|
+
* whole point of moving off keyword matching was to get judgements a human
|
|
106
|
+
* can inspect and disagree with.
|
|
107
|
+
*/
|
|
108
|
+
reasoning?: string;
|
|
109
|
+
/** Relevance per element index, 0-1. Missing index means zero. */
|
|
110
|
+
scores: Record<number, number>;
|
|
111
|
+
/** Which method produced these numbers. Never omit this from telemetry. */
|
|
112
|
+
source: "llm" | "keyword-fallback";
|
|
113
|
+
cached: boolean;
|
|
114
|
+
model?: string;
|
|
115
|
+
/** Present when the LLM path was attempted and could not be used. */
|
|
116
|
+
unavailable?: string;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* The keyword scorer this layer replaces, kept as the fallback.
|
|
120
|
+
*
|
|
121
|
+
* Deliberately identical to the historical behaviour so a downgrade changes the
|
|
122
|
+
* source field and nothing else.
|
|
123
|
+
*/
|
|
124
|
+
export declare function keywordRelevance(elementText: string, goal: string): number;
|
|
125
|
+
/**
|
|
126
|
+
* Judge how much each element deserves this persona's attention.
|
|
127
|
+
*
|
|
128
|
+
* Returns 0-1 per element index. Callers multiply their existing element weight
|
|
129
|
+
* by `1 + k * score` rather than replacing it — relevance modulates the
|
|
130
|
+
* element-type prior, it does not override it.
|
|
131
|
+
*/
|
|
132
|
+
export declare function judgeRelevance(elements: RelevanceElement[], ctx: RelevanceContext, getApiKey: () => string | null): Promise<RelevanceResult>;
|
|
133
|
+
/** One judged moment in a recording. */
|
|
134
|
+
export interface JudgedMoment {
|
|
135
|
+
/** Frame index this judgement was made at. */
|
|
136
|
+
frameIndex: number;
|
|
137
|
+
/** Milliseconds from capture start. */
|
|
138
|
+
tMs: number;
|
|
139
|
+
/** Why this moment scored the way it did, in the persona's terms. */
|
|
140
|
+
reasoning?: string;
|
|
141
|
+
/**
|
|
142
|
+
* Which method produced this moment's scores.
|
|
143
|
+
*
|
|
144
|
+
* Carried per-moment because a moment with no elements is otherwise
|
|
145
|
+
* indistinguishable from a moment that was never judged, and the two need
|
|
146
|
+
* completely different fixes.
|
|
147
|
+
*/
|
|
148
|
+
source?: "llm" | "keyword-fallback";
|
|
149
|
+
/** Why the judgement was unusable, when it was. */
|
|
150
|
+
unavailable?: string;
|
|
151
|
+
/** The elements that scored highest, already resolved to text. */
|
|
152
|
+
topElements: Array<{
|
|
153
|
+
text: string;
|
|
154
|
+
type: string;
|
|
155
|
+
score: number;
|
|
156
|
+
}>;
|
|
157
|
+
}
|
|
158
|
+
export interface CaptureSummary {
|
|
159
|
+
narrative: string;
|
|
160
|
+
source: "llm" | "unavailable";
|
|
161
|
+
unavailable?: string;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Narrate a whole recording from the moments that were judged along the way.
|
|
165
|
+
*
|
|
166
|
+
* A per-frame judgement answers "what did this persona attend to here". It does
|
|
167
|
+
* not answer "how did this experience go", which is the question a designer
|
|
168
|
+
* watching a recording actually has — where attention went first, what pulled it
|
|
169
|
+
* away, whether the thing they came to do ever became findable. That is a
|
|
170
|
+
* different question over a different unit (the run, not the frame), so it is a
|
|
171
|
+
* separate call rather than a field on the last frame.
|
|
172
|
+
*
|
|
173
|
+
* Costs one call per capture regardless of length, because it consumes the
|
|
174
|
+
* already-computed keyframe judgements rather than the frames.
|
|
175
|
+
*/
|
|
176
|
+
export declare function summarizeCapture(moments: JudgedMoment[], ctx: RelevanceContext & {
|
|
177
|
+
durationMs: number;
|
|
178
|
+
frameCount: number;
|
|
179
|
+
/**
|
|
180
|
+
* What actually happened, captured from the page.
|
|
181
|
+
*
|
|
182
|
+
* Without this the model had only timestamps, and read the gaps between them
|
|
183
|
+
* as a person deliberating. With it, silence is attributable: nothing
|
|
184
|
+
* happened because nothing was driven, which is a fact about the script and
|
|
185
|
+
* not about a user.
|
|
186
|
+
*/
|
|
187
|
+
actions?: Array<{
|
|
188
|
+
atMs: number;
|
|
189
|
+
type: string;
|
|
190
|
+
label: string;
|
|
191
|
+
}>;
|
|
192
|
+
}, getApiKey: () => string | null): Promise<CaptureSummary>;
|
|
193
|
+
//# sourceMappingURL=llm-relevance.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm-relevance.d.ts","sourceRoot":"","sources":["../../src/visual/llm-relevance.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAcH;;;;;;;;GAQG;AACH,MAAM,WAAW,gBAAgB;IAC/B,+EAA+E;IAC/E,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,0EAA0E;IAC1E,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,kCAAkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iDAAiD;IACjD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;;;;;;OAQG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,mBAAmB,CAAC,EAAE,WAAW,GAAG,YAAY,CAAC;IACjD,wDAAwD;IACxD,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C,WAAW,EAAE,MAAM,CAAC;IACpB,mEAAmE;IACnE,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,gFAAgF;IAChF,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,yCAAyC;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,6DAA6D;IAC7D,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kEAAkE;IAClE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,2EAA2E;IAC3E,MAAM,EAAE,KAAK,GAAG,kBAAkB,CAAC;IACnC,MAAM,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AA0GD;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAQ1E;AAkBD;;;;;;GAMG;AACH,wBAAsB,cAAc,CAClC,QAAQ,EAAE,gBAAgB,EAAE,EAC5B,GAAG,EAAE,gBAAgB,EACrB,SAAS,EAAE,MAAM,MAAM,GAAG,IAAI,GAC7B,OAAO,CAAC,eAAe,CAAC,CAqK1B;AAMD,wCAAwC;AACxC,MAAM,WAAW,YAAY;IAC3B,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,GAAG,EAAE,MAAM,CAAC;IACZ,qEAAqE;IACrE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;;;OAMG;IACH,MAAM,CAAC,EAAE,KAAK,GAAG,kBAAkB,CAAC;IACpC,mDAAmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,kEAAkE;IAClE,WAAW,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACnE;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,KAAK,GAAG,aAAa,CAAC;IAC9B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,gBAAgB,CACpC,OAAO,EAAE,YAAY,EAAE,EACvB,GAAG,EAAE,gBAAgB,GAAG;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAChE,EACD,SAAS,EAAE,MAAM,MAAM,GAAG,IAAI,GAC7B,OAAO,CAAC,cAAc,CAAC,CAwGzB"}
|