jevway 0.0.0-stage → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/INSTALL-FOR-AGENTS.md +138 -0
- package/LICENSE +11 -0
- package/LICENSE.jev-ultrafast +21 -0
- package/README.md +218 -2
- package/dist/browser/bot-wall.js +105 -0
- package/dist/browser/cdp.js +112 -0
- package/dist/browser/chrome.js +147 -0
- package/dist/browser/devices.js +20 -0
- package/dist/browser/frames.js +516 -0
- package/dist/browser/snapshot.js +611 -0
- package/dist/browser/tab.js +938 -0
- package/dist/browser/words.js +51 -0
- package/dist/cli.js +112 -0
- package/dist/jev/action-space.js +48 -0
- package/dist/jev/box.js +58 -0
- package/dist/jev/decide.js +405 -0
- package/dist/jev/instructions.js +67 -0
- package/dist/jev/model.js +189 -0
- package/dist/jev/replay.js +213 -0
- package/dist/jev/secrets.js +60 -0
- package/dist/journeys/diagnostics.js +85 -0
- package/dist/journeys/journey-doc.js +50 -0
- package/dist/journeys/journey-session.js +503 -0
- package/dist/journeys/pick-next-move.js +59 -0
- package/dist/journeys/render-viewer.js +153 -0
- package/dist/journeys/screen-boxes.js +49 -0
- package/dist/journeys/screen-judgments.js +149 -0
- package/dist/mcp.js +361 -0
- package/dist/setup/doctor-command.js +81 -0
- package/dist/setup/doctor.js +163 -0
- package/dist/setup/env-file.js +50 -0
- package/dist/task/checks.js +156 -0
- package/dist/task/evidence.js +96 -0
- package/dist/task/hosts.js +91 -0
- package/dist/task/look.js +17 -0
- package/dist/task/recording-page.js +310 -0
- package/dist/task/recording.js +232 -0
- package/dist/task/run-limit.js +66 -0
- package/dist/task/run-task.js +1607 -0
- package/dist/task/start-task.js +91 -0
- package/dist/task/upload-files.js +43 -0
- package/package.json +80 -4
- package/skill/jevway/SKILL.md +73 -0
|
@@ -0,0 +1,611 @@
|
|
|
1
|
+
(() => {
|
|
2
|
+
if (!document.body) return null;
|
|
3
|
+
// jevway: a frame's ids start at its own base (set by the tab), so ids stay unique across frames.
|
|
4
|
+
const cache = (window.__jevFast ||= { ids: new WeakMap(), nodes: new Map(), next: (window.__jevIdBase || 0) + 1 });
|
|
5
|
+
const identity = (e) => {
|
|
6
|
+
if (!cache.ids.has(e)) cache.ids.set(e, cache.next++);
|
|
7
|
+
const id = cache.ids.get(e);
|
|
8
|
+
cache.nodes.set(id, e);
|
|
9
|
+
return id;
|
|
10
|
+
};
|
|
11
|
+
for (const [id, e] of cache.nodes) if (!e.isConnected) cache.nodes.delete(id);
|
|
12
|
+
const cap = (text, max = 200) => (text.length > max ? text.slice(0, max - 1) + '…' : text);
|
|
13
|
+
const safe = (e) => !['password', 'file', 'hidden'].includes(e.type);
|
|
14
|
+
// jevway: a modal library (the aria-hidden package behind Radix, shadcn and react-aria) hides everything outside its
|
|
15
|
+
// sheet, including the popover it portaled for that sheet. A person still sees and taps that popover, so an element
|
|
16
|
+
// hidden only by that library's marker (data-aria-hidden) counts when it is on top at its centre. A plain aria-hidden
|
|
17
|
+
// or an inert ancestor still hides.
|
|
18
|
+
const hiddenOnlyByModalLibrary = (e) => {
|
|
19
|
+
if (e.closest('[inert]')) return false;
|
|
20
|
+
for (let h = e.closest('[aria-hidden="true"]'); h; h = h.parentElement?.closest('[aria-hidden="true"]'))
|
|
21
|
+
if (h.getAttribute('data-aria-hidden') !== 'true') return false;
|
|
22
|
+
return true;
|
|
23
|
+
};
|
|
24
|
+
const onTop = (e) => {
|
|
25
|
+
const r = e.getBoundingClientRect();
|
|
26
|
+
const hit = document.elementFromPoint(r.x + r.width / 2, r.y + r.height / 2);
|
|
27
|
+
return Boolean(hit) && e.contains(hit);
|
|
28
|
+
};
|
|
29
|
+
const visible = (e) => {
|
|
30
|
+
const hidden = e.closest('[aria-hidden="true"],[inert]');
|
|
31
|
+
if (hidden && !hiddenOnlyByModalLibrary(e)) return false;
|
|
32
|
+
return e.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true }) && (!hidden || onTop(e));
|
|
33
|
+
};
|
|
34
|
+
// jevway: an element whose centre is off-screen (a table row wider than its scroll box) is clicked at the
|
|
35
|
+
// centre of the part the viewport and its scroll containers leave visible. Everything else keeps its own rect.
|
|
36
|
+
cache.clickRect = (e) => {
|
|
37
|
+
const r = e.getBoundingClientRect();
|
|
38
|
+
const cx = r.x + r.width / 2,
|
|
39
|
+
cy = r.y + r.height / 2;
|
|
40
|
+
if (cx >= 0 && cy >= 0 && cx < innerWidth && cy < innerHeight) return r;
|
|
41
|
+
let left = Math.max(r.left, 0),
|
|
42
|
+
top = Math.max(r.top, 0),
|
|
43
|
+
right = Math.min(r.right, innerWidth),
|
|
44
|
+
bottom = Math.min(r.bottom, innerHeight);
|
|
45
|
+
for (let p = e.parentElement; p && p !== document.body; p = p.parentElement) {
|
|
46
|
+
if (getComputedStyle(p).overflow === 'visible') continue;
|
|
47
|
+
const c = p.getBoundingClientRect();
|
|
48
|
+
left = Math.max(left, c.left);
|
|
49
|
+
top = Math.max(top, c.top);
|
|
50
|
+
right = Math.min(right, c.right);
|
|
51
|
+
bottom = Math.min(bottom, c.bottom);
|
|
52
|
+
}
|
|
53
|
+
return { x: left, y: top, width: Math.max(0, right - left), height: Math.max(0, bottom - top) };
|
|
54
|
+
};
|
|
55
|
+
// jevway: display:contents has no box of its own (checkVisibility is false), yet its children render with the parent.
|
|
56
|
+
const renders = (n) =>
|
|
57
|
+
n.checkVisibility({ checkVisibilityCSS: true }) ||
|
|
58
|
+
(getComputedStyle(n).display === 'contents' && Boolean(n.parentElement) && renders(n.parentElement));
|
|
59
|
+
const name = (e, seen = new Set()) => {
|
|
60
|
+
if (!e || seen.has(e)) return '';
|
|
61
|
+
seen.add(e);
|
|
62
|
+
// jevway: accessible names leave out hidden descendants, so a dropdown reads as "Labels", not as its closed menu.
|
|
63
|
+
// An element that is itself hidden (one an aria-labelledby points at) keeps all of its text.
|
|
64
|
+
const shown = renders(e);
|
|
65
|
+
const referenced = (e.getAttribute('aria-labelledby') || '')
|
|
66
|
+
.split(/\s+/)
|
|
67
|
+
.map((id) => name(document.getElementById(id), seen))
|
|
68
|
+
.filter(Boolean)
|
|
69
|
+
.join(' ');
|
|
70
|
+
return (
|
|
71
|
+
referenced ||
|
|
72
|
+
e.getAttribute('aria-label') ||
|
|
73
|
+
[...(e.labels || [])]
|
|
74
|
+
.map((l) => name(l, seen))
|
|
75
|
+
.filter(Boolean)
|
|
76
|
+
.join(' ') ||
|
|
77
|
+
(['button', 'submit', 'reset'].includes(e.type) ? e.value : '') ||
|
|
78
|
+
e.getAttribute('alt') ||
|
|
79
|
+
(e.tagName === 'INPUT'
|
|
80
|
+
? ''
|
|
81
|
+
: [...e.childNodes]
|
|
82
|
+
.map((n) =>
|
|
83
|
+
n.nodeType === 3
|
|
84
|
+
? n.textContent
|
|
85
|
+
: n.nodeType === 1 &&
|
|
86
|
+
n.getAttribute('aria-hidden') !== 'true' &&
|
|
87
|
+
// An SVG <title> is never rendered, yet it is the svg's accessible name.
|
|
88
|
+
(!shown || n.tagName === 'title' || renders(n))
|
|
89
|
+
? name(n, seen)
|
|
90
|
+
: '',
|
|
91
|
+
)
|
|
92
|
+
.join(' ')
|
|
93
|
+
.trim()) ||
|
|
94
|
+
e.getAttribute('title') ||
|
|
95
|
+
e.getAttribute('placeholder') ||
|
|
96
|
+
''
|
|
97
|
+
);
|
|
98
|
+
};
|
|
99
|
+
const roles = [
|
|
100
|
+
'button',
|
|
101
|
+
'link',
|
|
102
|
+
'checkbox',
|
|
103
|
+
'radio',
|
|
104
|
+
'switch',
|
|
105
|
+
'tab',
|
|
106
|
+
'menuitem',
|
|
107
|
+
'menuitemradio',
|
|
108
|
+
'option',
|
|
109
|
+
'gridcell',
|
|
110
|
+
'combobox',
|
|
111
|
+
'textbox',
|
|
112
|
+
'searchbox',
|
|
113
|
+
'spinbutton',
|
|
114
|
+
];
|
|
115
|
+
// jevway: clickable rows and tabbable custom controls (React tables, cards) count when they show a pointer.
|
|
116
|
+
const selector =
|
|
117
|
+
'a[href],button,input,textarea,select,summary,[contenteditable="true"],tr,th,[role="columnheader"],[tabindex="0"],' +
|
|
118
|
+
roles.map((role) => '[role="' + role + '"]').join(',');
|
|
119
|
+
const pointer = (e) => getComputedStyle(e).cursor === 'pointer';
|
|
120
|
+
const role = (e) => {
|
|
121
|
+
if (popupItems.has(e)) return 'option';
|
|
122
|
+
const explicit = e.getAttribute('role');
|
|
123
|
+
if (roles.includes(explicit)) return explicit;
|
|
124
|
+
if (e.tagName === 'BUTTON' || e.tagName === 'SUMMARY') return 'button';
|
|
125
|
+
if (e.tagName === 'TR') return pointer(e) && !e.querySelector('th') ? 'row' : null;
|
|
126
|
+
// jevway: sortable column headers (a pointer or aria-sort); a select-all checkbox header is not one.
|
|
127
|
+
if (e.tagName === 'TH' || explicit === 'columnheader') {
|
|
128
|
+
const sortable = pointer(e) || e.hasAttribute('aria-sort');
|
|
129
|
+
return sortable && !e.querySelector('input,select') ? 'columnheader' : null;
|
|
130
|
+
}
|
|
131
|
+
if (e.getAttribute('tabindex') === '0' && !explicit && !['INPUT', 'TEXTAREA', 'SELECT', 'A'].includes(e.tagName)) {
|
|
132
|
+
return pointer(e) ? 'button' : null;
|
|
133
|
+
}
|
|
134
|
+
// jevway: a dropdown trigger that names itself with a role we don't list ("menu" on Fomantic UI's avatar menu) is
|
|
135
|
+
// still a disclosure button, as its aria-expanded says.
|
|
136
|
+
if (
|
|
137
|
+
explicit &&
|
|
138
|
+
e.getAttribute('tabindex') === '0' &&
|
|
139
|
+
e.hasAttribute('aria-expanded') &&
|
|
140
|
+
!['INPUT', 'TEXTAREA', 'SELECT', 'A'].includes(e.tagName)
|
|
141
|
+
) {
|
|
142
|
+
return 'button';
|
|
143
|
+
}
|
|
144
|
+
if (e.tagName === 'A') return 'link';
|
|
145
|
+
if (e.tagName === 'SELECT') return 'combobox';
|
|
146
|
+
if (e.tagName === 'TEXTAREA' || e.isContentEditable) return 'textbox';
|
|
147
|
+
if (e.tagName === 'INPUT') {
|
|
148
|
+
if (['checkbox', 'radio'].includes(e.type)) return e.type;
|
|
149
|
+
if (['button', 'submit', 'reset', 'image'].includes(e.type)) return 'button';
|
|
150
|
+
if (e.type === 'search') return 'searchbox';
|
|
151
|
+
if (e.type === 'number') return 'spinbutton';
|
|
152
|
+
if (['text', 'email', 'url', 'tel'].includes(e.type)) return 'textbox';
|
|
153
|
+
}
|
|
154
|
+
return null;
|
|
155
|
+
};
|
|
156
|
+
cache.pageKey = () => [
|
|
157
|
+
performance.timeOrigin,
|
|
158
|
+
location.href,
|
|
159
|
+
scrollX,
|
|
160
|
+
scrollY,
|
|
161
|
+
innerWidth,
|
|
162
|
+
innerHeight,
|
|
163
|
+
[...document.querySelectorAll('input,textarea,select')]
|
|
164
|
+
.filter(safe)
|
|
165
|
+
.map((e) => [identity(e), e.value, e.checked, e.selectedIndex, e.disabled, e.readOnly]),
|
|
166
|
+
];
|
|
167
|
+
// jevway: a hover-revealed target is invisible when observed and when checked, so it is guarded by identity, name
|
|
168
|
+
// and row text like any other; the visibility flag keeps a target that disappeared stale.
|
|
169
|
+
cache.guard = (e) => {
|
|
170
|
+
if (!e?.isConnected) return null;
|
|
171
|
+
const scope = e.closest('form,dialog,[role="dialog"],article,li,tr,[role="row"]') || e.parentElement;
|
|
172
|
+
return [
|
|
173
|
+
visible(e),
|
|
174
|
+
identity(e),
|
|
175
|
+
role(e),
|
|
176
|
+
name(e),
|
|
177
|
+
e.value ?? null,
|
|
178
|
+
e.checked ?? null,
|
|
179
|
+
e.selectedIndex ?? null,
|
|
180
|
+
e.readOnly ?? null,
|
|
181
|
+
e.matches(':disabled'),
|
|
182
|
+
e.getAttribute('aria-disabled'),
|
|
183
|
+
e.getAttribute('aria-expanded'),
|
|
184
|
+
e.getAttribute('aria-checked'),
|
|
185
|
+
e.getAttribute('aria-selected'),
|
|
186
|
+
e.getAttribute('href'),
|
|
187
|
+
scope?.innerText?.slice(0, 6000) || '',
|
|
188
|
+
];
|
|
189
|
+
};
|
|
190
|
+
// jevway: when modals are open, only their content is reachable, so read controls and text inside them.
|
|
191
|
+
// Sites often split one sheet into several modal layers (header + panel), so read all outermost ones.
|
|
192
|
+
const modals = [...document.querySelectorAll('[aria-modal="true"],dialog')].filter(
|
|
193
|
+
(m) => (m.tagName !== 'DIALOG' || m.matches(':modal')) && visible(m) && m.getBoundingClientRect().width > 0,
|
|
194
|
+
);
|
|
195
|
+
const roots = modals.length ? modals.filter((m) => !modals.some((o) => o !== m && o.contains(m))) : [document.body];
|
|
196
|
+
const actions = [];
|
|
197
|
+
// jevway: controls inside open shadow roots (web components such as MDN's search button) count too.
|
|
198
|
+
const deepQuery = (root) => [
|
|
199
|
+
...root.querySelectorAll(selector),
|
|
200
|
+
...[...root.querySelectorAll('*')].filter((host) => host.shadowRoot).flatMap((host) => deepQuery(host.shadowRoot)),
|
|
201
|
+
];
|
|
202
|
+
// jevway: open popups (date pickers, custom menus) often build their items from plain pointer-cursor divs:
|
|
203
|
+
// the outermost pointer element with text and no control inside counts as an option.
|
|
204
|
+
const wanted = Array.isArray(window.__jevWanted) ? window.__jevWanted : [];
|
|
205
|
+
const popupItems = new Set();
|
|
206
|
+
const popupSelector = '[role="dialog"],[role="menu"],[role="listbox"],[data-radix-popper-content-wrapper]';
|
|
207
|
+
for (const popup of roots.flatMap((root) => [...root.querySelectorAll(popupSelector)])) {
|
|
208
|
+
if (!visible(popup)) continue;
|
|
209
|
+
for (const e of popup.querySelectorAll('div,span,li,p')) {
|
|
210
|
+
const outermost = !e.parentElement || !pointer(e.parentElement);
|
|
211
|
+
if (pointer(e) && outermost && !e.closest(selector) && !e.querySelector(selector) && e.innerText.trim()) popupItems.add(e);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
// jevway: an unnamed checkbox is named by its row ("Select: Casa Flora"), not its value "on".
|
|
215
|
+
const unnamedToggle = (e, rname) => {
|
|
216
|
+
if (!['checkbox', 'radio', 'switch'].includes(rname)) return false;
|
|
217
|
+
const own = name(e);
|
|
218
|
+
return !own || own === e.value;
|
|
219
|
+
};
|
|
220
|
+
const rowContext = (e) => {
|
|
221
|
+
const text = (e.closest('tr,[role="row"],li,label')?.innerText || '').replace(/\s+/g, ' ').trim().slice(0, 80);
|
|
222
|
+
return text && 'Select: ' + text;
|
|
223
|
+
};
|
|
224
|
+
// jevway: an icon-only control is named the way a person would describe it: by the control it toggles, or by its icon.
|
|
225
|
+
const styleModifier = /^fa-(?:solid|regular|light|brands|lg|xs|sm|fw|\d+x)$/;
|
|
226
|
+
const iconHint = (el) => {
|
|
227
|
+
const testId = (el.getAttribute('data-testid') || '').match(/^icon-(.+)$/);
|
|
228
|
+
if (testId) return testId[1];
|
|
229
|
+
for (const token of (el.getAttribute('class') || '').split(/\s+/)) {
|
|
230
|
+
const hint = !styleModifier.test(token) && token.match(/^(?:lucide|fa|bi|mdi|icon)-(.+)$/);
|
|
231
|
+
if (hint) return hint[1];
|
|
232
|
+
}
|
|
233
|
+
return '';
|
|
234
|
+
};
|
|
235
|
+
const interactive =
|
|
236
|
+
'a,button,input,select,textarea,[contenteditable="true"],' +
|
|
237
|
+
['button', 'link', 'checkbox', 'radio', 'switch', 'tab', 'menuitem', 'option', 'combobox']
|
|
238
|
+
.map((role) => '[role="' + role + '"]')
|
|
239
|
+
.join(',');
|
|
240
|
+
const contextName = (e, hiddenInput, rname) => {
|
|
241
|
+
const own = hiddenInput ? name(hiddenInput).trim() : '';
|
|
242
|
+
if (own) return own;
|
|
243
|
+
const controlled = e.getAttribute('aria-controls');
|
|
244
|
+
if (controlled) {
|
|
245
|
+
const sibling = [...document.querySelectorAll('[aria-controls="' + CSS.escape(controlled) + '"]')].find(
|
|
246
|
+
(other) => other !== e && name(other).trim(),
|
|
247
|
+
);
|
|
248
|
+
const target = sibling ? name(sibling) : document.getElementById(controlled)?.getAttribute('aria-label');
|
|
249
|
+
if (target?.trim()) return 'Toggle ' + target.replace(/\s+/g, ' ').trim();
|
|
250
|
+
}
|
|
251
|
+
const icon = [e, ...e.querySelectorAll('svg,i,span,img')].map(iconHint).find(Boolean);
|
|
252
|
+
if (icon) return icon.replace(/-/g, ' ') + ' icon';
|
|
253
|
+
// jevway: with no name, row or icon, a control is named by the short text right around it, as a person reads it.
|
|
254
|
+
// Text inside another control (a neighbouring "Save") is skipped: it would name this one after that one.
|
|
255
|
+
const textAround = (container) => {
|
|
256
|
+
const walker = document.createTreeWalker(container, NodeFilter.SHOW_TEXT);
|
|
257
|
+
const words = [];
|
|
258
|
+
for (let node = walker.nextNode(); node; node = walker.nextNode()) {
|
|
259
|
+
const control = node.parentElement.closest(interactive);
|
|
260
|
+
const other = control && control !== e && container.contains(control);
|
|
261
|
+
if (!other && node.parentElement.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true }))
|
|
262
|
+
words.push(node.textContent);
|
|
263
|
+
}
|
|
264
|
+
return words.join(' ').replace(/\s+/g, ' ').trim();
|
|
265
|
+
};
|
|
266
|
+
const around = [e.parentElement, e.parentElement?.parentElement]
|
|
267
|
+
.filter((container) => container && container !== document.body && container !== document.documentElement)
|
|
268
|
+
.map(textAround)
|
|
269
|
+
.find((text) => text && text.length <= 80);
|
|
270
|
+
if (!around) return '';
|
|
271
|
+
return unnamedToggle(e, rname) ? 'Select: ' + around : around;
|
|
272
|
+
};
|
|
273
|
+
const queried = roots.flatMap(deepQuery);
|
|
274
|
+
// jevway: a checkbox or radio hidden with opacity or display (Grafana's quick time ranges) is chosen through its
|
|
275
|
+
// visible label, so the label stands in for the input. A visible input keeps its own entry; a disabled one stays out.
|
|
276
|
+
const labelOf = new Map();
|
|
277
|
+
for (const input of queried) {
|
|
278
|
+
if (input.tagName !== 'INPUT' || !['checkbox', 'radio'].includes(input.type) || visible(input)) continue;
|
|
279
|
+
const label = input.labels?.[0];
|
|
280
|
+
if (label && !labelOf.has(label) && !input.matches(':disabled') && !input.closest('[aria-disabled="true"],[inert]'))
|
|
281
|
+
labelOf.set(label, input);
|
|
282
|
+
}
|
|
283
|
+
const candidates = [...queried, ...popupItems, ...labelOf.keys()];
|
|
284
|
+
const eligible = (e) => safe(e) && visible(e) && !e.matches(':disabled') && !e.closest('[aria-disabled="true"]');
|
|
285
|
+
const outOfView = (r) => {
|
|
286
|
+
const x = r.x + r.width / 2,
|
|
287
|
+
y = r.y + r.height / 2;
|
|
288
|
+
return r.width <= 0 || r.height <= 0 || x < 0 || y < 0 || x >= innerWidth || y >= innerHeight;
|
|
289
|
+
};
|
|
290
|
+
// jevway: a control shown only while its container is hovered (a panel's menu, a row's Delete) is in the DOM with
|
|
291
|
+
// visibility:hidden or opacity:0. It is listed as "appears on hover", capped at 20, and the executor moves the pointer
|
|
292
|
+
// onto it before the click. Touch screens list them too: touch emulation still applies :hover to a pointer move, and a
|
|
293
|
+
// tap on the container could activate it. display:none is not laid out, so it does not count.
|
|
294
|
+
const hoverSuffix = ' (appears on hover)';
|
|
295
|
+
const standIns = new Set(labelOf.values());
|
|
296
|
+
let hoverListed = 0;
|
|
297
|
+
const hiddenUntilHover = (e) => {
|
|
298
|
+
if (getComputedStyle(e).visibility === 'hidden') return true;
|
|
299
|
+
for (let p = e, depth = 0; p && depth <= 3; p = p.parentElement, depth++)
|
|
300
|
+
if (getComputedStyle(p).opacity === '0') return true;
|
|
301
|
+
return false;
|
|
302
|
+
};
|
|
303
|
+
const revealedByHover = (e) =>
|
|
304
|
+
hoverListed < 20 &&
|
|
305
|
+
!standIns.has(e) &&
|
|
306
|
+
safe(e) &&
|
|
307
|
+
!e.closest('[aria-hidden="true"],[inert],[aria-disabled="true"]') &&
|
|
308
|
+
!e.matches(':disabled') &&
|
|
309
|
+
hiddenUntilHover(e) &&
|
|
310
|
+
!outOfView(e.getBoundingClientRect()) &&
|
|
311
|
+
Boolean(role(e)) &&
|
|
312
|
+
name(e).trim() !== '';
|
|
313
|
+
// jevway: an off-screen link, button or row holding a value the caller supplied is likely the target, but
|
|
314
|
+
// only when nothing on screen holds it, and at most 5 are listed: a common value ("ERROR") would flood the list.
|
|
315
|
+
const holdsWanted = (e, rname) =>
|
|
316
|
+
['link', 'button', 'row'].includes(rname) && wanted.some((w) => (e.innerText || '').toLowerCase().includes(w));
|
|
317
|
+
const wantedOnscreen =
|
|
318
|
+
wanted.length > 0 &&
|
|
319
|
+
candidates.some((e) => eligible(e) && !outOfView(cache.clickRect(e)) && holdsWanted(e, role(e)));
|
|
320
|
+
let wantedListed = 0;
|
|
321
|
+
// jevway: a link or button in an on-screen table row, scrolled sideways out of view inside the table's scroll box, is
|
|
322
|
+
// still a choice: a person swipes the table to reach it, and clicking scrolls it in. At most 2 per row and 20 in all.
|
|
323
|
+
const rowLinksListed = new Map();
|
|
324
|
+
let sidewaysListed = 0;
|
|
325
|
+
const sidewaysRowOf = (e, rname) => {
|
|
326
|
+
const row = ['link', 'button'].includes(rname) && sidewaysListed < 20 ? e.closest('tr,[role="row"]') : null;
|
|
327
|
+
if (!row || (rowLinksListed.get(row) || 0) >= 2 || e.getBoundingClientRect().width <= 0) return null;
|
|
328
|
+
const rowBox = row.getBoundingClientRect(),
|
|
329
|
+
rowY = rowBox.y + rowBox.height / 2;
|
|
330
|
+
if (rowY < 0 || rowY >= innerHeight) return null;
|
|
331
|
+
for (let p = e.parentElement; p && p !== document.body; p = p.parentElement)
|
|
332
|
+
if (['auto', 'scroll'].includes(getComputedStyle(p).overflowX) && p.scrollWidth > p.clientWidth) return row;
|
|
333
|
+
return null;
|
|
334
|
+
};
|
|
335
|
+
for (const e of candidates) {
|
|
336
|
+
const shown = eligible(e);
|
|
337
|
+
const hover = !shown && revealedByHover(e);
|
|
338
|
+
if (!shown && !hover) continue;
|
|
339
|
+
const hiddenInput = labelOf.get(e),
|
|
340
|
+
r = cache.clickRect(e),
|
|
341
|
+
y = r.y + r.height / 2,
|
|
342
|
+
rname = hiddenInput ? hiddenInput.type : role(e);
|
|
343
|
+
// jevway: a sortable header scrolled sideways out of its table is still a choice; clicking scrolls it in.
|
|
344
|
+
const sideways = rname === 'columnheader' && y >= 0 && y < innerHeight && e.getBoundingClientRect().width > 0;
|
|
345
|
+
const offscreen = outOfView(r);
|
|
346
|
+
const wantedOffscreen = offscreen && !wantedOnscreen && wantedListed < 5 && holdsWanted(e, rname);
|
|
347
|
+
if (wantedOffscreen) wantedListed++;
|
|
348
|
+
const sidewaysRow = rname && offscreen && !wantedOffscreen ? sidewaysRowOf(e, rname) : null;
|
|
349
|
+
if (!rname || (offscreen && !sideways && !wantedOffscreen && !sidewaysRow)) continue;
|
|
350
|
+
if (rname === 'gridcell' && e.querySelector('button,[role="button"]')) continue;
|
|
351
|
+
if (sidewaysRow) {
|
|
352
|
+
sidewaysListed++;
|
|
353
|
+
rowLinksListed.set(sidewaysRow, (rowLinksListed.get(sidewaysRow) || 0) + 1);
|
|
354
|
+
}
|
|
355
|
+
if (hover) hoverListed++;
|
|
356
|
+
const suffix = hover ? hoverSuffix : '';
|
|
357
|
+
const base = {
|
|
358
|
+
node: identity(e),
|
|
359
|
+
role: rname,
|
|
360
|
+
// jevway: a row is what it shows; its name would also pull in nested controls ("Select row"). Labels are
|
|
361
|
+
// capped: table rows holding whole JSON payloads otherwise blow the request past the model's token limit.
|
|
362
|
+
label:
|
|
363
|
+
cap(
|
|
364
|
+
((rname === 'row' || hiddenInput
|
|
365
|
+
? e.innerText.replace(/\s+/g, ' ').trim()
|
|
366
|
+
: unnamedToggle(e, rname)
|
|
367
|
+
? rowContext(e)
|
|
368
|
+
: name(e).replace(/\s+/g, ' ').trim()) ||
|
|
369
|
+
contextName(e, hiddenInput, rname) ||
|
|
370
|
+
rname) + (sidewaysRow ? ' — row: ' + sidewaysRow.innerText.replace(/\s+/g, ' ').trim() : ''),
|
|
371
|
+
200 - suffix.length,
|
|
372
|
+
) + suffix,
|
|
373
|
+
rect: { x: r.x, y: r.y, w: r.width, h: r.height },
|
|
374
|
+
};
|
|
375
|
+
if (hover) base.hover = true;
|
|
376
|
+
for (const key of ['checked', 'selected', 'expanded']) {
|
|
377
|
+
const value = e.getAttribute('aria-' + key);
|
|
378
|
+
if (value !== null) base[key] = value;
|
|
379
|
+
}
|
|
380
|
+
// jevway: a summary's section is open or closed, as its triangle shows; <details> carries no aria-expanded.
|
|
381
|
+
if (e.tagName === 'SUMMARY' && e.parentElement?.tagName === 'DETAILS' && base.expanded === undefined) {
|
|
382
|
+
base.expanded = String(e.parentElement.open);
|
|
383
|
+
}
|
|
384
|
+
const toggle = hiddenInput ?? e;
|
|
385
|
+
if (['checkbox', 'radio'].includes(toggle.type)) base.checked = String(toggle.checked);
|
|
386
|
+
if (e.tagName === 'SELECT') {
|
|
387
|
+
for (const o of e.options)
|
|
388
|
+
if (!o.selected && !o.disabled && !o.closest('optgroup[disabled]'))
|
|
389
|
+
actions.push({
|
|
390
|
+
...base,
|
|
391
|
+
kind: 'select',
|
|
392
|
+
value: o.value,
|
|
393
|
+
current_value: [...e.selectedOptions].map((o) => o.label).join(', '),
|
|
394
|
+
label: base.label + ' → ' + o.label,
|
|
395
|
+
});
|
|
396
|
+
} else {
|
|
397
|
+
const editable =
|
|
398
|
+
!e.readOnly &&
|
|
399
|
+
e.getAttribute('aria-readonly') !== 'true' &&
|
|
400
|
+
(['textbox', 'searchbox', 'spinbutton'].includes(rname) ||
|
|
401
|
+
// jevway: a contenteditable combobox is a search bar too (Langfuse's trace search).
|
|
402
|
+
(rname === 'combobox' && (['INPUT', 'TEXTAREA'].includes(e.tagName) || e.isContentEditable)));
|
|
403
|
+
const value =
|
|
404
|
+
'value' in e ? String(e.value) : e.isContentEditable || rname === 'combobox' ? e.innerText.trim() : '';
|
|
405
|
+
actions.push({ ...base, kind: editable ? 'fill' : 'click', value });
|
|
406
|
+
if (editable) actions.push({ ...base, kind: 'click', value, label: 'Open ' + base.label });
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
// jevway: only when the caller declared files, each enabled file input is a field Jev may upload one to. A hidden one
|
|
410
|
+
// (display:none behind a styled button, a drop zone) is reached the way a person does: through its visible label, else
|
|
411
|
+
// the one visible button in its nearest container, else that visible container. With nothing visible it is left out.
|
|
412
|
+
if (window.__jevFiles === true) {
|
|
413
|
+
const fileInputs = queried.filter((e) => e.tagName === 'INPUT' && e.type === 'file');
|
|
414
|
+
const shows = (e) => e.checkVisibility({ checkOpacity: true, checkVisibilityCSS: true }) && e.getBoundingClientRect().width > 0;
|
|
415
|
+
const standInFor = (input) => {
|
|
416
|
+
if (shows(input) && visible(input)) return input;
|
|
417
|
+
const label = [...(input.labels || [])].find(shows);
|
|
418
|
+
if (label) return label;
|
|
419
|
+
for (let p = input.parentElement, up = 0; p && up < 3 && p !== document.body; p = p.parentElement, up++) {
|
|
420
|
+
if (fileInputs.some((other) => other !== input && p.contains(other))) break;
|
|
421
|
+
const buttons = [...p.querySelectorAll('button,[role="button"],label')].filter(shows);
|
|
422
|
+
if (buttons.length === 1) return buttons[0];
|
|
423
|
+
if (shows(p) && !buttons.length) return p;
|
|
424
|
+
}
|
|
425
|
+
return null;
|
|
426
|
+
};
|
|
427
|
+
for (const input of fileInputs) {
|
|
428
|
+
if (input.matches(':disabled') || input.closest('[aria-disabled="true"],[inert]')) continue;
|
|
429
|
+
const anchor = standInFor(input);
|
|
430
|
+
if (!anchor || outOfView(anchor.getBoundingClientRect())) continue;
|
|
431
|
+
const r = cache.clickRect(anchor),
|
|
432
|
+
own = (anchor === input ? name(input) : name(input) || name(anchor)).replace(/\s+/g, ' ').trim();
|
|
433
|
+
actions.push({
|
|
434
|
+
node: identity(input),
|
|
435
|
+
role: 'file',
|
|
436
|
+
kind: 'upload',
|
|
437
|
+
label: cap(own || input.getAttribute('accept') || 'file', 180) + ' (file field)',
|
|
438
|
+
value: [...(input.files || [])].map((file) => file.name).join(', '),
|
|
439
|
+
rect: { x: r.x, y: r.y, w: r.width, h: r.height },
|
|
440
|
+
});
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
// jevway: controls sharing a role and label (a product grid's "Add to cart" buttons) are told apart by the card or
|
|
444
|
+
// row each sits in, as a person reads the page: "Add to cart — Sauce Labs Bike Light". Only colliding labels change;
|
|
445
|
+
// a nameless control (its label is its bare role) has nothing to share, and members whose cards read the same stay as is.
|
|
446
|
+
const squash = (text) => text.replace(/\s+/g, ' ').trim();
|
|
447
|
+
// an SVG element (a diagram's link or label) has no innerText, only its text content
|
|
448
|
+
const textOf = (e) => e.innerText ?? e.textContent;
|
|
449
|
+
const bareLabel = (a) => (a.hover ? a.label.slice(0, -hoverSuffix.length) : a.label);
|
|
450
|
+
const alone = (list, value) => list.indexOf(value) === list.lastIndexOf(value);
|
|
451
|
+
// The biggest ancestor (up to 4 levels) holding this control and none of the other members: its card or row.
|
|
452
|
+
const cardOf = (e, members) => {
|
|
453
|
+
let card = null;
|
|
454
|
+
for (let p = e.parentElement, depth = 0; p && p !== document.body && depth < 4; p = p.parentElement, depth++) {
|
|
455
|
+
if (members.some((other) => other !== e && p.contains(other))) break;
|
|
456
|
+
card = p;
|
|
457
|
+
}
|
|
458
|
+
return card;
|
|
459
|
+
};
|
|
460
|
+
// Ways to read a card, best first: its heading, a short link (a product name), then its own text outside controls.
|
|
461
|
+
const headings = 'h1,h2,h3,h4,h5,h6,[role="heading"]';
|
|
462
|
+
const apart = (members) => (el) => visible(el) && !members.some((m) => m.contains(el) || el.contains(m));
|
|
463
|
+
const cardReadings = [
|
|
464
|
+
(card, members) => [...card.querySelectorAll(headings)].filter(apart(members)).map((h) => squash(textOf(h))),
|
|
465
|
+
(card, members) =>
|
|
466
|
+
[...card.querySelectorAll('a[href]')]
|
|
467
|
+
.filter(apart(members))
|
|
468
|
+
.map((a) => squash(textOf(a)))
|
|
469
|
+
.filter((text) => text.length <= 60),
|
|
470
|
+
(card, members) => {
|
|
471
|
+
const walker = document.createTreeWalker(card, NodeFilter.SHOW_TEXT);
|
|
472
|
+
const words = [];
|
|
473
|
+
for (let node = walker.nextNode(); node; node = walker.nextNode()) {
|
|
474
|
+
const parent = node.parentElement,
|
|
475
|
+
control = parent.closest(interactive);
|
|
476
|
+
if ((control && card.contains(control)) || members.some((m) => m.contains(parent))) continue;
|
|
477
|
+
if (!parent.closest('script,style,noscript,template') && visible(parent)) words.push(node.textContent);
|
|
478
|
+
}
|
|
479
|
+
return [squash(words.join(' '))];
|
|
480
|
+
},
|
|
481
|
+
];
|
|
482
|
+
const sharedLabels = new Map();
|
|
483
|
+
for (const a of actions) {
|
|
484
|
+
const label = bareLabel(a);
|
|
485
|
+
if (label === a.role || label === 'Open ' + a.role) continue;
|
|
486
|
+
const key = a.role + '\0' + label;
|
|
487
|
+
sharedLabels.set(key, [...(sharedLabels.get(key) || []), a]);
|
|
488
|
+
}
|
|
489
|
+
for (const group of sharedLabels.values()) {
|
|
490
|
+
const members = [...new Set(group.map((a) => cache.nodes.get(a.node)))];
|
|
491
|
+
if (members.length < 2) continue;
|
|
492
|
+
const shared = bareLabel(group[0]).toLowerCase();
|
|
493
|
+
const cards = members.map((e) => cardOf(e, members));
|
|
494
|
+
// The first reading that tells the most members apart wins; a context already in the label adds nothing. Repeated
|
|
495
|
+
// cards all read as something: if one member's card says nothing, the members share a card (two links to one
|
|
496
|
+
// page) and a context from the other's surroundings would mislead.
|
|
497
|
+
let best = { contexts: [], count: 0 };
|
|
498
|
+
for (const read of cardReadings) {
|
|
499
|
+
const contexts = cards.map((card) => {
|
|
500
|
+
const context = card ? cap(read(card, members).find(Boolean) || '', 60) : '';
|
|
501
|
+
return shared.includes(context.toLowerCase()) ? '' : context;
|
|
502
|
+
});
|
|
503
|
+
if (!contexts.every(Boolean)) continue;
|
|
504
|
+
const count = contexts.filter((context) => context && alone(contexts, context)).length;
|
|
505
|
+
if (count > best.count) best = { contexts, count };
|
|
506
|
+
if (count === members.length) break;
|
|
507
|
+
}
|
|
508
|
+
for (const a of group) {
|
|
509
|
+
const context = best.contexts[members.indexOf(cache.nodes.get(a.node))];
|
|
510
|
+
if (!context || !alone(best.contexts, context)) continue;
|
|
511
|
+
const suffix = a.hover ? hoverSuffix : '';
|
|
512
|
+
a.label = cap(bareLabel(a), 200 - suffix.length - context.length - 3) + ' — ' + context + suffix;
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
// jevway: a person sees whether a password field has dots in it; the model gets only that fact, never the value.
|
|
516
|
+
// It goes out as data (`passwordFields`) and as lines before the walked text, so the text cap never cuts them. The value
|
|
517
|
+
// is only compared with '' and never kept.
|
|
518
|
+
const passwordFields = queried
|
|
519
|
+
.filter((e) => e.tagName === 'INPUT' && e.type === 'password' && visible(e))
|
|
520
|
+
.slice(0, 5)
|
|
521
|
+
.map((e) => ({ name: cap(name(e).replace(/\s+/g, ' ').trim(), 60) || 'unnamed', filled: e.value !== '' }));
|
|
522
|
+
const passwordLines = passwordFields.map(
|
|
523
|
+
(field, i) =>
|
|
524
|
+
'Password field "' +
|
|
525
|
+
field.name +
|
|
526
|
+
'": ' +
|
|
527
|
+
(field.filled ? 'filled' : 'empty') +
|
|
528
|
+
(i === 0 ? ' (passwords are typed by the user, not by you)' : ''),
|
|
529
|
+
);
|
|
530
|
+
const words = [];
|
|
531
|
+
// jevway: a frame reads only the part of it the page leaves on screen (set by the tab); a page reads its viewport.
|
|
532
|
+
const view = window.__jevView || { x: 0, y: 0, w: innerWidth, h: innerHeight };
|
|
533
|
+
const range = document.createRange();
|
|
534
|
+
let node,
|
|
535
|
+
length = 0;
|
|
536
|
+
for (const root of roots) {
|
|
537
|
+
const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT);
|
|
538
|
+
while ((node = walker.nextNode()) && length < 6000) {
|
|
539
|
+
const value = node.textContent.trim(),
|
|
540
|
+
parent = node.parentElement;
|
|
541
|
+
if (!value || !parent || parent.closest('script,style,noscript,template') || !visible(parent)) continue;
|
|
542
|
+
range.selectNodeContents(node);
|
|
543
|
+
const r = range.getBoundingClientRect();
|
|
544
|
+
if (r.width > 0 && r.height > 0 && r.bottom > view.y && r.top < view.y + view.h && r.right > view.x && r.left < view.x + view.w) {
|
|
545
|
+
words.push(value);
|
|
546
|
+
length += value.length;
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
const text = [...passwordLines, ...words].join('\n').slice(0, 6000),
|
|
551
|
+
height = document.documentElement.scrollHeight;
|
|
552
|
+
const page_key = cache.pageKey(),
|
|
553
|
+
guards = {};
|
|
554
|
+
for (const a of actions) if (!(a.node in guards)) guards[a.node] = cache.guard(cache.nodes.get(a.node));
|
|
555
|
+
// Compare meaning and identity. Geometry is always resolved and hit-tested just before input.
|
|
556
|
+
const semantics = actions.map(({ rect, ...action }) => action);
|
|
557
|
+
const marker = [
|
|
558
|
+
performance.timeOrigin,
|
|
559
|
+
location.href,
|
|
560
|
+
scrollX,
|
|
561
|
+
scrollY,
|
|
562
|
+
innerWidth,
|
|
563
|
+
innerHeight,
|
|
564
|
+
document.title,
|
|
565
|
+
text,
|
|
566
|
+
semantics,
|
|
567
|
+
page_key[6],
|
|
568
|
+
];
|
|
569
|
+
const omitted_actions = Math.max(0, actions.length - 250);
|
|
570
|
+
actions.splice(250);
|
|
571
|
+
actions.forEach((a, i) => (a.id = 'e' + (i + 1)));
|
|
572
|
+
// jevway: an app shell keeps the document still and scrolls a pane. When the document can't move one way, the
|
|
573
|
+
// largest visible box that can is offered instead, with the wheel aimed at the centre of its visible part (DOM rects
|
|
574
|
+
// only) and a step of 80% of its visible height, so a short pane never steps past a row.
|
|
575
|
+
const panes = [];
|
|
576
|
+
for (const e of [document.body, ...document.body.querySelectorAll('*')]) {
|
|
577
|
+
if (e.scrollHeight <= e.clientHeight + 1 || !/(auto|scroll|overlay)/.test(getComputedStyle(e).overflowY)) continue;
|
|
578
|
+
const r = e.getBoundingClientRect(),
|
|
579
|
+
left = Math.max(r.left, 0),
|
|
580
|
+
top = Math.max(r.top, 0),
|
|
581
|
+
w = Math.min(r.right, innerWidth) - left,
|
|
582
|
+
h = Math.min(r.bottom, innerHeight) - top;
|
|
583
|
+
if (w >= 50 && h >= 50 && visible(e))
|
|
584
|
+
panes.push({ e, w, h, at: { x: Math.round(left + w / 2), y: Math.round(top + h / 2) } });
|
|
585
|
+
}
|
|
586
|
+
panes.sort((a, b) => b.w * b.h - a.w * a.h);
|
|
587
|
+
const paneStep = (pane) => Math.max(100, Math.min(560, Math.round(pane.h * 0.8)));
|
|
588
|
+
const paneDown = panes.find(({ e }) => e.scrollTop + e.clientHeight < e.scrollHeight - 2);
|
|
589
|
+
const paneUp = panes.find(({ e }) => e.scrollTop > 0);
|
|
590
|
+
if (scrollY + innerHeight < height - 2)
|
|
591
|
+
actions.push({ id: 'scroll_down', kind: 'scroll', label: 'Scroll down', delta: 560 });
|
|
592
|
+
else if (paneDown)
|
|
593
|
+
actions.push({ id: 'scroll_down', kind: 'scroll', label: 'Scroll down', delta: paneStep(paneDown), at: paneDown.at });
|
|
594
|
+
if (scrollY > 0) actions.push({ id: 'scroll_up', kind: 'scroll', label: 'Scroll up', delta: -560 });
|
|
595
|
+
else if (paneUp) actions.push({ id: 'scroll_up', kind: 'scroll', label: 'Scroll up', delta: -paneStep(paneUp), at: paneUp.at });
|
|
596
|
+
actions.push({ id: 'wait', kind: 'wait', label: 'Wait for the page to update' });
|
|
597
|
+
return {
|
|
598
|
+
url: location.href,
|
|
599
|
+
title: document.title,
|
|
600
|
+
w: innerWidth,
|
|
601
|
+
h: innerHeight,
|
|
602
|
+
text,
|
|
603
|
+
passwordFields,
|
|
604
|
+
scroll: { y: scrollY, height, panes: panes.reduce((sum, { e }) => sum + e.scrollTop, 0) },
|
|
605
|
+
actions,
|
|
606
|
+
marker,
|
|
607
|
+
page_key,
|
|
608
|
+
guards,
|
|
609
|
+
omitted_actions,
|
|
610
|
+
};
|
|
611
|
+
})();
|