@namzu/browser 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/dist/classifier.d.ts +22 -1
- package/dist/classifier.d.ts.map +1 -1
- package/dist/classifier.js +30 -7
- package/dist/classifier.js.map +1 -1
- package/dist/errors.d.ts +19 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +26 -0
- package/dist/errors.js.map +1 -1
- package/dist/host.d.ts +42 -0
- package/dist/host.d.ts.map +1 -1
- package/dist/host.js +487 -70
- package/dist/host.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/page-scripts.d.ts +20 -0
- package/dist/page-scripts.d.ts.map +1 -1
- package/dist/page-scripts.js +110 -0
- package/dist/page-scripts.js.map +1 -1
- package/dist/reach.d.ts +66 -0
- package/dist/reach.d.ts.map +1 -0
- package/dist/reach.js +189 -0
- package/dist/reach.js.map +1 -0
- package/dist/settle.d.ts +40 -0
- package/dist/settle.d.ts.map +1 -0
- package/dist/settle.js +30 -0
- package/dist/settle.js.map +1 -0
- package/dist/snapshot.d.ts +80 -2
- package/dist/snapshot.d.ts.map +1 -1
- package/dist/snapshot.js +326 -20
- package/dist/snapshot.js.map +1 -1
- package/dist/windows-engine.d.ts +1 -1
- package/dist/windows-engine.d.ts.map +1 -1
- package/dist/windows-engine.js +3 -1
- package/dist/windows-engine.js.map +1 -1
- package/package.json +3 -2
- package/src/classifier.ts +44 -7
- package/src/errors.ts +32 -0
- package/src/host.ts +608 -89
- package/src/index.ts +4 -0
- package/src/page-scripts.ts +108 -0
- package/src/reach.ts +211 -0
- package/src/settle.ts +60 -0
- package/src/snapshot.ts +402 -20
- package/src/windows-engine.ts +4 -1
package/src/snapshot.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { BrowserRefDescription } from '@namzu/sdk'
|
|
1
|
+
import type { BrowserEffect, BrowserRefDescription } from '@namzu/sdk'
|
|
2
2
|
import type { Locator, Page } from 'playwright-core'
|
|
3
3
|
import { isCredentialField } from './classifier.js'
|
|
4
4
|
import { type InvisibleBoxes, type Rect, fieldFacts, invisibleBoxes } from './page-scripts.js'
|
|
@@ -116,15 +116,77 @@ function flags(node: AriaNode): string {
|
|
|
116
116
|
/** What a credential field's value is shown as. */
|
|
117
117
|
export const REDACTED = '[value hidden: password or one-time code]'
|
|
118
118
|
|
|
119
|
+
/** Roles whose children are the choices of a select. */
|
|
120
|
+
const CHOICE_PARENTS = new Set(['combobox', 'listbox'])
|
|
121
|
+
|
|
122
|
+
/** Most options of one select that are listed; the chosen ones are always kept. */
|
|
123
|
+
export const SELECT_OPTIONS_SHOWN = 40
|
|
124
|
+
|
|
125
|
+
/** Text controls: a line shows what they hold, or says they are empty. */
|
|
126
|
+
const TEXT_ROLES = new Set(['textbox', 'searchbox', 'spinbutton', 'combobox'])
|
|
127
|
+
|
|
128
|
+
function chosenOptions(node: AriaNode): string[] {
|
|
129
|
+
const out: string[] = []
|
|
130
|
+
for (const child of node.children ?? []) {
|
|
131
|
+
if (typeof child !== 'string' && child.role === 'option' && child.selected && child.name) {
|
|
132
|
+
out.push(flat(child.name))
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
return out
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function hasOptions(node: AriaNode): boolean {
|
|
139
|
+
return (node.children ?? []).some((c) => typeof c !== 'string' && c.role === 'option')
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* What a control's line says it holds: the text of a field, the chosen
|
|
144
|
+
* option(s) of a select, `undefined` when it holds none. A credential
|
|
145
|
+
* field's value is never shown.
|
|
146
|
+
*/
|
|
147
|
+
function heldValue(item: AriaNode, redact: ReadonlySet<string>): string | undefined {
|
|
148
|
+
if (item.ref !== undefined && redact.has(item.ref)) {
|
|
149
|
+
return item.text === undefined || item.text === '' ? undefined : REDACTED
|
|
150
|
+
}
|
|
151
|
+
if (item.text !== undefined && item.text !== '') return flat(item.text)
|
|
152
|
+
if (item.role === 'combobox' && hasOptions(item)) {
|
|
153
|
+
const chosen = chosenOptions(item)
|
|
154
|
+
return chosen.length > 0 ? chosen.join(', ') : undefined
|
|
155
|
+
}
|
|
156
|
+
return undefined
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** The element's own line, without its indent: role, name, flags, ref and the value it holds. */
|
|
160
|
+
function lineOf(item: AriaNode, redact: ReadonlySet<string>): string {
|
|
161
|
+
let line = `- ${item.role}`
|
|
162
|
+
if (item.name) line += ` ${quote(item.name)}`
|
|
163
|
+
line += flags(item)
|
|
164
|
+
if (item.ref) line += ` [ref=${item.ref}]`
|
|
165
|
+
const value = heldValue(item, redact)
|
|
166
|
+
if (value !== undefined) return `${line}: ${value}`
|
|
167
|
+
if (item.ref !== undefined && (item.role === 'textbox' || item.role === 'searchbox')) {
|
|
168
|
+
return `${line} [empty]`
|
|
169
|
+
}
|
|
170
|
+
return line
|
|
171
|
+
}
|
|
172
|
+
|
|
119
173
|
/**
|
|
120
174
|
* Render nodes as the snapshot text the model reads, one element per line:
|
|
121
175
|
*
|
|
122
176
|
* ```text
|
|
123
177
|
* - link "Pricing" [ref=e7]:
|
|
124
178
|
* - /url: /pricing
|
|
125
|
-
* - textbox "Email" [ref=e9]
|
|
179
|
+
* - textbox "Email" [ref=e9]: ada@example.com
|
|
180
|
+
* - combobox "Plan" [ref=e11]: Gold
|
|
181
|
+
* - option "Free"
|
|
182
|
+
* - option "Gold" [selected]
|
|
126
183
|
* ```
|
|
127
184
|
*
|
|
185
|
+
* Every control's line carries what it holds. A native select lists its
|
|
186
|
+
* options, the chosen ones marked: the browser reports them with empty boxes,
|
|
187
|
+
* which would read as invisible, so an option under a combobox or listbox is
|
|
188
|
+
* never dropped for its size.
|
|
189
|
+
*
|
|
128
190
|
* Dropped: `aria-hidden` subtrees, and the own text of nodes nobody can see
|
|
129
191
|
* (their visible children are kept, one level up). With `rootRef`, only that
|
|
130
192
|
* element's subtree; `undefined` when the ref is not in the tree.
|
|
@@ -138,7 +200,12 @@ export function renderAriaTree(
|
|
|
138
200
|
const lines: string[] = []
|
|
139
201
|
const refs = new Map<string, BrowserRefDescription>()
|
|
140
202
|
|
|
141
|
-
const emit = (
|
|
203
|
+
const emit = (
|
|
204
|
+
item: string | AriaNode,
|
|
205
|
+
depth: number,
|
|
206
|
+
inFrame: boolean,
|
|
207
|
+
parentRole?: string,
|
|
208
|
+
): void => {
|
|
142
209
|
const indent = ' '.repeat(depth)
|
|
143
210
|
if (typeof item === 'string') {
|
|
144
211
|
const text = flat(item)
|
|
@@ -147,38 +214,51 @@ export function renderAriaTree(
|
|
|
147
214
|
}
|
|
148
215
|
if (item.ariaHidden) return
|
|
149
216
|
const childFrame = inFrame || item.role === 'iframe'
|
|
150
|
-
|
|
217
|
+
const choice =
|
|
218
|
+
item.role === 'option' && parentRole !== undefined && CHOICE_PARENTS.has(parentRole)
|
|
219
|
+
if (!choice && unseen(item, hidden, inFrame)) {
|
|
151
220
|
for (const child of item.children ?? []) {
|
|
152
|
-
if (typeof child !== 'string') emit(child, depth, childFrame)
|
|
221
|
+
if (typeof child !== 'string') emit(child, depth, childFrame, item.role)
|
|
153
222
|
}
|
|
154
223
|
return
|
|
155
224
|
}
|
|
156
|
-
let line = `${indent}- ${item.role}`
|
|
157
|
-
if (item.name) line += ` ${quote(item.name)}`
|
|
158
|
-
line += flags(item)
|
|
159
225
|
if (item.ref) {
|
|
160
|
-
line += ` [ref=${item.ref}]`
|
|
161
226
|
refs.set(item.ref, {
|
|
162
227
|
role: item.role,
|
|
163
228
|
...(item.name ? { name: flat(item.name) } : {}),
|
|
164
229
|
})
|
|
165
230
|
}
|
|
166
|
-
const
|
|
167
|
-
item.text === undefined || item.text === ''
|
|
168
|
-
? ''
|
|
169
|
-
: item.ref !== undefined && redact.has(item.ref)
|
|
170
|
-
? REDACTED
|
|
171
|
-
: flat(item.text)
|
|
231
|
+
const holds = heldValue(item, redact) !== undefined
|
|
172
232
|
const hasChildren =
|
|
173
233
|
(item.children?.length ?? 0) > 0 || item.url !== undefined || item.placeholder !== undefined
|
|
174
|
-
|
|
175
|
-
|
|
234
|
+
let line = `${indent}${lineOf(item, redact)}`
|
|
235
|
+
if (!holds && hasChildren) line += ':'
|
|
176
236
|
lines.push(line)
|
|
177
237
|
const inner = `${indent} `
|
|
178
238
|
if (item.url !== undefined) lines.push(`${inner}- /url: ${flat(item.url)}`)
|
|
179
239
|
if (item.placeholder !== undefined)
|
|
180
240
|
lines.push(`${inner}- /placeholder: ${quote(item.placeholder)}`)
|
|
181
|
-
|
|
241
|
+
let children = item.children ?? []
|
|
242
|
+
let omitted = 0
|
|
243
|
+
if (CHOICE_PARENTS.has(item.role)) {
|
|
244
|
+
const options = children.filter((c) => typeof c !== 'string' && c.role === 'option')
|
|
245
|
+
if (options.length > SELECT_OPTIONS_SHOWN) {
|
|
246
|
+
const keep = new Set<string | AriaNode>(
|
|
247
|
+
options.filter((o) => typeof o !== 'string' && o.selected),
|
|
248
|
+
)
|
|
249
|
+
for (const o of options) if (keep.size < SELECT_OPTIONS_SHOWN) keep.add(o)
|
|
250
|
+
omitted = options.length - keep.size
|
|
251
|
+
children = children.filter(
|
|
252
|
+
(c) => typeof c === 'string' || c.role !== 'option' || keep.has(c),
|
|
253
|
+
)
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
for (const child of children) emit(child, depth + 1, childFrame, item.role)
|
|
257
|
+
if (omitted > 0) {
|
|
258
|
+
lines.push(
|
|
259
|
+
`${inner}- (${omitted} more options not listed; read the chosen one with the value action, or find an option by name with snapshot query)`,
|
|
260
|
+
)
|
|
261
|
+
}
|
|
182
262
|
}
|
|
183
263
|
|
|
184
264
|
if (rootRef === undefined) {
|
|
@@ -191,6 +271,15 @@ export function renderAriaTree(
|
|
|
191
271
|
return { text: lines.join('\n'), refs }
|
|
192
272
|
}
|
|
193
273
|
|
|
274
|
+
/** The element a ref names as a one-node tree, so a view can be cut to it; `undefined` when it is not there. */
|
|
275
|
+
export function subtreeOf(
|
|
276
|
+
nodes: readonly (string | AriaNode)[],
|
|
277
|
+
ref: string,
|
|
278
|
+
): readonly AriaNode[] | undefined {
|
|
279
|
+
const found = findRef(nodes, ref, false)
|
|
280
|
+
return found ? [found.node] : undefined
|
|
281
|
+
}
|
|
282
|
+
|
|
194
283
|
function findRef(
|
|
195
284
|
nodes: readonly (string | AriaNode)[],
|
|
196
285
|
ref: string,
|
|
@@ -205,6 +294,301 @@ function findRef(
|
|
|
205
294
|
return undefined
|
|
206
295
|
}
|
|
207
296
|
|
|
297
|
+
// ---------------------------------------------------------------------------
|
|
298
|
+
// Other views of the same tree
|
|
299
|
+
// ---------------------------------------------------------------------------
|
|
300
|
+
|
|
301
|
+
/** Roles a call can act on. */
|
|
302
|
+
const INTERACTIVE_ROLES = new Set([
|
|
303
|
+
'button',
|
|
304
|
+
'link',
|
|
305
|
+
'textbox',
|
|
306
|
+
'searchbox',
|
|
307
|
+
'combobox',
|
|
308
|
+
'checkbox',
|
|
309
|
+
'radio',
|
|
310
|
+
'switch',
|
|
311
|
+
'slider',
|
|
312
|
+
'spinbutton',
|
|
313
|
+
'tab',
|
|
314
|
+
'menuitem',
|
|
315
|
+
'menuitemcheckbox',
|
|
316
|
+
'menuitemradio',
|
|
317
|
+
'option',
|
|
318
|
+
'treeitem',
|
|
319
|
+
'listbox',
|
|
320
|
+
])
|
|
321
|
+
|
|
322
|
+
interface Seen {
|
|
323
|
+
readonly node: AriaNode
|
|
324
|
+
/** Ancestors, outermost first, that the page shows. */
|
|
325
|
+
readonly parents: readonly AriaNode[]
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** Every element the snapshot shows, in document order, with the ancestors that are shown. */
|
|
329
|
+
function shownNodes(
|
|
330
|
+
nodes: readonly (string | AriaNode)[],
|
|
331
|
+
hidden: InvisibleBoxes,
|
|
332
|
+
): { seen: Seen[]; strings: Map<AriaNode, string[]> } {
|
|
333
|
+
const seen: Seen[] = []
|
|
334
|
+
const strings = new Map<AriaNode, string[]>()
|
|
335
|
+
const walk = (
|
|
336
|
+
items: readonly (string | AriaNode)[],
|
|
337
|
+
parents: readonly AriaNode[],
|
|
338
|
+
inFrame: boolean,
|
|
339
|
+
parentRole?: string,
|
|
340
|
+
): void => {
|
|
341
|
+
for (const item of items) {
|
|
342
|
+
if (typeof item === 'string') continue
|
|
343
|
+
if (item.ariaHidden) continue
|
|
344
|
+
const childFrame = inFrame || item.role === 'iframe'
|
|
345
|
+
const choice =
|
|
346
|
+
item.role === 'option' && parentRole !== undefined && CHOICE_PARENTS.has(parentRole)
|
|
347
|
+
if (!choice && unseen(item, hidden, inFrame)) {
|
|
348
|
+
walk(item.children ?? [], parents, childFrame, item.role)
|
|
349
|
+
continue
|
|
350
|
+
}
|
|
351
|
+
seen.push({ node: item, parents })
|
|
352
|
+
strings.set(
|
|
353
|
+
item,
|
|
354
|
+
(item.children ?? []).filter((c): c is string => typeof c === 'string').map(flat),
|
|
355
|
+
)
|
|
356
|
+
walk(item.children ?? [], [...parents, item], childFrame, item.role)
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
walk(nodes, [], false)
|
|
360
|
+
return { seen, strings }
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/** The nearest ancestor that has a name, as `role "name"`; absent when none has. */
|
|
364
|
+
function contextOf(parents: readonly AriaNode[]): string | undefined {
|
|
365
|
+
for (let i = parents.length - 1; i >= 0; i--) {
|
|
366
|
+
const p = parents[i]
|
|
367
|
+
if (p?.name) return `${p.role} ${quote(p.name)}`
|
|
368
|
+
}
|
|
369
|
+
return undefined
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
/** The nearest ancestor that can be acted on or read by ref, as `role "name" [ref=eN]`. */
|
|
373
|
+
function refContextOf(parents: readonly AriaNode[]): string | undefined {
|
|
374
|
+
for (let i = parents.length - 1; i >= 0; i--) {
|
|
375
|
+
const p = parents[i]
|
|
376
|
+
if (p?.ref) return `${p.role}${p.name ? ` ${quote(p.name)}` : ''} [ref=${p.ref}]`
|
|
377
|
+
}
|
|
378
|
+
return undefined
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Only the elements a call can act on, one line each, in page order, with the
|
|
383
|
+
* heading or landmark they sit under. Cheap to read and to re-read; refs are
|
|
384
|
+
* the same as in the full tree.
|
|
385
|
+
*/
|
|
386
|
+
export function renderInteractive(
|
|
387
|
+
nodes: readonly (string | AriaNode)[],
|
|
388
|
+
hidden: InvisibleBoxes = NO_HIDDEN,
|
|
389
|
+
redact: ReadonlySet<string> = new Set(),
|
|
390
|
+
): string {
|
|
391
|
+
const { seen } = shownNodes(nodes, hidden)
|
|
392
|
+
const lines: string[] = []
|
|
393
|
+
for (const { node, parents } of seen) {
|
|
394
|
+
if (!node.ref || !INTERACTIVE_ROLES.has(node.role)) continue
|
|
395
|
+
if (node.role === 'option' && parents.at(-1)?.ref === undefined) continue
|
|
396
|
+
const context = contextOf(parents)
|
|
397
|
+
const url = node.url === undefined ? '' : ` -> ${flat(node.url)}`
|
|
398
|
+
lines.push(`${lineOf(node, redact)}${url}${context ? ` (in ${context})` : ''}`)
|
|
399
|
+
}
|
|
400
|
+
return lines.join('\n')
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/** What a find looks for. */
|
|
404
|
+
export interface FindQuery {
|
|
405
|
+
readonly query?: string
|
|
406
|
+
readonly role?: string
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
/** The result of a find: the matching lines (at most `max`) and how many matched in all. */
|
|
410
|
+
export interface FindResult {
|
|
411
|
+
readonly lines: readonly string[]
|
|
412
|
+
readonly total: number
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
/**
|
|
416
|
+
* Elements whose name, own text or direct text contains `query`
|
|
417
|
+
* (case-insensitive), and/or that have the given role. Each match is its tree
|
|
418
|
+
* line with its ref and the nearest named ancestor, so a model can reach
|
|
419
|
+
* something a long page keeps out of sight. A match with no ref of its own (a
|
|
420
|
+
* table cell, a label) is listed with its text and the nearest ref above it.
|
|
421
|
+
*/
|
|
422
|
+
export function findInTree(
|
|
423
|
+
nodes: readonly (string | AriaNode)[],
|
|
424
|
+
hidden: InvisibleBoxes,
|
|
425
|
+
redact: ReadonlySet<string>,
|
|
426
|
+
find: FindQuery,
|
|
427
|
+
max: number,
|
|
428
|
+
): FindResult {
|
|
429
|
+
const needle = find.query?.toLowerCase().trim()
|
|
430
|
+
const role = find.role?.toLowerCase().trim()
|
|
431
|
+
const { seen, strings } = shownNodes(nodes, hidden)
|
|
432
|
+
const out: string[] = []
|
|
433
|
+
let total = 0
|
|
434
|
+
for (const { node, parents } of seen) {
|
|
435
|
+
if (role && node.role.toLowerCase() !== role) continue
|
|
436
|
+
const own = strings.get(node) ?? []
|
|
437
|
+
if (needle) {
|
|
438
|
+
const haystack = [node.name ?? '', node.text ?? '', ...own].join(' ').toLowerCase()
|
|
439
|
+
if (!haystack.includes(needle)) continue
|
|
440
|
+
} else if (!node.ref) {
|
|
441
|
+
// Without a text to look for, only elements a call can name.
|
|
442
|
+
continue
|
|
443
|
+
}
|
|
444
|
+
total += 1
|
|
445
|
+
if (out.length >= max) continue
|
|
446
|
+
let line = lineOf(node, redact)
|
|
447
|
+
if (!node.name && own.length > 0 && heldValue(node, redact) === undefined) {
|
|
448
|
+
const text = own.join(' ')
|
|
449
|
+
line += `: ${text.length > 160 ? `${text.slice(0, 159)}…` : text}`
|
|
450
|
+
}
|
|
451
|
+
// A match that has no ref of its own (a table cell, a label) points at the
|
|
452
|
+
// nearest element above it that has one.
|
|
453
|
+
const context = node.ref ? contextOf(parents) : (refContextOf(parents) ?? contextOf(parents))
|
|
454
|
+
out.push(`${line}${context ? ` (in ${context})` : ''}`)
|
|
455
|
+
}
|
|
456
|
+
return { lines: out, total }
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
/**
|
|
460
|
+
* The page's text without markup: headings as `#` lines, then paragraphs,
|
|
461
|
+
* list items, cells and the like, one per line. For reading an article, where
|
|
462
|
+
* a tree costs more than it tells.
|
|
463
|
+
*/
|
|
464
|
+
export function renderReading(
|
|
465
|
+
nodes: readonly (string | AriaNode)[],
|
|
466
|
+
hidden: InvisibleBoxes = NO_HIDDEN,
|
|
467
|
+
redact: ReadonlySet<string> = new Set(),
|
|
468
|
+
): string {
|
|
469
|
+
const lines: string[] = []
|
|
470
|
+
const push = (text: string): void => {
|
|
471
|
+
const t = flat(text)
|
|
472
|
+
if (t && lines.at(-1) !== t) lines.push(t)
|
|
473
|
+
}
|
|
474
|
+
const walk = (
|
|
475
|
+
items: readonly (string | AriaNode)[],
|
|
476
|
+
inFrame: boolean,
|
|
477
|
+
parentRole?: string,
|
|
478
|
+
): void => {
|
|
479
|
+
for (const item of items) {
|
|
480
|
+
if (typeof item === 'string') {
|
|
481
|
+
push(item)
|
|
482
|
+
continue
|
|
483
|
+
}
|
|
484
|
+
if (item.ariaHidden) continue
|
|
485
|
+
const childFrame = inFrame || item.role === 'iframe'
|
|
486
|
+
// The choices of a select are not text to read; the chosen one is the control's value.
|
|
487
|
+
if (item.role === 'option' && parentRole !== undefined && CHOICE_PARENTS.has(parentRole))
|
|
488
|
+
continue
|
|
489
|
+
if (unseen(item, hidden, inFrame)) {
|
|
490
|
+
walk(item.children ?? [], childFrame, item.role)
|
|
491
|
+
continue
|
|
492
|
+
}
|
|
493
|
+
const ownText = heldValue(item, redact) ?? ''
|
|
494
|
+
const textChildren = (item.children ?? []).some((c) => typeof c === 'string')
|
|
495
|
+
if (item.role === 'heading')
|
|
496
|
+
push(`${'#'.repeat(Math.min(item.level ?? 1, 6))} ${item.name ?? ownText}`)
|
|
497
|
+
else if (item.role === 'img') {
|
|
498
|
+
if (item.name) push(`[image: ${flat(item.name)}]`)
|
|
499
|
+
} else if (ownText) push(ownText)
|
|
500
|
+
else if (!textChildren && item.name && (item.children?.length ?? 0) === 0) push(item.name)
|
|
501
|
+
walk(item.children ?? [], childFrame, item.role)
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
walk(nodes, false)
|
|
505
|
+
return lines.join('\n')
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
/** What changed between two renderings of a page's tree. */
|
|
509
|
+
export interface LineDiff {
|
|
510
|
+
readonly added: number
|
|
511
|
+
readonly removed: number
|
|
512
|
+
/** The first `shown` lines that appeared, in page order, indentation and the list dash dropped. */
|
|
513
|
+
readonly addedLines: readonly string[]
|
|
514
|
+
readonly removedLines: readonly string[]
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
/** Longest line kept in a diff, in characters. */
|
|
518
|
+
const DIFF_LINE_MAX = 160
|
|
519
|
+
|
|
520
|
+
/**
|
|
521
|
+
* Lines present after and not before (added) and before and not after
|
|
522
|
+
* (removed), compared as a multiset on the trimmed line. Refs of an element
|
|
523
|
+
* that stays do not change between snapshots, so a line that is unchanged is
|
|
524
|
+
* the same text on both sides; a control whose value changed shows as one
|
|
525
|
+
* line removed and one added.
|
|
526
|
+
*/
|
|
527
|
+
export function diffLines(before: string, after: string, shown = 8): LineDiff {
|
|
528
|
+
const norm = (text: string): string[] =>
|
|
529
|
+
text
|
|
530
|
+
.split('\n')
|
|
531
|
+
.map((l) => l.trim().replace(/^- /, ''))
|
|
532
|
+
.filter((l) => l !== '')
|
|
533
|
+
const a = norm(before)
|
|
534
|
+
const b = norm(after)
|
|
535
|
+
const counts = new Map<string, number>()
|
|
536
|
+
for (const l of a) counts.set(l, (counts.get(l) ?? 0) + 1)
|
|
537
|
+
const added: string[] = []
|
|
538
|
+
for (const l of b) {
|
|
539
|
+
const n = counts.get(l) ?? 0
|
|
540
|
+
if (n > 0) counts.set(l, n - 1)
|
|
541
|
+
else added.push(l)
|
|
542
|
+
}
|
|
543
|
+
const left = new Map<string, number>()
|
|
544
|
+
for (const l of b) left.set(l, (left.get(l) ?? 0) + 1)
|
|
545
|
+
const removed: string[] = []
|
|
546
|
+
for (const l of a) {
|
|
547
|
+
const n = left.get(l) ?? 0
|
|
548
|
+
if (n > 0) left.set(l, n - 1)
|
|
549
|
+
else removed.push(l)
|
|
550
|
+
}
|
|
551
|
+
const cut = (l: string): string =>
|
|
552
|
+
l.length > DIFF_LINE_MAX ? `${l.slice(0, DIFF_LINE_MAX - 1)}…` : l
|
|
553
|
+
return {
|
|
554
|
+
added: added.length,
|
|
555
|
+
removed: removed.length,
|
|
556
|
+
addedLines: added.slice(0, shown).map(cut),
|
|
557
|
+
removedLines: removed.slice(0, Math.max(1, Math.floor(shown / 2))).map(cut),
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
/** What the host knew of an action's surroundings, before and after it. */
|
|
562
|
+
export interface EffectFacts {
|
|
563
|
+
readonly beforeTree: string
|
|
564
|
+
readonly afterTree: string
|
|
565
|
+
readonly urlBefore: string
|
|
566
|
+
readonly urlNow: string
|
|
567
|
+
readonly tabsBefore: number
|
|
568
|
+
readonly tabsNow: number
|
|
569
|
+
readonly settle: { readonly quiet: boolean; readonly waitedMs: number; readonly capMs: number }
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
/**
|
|
573
|
+
* The effect of one action, computed from the tree before and after it. The
|
|
574
|
+
* host reads the two trees; everything the model is told about the result
|
|
575
|
+
* comes from here, so a test can feed two trees and check the line.
|
|
576
|
+
*/
|
|
577
|
+
export function buildEffect(facts: EffectFacts, shown = 10): BrowserEffect {
|
|
578
|
+
const diff = diffLines(facts.beforeTree, facts.afterTree, shown)
|
|
579
|
+
const urlChanged = facts.urlNow !== facts.urlBefore
|
|
580
|
+
const changed = diff.added > 0 || diff.removed > 0 || urlChanged
|
|
581
|
+
return {
|
|
582
|
+
...(urlChanged ? { urlChanged: { from: facts.urlBefore, to: facts.urlNow } } : {}),
|
|
583
|
+
added: diff.added,
|
|
584
|
+
removed: diff.removed,
|
|
585
|
+
addedLines: diff.addedLines,
|
|
586
|
+
removedLines: diff.removedLines,
|
|
587
|
+
...(!changed && facts.tabsNow > facts.tabsBefore ? { newTab: true } : {}),
|
|
588
|
+
settle: facts.settle,
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
|
|
208
592
|
/**
|
|
209
593
|
* Take a whole-page snapshot. `ariaSnapshotJSON` returns one root node or a
|
|
210
594
|
* list; either way the result is a list.
|
|
@@ -221,8 +605,6 @@ export async function captureAriaTree(
|
|
|
221
605
|
return { nodes, hidden, redact: await credentialRefs(page, nodes) }
|
|
222
606
|
}
|
|
223
607
|
|
|
224
|
-
const TEXT_ROLES = new Set(['textbox', 'searchbox', 'spinbutton', 'combobox'])
|
|
225
|
-
|
|
226
608
|
/**
|
|
227
609
|
* Refs of text fields that hold a value AND are a password or one-time-code
|
|
228
610
|
* field. The accessibility tree carries an input's value as its text, a
|
package/src/windows-engine.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { Browser, BrowserContext, CDPSession } from 'playwright-core'
|
|
2
2
|
import type { WindowsCdpBrowserPlan } from './detect.js'
|
|
3
3
|
import { type CdpRelay, startCdpRelay } from './relay.js'
|
|
4
4
|
import { type StartWindowsBridgeOptions, WindowsCdpBridge } from './windows-bridge.js'
|
|
@@ -63,6 +63,9 @@ export async function connectWindowsBrowser(
|
|
|
63
63
|
): Promise<WindowsBrowserConnection> {
|
|
64
64
|
const { plan } = options
|
|
65
65
|
const timeoutMs = options.timeoutMs ?? 60_000
|
|
66
|
+
// Loaded on the first Windows-browser launch, not when this module is, and
|
|
67
|
+
// before the bridge starts so a failed load leaves nothing running.
|
|
68
|
+
const { chromium } = await import('playwright-core')
|
|
66
69
|
const bridge = await WindowsCdpBridge.start({
|
|
67
70
|
powershell: plan.powershell,
|
|
68
71
|
env: windowsBridgeEnv(plan, options.env),
|