@mercuryo-ai/agentbrowse 0.2.60 → 0.2.63
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -1
- package/README.md +132 -14
- package/dist/browser-session-state.d.ts +40 -10
- package/dist/browser-session-state.d.ts.map +1 -1
- package/dist/browser-session-state.js +63 -5
- package/dist/commands/act.d.ts.map +1 -1
- package/dist/commands/act.js +548 -535
- package/dist/commands/attach.d.ts +1 -3
- package/dist/commands/attach.d.ts.map +1 -1
- package/dist/commands/attach.js +5 -12
- package/dist/commands/browser-connection-failure.d.ts +9 -0
- package/dist/commands/browser-connection-failure.d.ts.map +1 -0
- package/dist/commands/browser-connection-failure.js +15 -0
- package/dist/commands/browser-status.d.ts +0 -2
- package/dist/commands/browser-status.d.ts.map +1 -1
- package/dist/commands/browser-status.js +27 -37
- package/dist/commands/close.d.ts.map +1 -1
- package/dist/commands/close.js +5 -0
- package/dist/commands/extract.d.ts.map +1 -1
- package/dist/commands/extract.js +147 -144
- package/dist/commands/interaction-kernel.d.ts +1 -1
- package/dist/commands/interaction-kernel.d.ts.map +1 -1
- package/dist/commands/interaction-kernel.js +1 -1
- package/dist/commands/launch.d.ts +0 -1
- package/dist/commands/launch.d.ts.map +1 -1
- package/dist/commands/launch.js +11 -12
- package/dist/commands/navigate.d.ts.map +1 -1
- package/dist/commands/navigate.js +79 -73
- package/dist/commands/observe-accessibility.d.ts.map +1 -1
- package/dist/commands/observe-accessibility.js +36 -2
- package/dist/commands/observe-inventory.d.ts +50 -7
- package/dist/commands/observe-inventory.d.ts.map +1 -1
- package/dist/commands/observe-inventory.js +822 -99
- package/dist/commands/observe-persistence.d.ts.map +1 -1
- package/dist/commands/observe-persistence.js +49 -6
- package/dist/commands/observe-projection.d.ts +6 -2
- package/dist/commands/observe-projection.d.ts.map +1 -1
- package/dist/commands/observe-projection.js +251 -27
- package/dist/commands/observe-semantics.d.ts +1 -0
- package/dist/commands/observe-semantics.d.ts.map +1 -1
- package/dist/commands/observe-semantics.js +541 -135
- package/dist/commands/observe-signals.d.ts +4 -4
- package/dist/commands/observe-signals.d.ts.map +1 -1
- package/dist/commands/observe-signals.js +2 -2
- package/dist/commands/observe-surfaces.d.ts +2 -1
- package/dist/commands/observe-surfaces.d.ts.map +1 -1
- package/dist/commands/observe-surfaces.js +143 -45
- package/dist/commands/observe.d.ts +5 -1
- package/dist/commands/observe.d.ts.map +1 -1
- package/dist/commands/observe.js +266 -274
- package/dist/commands/screenshot.d.ts.map +1 -1
- package/dist/commands/screenshot.js +50 -64
- package/dist/commands/semantic-observe.d.ts.map +1 -1
- package/dist/commands/semantic-observe.js +43 -0
- package/dist/library.d.ts +3 -1
- package/dist/library.d.ts.map +1 -1
- package/dist/library.js +3 -1
- package/dist/match-resolve-fill.d.ts +196 -0
- package/dist/match-resolve-fill.d.ts.map +1 -0
- package/dist/match-resolve-fill.js +700 -0
- package/dist/match-resolve-fill.test-support.d.ts +34 -0
- package/dist/match-resolve-fill.test-support.d.ts.map +1 -0
- package/dist/match-resolve-fill.test-support.js +81 -0
- package/dist/protected-fill.d.ts.map +1 -1
- package/dist/protected-fill.js +46 -7
- package/dist/runtime-protected-state.d.ts.map +1 -1
- package/dist/runtime-protected-state.js +12 -0
- package/dist/runtime-state.d.ts +6 -0
- package/dist/runtime-state.d.ts.map +1 -1
- package/dist/runtime-state.js +6 -0
- package/dist/secrets/form-matcher.d.ts.map +1 -1
- package/dist/secrets/form-matcher.js +76 -27
- package/dist/secrets/protected-exact-value-redaction.d.ts.map +1 -1
- package/dist/secrets/protected-exact-value-redaction.js +6 -0
- package/dist/secrets/protected-fill.js +3 -3
- package/dist/session.d.ts +3 -3
- package/dist/session.d.ts.map +1 -1
- package/dist/session.js +2 -2
- package/dist/solver/browser-launcher.d.ts.map +1 -1
- package/dist/solver/browser-launcher.js +2 -1
- package/dist/sticky-owner-host-entry.d.ts +2 -0
- package/dist/sticky-owner-host-entry.d.ts.map +1 -0
- package/dist/sticky-owner-host-entry.js +97 -0
- package/dist/sticky-owner.d.ts +15 -0
- package/dist/sticky-owner.d.ts.map +1 -0
- package/dist/sticky-owner.js +431 -0
- package/dist/testing.d.ts +1 -0
- package/dist/testing.d.ts.map +1 -1
- package/dist/testing.js +1 -0
- package/docs/README.md +28 -11
- package/docs/api-reference.md +311 -19
- package/docs/assistive-runtime.md +41 -16
- package/docs/configuration.md +36 -4
- package/docs/getting-started.md +73 -5
- package/docs/integration-checklist.md +32 -3
- package/docs/match-resolve-fill.md +699 -0
- package/docs/protected-fill.md +373 -91
- package/docs/testing.md +147 -15
- package/docs/troubleshooting.md +47 -6
- package/examples/README.md +7 -0
- package/examples/match-resolve-fill.ts +107 -0
- package/package.json +4 -2
- package/dist/protected-fill-browser.d.ts +0 -22
- package/dist/protected-fill-browser.d.ts.map +0 -1
- package/dist/protected-fill-browser.js +0 -52
|
@@ -105,115 +105,501 @@ export function observedTargetKey(target) {
|
|
|
105
105
|
target.domSignature ??
|
|
106
106
|
`${target.formSelector ?? 'no-form'}|${target.ordinal ?? 'no-ordinal'}|${target.label ?? ''}`);
|
|
107
107
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
108
|
+
const EXPLICIT_BOUNDARY_OWNER_KINDS = new Set([
|
|
109
|
+
'dialog',
|
|
110
|
+
'listbox',
|
|
111
|
+
'menu',
|
|
112
|
+
'grid',
|
|
113
|
+
'tabpanel',
|
|
114
|
+
'popover',
|
|
115
|
+
'dropdown',
|
|
116
|
+
'datepicker',
|
|
117
|
+
'form',
|
|
118
|
+
]);
|
|
119
|
+
const SHELL_BOUNDARY_OWNER_KINDS = new Set([
|
|
120
|
+
'floating-panel',
|
|
121
|
+
'sticky-panel',
|
|
122
|
+
'section',
|
|
123
|
+
'aside',
|
|
124
|
+
'main',
|
|
125
|
+
]);
|
|
126
|
+
const ACTION_ONLY_OWNER_LABEL_RE = /^(?:select|book|buy|choose|continue|next|more|details|view|open|pay|submit|click)$/i;
|
|
127
|
+
const PRIMARY_ACTION_OWNER_LABEL_RE = /^(?:select|book|buy|choose|continue|next|pay|submit)$/i;
|
|
128
|
+
function normalizeOwnerText(value) {
|
|
129
|
+
const normalized = (value ?? '').replace(/\s+/g, ' ').trim();
|
|
130
|
+
return normalized || undefined;
|
|
111
131
|
}
|
|
112
|
-
function
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
const frameUrl = normalizeEmbeddedSurfaceText(target.frameUrl);
|
|
117
|
-
return frameUrl ? `url:${frameUrl}` : 'top';
|
|
132
|
+
function ownerCandidateSurfaceLabelOf(candidate) {
|
|
133
|
+
return (normalizeOwnerText(candidate.label) ??
|
|
134
|
+
normalizeOwnerText(candidate.text) ??
|
|
135
|
+
normalizeOwnerText(candidate.fallbackLabel));
|
|
118
136
|
}
|
|
119
|
-
function
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
return normalized;
|
|
124
|
-
}
|
|
137
|
+
function ownerCandidateRefinesContext(candidate) {
|
|
138
|
+
const surfaceLabel = ownerCandidateSurfaceLabelOf(candidate);
|
|
139
|
+
if (!surfaceLabel) {
|
|
140
|
+
return false;
|
|
125
141
|
}
|
|
126
|
-
return
|
|
142
|
+
return !ACTION_ONLY_OWNER_LABEL_RE.test(surfaceLabel);
|
|
127
143
|
}
|
|
128
|
-
function
|
|
129
|
-
|
|
130
|
-
|
|
144
|
+
function ownerCandidateContextBlockCountOf(candidate) {
|
|
145
|
+
return Math.max(candidate.directTextBlockCount ?? 0, estimatedTextBlockCount(ownerCandidateSurfaceLabelOf(candidate)));
|
|
146
|
+
}
|
|
147
|
+
function targetSharedOwnerSupportLabelOf(target) {
|
|
148
|
+
return (normalizeOwnerText(target.displayLabel) ??
|
|
149
|
+
normalizeOwnerText(target.label) ??
|
|
150
|
+
normalizeOwnerText(target.currentValue));
|
|
151
|
+
}
|
|
152
|
+
function targetSupportsSharedOwnerPrimaryAction(target) {
|
|
153
|
+
const label = targetSharedOwnerSupportLabelOf(target);
|
|
154
|
+
return Boolean(label && PRIMARY_ACTION_OWNER_LABEL_RE.test(label));
|
|
155
|
+
}
|
|
156
|
+
function targetSupportsSharedOwnerIdentity(target) {
|
|
157
|
+
const label = targetSharedOwnerSupportLabelOf(target);
|
|
158
|
+
if (label && !ACTION_ONLY_OWNER_LABEL_RE.test(label)) {
|
|
159
|
+
return true;
|
|
131
160
|
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
return
|
|
161
|
+
return Boolean(target.selection?.items?.some((item) => {
|
|
162
|
+
const itemLabel = normalizeOwnerText(item.label);
|
|
163
|
+
return Boolean(itemLabel && !ACTION_ONLY_OWNER_LABEL_RE.test(itemLabel));
|
|
164
|
+
}));
|
|
165
|
+
}
|
|
166
|
+
function ownerCandidateKindHintOf(candidate) {
|
|
167
|
+
return normalizeOwnerText(candidate.surfaceKind)?.toLowerCase();
|
|
168
|
+
}
|
|
169
|
+
function ownerCandidateKindOf(candidate) {
|
|
170
|
+
return ownerCandidateKindHintOf(candidate) ?? normalizeOwnerText(candidate.kind)?.toLowerCase();
|
|
171
|
+
}
|
|
172
|
+
function defaultPriorityForOwnerKind(kind) {
|
|
173
|
+
switch (kind) {
|
|
174
|
+
case 'dialog':
|
|
175
|
+
return 100;
|
|
176
|
+
case 'listbox':
|
|
177
|
+
case 'menu':
|
|
178
|
+
return 95;
|
|
179
|
+
case 'floating-panel':
|
|
180
|
+
return 92;
|
|
181
|
+
case 'datepicker':
|
|
182
|
+
return 90;
|
|
183
|
+
case 'sticky-panel':
|
|
184
|
+
return 88;
|
|
185
|
+
case 'popover':
|
|
186
|
+
case 'dropdown':
|
|
187
|
+
return 85;
|
|
188
|
+
case 'grid':
|
|
189
|
+
case 'tabpanel':
|
|
190
|
+
return 80;
|
|
191
|
+
case 'form':
|
|
192
|
+
return 70;
|
|
193
|
+
case 'card':
|
|
194
|
+
return 58;
|
|
195
|
+
case 'group':
|
|
196
|
+
return 55;
|
|
197
|
+
case 'section':
|
|
198
|
+
case 'aside':
|
|
199
|
+
return 52;
|
|
200
|
+
case 'listitem':
|
|
201
|
+
case 'row':
|
|
202
|
+
return 48;
|
|
203
|
+
default:
|
|
204
|
+
return undefined;
|
|
135
205
|
}
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
206
|
+
}
|
|
207
|
+
function estimatedTextBlockCount(value) {
|
|
208
|
+
const normalized = normalizeOwnerText(value);
|
|
209
|
+
if (!normalized) {
|
|
210
|
+
return 0;
|
|
140
211
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
212
|
+
const splitHints = normalized
|
|
213
|
+
.split(/(?: · | \| |, | {2,}| (?=\d{1,2}:\d{2}\b)| (?=\d+\s?[€$₽£]\b))/)
|
|
214
|
+
.map((part) => part.trim())
|
|
215
|
+
.filter(Boolean);
|
|
216
|
+
if (splitHints.length >= 2) {
|
|
217
|
+
return Math.min(splitHints.length, 4);
|
|
145
218
|
}
|
|
146
|
-
return
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
219
|
+
return normalized.length >= 28 ? 2 : 1;
|
|
220
|
+
}
|
|
221
|
+
function inferredFallbackOwnerCandidates(target) {
|
|
222
|
+
const candidates = [];
|
|
223
|
+
const appendCandidate = (node, defaults) => {
|
|
224
|
+
if (!node?.selector) {
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
const kind = ownerCandidateKindOf(node);
|
|
228
|
+
candidates.push({
|
|
229
|
+
...defaults,
|
|
230
|
+
...node,
|
|
231
|
+
kind: kind ?? node.kind,
|
|
232
|
+
surfaceKind: node.surfaceKind ?? kind,
|
|
233
|
+
surfacePriority: node.surfacePriority ?? defaults.surfacePriority ?? defaultPriorityForOwnerKind(kind),
|
|
234
|
+
});
|
|
235
|
+
};
|
|
236
|
+
appendCandidate(target.context?.item, {
|
|
237
|
+
depth: 0,
|
|
238
|
+
directTextBlockCount: estimatedTextBlockCount(target.context?.item?.label ?? target.context?.item?.text),
|
|
239
|
+
descendantInteractiveCount: 1,
|
|
240
|
+
viewportCoverage: 0.12,
|
|
241
|
+
relativeAreaToTarget: 6,
|
|
242
|
+
peerCount: 1,
|
|
243
|
+
});
|
|
244
|
+
appendCandidate(target.context?.group, {
|
|
245
|
+
depth: 1,
|
|
246
|
+
directTextBlockCount: estimatedTextBlockCount(target.context?.group?.label ?? target.context?.group?.text),
|
|
247
|
+
descendantInteractiveCount: 2,
|
|
248
|
+
viewportCoverage: 0.2,
|
|
249
|
+
relativeAreaToTarget: 14,
|
|
250
|
+
peerCount: 1,
|
|
251
|
+
});
|
|
252
|
+
appendCandidate(target.context?.container, {
|
|
253
|
+
depth: 1,
|
|
254
|
+
directTextBlockCount: estimatedTextBlockCount(target.context?.container?.label ?? target.context?.container?.text),
|
|
255
|
+
descendantInteractiveCount: 2,
|
|
256
|
+
viewportCoverage: 0.22,
|
|
257
|
+
relativeAreaToTarget: 14,
|
|
258
|
+
peerCount: 1,
|
|
259
|
+
shell: SHELL_BOUNDARY_OWNER_KINDS.has(normalizeOwnerText(target.context?.container?.kind)?.toLowerCase() ?? ''),
|
|
260
|
+
});
|
|
261
|
+
appendCandidate(target.context?.landmark, {
|
|
262
|
+
depth: 2,
|
|
263
|
+
directTextBlockCount: estimatedTextBlockCount(target.context?.landmark?.label ?? target.context?.landmark?.text),
|
|
264
|
+
descendantInteractiveCount: 4,
|
|
265
|
+
viewportCoverage: 0.45,
|
|
266
|
+
relativeAreaToTarget: 24,
|
|
267
|
+
boundary: true,
|
|
268
|
+
shell: true,
|
|
269
|
+
});
|
|
270
|
+
return candidates;
|
|
271
|
+
}
|
|
272
|
+
function resolutionCandidatesFor(target) {
|
|
273
|
+
const explicitCandidates = target.context?.ownerCandidates?.filter((candidate) => Boolean(candidate.selector)) ?? [];
|
|
274
|
+
return explicitCandidates.length > 0
|
|
275
|
+
? explicitCandidates
|
|
276
|
+
: inferredFallbackOwnerCandidates(target);
|
|
157
277
|
}
|
|
158
|
-
function
|
|
159
|
-
if (
|
|
278
|
+
function canonicalStructuredGridSurfaceOf(target) {
|
|
279
|
+
if (target.structure?.family !== 'structured-grid') {
|
|
160
280
|
return undefined;
|
|
161
281
|
}
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
kind: normalizeEmbeddedSurfaceText(target.context?.group?.kind)?.toLowerCase(),
|
|
166
|
-
label: compactEmbeddedSurfaceLabel(target.context?.group?.label, target.context?.landmark?.label, target.context?.container?.label, target.context?.hintText),
|
|
167
|
-
},
|
|
168
|
-
{
|
|
169
|
-
selector: normalizeEmbeddedSurfaceText(target.context?.container?.selector),
|
|
170
|
-
kind: normalizeEmbeddedSurfaceText(target.context?.container?.kind)?.toLowerCase(),
|
|
171
|
-
label: compactEmbeddedSurfaceLabel(target.context?.container?.label, target.context?.landmark?.label, target.context?.hintText),
|
|
172
|
-
},
|
|
173
|
-
];
|
|
174
|
-
const surfaceSeed = selectorCandidates.find((candidate) => candidate.selector);
|
|
175
|
-
if (!surfaceSeed?.selector) {
|
|
282
|
+
const explicitSurfaceSelector = normalizeOwnerText(target.explicitSurfaceSelector) ??
|
|
283
|
+
normalizeOwnerText(target.surfaceSelector);
|
|
284
|
+
if (!explicitSurfaceSelector) {
|
|
176
285
|
return undefined;
|
|
177
286
|
}
|
|
178
|
-
const
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
287
|
+
const explicitSurfaceKind = normalizeOwnerText(target.explicitSurfaceKind)?.toLowerCase() ??
|
|
288
|
+
normalizeOwnerText(target.surfaceKind)?.toLowerCase();
|
|
289
|
+
if (!explicitSurfaceKind) {
|
|
290
|
+
return undefined;
|
|
291
|
+
}
|
|
292
|
+
const surfaceLabel = normalizeOwnerText(target.explicitSurfaceLabel) ??
|
|
293
|
+
normalizeOwnerText(target.surfaceLabel) ??
|
|
294
|
+
normalizeOwnerText(target.fallbackSurfaceLabel);
|
|
295
|
+
const surfaceSelectors = mergeSurfaceSelectors(explicitSurfaceSelector, target.explicitSurfaceSelectors ?? target.surfaceSelectors);
|
|
296
|
+
const surfacePriority = target.explicitSurfacePriority ??
|
|
297
|
+
target.surfacePriority ??
|
|
298
|
+
defaultPriorityForOwnerKind(explicitSurfaceKind);
|
|
183
299
|
return {
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
surfacePriority
|
|
300
|
+
surfaceKind: explicitSurfaceKind,
|
|
301
|
+
surfaceLabel,
|
|
302
|
+
fallbackSurfaceLabel: surfaceLabel ?? target.fallbackSurfaceLabel,
|
|
303
|
+
surfaceSelector: explicitSurfaceSelector,
|
|
304
|
+
surfaceSelectors,
|
|
305
|
+
surfacePriority,
|
|
190
306
|
};
|
|
191
307
|
}
|
|
192
|
-
function
|
|
193
|
-
|
|
194
|
-
|
|
308
|
+
function ownerCandidateSelectorOf(candidate) {
|
|
309
|
+
return normalizeOwnerText(candidate.selector);
|
|
310
|
+
}
|
|
311
|
+
function ownerCandidateActsAsBoundary(candidate) {
|
|
312
|
+
const surfaceKind = ownerCandidateKindOf(candidate);
|
|
313
|
+
return (candidate.boundary === true ||
|
|
314
|
+
candidate.shell === true ||
|
|
315
|
+
(surfaceKind !== undefined &&
|
|
316
|
+
(EXPLICIT_BOUNDARY_OWNER_KINDS.has(surfaceKind) ||
|
|
317
|
+
SHELL_BOUNDARY_OWNER_KINDS.has(surfaceKind))));
|
|
318
|
+
}
|
|
319
|
+
function ownerBoundaryKeyOf(target, candidates) {
|
|
320
|
+
for (const candidate of candidates) {
|
|
321
|
+
const selector = ownerCandidateSelectorOf(candidate);
|
|
322
|
+
if (selector && ownerCandidateActsAsBoundary(candidate)) {
|
|
323
|
+
return `boundary:${selector}`;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
const explicitSurfaceSelector = normalizeOwnerText(target.explicitSurfaceSelector);
|
|
327
|
+
if (explicitSurfaceSelector) {
|
|
328
|
+
return `explicit:${explicitSurfaceSelector}`;
|
|
329
|
+
}
|
|
330
|
+
const formKey = formGroupingKeyOf(target);
|
|
331
|
+
if (formKey) {
|
|
332
|
+
return `form:${formKey}`;
|
|
333
|
+
}
|
|
334
|
+
const landmarkSelector = normalizeOwnerText(target.context?.landmark?.selector);
|
|
335
|
+
if (landmarkSelector) {
|
|
336
|
+
return `landmark:${landmarkSelector}`;
|
|
337
|
+
}
|
|
338
|
+
const surfaceSelector = normalizeOwnerText(target.surfaceSelector);
|
|
339
|
+
if (surfaceSelector) {
|
|
340
|
+
return `surface:${surfaceSelector}`;
|
|
341
|
+
}
|
|
342
|
+
const pageKey = normalizeOwnerText(target.pageSignature) ?? 'unknown-page';
|
|
343
|
+
const frameKey = target.framePath?.join('>') ??
|
|
344
|
+
(normalizeOwnerText(target.frameUrl) ? `url:${normalizeOwnerText(target.frameUrl)}` : 'top');
|
|
345
|
+
return `page:${pageKey}|${frameKey}`;
|
|
346
|
+
}
|
|
347
|
+
function ownerCandidateEligibleForSharedResolution(candidate) {
|
|
348
|
+
const contextBlockCount = ownerCandidateContextBlockCountOf(candidate);
|
|
349
|
+
const descendantInteractiveCount = candidate.descendantInteractiveCount ?? 0;
|
|
350
|
+
const viewportCoverage = candidate.viewportCoverage ?? 1;
|
|
351
|
+
const relativeAreaToTarget = candidate.relativeAreaToTarget ?? 0;
|
|
352
|
+
return Boolean(ownerCandidateSelectorOf(candidate) &&
|
|
353
|
+
!ownerCandidateActsAsBoundary(candidate) &&
|
|
354
|
+
ownerCandidateRefinesContext(candidate) &&
|
|
355
|
+
contextBlockCount >= 2 &&
|
|
356
|
+
descendantInteractiveCount >= 1 &&
|
|
357
|
+
descendantInteractiveCount <= 8 &&
|
|
358
|
+
viewportCoverage <= 0.55 &&
|
|
359
|
+
relativeAreaToTarget >= 2);
|
|
360
|
+
}
|
|
361
|
+
function nearestLocalOwnerBranchKeyOf(target, candidates) {
|
|
362
|
+
for (const candidate of candidates) {
|
|
363
|
+
const selector = ownerCandidateSelectorOf(candidate);
|
|
364
|
+
if (!selector || ownerCandidateActsAsBoundary(candidate)) {
|
|
365
|
+
continue;
|
|
366
|
+
}
|
|
367
|
+
return selector;
|
|
368
|
+
}
|
|
369
|
+
return `target:${observedTargetKey(target)}`;
|
|
370
|
+
}
|
|
371
|
+
function inferSharedOwnerSelectors(targets) {
|
|
372
|
+
const supportByBoundary = new Map();
|
|
373
|
+
const candidatesByTarget = targets.map((target) => resolutionCandidatesFor(target));
|
|
195
374
|
for (const [index, target] of targets.entries()) {
|
|
196
|
-
const
|
|
197
|
-
if (
|
|
375
|
+
const candidates = candidatesByTarget[index] ?? [];
|
|
376
|
+
if (candidates.length === 0) {
|
|
198
377
|
continue;
|
|
199
378
|
}
|
|
200
|
-
|
|
201
|
-
|
|
379
|
+
const boundaryKey = ownerBoundaryKeyOf(target, candidates);
|
|
380
|
+
const branchKey = nearestLocalOwnerBranchKeyOf(target, candidates);
|
|
381
|
+
const supportBySelector = supportByBoundary.get(boundaryKey) ?? new Map();
|
|
382
|
+
for (const candidate of candidates) {
|
|
383
|
+
const selector = ownerCandidateSelectorOf(candidate);
|
|
384
|
+
if (!selector || !ownerCandidateEligibleForSharedResolution(candidate)) {
|
|
385
|
+
continue;
|
|
386
|
+
}
|
|
387
|
+
const existing = supportBySelector.get(selector) ?? {
|
|
388
|
+
branchRefs: new Set(),
|
|
389
|
+
targetIndexes: new Set(),
|
|
390
|
+
hasPrimaryActionSupport: false,
|
|
391
|
+
hasIdentitySupport: false,
|
|
392
|
+
};
|
|
393
|
+
existing.branchRefs.add(branchKey);
|
|
394
|
+
existing.targetIndexes.add(index);
|
|
395
|
+
existing.hasPrimaryActionSupport ||= targetSupportsSharedOwnerPrimaryAction(target);
|
|
396
|
+
existing.hasIdentitySupport ||= targetSupportsSharedOwnerIdentity(target);
|
|
397
|
+
supportBySelector.set(selector, existing);
|
|
398
|
+
}
|
|
399
|
+
supportByBoundary.set(boundaryKey, supportBySelector);
|
|
202
400
|
}
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
401
|
+
const preferredSelectors = new Map();
|
|
402
|
+
for (const [index, target] of targets.entries()) {
|
|
403
|
+
const candidates = candidatesByTarget[index] ?? [];
|
|
404
|
+
if (candidates.length === 0) {
|
|
405
|
+
continue;
|
|
207
406
|
}
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
407
|
+
const boundaryKey = ownerBoundaryKeyOf(target, candidates);
|
|
408
|
+
const supportBySelector = supportByBoundary.get(boundaryKey);
|
|
409
|
+
if (!supportBySelector) {
|
|
410
|
+
continue;
|
|
411
|
+
}
|
|
412
|
+
for (const candidate of candidates) {
|
|
413
|
+
const selector = ownerCandidateSelectorOf(candidate);
|
|
414
|
+
if (!selector || !ownerCandidateEligibleForSharedResolution(candidate)) {
|
|
415
|
+
continue;
|
|
416
|
+
}
|
|
417
|
+
const support = supportBySelector.get(selector);
|
|
418
|
+
if (!support) {
|
|
419
|
+
continue;
|
|
420
|
+
}
|
|
421
|
+
if (support.branchRefs.size >= 2 &&
|
|
422
|
+
support.targetIndexes.size >= 2 &&
|
|
423
|
+
support.hasPrimaryActionSupport &&
|
|
424
|
+
support.hasIdentitySupport) {
|
|
425
|
+
preferredSelectors.set(index, selector);
|
|
426
|
+
break;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
return preferredSelectors;
|
|
431
|
+
}
|
|
432
|
+
function ownerCandidateTierOf(candidate) {
|
|
433
|
+
const surfaceKind = ownerCandidateKindOf(candidate);
|
|
434
|
+
if (surfaceKind === 'token' || surfaceKind === 'chip') {
|
|
435
|
+
return 0;
|
|
436
|
+
}
|
|
437
|
+
const explicitBoundary = surfaceKind ? EXPLICIT_BOUNDARY_OWNER_KINDS.has(surfaceKind) : false;
|
|
438
|
+
const shellLike = candidate.shell === true || (surfaceKind ? SHELL_BOUNDARY_OWNER_KINDS.has(surfaceKind) : false);
|
|
439
|
+
const boundary = candidate.boundary === true;
|
|
440
|
+
const directTextBlockCount = candidate.directTextBlockCount ?? 0;
|
|
441
|
+
const descendantInteractiveCount = candidate.descendantInteractiveCount ?? 0;
|
|
442
|
+
const viewportCoverage = candidate.viewportCoverage ?? 1;
|
|
443
|
+
const relativeAreaToTarget = candidate.relativeAreaToTarget ?? 0;
|
|
444
|
+
const peerCount = candidate.peerCount ?? 0;
|
|
445
|
+
if (relativeAreaToTarget < 1.5 ||
|
|
446
|
+
viewportCoverage > 0.8 ||
|
|
447
|
+
descendantInteractiveCount > 12 ||
|
|
448
|
+
!ownerCandidateRefinesContext(candidate)) {
|
|
449
|
+
return 0;
|
|
450
|
+
}
|
|
451
|
+
const richLocalOwner = directTextBlockCount >= 2 &&
|
|
452
|
+
descendantInteractiveCount >= 1 &&
|
|
453
|
+
descendantInteractiveCount <= 8 &&
|
|
454
|
+
viewportCoverage <= 0.45;
|
|
455
|
+
const repeatedItemLike = peerCount >= 2;
|
|
456
|
+
const itemLikeSurface = surfaceKind === 'card' || surfaceKind === 'listitem' || surfaceKind === 'row';
|
|
457
|
+
if (richLocalOwner && (repeatedItemLike || itemLikeSurface)) {
|
|
458
|
+
return 320;
|
|
459
|
+
}
|
|
460
|
+
if (richLocalOwner && !shellLike) {
|
|
461
|
+
return 280;
|
|
462
|
+
}
|
|
463
|
+
if (explicitBoundary &&
|
|
464
|
+
directTextBlockCount >= 1 &&
|
|
465
|
+
descendantInteractiveCount >= 1 &&
|
|
466
|
+
descendantInteractiveCount <= 8 &&
|
|
467
|
+
viewportCoverage <= 0.55) {
|
|
468
|
+
return 240;
|
|
469
|
+
}
|
|
470
|
+
if (directTextBlockCount >= 1 &&
|
|
471
|
+
repeatedItemLike &&
|
|
472
|
+
descendantInteractiveCount >= 1 &&
|
|
473
|
+
descendantInteractiveCount <= 4 &&
|
|
474
|
+
viewportCoverage <= 0.35) {
|
|
475
|
+
return 220;
|
|
476
|
+
}
|
|
477
|
+
if (shellLike &&
|
|
478
|
+
directTextBlockCount >= 1 &&
|
|
479
|
+
descendantInteractiveCount >= 1 &&
|
|
480
|
+
descendantInteractiveCount <= 8 &&
|
|
481
|
+
viewportCoverage <= 0.5) {
|
|
482
|
+
return 140;
|
|
483
|
+
}
|
|
484
|
+
if (boundary &&
|
|
485
|
+
directTextBlockCount >= 1 &&
|
|
486
|
+
descendantInteractiveCount >= 1 &&
|
|
487
|
+
descendantInteractiveCount <= 8 &&
|
|
488
|
+
viewportCoverage <= 0.5) {
|
|
489
|
+
return 180;
|
|
490
|
+
}
|
|
491
|
+
return 0;
|
|
492
|
+
}
|
|
493
|
+
function compareOwnerCandidates(left, right) {
|
|
494
|
+
const tierDelta = ownerCandidateTierOf(right) - ownerCandidateTierOf(left);
|
|
495
|
+
if (tierDelta !== 0) {
|
|
496
|
+
return tierDelta;
|
|
497
|
+
}
|
|
498
|
+
const leftDepth = left.depth ?? Number.MAX_SAFE_INTEGER;
|
|
499
|
+
const rightDepth = right.depth ?? Number.MAX_SAFE_INTEGER;
|
|
500
|
+
if (leftDepth !== rightDepth) {
|
|
501
|
+
return leftDepth - rightDepth;
|
|
502
|
+
}
|
|
503
|
+
const leftPriority = left.surfacePriority ?? 0;
|
|
504
|
+
const rightPriority = right.surfacePriority ?? 0;
|
|
505
|
+
if (leftPriority !== rightPriority) {
|
|
506
|
+
return rightPriority - leftPriority;
|
|
507
|
+
}
|
|
508
|
+
const leftTextBlocks = left.directTextBlockCount ?? 0;
|
|
509
|
+
const rightTextBlocks = right.directTextBlockCount ?? 0;
|
|
510
|
+
return rightTextBlocks - leftTextBlocks;
|
|
511
|
+
}
|
|
512
|
+
function mergeSurfaceSelectors(primary, additional) {
|
|
513
|
+
const selectors = new Set();
|
|
514
|
+
if (primary) {
|
|
515
|
+
selectors.add(primary);
|
|
516
|
+
}
|
|
517
|
+
for (const selector of additional ?? []) {
|
|
518
|
+
if (selector) {
|
|
519
|
+
selectors.add(selector);
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
return selectors.size > 0 ? [...selectors] : undefined;
|
|
523
|
+
}
|
|
524
|
+
function inferredOwnerSurfaceKindOf(candidate, options) {
|
|
525
|
+
const hintedKind = ownerCandidateKindHintOf(candidate) ?? normalizeOwnerText(candidate.kind)?.toLowerCase();
|
|
526
|
+
if (hintedKind && hintedKind !== 'div') {
|
|
527
|
+
return hintedKind;
|
|
528
|
+
}
|
|
529
|
+
const directTextBlockCount = candidate.directTextBlockCount ?? 0;
|
|
530
|
+
const contextBlockCount = options?.sharedOwner === true
|
|
531
|
+
? ownerCandidateContextBlockCountOf(candidate)
|
|
532
|
+
: directTextBlockCount;
|
|
533
|
+
const descendantInteractiveCount = candidate.descendantInteractiveCount ?? 0;
|
|
534
|
+
const viewportCoverage = candidate.viewportCoverage ?? 1;
|
|
535
|
+
const relativeAreaToTarget = candidate.relativeAreaToTarget ?? 0;
|
|
536
|
+
const peerCount = candidate.peerCount ?? 0;
|
|
537
|
+
const shellLike = candidate.shell === true || (hintedKind ? SHELL_BOUNDARY_OWNER_KINDS.has(hintedKind) : false);
|
|
538
|
+
if (shellLike) {
|
|
539
|
+
return undefined;
|
|
540
|
+
}
|
|
541
|
+
const maxInteractiveCount = options?.sharedOwner === true ? 8 : 4;
|
|
542
|
+
const richLocalOwner = contextBlockCount >= 2 &&
|
|
543
|
+
descendantInteractiveCount >= 1 &&
|
|
544
|
+
descendantInteractiveCount <= maxInteractiveCount &&
|
|
545
|
+
viewportCoverage <= 0.45;
|
|
546
|
+
if (richLocalOwner &&
|
|
547
|
+
(peerCount >= 1 || relativeAreaToTarget >= 4 || options?.sharedOwner === true)) {
|
|
548
|
+
return 'card';
|
|
549
|
+
}
|
|
550
|
+
if (contextBlockCount >= 1 &&
|
|
551
|
+
descendantInteractiveCount >= 1 &&
|
|
552
|
+
descendantInteractiveCount <= 4 &&
|
|
553
|
+
viewportCoverage <= 0.35 &&
|
|
554
|
+
relativeAreaToTarget >= 2) {
|
|
555
|
+
return 'group';
|
|
556
|
+
}
|
|
557
|
+
return undefined;
|
|
558
|
+
}
|
|
559
|
+
function ownerCandidateReusesExistingSurface(target, candidate) {
|
|
560
|
+
const candidateSelector = normalizeOwnerText(candidate.selector);
|
|
561
|
+
const currentSurfaceSelector = normalizeOwnerText(target.surfaceSelector);
|
|
562
|
+
return Boolean(candidateSelector && currentSurfaceSelector && candidateSelector === currentSurfaceSelector);
|
|
563
|
+
}
|
|
564
|
+
function resolveTargetOwnedSurface(target, preferredSharedOwnerSelector) {
|
|
565
|
+
const canonicalStructuredGridSurface = canonicalStructuredGridSurfaceOf(target);
|
|
566
|
+
if (canonicalStructuredGridSurface) {
|
|
567
|
+
return canonicalStructuredGridSurface;
|
|
568
|
+
}
|
|
569
|
+
const candidates = resolutionCandidatesFor(target);
|
|
570
|
+
if (!candidates || candidates.length === 0) {
|
|
571
|
+
return undefined;
|
|
572
|
+
}
|
|
573
|
+
const scoredCandidates = candidates.filter((candidate) => ownerCandidateTierOf(candidate) > 0);
|
|
574
|
+
const preferredCandidate = preferredSharedOwnerSelector
|
|
575
|
+
? candidates.find((candidate) => ownerCandidateSelectorOf(candidate) === preferredSharedOwnerSelector &&
|
|
576
|
+
ownerCandidateEligibleForSharedResolution(candidate))
|
|
577
|
+
: undefined;
|
|
578
|
+
const bestCandidate = preferredCandidate ?? [...scoredCandidates].sort(compareOwnerCandidates)[0];
|
|
579
|
+
if (!bestCandidate?.selector) {
|
|
580
|
+
return undefined;
|
|
581
|
+
}
|
|
582
|
+
const reusesExistingSurface = ownerCandidateReusesExistingSurface(target, bestCandidate);
|
|
583
|
+
const sharedOwnerSelected = preferredCandidate !== undefined && bestCandidate === preferredCandidate;
|
|
584
|
+
const surfaceKind = inferredOwnerSurfaceKindOf(bestCandidate, {
|
|
585
|
+
sharedOwner: sharedOwnerSelected,
|
|
586
|
+
}) ?? (reusesExistingSurface ? target.surfaceKind : undefined);
|
|
587
|
+
const surfaceLabel = ownerCandidateSurfaceLabelOf(bestCandidate) ??
|
|
588
|
+
(reusesExistingSurface
|
|
589
|
+
? (normalizeOwnerText(target.surfaceLabel) ?? normalizeOwnerText(target.fallbackSurfaceLabel))
|
|
590
|
+
: undefined);
|
|
591
|
+
const surfaceSelectors = mergeSurfaceSelectors(bestCandidate.selector, reusesExistingSurface ? target.surfaceSelectors : undefined);
|
|
592
|
+
const surfacePriority = bestCandidate.surfacePriority ??
|
|
593
|
+
defaultPriorityForOwnerKind(surfaceKind) ??
|
|
594
|
+
(reusesExistingSurface ? target.surfacePriority : undefined);
|
|
595
|
+
return {
|
|
596
|
+
surfaceKind,
|
|
597
|
+
surfaceLabel,
|
|
598
|
+
fallbackSurfaceLabel: surfaceLabel ?? (reusesExistingSurface ? target.fallbackSurfaceLabel : undefined),
|
|
599
|
+
surfaceSelector: bestCandidate.selector,
|
|
600
|
+
surfaceSelectors,
|
|
601
|
+
surfacePriority,
|
|
602
|
+
};
|
|
217
603
|
}
|
|
218
604
|
function parseNumericAmount(value) {
|
|
219
605
|
const normalized = value.replace(/,/g, '.').trim();
|
|
@@ -417,52 +803,63 @@ function inferCapability(target, allowedActions) {
|
|
|
417
803
|
}
|
|
418
804
|
return 'actionable';
|
|
419
805
|
}
|
|
420
|
-
|
|
421
|
-
const
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
806
|
+
function withAnnotatedSemantics(target) {
|
|
807
|
+
const allowedActions = inferAllowedActions(target);
|
|
808
|
+
const controlFamily = inferControlFamilyFromFacts({
|
|
809
|
+
kind: target.kind,
|
|
810
|
+
role: target.role,
|
|
811
|
+
label: target.label,
|
|
812
|
+
displayLabel: target.displayLabel,
|
|
813
|
+
interactionHint: target.interactionHint,
|
|
814
|
+
text: target.text,
|
|
815
|
+
placeholder: target.placeholder,
|
|
816
|
+
inputName: target.inputName,
|
|
817
|
+
inputType: target.inputType,
|
|
818
|
+
autocomplete: target.autocomplete,
|
|
819
|
+
ariaAutocomplete: target.ariaAutocomplete,
|
|
820
|
+
surfaceKind: target.surfaceKind,
|
|
821
|
+
controlsSurfaceSelector: target.controlsSurfaceSelector,
|
|
822
|
+
states: target.states,
|
|
823
|
+
structure: target.structure,
|
|
824
|
+
}, allowedActions);
|
|
825
|
+
const availability = inferAvailability(target);
|
|
826
|
+
const capability = inferCapability(target, allowedActions);
|
|
827
|
+
const acceptancePolicy = inferAcceptancePolicy(target, allowedActions);
|
|
828
|
+
return {
|
|
829
|
+
...target,
|
|
830
|
+
capability,
|
|
831
|
+
availability,
|
|
832
|
+
allowedActions,
|
|
833
|
+
controlFamily,
|
|
834
|
+
acceptancePolicy,
|
|
835
|
+
};
|
|
836
|
+
}
|
|
837
|
+
function withResolvedSurface(target, preferredSharedOwnerSelector) {
|
|
838
|
+
const ownedSurface = resolveTargetOwnedSurface(target, preferredSharedOwnerSelector);
|
|
839
|
+
const surfaceResolvedTarget = ownedSurface ? { ...target, ...ownedSurface } : target;
|
|
840
|
+
const explicitSurfaceRef = buildSurfaceRef(surfaceResolvedTarget);
|
|
841
|
+
const syntheticFormSurfaceRef = explicitSurfaceRef === undefined
|
|
842
|
+
? buildSyntheticFormSurfaceId(surfaceResolvedTarget)
|
|
843
|
+
: undefined;
|
|
844
|
+
const surfaceRef = explicitSurfaceRef ?? syntheticFormSurfaceRef;
|
|
845
|
+
const surfaceKind = surfaceResolvedTarget.surfaceKind ?? (syntheticFormSurfaceRef ? 'form' : undefined);
|
|
846
|
+
const surfaceLabel = surfaceResolvedTarget.surfaceLabel ??
|
|
847
|
+
(syntheticFormSurfaceRef
|
|
848
|
+
? (surfaceResolvedTarget.context?.landmark?.label ??
|
|
849
|
+
surfaceResolvedTarget.context?.group?.label ??
|
|
850
|
+
'Form')
|
|
851
|
+
: undefined);
|
|
852
|
+
const surfacePriority = surfaceResolvedTarget.surfacePriority ?? (surfaceKind === 'form' ? 70 : undefined);
|
|
853
|
+
return {
|
|
854
|
+
...surfaceResolvedTarget,
|
|
855
|
+
surfaceKind,
|
|
856
|
+
surfaceLabel,
|
|
857
|
+
surfacePriority,
|
|
858
|
+
surfaceRef,
|
|
859
|
+
};
|
|
860
|
+
}
|
|
861
|
+
function attachOwnerIndices(targets) {
|
|
862
|
+
return targets.map((target, index, current) => {
|
|
466
863
|
const compactRemoveItemLabel = compactRemoveItemLabelOf(target);
|
|
467
864
|
const compactRemoveOwnerIndex = isCompactRemoveActionTarget(target) && compactRemoveItemLabel
|
|
468
865
|
? current.findIndex((candidate, candidateIndex) => {
|
|
@@ -515,14 +912,20 @@ export function annotateDomTargets(targets) {
|
|
|
515
912
|
return true;
|
|
516
913
|
if (candidate.structure?.family === 'structured-grid')
|
|
517
914
|
return false;
|
|
518
|
-
if (candidate.capability === 'informational' && !explicitController)
|
|
915
|
+
if (candidate.capability === 'informational' && !explicitController) {
|
|
519
916
|
return false;
|
|
917
|
+
}
|
|
520
918
|
return (candidate.allowedActions?.includes('click') &&
|
|
521
919
|
['input', 'select', 'button', 'combobox'].includes((candidate.kind || '').toLowerCase()));
|
|
522
920
|
});
|
|
523
921
|
return ownerIndex >= 0 ? { ...target, ownerIndex } : target;
|
|
524
922
|
});
|
|
525
923
|
}
|
|
924
|
+
export function annotateDomTargets(targets) {
|
|
925
|
+
const semanticallyAnnotatedTargets = targets.map(withAnnotatedSemantics);
|
|
926
|
+
const preferredSharedOwnerSelectors = inferSharedOwnerSelectors(semanticallyAnnotatedTargets);
|
|
927
|
+
return attachOwnerIndices(semanticallyAnnotatedTargets.map((target, index) => withResolvedSurface(target, preferredSharedOwnerSelectors.get(index))));
|
|
928
|
+
}
|
|
526
929
|
export function orderBySurfaceCompetition(targets) {
|
|
527
930
|
const iframeFieldLike = (target) => {
|
|
528
931
|
const kind = (target.kind ?? '').toLowerCase();
|
|
@@ -603,3 +1006,6 @@ export function orderBySurfaceCompetition(targets) {
|
|
|
603
1006
|
return (left.ordinal ?? 0) - (right.ordinal ?? 0);
|
|
604
1007
|
});
|
|
605
1008
|
}
|
|
1009
|
+
export function normalizePostEnrichmentDomTargets(targets) {
|
|
1010
|
+
return compressSemanticallyDuplicateTargets(orderBySurfaceCompetition(annotateDomTargets(targets)));
|
|
1011
|
+
}
|