@mercuryo-ai/agentbrowse 0.2.61 → 0.2.63

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. package/CHANGELOG.md +33 -1
  2. package/README.md +102 -9
  3. package/dist/browser-session-state.d.ts +2 -11
  4. package/dist/browser-session-state.d.ts.map +1 -1
  5. package/dist/browser-session-state.js +0 -4
  6. package/dist/commands/act.d.ts.map +1 -1
  7. package/dist/commands/act.js +14 -5
  8. package/dist/commands/attach.d.ts +1 -3
  9. package/dist/commands/attach.d.ts.map +1 -1
  10. package/dist/commands/attach.js +0 -2
  11. package/dist/commands/browser-status.d.ts +0 -2
  12. package/dist/commands/browser-status.d.ts.map +1 -1
  13. package/dist/commands/browser-status.js +1 -7
  14. package/dist/commands/interaction-kernel.d.ts +1 -1
  15. package/dist/commands/interaction-kernel.d.ts.map +1 -1
  16. package/dist/commands/interaction-kernel.js +1 -1
  17. package/dist/commands/launch.d.ts +0 -1
  18. package/dist/commands/launch.d.ts.map +1 -1
  19. package/dist/commands/launch.js +0 -4
  20. package/dist/commands/observe-accessibility.d.ts.map +1 -1
  21. package/dist/commands/observe-accessibility.js +36 -2
  22. package/dist/commands/observe-inventory.d.ts +49 -7
  23. package/dist/commands/observe-inventory.d.ts.map +1 -1
  24. package/dist/commands/observe-inventory.js +807 -96
  25. package/dist/commands/observe-persistence.d.ts.map +1 -1
  26. package/dist/commands/observe-persistence.js +49 -6
  27. package/dist/commands/observe-projection.d.ts +6 -2
  28. package/dist/commands/observe-projection.d.ts.map +1 -1
  29. package/dist/commands/observe-projection.js +251 -27
  30. package/dist/commands/observe-semantics.d.ts +1 -0
  31. package/dist/commands/observe-semantics.d.ts.map +1 -1
  32. package/dist/commands/observe-semantics.js +541 -135
  33. package/dist/commands/observe-signals.d.ts +4 -4
  34. package/dist/commands/observe-signals.d.ts.map +1 -1
  35. package/dist/commands/observe-signals.js +2 -2
  36. package/dist/commands/observe-surfaces.d.ts +2 -1
  37. package/dist/commands/observe-surfaces.d.ts.map +1 -1
  38. package/dist/commands/observe-surfaces.js +143 -45
  39. package/dist/commands/observe.d.ts +5 -1
  40. package/dist/commands/observe.d.ts.map +1 -1
  41. package/dist/commands/observe.js +15 -11
  42. package/dist/commands/semantic-observe.d.ts.map +1 -1
  43. package/dist/commands/semantic-observe.js +43 -0
  44. package/dist/library.d.ts +2 -1
  45. package/dist/library.d.ts.map +1 -1
  46. package/dist/library.js +2 -1
  47. package/dist/match-resolve-fill.d.ts +196 -0
  48. package/dist/match-resolve-fill.d.ts.map +1 -0
  49. package/dist/match-resolve-fill.js +700 -0
  50. package/dist/match-resolve-fill.test-support.d.ts +34 -0
  51. package/dist/match-resolve-fill.test-support.d.ts.map +1 -0
  52. package/dist/match-resolve-fill.test-support.js +81 -0
  53. package/dist/runtime-protected-state.d.ts.map +1 -1
  54. package/dist/runtime-protected-state.js +12 -0
  55. package/dist/runtime-state.d.ts +6 -0
  56. package/dist/runtime-state.d.ts.map +1 -1
  57. package/dist/runtime-state.js +6 -0
  58. package/dist/secrets/form-matcher.d.ts.map +1 -1
  59. package/dist/secrets/form-matcher.js +76 -27
  60. package/dist/secrets/protected-exact-value-redaction.d.ts.map +1 -1
  61. package/dist/secrets/protected-exact-value-redaction.js +6 -0
  62. package/dist/secrets/protected-fill.js +3 -3
  63. package/dist/session.d.ts +3 -3
  64. package/dist/session.d.ts.map +1 -1
  65. package/dist/session.js +2 -2
  66. package/dist/solver/browser-launcher.d.ts.map +1 -1
  67. package/dist/solver/browser-launcher.js +2 -1
  68. package/dist/testing.d.ts +1 -0
  69. package/dist/testing.d.ts.map +1 -1
  70. package/dist/testing.js +1 -0
  71. package/docs/README.md +28 -11
  72. package/docs/api-reference.md +311 -19
  73. package/docs/assistive-runtime.md +41 -16
  74. package/docs/getting-started.md +45 -1
  75. package/docs/integration-checklist.md +32 -3
  76. package/docs/match-resolve-fill.md +699 -0
  77. package/docs/protected-fill.md +373 -91
  78. package/docs/testing.md +147 -15
  79. package/docs/troubleshooting.md +5 -0
  80. package/examples/README.md +7 -0
  81. package/examples/match-resolve-fill.ts +107 -0
  82. package/package.json +4 -2
@@ -105,115 +105,501 @@ export function observedTargetKey(target) {
105
105
  target.domSignature ??
106
106
  `${target.formSelector ?? 'no-form'}|${target.ordinal ?? 'no-ordinal'}|${target.label ?? ''}`);
107
107
  }
108
- function normalizeEmbeddedSurfaceText(value) {
109
- const normalized = value?.replace(/\s+/g, ' ').trim();
110
- return normalized ? normalized : undefined;
108
+ const EXPLICIT_BOUNDARY_OWNER_KINDS = new Set([
109
+ 'dialog',
110
+ 'listbox',
111
+ 'menu',
112
+ 'grid',
113
+ 'tabpanel',
114
+ 'popover',
115
+ 'dropdown',
116
+ 'datepicker',
117
+ 'form',
118
+ ]);
119
+ const SHELL_BOUNDARY_OWNER_KINDS = new Set([
120
+ 'floating-panel',
121
+ 'sticky-panel',
122
+ 'section',
123
+ 'aside',
124
+ 'main',
125
+ ]);
126
+ const ACTION_ONLY_OWNER_LABEL_RE = /^(?:select|book|buy|choose|continue|next|more|details|view|open|pay|submit|click)$/i;
127
+ const PRIMARY_ACTION_OWNER_LABEL_RE = /^(?:select|book|buy|choose|continue|next|pay|submit)$/i;
128
+ function normalizeOwnerText(value) {
129
+ const normalized = (value ?? '').replace(/\s+/g, ' ').trim();
130
+ return normalized || undefined;
111
131
  }
112
- function embeddedSurfaceFrameKeyOf(target) {
113
- if (target.framePath?.length) {
114
- return target.framePath.join('>');
115
- }
116
- const frameUrl = normalizeEmbeddedSurfaceText(target.frameUrl);
117
- return frameUrl ? `url:${frameUrl}` : 'top';
132
+ function ownerCandidateSurfaceLabelOf(candidate) {
133
+ return (normalizeOwnerText(candidate.label) ??
134
+ normalizeOwnerText(candidate.text) ??
135
+ normalizeOwnerText(candidate.fallbackLabel));
118
136
  }
119
- function compactEmbeddedSurfaceLabel(...values) {
120
- for (const value of values) {
121
- const normalized = normalizeEmbeddedSurfaceText(value);
122
- if (normalized && normalized.length <= 80) {
123
- return normalized;
124
- }
137
+ function ownerCandidateRefinesContext(candidate) {
138
+ const surfaceLabel = ownerCandidateSurfaceLabelOf(candidate);
139
+ if (!surfaceLabel) {
140
+ return false;
125
141
  }
126
- return undefined;
142
+ return !ACTION_ONLY_OWNER_LABEL_RE.test(surfaceLabel);
127
143
  }
128
- function isEmbeddedInteractiveControl(target) {
129
- if ((target.surfaceRef ?? '').trim().length > 0) {
130
- return false;
144
+ function ownerCandidateContextBlockCountOf(candidate) {
145
+ return Math.max(candidate.directTextBlockCount ?? 0, estimatedTextBlockCount(ownerCandidateSurfaceLabelOf(candidate)));
146
+ }
147
+ function targetSharedOwnerSupportLabelOf(target) {
148
+ return (normalizeOwnerText(target.displayLabel) ??
149
+ normalizeOwnerText(target.label) ??
150
+ normalizeOwnerText(target.currentValue));
151
+ }
152
+ function targetSupportsSharedOwnerPrimaryAction(target) {
153
+ const label = targetSharedOwnerSupportLabelOf(target);
154
+ return Boolean(label && PRIMARY_ACTION_OWNER_LABEL_RE.test(label));
155
+ }
156
+ function targetSupportsSharedOwnerIdentity(target) {
157
+ const label = targetSharedOwnerSupportLabelOf(target);
158
+ if (label && !ACTION_ONLY_OWNER_LABEL_RE.test(label)) {
159
+ return true;
131
160
  }
132
- const allowedActions = target.allowedActions ?? [];
133
- if (allowedActions.length === 0) {
134
- return false;
161
+ return Boolean(target.selection?.items?.some((item) => {
162
+ const itemLabel = normalizeOwnerText(item.label);
163
+ return Boolean(itemLabel && !ACTION_ONLY_OWNER_LABEL_RE.test(itemLabel));
164
+ }));
165
+ }
166
+ function ownerCandidateKindHintOf(candidate) {
167
+ return normalizeOwnerText(candidate.surfaceKind)?.toLowerCase();
168
+ }
169
+ function ownerCandidateKindOf(candidate) {
170
+ return ownerCandidateKindHintOf(candidate) ?? normalizeOwnerText(candidate.kind)?.toLowerCase();
171
+ }
172
+ function defaultPriorityForOwnerKind(kind) {
173
+ switch (kind) {
174
+ case 'dialog':
175
+ return 100;
176
+ case 'listbox':
177
+ case 'menu':
178
+ return 95;
179
+ case 'floating-panel':
180
+ return 92;
181
+ case 'datepicker':
182
+ return 90;
183
+ case 'sticky-panel':
184
+ return 88;
185
+ case 'popover':
186
+ case 'dropdown':
187
+ return 85;
188
+ case 'grid':
189
+ case 'tabpanel':
190
+ return 80;
191
+ case 'form':
192
+ return 70;
193
+ case 'card':
194
+ return 58;
195
+ case 'group':
196
+ return 55;
197
+ case 'section':
198
+ case 'aside':
199
+ return 52;
200
+ case 'listitem':
201
+ case 'row':
202
+ return 48;
203
+ default:
204
+ return undefined;
135
205
  }
136
- const kind = (target.kind ?? '').trim().toLowerCase();
137
- const role = (target.role ?? '').trim().toLowerCase();
138
- if (kind === 'link' || role === 'link') {
139
- return false;
206
+ }
207
+ function estimatedTextBlockCount(value) {
208
+ const normalized = normalizeOwnerText(value);
209
+ if (!normalized) {
210
+ return 0;
140
211
  }
141
- if (allowedActions.includes('fill') ||
142
- allowedActions.includes('type') ||
143
- allowedActions.includes('select')) {
144
- return true;
212
+ const splitHints = normalized
213
+ .split(/(?: · | \| |, | {2,}| (?=\d{1,2}:\d{2}\b)| (?=\d+\s?[€$₽£]\b))/)
214
+ .map((part) => part.trim())
215
+ .filter(Boolean);
216
+ if (splitHints.length >= 2) {
217
+ return Math.min(splitHints.length, 4);
145
218
  }
146
- return (kind === 'button' ||
147
- kind === 'radio' ||
148
- kind === 'checkbox' ||
149
- kind === 'input' ||
150
- kind === 'select' ||
151
- kind === 'textarea' ||
152
- role === 'button' ||
153
- role === 'radio' ||
154
- role === 'checkbox' ||
155
- role === 'textbox' ||
156
- role === 'combobox');
219
+ return normalized.length >= 28 ? 2 : 1;
220
+ }
221
+ function inferredFallbackOwnerCandidates(target) {
222
+ const candidates = [];
223
+ const appendCandidate = (node, defaults) => {
224
+ if (!node?.selector) {
225
+ return;
226
+ }
227
+ const kind = ownerCandidateKindOf(node);
228
+ candidates.push({
229
+ ...defaults,
230
+ ...node,
231
+ kind: kind ?? node.kind,
232
+ surfaceKind: node.surfaceKind ?? kind,
233
+ surfacePriority: node.surfacePriority ?? defaults.surfacePriority ?? defaultPriorityForOwnerKind(kind),
234
+ });
235
+ };
236
+ appendCandidate(target.context?.item, {
237
+ depth: 0,
238
+ directTextBlockCount: estimatedTextBlockCount(target.context?.item?.label ?? target.context?.item?.text),
239
+ descendantInteractiveCount: 1,
240
+ viewportCoverage: 0.12,
241
+ relativeAreaToTarget: 6,
242
+ peerCount: 1,
243
+ });
244
+ appendCandidate(target.context?.group, {
245
+ depth: 1,
246
+ directTextBlockCount: estimatedTextBlockCount(target.context?.group?.label ?? target.context?.group?.text),
247
+ descendantInteractiveCount: 2,
248
+ viewportCoverage: 0.2,
249
+ relativeAreaToTarget: 14,
250
+ peerCount: 1,
251
+ });
252
+ appendCandidate(target.context?.container, {
253
+ depth: 1,
254
+ directTextBlockCount: estimatedTextBlockCount(target.context?.container?.label ?? target.context?.container?.text),
255
+ descendantInteractiveCount: 2,
256
+ viewportCoverage: 0.22,
257
+ relativeAreaToTarget: 14,
258
+ peerCount: 1,
259
+ shell: SHELL_BOUNDARY_OWNER_KINDS.has(normalizeOwnerText(target.context?.container?.kind)?.toLowerCase() ?? ''),
260
+ });
261
+ appendCandidate(target.context?.landmark, {
262
+ depth: 2,
263
+ directTextBlockCount: estimatedTextBlockCount(target.context?.landmark?.label ?? target.context?.landmark?.text),
264
+ descendantInteractiveCount: 4,
265
+ viewportCoverage: 0.45,
266
+ relativeAreaToTarget: 24,
267
+ boundary: true,
268
+ shell: true,
269
+ });
270
+ return candidates;
271
+ }
272
+ function resolutionCandidatesFor(target) {
273
+ const explicitCandidates = target.context?.ownerCandidates?.filter((candidate) => Boolean(candidate.selector)) ?? [];
274
+ return explicitCandidates.length > 0
275
+ ? explicitCandidates
276
+ : inferredFallbackOwnerCandidates(target);
157
277
  }
158
- function embeddedSurfaceSeedOf(target) {
159
- if (!isEmbeddedInteractiveControl(target)) {
278
+ function canonicalStructuredGridSurfaceOf(target) {
279
+ if (target.structure?.family !== 'structured-grid') {
160
280
  return undefined;
161
281
  }
162
- const selectorCandidates = [
163
- {
164
- selector: normalizeEmbeddedSurfaceText(target.context?.group?.selector),
165
- kind: normalizeEmbeddedSurfaceText(target.context?.group?.kind)?.toLowerCase(),
166
- label: compactEmbeddedSurfaceLabel(target.context?.group?.label, target.context?.landmark?.label, target.context?.container?.label, target.context?.hintText),
167
- },
168
- {
169
- selector: normalizeEmbeddedSurfaceText(target.context?.container?.selector),
170
- kind: normalizeEmbeddedSurfaceText(target.context?.container?.kind)?.toLowerCase(),
171
- label: compactEmbeddedSurfaceLabel(target.context?.container?.label, target.context?.landmark?.label, target.context?.hintText),
172
- },
173
- ];
174
- const surfaceSeed = selectorCandidates.find((candidate) => candidate.selector);
175
- if (!surfaceSeed?.selector) {
282
+ const explicitSurfaceSelector = normalizeOwnerText(target.explicitSurfaceSelector) ??
283
+ normalizeOwnerText(target.surfaceSelector);
284
+ if (!explicitSurfaceSelector) {
176
285
  return undefined;
177
286
  }
178
- const pageKey = normalizeEmbeddedSurfaceText(target.pageSignature) ?? 'unknown-page';
179
- const frameKey = embeddedSurfaceFrameKeyOf(target);
180
- const surfaceKind = surfaceSeed.kind && !['div', 'tbody', 'table', 'tr', 'td', 'span'].includes(surfaceSeed.kind)
181
- ? surfaceSeed.kind
182
- : 'group';
287
+ const explicitSurfaceKind = normalizeOwnerText(target.explicitSurfaceKind)?.toLowerCase() ??
288
+ normalizeOwnerText(target.surfaceKind)?.toLowerCase();
289
+ if (!explicitSurfaceKind) {
290
+ return undefined;
291
+ }
292
+ const surfaceLabel = normalizeOwnerText(target.explicitSurfaceLabel) ??
293
+ normalizeOwnerText(target.surfaceLabel) ??
294
+ normalizeOwnerText(target.fallbackSurfaceLabel);
295
+ const surfaceSelectors = mergeSurfaceSelectors(explicitSurfaceSelector, target.explicitSurfaceSelectors ?? target.surfaceSelectors);
296
+ const surfacePriority = target.explicitSurfacePriority ??
297
+ target.surfacePriority ??
298
+ defaultPriorityForOwnerKind(explicitSurfaceKind);
183
299
  return {
184
- clusterKey: `${pageKey}|${frameKey}|${surfaceSeed.selector}`,
185
- surfaceRef: `${pageKey}|${frameKey}|embedded|${surfaceSeed.selector}`,
186
- surfaceKind,
187
- surfaceLabel: surfaceSeed.label ?? 'Embedded controls',
188
- surfaceSelector: surfaceSeed.selector,
189
- surfacePriority: 56,
300
+ surfaceKind: explicitSurfaceKind,
301
+ surfaceLabel,
302
+ fallbackSurfaceLabel: surfaceLabel ?? target.fallbackSurfaceLabel,
303
+ surfaceSelector: explicitSurfaceSelector,
304
+ surfaceSelectors,
305
+ surfacePriority,
190
306
  };
191
307
  }
192
- function materializeEmbeddedControlSurfaces(targets) {
193
- const clusterCounts = new Map();
194
- const seedsByIndex = new Map();
308
+ function ownerCandidateSelectorOf(candidate) {
309
+ return normalizeOwnerText(candidate.selector);
310
+ }
311
+ function ownerCandidateActsAsBoundary(candidate) {
312
+ const surfaceKind = ownerCandidateKindOf(candidate);
313
+ return (candidate.boundary === true ||
314
+ candidate.shell === true ||
315
+ (surfaceKind !== undefined &&
316
+ (EXPLICIT_BOUNDARY_OWNER_KINDS.has(surfaceKind) ||
317
+ SHELL_BOUNDARY_OWNER_KINDS.has(surfaceKind))));
318
+ }
319
+ function ownerBoundaryKeyOf(target, candidates) {
320
+ for (const candidate of candidates) {
321
+ const selector = ownerCandidateSelectorOf(candidate);
322
+ if (selector && ownerCandidateActsAsBoundary(candidate)) {
323
+ return `boundary:${selector}`;
324
+ }
325
+ }
326
+ const explicitSurfaceSelector = normalizeOwnerText(target.explicitSurfaceSelector);
327
+ if (explicitSurfaceSelector) {
328
+ return `explicit:${explicitSurfaceSelector}`;
329
+ }
330
+ const formKey = formGroupingKeyOf(target);
331
+ if (formKey) {
332
+ return `form:${formKey}`;
333
+ }
334
+ const landmarkSelector = normalizeOwnerText(target.context?.landmark?.selector);
335
+ if (landmarkSelector) {
336
+ return `landmark:${landmarkSelector}`;
337
+ }
338
+ const surfaceSelector = normalizeOwnerText(target.surfaceSelector);
339
+ if (surfaceSelector) {
340
+ return `surface:${surfaceSelector}`;
341
+ }
342
+ const pageKey = normalizeOwnerText(target.pageSignature) ?? 'unknown-page';
343
+ const frameKey = target.framePath?.join('>') ??
344
+ (normalizeOwnerText(target.frameUrl) ? `url:${normalizeOwnerText(target.frameUrl)}` : 'top');
345
+ return `page:${pageKey}|${frameKey}`;
346
+ }
347
+ function ownerCandidateEligibleForSharedResolution(candidate) {
348
+ const contextBlockCount = ownerCandidateContextBlockCountOf(candidate);
349
+ const descendantInteractiveCount = candidate.descendantInteractiveCount ?? 0;
350
+ const viewportCoverage = candidate.viewportCoverage ?? 1;
351
+ const relativeAreaToTarget = candidate.relativeAreaToTarget ?? 0;
352
+ return Boolean(ownerCandidateSelectorOf(candidate) &&
353
+ !ownerCandidateActsAsBoundary(candidate) &&
354
+ ownerCandidateRefinesContext(candidate) &&
355
+ contextBlockCount >= 2 &&
356
+ descendantInteractiveCount >= 1 &&
357
+ descendantInteractiveCount <= 8 &&
358
+ viewportCoverage <= 0.55 &&
359
+ relativeAreaToTarget >= 2);
360
+ }
361
+ function nearestLocalOwnerBranchKeyOf(target, candidates) {
362
+ for (const candidate of candidates) {
363
+ const selector = ownerCandidateSelectorOf(candidate);
364
+ if (!selector || ownerCandidateActsAsBoundary(candidate)) {
365
+ continue;
366
+ }
367
+ return selector;
368
+ }
369
+ return `target:${observedTargetKey(target)}`;
370
+ }
371
+ function inferSharedOwnerSelectors(targets) {
372
+ const supportByBoundary = new Map();
373
+ const candidatesByTarget = targets.map((target) => resolutionCandidatesFor(target));
195
374
  for (const [index, target] of targets.entries()) {
196
- const seed = embeddedSurfaceSeedOf(target);
197
- if (!seed) {
375
+ const candidates = candidatesByTarget[index] ?? [];
376
+ if (candidates.length === 0) {
198
377
  continue;
199
378
  }
200
- seedsByIndex.set(index, seed);
201
- clusterCounts.set(seed.clusterKey, (clusterCounts.get(seed.clusterKey) ?? 0) + 1);
379
+ const boundaryKey = ownerBoundaryKeyOf(target, candidates);
380
+ const branchKey = nearestLocalOwnerBranchKeyOf(target, candidates);
381
+ const supportBySelector = supportByBoundary.get(boundaryKey) ?? new Map();
382
+ for (const candidate of candidates) {
383
+ const selector = ownerCandidateSelectorOf(candidate);
384
+ if (!selector || !ownerCandidateEligibleForSharedResolution(candidate)) {
385
+ continue;
386
+ }
387
+ const existing = supportBySelector.get(selector) ?? {
388
+ branchRefs: new Set(),
389
+ targetIndexes: new Set(),
390
+ hasPrimaryActionSupport: false,
391
+ hasIdentitySupport: false,
392
+ };
393
+ existing.branchRefs.add(branchKey);
394
+ existing.targetIndexes.add(index);
395
+ existing.hasPrimaryActionSupport ||= targetSupportsSharedOwnerPrimaryAction(target);
396
+ existing.hasIdentitySupport ||= targetSupportsSharedOwnerIdentity(target);
397
+ supportBySelector.set(selector, existing);
398
+ }
399
+ supportByBoundary.set(boundaryKey, supportBySelector);
202
400
  }
203
- return targets.map((target, index) => {
204
- const seed = seedsByIndex.get(index);
205
- if (!seed || (clusterCounts.get(seed.clusterKey) ?? 0) < 2) {
206
- return target;
401
+ const preferredSelectors = new Map();
402
+ for (const [index, target] of targets.entries()) {
403
+ const candidates = candidatesByTarget[index] ?? [];
404
+ if (candidates.length === 0) {
405
+ continue;
207
406
  }
208
- return {
209
- ...target,
210
- surfaceRef: target.surfaceRef ?? seed.surfaceRef,
211
- surfaceKind: target.surfaceKind ?? seed.surfaceKind,
212
- surfaceLabel: target.surfaceLabel ?? seed.surfaceLabel,
213
- surfaceSelector: target.surfaceSelector ?? seed.surfaceSelector,
214
- surfacePriority: target.surfacePriority ?? seed.surfacePriority,
215
- };
216
- });
407
+ const boundaryKey = ownerBoundaryKeyOf(target, candidates);
408
+ const supportBySelector = supportByBoundary.get(boundaryKey);
409
+ if (!supportBySelector) {
410
+ continue;
411
+ }
412
+ for (const candidate of candidates) {
413
+ const selector = ownerCandidateSelectorOf(candidate);
414
+ if (!selector || !ownerCandidateEligibleForSharedResolution(candidate)) {
415
+ continue;
416
+ }
417
+ const support = supportBySelector.get(selector);
418
+ if (!support) {
419
+ continue;
420
+ }
421
+ if (support.branchRefs.size >= 2 &&
422
+ support.targetIndexes.size >= 2 &&
423
+ support.hasPrimaryActionSupport &&
424
+ support.hasIdentitySupport) {
425
+ preferredSelectors.set(index, selector);
426
+ break;
427
+ }
428
+ }
429
+ }
430
+ return preferredSelectors;
431
+ }
432
+ function ownerCandidateTierOf(candidate) {
433
+ const surfaceKind = ownerCandidateKindOf(candidate);
434
+ if (surfaceKind === 'token' || surfaceKind === 'chip') {
435
+ return 0;
436
+ }
437
+ const explicitBoundary = surfaceKind ? EXPLICIT_BOUNDARY_OWNER_KINDS.has(surfaceKind) : false;
438
+ const shellLike = candidate.shell === true || (surfaceKind ? SHELL_BOUNDARY_OWNER_KINDS.has(surfaceKind) : false);
439
+ const boundary = candidate.boundary === true;
440
+ const directTextBlockCount = candidate.directTextBlockCount ?? 0;
441
+ const descendantInteractiveCount = candidate.descendantInteractiveCount ?? 0;
442
+ const viewportCoverage = candidate.viewportCoverage ?? 1;
443
+ const relativeAreaToTarget = candidate.relativeAreaToTarget ?? 0;
444
+ const peerCount = candidate.peerCount ?? 0;
445
+ if (relativeAreaToTarget < 1.5 ||
446
+ viewportCoverage > 0.8 ||
447
+ descendantInteractiveCount > 12 ||
448
+ !ownerCandidateRefinesContext(candidate)) {
449
+ return 0;
450
+ }
451
+ const richLocalOwner = directTextBlockCount >= 2 &&
452
+ descendantInteractiveCount >= 1 &&
453
+ descendantInteractiveCount <= 8 &&
454
+ viewportCoverage <= 0.45;
455
+ const repeatedItemLike = peerCount >= 2;
456
+ const itemLikeSurface = surfaceKind === 'card' || surfaceKind === 'listitem' || surfaceKind === 'row';
457
+ if (richLocalOwner && (repeatedItemLike || itemLikeSurface)) {
458
+ return 320;
459
+ }
460
+ if (richLocalOwner && !shellLike) {
461
+ return 280;
462
+ }
463
+ if (explicitBoundary &&
464
+ directTextBlockCount >= 1 &&
465
+ descendantInteractiveCount >= 1 &&
466
+ descendantInteractiveCount <= 8 &&
467
+ viewportCoverage <= 0.55) {
468
+ return 240;
469
+ }
470
+ if (directTextBlockCount >= 1 &&
471
+ repeatedItemLike &&
472
+ descendantInteractiveCount >= 1 &&
473
+ descendantInteractiveCount <= 4 &&
474
+ viewportCoverage <= 0.35) {
475
+ return 220;
476
+ }
477
+ if (shellLike &&
478
+ directTextBlockCount >= 1 &&
479
+ descendantInteractiveCount >= 1 &&
480
+ descendantInteractiveCount <= 8 &&
481
+ viewportCoverage <= 0.5) {
482
+ return 140;
483
+ }
484
+ if (boundary &&
485
+ directTextBlockCount >= 1 &&
486
+ descendantInteractiveCount >= 1 &&
487
+ descendantInteractiveCount <= 8 &&
488
+ viewportCoverage <= 0.5) {
489
+ return 180;
490
+ }
491
+ return 0;
492
+ }
493
+ function compareOwnerCandidates(left, right) {
494
+ const tierDelta = ownerCandidateTierOf(right) - ownerCandidateTierOf(left);
495
+ if (tierDelta !== 0) {
496
+ return tierDelta;
497
+ }
498
+ const leftDepth = left.depth ?? Number.MAX_SAFE_INTEGER;
499
+ const rightDepth = right.depth ?? Number.MAX_SAFE_INTEGER;
500
+ if (leftDepth !== rightDepth) {
501
+ return leftDepth - rightDepth;
502
+ }
503
+ const leftPriority = left.surfacePriority ?? 0;
504
+ const rightPriority = right.surfacePriority ?? 0;
505
+ if (leftPriority !== rightPriority) {
506
+ return rightPriority - leftPriority;
507
+ }
508
+ const leftTextBlocks = left.directTextBlockCount ?? 0;
509
+ const rightTextBlocks = right.directTextBlockCount ?? 0;
510
+ return rightTextBlocks - leftTextBlocks;
511
+ }
512
+ function mergeSurfaceSelectors(primary, additional) {
513
+ const selectors = new Set();
514
+ if (primary) {
515
+ selectors.add(primary);
516
+ }
517
+ for (const selector of additional ?? []) {
518
+ if (selector) {
519
+ selectors.add(selector);
520
+ }
521
+ }
522
+ return selectors.size > 0 ? [...selectors] : undefined;
523
+ }
524
+ function inferredOwnerSurfaceKindOf(candidate, options) {
525
+ const hintedKind = ownerCandidateKindHintOf(candidate) ?? normalizeOwnerText(candidate.kind)?.toLowerCase();
526
+ if (hintedKind && hintedKind !== 'div') {
527
+ return hintedKind;
528
+ }
529
+ const directTextBlockCount = candidate.directTextBlockCount ?? 0;
530
+ const contextBlockCount = options?.sharedOwner === true
531
+ ? ownerCandidateContextBlockCountOf(candidate)
532
+ : directTextBlockCount;
533
+ const descendantInteractiveCount = candidate.descendantInteractiveCount ?? 0;
534
+ const viewportCoverage = candidate.viewportCoverage ?? 1;
535
+ const relativeAreaToTarget = candidate.relativeAreaToTarget ?? 0;
536
+ const peerCount = candidate.peerCount ?? 0;
537
+ const shellLike = candidate.shell === true || (hintedKind ? SHELL_BOUNDARY_OWNER_KINDS.has(hintedKind) : false);
538
+ if (shellLike) {
539
+ return undefined;
540
+ }
541
+ const maxInteractiveCount = options?.sharedOwner === true ? 8 : 4;
542
+ const richLocalOwner = contextBlockCount >= 2 &&
543
+ descendantInteractiveCount >= 1 &&
544
+ descendantInteractiveCount <= maxInteractiveCount &&
545
+ viewportCoverage <= 0.45;
546
+ if (richLocalOwner &&
547
+ (peerCount >= 1 || relativeAreaToTarget >= 4 || options?.sharedOwner === true)) {
548
+ return 'card';
549
+ }
550
+ if (contextBlockCount >= 1 &&
551
+ descendantInteractiveCount >= 1 &&
552
+ descendantInteractiveCount <= 4 &&
553
+ viewportCoverage <= 0.35 &&
554
+ relativeAreaToTarget >= 2) {
555
+ return 'group';
556
+ }
557
+ return undefined;
558
+ }
559
+ function ownerCandidateReusesExistingSurface(target, candidate) {
560
+ const candidateSelector = normalizeOwnerText(candidate.selector);
561
+ const currentSurfaceSelector = normalizeOwnerText(target.surfaceSelector);
562
+ return Boolean(candidateSelector && currentSurfaceSelector && candidateSelector === currentSurfaceSelector);
563
+ }
564
+ function resolveTargetOwnedSurface(target, preferredSharedOwnerSelector) {
565
+ const canonicalStructuredGridSurface = canonicalStructuredGridSurfaceOf(target);
566
+ if (canonicalStructuredGridSurface) {
567
+ return canonicalStructuredGridSurface;
568
+ }
569
+ const candidates = resolutionCandidatesFor(target);
570
+ if (!candidates || candidates.length === 0) {
571
+ return undefined;
572
+ }
573
+ const scoredCandidates = candidates.filter((candidate) => ownerCandidateTierOf(candidate) > 0);
574
+ const preferredCandidate = preferredSharedOwnerSelector
575
+ ? candidates.find((candidate) => ownerCandidateSelectorOf(candidate) === preferredSharedOwnerSelector &&
576
+ ownerCandidateEligibleForSharedResolution(candidate))
577
+ : undefined;
578
+ const bestCandidate = preferredCandidate ?? [...scoredCandidates].sort(compareOwnerCandidates)[0];
579
+ if (!bestCandidate?.selector) {
580
+ return undefined;
581
+ }
582
+ const reusesExistingSurface = ownerCandidateReusesExistingSurface(target, bestCandidate);
583
+ const sharedOwnerSelected = preferredCandidate !== undefined && bestCandidate === preferredCandidate;
584
+ const surfaceKind = inferredOwnerSurfaceKindOf(bestCandidate, {
585
+ sharedOwner: sharedOwnerSelected,
586
+ }) ?? (reusesExistingSurface ? target.surfaceKind : undefined);
587
+ const surfaceLabel = ownerCandidateSurfaceLabelOf(bestCandidate) ??
588
+ (reusesExistingSurface
589
+ ? (normalizeOwnerText(target.surfaceLabel) ?? normalizeOwnerText(target.fallbackSurfaceLabel))
590
+ : undefined);
591
+ const surfaceSelectors = mergeSurfaceSelectors(bestCandidate.selector, reusesExistingSurface ? target.surfaceSelectors : undefined);
592
+ const surfacePriority = bestCandidate.surfacePriority ??
593
+ defaultPriorityForOwnerKind(surfaceKind) ??
594
+ (reusesExistingSurface ? target.surfacePriority : undefined);
595
+ return {
596
+ surfaceKind,
597
+ surfaceLabel,
598
+ fallbackSurfaceLabel: surfaceLabel ?? (reusesExistingSurface ? target.fallbackSurfaceLabel : undefined),
599
+ surfaceSelector: bestCandidate.selector,
600
+ surfaceSelectors,
601
+ surfacePriority,
602
+ };
217
603
  }
218
604
  function parseNumericAmount(value) {
219
605
  const normalized = value.replace(/,/g, '.').trim();
@@ -417,52 +803,63 @@ function inferCapability(target, allowedActions) {
417
803
  }
418
804
  return 'actionable';
419
805
  }
420
- export function annotateDomTargets(targets) {
421
- const annotated = materializeEmbeddedControlSurfaces(targets.map((target) => {
422
- const allowedActions = inferAllowedActions(target);
423
- const controlFamily = inferControlFamilyFromFacts({
424
- kind: target.kind,
425
- role: target.role,
426
- label: target.label,
427
- displayLabel: target.displayLabel,
428
- interactionHint: target.interactionHint,
429
- text: target.text,
430
- placeholder: target.placeholder,
431
- inputName: target.inputName,
432
- inputType: target.inputType,
433
- autocomplete: target.autocomplete,
434
- ariaAutocomplete: target.ariaAutocomplete,
435
- surfaceKind: target.surfaceKind,
436
- controlsSurfaceSelector: target.controlsSurfaceSelector,
437
- states: target.states,
438
- structure: target.structure,
439
- }, allowedActions);
440
- const availability = inferAvailability(target);
441
- const capability = inferCapability(target, allowedActions);
442
- const acceptancePolicy = inferAcceptancePolicy(target, allowedActions);
443
- const explicitSurfaceRef = buildSurfaceRef(target);
444
- const syntheticFormSurfaceRef = explicitSurfaceRef === undefined ? buildSyntheticFormSurfaceId(target) : undefined;
445
- const surfaceRef = explicitSurfaceRef ?? syntheticFormSurfaceRef;
446
- const surfaceKind = target.surfaceKind ?? (syntheticFormSurfaceRef ? 'form' : undefined);
447
- const surfaceLabel = target.surfaceLabel ??
448
- (syntheticFormSurfaceRef
449
- ? (target.context?.landmark?.label ?? target.context?.group?.label ?? 'Form')
450
- : undefined);
451
- const surfacePriority = target.surfacePriority ?? (surfaceKind === 'form' ? 70 : undefined);
452
- return {
453
- ...target,
454
- surfaceKind,
455
- surfaceLabel,
456
- surfacePriority,
457
- capability,
458
- availability,
459
- allowedActions,
460
- controlFamily,
461
- acceptancePolicy,
462
- surfaceRef,
463
- };
464
- }));
465
- return annotated.map((target, index, current) => {
806
+ function withAnnotatedSemantics(target) {
807
+ const allowedActions = inferAllowedActions(target);
808
+ const controlFamily = inferControlFamilyFromFacts({
809
+ kind: target.kind,
810
+ role: target.role,
811
+ label: target.label,
812
+ displayLabel: target.displayLabel,
813
+ interactionHint: target.interactionHint,
814
+ text: target.text,
815
+ placeholder: target.placeholder,
816
+ inputName: target.inputName,
817
+ inputType: target.inputType,
818
+ autocomplete: target.autocomplete,
819
+ ariaAutocomplete: target.ariaAutocomplete,
820
+ surfaceKind: target.surfaceKind,
821
+ controlsSurfaceSelector: target.controlsSurfaceSelector,
822
+ states: target.states,
823
+ structure: target.structure,
824
+ }, allowedActions);
825
+ const availability = inferAvailability(target);
826
+ const capability = inferCapability(target, allowedActions);
827
+ const acceptancePolicy = inferAcceptancePolicy(target, allowedActions);
828
+ return {
829
+ ...target,
830
+ capability,
831
+ availability,
832
+ allowedActions,
833
+ controlFamily,
834
+ acceptancePolicy,
835
+ };
836
+ }
837
+ function withResolvedSurface(target, preferredSharedOwnerSelector) {
838
+ const ownedSurface = resolveTargetOwnedSurface(target, preferredSharedOwnerSelector);
839
+ const surfaceResolvedTarget = ownedSurface ? { ...target, ...ownedSurface } : target;
840
+ const explicitSurfaceRef = buildSurfaceRef(surfaceResolvedTarget);
841
+ const syntheticFormSurfaceRef = explicitSurfaceRef === undefined
842
+ ? buildSyntheticFormSurfaceId(surfaceResolvedTarget)
843
+ : undefined;
844
+ const surfaceRef = explicitSurfaceRef ?? syntheticFormSurfaceRef;
845
+ const surfaceKind = surfaceResolvedTarget.surfaceKind ?? (syntheticFormSurfaceRef ? 'form' : undefined);
846
+ const surfaceLabel = surfaceResolvedTarget.surfaceLabel ??
847
+ (syntheticFormSurfaceRef
848
+ ? (surfaceResolvedTarget.context?.landmark?.label ??
849
+ surfaceResolvedTarget.context?.group?.label ??
850
+ 'Form')
851
+ : undefined);
852
+ const surfacePriority = surfaceResolvedTarget.surfacePriority ?? (surfaceKind === 'form' ? 70 : undefined);
853
+ return {
854
+ ...surfaceResolvedTarget,
855
+ surfaceKind,
856
+ surfaceLabel,
857
+ surfacePriority,
858
+ surfaceRef,
859
+ };
860
+ }
861
+ function attachOwnerIndices(targets) {
862
+ return targets.map((target, index, current) => {
466
863
  const compactRemoveItemLabel = compactRemoveItemLabelOf(target);
467
864
  const compactRemoveOwnerIndex = isCompactRemoveActionTarget(target) && compactRemoveItemLabel
468
865
  ? current.findIndex((candidate, candidateIndex) => {
@@ -515,14 +912,20 @@ export function annotateDomTargets(targets) {
515
912
  return true;
516
913
  if (candidate.structure?.family === 'structured-grid')
517
914
  return false;
518
- if (candidate.capability === 'informational' && !explicitController)
915
+ if (candidate.capability === 'informational' && !explicitController) {
519
916
  return false;
917
+ }
520
918
  return (candidate.allowedActions?.includes('click') &&
521
919
  ['input', 'select', 'button', 'combobox'].includes((candidate.kind || '').toLowerCase()));
522
920
  });
523
921
  return ownerIndex >= 0 ? { ...target, ownerIndex } : target;
524
922
  });
525
923
  }
924
+ export function annotateDomTargets(targets) {
925
+ const semanticallyAnnotatedTargets = targets.map(withAnnotatedSemantics);
926
+ const preferredSharedOwnerSelectors = inferSharedOwnerSelectors(semanticallyAnnotatedTargets);
927
+ return attachOwnerIndices(semanticallyAnnotatedTargets.map((target, index) => withResolvedSurface(target, preferredSharedOwnerSelectors.get(index))));
928
+ }
526
929
  export function orderBySurfaceCompetition(targets) {
527
930
  const iframeFieldLike = (target) => {
528
931
  const kind = (target.kind ?? '').toLowerCase();
@@ -603,3 +1006,6 @@ export function orderBySurfaceCompetition(targets) {
603
1006
  return (left.ordinal ?? 0) - (right.ordinal ?? 0);
604
1007
  });
605
1008
  }
1009
+ export function normalizePostEnrichmentDomTargets(targets) {
1010
+ return compressSemanticallyDuplicateTargets(orderBySurfaceCompetition(annotateDomTargets(targets)));
1011
+ }