@browserbridge/bbx 1.11.5 → 1.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/PRIVACY.md +1 -0
- package/package.json +3 -1
- package/packages/agent-client/src/client.js +18 -1
- package/packages/agent-client/src/types.ts +2 -0
- package/packages/mcp-server/src/handlers-capture.js +17 -7
- package/packages/mcp-server/src/handlers-dom.js +6 -1
- package/packages/mcp-server/src/handlers-utils.js +3 -0
- package/packages/mcp-server/src/server.js +49 -2
- package/packages/native-host/src/daemon.js +78 -4
- package/packages/native-host/src/native-host.js +40 -2
- package/packages/native-host/src/window-actions.js +115 -0
- package/packages/protocol/src/protocol.js +146 -15
- package/packages/protocol/src/registry.js +8 -6
- package/packages/protocol/src/summary.js +184 -32
- package/packages/protocol/src/types.ts +58 -7
- package/packages/protocol/src/window-access.ts +29 -0
- package/packages/protocol/src/window-actions.js +46 -0
- package/skills/browser-bridge/SKILL.md +18 -12
- package/skills/browser-bridge/references/interaction.md +12 -16
- package/skills/browser-bridge/references/protocol.md +16 -6
- package/skills/browser-bridge/references/token-efficiency.md +12 -6
|
@@ -101,6 +101,126 @@ function describeTabResult(method, result) {
|
|
|
101
101
|
}
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
+
const ROUTINE_STRATEGIES = new Set(['elementRef', 'selector-first', 'locator-first']);
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Keep input evidence lean: resolution/execution metadata only when something
|
|
108
|
+
* notable happened (scroll, wait, ranking, recovery, CDP, fallback), plus any
|
|
109
|
+
* observed effects. Routine successes cost a few tokens instead of ~100.
|
|
110
|
+
*
|
|
111
|
+
* @param {Record<string, unknown>} r
|
|
112
|
+
* @param {Record<string, unknown>} [extra]
|
|
113
|
+
* @returns {Record<string, unknown>}
|
|
114
|
+
*/
|
|
115
|
+
function compactInputEvidence(r, extra = {}) {
|
|
116
|
+
/** @type {Record<string, unknown>} */
|
|
117
|
+
const evidence = { elementRef: r.elementRef ?? null, ...extra };
|
|
118
|
+
const resolution = toRecord(r.resolution);
|
|
119
|
+
if (
|
|
120
|
+
resolution.scrolled === true ||
|
|
121
|
+
resolution.recovered === true ||
|
|
122
|
+
(typeof resolution.waitedMs === 'number' && resolution.waitedMs > 0) ||
|
|
123
|
+
(typeof resolution.strategy === 'string' && !ROUTINE_STRATEGIES.has(resolution.strategy))
|
|
124
|
+
) {
|
|
125
|
+
/** @type {Record<string, unknown>} */
|
|
126
|
+
const notable = { strategy: resolution.strategy };
|
|
127
|
+
for (const key of ['scrolled', 'recovered', 'waitedMs', 'candidateCount', 'oldRef', 'newRef']) {
|
|
128
|
+
const value = resolution[key];
|
|
129
|
+
if (value !== undefined && value !== false && !(key === 'candidateCount' && value === 1)) {
|
|
130
|
+
notable[key] = value;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
evidence.resolution = notable;
|
|
134
|
+
}
|
|
135
|
+
const execution = toRecord(r.execution);
|
|
136
|
+
if (execution.actualMode === 'cdp' || execution.fallbackReason) {
|
|
137
|
+
evidence.execution = {
|
|
138
|
+
actualMode: execution.actualMode,
|
|
139
|
+
...(execution.fallbackReason ? { fallbackReason: execution.fallbackReason } : {}),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
if (r.effects && typeof r.effects === 'object') evidence.effects = r.effects;
|
|
143
|
+
return Object.fromEntries(Object.entries(evidence).filter(([, value]) => value !== undefined));
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Health evidence without idle diagnostics: zero-valued recovery counters,
|
|
148
|
+
* internal latency stats, and idle debugger/capture state are dropped, which
|
|
149
|
+
* cuts a routine health check from ~1k tokens to ~200.
|
|
150
|
+
*
|
|
151
|
+
* @param {Record<string, unknown>} result
|
|
152
|
+
* @returns {Record<string, unknown>}
|
|
153
|
+
*/
|
|
154
|
+
function compactHealthEvidence(result) {
|
|
155
|
+
const { recovery, domBaselines, debugger: debuggerState, capture, ...rest } = result;
|
|
156
|
+
/** @type {Record<string, unknown>} */
|
|
157
|
+
const evidence = { ...rest };
|
|
158
|
+
/** @type {Record<string, Record<string, unknown>>} */
|
|
159
|
+
const activeRecovery = {};
|
|
160
|
+
for (const [scope, value] of Object.entries(toRecord(recovery))) {
|
|
161
|
+
const events = toRecord(toRecord(value).events);
|
|
162
|
+
for (const [event, stats] of Object.entries(events)) {
|
|
163
|
+
const record = toRecord(stats);
|
|
164
|
+
if (record.attempts || record.pending || record.activeLoop) {
|
|
165
|
+
activeRecovery[`${scope}.${event}`] = {
|
|
166
|
+
attempts: record.attempts,
|
|
167
|
+
failures: record.failures,
|
|
168
|
+
...(record.activeLoop ? { activeLoop: true } : {}),
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
if (Object.keys(activeRecovery).length) evidence.recovery = activeRecovery;
|
|
174
|
+
const baselineCount = toRecord(domBaselines).baselineCount;
|
|
175
|
+
if (typeof baselineCount === 'number' && baselineCount > 0) {
|
|
176
|
+
evidence.domBaselines = { baselineCount, bytes: toRecord(domBaselines).bytes };
|
|
177
|
+
}
|
|
178
|
+
const debuggerRecord = toRecord(debuggerState);
|
|
179
|
+
if (debuggerState && (debuggerRecord.status !== 'idle' || debuggerRecord.attachedTabCount)) {
|
|
180
|
+
evidence.debugger = debuggerState;
|
|
181
|
+
}
|
|
182
|
+
const captureRecord = toRecord(capture);
|
|
183
|
+
if (capture && captureRecord.state !== 'stopped') evidence.capture = capture;
|
|
184
|
+
return evidence;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* One-line digest of observed post-action effects.
|
|
189
|
+
*
|
|
190
|
+
* @param {unknown} value
|
|
191
|
+
* @returns {string}
|
|
192
|
+
*/
|
|
193
|
+
function describeEffects(value) {
|
|
194
|
+
if (!value || typeof value !== 'object') return '';
|
|
195
|
+
const effects = /** @type {Record<string, unknown>} */ (value);
|
|
196
|
+
/** @type {string[]} */
|
|
197
|
+
const parts = [];
|
|
198
|
+
if (effects.navigation === 'unloading') parts.push('page is navigating');
|
|
199
|
+
if (typeof effects.url === 'string') parts.push(`URL → ${truncateUrl(effects.url)}`);
|
|
200
|
+
for (const dialog of Array.isArray(effects.dialogs) ? effects.dialogs : []) {
|
|
201
|
+
const record = toRecord(dialog);
|
|
202
|
+
parts.push(
|
|
203
|
+
`${record.role ?? 'dialog'}${record.name ? ` "${record.name}"` : ''} opened [${record.ref}]`
|
|
204
|
+
);
|
|
205
|
+
}
|
|
206
|
+
for (const message of Array.isArray(effects.messages) ? effects.messages : []) {
|
|
207
|
+
const record = toRecord(message);
|
|
208
|
+
parts.push(`${record.role ?? 'status'}: "${record.text}"`);
|
|
209
|
+
}
|
|
210
|
+
const focused = toRecord(effects.focused);
|
|
211
|
+
if (focused.ref)
|
|
212
|
+
parts.push(
|
|
213
|
+
`focus → ${focused.role ?? ''}${focused.name ? ` "${focused.name}"` : ''} [${focused.ref}]`
|
|
214
|
+
);
|
|
215
|
+
const dom = toRecord(effects.dom);
|
|
216
|
+
if (typeof dom.added === 'number' || typeof dom.removed === 'number') {
|
|
217
|
+
parts.push(`DOM +${dom.added ?? 0}/-${dom.removed ?? 0}`);
|
|
218
|
+
}
|
|
219
|
+
if (toRecord(effects.target).removed === true) parts.push('target removed');
|
|
220
|
+
if (!parts.length) return effects.changed === false ? ' No visible effect.' : '';
|
|
221
|
+
return ` Effects: ${parts.join('; ')}.`;
|
|
222
|
+
}
|
|
223
|
+
|
|
104
224
|
/** @type {Record<string, ActionSummary>} */
|
|
105
225
|
const ACTION_SUMMARIES = {
|
|
106
226
|
performed: {
|
|
@@ -118,8 +238,9 @@ const ACTION_SUMMARIES = {
|
|
|
118
238
|
evidence: (r) => r,
|
|
119
239
|
},
|
|
120
240
|
hovered: {
|
|
121
|
-
text: (r) =>
|
|
122
|
-
|
|
241
|
+
text: (r) =>
|
|
242
|
+
`Hover ${r.hovered ? 'active' : 'failed'} on ${r.elementRef}.${describeEffects(r.effects)}`,
|
|
243
|
+
evidence: (r) => compactInputEvidence(r),
|
|
123
244
|
},
|
|
124
245
|
dragged: {
|
|
125
246
|
text: (r) =>
|
|
@@ -130,33 +251,25 @@ const ACTION_SUMMARIES = {
|
|
|
130
251
|
clicked: {
|
|
131
252
|
text: (r) => {
|
|
132
253
|
const mode = toRecord(r.execution).actualMode;
|
|
133
|
-
return `Clicked ${r.elementRef ?? 'element'}${typeof mode === 'string' ? ` via ${mode}` : ''}
|
|
254
|
+
return `Clicked ${r.elementRef ?? 'element'}${typeof mode === 'string' ? ` via ${mode}` : ''}.${describeEffects(r.effects)}`;
|
|
134
255
|
},
|
|
135
|
-
evidence: (r) => (
|
|
136
|
-
elementRef: r.elementRef,
|
|
137
|
-
resolution: r.resolution,
|
|
138
|
-
execution: r.execution,
|
|
139
|
-
}),
|
|
256
|
+
evidence: (r) => compactInputEvidence(r),
|
|
140
257
|
},
|
|
141
258
|
focused: {
|
|
142
|
-
text: (r) => `Focused ${r.elementRef ?? 'element'}
|
|
143
|
-
evidence: (r) => (
|
|
144
|
-
elementRef: r.elementRef,
|
|
145
|
-
resolution: r.resolution,
|
|
146
|
-
execution: r.execution,
|
|
147
|
-
}),
|
|
259
|
+
text: (r) => `Focused ${r.elementRef ?? 'element'}.${describeEffects(r.effects)}`,
|
|
260
|
+
evidence: (r) => compactInputEvidence(r),
|
|
148
261
|
},
|
|
149
262
|
typed: {
|
|
150
|
-
text: (r) => `Typed into ${r.elementRef ?? 'element'}
|
|
151
|
-
evidence: (r) => ({
|
|
152
|
-
elementRef: r.elementRef,
|
|
153
|
-
resolution: r.resolution,
|
|
154
|
-
execution: r.execution,
|
|
155
|
-
}),
|
|
263
|
+
text: (r) => `Typed into ${r.elementRef ?? 'element'}.${describeEffects(r.effects)}`,
|
|
264
|
+
evidence: (r) => compactInputEvidence(r, { value: r.value }),
|
|
156
265
|
},
|
|
157
266
|
pressed: {
|
|
158
|
-
text: (r) => `Key pressed${r.key ? ` (${r.key})` : ''}
|
|
159
|
-
evidence: (r) =>
|
|
267
|
+
text: (r) => `Key pressed${r.key ? ` (${r.key})` : ''}.${describeEffects(r.effects)}`,
|
|
268
|
+
evidence: (r) => {
|
|
269
|
+
const { resolution: _resolution, execution: _execution, effects: _effects, ...rest } = r;
|
|
270
|
+
const { elementRef: _elementRef, ...compact } = compactInputEvidence(r);
|
|
271
|
+
return { ...rest, ...compact };
|
|
272
|
+
},
|
|
160
273
|
},
|
|
161
274
|
navigated: {
|
|
162
275
|
text: (r) => `Navigated to ${r.url ?? 'page'}.`,
|
|
@@ -242,6 +355,19 @@ function getMetaCostClass(meta, field) {
|
|
|
242
355
|
* @returns {BridgeSummary}
|
|
243
356
|
*/
|
|
244
357
|
export function summarizeBridgeResponse(response, method) {
|
|
358
|
+
const summarized = summarizeBridgeResponseBody(response, method);
|
|
359
|
+
const routingNote = response.ok ? getTabRoutingNote(response.meta) : null;
|
|
360
|
+
return routingNote
|
|
361
|
+
? { ...summarized, summary: `${summarized.summary} ${routingNote}` }
|
|
362
|
+
: summarized;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* @param {BridgeResponse} response
|
|
367
|
+
* @param {string} [method]
|
|
368
|
+
* @returns {BridgeSummary}
|
|
369
|
+
*/
|
|
370
|
+
function summarizeBridgeResponseBody(response, method) {
|
|
245
371
|
const protocolWarning = getProtocolWarning(response.meta);
|
|
246
372
|
if (!response.ok) {
|
|
247
373
|
const recovery = response.error.recovery ?? getErrorRecovery(response.error.code);
|
|
@@ -350,7 +476,7 @@ export function summarizeBridgeResponse(response, method) {
|
|
|
350
476
|
access == null
|
|
351
477
|
? ''
|
|
352
478
|
: access.enabled
|
|
353
|
-
? ` Access: ${access.routeReady ? `ready on tab ${access.routeTabId}.` : `enabled${typeof access.reason === 'string' ? ` (${access.reason})` : '.'}`}`
|
|
479
|
+
? ` Access: ${access.routeReady ? `ready on tab ${access.routeTabId}.` : `enabled${typeof access.reason === 'string' ? ` (${access.reason})` : '.'}`}${typeof access.workingTabId === 'number' && access.workingTabId !== access.routeTabId ? ` Your working tab is ${access.workingTabId}.` : ''}`
|
|
354
480
|
: ' Access: disabled.';
|
|
355
481
|
const connectedExtensions = Array.isArray(result.connectedExtensions)
|
|
356
482
|
? /** @type {Array<Record<string, unknown>>} */ (result.connectedExtensions)
|
|
@@ -369,7 +495,7 @@ export function summarizeBridgeResponse(response, method) {
|
|
|
369
495
|
`Daemon: ${result.daemon}. Extension: ${extensionSummary}.${accessSummary}`,
|
|
370
496
|
protocolWarning
|
|
371
497
|
),
|
|
372
|
-
evidence: result,
|
|
498
|
+
evidence: compactHealthEvidence(result),
|
|
373
499
|
};
|
|
374
500
|
}
|
|
375
501
|
if (Array.isArray(result.mcpClients) && Array.isArray(result.skillTargets)) {
|
|
@@ -470,6 +596,18 @@ export function summarizeBridgeResponse(response, method) {
|
|
|
470
596
|
})),
|
|
471
597
|
};
|
|
472
598
|
}
|
|
599
|
+
if (typeof result.outline === 'string') {
|
|
600
|
+
const count = typeof result.count === 'number' ? result.count : 0;
|
|
601
|
+
const refs = result.source === 'dom' ? '; refs work as input targets' : '';
|
|
602
|
+
return {
|
|
603
|
+
ok: true,
|
|
604
|
+
summary: appendProtocolWarning(
|
|
605
|
+
`Accessibility outline: ${count} node(s)${result.truncated === true ? ', truncated' : ''}${refs}.`,
|
|
606
|
+
protocolWarning
|
|
607
|
+
),
|
|
608
|
+
evidence: result.outline,
|
|
609
|
+
};
|
|
610
|
+
}
|
|
473
611
|
if (
|
|
474
612
|
Array.isArray(result.nodes) &&
|
|
475
613
|
typeof result.total === 'number' &&
|
|
@@ -528,6 +666,7 @@ export function summarizeBridgeResponse(response, method) {
|
|
|
528
666
|
if (Array.isArray(n.children) && n.children.length) {
|
|
529
667
|
entry.childCount = n.children.length;
|
|
530
668
|
}
|
|
669
|
+
if (n.visible === false) entry.visible = false;
|
|
531
670
|
return entry;
|
|
532
671
|
});
|
|
533
672
|
const isFind = method === 'dom.find_by_text' || method === 'dom.find_by_role';
|
|
@@ -545,6 +684,9 @@ export function summarizeBridgeResponse(response, method) {
|
|
|
545
684
|
} else {
|
|
546
685
|
summary = `Found ${count} matching element(s)${scanned === null ? '' : ` after scanning ${scanned}`}.`;
|
|
547
686
|
}
|
|
687
|
+
if (typeof result.hiddenMatches === 'number' && result.hiddenMatches > 0) {
|
|
688
|
+
summary += ` ${result.hiddenMatches} hidden match(es) omitted (includeHidden: true to list).`;
|
|
689
|
+
}
|
|
548
690
|
return {
|
|
549
691
|
ok: true,
|
|
550
692
|
summary: appendProtocolWarning(summary, protocolWarning),
|
|
@@ -754,20 +896,18 @@ export function summarizeBridgeResponse(response, method) {
|
|
|
754
896
|
return {
|
|
755
897
|
ok: true,
|
|
756
898
|
summary: appendProtocolWarning(
|
|
757
|
-
`${method} completed via ${execution.actualMode ?? 'dom'}${result.elementRef ? ` on ${result.elementRef}` : ''}
|
|
899
|
+
`${method} completed via ${execution.actualMode ?? 'dom'}${result.elementRef ? ` on ${result.elementRef}` : ''}.${describeEffects(result.effects)}`,
|
|
758
900
|
protocolWarning
|
|
759
901
|
),
|
|
760
|
-
evidence: {
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
destinationRef: result.destinationRef ?? null,
|
|
902
|
+
evidence: compactInputEvidence(result, {
|
|
903
|
+
sourceRef: result.sourceRef ?? undefined,
|
|
904
|
+
destinationRef: result.destinationRef ?? undefined,
|
|
764
905
|
typed: result.typed,
|
|
765
906
|
checked: result.checked,
|
|
766
907
|
selectedValues: result.selectedValues,
|
|
767
908
|
mode: result.mode,
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
},
|
|
909
|
+
value: result.value,
|
|
910
|
+
}),
|
|
771
911
|
};
|
|
772
912
|
}
|
|
773
913
|
if (typeof result.tabId === 'number' && typeof result.url === 'string') {
|
|
@@ -1006,6 +1146,18 @@ function getProtocolWarning(meta) {
|
|
|
1006
1146
|
: null;
|
|
1007
1147
|
}
|
|
1008
1148
|
|
|
1149
|
+
/**
|
|
1150
|
+
* Tell the agent when it kept working in its own tab while the user looked at
|
|
1151
|
+
* a different one, so "this tab" requests can be redirected explicitly.
|
|
1152
|
+
*
|
|
1153
|
+
* @param {Record<string, unknown> | null | undefined} meta
|
|
1154
|
+
* @returns {string | null}
|
|
1155
|
+
*/
|
|
1156
|
+
function getTabRoutingNote(meta) {
|
|
1157
|
+
if (typeof meta?.tab_id !== 'number' || typeof meta.active_tab_id !== 'number') return null;
|
|
1158
|
+
return `Ran in working tab ${meta.tab_id}; the user's active tab is ${meta.active_tab_id} (pass tabId to switch).`;
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1009
1161
|
/**
|
|
1010
1162
|
* @param {string} summary
|
|
1011
1163
|
* @param {string | null} warning
|
|
@@ -166,6 +166,14 @@ export interface BridgeMeta {
|
|
|
166
166
|
continuation_hint?: string | null;
|
|
167
167
|
automatic_retry?: { attempt: 2; reason: 'retryable_error' };
|
|
168
168
|
stale_recovery?: 'success' | 'failure';
|
|
169
|
+
/** Agent session for sticky working-tab routing; unset callers share a default. */
|
|
170
|
+
agent_session?: string;
|
|
171
|
+
/** Tab the request ran in. */
|
|
172
|
+
tab_id?: number;
|
|
173
|
+
/** How the tab was chosen: explicit tabId, the agent's working tab, or the active tab. */
|
|
174
|
+
tab_routing?: 'explicit' | 'working' | 'active';
|
|
175
|
+
/** Present when the user's active tab differs from the tab the request ran in. */
|
|
176
|
+
active_tab_id?: number;
|
|
169
177
|
[key: string]: unknown;
|
|
170
178
|
}
|
|
171
179
|
|
|
@@ -444,13 +452,42 @@ export interface NormalizedStyleQuery extends BridgeParams {
|
|
|
444
452
|
|
|
445
453
|
export interface InputTarget {
|
|
446
454
|
elementRef?: string;
|
|
455
|
+
/** CSS selector; with a locator it narrows the locator's candidates. */
|
|
447
456
|
selector?: string;
|
|
457
|
+
/** ARIA role, e.g. "button" or "textbox". */
|
|
458
|
+
role?: string;
|
|
459
|
+
/** Accessible name (substring, case-insensitive unless exact). */
|
|
460
|
+
name?: string;
|
|
461
|
+
/** Visible text of the innermost matching element. */
|
|
462
|
+
text?: string;
|
|
463
|
+
/** Associated label text for form controls. */
|
|
464
|
+
label?: string;
|
|
465
|
+
placeholder?: string;
|
|
466
|
+
/** data-testid / data-test-id / data-test / data-qa value. */
|
|
467
|
+
testId?: string;
|
|
468
|
+
exact?: boolean;
|
|
469
|
+
/** Zero-based index among visible locator matches. */
|
|
470
|
+
nth?: number;
|
|
448
471
|
}
|
|
449
472
|
|
|
450
|
-
export
|
|
473
|
+
export interface InputWaitObserveParams {
|
|
474
|
+
/** Wait up to this long for the target to exist and become actionable (0 disables). */
|
|
475
|
+
timeoutMs?: number;
|
|
476
|
+
/** Report post-action effects; true or { settleMs }. Defaults on for click/press_key/select/check. */
|
|
477
|
+
observe?: boolean | { settleMs?: number } | null;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
/** auto picks cdp only when trusted input is likely required (or the debugger is already attached). */
|
|
481
|
+
export type InputExecutionMode = 'dom' | 'cdp' | 'auto';
|
|
451
482
|
|
|
452
483
|
export interface InputResolutionMetadata {
|
|
453
|
-
strategy:
|
|
484
|
+
strategy:
|
|
485
|
+
| 'elementRef'
|
|
486
|
+
| 'selector-first'
|
|
487
|
+
| 'selector-ranked'
|
|
488
|
+
| 'locator-first'
|
|
489
|
+
| 'locator-ranked'
|
|
490
|
+
| 'stale-recovery';
|
|
454
491
|
candidateCount: number;
|
|
455
492
|
evaluatedCount: number;
|
|
456
493
|
scrolled: boolean;
|
|
@@ -460,6 +497,8 @@ export interface InputResolutionMetadata {
|
|
|
460
497
|
newRef?: string;
|
|
461
498
|
matchedFields?: string[];
|
|
462
499
|
confidenceBasis?: string;
|
|
500
|
+
/** Time spent waiting for the target to become actionable. */
|
|
501
|
+
waitedMs?: number;
|
|
463
502
|
}
|
|
464
503
|
|
|
465
504
|
export interface InputExecutionMetadata {
|
|
@@ -470,7 +509,7 @@ export interface InputExecutionMetadata {
|
|
|
470
509
|
targetCoordinates?: { x: number; y: number };
|
|
471
510
|
}
|
|
472
511
|
|
|
473
|
-
export interface InputActionParams {
|
|
512
|
+
export interface InputActionParams extends InputWaitObserveParams {
|
|
474
513
|
target?: InputTarget;
|
|
475
514
|
button?: 'left' | 'middle' | 'right';
|
|
476
515
|
clickCount?: number;
|
|
@@ -625,7 +664,7 @@ export interface NormalizedCdpDomSnapshotParams extends BridgeParams {
|
|
|
625
664
|
computedStyles: string[];
|
|
626
665
|
}
|
|
627
666
|
|
|
628
|
-
export interface CheckedActionParams {
|
|
667
|
+
export interface CheckedActionParams extends InputWaitObserveParams {
|
|
629
668
|
target?: InputTarget;
|
|
630
669
|
checked?: boolean;
|
|
631
670
|
executionMode?: InputExecutionMode;
|
|
@@ -639,7 +678,7 @@ export interface NormalizedCheckedAction extends BridgeParams {
|
|
|
639
678
|
recoverStale: boolean;
|
|
640
679
|
}
|
|
641
680
|
|
|
642
|
-
export interface SelectActionParams {
|
|
681
|
+
export interface SelectActionParams extends InputWaitObserveParams {
|
|
643
682
|
target?: InputTarget;
|
|
644
683
|
values?: string[];
|
|
645
684
|
labels?: string[];
|
|
@@ -816,6 +855,7 @@ export interface FindByTextParams {
|
|
|
816
855
|
exact?: boolean;
|
|
817
856
|
selector?: string;
|
|
818
857
|
maxResults?: number;
|
|
858
|
+
includeHidden?: boolean;
|
|
819
859
|
}
|
|
820
860
|
|
|
821
861
|
export interface NormalizedFindByTextParams extends BridgeParams {
|
|
@@ -823,20 +863,25 @@ export interface NormalizedFindByTextParams extends BridgeParams {
|
|
|
823
863
|
exact: boolean;
|
|
824
864
|
selector: string;
|
|
825
865
|
maxResults: number;
|
|
866
|
+
includeHidden: boolean;
|
|
826
867
|
}
|
|
827
868
|
|
|
828
869
|
export interface FindByRoleParams {
|
|
829
870
|
role?: string;
|
|
830
871
|
name?: string;
|
|
872
|
+
exact?: boolean;
|
|
831
873
|
selector?: string;
|
|
832
874
|
maxResults?: number;
|
|
875
|
+
includeHidden?: boolean;
|
|
833
876
|
}
|
|
834
877
|
|
|
835
878
|
export interface NormalizedFindByRoleParams extends BridgeParams {
|
|
836
879
|
role: string;
|
|
837
880
|
name: string;
|
|
881
|
+
exact: boolean;
|
|
838
882
|
selector: string;
|
|
839
883
|
maxResults: number;
|
|
884
|
+
includeHidden: boolean;
|
|
840
885
|
}
|
|
841
886
|
|
|
842
887
|
export interface GetHtmlParams {
|
|
@@ -853,7 +898,7 @@ export interface NormalizedGetHtmlParams extends BridgeParams {
|
|
|
853
898
|
maxLength: number;
|
|
854
899
|
}
|
|
855
900
|
|
|
856
|
-
export interface HoverParams {
|
|
901
|
+
export interface HoverParams extends InputWaitObserveParams {
|
|
857
902
|
target?: InputTarget;
|
|
858
903
|
duration?: number;
|
|
859
904
|
modifiers?: string[];
|
|
@@ -869,7 +914,7 @@ export interface NormalizedHoverParams extends BridgeParams {
|
|
|
869
914
|
recoverStale: boolean;
|
|
870
915
|
}
|
|
871
916
|
|
|
872
|
-
export interface DragParams {
|
|
917
|
+
export interface DragParams extends InputWaitObserveParams {
|
|
873
918
|
source?: InputTarget;
|
|
874
919
|
destination?: InputTarget;
|
|
875
920
|
offsetX?: number;
|
|
@@ -1053,6 +1098,10 @@ export interface AccessibilityTreeParams {
|
|
|
1053
1098
|
maxNodes?: number;
|
|
1054
1099
|
compact?: boolean;
|
|
1055
1100
|
interactiveOnly?: boolean;
|
|
1101
|
+
/** cdp (default): Chrome's AX tree via the debugger. dom: content-script outline with elementRefs, no debugger. */
|
|
1102
|
+
source?: 'cdp' | 'dom';
|
|
1103
|
+
/** outline: one compact `- role "name" [ref]` line per node (default for source=dom). */
|
|
1104
|
+
format?: 'tree' | 'outline';
|
|
1056
1105
|
}
|
|
1057
1106
|
|
|
1058
1107
|
export interface NormalizedAccessibilityTreeParams extends BridgeParams {
|
|
@@ -1061,6 +1110,8 @@ export interface NormalizedAccessibilityTreeParams extends BridgeParams {
|
|
|
1061
1110
|
maxNodes: number;
|
|
1062
1111
|
compact: boolean;
|
|
1063
1112
|
interactiveOnly: boolean;
|
|
1113
|
+
source?: 'dom';
|
|
1114
|
+
format?: 'outline';
|
|
1064
1115
|
}
|
|
1065
1116
|
|
|
1066
1117
|
export interface AccessibilityTreeNode {
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/** Window access metadata shared by the native relay and extension UI. */
|
|
2
|
+
export type EnabledWindowInfo = {
|
|
3
|
+
windowId: number;
|
|
4
|
+
title: string;
|
|
5
|
+
enabledAt: number;
|
|
6
|
+
};
|
|
7
|
+
|
|
8
|
+
export type BrowserWindowAccess = {
|
|
9
|
+
extensionId: string;
|
|
10
|
+
browserName: string | null;
|
|
11
|
+
profileLabel: string | null;
|
|
12
|
+
window: EnabledWindowInfo | null;
|
|
13
|
+
canControl?: boolean;
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
/** Local extension UI actions. These are not agent RPC methods. */
|
|
17
|
+
export type WindowAction = 'focus' | 'disable';
|
|
18
|
+
|
|
19
|
+
export type WindowActionCommand = {
|
|
20
|
+
requestId: string;
|
|
21
|
+
action: WindowAction;
|
|
22
|
+
windowId: number;
|
|
23
|
+
enabledAt: number;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export type WindowActionRequest = WindowActionCommand & { extensionId: string };
|
|
27
|
+
|
|
28
|
+
export type WindowActionResult = { requestId: string; ok: boolean; error?: string };
|
|
29
|
+
export type WindowActionUiResult = { action: WindowAction; ok: boolean; error?: string };
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* @param {unknown} value
|
|
5
|
+
* @returns {import('./window-access.js').WindowActionCommand | null}
|
|
6
|
+
*/
|
|
7
|
+
export function parseWindowActionCommand(value) {
|
|
8
|
+
if (!value || typeof value !== 'object') return null;
|
|
9
|
+
const message = /** @type {Record<string, unknown>} */ (value);
|
|
10
|
+
if (
|
|
11
|
+
typeof message.requestId !== 'string' ||
|
|
12
|
+
!message.requestId ||
|
|
13
|
+
message.requestId.length > 80 ||
|
|
14
|
+
(message.action !== 'focus' && message.action !== 'disable') ||
|
|
15
|
+
typeof message.windowId !== 'number' ||
|
|
16
|
+
!Number.isSafeInteger(message.windowId) ||
|
|
17
|
+
message.windowId <= 0 ||
|
|
18
|
+
typeof message.enabledAt !== 'number' ||
|
|
19
|
+
!Number.isFinite(message.enabledAt) ||
|
|
20
|
+
message.enabledAt < 0
|
|
21
|
+
)
|
|
22
|
+
return null;
|
|
23
|
+
return {
|
|
24
|
+
requestId: message.requestId,
|
|
25
|
+
action: message.action,
|
|
26
|
+
windowId: message.windowId,
|
|
27
|
+
enabledAt: message.enabledAt,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @param {unknown} value
|
|
33
|
+
* @returns {import('./window-access.js').WindowActionRequest | null}
|
|
34
|
+
*/
|
|
35
|
+
export function parseWindowActionRequest(value) {
|
|
36
|
+
const command = parseWindowActionCommand(value);
|
|
37
|
+
if (!command) return null;
|
|
38
|
+
const message = /** @type {Record<string, unknown>} */ (value);
|
|
39
|
+
if (
|
|
40
|
+
typeof message.extensionId !== 'string' ||
|
|
41
|
+
!message.extensionId ||
|
|
42
|
+
message.extensionId.length > 80
|
|
43
|
+
)
|
|
44
|
+
return null;
|
|
45
|
+
return { ...command, extensionId: message.extensionId };
|
|
46
|
+
}
|
|
@@ -53,7 +53,8 @@ bbx box <ref> # box model dimensions
|
|
|
53
53
|
bbx find <text> # find by text content
|
|
54
54
|
bbx find-role <role> [name] # find by ARIA role
|
|
55
55
|
bbx wait <selector> [timeoutMs] # wait for DOM element
|
|
56
|
-
bbx a11y-tree [maxNodes] [maxDepth] # accessibility tree
|
|
56
|
+
bbx a11y-tree [maxNodes] [maxDepth] # accessibility tree (CDP)
|
|
57
|
+
bbx call dom.get_accessibility_tree '{"source":"dom","interactiveOnly":true}' # actionable outline with refs
|
|
57
58
|
```
|
|
58
59
|
|
|
59
60
|
### Page & Evaluate
|
|
@@ -90,6 +91,8 @@ bbx press-key <key> [ref] # send key event
|
|
|
90
91
|
bbx cdp-press-key --tab <id> Escape # CDP key event without foreground focus
|
|
91
92
|
bbx hover <ref> # hover over element
|
|
92
93
|
bbx call input.scroll_into_view '{"target":{"elementRef":"el_123"}}' # ensure target is visible
|
|
94
|
+
bbx call input.click '{"target":{"role":"button","name":"Save"}}' # find + wait + click + effects in one call
|
|
95
|
+
bbx call input.fill '{"target":{"label":"Email"},"value":"a@b.co"}' # locate a field by its label
|
|
93
96
|
bbx call input.click '{"target":{"elementRef":"el_123"},"executionMode":"cdp"}' # optional native click
|
|
94
97
|
bbx patch-style <ref> prop=val... # apply style patch
|
|
95
98
|
bbx patch-text <ref> <text...> # apply text patch
|
|
@@ -131,10 +134,13 @@ After the Enable cue appears:
|
|
|
131
134
|
|
|
132
135
|
After access is enabled:
|
|
133
136
|
|
|
134
|
-
1.
|
|
135
|
-
2. If the user switches
|
|
136
|
-
3.
|
|
137
|
-
4.
|
|
137
|
+
1. Your first tab-bound call binds your session to the active tab: that becomes your **working tab**.
|
|
138
|
+
2. The working tab is sticky. If the user switches to another tab (to read mail, say), your calls keep running in your working tab; they never navigate or type into the tab the user is looking at.
|
|
139
|
+
3. Responses report `meta.tab_id`; when the user's active tab differs, the summary says so (`Ran in working tab 12; the user's active tab is 34`). If the user asks you to work in "this tab", pass that `tabId` once; the explicit tab becomes your new working tab. `tabs.create` and `tabs.activate` also move your working tab.
|
|
140
|
+
4. If your working tab was closed, calls fail with `TAB_MISMATCH` (`reason: working_tab_closed`) instead of silently using another tab; pick a tab with `tabId` or `tabs.list` (which marks `working: true`).
|
|
141
|
+
5. The binding expires after 15 idle minutes.
|
|
142
|
+
6. Do not stop at a generic "no access" message before making a real Browser Bridge call, because the first denied call is what triggers the UI cue.
|
|
143
|
+
7. CLI agents running separate `bbx` processes share one working tab unless each sets `BBX_SESSION=<name>`; MCP sessions are separated automatically.
|
|
138
144
|
|
|
139
145
|
## Error Recovery
|
|
140
146
|
|
|
@@ -148,7 +154,7 @@ After access is enabled:
|
|
|
148
154
|
| `INPUT_FOCUS_CHANGED` | No | Inspect focus handlers; do not replay native text automatically |
|
|
149
155
|
| `DIALOG_NOT_OPEN` | No | Trigger or inspect the dialog again |
|
|
150
156
|
| `DIALOG_ACTION_CONFLICT` | No | Inspect current dialog state; never auto-repeat accept/dismiss |
|
|
151
|
-
| `TAB_MISMATCH` | No | Tab closed or not found - use `tabs.list`
|
|
157
|
+
| `TAB_MISMATCH` | No | Tab closed or not found (`working_tab_closed`: your working tab is gone) - pass `tabId` or use `tabs.list` |
|
|
152
158
|
| `TIMEOUT` | Once | Retry once; if still failing, simplify (smaller `maxNodes`, narrower selector) |
|
|
153
159
|
| `CONTENT_SCRIPT_UNAVAILABLE` | No | Switch to a normal http(s) page in the enabled window |
|
|
154
160
|
| `EXTENSION_DISCONNECTED` | After 3s | Check Chrome is running; `bbx status` to verify, then retry |
|
|
@@ -181,22 +187,22 @@ they contain no page data, URLs, selectors, payloads, errors, or group names.
|
|
|
181
187
|
10. **Avoid debugger first** - prefer DOM/content-script methods (`dom.*`, `styles.*`, `layout.get_box_model`, `page.get_console`, `page.get_text`, `page.get_storage`, `page.get_network`) before any debugger-backed method. Escalate to CDP only when those cannot answer the question.
|
|
182
188
|
11. **Evaluate only when needed** - `page.evaluate` is powerful but debugger-backed; use it only when DOM, storage, console, network, or text reads cannot expose the needed state.
|
|
183
189
|
12. **Treat exact storage as sensitive** - use `page.get_storage` for metadata first. Call `sensitive.read` for one exact key only when necessary; never batch or automatically retry it, and never repeat a failed attempt without new evidence.
|
|
184
|
-
12. **Debugger-backed methods are last resort** - treat `page.evaluate`, `page.handle_dialog`, `dom.get_accessibility_tree
|
|
190
|
+
12. **Debugger-backed methods are last resort** - treat `page.evaluate`, `page.handle_dialog`, `dom.get_accessibility_tree` with the default CDP source, `page.get_network` with `source: 'cdp'`, input with `executionMode: 'cdp'`, `viewport.resize`, `performance.get_metrics`, `screenshot.capture_*`, and all `cdp.*` methods as escalation steps because they attach `chrome.debugger`. `performance.get_metrics` is a raw browser-maintained CDP counter point sample, not Web Vitals.
|
|
185
191
|
13. **Wait after change** - after editing source, wait for expected new text, a selector matching the changed attribute, or a detach/attach remount; waiting for `attached` on a selector that already exists does not prove HMR ran. Use `page.wait_for_load_state` for navigation, not HMR.
|
|
186
192
|
14. **Prime event buffers** - before reproducing a console or fetch/XHR issue, call `page.get_console` and default `page.get_network` once with `clear: true`; then reproduce and read without clearing. CDP all-resource capture instead requires explicit `start`, reproduce, `read`, and `stop` calls.
|
|
187
|
-
15. **
|
|
193
|
+
15. **Act on what you can name** - when you know the label but not the selector, put a locator straight into the input target (`{"role":"button","name":"Save"}`, `{"label":"Email"}`, `{"text":"Load more"}`, `{"placeholder":"Search"}`, `{"testId":"submit"}`; add `nth` to pick among matches). Use `dom.find_by_text` / `dom.find_by_role` only when you need to look before acting. Locators and finders search shadow DOM and iframes, use real accessible names (`<label for>`, wrapping labels, `aria-*`, `alt`), and skip hidden matches (`hiddenMatches` counts them; `includeHidden: true` lists them).
|
|
188
194
|
16. **Text extraction** - use `page.extract_content` for articles/documentation, or `page.get_text` for all visible UI text, instead of `dom.query` on body.
|
|
189
195
|
17. **Network monitoring** - use `page.get_network` to inspect API calls; auto-installs interceptor.
|
|
190
|
-
18. **
|
|
196
|
+
18. **Overview with the DOM outline** - for "what can I do on this page?", call `dom.get_accessibility_tree` with `source: "dom"` (add `interactiveOnly: true` for controls only). It returns compact lines like `- button "Save" [el_k7q2_1f] disabled` whose refs work directly as input targets, includes shadow DOM and iframes, and needs no debugger. Reserve the default CDP source for accessibility audits.
|
|
191
197
|
19. **Tailwind-aware** - when `page.get_state` returns `hints.tailwind: true`, load `references/tailwind.md`; avoid selecting by utility classes, prefer `find_by_text`/`find_by_role`; `dom.query` auto-escapes `[]` brackets.
|
|
192
|
-
20. **
|
|
193
|
-
21. **
|
|
198
|
+
20. **Read the effects before reading the page** - click, press_key, set_checked, select_option, and submitting type/fill return `effects` after a short settle: same-document `url` changes, `navigation: "unloading"`, newly opened `dialogs`, alert/status/live-region `messages`, the `focused` element, `dom` churn, and the target's new state (`value`, `checked`, `expanded`, `removed`). The summary digests them (`Effects: URL → /done; alert: "Saved"`). Only follow up with a read when the effects do not answer the question; `changed: false` means nothing visible happened. Pass `observe: true` to other actions, or `observe: false` to skip the settle.
|
|
199
|
+
21. **Let actions wait** - targeted actions wait up to `timeoutMs` (default 2500, max 15000, `0` disables) for the target to appear, become enabled/visible/unobscured, and stop animating. Do not add a separate `dom.wait_for` before an action. Honor the errors that remain after the wait, plus `resolution.waitedMs`. `cdp_press_key` and `scroll_into_view` use separate contracts; `input.fill.mode: auto` is a separate DOM setter/keystroke strategy.
|
|
194
200
|
22. **Stale recovery stays opt-in** - prefer re-querying. Use `recoverStale: true` once only when the same document/URL and a strong unique semantic descriptor should still identify the target; inspect `recovered`, old/new refs, and matched fields. A scan with more than 100 same-tag candidates fails as `ELEMENT_AMBIGUOUS`/`scan_incomplete` because uniqueness is not provable.
|
|
195
201
|
23. **Dialogs are explicit** - inspect first, then accept or dismiss only when intended. `expectedDialogId` is a pre-dispatch stale-decision check, not an atomic CDP binding; never auto-repeat `DIALOG_ACTION_CONFLICT`.
|
|
196
202
|
24. **DOM diffs use explicit baselines** - call `dom.baseline.create` before the action, `dom.baseline.compare` afterward, then `dom.baseline.release`. Use a narrow selector and bounded evidence. Never substitute unrelated DOM reads for the retained baseline, and recreate it after `DOM_BASELINE_INVALIDATED`.
|
|
197
203
|
25. **HAR export reads an armed capture** - use CDP capture `start`, reproduce, `network.export_har`/`bbx har`, then `stop`. Export never starts, stops, or clears capture. Inspect truncation plus `dropped`, `abandoned`, and `inflight` before claiming completeness.
|
|
198
204
|
26. **Keep timing inside the browser** - for rhythm, games, gestures, or any steps that must happen at set times or back-to-back, send one `input.perform` with the whole schedule (`atMs` from the sequence start, or `delayMs` after the previous step). Never spread timed input across tool calls (each call costs an agent round-trip) and never issue input as parallel tool calls (they land all at once).
|
|
199
|
-
27. **Use trusted input when the page needs a user gesture** - DOM-mode events are untrusted, so Chrome may refuse audio or media start, popups, clipboard writes, and fullscreen. Use `executionMode: "cdp"` (click, hover, drag, type, fill, press_key, touch)
|
|
205
|
+
27. **Use trusted input when the page needs a user gesture** - DOM-mode events are untrusted, so Chrome may refuse audio or media start, popups, clipboard writes, and fullscreen. `executionMode: "auto"` picks debugger input only for file inputs, media, `target=_blank` links, rich-text editors, or when the debugger is already attached (`execution.selectionReason` says why). Use `executionMode: "cdp"` to force it (click, hover, drag, type, fill, press_key, touch); `input.perform` accepts it once for every step. If a DOM click reports `changed: false` on a page that should react, retry once with `cdp` rather than repeating the DOM click. CDP input is not available for targets inside iframes.
|
|
200
206
|
28. **Simultaneous presses need touch** - a mouse has one pointer. For chords or multi-finger gestures use `input.touch` with several `points` (optionally `to` end positions for swipes and pinches); add `holdMs` to `input.click`/`input.press_key` when the press must last.
|
|
201
207
|
|
|
202
208
|
## Token Budget Quick Rules
|