@browserbridge/bbx 1.11.3 → 1.11.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +12 -8
- package/packages/mcp-server/src/handlers-capture.js +3 -1
- package/packages/mcp-server/src/server.js +10 -1
- package/packages/protocol/src/defaults.js +16 -0
- package/packages/protocol/src/protocol.js +223 -0
- package/packages/protocol/src/registry.js +26 -4
- package/packages/protocol/src/summary.js +65 -6
- package/packages/protocol/src/types.ts +99 -0
- package/skills/browser-bridge/SKILL.md +6 -2
- package/skills/browser-bridge/references/interaction.md +75 -6
- package/skills/browser-bridge/references/protocol.md +24 -5
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@browserbridge/bbx",
|
|
3
|
-
"version": "1.11.
|
|
3
|
+
"version": "1.11.5",
|
|
4
4
|
"private": false,
|
|
5
5
|
"keywords": [
|
|
6
6
|
"agent-tools",
|
|
@@ -73,25 +73,29 @@
|
|
|
73
73
|
"install-manifest": "node packages/native-host/bin/install-manifest.js"
|
|
74
74
|
},
|
|
75
75
|
"dependencies": {
|
|
76
|
-
"@modelcontextprotocol/server": "^2.
|
|
76
|
+
"@modelcontextprotocol/server": "^2.3.1",
|
|
77
77
|
"@mozilla/readability": "^0.6.0",
|
|
78
78
|
"linkedom": "^0.18.13",
|
|
79
79
|
"zod": "^4.6.5"
|
|
80
80
|
},
|
|
81
81
|
"devDependencies": {
|
|
82
|
-
"@modelcontextprotocol/client": "^2.
|
|
83
|
-
"@types/chrome": "^0.3.
|
|
84
|
-
"@types/node": "^26.6.
|
|
82
|
+
"@modelcontextprotocol/client": "^2.3.1",
|
|
83
|
+
"@types/chrome": "^0.3.4",
|
|
84
|
+
"@types/node": "^26.6.4",
|
|
85
85
|
"ajv": "^8.20.0",
|
|
86
86
|
"ajv-formats": "^3.0.1",
|
|
87
87
|
"c8": "^12.0.0",
|
|
88
88
|
"har-schema": "^2.0.0",
|
|
89
|
-
"oxfmt": "^0.
|
|
90
|
-
"oxlint": "^1.
|
|
91
|
-
"tsx": "^4.23.
|
|
89
|
+
"oxfmt": "^0.72.0",
|
|
90
|
+
"oxlint": "^1.87.0",
|
|
91
|
+
"tsx": "^4.23.15",
|
|
92
92
|
"typescript": "^7.0.2"
|
|
93
93
|
},
|
|
94
94
|
"engines": {
|
|
95
95
|
"node": ">=20"
|
|
96
|
+
},
|
|
97
|
+
"allowScripts": {
|
|
98
|
+
"esbuild@0.28.1": true,
|
|
99
|
+
"fsevents@2.3.3": true
|
|
96
100
|
}
|
|
97
101
|
}
|
|
@@ -348,7 +348,7 @@ function createInputOptions(args) {
|
|
|
348
348
|
}
|
|
349
349
|
|
|
350
350
|
/**
|
|
351
|
-
* @param {{ action: string, elementRef?: string, selector?: string, button?: string, clickCount?: number, text?: string, value?: string, mode?: 'auto' | 'setter' | 'keystrokes', executionMode?: 'dom' | 'cdp', recoverStale?: boolean, clear?: boolean, submit?: boolean, key?: string, code?: string, modifiers?: string[], checked?: boolean, values?: string[], labels?: string[], indexes?: number[], duration?: number, sourceElementRef?: string, sourceSelector?: string, destinationElementRef?: string, destinationSelector?: string, offsetX?: number, offsetY?: number, tabId?: number, destinationId?: string, budgetPreset?: 'quick' | 'normal' | 'deep' }} args
|
|
351
|
+
* @param {{ action: string, elementRef?: string, selector?: string, button?: string, clickCount?: number, holdMs?: number, text?: string, value?: string, mode?: 'auto' | 'setter' | 'keystrokes', executionMode?: 'dom' | 'cdp', recoverStale?: boolean, clear?: boolean, submit?: boolean, key?: string, code?: string, modifiers?: string[], checked?: boolean, values?: string[], labels?: string[], indexes?: number[], duration?: number, sourceElementRef?: string, sourceSelector?: string, destinationElementRef?: string, destinationSelector?: string, offsetX?: number, offsetY?: number, tabId?: number, destinationId?: string, budgetPreset?: 'quick' | 'normal' | 'deep' }} args
|
|
352
352
|
* @returns {Promise<ToolResult>}
|
|
353
353
|
*/
|
|
354
354
|
export async function handleInputTool(args) {
|
|
@@ -391,6 +391,7 @@ export async function handleInputTool(args) {
|
|
|
391
391
|
button: args.button,
|
|
392
392
|
clickCount: args.clickCount,
|
|
393
393
|
modifiers: args.modifiers,
|
|
394
|
+
...(typeof args.holdMs === 'number' ? { holdMs: args.holdMs } : {}),
|
|
394
395
|
...createInputOptions(args),
|
|
395
396
|
},
|
|
396
397
|
{
|
|
@@ -464,6 +465,7 @@ export async function handleInputTool(args) {
|
|
|
464
465
|
target,
|
|
465
466
|
key: args.key,
|
|
466
467
|
modifiers: args.modifiers,
|
|
468
|
+
...(typeof args.holdMs === 'number' ? { holdMs: args.holdMs } : {}),
|
|
467
469
|
...createInputOptions(args),
|
|
468
470
|
},
|
|
469
471
|
{
|
|
@@ -707,7 +707,7 @@ export function createBridgeMcpServer(options = {}) {
|
|
|
707
707
|
.enum(['dom', 'cdp'])
|
|
708
708
|
.optional()
|
|
709
709
|
.describe(
|
|
710
|
-
'Input execution path (default: dom; CDP supports click, hover, drag, type, fill)'
|
|
710
|
+
'Input execution path (default: dom; CDP supports click, hover, drag, type, fill, press_key). Use cdp when the page needs trusted input, e.g. to start audio or media.'
|
|
711
711
|
),
|
|
712
712
|
recoverStale: z
|
|
713
713
|
.boolean()
|
|
@@ -729,6 +729,15 @@ export function createBridgeMcpServer(options = {}) {
|
|
|
729
729
|
.array(z.enum(['Alt', 'Control', 'Meta', 'Shift']))
|
|
730
730
|
.optional()
|
|
731
731
|
.describe('Modifier keys'),
|
|
732
|
+
holdMs: z
|
|
733
|
+
.number()
|
|
734
|
+
.int()
|
|
735
|
+
.min(0)
|
|
736
|
+
.max(10000)
|
|
737
|
+
.optional()
|
|
738
|
+
.describe(
|
|
739
|
+
'Keep the mouse button or key pressed this long (click, press_key; default: 0)'
|
|
740
|
+
),
|
|
732
741
|
checked: z.boolean().optional().describe('Checked state (for set_checked action)'),
|
|
733
742
|
values: z.array(z.string()).optional().describe('Option values to select'),
|
|
734
743
|
labels: z
|
|
@@ -45,6 +45,22 @@ export const MAX_BATCH_CALLS = 20;
|
|
|
45
45
|
|
|
46
46
|
/** Maximum read-only batch calls executing concurrently. */
|
|
47
47
|
export const MAX_BATCH_CONCURRENCY = 5;
|
|
48
|
+
|
|
49
|
+
/** Maximum press-and-hold time for one pointer, key, or touch action (ms). */
|
|
50
|
+
export const MAX_INPUT_HOLD_MS = 10_000;
|
|
51
|
+
|
|
52
|
+
/** Default contact time for an input.touch gesture without explicit holdMs (ms). */
|
|
53
|
+
export const DEFAULT_TOUCH_HOLD_MS = 50;
|
|
54
|
+
|
|
55
|
+
/** Maximum simultaneous touch points in one input.touch gesture. */
|
|
56
|
+
export const MAX_TOUCH_POINTS = 10;
|
|
57
|
+
|
|
58
|
+
/** Maximum steps accepted by one input.perform sequence. */
|
|
59
|
+
export const MAX_PERFORM_STEPS = 200;
|
|
60
|
+
|
|
61
|
+
/** Default and maximum total budget for one input.perform sequence (ms). */
|
|
62
|
+
export const DEFAULT_PERFORM_TIMEOUT_MS = 30_000;
|
|
63
|
+
export const MAX_PERFORM_TIMEOUT_MS = 120_000;
|
|
48
64
|
export const MAX_INTERCEPT_RULES_PER_TAB = 32;
|
|
49
65
|
export const MAX_INTERCEPT_URL_PATTERN_LENGTH = 2_048;
|
|
50
66
|
export const MAX_INTERCEPT_BODY_BYTES = 262_144;
|
|
@@ -19,19 +19,25 @@ import {
|
|
|
19
19
|
DEFAULT_NETWORK_INTERCEPT_ACTION,
|
|
20
20
|
DEFAULT_NETWORK_LIMIT,
|
|
21
21
|
DEFAULT_PAGE_TEXT_BUDGET,
|
|
22
|
+
DEFAULT_PERFORM_TIMEOUT_MS,
|
|
22
23
|
DEFAULT_TEXT_BUDGET,
|
|
24
|
+
DEFAULT_TOUCH_HOLD_MS,
|
|
23
25
|
DEFAULT_VIEWPORT_HEIGHT,
|
|
24
26
|
DEFAULT_VIEWPORT_WIDTH,
|
|
25
27
|
DEFAULT_WAIT_TIMEOUT_MS,
|
|
26
28
|
MAX_EXTRACT_SETTLE_TIMEOUT_MS,
|
|
27
29
|
MAX_ARTIFACT_BYTES,
|
|
28
30
|
MAX_HAR_ENTRIES,
|
|
31
|
+
MAX_INPUT_HOLD_MS,
|
|
29
32
|
MAX_INTERCEPT_BODY_BYTES,
|
|
30
33
|
MAX_INTERCEPT_HEADER_BYTES,
|
|
31
34
|
MAX_INTERCEPT_HEADERS,
|
|
32
35
|
MAX_INTERCEPT_HEADER_VALUE_BYTES,
|
|
33
36
|
MAX_INTERCEPT_URL_PATTERN_LENGTH,
|
|
37
|
+
MAX_PERFORM_STEPS,
|
|
38
|
+
MAX_PERFORM_TIMEOUT_MS,
|
|
34
39
|
MAX_SENSITIVE_VALUE_BYTES,
|
|
40
|
+
MAX_TOUCH_POINTS,
|
|
35
41
|
} from './defaults.js';
|
|
36
42
|
import { BridgeError, ERROR_CODES, getErrorRecovery } from './errors.js';
|
|
37
43
|
import { BRIDGE_METHODS, METHOD_SET, createBridgeMethodGroups } from './registry.js';
|
|
@@ -96,6 +102,8 @@ import { BRIDGE_METHODS, METHOD_SET, createBridgeMethodGroups } from './registry
|
|
|
96
102
|
/** @typedef {import('./types.js').NormalizedNetworkParams} NormalizedNetworkParams */
|
|
97
103
|
/** @typedef {import('./types.js').NormalizedPageTextParams} NormalizedPageTextParams */
|
|
98
104
|
/** @typedef {import('./types.js').NormalizedPatchOperation} NormalizedPatchOperation */
|
|
105
|
+
/** @typedef {import('./types.js').NormalizedPerformParams} NormalizedPerformParams */
|
|
106
|
+
/** @typedef {import('./types.js').NormalizedPerformStep} NormalizedPerformStep */
|
|
99
107
|
/** @typedef {import('./types.js').NormalizedSelectAction} NormalizedSelectAction */
|
|
100
108
|
/** @typedef {import('./types.js').NormalizedScreenshotParams} NormalizedScreenshotParams */
|
|
101
109
|
/** @typedef {import('./types.js').NormalizedStorageParams} NormalizedStorageParams */
|
|
@@ -103,11 +111,17 @@ import { BRIDGE_METHODS, METHOD_SET, createBridgeMethodGroups } from './registry
|
|
|
103
111
|
/** @typedef {import('./types.js').NormalizedStyleQuery} NormalizedStyleQuery */
|
|
104
112
|
/** @typedef {import('./types.js').NormalizedTabCloseParams} NormalizedTabCloseParams */
|
|
105
113
|
/** @typedef {import('./types.js').NormalizedTabCreateParams} NormalizedTabCreateParams */
|
|
114
|
+
/** @typedef {import('./types.js').NormalizedTouchParams} NormalizedTouchParams */
|
|
115
|
+
/** @typedef {import('./types.js').NormalizedTouchPoint} NormalizedTouchPoint */
|
|
116
|
+
/** @typedef {import('./types.js').NormalizedTouchPosition} NormalizedTouchPosition */
|
|
106
117
|
/** @typedef {import('./types.js').NormalizedViewportAction} NormalizedViewportAction */
|
|
107
118
|
/** @typedef {import('./types.js').NormalizedViewportResizeParams} NormalizedViewportResizeParams */
|
|
108
119
|
/** @typedef {import('./types.js').NormalizedWaitForLoadStateParams} NormalizedWaitForLoadStateParams */
|
|
109
120
|
/** @typedef {import('./types.js').NormalizedWaitForParams} NormalizedWaitForParams */
|
|
110
121
|
/** @typedef {import('./types.js').PageTextParams} PageTextParams */
|
|
122
|
+
/** @typedef {import('./types.js').PerformParams} PerformParams */
|
|
123
|
+
/** @typedef {import('./types.js').PerformStepMethod} PerformStepMethod */
|
|
124
|
+
/** @typedef {import('./types.js').TouchParams} TouchParams */
|
|
111
125
|
/** @typedef {import('./types.js').PatchOperationParams} PatchOperationParams */
|
|
112
126
|
/** @typedef {import('./types.js').SelectActionParams} SelectActionParams */
|
|
113
127
|
/** @typedef {import('./types.js').ScreenshotParams} ScreenshotParams */
|
|
@@ -528,6 +542,10 @@ function normalizeRequestParams(method, params) {
|
|
|
528
542
|
return normalizeHoverParams(params);
|
|
529
543
|
case 'input.drag':
|
|
530
544
|
return normalizeDragParams(params);
|
|
545
|
+
case 'input.touch':
|
|
546
|
+
return normalizeTouchParams(params);
|
|
547
|
+
case 'input.perform':
|
|
548
|
+
return normalizeInputPerformParams(params);
|
|
531
549
|
case 'screenshot.capture_region':
|
|
532
550
|
return { ...params, ...normalizeScreenshotParams(params) };
|
|
533
551
|
case 'screenshot.capture_element':
|
|
@@ -604,11 +622,23 @@ export function getBridgeOperationTimeoutMs(method, params = {}) {
|
|
|
604
622
|
}
|
|
605
623
|
case 'dom.wait_for':
|
|
606
624
|
return clampInt(params.timeoutMs, 100, 30_000, 5_000);
|
|
625
|
+
case 'input.perform':
|
|
626
|
+
return normalizeInputPerformParams(params).timeoutMs;
|
|
627
|
+
case 'input.click':
|
|
628
|
+
case 'input.press_key': {
|
|
629
|
+
const holdMs = normalizeHoldMs(params.holdMs, 0);
|
|
630
|
+
return holdMs > 0 ? holdMs + INPUT_HOLD_TIMEOUT_MARGIN_MS : null;
|
|
631
|
+
}
|
|
632
|
+
case 'input.touch':
|
|
633
|
+
return normalizeHoldMs(params.holdMs, DEFAULT_TOUCH_HOLD_MS) + INPUT_HOLD_TIMEOUT_MARGIN_MS;
|
|
607
634
|
default:
|
|
608
635
|
return null;
|
|
609
636
|
}
|
|
610
637
|
}
|
|
611
638
|
|
|
639
|
+
/** Resolution and dispatch allowance added around an input hold (ms). */
|
|
640
|
+
const INPUT_HOLD_TIMEOUT_MARGIN_MS = 5_000;
|
|
641
|
+
|
|
612
642
|
/**
|
|
613
643
|
* @param {DomQueryParams} [params={}]
|
|
614
644
|
* @returns {NormalizedDomQuery}
|
|
@@ -774,11 +804,204 @@ export function normalizeInputAction(params = {}) {
|
|
|
774
804
|
modifiers: Array.isArray(params.modifiers)
|
|
775
805
|
? params.modifiers.filter((modifier) => typeof modifier === 'string' && modifier.trim())
|
|
776
806
|
: [],
|
|
807
|
+
holdMs: normalizeHoldMs(params.holdMs, 0),
|
|
808
|
+
executionMode: normalizeInputExecutionMode(params.executionMode),
|
|
809
|
+
recoverStale: params.recoverStale === true,
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
/**
|
|
814
|
+
* Normalize a press-and-hold duration, honoring an explicit zero.
|
|
815
|
+
*
|
|
816
|
+
* @param {unknown} value
|
|
817
|
+
* @param {number} fallback
|
|
818
|
+
* @returns {number}
|
|
819
|
+
*/
|
|
820
|
+
function normalizeHoldMs(value, fallback) {
|
|
821
|
+
return Math.round(clampNumber(value, 0, MAX_INPUT_HOLD_MS, fallback));
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
/**
|
|
825
|
+
* @param {unknown} value
|
|
826
|
+
* @param {string} label
|
|
827
|
+
* @returns {NormalizedTouchPosition}
|
|
828
|
+
*/
|
|
829
|
+
function normalizeTouchPosition(value, label) {
|
|
830
|
+
if (!isPlainRecord(value)) {
|
|
831
|
+
throw new BridgeError(
|
|
832
|
+
ERROR_CODES.INVALID_REQUEST,
|
|
833
|
+
`${label} must be an object with either target or numeric x and y.`
|
|
834
|
+
);
|
|
835
|
+
}
|
|
836
|
+
const hasTarget = isPlainRecord(value.target);
|
|
837
|
+
const hasPoint =
|
|
838
|
+
typeof value.x === 'number' &&
|
|
839
|
+
Number.isFinite(value.x) &&
|
|
840
|
+
typeof value.y === 'number' &&
|
|
841
|
+
Number.isFinite(value.y);
|
|
842
|
+
if (hasTarget === hasPoint) {
|
|
843
|
+
throw new BridgeError(
|
|
844
|
+
ERROR_CODES.INVALID_REQUEST,
|
|
845
|
+
`${label} needs exactly one of target or numeric x and y.`
|
|
846
|
+
);
|
|
847
|
+
}
|
|
848
|
+
if (hasTarget) {
|
|
849
|
+
return {
|
|
850
|
+
target: normalizeTarget(
|
|
851
|
+
/** @type {{ elementRef?: string, selector?: string }} */ (value.target)
|
|
852
|
+
),
|
|
853
|
+
x: null,
|
|
854
|
+
y: null,
|
|
855
|
+
};
|
|
856
|
+
}
|
|
857
|
+
return { target: null, x: /** @type {number} */ (value.x), y: /** @type {number} */ (value.y) };
|
|
858
|
+
}
|
|
859
|
+
|
|
860
|
+
/**
|
|
861
|
+
* Normalize a simultaneous touch gesture: one to MAX_TOUCH_POINTS fingers,
|
|
862
|
+
* each placed on a target or viewport point and optionally moved to an end
|
|
863
|
+
* position while all fingers stay down for holdMs.
|
|
864
|
+
*
|
|
865
|
+
* @param {TouchParams} [params={}]
|
|
866
|
+
* @returns {NormalizedTouchParams}
|
|
867
|
+
*/
|
|
868
|
+
export function normalizeTouchParams(params = {}) {
|
|
869
|
+
const rawPoints = params.points;
|
|
870
|
+
if (!Array.isArray(rawPoints) || rawPoints.length < 1 || rawPoints.length > MAX_TOUCH_POINTS) {
|
|
871
|
+
throw new BridgeError(
|
|
872
|
+
ERROR_CODES.INVALID_REQUEST,
|
|
873
|
+
`points must be an array of 1 to ${MAX_TOUCH_POINTS} touch points.`
|
|
874
|
+
);
|
|
875
|
+
}
|
|
876
|
+
/** @type {NormalizedTouchPoint[]} */
|
|
877
|
+
const points = rawPoints.map((point, index) => {
|
|
878
|
+
const start = normalizeTouchPosition(point, `points[${index}]`);
|
|
879
|
+
const rawEnd = isPlainRecord(point) ? point.to : undefined;
|
|
880
|
+
return {
|
|
881
|
+
...start,
|
|
882
|
+
to: rawEnd == null ? null : normalizeTouchPosition(rawEnd, `points[${index}].to`),
|
|
883
|
+
};
|
|
884
|
+
});
|
|
885
|
+
return {
|
|
886
|
+
points,
|
|
887
|
+
holdMs: normalizeHoldMs(params.holdMs, DEFAULT_TOUCH_HOLD_MS),
|
|
888
|
+
moveSteps: clampInt(params.moveSteps, 1, 60, 10),
|
|
777
889
|
executionMode: normalizeInputExecutionMode(params.executionMode),
|
|
778
890
|
recoverStale: params.recoverStale === true,
|
|
779
891
|
};
|
|
780
892
|
}
|
|
781
893
|
|
|
894
|
+
/** Methods that may appear as input.perform steps. */
|
|
895
|
+
const PERFORM_STEP_METHODS = new Set([
|
|
896
|
+
'input.click',
|
|
897
|
+
'input.focus',
|
|
898
|
+
'input.type',
|
|
899
|
+
'input.fill',
|
|
900
|
+
'input.press_key',
|
|
901
|
+
'input.set_checked',
|
|
902
|
+
'input.select_option',
|
|
903
|
+
'input.hover',
|
|
904
|
+
'input.drag',
|
|
905
|
+
'input.touch',
|
|
906
|
+
'input.scroll_into_view',
|
|
907
|
+
'viewport.scroll',
|
|
908
|
+
'dom.wait_for',
|
|
909
|
+
'page.wait_for_load_state',
|
|
910
|
+
]);
|
|
911
|
+
|
|
912
|
+
/** Step methods that accept executionMode=cdp. */
|
|
913
|
+
const CDP_CAPABLE_STEP_METHODS = new Set([
|
|
914
|
+
'input.click',
|
|
915
|
+
'input.hover',
|
|
916
|
+
'input.drag',
|
|
917
|
+
'input.type',
|
|
918
|
+
'input.fill',
|
|
919
|
+
'input.press_key',
|
|
920
|
+
'input.touch',
|
|
921
|
+
]);
|
|
922
|
+
|
|
923
|
+
/**
|
|
924
|
+
* @param {unknown} value
|
|
925
|
+
* @param {string} label
|
|
926
|
+
* @param {number} maximum
|
|
927
|
+
* @returns {number | null}
|
|
928
|
+
*/
|
|
929
|
+
function normalizeStepTiming(value, label, maximum) {
|
|
930
|
+
if (value == null) return null;
|
|
931
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > maximum) {
|
|
932
|
+
throw new BridgeError(
|
|
933
|
+
ERROR_CODES.INVALID_REQUEST,
|
|
934
|
+
`${label} must be a number between 0 and the sequence timeoutMs (${maximum}).`
|
|
935
|
+
);
|
|
936
|
+
}
|
|
937
|
+
return Math.round(value);
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
/**
|
|
941
|
+
* Normalize an ordered, timed input sequence. Each step reuses an existing
|
|
942
|
+
* bridge method and its own normalizer; the sequence adds only scheduling.
|
|
943
|
+
*
|
|
944
|
+
* @param {PerformParams} [params={}]
|
|
945
|
+
* @returns {NormalizedPerformParams}
|
|
946
|
+
*/
|
|
947
|
+
export function normalizeInputPerformParams(params = {}) {
|
|
948
|
+
const timeoutMs = clampInt(
|
|
949
|
+
params.timeoutMs,
|
|
950
|
+
100,
|
|
951
|
+
MAX_PERFORM_TIMEOUT_MS,
|
|
952
|
+
DEFAULT_PERFORM_TIMEOUT_MS
|
|
953
|
+
);
|
|
954
|
+
const executionMode =
|
|
955
|
+
params.executionMode === undefined ? null : normalizeInputExecutionMode(params.executionMode);
|
|
956
|
+
const rawSteps = params.steps;
|
|
957
|
+
if (!Array.isArray(rawSteps) || rawSteps.length < 1 || rawSteps.length > MAX_PERFORM_STEPS) {
|
|
958
|
+
throw new BridgeError(
|
|
959
|
+
ERROR_CODES.INVALID_REQUEST,
|
|
960
|
+
`steps must be an array of 1 to ${MAX_PERFORM_STEPS} steps.`
|
|
961
|
+
);
|
|
962
|
+
}
|
|
963
|
+
/** @type {NormalizedPerformStep[]} */
|
|
964
|
+
const steps = rawSteps.map((step, index) => {
|
|
965
|
+
if (!isPlainRecord(step)) {
|
|
966
|
+
throw new BridgeError(ERROR_CODES.INVALID_REQUEST, `steps[${index}] must be an object.`);
|
|
967
|
+
}
|
|
968
|
+
const method = step.method;
|
|
969
|
+
if (typeof method !== 'string' || !PERFORM_STEP_METHODS.has(method)) {
|
|
970
|
+
throw new BridgeError(
|
|
971
|
+
ERROR_CODES.INVALID_REQUEST,
|
|
972
|
+
`steps[${index}].method must be one of ${[...PERFORM_STEP_METHODS].join(', ')}.`
|
|
973
|
+
);
|
|
974
|
+
}
|
|
975
|
+
if (step.params !== undefined && !isPlainRecord(step.params)) {
|
|
976
|
+
throw new BridgeError(
|
|
977
|
+
ERROR_CODES.INVALID_REQUEST,
|
|
978
|
+
`steps[${index}].params must be an object.`
|
|
979
|
+
);
|
|
980
|
+
}
|
|
981
|
+
const rawParams = /** @type {Record<string, unknown>} */ (step.params ?? {});
|
|
982
|
+
const stepParams =
|
|
983
|
+
executionMode && CDP_CAPABLE_STEP_METHODS.has(method) && rawParams.executionMode === undefined
|
|
984
|
+
? { ...rawParams, executionMode }
|
|
985
|
+
: rawParams;
|
|
986
|
+
const delayMs = normalizeStepTiming(step.delayMs, `steps[${index}].delayMs`, timeoutMs);
|
|
987
|
+
const atMs = normalizeStepTiming(step.atMs, `steps[${index}].atMs`, timeoutMs);
|
|
988
|
+
if (delayMs && atMs !== null) {
|
|
989
|
+
throw new BridgeError(
|
|
990
|
+
ERROR_CODES.INVALID_REQUEST,
|
|
991
|
+
`steps[${index}] accepts either delayMs or atMs, not both.`
|
|
992
|
+
);
|
|
993
|
+
}
|
|
994
|
+
const stepMethod = /** @type {PerformStepMethod} */ (method);
|
|
995
|
+
return {
|
|
996
|
+
method: stepMethod,
|
|
997
|
+
params: normalizeRequestParams(stepMethod, stepParams),
|
|
998
|
+
delayMs: delayMs ?? 0,
|
|
999
|
+
atMs,
|
|
1000
|
+
};
|
|
1001
|
+
});
|
|
1002
|
+
return { steps, timeoutMs, continueOnError: params.continueOnError === true };
|
|
1003
|
+
}
|
|
1004
|
+
|
|
782
1005
|
/**
|
|
783
1006
|
* @param {CdpDispatchKeyEventParams} [params={}]
|
|
784
1007
|
* @returns {NormalizedCdpDispatchKeyEventParams}
|
|
@@ -153,16 +153,22 @@ const BRIDGE_METHOD_DESCRIPTIONS = Object.freeze({
|
|
|
153
153
|
'Read element classes and inline style context (not stylesheet cascade data).',
|
|
154
154
|
'viewport.scroll': 'Scroll the viewport or a scrollable element.',
|
|
155
155
|
'viewport.resize': 'Resize or reset the tab viewport.',
|
|
156
|
-
'input.click':
|
|
156
|
+
'input.click':
|
|
157
|
+
'Actionability-check and click an element through DOM or optional CDP input; holdMs keeps the button pressed.',
|
|
157
158
|
'input.focus': 'Actionability-check and focus an element through DOM input.',
|
|
158
159
|
'input.type': 'Actionability-check and type through DOM or optional CDP text input.',
|
|
159
160
|
'input.fill':
|
|
160
161
|
'Fill an editable target using a DOM strategy or optional CDP text input; verify afterward.',
|
|
161
|
-
'input.press_key':
|
|
162
|
+
'input.press_key':
|
|
163
|
+
'Send a key press to the page or an element through DOM or optional CDP input; holdMs keeps the key down.',
|
|
162
164
|
'input.set_checked': 'Set checkbox or radio checked state.',
|
|
163
165
|
'input.select_option': 'Select options in a select element.',
|
|
164
166
|
'input.hover': 'Actionability-check and hover through DOM or optional CDP pointer input.',
|
|
165
167
|
'input.drag': 'Actionability-check and drag through DOM or optional CDP pointer input.',
|
|
168
|
+
'input.touch':
|
|
169
|
+
'Press one or more touch points at the same time (chords, multi-finger taps, swipes, pinches) through DOM or optional CDP touch input.',
|
|
170
|
+
'input.perform':
|
|
171
|
+
'Run an ordered, timed sequence of input steps inside the browser in one call; use for rhythm, games, gestures, and multi-step flows.',
|
|
166
172
|
'input.scroll_into_view': 'Scroll an element into the visible viewport.',
|
|
167
173
|
'screenshot.capture_region': 'Capture a screenshot of a viewport region.',
|
|
168
174
|
'screenshot.capture_element': 'Capture a screenshot of one element.',
|
|
@@ -600,7 +606,7 @@ export const BRIDGE_METHOD_REGISTRY = Object.freeze({
|
|
|
600
606
|
'input.click',
|
|
601
607
|
'interact',
|
|
602
608
|
true,
|
|
603
|
-
['target', 'button', 'clickCount', 'modifiers', 'executionMode', 'recoverStale'],
|
|
609
|
+
['target', 'button', 'clickCount', 'modifiers', 'holdMs', 'executionMode', 'recoverStale'],
|
|
604
610
|
'low',
|
|
605
611
|
METHOD_POLICIES.automationInput
|
|
606
612
|
),
|
|
@@ -632,7 +638,7 @@ export const BRIDGE_METHOD_REGISTRY = Object.freeze({
|
|
|
632
638
|
'input.press_key',
|
|
633
639
|
'interact',
|
|
634
640
|
true,
|
|
635
|
-
['target', 'key', 'modifiers', 'executionMode', 'recoverStale'],
|
|
641
|
+
['target', 'key', 'modifiers', 'holdMs', 'executionMode', 'recoverStale'],
|
|
636
642
|
'low',
|
|
637
643
|
METHOD_POLICIES.automationInput
|
|
638
644
|
),
|
|
@@ -668,6 +674,22 @@ export const BRIDGE_METHOD_REGISTRY = Object.freeze({
|
|
|
668
674
|
'moderate',
|
|
669
675
|
METHOD_POLICIES.automationInput
|
|
670
676
|
),
|
|
677
|
+
'input.touch': createRegistryEntry(
|
|
678
|
+
'input.touch',
|
|
679
|
+
'interact',
|
|
680
|
+
true,
|
|
681
|
+
['points', 'holdMs', 'moveSteps', 'executionMode', 'recoverStale'],
|
|
682
|
+
'moderate',
|
|
683
|
+
METHOD_POLICIES.automationInput
|
|
684
|
+
),
|
|
685
|
+
'input.perform': createRegistryEntry(
|
|
686
|
+
'input.perform',
|
|
687
|
+
'interact',
|
|
688
|
+
true,
|
|
689
|
+
['steps', 'executionMode', 'timeoutMs', 'continueOnError'],
|
|
690
|
+
'moderate',
|
|
691
|
+
METHOD_POLICIES.automationInput
|
|
692
|
+
),
|
|
671
693
|
'input.scroll_into_view': createRegistryEntry(
|
|
672
694
|
'input.scroll_into_view',
|
|
673
695
|
'interact',
|
|
@@ -56,8 +56,67 @@ import { sanitizeIncidentalUrl } from './incidental-sanitizer.js';
|
|
|
56
56
|
* }} BatchItemSummary
|
|
57
57
|
*/
|
|
58
58
|
|
|
59
|
+
/** Evaluate values longer than this stay out of the summary; evidence holds them. */
|
|
60
|
+
const MAX_EVALUATE_SUMMARY_CHARS = 160;
|
|
61
|
+
const EVALUATE_PREVIEW_CHARS = 80;
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Keep evaluate summaries short: small values inline, larger ones as a shape
|
|
65
|
+
* description, because the full value is already delivered in evidence.
|
|
66
|
+
*
|
|
67
|
+
* @param {unknown} value
|
|
68
|
+
* @returns {string}
|
|
69
|
+
*/
|
|
70
|
+
function summarizeEvaluateValue(value) {
|
|
71
|
+
const json = JSON.stringify(value) ?? String(value);
|
|
72
|
+
if (json.length <= MAX_EVALUATE_SUMMARY_CHARS) return json;
|
|
73
|
+
const shape = Array.isArray(value)
|
|
74
|
+
? `${value.length} item(s)`
|
|
75
|
+
: value && typeof value === 'object'
|
|
76
|
+
? `${Object.keys(value).length} key(s)`
|
|
77
|
+
: `${json.length} chars`;
|
|
78
|
+
return `${shape}, ${json.length} chars as JSON; full value in evidence.value`;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* @param {string | undefined} method
|
|
83
|
+
* @param {Record<string, unknown>} result
|
|
84
|
+
* @returns {string}
|
|
85
|
+
*/
|
|
86
|
+
function describeTabResult(method, result) {
|
|
87
|
+
const url = typeof result.url === 'string' && result.url ? ` (${result.url})` : '';
|
|
88
|
+
switch (method) {
|
|
89
|
+
case 'navigation.reload':
|
|
90
|
+
return `Reloaded tab ${result.tabId}${url}.`;
|
|
91
|
+
case 'navigation.go_back':
|
|
92
|
+
return `Navigated back in tab ${result.tabId}${url}.`;
|
|
93
|
+
case 'navigation.go_forward':
|
|
94
|
+
return `Navigated forward in tab ${result.tabId}${url}.`;
|
|
95
|
+
case 'tabs.activate':
|
|
96
|
+
return `Tab ${result.tabId} activated${url}.`;
|
|
97
|
+
case 'page.wait_for_load_state':
|
|
98
|
+
return `Tab ${result.tabId} ${typeof result.status === 'string' ? result.status : 'ready'}${url}.`;
|
|
99
|
+
default:
|
|
100
|
+
return `Tab ${result.tabId} created${url}.`;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
59
104
|
/** @type {Record<string, ActionSummary>} */
|
|
60
105
|
const ACTION_SUMMARIES = {
|
|
106
|
+
performed: {
|
|
107
|
+
text: (r) => {
|
|
108
|
+
const failures = Array.isArray(r.failures) ? r.failures.length : 0;
|
|
109
|
+
return `Performed ${r.completed ?? 0}/${r.total ?? 0} step(s) in ${r.elapsedMs ?? 0}ms${failures ? ` with ${failures} failed step(s)` : ''}.`;
|
|
110
|
+
},
|
|
111
|
+
evidence: (r) => r,
|
|
112
|
+
},
|
|
113
|
+
touched: {
|
|
114
|
+
text: (r) => {
|
|
115
|
+
const mode = toRecord(r.execution).actualMode;
|
|
116
|
+
return `Touched ${r.pointCount ?? 0} point(s)${typeof mode === 'string' ? ` via ${mode}` : ''}.`;
|
|
117
|
+
},
|
|
118
|
+
evidence: (r) => r,
|
|
119
|
+
},
|
|
61
120
|
hovered: {
|
|
62
121
|
text: (r) => `Hover ${r.hovered ? 'active' : 'failed'} on ${r.elementRef}.`,
|
|
63
122
|
evidence: (r) => r,
|
|
@@ -545,7 +604,10 @@ export function summarizeBridgeResponse(response, method) {
|
|
|
545
604
|
} else if (isNull) {
|
|
546
605
|
repr = '';
|
|
547
606
|
} else if (typeof result.value === 'string') {
|
|
548
|
-
repr =
|
|
607
|
+
repr =
|
|
608
|
+
result.value.length > MAX_EVALUATE_SUMMARY_CHARS
|
|
609
|
+
? `${result.value.length} chars, starts ${JSON.stringify(result.value.slice(0, EVALUATE_PREVIEW_CHARS))}\u2026`
|
|
610
|
+
: result.value;
|
|
549
611
|
} else if (
|
|
550
612
|
typeof result.value === 'object' &&
|
|
551
613
|
result.value !== null &&
|
|
@@ -553,7 +615,7 @@ export function summarizeBridgeResponse(response, method) {
|
|
|
553
615
|
) {
|
|
554
616
|
repr = '(empty - may be a Promise, Map, or non-serializable value)';
|
|
555
617
|
} else {
|
|
556
|
-
repr =
|
|
618
|
+
repr = summarizeEvaluateValue(result.value);
|
|
557
619
|
}
|
|
558
620
|
const typeLabel = isNull ? 'null' : result.type;
|
|
559
621
|
return {
|
|
@@ -724,10 +786,7 @@ export function summarizeBridgeResponse(response, method) {
|
|
|
724
786
|
}
|
|
725
787
|
return {
|
|
726
788
|
ok: true,
|
|
727
|
-
summary: appendProtocolWarning(
|
|
728
|
-
`Tab ${result.tabId} created${result.url ? ` (${result.url})` : ''}.`,
|
|
729
|
-
protocolWarning
|
|
730
|
-
),
|
|
789
|
+
summary: appendProtocolWarning(describeTabResult(actionMethod, result), protocolWarning),
|
|
731
790
|
evidence: result,
|
|
732
791
|
};
|
|
733
792
|
}
|
|
@@ -119,6 +119,8 @@ export type BridgeMethod =
|
|
|
119
119
|
| 'input.select_option'
|
|
120
120
|
| 'input.hover'
|
|
121
121
|
| 'input.drag'
|
|
122
|
+
| 'input.touch'
|
|
123
|
+
| 'input.perform'
|
|
122
124
|
| 'input.scroll_into_view'
|
|
123
125
|
| 'screenshot.capture_region'
|
|
124
126
|
| 'screenshot.capture_element'
|
|
@@ -479,6 +481,7 @@ export interface InputActionParams {
|
|
|
479
481
|
submit?: boolean;
|
|
480
482
|
key?: string;
|
|
481
483
|
modifiers?: string[];
|
|
484
|
+
holdMs?: number;
|
|
482
485
|
executionMode?: InputExecutionMode;
|
|
483
486
|
recoverStale?: boolean;
|
|
484
487
|
}
|
|
@@ -494,10 +497,106 @@ export interface NormalizedInputAction extends BridgeParams {
|
|
|
494
497
|
submit: boolean;
|
|
495
498
|
key: string;
|
|
496
499
|
modifiers: string[];
|
|
500
|
+
holdMs: number;
|
|
497
501
|
executionMode: InputExecutionMode;
|
|
498
502
|
recoverStale: boolean;
|
|
499
503
|
}
|
|
500
504
|
|
|
505
|
+
// Nullable fields let already-normalized params pass through normalization again.
|
|
506
|
+
export interface TouchPositionParams {
|
|
507
|
+
target?: InputTarget | null;
|
|
508
|
+
x?: number | null;
|
|
509
|
+
y?: number | null;
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
export interface TouchPointParams extends TouchPositionParams {
|
|
513
|
+
to?: TouchPositionParams | null;
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
export interface TouchParams {
|
|
517
|
+
points?: TouchPointParams[];
|
|
518
|
+
holdMs?: number;
|
|
519
|
+
moveSteps?: number;
|
|
520
|
+
executionMode?: InputExecutionMode;
|
|
521
|
+
recoverStale?: boolean;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
export type NormalizedTouchPosition =
|
|
525
|
+
| { target: InputTarget; x: null; y: null }
|
|
526
|
+
| { target: null; x: number; y: number };
|
|
527
|
+
|
|
528
|
+
export type NormalizedTouchPoint = NormalizedTouchPosition & {
|
|
529
|
+
to: NormalizedTouchPosition | null;
|
|
530
|
+
};
|
|
531
|
+
|
|
532
|
+
export interface NormalizedTouchParams extends BridgeParams {
|
|
533
|
+
points: NormalizedTouchPoint[];
|
|
534
|
+
holdMs: number;
|
|
535
|
+
moveSteps: number;
|
|
536
|
+
executionMode: InputExecutionMode;
|
|
537
|
+
recoverStale: boolean;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
export type PerformStepMethod =
|
|
541
|
+
| 'input.click'
|
|
542
|
+
| 'input.focus'
|
|
543
|
+
| 'input.type'
|
|
544
|
+
| 'input.fill'
|
|
545
|
+
| 'input.press_key'
|
|
546
|
+
| 'input.set_checked'
|
|
547
|
+
| 'input.select_option'
|
|
548
|
+
| 'input.hover'
|
|
549
|
+
| 'input.drag'
|
|
550
|
+
| 'input.touch'
|
|
551
|
+
| 'input.scroll_into_view'
|
|
552
|
+
| 'viewport.scroll'
|
|
553
|
+
| 'dom.wait_for'
|
|
554
|
+
| 'page.wait_for_load_state';
|
|
555
|
+
|
|
556
|
+
export interface PerformStepParams {
|
|
557
|
+
method?: string;
|
|
558
|
+
params?: Record<string, unknown>;
|
|
559
|
+
delayMs?: number | null;
|
|
560
|
+
atMs?: number | null;
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
export interface PerformParams {
|
|
564
|
+
steps?: PerformStepParams[];
|
|
565
|
+
executionMode?: InputExecutionMode;
|
|
566
|
+
timeoutMs?: number;
|
|
567
|
+
continueOnError?: boolean;
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
export interface NormalizedPerformStep {
|
|
571
|
+
method: PerformStepMethod;
|
|
572
|
+
params: Record<string, unknown>;
|
|
573
|
+
delayMs: number;
|
|
574
|
+
atMs: number | null;
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
export interface NormalizedPerformParams extends BridgeParams {
|
|
578
|
+
steps: NormalizedPerformStep[];
|
|
579
|
+
timeoutMs: number;
|
|
580
|
+
continueOnError: boolean;
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
export interface PerformStepFailure {
|
|
584
|
+
index: number;
|
|
585
|
+
method: PerformStepMethod;
|
|
586
|
+
code: string;
|
|
587
|
+
message: string;
|
|
588
|
+
details?: unknown;
|
|
589
|
+
}
|
|
590
|
+
|
|
591
|
+
export interface PerformResult {
|
|
592
|
+
performed: true;
|
|
593
|
+
completed: number;
|
|
594
|
+
total: number;
|
|
595
|
+
elapsedMs: number;
|
|
596
|
+
startedAtMs: number[];
|
|
597
|
+
failures: PerformStepFailure[];
|
|
598
|
+
}
|
|
599
|
+
|
|
501
600
|
export interface CdpDispatchKeyEventParams {
|
|
502
601
|
key?: string;
|
|
503
602
|
code?: string;
|
|
@@ -195,6 +195,9 @@ they contain no page data, URLs, selectors, payloads, errors, or group names.
|
|
|
195
195
|
23. **Dialogs are explicit** - inspect first, then accept or dismiss only when intended. `expectedDialogId` is a pre-dispatch stale-decision check, not an atomic CDP binding; never auto-repeat `DIALOG_ACTION_CONFLICT`.
|
|
196
196
|
24. **DOM diffs use explicit baselines** - call `dom.baseline.create` before the action, `dom.baseline.compare` afterward, then `dom.baseline.release`. Use a narrow selector and bounded evidence. Never substitute unrelated DOM reads for the retained baseline, and recreate it after `DOM_BASELINE_INVALIDATED`.
|
|
197
197
|
25. **HAR export reads an armed capture** - use CDP capture `start`, reproduce, `network.export_har`/`bbx har`, then `stop`. Export never starts, stops, or clears capture. Inspect truncation plus `dropped`, `abandoned`, and `inflight` before claiming completeness.
|
|
198
|
+
26. **Keep timing inside the browser** - for rhythm, games, gestures, or any steps that must happen at set times or back-to-back, send one `input.perform` with the whole schedule (`atMs` from the sequence start, or `delayMs` after the previous step). Never spread timed input across tool calls (each call costs an agent round-trip) and never issue input as parallel tool calls (they land all at once).
|
|
199
|
+
27. **Use trusted input when the page needs a user gesture** - DOM-mode events are untrusted, so Chrome may refuse audio or media start, popups, clipboard writes, and fullscreen. Use `executionMode: "cdp"` (click, hover, drag, type, fill, press_key, touch) for those pages; `input.perform` accepts it once for every step.
|
|
200
|
+
28. **Simultaneous presses need touch** - a mouse has one pointer. For chords or multi-finger gestures use `input.touch` with several `points` (optionally `to` end positions for swipes and pinches); add `holdMs` to `input.click`/`input.press_key` when the press must last.
|
|
198
201
|
|
|
199
202
|
## Token Budget Quick Rules
|
|
200
203
|
|
|
@@ -267,7 +270,7 @@ bbx page-text 2000 # extract page content
|
|
|
267
270
|
| Page State | `page.get_console`, `page.get_storage`, `page.get_text`, `page.wait_for_load_state`, `page.evaluate` (debugger-backed) |
|
|
268
271
|
| Dialogs | `page.handle_dialog` (debugger-backed; explicit inspect/accept/dismiss only) |
|
|
269
272
|
| Network | `page.get_network` (fetch/XHR default; optional CDP lifecycle), `network.export_har`, `network.intercept.add/remove/list/clear` (debugger-backed) |
|
|
270
|
-
| Interact | `input.click`, `input.type`, `input.fill`, `input.focus`, `input.press_key`, `cdp.dispatch_key_event`, `input.hover`, `input.drag`, `input.scroll_into_view` |
|
|
273
|
+
| Interact | `input.click`, `input.type`, `input.fill`, `input.focus`, `input.press_key`, `cdp.dispatch_key_event`, `input.hover`, `input.drag`, `input.touch`, `input.perform` (timed sequences), `input.scroll_into_view` |
|
|
271
274
|
| Tabs | `tabs.list` (preferred), `tabs.create` (avoid unless necessary), `tabs.close`, `tabs.activate` |
|
|
272
275
|
| Patch | `patch.apply_styles`, `patch.apply_dom`, `patch.rollback` |
|
|
273
276
|
| Navigate | `navigation.navigate`, `viewport.scroll`, `viewport.resize` |
|
|
@@ -349,7 +352,8 @@ Every CLI shortcut command produces consistent `{ok, summary, evidence}` JSON. U
|
|
|
349
352
|
bbx call input.click '{"elementRef":"el_xxx"}'
|
|
350
353
|
```
|
|
351
354
|
- `input.drag` uses `source`, `destination`, and optional destination offsets `offsetX` / `offsetY`.
|
|
352
|
-
- `executionMode` accepts `dom` or `cdp` and defaults to `dom`; CDP supports
|
|
355
|
+
- `executionMode` accepts `dom` or `cdp` and defaults to `dom`; CDP supports click, hover, drag, type, fill, press_key, and touch. `input.fill.mode` (`auto`, `setter`, `keystrokes`) is a separate DOM strategy.
|
|
356
|
+
- `input.perform` takes `steps: [{ method, params, atMs | delayMs }]` using the same params each method takes alone. Steps run in order inside the extension; it stops at the first failed step (or a `dom.wait_for` that times out) unless `continueOnError: true`, and reports `startedAtMs` for every step.
|
|
353
357
|
- `page.get_network` defaults to low-cost fetch/XHR instrumentation. All-resource CDP capture must be armed before the activity with `source: "cdp", capture: "start"`, read later, and explicitly stopped.
|
|
354
358
|
- `network.export_har` requires that explicit capture to still be armed. Params are `limit` (1-200), sanitized-URL substring `urlPattern`, and `delivery` (`auto`, `inline`, `artifact`). Inline bounds remove whole oldest entries; export never changes capture state. `bbx har` uses the same client to download, length/SHA-256 verify, delete, validate, and atomically write artifacts locally.
|
|
355
359
|
- `page.handle_dialog` mutations are not atomically bound to `expectedDialogId`; treat `commandDispatched` as dispatch evidence and verify follow-up state.
|
|
@@ -4,16 +4,18 @@
|
|
|
4
4
|
|
|
5
5
|
| Method | CLI Shortcut | Purpose |
|
|
6
6
|
| ------------------------ | ------------------------------------- | ----------------------------------------------------------- |
|
|
7
|
-
| `input.click` | `click <ref> [button]` | Actionability-aware DOM click; optional CDP native click
|
|
7
|
+
| `input.click` | `click <ref> [button]` | Actionability-aware DOM click; optional CDP native click; `holdMs` keeps the button down |
|
|
8
8
|
| `input.focus` | `focus <ref>` | Focus an element |
|
|
9
9
|
| `input.type` | `type <ref> <text>` | DOM key sequence; optional CDP native text insertion |
|
|
10
10
|
| `input.fill` | `fill <ref> <value>` | DOM fill strategy; optional CDP clear and text insertion |
|
|
11
|
-
| `input.press_key` | `press-key <key> [ref]` | Send keyboard key (Enter, Backspace, etc.)
|
|
11
|
+
| `input.press_key` | `press-key <key> [ref]` | Send keyboard key (Enter, Backspace, etc.); optional CDP; `holdMs` |
|
|
12
12
|
| `cdp.dispatch_key_event` | `cdp-press-key --tab <id> <key>` | CDP keyDown/keyUp without focusing the target tab |
|
|
13
13
|
| `input.set_checked` | `call input.set_checked '{...}'` | Toggle checkbox/radio |
|
|
14
14
|
| `input.select_option` | `call input.select_option '{...}'` | Select native `<select>` by value/label/index |
|
|
15
15
|
| `input.hover` | `hover <ref>` | DOM hover events or optional CDP pointer move |
|
|
16
|
-
| `input.drag` | `call input.drag '{...}'` |
|
|
16
|
+
| `input.drag` | `call input.drag '{...}'` | HTML5 drag for draggable sources, pointer drag otherwise; optional CDP |
|
|
17
|
+
| `input.touch` | `call input.touch '{...}'` | One or more simultaneous touch points: chords, taps, swipes, pinches |
|
|
18
|
+
| `input.perform` | `call input.perform '{...}'` | Ordered, timed sequence of input steps run inside the browser |
|
|
17
19
|
| `input.scroll_into_view` | `call input.scroll_into_view '{...}'` | Ensure a target is visible before inspect/capture |
|
|
18
20
|
|
|
19
21
|
## Navigation
|
|
@@ -225,7 +227,69 @@ bbx type el_123 hello # simulate per-character keystrokes
|
|
|
225
227
|
|
|
226
228
|
Prefer `fill` for setting form values: it uses the native prototype setter plus `input`/`change`/`blur` events, which React, Vue, and Angular pick up reliably. `mode` defaults to `auto` (setter first, keystroke fallback if the value did not stick); pass `"mode":"keystrokes"` via `bbx call input.fill` for components that only react to per-key events. Use `type` when page logic depends on individual key events (autocomplete, masked inputs).
|
|
227
229
|
|
|
228
|
-
`mode` is not `executionMode`. The latter accepts only `dom` or `cdp`, defaults to `dom` for compatibility, and selects the dispatch path. CDP execution is available
|
|
230
|
+
`mode` is not `executionMode`. The latter accepts only `dom` or `cdp`, defaults to `dom` for compatibility, and selects the dispatch path. CDP execution is available for click, hover, drag, type, fill, press_key, and touch; unsupported combinations fail with `INPUT_UNSUPPORTED` instead of silently changing paths.
|
|
231
|
+
|
|
232
|
+
## DOM vs CDP Input
|
|
233
|
+
|
|
234
|
+
DOM mode dispatches the same event sequence a real device produces, so most pages cannot tell the difference:
|
|
235
|
+
|
|
236
|
+
- **Clicks**: `pointerover`/`pointerenter`/`mouseover`/`mouseenter` (only when the virtual pointer moves onto a new element), `pointermove`/`mousemove`, `pointerdown`/`mousedown`, focus, optional hold, `pointerup`/`mouseup`, then `click` (`contextmenu` for right, `auxclick` for middle) carrying coordinates, modifiers, `detail`, and `pointerType`. A canceled `pointerdown` suppresses the mouse events; a canceled `mousedown` keeps focus where it was. Leaving an element fires `pointerout`/`mouseout`/`leave` on it.
|
|
237
|
+
- **Keys**: `keydown` with `key`, `code`, and `keyCode`, `keypress` for character keys, the editing action, then `keyup`. A canceled `keydown` suppresses the character, as in browsers.
|
|
238
|
+
|
|
239
|
+
DOM events are still untrusted (`isTrusted: false`) and do not count as a user gesture, so Chrome may refuse what a page gates on one: starting audio or media (unless the site already has autoplay engagement), popups, clipboard writes, and fullscreen. When a page needs a real gesture, use `executionMode: "cdp"`, which dispatches trusted input through Chrome. If sound or video stays silent after a DOM click, retry the first interaction with CDP.
|
|
240
|
+
|
|
241
|
+
## Press and Hold
|
|
242
|
+
|
|
243
|
+
`holdMs` (0-10000) keeps a mouse button or key pressed between down and up:
|
|
244
|
+
|
|
245
|
+
```bash
|
|
246
|
+
bbx call input.click '{"target":{"selector":"[data-midi=\"60\"]"},"holdMs":400}'
|
|
247
|
+
bbx call input.press_key '{"key":"q","holdMs":400,"executionMode":"cdp"}'
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
Use it for long-press menus, sustained notes, and buttons that act while pressed.
|
|
251
|
+
|
|
252
|
+
## Timed Sequences
|
|
253
|
+
|
|
254
|
+
Every tool call costs an agent round-trip, so steps sent one call at a time arrive seconds apart, and steps sent as parallel tool calls all land at once. When timing matters (music, games, animations, gestures, quick reactions), send the whole schedule as one `input.perform`:
|
|
255
|
+
|
|
256
|
+
```bash
|
|
257
|
+
bbx call input.perform '{
|
|
258
|
+
"executionMode": "cdp",
|
|
259
|
+
"steps": [
|
|
260
|
+
{"method":"input.click","params":{"target":{"selector":"[data-midi=\"60\"]"},"holdMs":350},"atMs":0},
|
|
261
|
+
{"method":"input.click","params":{"target":{"selector":"[data-midi=\"62\"]"},"holdMs":350},"atMs":400},
|
|
262
|
+
{"method":"input.touch","params":{"points":[{"target":{"selector":"[data-midi=\"60\"]"}},{"target":{"selector":"[data-midi=\"64\"]"}},{"target":{"selector":"[data-midi=\"67\"]"}}],"holdMs":700},"atMs":800}
|
|
263
|
+
]
|
|
264
|
+
}'
|
|
265
|
+
```
|
|
266
|
+
|
|
267
|
+
- Each step is `{ method, params, atMs | delayMs }`; `params` are exactly what the method takes on its own.
|
|
268
|
+
- Allowed step methods: `input.click`, `input.focus`, `input.type`, `input.fill`, `input.press_key`, `input.set_checked`, `input.select_option`, `input.hover`, `input.drag`, `input.touch`, `input.scroll_into_view`, `viewport.scroll`, `dom.wait_for`, `page.wait_for_load_state`.
|
|
269
|
+
- `atMs` is the earliest start measured from the sequence start, so delays never accumulate; `delayMs` waits after the previous step finished. Steps always run in order, so a step whose `atMs` has already passed starts immediately.
|
|
270
|
+
- A top-level `executionMode` applies to every step that supports it unless the step sets its own.
|
|
271
|
+
- Add `dom.wait_for` steps to react to the page without a round-trip, e.g. click, wait for `.menu`, click the item. A wait that times out counts as a failed step.
|
|
272
|
+
- The sequence stops at the first failure with that step's error code and `details.failedStep`, `completed`, and `startedAtMs`. Set `continueOnError: true` to run every step and collect `failures`.
|
|
273
|
+
- `timeoutMs` (default 30000, max 120000) bounds the whole sequence, and waiting steps are capped to the remaining budget. At most 200 steps.
|
|
274
|
+
- The result reports `startedAtMs` (actual start of every step), so you can verify the rhythm without a screenshot. Then verify application state as usual.
|
|
275
|
+
|
|
276
|
+
## Touch and Multi-Finger Input
|
|
277
|
+
|
|
278
|
+
A mouse is one pointer, so it cannot press two things at once. `input.touch` puts every point down together, holds them for `holdMs` (default 50), and lifts them together:
|
|
279
|
+
|
|
280
|
+
```bash
|
|
281
|
+
# Three-finger chord
|
|
282
|
+
bbx call input.touch '{"points":[{"target":{"selector":"#c4"}},{"target":{"selector":"#e4"}},{"target":{"selector":"#g4"}}],"holdMs":600,"executionMode":"cdp"}'
|
|
283
|
+
# Swipe, and pinch-out with viewport coordinates
|
|
284
|
+
bbx call input.touch '{"points":[{"x":300,"y":400,"to":{"x":60,"y":400}}],"holdMs":250}'
|
|
285
|
+
bbx call input.touch '{"points":[{"x":200,"y":300,"to":{"x":120,"y":300}},{"x":240,"y":300,"to":{"x":320,"y":300}}],"holdMs":300,"moveSteps":12}'
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
- Each point is either `target` (`elementRef`/`selector`, actionability-checked at its center) or viewport `x`/`y`, plus an optional `to` end position. Up to 10 points.
|
|
289
|
+
- Points with `to` move in `moveSteps` (default 10) interpolated steps spread across `holdMs`.
|
|
290
|
+
- DOM mode fires per-finger `pointerdown`/`pointermove`/`pointerup` (`pointerType: "touch"`, distinct `pointerId`, first finger primary) and `touchstart`/`touchmove`/`touchend` with accurate `touches`. A single-finger tap without movement also produces the compatibility `mousedown`/`mouseup`/`click`.
|
|
291
|
+
- CDP mode sends trusted `Input.dispatchTouchEvent` events; fingers are always lifted, even after an error. Pages that only enable touch handlers after feature-detecting a touch screen may still ignore desktop touch input.
|
|
292
|
+
- Result: `pointCount`, per-point `elementRef` and coordinates, and in DOM mode `canceled` (the page called `preventDefault`) and `clicked`.
|
|
229
293
|
|
|
230
294
|
## Actionability And Stale Refs
|
|
231
295
|
|
|
@@ -256,7 +320,7 @@ bbx hover el_abc123
|
|
|
256
320
|
bbx call input.hover '{"target":{"elementRef":"el_abc123"}}'
|
|
257
321
|
```
|
|
258
322
|
|
|
259
|
-
**Hold hover for inspection:** set `duration` (ms) to
|
|
323
|
+
**Hold hover for inspection:** set `duration` (ms) to wait after hovering before the call returns. The pointer stays over the element until a later input moves it elsewhere, which then fires `mouseout`/`mouseleave`:
|
|
260
324
|
|
|
261
325
|
```bash
|
|
262
326
|
bbx call input.hover '{"target":{"elementRef":"el_abc123"},"duration":2000}'
|
|
@@ -283,7 +347,12 @@ With pixel offsets for precise positioning:
|
|
|
283
347
|
bbx call input.drag '{"source":{"elementRef":"el_src"},"destination":{"elementRef":"el_dst"},"offsetX":5,"offsetY":5}'
|
|
284
348
|
```
|
|
285
349
|
|
|
286
|
-
|
|
350
|
+
DOM mode picks the sequence a browser would use:
|
|
351
|
+
|
|
352
|
+
- **Draggable sources** (`draggable="true"`, links, images): `pointerdown → mousedown → dragstart → pointercancel → drag → dragenter → dragover → drop → dragend`. `drop` only fires when the destination cancels `dragover`, as real drop zones must; otherwise it gets `dragleave`.
|
|
353
|
+
- **Other sources** (sortable lists, sliders, canvases, and other pointer-driven UIs): `pointerdown/mousedown`, ten interpolated `pointermove/mousemove` steps with the button held across whatever elements lie under the path (with over/out transitions), then `pointerup/mouseup` at the destination.
|
|
354
|
+
|
|
355
|
+
The result's `strategy` reports `html5` or `pointer`. CDP mode drives a real pointer drag either way.
|
|
287
356
|
|
|
288
357
|
Typical workflow - reorder a list:
|
|
289
358
|
|
|
@@ -85,15 +85,17 @@ omit both for the group index.
|
|
|
85
85
|
| `styles.get_matched_rules` | Yes | - | inspect | `styles.read` | Element ref, class list, and inline style only; not stylesheet cascade data |
|
|
86
86
|
| `viewport.scroll` | Yes | - | navigate | `viewport.control` | Window or element scroll |
|
|
87
87
|
| `viewport.resize` | Yes | CDP | navigate | `viewport.control` | Set viewport via device emulation; `reset: true` |
|
|
88
|
-
| `input.click` | Yes | Optional | interact | `automation.input` | Actionability-aware DOM or CDP click
|
|
88
|
+
| `input.click` | Yes | Optional | interact | `automation.input` | Actionability-aware DOM or CDP click; `holdMs` keeps the button pressed |
|
|
89
89
|
| `input.focus` | Yes | - | interact | `automation.input` | Actionability-aware DOM focus |
|
|
90
90
|
| `input.type` | Yes | Optional | interact | `automation.input` | DOM key sequence or CDP text insertion |
|
|
91
91
|
| `input.fill` | Yes | Optional | interact | `automation.input` | DOM fill strategy or CDP clear/insert; see `mode` versus `executionMode` |
|
|
92
|
-
| `input.press_key` | Yes |
|
|
92
|
+
| `input.press_key` | Yes | Optional | interact | `automation.input` | DOM key events (with `code`/`keyCode`) or trusted CDP key pair; `holdMs` |
|
|
93
93
|
| `input.set_checked` | Yes | - | interact | `automation.input` | Checkbox/radio toggle |
|
|
94
94
|
| `input.select_option` | Yes | - | interact | `automation.input` | Native select by value/label/index |
|
|
95
95
|
| `input.hover` | Yes | Optional | interact | `automation.input` | DOM hover events or native CDP pointer move |
|
|
96
|
-
| `input.drag` | Yes | Optional | interact | `automation.input` |
|
|
96
|
+
| `input.drag` | Yes | Optional | interact | `automation.input` | HTML5 drag for draggable sources, pointer drag otherwise; or native CDP pointer drag |
|
|
97
|
+
| `input.touch` | Yes | Optional | interact | `automation.input` | 1-10 simultaneous touch points (chords, taps, swipes, pinches) held for `holdMs` |
|
|
98
|
+
| `input.perform` | Yes | Per step | interact | `automation.input` | Ordered, timed sequence of input steps executed inside the extension |
|
|
97
99
|
| `input.scroll_into_view` | Yes | - | interact | `automation.input` | Explicitly scroll target into view before inspect/capture |
|
|
98
100
|
| `screenshot.capture_element` | Yes | CDP | capture | `screenshot.partial` | Complete element screenshot with explicit completeness metadata |
|
|
99
101
|
| `screenshot.capture_region` | Yes | CDP | capture | `screenshot.partial` | Cropped viewport region in PNG, JPEG, or WebP |
|
|
@@ -294,7 +296,7 @@ When `properties` is omitted, `styles.get_computed` returns exactly `display`, `
|
|
|
294
296
|
|
|
295
297
|
Input calls preserve an explicit `elementRef`. For selectors, the first actionable match wins; if it is not actionable, Browser Bridge evaluates at most 25 matches and chooses only a uniquely better candidate. It scrolls the selected target if needed and rechecks visibility, disabled/inert state, rendered bounds, and pointer hit testing before dispatch. Failures use `ELEMENT_NOT_FOUND`, `ELEMENT_NOT_ACTIONABLE`, `ELEMENT_OBSCURED`, or `ELEMENT_AMBIGUOUS` with bounded details.
|
|
296
298
|
|
|
297
|
-
Successful targeted click, focus, type, fill, press-key, checked-state, option-selection, hover, and drag results include `resolution` (`strategy`, candidate/evaluated counts, scrolling, hit test, and recovery fields) and `execution` (`requestedMode`, `actualMode`, `fallbackReason`, `debuggerUsed`, and coordinates). `cdp.dispatch_key_event`/MCP `cdp_press_key` and `input.scroll_into_view` use separate response contracts. `executionMode` is the dispatch path and accepts only `dom` or `cdp`, defaulting to `dom` for compatibility. CDP is supported
|
|
299
|
+
Successful targeted click, focus, type, fill, press-key, checked-state, option-selection, hover, and drag results include `resolution` (`strategy`, candidate/evaluated counts, scrolling, hit test, and recovery fields) and `execution` (`requestedMode`, `actualMode`, `fallbackReason`, `debuggerUsed`, and coordinates). `cdp.dispatch_key_event`/MCP `cdp_press_key` and `input.scroll_into_view` use separate response contracts. `executionMode` is the dispatch path and accepts only `dom` or `cdp`, defaulting to `dom` for compatibility. CDP is supported by click, hover, drag, type, fill, press_key, and touch; unsupported combinations return `INPUT_UNSUPPORTED` instead of falling back silently. DOM events mirror real device event order (pointer events, boundary transitions, `code`/`keyCode`) but are untrusted, so Chrome may refuse gesture-gated features such as audio/media start; use CDP for those.
|
|
298
300
|
|
|
299
301
|
`input.fill.mode` is separate: `auto`, `setter`, and `keystrokes` choose the DOM fill strategy. `mode: "auto"` may fall back from the setter to keystrokes and reports the used `mode`; it does not select CDP. With `executionMode: "cdp"`, fill uses native clear/text dispatch and reports `mode: "cdp"`.
|
|
300
302
|
|
|
@@ -313,13 +315,30 @@ bbx call input.hover '{"target":{"elementRef":"el_abc123"},"duration":1000}'
|
|
|
313
315
|
|
|
314
316
|
### input.drag
|
|
315
317
|
|
|
316
|
-
|
|
318
|
+
For natively draggable sources the DOM path dispatches `pointerdown → mousedown → dragstart → pointercancel → drag → dragenter → dragover → drop → dragend` (`drop` only when the destination cancels `dragover`). Other sources get a pointer drag: press, ten interpolated moves with the button held, release at the destination. `strategy` reports `html5` or `pointer`. The CDP path performs a bounded interpolated native pointer drag and guarantees a release attempt after failure. Both accept source, destination, and optional destination offsets.
|
|
317
319
|
|
|
318
320
|
```bash
|
|
319
321
|
bbx call input.drag '{"source":{"elementRef":"el_src"},"destination":{"elementRef":"el_dst"}}'
|
|
320
322
|
bbx call input.drag '{"source":{"elementRef":"el_src"},"destination":{"elementRef":"el_dst"},"offsetX":10,"offsetY":10}'
|
|
321
323
|
```
|
|
322
324
|
|
|
325
|
+
### input.touch
|
|
326
|
+
|
|
327
|
+
Put 1-10 fingers down at once, hold them for `holdMs` (default 50), optionally move points that have a `to` end position over `moveSteps` (default 10), then lift them. Each point is `{ target }` or viewport `{ x, y }`. DOM mode fires per-finger pointer events (`pointerType: "touch"`) and touch events; CDP mode uses trusted `Input.dispatchTouchEvent`.
|
|
328
|
+
|
|
329
|
+
```bash
|
|
330
|
+
bbx call input.touch '{"points":[{"target":{"selector":"#c4"}},{"target":{"selector":"#e4"}}],"holdMs":500,"executionMode":"cdp"}'
|
|
331
|
+
bbx call input.touch '{"points":[{"x":300,"y":400,"to":{"x":60,"y":400}}],"holdMs":250}'
|
|
332
|
+
```
|
|
333
|
+
|
|
334
|
+
### input.perform
|
|
335
|
+
|
|
336
|
+
Run `steps: [{ method, params, atMs | delayMs }]` in order inside the extension, so timing does not depend on agent round-trips. Step methods: input click/focus/type/fill/press_key/set_checked/select_option/hover/drag/touch/scroll_into_view, `viewport.scroll`, `dom.wait_for`, and `page.wait_for_load_state`. `atMs` is measured from the sequence start (no drift); `delayMs` follows the previous step. A top-level `executionMode` applies to steps that support it. The sequence stops at the first failed step (including a timed-out `dom.wait_for`) with that step's code and `details.failedStep`, unless `continueOnError: true`. `timeoutMs` defaults to 30000 (max 120000); at most 200 steps. Success returns `completed`, `total`, `elapsedMs`, `startedAtMs`, and `failures`.
|
|
337
|
+
|
|
338
|
+
```bash
|
|
339
|
+
bbx call input.perform '{"executionMode":"cdp","steps":[{"method":"input.click","params":{"target":{"selector":"#c4"},"holdMs":300},"atMs":0},{"method":"input.click","params":{"target":{"selector":"#d4"},"holdMs":300},"atMs":400}]}'
|
|
340
|
+
```
|
|
341
|
+
|
|
323
342
|
### input.scroll_into_view
|
|
324
343
|
|
|
325
344
|
Explicitly scroll an element into the visible viewport before inspecting, hovering, or capturing it.
|