agentic-browser 2.8.0 → 2.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cli/index.mjs +38 -79
- package/dist/index.d.mts +77 -4
- package/dist/index.mjs +1 -1
- package/dist/mcp/index.mjs +29 -73
- package/dist/{runtime-DdEXOpiU.mjs → runtime-CgX6XW9u.mjs} +202 -60
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -144,7 +144,7 @@ agentic-browser agent elements --roles button,link,input --visible-only --limit
|
|
|
144
144
|
agentic-browser agent elements --selector "#main-content"
|
|
145
145
|
```
|
|
146
146
|
|
|
147
|
-
Returns a JSON array of elements with selectors and fallback selectors usable in the typed agent commands like `agent click`, `agent type`, and `agent select`. Main-content elements come first; header, nav and footer links come last. Each element also gets a short ref (`e1`, `e2`, ...): `agent click e3` resolves it to the stored selector. Refs
|
|
147
|
+
Returns a JSON array of elements with selectors and fallback selectors usable in the typed agent commands like `agent click`, `agent type`, and `agent select`. Main-content elements come first; header, nav and footer links come last. Each element also gets a short ref (`e1`, `e2`, ...): `agent click e3` resolves it to the stored selector. Refs stay valid while the page URL is unchanged; after navigating, run `agent elements` again:
|
|
148
148
|
|
|
149
149
|
```json
|
|
150
150
|
{
|
|
@@ -275,7 +275,7 @@ agentic-browser memory:stats
|
|
|
275
275
|
|
|
276
276
|
1. `agent start` — launch Chrome and persist session.
|
|
277
277
|
2. `agent elements` — discover what's on the page.
|
|
278
|
-
3. `agent navigate`, `agent click`, `agent type`, `agent select`, and related typed commands — execute actions using element refs or selectors. Pass `--intent "<task>"` to group the steps of one task in memory.
|
|
278
|
+
3. `agent navigate`, `agent click`, `agent type`, `agent select`, and related typed commands — execute actions using element refs or selectors. Pass `--intent "<task>"` (an `agent`-level option, accepted before or after the subcommand) to group the steps of one task in memory. Consecutive steps with the same intent form one run; a new run replaces the stored recipe, so it always holds the latest run's steps in order.
|
|
279
279
|
4. `agent content` — read page content after actions.
|
|
280
280
|
5. `agent memory-search "<task>"` — matching is fuzzy; results include a `recipe` with the steps that worked last time (`needsInput` marks typed text that was redacted).
|
|
281
281
|
6. `agent stop` — terminate when done.
|
package/dist/cli/index.mjs
CHANGED
|
@@ -1,35 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
2
|
+
import { c as writePrivateFileSync, o as isConnectionError, r as createCliRuntime, s as ensurePrivateDir } from "../runtime-CgX6XW9u.mjs";
|
|
3
3
|
import fs from "node:fs";
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import crypto from "node:crypto";
|
|
6
6
|
import { Command } from "commander";
|
|
7
7
|
// src/cli/commands/agent.ts
|
|
8
8
|
const STATE_FILE_NAME = "agent-state.json";
|
|
9
|
-
const REF_PATTERN = /^@?(e\d+)$/;
|
|
10
|
-
function storeRefs(runtime, refs) {
|
|
11
|
-
saveState(runtime, {
|
|
12
|
-
...loadState(runtime),
|
|
13
|
-
refs
|
|
14
|
-
});
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Swap a short ref ("e3" or "@e3") from the last `agent elements` call for the
|
|
18
|
-
* selector it stands for. Plain selectors pass through untouched.
|
|
19
|
-
*/
|
|
20
|
-
function resolveRef(runtime, sessionId, input) {
|
|
21
|
-
const match = input.selector ? REF_PATTERN.exec(input.selector) : null;
|
|
22
|
-
if (!match) return input;
|
|
23
|
-
const state = loadState(runtime);
|
|
24
|
-
const stored = state.refs?.sessionId === sessionId ? state.refs.byRef[match[1]] : void 0;
|
|
25
|
-
if (!stored) throw new Error(`Unknown or stale ref "${input.selector}". Run "agent elements" to refresh element refs.`);
|
|
26
|
-
return {
|
|
27
|
-
...input,
|
|
28
|
-
selector: stored.selector,
|
|
29
|
-
fallbackSelectors: [...stored.fallbackSelectors ?? [], ...input.fallbackSelectors ?? []],
|
|
30
|
-
frame: input.frame ?? stored.frame
|
|
31
|
-
};
|
|
32
|
-
}
|
|
33
9
|
function stateFilePath(runtime) {
|
|
34
10
|
return path.join(runtime.context.config.dataDir, STATE_FILE_NAME);
|
|
35
11
|
}
|
|
@@ -111,7 +87,8 @@ async function executeAgentCommand(runtime, input) {
|
|
|
111
87
|
const attempt = () => runtime.api.executeCommand(sessionId, {
|
|
112
88
|
commandId: nextCommandId(),
|
|
113
89
|
type: input.type,
|
|
114
|
-
payload: input.payload
|
|
90
|
+
payload: input.payload,
|
|
91
|
+
intent: input.intent
|
|
115
92
|
});
|
|
116
93
|
const retryAfterRestart = async () => {
|
|
117
94
|
await runtime.api.restartSession(sessionId);
|
|
@@ -135,16 +112,12 @@ async function executeAgentCommand(runtime, input) {
|
|
|
135
112
|
async function agentNavigate(runtime, input) {
|
|
136
113
|
const result = await executeAgentCommand(runtime, {
|
|
137
114
|
type: "navigate",
|
|
138
|
-
payload: input.
|
|
139
|
-
|
|
140
|
-
intent: input.intent
|
|
141
|
-
} : { url: input.url }
|
|
115
|
+
payload: { url: input.url },
|
|
116
|
+
intent: input.intent
|
|
142
117
|
});
|
|
143
|
-
storeRefs(runtime, void 0);
|
|
144
118
|
let cookieBanner;
|
|
145
119
|
if (result.resultStatus === "success") try {
|
|
146
|
-
const
|
|
147
|
-
const dismissed = await runtime.api.dismissCookieBanner(sessionId);
|
|
120
|
+
const dismissed = await runtime.api.dismissCookieBanner(result.sessionId);
|
|
148
121
|
if (dismissed.dismissed) cookieBanner = dismissed;
|
|
149
122
|
} catch {}
|
|
150
123
|
const page = await tryPageState(runtime, result.sessionId);
|
|
@@ -156,10 +129,13 @@ async function agentNavigate(runtime, input) {
|
|
|
156
129
|
cookieBanner
|
|
157
130
|
};
|
|
158
131
|
}
|
|
132
|
+
/** `selector` may be a ref ("e3") from `agent elements`; the core resolves it. */
|
|
159
133
|
async function agentInteract(runtime, input) {
|
|
134
|
+
const { intent, ...payload } = input;
|
|
160
135
|
const result = await executeAgentCommand(runtime, {
|
|
161
136
|
type: "interact",
|
|
162
|
-
payload
|
|
137
|
+
payload,
|
|
138
|
+
intent
|
|
163
139
|
});
|
|
164
140
|
const page = await tryPageState(runtime, result.sessionId);
|
|
165
141
|
return {
|
|
@@ -390,37 +366,18 @@ async function agentScreenshot(runtime, input) {
|
|
|
390
366
|
async function agentElements(runtime, input) {
|
|
391
367
|
const sessionId = requireSessionId(runtime);
|
|
392
368
|
await ensureReady(runtime, sessionId);
|
|
393
|
-
const result = await runtime.api.getInteractiveElements(sessionId, {
|
|
394
|
-
roles: input.roles,
|
|
395
|
-
visibleOnly: input.visibleOnly,
|
|
396
|
-
limit: input.limit,
|
|
397
|
-
selector: input.selector,
|
|
398
|
-
href: input.href,
|
|
399
|
-
text: input.text,
|
|
400
|
-
frame: input.frame
|
|
401
|
-
});
|
|
402
|
-
const byRef = {};
|
|
403
|
-
const elements = result.elements.map((element, index) => {
|
|
404
|
-
const ref = `e${index + 1}`;
|
|
405
|
-
byRef[ref] = {
|
|
406
|
-
selector: element.selector,
|
|
407
|
-
fallbackSelectors: element.fallbackSelectors,
|
|
408
|
-
frame: input.frame
|
|
409
|
-
};
|
|
410
|
-
return {
|
|
411
|
-
ref,
|
|
412
|
-
...element
|
|
413
|
-
};
|
|
414
|
-
});
|
|
415
|
-
storeRefs(runtime, {
|
|
416
|
-
sessionId,
|
|
417
|
-
byRef
|
|
418
|
-
});
|
|
419
369
|
return {
|
|
420
370
|
ok: true,
|
|
421
371
|
action: "elements",
|
|
422
|
-
...
|
|
423
|
-
|
|
372
|
+
...await runtime.api.getInteractiveElements(sessionId, {
|
|
373
|
+
roles: input.roles,
|
|
374
|
+
visibleOnly: input.visibleOnly,
|
|
375
|
+
limit: input.limit,
|
|
376
|
+
selector: input.selector,
|
|
377
|
+
href: input.href,
|
|
378
|
+
text: input.text,
|
|
379
|
+
frame: input.frame
|
|
380
|
+
})
|
|
424
381
|
};
|
|
425
382
|
}
|
|
426
383
|
async function agentMemorySearch(runtime, input) {
|
|
@@ -602,7 +559,9 @@ async function main() {
|
|
|
602
559
|
const result = await runMemoryStats(runtime);
|
|
603
560
|
console.log(JSON.stringify(result));
|
|
604
561
|
});
|
|
605
|
-
const agent = program.command("agent").description("Stateful agent wrapper with session persistence and auto-retry");
|
|
562
|
+
const agent = program.command("agent").description("Stateful agent wrapper with session persistence and auto-retry").option("--intent <task>", INTENT_HELP);
|
|
563
|
+
/** --intent of the current agent step command; memory groups steps by it. */
|
|
564
|
+
const stepIntent = () => agent.opts().intent;
|
|
606
565
|
agent.command("start").option("--cdp-url <url>", "connect to existing Chrome via CDP endpoint URL").option("--user-profile <path>", "use 'default' for system Chrome profile or an absolute path").option("--headless", "run Chrome in headless mode (no visible window)").option("--user-agent <string>", "override the browser user-agent string").action(async (options) => {
|
|
607
566
|
if (options.cdpUrl) runtime.context.config.cdpUrl = options.cdpUrl;
|
|
608
567
|
if (options.userProfile) runtime.context.config.userProfileDir = options.userProfile === "true" || options.userProfile === "default" ? "default" : options.userProfile;
|
|
@@ -619,24 +578,24 @@ async function main() {
|
|
|
619
578
|
const result = await agentStop(runtime);
|
|
620
579
|
console.log(JSON.stringify(result));
|
|
621
580
|
});
|
|
622
|
-
agent.command("navigate").argument("<url>").
|
|
581
|
+
agent.command("navigate").argument("<url>").action(async (url) => {
|
|
623
582
|
const result = await agentNavigate(runtime, {
|
|
624
583
|
url,
|
|
625
|
-
intent:
|
|
584
|
+
intent: stepIntent()
|
|
626
585
|
});
|
|
627
586
|
console.log(JSON.stringify(result));
|
|
628
587
|
});
|
|
629
|
-
agent.command("click").argument("[selector]", "element ref (e3) from agent elements, CSS selector, or composed locator (a >>> b)").option("--text <text>", "click the element with this visible text instead of a selector").option("--exact", "with --text: require an exact text match").option("--frame <selector>", FRAME_HELP).option("--fallback <selector>", "backup CSS selector", collect, []).
|
|
588
|
+
agent.command("click").argument("[selector]", "element ref (e3) from agent elements, CSS selector, or composed locator (a >>> b)").option("--text <text>", "click the element with this visible text instead of a selector").option("--exact", "with --text: require an exact text match").option("--frame <selector>", FRAME_HELP).option("--fallback <selector>", "backup CSS selector", collect, []).action(async (selector, options) => {
|
|
630
589
|
const result = await agentInteract(runtime, {
|
|
631
590
|
action: "click",
|
|
632
591
|
selector: resolveTarget(selector, options),
|
|
633
592
|
frame: options.frame,
|
|
634
593
|
fallbackSelectors: options.fallback,
|
|
635
|
-
intent:
|
|
594
|
+
intent: stepIntent()
|
|
636
595
|
});
|
|
637
596
|
console.log(JSON.stringify(result));
|
|
638
597
|
});
|
|
639
|
-
agent.command("type").argument("<selectorOrText>", "element ref (e3), CSS selector, or the text to type when --into-text is used").argument("[text]").option("--into-text <label>", "target the field by its visible text/label/placeholder").option("--frame <selector>", FRAME_HELP).option("--fallback <selector>", "backup CSS selector", collect, []).
|
|
598
|
+
agent.command("type").argument("<selectorOrText>", "element ref (e3), CSS selector, or the text to type when --into-text is used").argument("[text]").option("--into-text <label>", "target the field by its visible text/label/placeholder").option("--frame <selector>", FRAME_HELP).option("--fallback <selector>", "backup CSS selector", collect, []).action(async (first, second, options) => {
|
|
640
599
|
const selector = options.intoText ? `text=${options.intoText}` : first;
|
|
641
600
|
const text = options.intoText ? first : second;
|
|
642
601
|
if (text === void 0) throw new Error("Missing text to type");
|
|
@@ -646,15 +605,15 @@ async function main() {
|
|
|
646
605
|
text,
|
|
647
606
|
frame: options.frame,
|
|
648
607
|
fallbackSelectors: options.fallback,
|
|
649
|
-
intent:
|
|
608
|
+
intent: stepIntent()
|
|
650
609
|
});
|
|
651
610
|
console.log(JSON.stringify(result));
|
|
652
611
|
});
|
|
653
|
-
agent.command("press").argument("<key>").
|
|
612
|
+
agent.command("press").argument("<key>").action(async (key) => {
|
|
654
613
|
const result = await agentInteract(runtime, {
|
|
655
614
|
action: "press",
|
|
656
615
|
key,
|
|
657
|
-
intent:
|
|
616
|
+
intent: stepIntent()
|
|
658
617
|
});
|
|
659
618
|
console.log(JSON.stringify(result));
|
|
660
619
|
});
|
|
@@ -669,43 +628,43 @@ async function main() {
|
|
|
669
628
|
});
|
|
670
629
|
console.log(JSON.stringify(result));
|
|
671
630
|
});
|
|
672
|
-
agent.command("hover").argument("[selector]").option("--text <text>", "hover the element with this visible text").option("--exact", "with --text: require an exact text match").option("--frame <selector>", FRAME_HELP).option("--fallback <selector>", "backup CSS selector", collect, []).
|
|
631
|
+
agent.command("hover").argument("[selector]").option("--text <text>", "hover the element with this visible text").option("--exact", "with --text: require an exact text match").option("--frame <selector>", FRAME_HELP).option("--fallback <selector>", "backup CSS selector", collect, []).action(async (selector, options) => {
|
|
673
632
|
const result = await agentInteract(runtime, {
|
|
674
633
|
action: "hover",
|
|
675
634
|
selector: resolveTarget(selector, options),
|
|
676
635
|
frame: options.frame,
|
|
677
636
|
fallbackSelectors: options.fallback,
|
|
678
|
-
intent:
|
|
637
|
+
intent: stepIntent()
|
|
679
638
|
});
|
|
680
639
|
console.log(JSON.stringify(result));
|
|
681
640
|
});
|
|
682
|
-
agent.command("select").argument("<selector>").argument("<value>").option("--frame <selector>", FRAME_HELP).option("--fallback <selector>", "backup CSS selector", collect, []).
|
|
641
|
+
agent.command("select").argument("<selector>").argument("<value>").option("--frame <selector>", FRAME_HELP).option("--fallback <selector>", "backup CSS selector", collect, []).action(async (selector, value, options) => {
|
|
683
642
|
const result = await agentInteract(runtime, {
|
|
684
643
|
action: "select",
|
|
685
644
|
selector,
|
|
686
645
|
value,
|
|
687
646
|
frame: options.frame,
|
|
688
647
|
fallbackSelectors: options.fallback,
|
|
689
|
-
intent:
|
|
648
|
+
intent: stepIntent()
|
|
690
649
|
});
|
|
691
650
|
console.log(JSON.stringify(result));
|
|
692
651
|
});
|
|
693
|
-
agent.command("toggle").argument("<selector>").option("--frame <selector>", FRAME_HELP).option("--fallback <selector>", "backup CSS selector", collect, []).
|
|
652
|
+
agent.command("toggle").argument("<selector>").option("--frame <selector>", FRAME_HELP).option("--fallback <selector>", "backup CSS selector", collect, []).action(async (selector, options) => {
|
|
694
653
|
const result = await agentInteract(runtime, {
|
|
695
654
|
action: "toggle",
|
|
696
655
|
selector,
|
|
697
656
|
frame: options.frame,
|
|
698
657
|
fallbackSelectors: options.fallback,
|
|
699
|
-
intent:
|
|
658
|
+
intent: stepIntent()
|
|
700
659
|
});
|
|
701
660
|
console.log(JSON.stringify(result));
|
|
702
661
|
});
|
|
703
|
-
agent.command("scroll").option("--x <pixels>", "horizontal scroll delta", "0").option("--y <pixels>", "vertical scroll delta", "0").option("--selector <selector>", "scroll a specific element instead of the page").option("--fallback <selector>", "backup CSS selector", collect, []).
|
|
662
|
+
agent.command("scroll").option("--x <pixels>", "horizontal scroll delta", "0").option("--y <pixels>", "vertical scroll delta", "0").option("--selector <selector>", "scroll a specific element instead of the page").option("--fallback <selector>", "backup CSS selector", collect, []).action(async (options) => {
|
|
704
663
|
const result = await agentInteract(runtime, {
|
|
705
664
|
action: "scroll",
|
|
706
665
|
selector: options.selector,
|
|
707
666
|
fallbackSelectors: options.fallback,
|
|
708
|
-
intent:
|
|
667
|
+
intent: stepIntent(),
|
|
709
668
|
scrollX: Number.parseInt(options.x, 10),
|
|
710
669
|
scrollY: Number.parseInt(options.y, 10)
|
|
711
670
|
});
|
package/dist/index.d.mts
CHANGED
|
@@ -49,6 +49,8 @@ interface InteractiveElement {
|
|
|
49
49
|
ariaLabel?: string;
|
|
50
50
|
placeholder?: string;
|
|
51
51
|
currentValue?: string;
|
|
52
|
+
/** Short handle ("e3") usable in place of the selector until the page changes. */
|
|
53
|
+
ref?: string;
|
|
52
54
|
}
|
|
53
55
|
interface InteractiveElementsOptions {
|
|
54
56
|
roles?: InteractiveElementRole[];
|
|
@@ -171,6 +173,7 @@ declare const TaskInsightSchema: z.ZodObject<{
|
|
|
171
173
|
selector: z.ZodOptional<z.ZodString>;
|
|
172
174
|
payload: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
173
175
|
}, z.core.$strip>>;
|
|
176
|
+
recipeRunId: z.ZodOptional<z.ZodString>;
|
|
174
177
|
expectedOutcome: z.ZodString;
|
|
175
178
|
confidence: z.ZodNumber;
|
|
176
179
|
successCount: z.ZodNumber;
|
|
@@ -408,6 +411,11 @@ interface RecordExecutionInput {
|
|
|
408
411
|
expectedOutcome: string;
|
|
409
412
|
url?: string;
|
|
410
413
|
selector?: string;
|
|
414
|
+
/**
|
|
415
|
+
* Run this step belongs to. A step from a new run starts the recipe over, so
|
|
416
|
+
* the recipe is always the latest run's steps in order. Defaults to commandId.
|
|
417
|
+
*/
|
|
418
|
+
runId?: string;
|
|
411
419
|
}
|
|
412
420
|
declare class MemoryService {
|
|
413
421
|
private readonly store;
|
|
@@ -461,6 +469,7 @@ declare class MemoryService {
|
|
|
461
469
|
private createEvidence;
|
|
462
470
|
private maybeGenerateAliases;
|
|
463
471
|
private deriveAliasName;
|
|
472
|
+
/** Same run: append the step. New run: its first step replaces the old recipe. */
|
|
464
473
|
private mergeRecipe;
|
|
465
474
|
}
|
|
466
475
|
interface AppContext {
|
|
@@ -470,12 +479,43 @@ interface AppContext {
|
|
|
470
479
|
tokenService: SessionTokenService;
|
|
471
480
|
memoryService: MemoryService;
|
|
472
481
|
}
|
|
482
|
+
/**
|
|
483
|
+
* Short, stable handles ("e1", "e2", …) an agent can point at instead of
|
|
484
|
+
* authoring CSS selectors — the way a human points at what they see rather
|
|
485
|
+
* than reasoning about the DOM.
|
|
486
|
+
*
|
|
487
|
+
* Refs are ephemeral per "look": each element listing replaces the session's
|
|
488
|
+
* ref set, and a ref only resolves while the page URL is the one it was taken
|
|
489
|
+
* on. SessionManager persists them on the session record, so the CLI (one
|
|
490
|
+
* process per command) and the MCP server share the same mechanism.
|
|
491
|
+
*/
|
|
492
|
+
interface ResolvedRef {
|
|
493
|
+
selector: string;
|
|
494
|
+
fallbackSelectors?: string[];
|
|
495
|
+
/** Iframe the element was discovered in, so the ref acts inside it. */
|
|
496
|
+
frame?: string;
|
|
497
|
+
}
|
|
498
|
+
interface StoredRefs {
|
|
499
|
+
/** Page the refs were taken on; on any other page they are stale. */
|
|
500
|
+
url?: string;
|
|
501
|
+
byRef: Record<string, ResolvedRef>;
|
|
502
|
+
}
|
|
473
503
|
interface StoredSessionRecord {
|
|
474
504
|
session: Session;
|
|
475
505
|
cdpUrl: string;
|
|
476
506
|
targetWsUrl: string;
|
|
477
507
|
pid: number;
|
|
478
508
|
lastUrl?: string;
|
|
509
|
+
/** Element refs from the last element listing. */
|
|
510
|
+
refs?: StoredRefs;
|
|
511
|
+
/**
|
|
512
|
+
* The task run in progress: consecutive commands with the same explicit
|
|
513
|
+
* intent share `runId`, so memory can rebuild that task's recipe per run.
|
|
514
|
+
*/
|
|
515
|
+
intentRun?: {
|
|
516
|
+
intent: string;
|
|
517
|
+
runId: string;
|
|
518
|
+
};
|
|
479
519
|
}
|
|
480
520
|
interface CreateSessionInput {
|
|
481
521
|
browser: "chrome";
|
|
@@ -484,6 +524,8 @@ interface ExecuteCommandInput$1 {
|
|
|
484
524
|
commandId: string;
|
|
485
525
|
type: CommandType;
|
|
486
526
|
payload: Record<string, unknown>;
|
|
527
|
+
/** Task this step belongs to (e.g. "log in"); memory groups steps by it. */
|
|
528
|
+
intent?: string;
|
|
487
529
|
}
|
|
488
530
|
interface CleanupInput {
|
|
489
531
|
maxAgeDays?: number;
|
|
@@ -504,9 +546,15 @@ declare class SessionManager {
|
|
|
504
546
|
getSession(sessionId: string): Session;
|
|
505
547
|
/** Return a healthy session, recovering automatically if needed. */
|
|
506
548
|
ensureSession(sessionId: string): Promise<StoredSessionRecord>;
|
|
507
|
-
executeCommand(sessionId: string,
|
|
549
|
+
executeCommand(sessionId: string, rawInput: ExecuteCommandInput$1): Promise<Command>;
|
|
508
550
|
getContent(sessionId: string, options: PageContentOptions): Promise<PageContentResult>;
|
|
509
|
-
|
|
551
|
+
/** Lists interactive elements and assigns fresh refs (e1, e2, …) for this page. */
|
|
552
|
+
getInteractiveElements(sessionId: string, options: InteractiveElementsOptions): Promise<{
|
|
553
|
+
totalFound: number;
|
|
554
|
+
truncated: boolean;
|
|
555
|
+
crossOriginFrames?: CrossOriginFrameInfo[];
|
|
556
|
+
elements: InteractiveElement[];
|
|
557
|
+
}>;
|
|
510
558
|
screenshot(sessionId: string, options?: ScreenshotOptions): Promise<ScreenshotResult>;
|
|
511
559
|
getCurrentUrl(sessionId: string): Promise<string>;
|
|
512
560
|
getPageState(sessionId: string): Promise<PageState>;
|
|
@@ -544,6 +592,7 @@ declare class SessionManager {
|
|
|
544
592
|
selector?: string | undefined;
|
|
545
593
|
payload: Record<string, unknown>;
|
|
546
594
|
}[];
|
|
595
|
+
recipeRunId?: string | undefined;
|
|
547
596
|
expectedOutcome: string;
|
|
548
597
|
confidence: number;
|
|
549
598
|
successCount: number;
|
|
@@ -581,6 +630,7 @@ declare class SessionManager {
|
|
|
581
630
|
selector?: string | undefined;
|
|
582
631
|
payload: Record<string, unknown>;
|
|
583
632
|
}[];
|
|
633
|
+
recipeRunId?: string | undefined;
|
|
584
634
|
expectedOutcome: string;
|
|
585
635
|
confidence: number;
|
|
586
636
|
successCount: number;
|
|
@@ -625,6 +675,12 @@ declare class SessionManager {
|
|
|
625
675
|
private mustGetRecord;
|
|
626
676
|
private recordEvent;
|
|
627
677
|
private buildMemoryContext;
|
|
678
|
+
private resolveRef;
|
|
679
|
+
/**
|
|
680
|
+
* Consecutive commands with the same explicit intent form one run; any other
|
|
681
|
+
* command ends it. Commands without an intent are each their own run.
|
|
682
|
+
*/
|
|
683
|
+
private trackIntentRun;
|
|
628
684
|
private parseUrl;
|
|
629
685
|
}
|
|
630
686
|
declare class ControlApi {
|
|
@@ -653,6 +709,7 @@ declare class ControlApi {
|
|
|
653
709
|
commandId: string;
|
|
654
710
|
type: "navigate" | "interact" | "restart" | "terminate";
|
|
655
711
|
payload: Record<string, unknown>;
|
|
712
|
+
intent?: string;
|
|
656
713
|
}): Promise<{
|
|
657
714
|
commandId: string;
|
|
658
715
|
sessionId: string;
|
|
@@ -677,7 +734,12 @@ declare class ControlApi {
|
|
|
677
734
|
selector?: string;
|
|
678
735
|
maxChars?: number;
|
|
679
736
|
}): Promise<PageContentResult>;
|
|
680
|
-
getInteractiveElements(sessionId: string, options: InteractiveElementsOptions): Promise<
|
|
737
|
+
getInteractiveElements(sessionId: string, options: InteractiveElementsOptions): Promise<{
|
|
738
|
+
totalFound: number;
|
|
739
|
+
truncated: boolean;
|
|
740
|
+
crossOriginFrames?: CrossOriginFrameInfo[];
|
|
741
|
+
elements: InteractiveElement[];
|
|
742
|
+
}>;
|
|
681
743
|
dismissCookieBanner(sessionId: string): Promise<DismissCookieBannerResult>;
|
|
682
744
|
screenshot(sessionId: string, options?: ScreenshotOptions): Promise<ScreenshotResult>;
|
|
683
745
|
getCurrentUrl(sessionId: string): Promise<string>;
|
|
@@ -719,6 +781,7 @@ declare class ControlApi {
|
|
|
719
781
|
selector?: string | undefined;
|
|
720
782
|
payload: Record<string, unknown>;
|
|
721
783
|
}[];
|
|
784
|
+
recipeRunId?: string | undefined;
|
|
722
785
|
expectedOutcome: string;
|
|
723
786
|
confidence: number;
|
|
724
787
|
successCount: number;
|
|
@@ -756,6 +819,7 @@ declare class ControlApi {
|
|
|
756
819
|
selector?: string | undefined;
|
|
757
820
|
payload: Record<string, unknown>;
|
|
758
821
|
}[];
|
|
822
|
+
recipeRunId?: string | undefined;
|
|
759
823
|
expectedOutcome: string;
|
|
760
824
|
confidence: number;
|
|
761
825
|
successCount: number;
|
|
@@ -806,6 +870,8 @@ interface ExecuteCommandInput {
|
|
|
806
870
|
commandId?: string;
|
|
807
871
|
type: CommandType;
|
|
808
872
|
payload: Record<string, unknown>;
|
|
873
|
+
/** Task this step belongs to (e.g. "log in"); memory groups steps by it. */
|
|
874
|
+
intent?: string;
|
|
809
875
|
}
|
|
810
876
|
export declare class AgenticBrowserCore {
|
|
811
877
|
readonly context: AppContext;
|
|
@@ -863,7 +929,12 @@ export declare class AgenticBrowserCore {
|
|
|
863
929
|
href?: string;
|
|
864
930
|
text?: string;
|
|
865
931
|
frame?: string;
|
|
866
|
-
}): Promise<
|
|
932
|
+
}): Promise<{
|
|
933
|
+
totalFound: number;
|
|
934
|
+
truncated: boolean;
|
|
935
|
+
crossOriginFrames?: CrossOriginFrameInfo[];
|
|
936
|
+
elements: InteractiveElement[];
|
|
937
|
+
}>;
|
|
867
938
|
dismissCookieBanner(sessionId: string): Promise<DismissCookieBannerResult>;
|
|
868
939
|
screenshot(sessionId: string, options?: ScreenshotOptions): Promise<ScreenshotResult>;
|
|
869
940
|
getCurrentUrl(sessionId: string): Promise<string>;
|
|
@@ -905,6 +976,7 @@ export declare class AgenticBrowserCore {
|
|
|
905
976
|
selector?: string | undefined;
|
|
906
977
|
payload: Record<string, unknown>;
|
|
907
978
|
}[];
|
|
979
|
+
recipeRunId?: string | undefined;
|
|
908
980
|
expectedOutcome: string;
|
|
909
981
|
confidence: number;
|
|
910
982
|
successCount: number;
|
|
@@ -942,6 +1014,7 @@ export declare class AgenticBrowserCore {
|
|
|
942
1014
|
selector?: string | undefined;
|
|
943
1015
|
payload: Record<string, unknown>;
|
|
944
1016
|
}[];
|
|
1017
|
+
recipeRunId?: string | undefined;
|
|
945
1018
|
expectedOutcome: string;
|
|
946
1019
|
confidence: number;
|
|
947
1020
|
successCount: number;
|
package/dist/index.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
export { t as AgenticBrowserCore, n as createAgenticBrowserCore, i as createMockAgenticBrowserCore } from "./runtime-
|
|
2
|
+
export { t as AgenticBrowserCore, n as createAgenticBrowserCore, i as createMockAgenticBrowserCore } from "./runtime-CgX6XW9u.mjs";
|
package/dist/mcp/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { n as createAgenticBrowserCore } from "../runtime-
|
|
2
|
+
import { a as StaleRefError, n as createAgenticBrowserCore } from "../runtime-CgX6XW9u.mjs";
|
|
3
3
|
import { readFileSync } from "node:fs";
|
|
4
4
|
import { dirname, join } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
@@ -204,46 +204,6 @@ function compactMemoryResults(results) {
|
|
|
204
204
|
return compact;
|
|
205
205
|
}) };
|
|
206
206
|
}
|
|
207
|
-
// src/mcp/element-refs.ts
|
|
208
|
-
var ElementRefRegistry = class {
|
|
209
|
-
bySession = /* @__PURE__ */ new Map();
|
|
210
|
-
/**
|
|
211
|
-
* Assign fresh refs to a freshly-discovered element list, replacing any prior
|
|
212
|
-
* refs for the session. Elements without a selector (nothing to act on) are
|
|
213
|
-
* still returned but get no ref.
|
|
214
|
-
*/
|
|
215
|
-
assign(sessionId, elements) {
|
|
216
|
-
const map = /* @__PURE__ */ new Map();
|
|
217
|
-
let n = 0;
|
|
218
|
-
const annotated = elements.map((el) => {
|
|
219
|
-
if (!el.selector) return {
|
|
220
|
-
...el,
|
|
221
|
-
ref: ""
|
|
222
|
-
};
|
|
223
|
-
const ref = `e${++n}`;
|
|
224
|
-
map.set(ref, {
|
|
225
|
-
selector: el.selector,
|
|
226
|
-
fallbackSelectors: el.fallbackSelectors,
|
|
227
|
-
text: el.text,
|
|
228
|
-
role: el.role
|
|
229
|
-
});
|
|
230
|
-
return {
|
|
231
|
-
...el,
|
|
232
|
-
ref
|
|
233
|
-
};
|
|
234
|
-
});
|
|
235
|
-
this.bySession.set(sessionId, map);
|
|
236
|
-
return { elements: annotated };
|
|
237
|
-
}
|
|
238
|
-
/** Resolve a ref to its selector target, or undefined if unknown/stale. */
|
|
239
|
-
resolve(sessionId, ref) {
|
|
240
|
-
return this.bySession.get(sessionId)?.get(ref);
|
|
241
|
-
}
|
|
242
|
-
/** Drop a session's refs (on stop/restart) so stale handles can't resolve. */
|
|
243
|
-
clear(sessionId) {
|
|
244
|
-
this.bySession.delete(sessionId);
|
|
245
|
-
}
|
|
246
|
-
};
|
|
247
207
|
// src/mcp/session-resolver.ts
|
|
248
208
|
/**
|
|
249
209
|
* Tracks the active session and auto-starts one on demand. Concurrent callers
|
|
@@ -318,7 +278,6 @@ function getCore() {
|
|
|
318
278
|
function genId(prefix) {
|
|
319
279
|
return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
320
280
|
}
|
|
321
|
-
const refs = new ElementRefRegistry();
|
|
322
281
|
/**
|
|
323
282
|
* Discover interactive elements, assign fresh refs for the session, and return
|
|
324
283
|
* the compact agent-facing view (refs + role/text, no raw selectors). This is
|
|
@@ -334,8 +293,7 @@ async function snapshotElements(sessionId, opts = {}) {
|
|
|
334
293
|
selector: opts.selector,
|
|
335
294
|
roles: opts.roles
|
|
336
295
|
});
|
|
337
|
-
|
|
338
|
-
return compactInteractiveElementsResult(elements, {
|
|
296
|
+
return compactInteractiveElementsResult(result.elements, {
|
|
339
297
|
totalFound: result.totalFound,
|
|
340
298
|
truncated: result.truncated
|
|
341
299
|
}, visibleOnly);
|
|
@@ -347,6 +305,7 @@ async function snapshotElements(sessionId, opts = {}) {
|
|
|
347
305
|
async function resolveSession(sessionId) {
|
|
348
306
|
return sessionResolver.resolve(sessionId);
|
|
349
307
|
}
|
|
308
|
+
const intentParam = z.string().optional().describe("Task this step belongs to, in your own words (e.g. \"log in\", \"search products\"). Steps sharing an intent are remembered together and returned as a recipe by browser_search_memory");
|
|
350
309
|
const server = new McpServer({
|
|
351
310
|
name: "agentic-browser",
|
|
352
311
|
version: readPackageVersion()
|
|
@@ -362,7 +321,7 @@ server.tool("browser_start_session", "Start a new Chrome browser session (or ret
|
|
|
362
321
|
});
|
|
363
322
|
server.tool("browser_navigate", "Navigate the browser to a URL. A session is auto-started if needed.", {
|
|
364
323
|
url: z.string().describe("The URL to navigate to"),
|
|
365
|
-
intent:
|
|
324
|
+
intent: intentParam,
|
|
366
325
|
sessionId: z.string().optional().describe("Session ID (auto-resolved if omitted)")
|
|
367
326
|
}, async ({ url, intent, sessionId }) => {
|
|
368
327
|
const sid = await resolveSession(sessionId);
|
|
@@ -370,10 +329,8 @@ server.tool("browser_navigate", "Navigate the browser to a URL. A session is aut
|
|
|
370
329
|
sessionId: sid,
|
|
371
330
|
commandId: genId("nav"),
|
|
372
331
|
type: "navigate",
|
|
373
|
-
payload:
|
|
374
|
-
|
|
375
|
-
intent
|
|
376
|
-
} : { url }
|
|
332
|
+
payload: { url },
|
|
333
|
+
intent
|
|
377
334
|
});
|
|
378
335
|
let cookieBanner;
|
|
379
336
|
if (result.resultStatus === "success") try {
|
|
@@ -420,40 +377,40 @@ server.tool("browser_interact", "Interact with a page element or perform browser
|
|
|
420
377
|
scrollX: z.number().optional().describe("Horizontal scroll delta in pixels (for \"scroll\" action)"),
|
|
421
378
|
scrollY: z.number().optional().describe("Vertical scroll delta in pixels (for \"scroll\" action, positive = down)"),
|
|
422
379
|
timeoutMs: z.number().optional().describe("Timeout in milliseconds (for \"waitFor\" action, default 4000)"),
|
|
423
|
-
intent:
|
|
380
|
+
intent: intentParam,
|
|
424
381
|
sessionId: z.string().optional().describe("Session ID (auto-resolved if omitted)")
|
|
425
382
|
}, async ({ action, ref, selector, fallbackSelectors, text, key, value, scrollX, scrollY, timeoutMs, intent, sessionId }) => {
|
|
426
383
|
const sid = await resolveSession(sessionId);
|
|
427
384
|
const payload = { action };
|
|
428
|
-
|
|
429
|
-
if (
|
|
430
|
-
|
|
431
|
-
|
|
385
|
+
const target = selector ?? ref;
|
|
386
|
+
if (target) payload.selector = target;
|
|
387
|
+
if (fallbackSelectors) payload.fallbackSelectors = fallbackSelectors;
|
|
388
|
+
if (text) payload.text = text;
|
|
389
|
+
if (key) payload.key = key;
|
|
390
|
+
if (value) payload.value = value;
|
|
391
|
+
if (scrollX !== void 0) payload.scrollX = scrollX;
|
|
392
|
+
if (scrollY !== void 0) payload.scrollY = scrollY;
|
|
393
|
+
if (timeoutMs) payload.timeoutMs = timeoutMs;
|
|
394
|
+
let result;
|
|
395
|
+
try {
|
|
396
|
+
result = await getCore().runCommand({
|
|
397
|
+
sessionId: sid,
|
|
398
|
+
commandId: genId("int"),
|
|
399
|
+
type: "interact",
|
|
400
|
+
payload,
|
|
401
|
+
intent
|
|
402
|
+
});
|
|
403
|
+
} catch (error) {
|
|
404
|
+
if (!(error instanceof StaleRefError)) throw error;
|
|
405
|
+
return { content: [{
|
|
432
406
|
type: "text",
|
|
433
407
|
text: JSON.stringify({
|
|
434
408
|
resultStatus: "failed",
|
|
435
|
-
resultMessage:
|
|
409
|
+
resultMessage: error.message,
|
|
436
410
|
errorCode: "SELECTOR_NOT_FOUND"
|
|
437
411
|
})
|
|
438
412
|
}] };
|
|
439
|
-
payload.selector = resolved.selector;
|
|
440
|
-
if (resolved.fallbackSelectors) payload.fallbackSelectors = resolved.fallbackSelectors;
|
|
441
|
-
} else {
|
|
442
|
-
if (selector) payload.selector = selector;
|
|
443
|
-
if (fallbackSelectors) payload.fallbackSelectors = fallbackSelectors;
|
|
444
413
|
}
|
|
445
|
-
if (text) payload.text = text;
|
|
446
|
-
if (key) payload.key = key;
|
|
447
|
-
if (value) payload.value = value;
|
|
448
|
-
if (scrollX !== void 0) payload.scrollX = scrollX;
|
|
449
|
-
if (scrollY !== void 0) payload.scrollY = scrollY;
|
|
450
|
-
if (timeoutMs) payload.timeoutMs = timeoutMs;
|
|
451
|
-
const result = await getCore().runCommand({
|
|
452
|
-
sessionId: sid,
|
|
453
|
-
commandId: genId("int"),
|
|
454
|
-
type: "interact",
|
|
455
|
-
payload
|
|
456
|
-
});
|
|
457
414
|
let currentUrl;
|
|
458
415
|
try {
|
|
459
416
|
currentUrl = await getCore().getCurrentUrl(sid);
|
|
@@ -580,7 +537,6 @@ server.tool("browser_stop_session", "Stop the browser session and terminate Chro
|
|
|
580
537
|
})
|
|
581
538
|
}] };
|
|
582
539
|
await getCore().stopSession(sid);
|
|
583
|
-
refs.clear(sid);
|
|
584
540
|
if (sessionResolver.current() === sid) sessionResolver.clear();
|
|
585
541
|
return { content: [{
|
|
586
542
|
type: "text",
|
|
@@ -454,6 +454,22 @@ function resolveDefaultProfileDir() {
|
|
|
454
454
|
return path.join(home, ".config", "google-chrome");
|
|
455
455
|
}
|
|
456
456
|
const CONTENT_TRUNCATION_HINT = "Use a CSS selector to scope the content, or use mode=\"summary\" for a lower-token overview.";
|
|
457
|
+
/**
|
|
458
|
+
* In-page visibility check shared by element discovery and cookie dismissal.
|
|
459
|
+
* Uses the element's own window so it also works inside same-origin iframes.
|
|
460
|
+
*/
|
|
461
|
+
const IN_PAGE_IS_VISIBLE = String.raw`
|
|
462
|
+
function isVisible(el) {
|
|
463
|
+
if (!el) return false;
|
|
464
|
+
const view = (el.ownerDocument && el.ownerDocument.defaultView) || window;
|
|
465
|
+
const style = view.getComputedStyle(el);
|
|
466
|
+
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
|
|
467
|
+
return false;
|
|
468
|
+
}
|
|
469
|
+
const rect = el.getBoundingClientRect();
|
|
470
|
+
return rect.width > 0 && rect.height > 0;
|
|
471
|
+
}
|
|
472
|
+
`;
|
|
457
473
|
const LOCATOR_RUNTIME_HELPERS = String.raw`
|
|
458
474
|
const LOCATOR_SEPARATOR = ' >>> ';
|
|
459
475
|
|
|
@@ -2235,12 +2251,23 @@ var ChromeCdpBrowserController = class {
|
|
|
2235
2251
|
current = parent;
|
|
2236
2252
|
}
|
|
2237
2253
|
// Prefer the shortest unique tail of the path: long nth-of-type chains
|
|
2238
|
-
// cost tokens and break on the slightest layout change.
|
|
2239
|
-
|
|
2254
|
+
// cost tokens and break on the slightest layout change. A longer tail
|
|
2255
|
+
// matches a subset of a shorter one, so uniqueness is monotonic in the
|
|
2256
|
+
// tail length and a binary search finds the shortest unique tail.
|
|
2257
|
+
let best = parts.join(' > ');
|
|
2258
|
+
let lo = 1;
|
|
2259
|
+
let hi = parts.length - 1;
|
|
2260
|
+
while (lo <= hi) {
|
|
2261
|
+
const size = (lo + hi) >> 1;
|
|
2240
2262
|
const candidate = tryUniqueSelector(root, parts.slice(-size).join(' > '));
|
|
2241
|
-
if (candidate)
|
|
2263
|
+
if (candidate) {
|
|
2264
|
+
best = candidate;
|
|
2265
|
+
hi = size - 1;
|
|
2266
|
+
} else {
|
|
2267
|
+
lo = size + 1;
|
|
2268
|
+
}
|
|
2242
2269
|
}
|
|
2243
|
-
return
|
|
2270
|
+
return best;
|
|
2244
2271
|
}
|
|
2245
2272
|
|
|
2246
2273
|
function buildFallbackSelectors(root, path, el, primarySelector) {
|
|
@@ -2334,18 +2361,7 @@ var ChromeCdpBrowserController = class {
|
|
|
2334
2361
|
}
|
|
2335
2362
|
}
|
|
2336
2363
|
|
|
2337
|
-
|
|
2338
|
-
return el.ownerDocument && el.ownerDocument.defaultView ? el.ownerDocument.defaultView : window;
|
|
2339
|
-
}
|
|
2340
|
-
|
|
2341
|
-
function isVisible(el) {
|
|
2342
|
-
const style = viewForElement(el).getComputedStyle(el);
|
|
2343
|
-
if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
|
|
2344
|
-
return false;
|
|
2345
|
-
}
|
|
2346
|
-
const rect = el.getBoundingClientRect();
|
|
2347
|
-
return rect.width > 0 && rect.height > 0;
|
|
2348
|
-
}
|
|
2364
|
+
${IN_PAGE_IS_VISIBLE}
|
|
2349
2365
|
|
|
2350
2366
|
function getText(el) {
|
|
2351
2367
|
const ariaLabel = el.getAttribute('aria-label');
|
|
@@ -2437,17 +2453,33 @@ var ChromeCdpBrowserController = class {
|
|
|
2437
2453
|
}
|
|
2438
2454
|
|
|
2439
2455
|
totalFound += 1;
|
|
2440
|
-
candidates.push({ root, path, el, role, visible, rank: rankElement(el, visible)
|
|
2456
|
+
candidates.push({ root, path, el, role, visible, rank: rankElement(el, visible) });
|
|
2441
2457
|
}
|
|
2442
2458
|
|
|
2443
2459
|
// Lower rank sorts first: main content, then the rest of the page, then
|
|
2444
2460
|
// site chrome (header, nav, footer), then hidden elements. Ties keep
|
|
2445
2461
|
// document order.
|
|
2462
|
+
// Region flags per ancestor, memoized so siblings share the walk up the
|
|
2463
|
+
// tree instead of each element running three closest() scans.
|
|
2464
|
+
const FOOTER = 1, MAIN = 2, CHROME = 4;
|
|
2465
|
+
const regionMemo = new Map();
|
|
2466
|
+
function regionFlags(el) {
|
|
2467
|
+
if (!el) return 0;
|
|
2468
|
+
let flags = regionMemo.get(el);
|
|
2469
|
+
if (flags !== undefined) return flags;
|
|
2470
|
+
flags = regionFlags(el.parentElement);
|
|
2471
|
+
if (el.matches('footer, [role="contentinfo"]')) flags |= FOOTER;
|
|
2472
|
+
if (el.matches('main, [role="main"], article')) flags |= MAIN;
|
|
2473
|
+
if (el.matches('header, nav, [role="banner"], [role="navigation"]')) flags |= CHROME;
|
|
2474
|
+
regionMemo.set(el, flags);
|
|
2475
|
+
return flags;
|
|
2476
|
+
}
|
|
2446
2477
|
function rankElement(el, visible) {
|
|
2447
2478
|
if (!visible) return 3;
|
|
2448
|
-
|
|
2449
|
-
if (
|
|
2450
|
-
if (
|
|
2479
|
+
const flags = regionFlags(el);
|
|
2480
|
+
if (flags & FOOTER) return 2;
|
|
2481
|
+
if (flags & MAIN) return 0;
|
|
2482
|
+
if (flags & CHROME) return 2;
|
|
2451
2483
|
return 1;
|
|
2452
2484
|
}
|
|
2453
2485
|
|
|
@@ -2521,7 +2553,8 @@ var ChromeCdpBrowserController = class {
|
|
|
2521
2553
|
}
|
|
2522
2554
|
|
|
2523
2555
|
walkRoot(document, []);
|
|
2524
|
-
|
|
2556
|
+
// Array sort is stable, so ties keep document order.
|
|
2557
|
+
candidates.sort((a, b) => a.rank - b.rank);
|
|
2525
2558
|
const results = candidates.slice(0, limit).map(buildEntry);
|
|
2526
2559
|
const output = { elements: results, totalFound, truncated: totalFound > results.length };
|
|
2527
2560
|
if (crossOriginFrames.length) output.crossOriginFrames = crossOriginFrames;
|
|
@@ -2607,11 +2640,7 @@ var ChromeCdpBrowserController = class {
|
|
|
2607
2640
|
}
|
|
2608
2641
|
async dismissViaDom(targetWsUrl) {
|
|
2609
2642
|
const expression = `(() => {
|
|
2610
|
-
|
|
2611
|
-
if (!el) return false;
|
|
2612
|
-
const style = getComputedStyle(el);
|
|
2613
|
-
return style.display !== 'none' && style.visibility !== 'hidden' && style.opacity !== '0' && el.offsetWidth > 0;
|
|
2614
|
-
}
|
|
2643
|
+
${IN_PAGE_IS_VISIBLE}
|
|
2615
2644
|
|
|
2616
2645
|
// Comprehensive CMP selectors (ordered by specificity/popularity)
|
|
2617
2646
|
const selectors = [
|
|
@@ -2914,12 +2943,17 @@ var ChromeCdpBrowserController = class {
|
|
|
2914
2943
|
'iframe[src*="consentmanager"]',
|
|
2915
2944
|
];
|
|
2916
2945
|
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2946
|
+
// Consent prompts overlay the page: the frame or a close ancestor is
|
|
2947
|
+
// fixed/sticky or stacked high. In-flow frames (ads, embeds) are skipped.
|
|
2948
|
+
function isOverlay(iframe) {
|
|
2949
|
+
let el = iframe;
|
|
2950
|
+
for (let depth = 0; el && el !== document.body && depth < 10; depth++) {
|
|
2951
|
+
const style = getComputedStyle(el);
|
|
2952
|
+
if (style.position === 'fixed' || style.position === 'sticky') return true;
|
|
2953
|
+
if ((parseInt(style.zIndex, 10) || 0) >= 1000) return true;
|
|
2954
|
+
el = el.parentElement;
|
|
2955
|
+
}
|
|
2956
|
+
return false;
|
|
2923
2957
|
}
|
|
2924
2958
|
function isShown(iframe) {
|
|
2925
2959
|
const rect = iframe.getBoundingClientRect();
|
|
@@ -2942,7 +2976,7 @@ var ChromeCdpBrowserController = class {
|
|
|
2942
2976
|
}
|
|
2943
2977
|
function findFrame(sel) {
|
|
2944
2978
|
for (const iframe of document.querySelectorAll(sel)) {
|
|
2945
|
-
if (
|
|
2979
|
+
if (!isShown(iframe) || !srcKeywordMatches(iframe, sel) || !isOverlay(iframe)) continue;
|
|
2946
2980
|
return iframe;
|
|
2947
2981
|
}
|
|
2948
2982
|
return null;
|
|
@@ -3413,6 +3447,40 @@ z.object({
|
|
|
3413
3447
|
createdAt: z.string().datetime(),
|
|
3414
3448
|
severity: EventSeveritySchema
|
|
3415
3449
|
});
|
|
3450
|
+
// src/session/element-refs.ts
|
|
3451
|
+
/** Matches "e3" or "@e3"; group 1 is the bare ref. */
|
|
3452
|
+
const REF_PATTERN = /^@?(e\d+)$/;
|
|
3453
|
+
/**
|
|
3454
|
+
* Number elements e1, e2, … Elements without a selector (nothing to act on)
|
|
3455
|
+
* are still returned but get no ref.
|
|
3456
|
+
*/
|
|
3457
|
+
function assignRefs(elements, frame) {
|
|
3458
|
+
const byRef = {};
|
|
3459
|
+
let n = 0;
|
|
3460
|
+
return {
|
|
3461
|
+
elements: elements.map((el) => {
|
|
3462
|
+
if (!el.selector) return el;
|
|
3463
|
+
const ref = `e${++n}`;
|
|
3464
|
+
byRef[ref] = {
|
|
3465
|
+
selector: el.selector,
|
|
3466
|
+
fallbackSelectors: el.fallbackSelectors,
|
|
3467
|
+
frame
|
|
3468
|
+
};
|
|
3469
|
+
return {
|
|
3470
|
+
...el,
|
|
3471
|
+
ref
|
|
3472
|
+
};
|
|
3473
|
+
}),
|
|
3474
|
+
byRef
|
|
3475
|
+
};
|
|
3476
|
+
}
|
|
3477
|
+
/** A ref that was never assigned, or was taken on a different page. */
|
|
3478
|
+
var StaleRefError = class extends Error {
|
|
3479
|
+
constructor(ref) {
|
|
3480
|
+
super(`Unknown or stale ref "${ref}". List the page's elements again (agent elements / browser_get_elements) to get fresh refs.`);
|
|
3481
|
+
this.name = "StaleRefError";
|
|
3482
|
+
}
|
|
3483
|
+
};
|
|
3416
3484
|
// src/session/session-store.ts
|
|
3417
3485
|
var SessionStore = class {
|
|
3418
3486
|
filePath;
|
|
@@ -3457,12 +3525,16 @@ var SessionStore = class {
|
|
|
3457
3525
|
}
|
|
3458
3526
|
}
|
|
3459
3527
|
setLastUrl(sessionId, lastUrl) {
|
|
3528
|
+
this.update(sessionId, { lastUrl });
|
|
3529
|
+
}
|
|
3530
|
+
/** Merge fields into a stored record; `undefined` values clear the field. */
|
|
3531
|
+
update(sessionId, patch) {
|
|
3460
3532
|
const state = this.read();
|
|
3461
3533
|
const existing = state.sessions[sessionId];
|
|
3462
3534
|
if (!existing) throw new Error("Session not found in store");
|
|
3463
3535
|
state.sessions[sessionId] = {
|
|
3464
3536
|
...existing,
|
|
3465
|
-
|
|
3537
|
+
...patch
|
|
3466
3538
|
};
|
|
3467
3539
|
this.write(state);
|
|
3468
3540
|
}
|
|
@@ -3625,8 +3697,13 @@ var SessionManager = class {
|
|
|
3625
3697
|
}
|
|
3626
3698
|
throw new Error("Session is terminated. Start a new session.");
|
|
3627
3699
|
}
|
|
3628
|
-
async executeCommand(sessionId,
|
|
3700
|
+
async executeCommand(sessionId, rawInput) {
|
|
3629
3701
|
const record = await this.ensureSession(sessionId);
|
|
3702
|
+
const input = rawInput.type === "interact" ? {
|
|
3703
|
+
...rawInput,
|
|
3704
|
+
payload: await this.resolveRef(record, rawInput.payload)
|
|
3705
|
+
} : rawInput;
|
|
3706
|
+
const runId = this.trackIntentRun(sessionId, record, input);
|
|
3630
3707
|
const command = {
|
|
3631
3708
|
commandId: input.commandId,
|
|
3632
3709
|
sessionId,
|
|
@@ -3655,6 +3732,8 @@ var SessionManager = class {
|
|
|
3655
3732
|
const parsedPayload = InteractPayloadSchema.safeParse(input.payload);
|
|
3656
3733
|
if (!parsedPayload.success) throw new Error(`Invalid interact payload: ${parsedPayload.error.message}`);
|
|
3657
3734
|
resultMessage = `Interaction result: ${await this.browser.interact(record.targetWsUrl, parsedPayload.data)}`;
|
|
3735
|
+
const currentUrl = await this.browser.getCurrentUrl(record.targetWsUrl).catch(() => void 0);
|
|
3736
|
+
if (currentUrl) this.store.setLastUrl(sessionId, currentUrl);
|
|
3658
3737
|
} else if (input.type === "restart") {
|
|
3659
3738
|
await this.restartSession(sessionId);
|
|
3660
3739
|
resultMessage = "Session restarted";
|
|
@@ -3681,7 +3760,8 @@ var SessionManager = class {
|
|
|
3681
3760
|
expectedOutcome: resultMessage,
|
|
3682
3761
|
step: memoryContext.step,
|
|
3683
3762
|
selector: memoryContext.selector,
|
|
3684
|
-
url: memoryContext.url
|
|
3763
|
+
url: memoryContext.url,
|
|
3764
|
+
runId
|
|
3685
3765
|
});
|
|
3686
3766
|
else this.ctx.memoryService.recordFailure({
|
|
3687
3767
|
commandId: input.commandId,
|
|
@@ -3711,9 +3791,20 @@ var SessionManager = class {
|
|
|
3711
3791
|
}
|
|
3712
3792
|
return result;
|
|
3713
3793
|
}
|
|
3794
|
+
/** Lists interactive elements and assigns fresh refs (e1, e2, …) for this page. */
|
|
3714
3795
|
async getInteractiveElements(sessionId, options) {
|
|
3715
3796
|
const record = await this.ensureSession(sessionId);
|
|
3716
|
-
|
|
3797
|
+
const result = await this.browser.getInteractiveElements(record.targetWsUrl, options);
|
|
3798
|
+
const { elements, byRef } = assignRefs(result.elements, options.frame);
|
|
3799
|
+
const url = await this.browser.getCurrentUrl(record.targetWsUrl).catch(() => void 0);
|
|
3800
|
+
this.store.update(sessionId, { refs: {
|
|
3801
|
+
url,
|
|
3802
|
+
byRef
|
|
3803
|
+
} });
|
|
3804
|
+
return {
|
|
3805
|
+
...result,
|
|
3806
|
+
elements
|
|
3807
|
+
};
|
|
3717
3808
|
}
|
|
3718
3809
|
async screenshot(sessionId, options) {
|
|
3719
3810
|
const record = await this.ensureSession(sessionId);
|
|
@@ -3848,7 +3939,8 @@ var SessionManager = class {
|
|
|
3848
3939
|
session: restarted,
|
|
3849
3940
|
cdpUrl: relaunched.cdpUrl,
|
|
3850
3941
|
targetWsUrl: relaunched.targetWsUrl,
|
|
3851
|
-
pid: relaunched.pid
|
|
3942
|
+
pid: relaunched.pid,
|
|
3943
|
+
intentRun: record.intentRun
|
|
3852
3944
|
});
|
|
3853
3945
|
this.recordEvent(sessionId, "lifecycle", "info", "Session restarted");
|
|
3854
3946
|
return restarted;
|
|
@@ -3992,7 +4084,7 @@ var SessionManager = class {
|
|
|
3992
4084
|
const parsed = this.parseUrl(inputUrl);
|
|
3993
4085
|
const action = typeof input.payload.action === "string" ? input.payload.action : void 0;
|
|
3994
4086
|
const defaultIntent = input.type === "navigate" ? `navigate:${parsed.domain}` : input.type === "interact" ? `interact:${action ?? "action"}:${parsed.domain}` : `${input.type}:${parsed.domain}`;
|
|
3995
|
-
const taskIntent =
|
|
4087
|
+
const taskIntent = explicitIntent(input) ?? defaultIntent;
|
|
3996
4088
|
const expectedOutcome = typeof input.payload.expectedOutcome === "string" && input.payload.expectedOutcome.trim() ? input.payload.expectedOutcome : `${input.type} command succeeds`;
|
|
3997
4089
|
return {
|
|
3998
4090
|
taskIntent,
|
|
@@ -4009,6 +4101,36 @@ var SessionManager = class {
|
|
|
4009
4101
|
url: inputUrl
|
|
4010
4102
|
};
|
|
4011
4103
|
}
|
|
4104
|
+
async resolveRef(record, payload) {
|
|
4105
|
+
const selector = typeof payload.selector === "string" ? payload.selector : void 0;
|
|
4106
|
+
const ref = selector ? REF_PATTERN.exec(selector)?.[1] : void 0;
|
|
4107
|
+
if (!selector || !ref) return payload;
|
|
4108
|
+
const { refs } = record;
|
|
4109
|
+
const stored = refs?.byRef[ref];
|
|
4110
|
+
const onSamePage = !refs?.url || await this.browser.getCurrentUrl(record.targetWsUrl).catch(() => void 0) === refs.url;
|
|
4111
|
+
if (!stored || !onSamePage) throw new StaleRefError(selector);
|
|
4112
|
+
const extra = Array.isArray(payload.fallbackSelectors) ? payload.fallbackSelectors : [];
|
|
4113
|
+
return {
|
|
4114
|
+
...payload,
|
|
4115
|
+
selector: stored.selector,
|
|
4116
|
+
fallbackSelectors: [...stored.fallbackSelectors ?? [], ...extra],
|
|
4117
|
+
frame: payload.frame ?? stored.frame
|
|
4118
|
+
};
|
|
4119
|
+
}
|
|
4120
|
+
/**
|
|
4121
|
+
* Consecutive commands with the same explicit intent form one run; any other
|
|
4122
|
+
* command ends it. Commands without an intent are each their own run.
|
|
4123
|
+
*/
|
|
4124
|
+
trackIntentRun(sessionId, record, input) {
|
|
4125
|
+
const intent = explicitIntent(input);
|
|
4126
|
+
if (intent && record.intentRun?.intent === intent) return record.intentRun.runId;
|
|
4127
|
+
const intentRun = intent ? {
|
|
4128
|
+
intent,
|
|
4129
|
+
runId: input.commandId
|
|
4130
|
+
} : void 0;
|
|
4131
|
+
if (intentRun || record.intentRun) this.store.update(sessionId, { intentRun });
|
|
4132
|
+
return input.commandId;
|
|
4133
|
+
}
|
|
4012
4134
|
parseUrl(rawUrl) {
|
|
4013
4135
|
if (!rawUrl) return {
|
|
4014
4136
|
domain: "unknown",
|
|
@@ -4019,7 +4141,7 @@ var SessionManager = class {
|
|
|
4019
4141
|
const firstSegment = parsed.pathname.split("/").filter(Boolean)[0];
|
|
4020
4142
|
const pathPattern = firstSegment ? `/${firstSegment}/*` : "/";
|
|
4021
4143
|
return {
|
|
4022
|
-
domain: parsed.hostname.toLowerCase(),
|
|
4144
|
+
domain: parsed.hostname.toLowerCase() || parsed.protocol.replace(/:$/, ""),
|
|
4023
4145
|
pathPattern
|
|
4024
4146
|
};
|
|
4025
4147
|
} catch {
|
|
@@ -4030,6 +4152,11 @@ var SessionManager = class {
|
|
|
4030
4152
|
}
|
|
4031
4153
|
}
|
|
4032
4154
|
};
|
|
4155
|
+
/** The caller's task intent, if any. payload.intent is the older spelling, still honored. */
|
|
4156
|
+
function explicitIntent(input) {
|
|
4157
|
+
const raw = input.intent ?? input.payload.intent;
|
|
4158
|
+
return typeof raw === "string" && raw.trim() ? raw.trim() : void 0;
|
|
4159
|
+
}
|
|
4033
4160
|
// src/transport/control-api.ts
|
|
4034
4161
|
var ControlApi = class {
|
|
4035
4162
|
sessions;
|
|
@@ -4303,11 +4430,10 @@ function intentTokens(value, domain) {
|
|
|
4303
4430
|
return tokens;
|
|
4304
4431
|
}
|
|
4305
4432
|
/** 0 = unrelated; 1 = identical intent. Word overlap covers rephrased queries. */
|
|
4306
|
-
function intentSimilarity(insight, normalizedIntent) {
|
|
4433
|
+
function intentSimilarity(insight, normalizedIntent, query) {
|
|
4307
4434
|
const insightIntent = normalize(insight.taskIntent);
|
|
4308
4435
|
if (insightIntent === normalizedIntent) return 1;
|
|
4309
4436
|
if (insightIntent.includes(normalizedIntent) || normalizedIntent.includes(insightIntent)) return .65;
|
|
4310
|
-
const query = intentTokens(normalizedIntent);
|
|
4311
4437
|
if (!query.size) return 0;
|
|
4312
4438
|
const stored = intentTokens(insightIntent, insight.siteDomain);
|
|
4313
4439
|
let shared = 0;
|
|
@@ -4357,12 +4483,19 @@ function selectorSignal(insight) {
|
|
|
4357
4483
|
const evidenceStrength = Math.min(selectorEvidence / 5, 1);
|
|
4358
4484
|
return .7 * recipeCoverage + .3 * evidenceStrength;
|
|
4359
4485
|
}
|
|
4360
|
-
function scoreInsight(insight, normalizedIntent,
|
|
4361
|
-
const intentSignal = intentSimilarity(insight, normalizedIntent);
|
|
4486
|
+
function scoreInsight(insight, normalizedIntent, queryTokens, normalizedDomain) {
|
|
4487
|
+
const intentSignal = intentSimilarity(insight, normalizedIntent, queryTokens);
|
|
4362
4488
|
const domainMatch = normalizedDomain && normalize(insight.siteDomain) === normalizedDomain ? 1 : normalizedDomain ? 0 : .6;
|
|
4363
4489
|
const reliability = .6 * confidenceFromCounts(insight.successCount, insight.failureCount) + .4 * freshnessWeight(insight.freshness);
|
|
4364
4490
|
const selectorQuality = selectorSignal(insight);
|
|
4365
|
-
|
|
4491
|
+
return {
|
|
4492
|
+
insight,
|
|
4493
|
+
score: .5 * intentSignal + .2 * domainMatch + .15 * reliability + .15 * selectorQuality,
|
|
4494
|
+
intentSignal
|
|
4495
|
+
};
|
|
4496
|
+
}
|
|
4497
|
+
/** Builds the full result (recipe, hints, profile) — only for insights that are returned. */
|
|
4498
|
+
function toSearchResult({ insight, score }, profileLookup) {
|
|
4366
4499
|
const result = {
|
|
4367
4500
|
insightId: insight.insightId,
|
|
4368
4501
|
taskIntent: insight.taskIntent,
|
|
@@ -4382,10 +4515,7 @@ function scoreInsight(insight, normalizedIntent, normalizedDomain, profileLookup
|
|
|
4382
4515
|
cookieBanner: profile.cookieBanner,
|
|
4383
4516
|
visitCount: profile.visitCount
|
|
4384
4517
|
};
|
|
4385
|
-
return
|
|
4386
|
-
result,
|
|
4387
|
-
intentSignal
|
|
4388
|
-
};
|
|
4518
|
+
return result;
|
|
4389
4519
|
}
|
|
4390
4520
|
var MemoryIndex = class {
|
|
4391
4521
|
/** Domain → insights index, rebuilt lazily when insight list changes. */
|
|
@@ -4408,6 +4538,7 @@ var MemoryIndex = class {
|
|
|
4408
4538
|
}
|
|
4409
4539
|
search(insights, input, profileLookup) {
|
|
4410
4540
|
const normalizedIntent = normalize(input.taskIntent);
|
|
4541
|
+
const queryTokens = intentTokens(normalizedIntent);
|
|
4411
4542
|
const normalizedDomain = input.siteDomain ? normalize(input.siteDomain) : void 0;
|
|
4412
4543
|
const limit = input.limit ?? 10;
|
|
4413
4544
|
this.ensureIndex(insights);
|
|
@@ -4416,16 +4547,16 @@ var MemoryIndex = class {
|
|
|
4416
4547
|
if (limit === 1 && normalizedDomain) {
|
|
4417
4548
|
let best;
|
|
4418
4549
|
for (const insight of candidates) {
|
|
4419
|
-
const scored = scoreInsight(insight, normalizedIntent,
|
|
4550
|
+
const scored = scoreInsight(insight, normalizedIntent, queryTokens, normalizedDomain);
|
|
4420
4551
|
if (!relevant(scored)) continue;
|
|
4421
|
-
if (!best || scored.
|
|
4552
|
+
if (!best || scored.score > best.score) {
|
|
4422
4553
|
best = scored;
|
|
4423
|
-
if (best.
|
|
4554
|
+
if (best.score >= 1) break;
|
|
4424
4555
|
}
|
|
4425
4556
|
}
|
|
4426
|
-
return best ? [best
|
|
4557
|
+
return best ? [toSearchResult(best, profileLookup)] : [];
|
|
4427
4558
|
}
|
|
4428
|
-
return candidates.map((insight) => scoreInsight(insight, normalizedIntent,
|
|
4559
|
+
return candidates.map((insight) => scoreInsight(insight, normalizedIntent, queryTokens, normalizedDomain)).filter(relevant).toSorted((a, b) => b.score - a.score).slice(0, limit).map((scored) => toSearchResult(scored, profileLookup));
|
|
4429
4560
|
}
|
|
4430
4561
|
};
|
|
4431
4562
|
// src/memory/staleness-detector.ts
|
|
@@ -4569,6 +4700,8 @@ const TaskInsightSchema = z.object({
|
|
|
4569
4700
|
siteDomain: z.string().min(1),
|
|
4570
4701
|
sitePathPattern: z.string().min(1),
|
|
4571
4702
|
actionRecipe: z.array(TaskStepSchema),
|
|
4703
|
+
/** Run that produced actionRecipe; steps from another run start it over. */
|
|
4704
|
+
recipeRunId: z.string().optional(),
|
|
4572
4705
|
expectedOutcome: z.string().min(1),
|
|
4573
4706
|
confidence: z.number().min(0).max(1),
|
|
4574
4707
|
successCount: z.number().int().nonnegative(),
|
|
@@ -4983,6 +5116,7 @@ var MemoryService = class {
|
|
|
4983
5116
|
siteDomain: input.siteDomain,
|
|
4984
5117
|
sitePathPattern: input.sitePathPattern,
|
|
4985
5118
|
actionRecipe: [input.step],
|
|
5119
|
+
recipeRunId: input.runId ?? input.commandId,
|
|
4986
5120
|
expectedOutcome: input.expectedOutcome,
|
|
4987
5121
|
confidence: 1,
|
|
4988
5122
|
successCount: 1,
|
|
@@ -5005,7 +5139,7 @@ var MemoryService = class {
|
|
|
5005
5139
|
...matched,
|
|
5006
5140
|
useCount: matched.useCount + 1,
|
|
5007
5141
|
evidence: [...matched.evidence.slice(-49), evidence],
|
|
5008
|
-
|
|
5142
|
+
...this.mergeRecipe(matched, input.step, input.runId ?? input.commandId),
|
|
5009
5143
|
expectedOutcome: input.expectedOutcome
|
|
5010
5144
|
});
|
|
5011
5145
|
if (matched.freshness === "suspect" || matched.freshness === "stale") {
|
|
@@ -5214,9 +5348,16 @@ var MemoryService = class {
|
|
|
5214
5348
|
const testMatch = selector.match(/\[data-test="([^"]+)"\]/);
|
|
5215
5349
|
if (testMatch) return testMatch[1];
|
|
5216
5350
|
}
|
|
5217
|
-
|
|
5218
|
-
|
|
5219
|
-
|
|
5351
|
+
/** Same run: append the step. New run: its first step replaces the old recipe. */
|
|
5352
|
+
mergeRecipe(insight, step, runId) {
|
|
5353
|
+
if (insight.recipeRunId !== runId) return {
|
|
5354
|
+
actionRecipe: [step],
|
|
5355
|
+
recipeRunId: runId
|
|
5356
|
+
};
|
|
5357
|
+
return {
|
|
5358
|
+
actionRecipe: [...insight.actionRecipe, step].slice(-8),
|
|
5359
|
+
recipeRunId: runId
|
|
5360
|
+
};
|
|
5220
5361
|
}
|
|
5221
5362
|
};
|
|
5222
5363
|
// src/cli/app.ts
|
|
@@ -5266,7 +5407,8 @@ var AgenticBrowserCore = class {
|
|
|
5266
5407
|
return await this.api.executeCommand(input.sessionId, {
|
|
5267
5408
|
commandId,
|
|
5268
5409
|
type: input.type,
|
|
5269
|
-
payload: input.payload
|
|
5410
|
+
payload: input.payload,
|
|
5411
|
+
intent: input.intent
|
|
5270
5412
|
});
|
|
5271
5413
|
}
|
|
5272
5414
|
async getPageContent(input) {
|
|
@@ -5343,4 +5485,4 @@ function createMockAgenticBrowserCore(env) {
|
|
|
5343
5485
|
function createCliRuntime() {
|
|
5344
5486
|
return createAgenticBrowserCore();
|
|
5345
5487
|
}
|
|
5346
|
-
export {
|
|
5488
|
+
export { StaleRefError as a, writePrivateFileSync as c, createMockAgenticBrowserCore as i, createAgenticBrowserCore as n, isConnectionError as o, createCliRuntime as r, ensurePrivateDir as s, AgenticBrowserCore as t };
|