@oh-my-pi/pi-coding-agent 11.8.2 → 11.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +42 -0
- package/docs/tui.md +9 -9
- package/package.json +7 -7
- package/src/capability/mcp.ts +9 -0
- package/src/cli/file-processor.ts +8 -13
- package/src/cli/oclif-help.ts +1 -1
- package/src/cli.ts +14 -0
- package/src/commit/git/index.ts +16 -16
- package/src/config/file-lock.ts +1 -1
- package/src/config/keybindings.ts +11 -11
- package/src/config/model-registry.ts +31 -66
- package/src/config/settings.ts +88 -95
- package/src/config.ts +2 -2
- package/src/cursor.ts +4 -4
- package/src/debug/index.ts +28 -28
- package/src/discovery/builtin.ts +48 -0
- package/src/discovery/codex.ts +5 -13
- package/src/discovery/cursor.ts +2 -7
- package/src/discovery/mcp-json.ts +33 -0
- package/src/exa/mcp-client.ts +2 -2
- package/src/exa/websets.ts +2 -2
- package/src/export/html/index.ts +3 -3
- package/src/export/ttsr.ts +27 -27
- package/src/extensibility/custom-tools/loader.ts +9 -9
- package/src/extensibility/extensions/runner.ts +64 -64
- package/src/extensibility/hooks/runner.ts +46 -46
- package/src/extensibility/plugins/manager.ts +49 -49
- package/src/extensibility/slash-commands.ts +1 -0
- package/src/index.ts +0 -3
- package/src/internal-urls/router.ts +5 -5
- package/src/ipy/kernel.ts +61 -57
- package/src/lsp/client.ts +1 -1
- package/src/lsp/clients/biome-client.ts +2 -2
- package/src/lsp/clients/lsp-linter-client.ts +7 -7
- package/src/lsp/index.ts +9 -9
- package/src/mcp/config-writer.ts +194 -0
- package/src/mcp/config.ts +20 -6
- package/src/mcp/index.ts +4 -0
- package/src/mcp/loader.ts +6 -0
- package/src/mcp/manager.ts +139 -50
- package/src/mcp/oauth-discovery.ts +274 -0
- package/src/mcp/oauth-flow.ts +229 -0
- package/src/mcp/tool-bridge.ts +20 -20
- package/src/mcp/transports/http.ts +107 -66
- package/src/mcp/transports/stdio.ts +74 -59
- package/src/mcp/types.ts +15 -1
- package/src/modes/components/assistant-message.ts +25 -25
- package/src/modes/components/bash-execution.ts +51 -51
- package/src/modes/components/bordered-loader.ts +7 -7
- package/src/modes/components/branch-summary-message.ts +7 -7
- package/src/modes/components/compaction-summary-message.ts +7 -7
- package/src/modes/components/countdown-timer.ts +15 -15
- package/src/modes/components/custom-editor.ts +22 -22
- package/src/modes/components/custom-message.ts +21 -21
- package/src/modes/components/dynamic-border.ts +3 -3
- package/src/modes/components/extensions/extension-dashboard.ts +72 -72
- package/src/modes/components/extensions/extension-list.ts +99 -97
- package/src/modes/components/extensions/inspector-panel.ts +26 -26
- package/src/modes/components/footer.ts +36 -36
- package/src/modes/components/history-search.ts +52 -52
- package/src/modes/components/hook-editor.ts +20 -20
- package/src/modes/components/hook-input.ts +20 -20
- package/src/modes/components/hook-message.ts +22 -22
- package/src/modes/components/hook-selector.ts +52 -52
- package/src/modes/components/index.ts +0 -1
- package/src/modes/components/login-dialog.ts +57 -57
- package/src/modes/components/mcp-add-wizard.ts +1286 -0
- package/src/modes/components/model-selector.ts +173 -173
- package/src/modes/components/oauth-selector.ts +45 -45
- package/src/modes/components/plugin-settings.ts +52 -52
- package/src/modes/components/python-execution.ts +53 -53
- package/src/modes/components/queue-mode-selector.ts +7 -7
- package/src/modes/components/read-tool-group.ts +23 -23
- package/src/modes/components/session-selector.ts +40 -37
- package/src/modes/components/settings-selector.ts +80 -80
- package/src/modes/components/show-images-selector.ts +7 -7
- package/src/modes/components/skill-message.ts +27 -27
- package/src/modes/components/status-line-segment-editor.ts +81 -81
- package/src/modes/components/status-line.ts +73 -73
- package/src/modes/components/theme-selector.ts +11 -11
- package/src/modes/components/thinking-selector.ts +7 -7
- package/src/modes/components/todo-display.ts +19 -19
- package/src/modes/components/todo-reminder.ts +9 -9
- package/src/modes/components/tool-execution.ts +212 -216
- package/src/modes/components/tree-selector.ts +144 -144
- package/src/modes/components/ttsr-notification.ts +17 -17
- package/src/modes/components/user-message-selector.ts +18 -18
- package/src/modes/components/welcome.ts +10 -10
- package/src/modes/controllers/command-controller.ts +0 -7
- package/src/modes/controllers/event-controller.ts +23 -23
- package/src/modes/controllers/extension-ui-controller.ts +13 -13
- package/src/modes/controllers/input-controller.ts +12 -9
- package/src/modes/controllers/mcp-command-controller.ts +1223 -0
- package/src/modes/interactive-mode.ts +240 -241
- package/src/modes/rpc/rpc-client.ts +77 -77
- package/src/modes/rpc/rpc-mode.ts +5 -5
- package/src/modes/theme/theme.ts +113 -113
- package/src/modes/types.ts +1 -1
- package/src/patch/index.ts +45 -45
- package/src/prompts/tools/task.md +22 -2
- package/src/sdk.ts +1 -0
- package/src/session/agent-session.ts +512 -476
- package/src/session/agent-storage.ts +72 -75
- package/src/session/auth-storage.ts +186 -252
- package/src/session/history-storage.ts +36 -38
- package/src/session/session-manager.ts +300 -299
- package/src/session/session-storage.ts +65 -90
- package/src/ssh/connection-manager.ts +9 -9
- package/src/system-prompt.ts +2 -3
- package/src/task/agents.ts +1 -1
- package/src/task/executor.ts +28 -40
- package/src/task/index.ts +13 -12
- package/src/task/subprocess-tool-registry.ts +5 -5
- package/src/task/worktree.ts +8 -5
- package/src/tools/ask.ts +7 -7
- package/src/tools/bash.ts +15 -10
- package/src/tools/browser.ts +130 -127
- package/src/tools/calculator.ts +46 -46
- package/src/tools/context.ts +9 -9
- package/src/tools/exit-plan-mode.ts +5 -5
- package/src/tools/fetch.ts +5 -5
- package/src/tools/find.ts +16 -16
- package/src/tools/grep.ts +12 -24
- package/src/tools/index.ts +1 -1
- package/src/tools/notebook.ts +6 -6
- package/src/tools/output-meta.ts +10 -2
- package/src/tools/python.ts +12 -11
- package/src/tools/read.ts +17 -17
- package/src/tools/ssh.ts +9 -9
- package/src/tools/submit-result.ts +13 -13
- package/src/tools/todo-write.ts +6 -6
- package/src/tools/write.ts +10 -10
- package/src/tui/output-block.ts +6 -6
- package/src/tui/utils.ts +9 -9
- package/src/utils/event-bus.ts +13 -11
- package/src/utils/frontmatter.ts +1 -1
- package/src/utils/ignore-files.ts +1 -1
- package/src/web/search/index.ts +5 -5
- package/src/web/search/providers/anthropic.ts +7 -2
- package/examples/hooks/snake.ts +0 -342
- package/src/modes/components/armin.ts +0 -379
package/src/tools/browser.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import * as os from "node:os";
|
|
2
2
|
import * as path from "node:path";
|
|
3
3
|
import { Readability } from "@mozilla/readability";
|
|
4
4
|
import type { AgentTool, AgentToolContext, AgentToolResult, AgentToolUpdateCallback } from "@oh-my-pi/pi-agent-core";
|
|
@@ -428,44 +428,44 @@ function formatEvaluateResult(value: unknown): string {
|
|
|
428
428
|
* Puppeteer tool for headless browser automation.
|
|
429
429
|
*/
|
|
430
430
|
export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolDetails> {
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
431
|
+
readonly name = "puppeteer";
|
|
432
|
+
readonly label = "Puppeteer";
|
|
433
|
+
readonly description: string;
|
|
434
|
+
readonly parameters = browserSchema;
|
|
435
|
+
#browser: Browser | null = null;
|
|
436
|
+
#page: Page | null = null;
|
|
437
|
+
#currentHeadless: boolean | null = null;
|
|
438
|
+
#browserSession: CDPSession | null = null;
|
|
439
|
+
#userAgentOverride: UserAgentOverride | null = null;
|
|
440
|
+
#elementIdCounter = 0;
|
|
441
|
+
readonly #elementCache = new Map<number, ElementHandle>();
|
|
442
|
+
readonly #patchedClients = new WeakSet<object>();
|
|
443
443
|
|
|
444
444
|
constructor(private readonly session: ToolSession) {
|
|
445
445
|
this.description = renderPromptTemplate(browserDescription, {});
|
|
446
446
|
}
|
|
447
447
|
|
|
448
|
-
|
|
449
|
-
await this
|
|
450
|
-
if (this
|
|
451
|
-
await this
|
|
448
|
+
async #closeBrowser(): Promise<void> {
|
|
449
|
+
await this.#clearElementCache();
|
|
450
|
+
if (this.#page && !this.#page.isClosed()) {
|
|
451
|
+
await this.#page.close();
|
|
452
452
|
}
|
|
453
|
-
this
|
|
454
|
-
if (this
|
|
455
|
-
await this
|
|
453
|
+
this.#page = null;
|
|
454
|
+
if (this.#browser?.connected) {
|
|
455
|
+
await this.#browser.close();
|
|
456
456
|
}
|
|
457
|
-
this
|
|
458
|
-
this
|
|
459
|
-
this
|
|
457
|
+
this.#browser = null;
|
|
458
|
+
this.#browserSession = null;
|
|
459
|
+
this.#userAgentOverride = null;
|
|
460
460
|
}
|
|
461
461
|
|
|
462
|
-
|
|
463
|
-
await this
|
|
464
|
-
this
|
|
462
|
+
async #resetBrowser(params?: BrowserParams): Promise<Page> {
|
|
463
|
+
await this.#closeBrowser();
|
|
464
|
+
this.#currentHeadless = this.session.settings.get("browser.headless");
|
|
465
465
|
const initialViewport = params?.viewport ?? DEFAULT_VIEWPORT;
|
|
466
|
-
this
|
|
467
|
-
headless: this
|
|
468
|
-
defaultViewport: this
|
|
466
|
+
this.#browser = await puppeteer.launch({
|
|
467
|
+
headless: this.#currentHeadless,
|
|
468
|
+
defaultViewport: this.#currentHeadless ? initialViewport : null,
|
|
469
469
|
args: [
|
|
470
470
|
"--no-sandbox",
|
|
471
471
|
"--disable-setuid-sandbox",
|
|
@@ -474,46 +474,46 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
474
474
|
],
|
|
475
475
|
ignoreDefaultArgs: [...STEALTH_IGNORE_DEFAULT_ARGS],
|
|
476
476
|
});
|
|
477
|
-
this
|
|
478
|
-
await this
|
|
479
|
-
if (this
|
|
480
|
-
await this
|
|
477
|
+
this.#page = await this.#browser.newPage();
|
|
478
|
+
await this.#applyStealthPatches(this.#page);
|
|
479
|
+
if (this.#currentHeadless || params?.viewport) {
|
|
480
|
+
await this.#applyViewport(this.#page, params?.viewport);
|
|
481
481
|
}
|
|
482
|
-
return this
|
|
482
|
+
return this.#page;
|
|
483
483
|
}
|
|
484
484
|
|
|
485
|
-
|
|
485
|
+
async #ensurePage(params?: BrowserParams): Promise<Page> {
|
|
486
486
|
const desiredHeadless = this.session.settings.get("browser.headless");
|
|
487
|
-
if (this
|
|
488
|
-
return this
|
|
487
|
+
if (this.#currentHeadless !== null && this.#currentHeadless !== desiredHeadless) {
|
|
488
|
+
return this.#resetBrowser(params);
|
|
489
489
|
}
|
|
490
|
-
if (this
|
|
491
|
-
return this
|
|
490
|
+
if (this.#page && !this.#page.isClosed()) {
|
|
491
|
+
return this.#page;
|
|
492
492
|
}
|
|
493
|
-
if (!this
|
|
494
|
-
return this
|
|
493
|
+
if (!this.#browser || !this.#browser.isConnected()) {
|
|
494
|
+
return this.#resetBrowser(params);
|
|
495
495
|
}
|
|
496
|
-
this
|
|
497
|
-
await this
|
|
498
|
-
if (this
|
|
499
|
-
await this
|
|
496
|
+
this.#page = await this.#browser.newPage();
|
|
497
|
+
await this.#applyStealthPatches(this.#page);
|
|
498
|
+
if (this.#currentHeadless || params?.viewport) {
|
|
499
|
+
await this.#applyViewport(this.#page, params?.viewport);
|
|
500
500
|
}
|
|
501
|
-
return this
|
|
501
|
+
return this.#page;
|
|
502
502
|
}
|
|
503
503
|
|
|
504
|
-
|
|
504
|
+
async #applyViewport(page: Page, viewport?: BrowserParams["viewport"]): Promise<void> {
|
|
505
505
|
const target = viewport ?? DEFAULT_VIEWPORT;
|
|
506
506
|
await page.setViewport(target);
|
|
507
507
|
}
|
|
508
508
|
|
|
509
|
-
|
|
510
|
-
if (this
|
|
511
|
-
this
|
|
509
|
+
async #clearElementCache(): Promise<void> {
|
|
510
|
+
if (this.#elementCache.size === 0) {
|
|
511
|
+
this.#elementIdCounter = 0;
|
|
512
512
|
return;
|
|
513
513
|
}
|
|
514
|
-
const handles = Array.from(this
|
|
515
|
-
this
|
|
516
|
-
this
|
|
514
|
+
const handles = Array.from(this.#elementCache.values());
|
|
515
|
+
this.#elementCache.clear();
|
|
516
|
+
this.#elementIdCounter = 0;
|
|
517
517
|
await Promise.all(
|
|
518
518
|
handles.map(async handle => {
|
|
519
519
|
try {
|
|
@@ -525,25 +525,25 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
525
525
|
);
|
|
526
526
|
}
|
|
527
527
|
|
|
528
|
-
|
|
529
|
-
const handle = this
|
|
528
|
+
async #resolveCachedHandle(id: number): Promise<ElementHandle> {
|
|
529
|
+
const handle = this.#elementCache.get(id);
|
|
530
530
|
if (!handle) {
|
|
531
531
|
throw new ToolError(`Unknown element_id ${id}. Run observe to refresh the element list.`);
|
|
532
532
|
}
|
|
533
533
|
try {
|
|
534
534
|
const isConnected = (await handle.evaluate(el => el.isConnected)) as boolean;
|
|
535
535
|
if (!isConnected) {
|
|
536
|
-
await this
|
|
536
|
+
await this.#clearElementCache();
|
|
537
537
|
throw new ToolError(`Element_id ${id} is stale. Run observe again.`);
|
|
538
538
|
}
|
|
539
539
|
} catch {
|
|
540
|
-
await this
|
|
540
|
+
await this.#clearElementCache();
|
|
541
541
|
throw new ToolError(`Element_id ${id} is stale. Run observe again.`);
|
|
542
542
|
}
|
|
543
543
|
return handle;
|
|
544
544
|
}
|
|
545
545
|
|
|
546
|
-
|
|
546
|
+
#isInteractiveNode(node: SerializedAXNode): boolean {
|
|
547
547
|
if (INTERACTIVE_AX_ROLES.has(node.role)) return true;
|
|
548
548
|
return (
|
|
549
549
|
node.checked !== undefined ||
|
|
@@ -554,12 +554,12 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
554
554
|
);
|
|
555
555
|
}
|
|
556
556
|
|
|
557
|
-
|
|
557
|
+
async #collectObservationEntries(
|
|
558
558
|
node: SerializedAXNode,
|
|
559
559
|
entries: ObservationEntry[],
|
|
560
560
|
options: { viewportOnly: boolean; includeAll: boolean },
|
|
561
561
|
): Promise<void> {
|
|
562
|
-
if (options.includeAll || this
|
|
562
|
+
if (options.includeAll || this.#isInteractiveNode(node)) {
|
|
563
563
|
const handle = await node.elementHandle();
|
|
564
564
|
if (handle) {
|
|
565
565
|
let inViewport = true;
|
|
@@ -571,7 +571,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
571
571
|
}
|
|
572
572
|
}
|
|
573
573
|
if (inViewport) {
|
|
574
|
-
const id = ++this
|
|
574
|
+
const id = ++this.#elementIdCounter;
|
|
575
575
|
const states: string[] = [];
|
|
576
576
|
if (node.disabled) states.push("disabled");
|
|
577
577
|
if (node.checked !== undefined) states.push(`checked=${String(node.checked)}`);
|
|
@@ -584,7 +584,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
584
584
|
if (node.multiline) states.push("multiline");
|
|
585
585
|
if (node.modal) states.push("modal");
|
|
586
586
|
if (node.focused) states.push("focused");
|
|
587
|
-
this
|
|
587
|
+
this.#elementCache.set(id, handle);
|
|
588
588
|
entries.push({
|
|
589
589
|
id,
|
|
590
590
|
role: node.role,
|
|
@@ -600,11 +600,11 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
600
600
|
}
|
|
601
601
|
}
|
|
602
602
|
for (const child of node.children ?? []) {
|
|
603
|
-
await this
|
|
603
|
+
await this.#collectObservationEntries(child, entries, options);
|
|
604
604
|
}
|
|
605
605
|
}
|
|
606
606
|
|
|
607
|
-
|
|
607
|
+
#formatObservation(observation: Observation): string {
|
|
608
608
|
const viewport = `${observation.viewport.width}x${observation.viewport.height}`;
|
|
609
609
|
const scroll = `x=${observation.scroll.x} y=${observation.scroll.y} viewport=${observation.scroll.width}x${observation.scroll.height} doc=${observation.scroll.scrollWidth}x${observation.scroll.scrollHeight}`;
|
|
610
610
|
const lines = [
|
|
@@ -628,26 +628,26 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
628
628
|
/**
|
|
629
629
|
* Restart the browser to apply changes like headless mode.
|
|
630
630
|
*/
|
|
631
|
-
|
|
632
|
-
await this
|
|
631
|
+
async restartForModeChange(): Promise<void> {
|
|
632
|
+
await this.#resetBrowser();
|
|
633
633
|
}
|
|
634
634
|
|
|
635
|
-
|
|
636
|
-
this
|
|
637
|
-
await this
|
|
638
|
-
await this
|
|
635
|
+
async #applyStealthPatches(page: Page): Promise<void> {
|
|
636
|
+
this.#patchSourceUrl(page);
|
|
637
|
+
await this.#applyUserAgentOverride(page);
|
|
638
|
+
await this.#injectStealthScripts(page);
|
|
639
639
|
}
|
|
640
640
|
|
|
641
|
-
|
|
641
|
+
async #applyUserAgentOverride(page: Page): Promise<void> {
|
|
642
642
|
const client = resolvePageClient(page);
|
|
643
643
|
if (!client) return;
|
|
644
|
-
const override = await this
|
|
645
|
-
await this
|
|
646
|
-
await this
|
|
644
|
+
const override = await this.#resolveUserAgentOverride(page);
|
|
645
|
+
await this.#sendUserAgentOverride(client, override);
|
|
646
|
+
await this.#configureUserAgentTargets(override);
|
|
647
647
|
}
|
|
648
648
|
|
|
649
|
-
|
|
650
|
-
if (this
|
|
649
|
+
async #resolveUserAgentOverride(page: Page): Promise<UserAgentOverride> {
|
|
650
|
+
if (this.#userAgentOverride) return this.#userAgentOverride;
|
|
651
651
|
const rawUserAgent = await page.browser().userAgent();
|
|
652
652
|
let userAgent = rawUserAgent.replace("HeadlessChrome/", "Chrome/");
|
|
653
653
|
if (userAgent.includes("Linux") && !userAgent.includes("Android")) {
|
|
@@ -699,7 +699,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
699
699
|
brands[order[1]] = { brand: "Chromium", version: String(majorVersion) };
|
|
700
700
|
brands[order[2]] = { brand: "Google Chrome", version: String(majorVersion) };
|
|
701
701
|
|
|
702
|
-
this
|
|
702
|
+
this.#userAgentOverride = {
|
|
703
703
|
userAgent,
|
|
704
704
|
platform,
|
|
705
705
|
acceptLanguage: STEALTH_ACCEPT_LANGUAGE,
|
|
@@ -713,42 +713,42 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
713
713
|
mobile: isAndroid,
|
|
714
714
|
},
|
|
715
715
|
};
|
|
716
|
-
return this
|
|
716
|
+
return this.#userAgentOverride;
|
|
717
717
|
}
|
|
718
718
|
|
|
719
|
-
|
|
720
|
-
if (!this
|
|
721
|
-
if (!this
|
|
722
|
-
this
|
|
723
|
-
await this
|
|
719
|
+
async #configureUserAgentTargets(override: UserAgentOverride): Promise<void> {
|
|
720
|
+
if (!this.#browser) return;
|
|
721
|
+
if (!this.#browserSession) {
|
|
722
|
+
this.#browserSession = await this.#browser.target().createCDPSession();
|
|
723
|
+
await this.#browserSession.send("Target.setAutoAttach", {
|
|
724
724
|
autoAttach: true,
|
|
725
725
|
waitForDebuggerOnStart: false,
|
|
726
726
|
flatten: true,
|
|
727
727
|
});
|
|
728
|
-
this
|
|
729
|
-
const connection = this
|
|
728
|
+
this.#browserSession.on("Target.attachedToTarget", async (event: { sessionId: string }) => {
|
|
729
|
+
const connection = this.#browserSession?.connection();
|
|
730
730
|
const session = connection?.session(event.sessionId);
|
|
731
|
-
if (!session || !this
|
|
732
|
-
await this
|
|
731
|
+
if (!session || !this.#userAgentOverride) return;
|
|
732
|
+
await this.#sendUserAgentOverride(this.#wrapSession(session), this.#userAgentOverride);
|
|
733
733
|
});
|
|
734
734
|
}
|
|
735
735
|
|
|
736
|
-
const targets = this
|
|
736
|
+
const targets = this.#browser.targets();
|
|
737
737
|
await Promise.all(
|
|
738
738
|
targets.map(async target => {
|
|
739
739
|
const session = await target.createCDPSession();
|
|
740
|
-
await this
|
|
740
|
+
await this.#sendUserAgentOverride(this.#wrapSession(session), override);
|
|
741
741
|
}),
|
|
742
742
|
);
|
|
743
743
|
}
|
|
744
744
|
|
|
745
|
-
|
|
745
|
+
#wrapSession(session: CDPSession): PuppeteerCdpClient {
|
|
746
746
|
return {
|
|
747
747
|
send: async (method, params) => session.send(method as never, params as never),
|
|
748
748
|
};
|
|
749
749
|
}
|
|
750
750
|
|
|
751
|
-
|
|
751
|
+
async #sendUserAgentOverride(client: PuppeteerCdpClient, override: UserAgentOverride): Promise<void> {
|
|
752
752
|
try {
|
|
753
753
|
await client.send("Network.enable");
|
|
754
754
|
} catch {}
|
|
@@ -768,12 +768,12 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
768
768
|
}
|
|
769
769
|
}
|
|
770
770
|
|
|
771
|
-
|
|
771
|
+
#patchSourceUrl(page: Page): void {
|
|
772
772
|
const client = resolvePageClient(page);
|
|
773
773
|
if (!client) return;
|
|
774
774
|
const clientKey = client as object;
|
|
775
|
-
if (this
|
|
776
|
-
this
|
|
775
|
+
if (this.#patchedClients.has(clientKey)) return;
|
|
776
|
+
this.#patchedClients.add(clientKey);
|
|
777
777
|
const originalSend = client.send.bind(client);
|
|
778
778
|
client.send = async (method: string, params?: Record<string, unknown>) => {
|
|
779
779
|
const next = async (payload?: Record<string, unknown>) => {
|
|
@@ -813,7 +813,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
813
813
|
}
|
|
814
814
|
|
|
815
815
|
/** Injects stealth scripts that cover common puppeteer detection surfaces. */
|
|
816
|
-
|
|
816
|
+
async #injectStealthScripts(page: Page): Promise<void> {
|
|
817
817
|
const scripts = [
|
|
818
818
|
stealthTamperingScript,
|
|
819
819
|
stealthActivityScript,
|
|
@@ -880,7 +880,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
880
880
|
document.head.removeChild(iframe);})();`);
|
|
881
881
|
}
|
|
882
882
|
|
|
883
|
-
|
|
883
|
+
async execute(
|
|
884
884
|
_toolCallId: string,
|
|
885
885
|
params: BrowserParams,
|
|
886
886
|
signal?: AbortSignal,
|
|
@@ -895,21 +895,21 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
895
895
|
|
|
896
896
|
switch (params.action) {
|
|
897
897
|
case "open": {
|
|
898
|
-
const page = await untilAborted(signal, () => this
|
|
898
|
+
const page = await untilAborted(signal, () => this.#resetBrowser(params));
|
|
899
899
|
const viewport = page.viewport();
|
|
900
900
|
details.viewport = viewport ?? DEFAULT_VIEWPORT;
|
|
901
901
|
return toolResult(details).text("Opened headless browser session").done();
|
|
902
902
|
}
|
|
903
903
|
case "close": {
|
|
904
|
-
await untilAborted(signal, () => this
|
|
904
|
+
await untilAborted(signal, () => this.#closeBrowser());
|
|
905
905
|
return toolResult(details).text("Closed headless browser session").done();
|
|
906
906
|
}
|
|
907
907
|
case "goto": {
|
|
908
908
|
const url = ensureParam(params.url, "url", params.action);
|
|
909
909
|
details.url = url;
|
|
910
|
-
const page = await this
|
|
910
|
+
const page = await this.#ensurePage(params);
|
|
911
911
|
const waitUntil = params.wait_until ?? "networkidle2";
|
|
912
|
-
await this
|
|
912
|
+
await this.#clearElementCache();
|
|
913
913
|
await untilAborted(signal, () => page.goto(url, { waitUntil, timeout: timeoutMs }));
|
|
914
914
|
const finalUrl = page.url();
|
|
915
915
|
const title = (await untilAborted(signal, () => page.title())) as string;
|
|
@@ -920,10 +920,10 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
920
920
|
.done();
|
|
921
921
|
}
|
|
922
922
|
case "observe": {
|
|
923
|
-
const page = await this
|
|
923
|
+
const page = await this.#ensurePage(params);
|
|
924
924
|
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
925
925
|
const observeSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
|
926
|
-
await this
|
|
926
|
+
await this.#clearElementCache();
|
|
927
927
|
const snapshot = (await untilAborted(observeSignal, () =>
|
|
928
928
|
page.accessibility.snapshot({ interestingOnly: !(params.include_all ?? false) }),
|
|
929
929
|
)) as SerializedAXNode | null;
|
|
@@ -931,7 +931,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
931
931
|
throw new ToolError("Accessibility snapshot unavailable");
|
|
932
932
|
}
|
|
933
933
|
const entries: ObservationEntry[] = [];
|
|
934
|
-
await this
|
|
934
|
+
await this.#collectObservationEntries(snapshot, entries, {
|
|
935
935
|
viewportOnly: params.viewport_only ?? false,
|
|
936
936
|
includeAll: params.include_all ?? false,
|
|
937
937
|
});
|
|
@@ -969,12 +969,12 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
969
969
|
details.viewport = viewport;
|
|
970
970
|
details.observation = observation;
|
|
971
971
|
details.result = `${entries.length} elements`;
|
|
972
|
-
return toolResult(details).text(this
|
|
972
|
+
return toolResult(details).text(this.#formatObservation(observation)).done();
|
|
973
973
|
}
|
|
974
974
|
case "click": {
|
|
975
975
|
const selector = ensureParam(params.selector, "selector", params.action);
|
|
976
976
|
details.selector = selector;
|
|
977
|
-
const page = await this
|
|
977
|
+
const page = await this.#ensurePage(params);
|
|
978
978
|
const resolvedSelector = normalizeSelector(selector);
|
|
979
979
|
if (resolvedSelector.startsWith("text/")) {
|
|
980
980
|
await clickQueryHandlerText(page, resolvedSelector, timeoutMs, signal);
|
|
@@ -987,11 +987,11 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
987
987
|
case "click_id": {
|
|
988
988
|
const elementId = ensureParam(params.element_id, "element_id", params.action);
|
|
989
989
|
details.elementId = elementId;
|
|
990
|
-
const handle = await this
|
|
990
|
+
const handle = await this.#resolveCachedHandle(elementId);
|
|
991
991
|
try {
|
|
992
992
|
await untilAborted(signal, () => handle.click());
|
|
993
993
|
} catch {
|
|
994
|
-
await this
|
|
994
|
+
await this.#clearElementCache();
|
|
995
995
|
throw new ToolError(`Element_id ${elementId} is stale. Run observe again.`);
|
|
996
996
|
}
|
|
997
997
|
return toolResult(details).text(`Clicked element ${elementId}`).done();
|
|
@@ -1000,7 +1000,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1000
1000
|
const selector = ensureParam(params.selector, "selector", params.action);
|
|
1001
1001
|
const text = ensureParam(params.text, "text", params.action);
|
|
1002
1002
|
details.selector = selector;
|
|
1003
|
-
const page = await this
|
|
1003
|
+
const page = await this.#ensurePage(params);
|
|
1004
1004
|
const resolvedSelector = normalizeSelector(selector);
|
|
1005
1005
|
const locator = page.locator(resolvedSelector).setTimeout(timeoutMs);
|
|
1006
1006
|
const handle = (await untilAborted(signal, () => locator.waitHandle())) as ElementHandle;
|
|
@@ -1012,13 +1012,13 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1012
1012
|
const elementId = ensureParam(params.element_id, "element_id", params.action);
|
|
1013
1013
|
const text = ensureParam(params.text, "text", params.action);
|
|
1014
1014
|
details.elementId = elementId;
|
|
1015
|
-
const page = await this
|
|
1016
|
-
const handle = await this
|
|
1015
|
+
const page = await this.#ensurePage(params);
|
|
1016
|
+
const handle = await this.#resolveCachedHandle(elementId);
|
|
1017
1017
|
try {
|
|
1018
1018
|
await untilAborted(signal, () => handle.focus());
|
|
1019
1019
|
await untilAborted(signal, () => page.keyboard.type(text, { delay: 0 }));
|
|
1020
1020
|
} catch {
|
|
1021
|
-
await this
|
|
1021
|
+
await this.#clearElementCache();
|
|
1022
1022
|
throw new ToolError(`Element_id ${elementId} is stale. Run observe again.`);
|
|
1023
1023
|
}
|
|
1024
1024
|
return toolResult(details).text(`Typed into element ${elementId}`).done();
|
|
@@ -1027,7 +1027,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1027
1027
|
const selector = ensureParam(params.selector, "selector", params.action);
|
|
1028
1028
|
const value = ensureParam(params.value, "value", params.action);
|
|
1029
1029
|
details.selector = selector;
|
|
1030
|
-
const page = await this
|
|
1030
|
+
const page = await this.#ensurePage(params);
|
|
1031
1031
|
const resolvedSelector = normalizeSelector(selector);
|
|
1032
1032
|
const locator = page.locator(resolvedSelector).setTimeout(timeoutMs);
|
|
1033
1033
|
await untilAborted(signal, () => locator.fill(value));
|
|
@@ -1037,7 +1037,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1037
1037
|
const elementId = ensureParam(params.element_id, "element_id", params.action);
|
|
1038
1038
|
const value = ensureParam(params.value, "value", params.action);
|
|
1039
1039
|
details.elementId = elementId;
|
|
1040
|
-
const handle = await this
|
|
1040
|
+
const handle = await this.#resolveCachedHandle(elementId);
|
|
1041
1041
|
try {
|
|
1042
1042
|
await untilAborted(signal, () =>
|
|
1043
1043
|
handle.evaluate((el, inputValue) => {
|
|
@@ -1051,14 +1051,14 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1051
1051
|
}, value),
|
|
1052
1052
|
);
|
|
1053
1053
|
} catch {
|
|
1054
|
-
await this
|
|
1054
|
+
await this.#clearElementCache();
|
|
1055
1055
|
throw new ToolError(`Element_id ${elementId} is stale. Run observe again.`);
|
|
1056
1056
|
}
|
|
1057
1057
|
return toolResult(details).text(`Filled element ${elementId}`).done();
|
|
1058
1058
|
}
|
|
1059
1059
|
case "press": {
|
|
1060
1060
|
const key = ensureParam(params.key, "key", params.action) as KeyInput;
|
|
1061
|
-
const page = await this
|
|
1061
|
+
const page = await this.#ensurePage(params);
|
|
1062
1062
|
if (params.selector) {
|
|
1063
1063
|
const resolvedSelector = normalizeSelector(params.selector as string);
|
|
1064
1064
|
await untilAborted(signal, () => page.focus(resolvedSelector));
|
|
@@ -1069,14 +1069,14 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1069
1069
|
case "scroll": {
|
|
1070
1070
|
const deltaY = ensureParam(params.delta_y, "delta_y", params.action);
|
|
1071
1071
|
const deltaX = params.delta_x ?? 0;
|
|
1072
|
-
const page = await this
|
|
1072
|
+
const page = await this.#ensurePage(params);
|
|
1073
1073
|
await untilAborted(signal, () => page.mouse.wheel({ deltaX, deltaY }));
|
|
1074
1074
|
return toolResult(details).text(`Scrolled by ${deltaX}, ${deltaY}`).done();
|
|
1075
1075
|
}
|
|
1076
1076
|
case "drag": {
|
|
1077
1077
|
const fromSelector = ensureParam(params.from_selector, "from_selector", params.action);
|
|
1078
1078
|
const toSelector = ensureParam(params.to_selector, "to_selector", params.action);
|
|
1079
|
-
const page = await this
|
|
1079
|
+
const page = await this.#ensurePage(params);
|
|
1080
1080
|
const resolvedFromSelector = normalizeSelector(fromSelector);
|
|
1081
1081
|
const resolvedToSelector = normalizeSelector(toSelector);
|
|
1082
1082
|
const fromHandle = (await untilAborted(signal, () =>
|
|
@@ -1116,7 +1116,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1116
1116
|
case "wait_for_selector": {
|
|
1117
1117
|
const selector = ensureParam(params.selector, "selector", params.action);
|
|
1118
1118
|
details.selector = selector;
|
|
1119
|
-
const page = await this
|
|
1119
|
+
const page = await this.#ensurePage(params);
|
|
1120
1120
|
const resolvedSelector = normalizeSelector(selector);
|
|
1121
1121
|
const locator = page.locator(resolvedSelector).setTimeout(timeoutMs);
|
|
1122
1122
|
await untilAborted(signal, () => locator.wait());
|
|
@@ -1124,11 +1124,14 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1124
1124
|
}
|
|
1125
1125
|
case "evaluate": {
|
|
1126
1126
|
const script = ensureParam(params.script, "script", params.action);
|
|
1127
|
-
const page = await this
|
|
1127
|
+
const page = await this.#ensurePage(params);
|
|
1128
1128
|
const value = (await untilAborted(signal, () =>
|
|
1129
1129
|
page.evaluate((source: string) => {
|
|
1130
|
-
|
|
1131
|
-
|
|
1130
|
+
try {
|
|
1131
|
+
return new Function(`return (${source});`)();
|
|
1132
|
+
} catch {
|
|
1133
|
+
return new Function(source)();
|
|
1134
|
+
}
|
|
1132
1135
|
}, script),
|
|
1133
1136
|
)) as unknown;
|
|
1134
1137
|
const output = formatEvaluateResult(value);
|
|
@@ -1136,7 +1139,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1136
1139
|
return toolResult(details).text(output).done();
|
|
1137
1140
|
}
|
|
1138
1141
|
case "get_text": {
|
|
1139
|
-
const page = await this
|
|
1142
|
+
const page = await this.#ensurePage(params);
|
|
1140
1143
|
if (params.args?.length) {
|
|
1141
1144
|
const values = (await Promise.all(
|
|
1142
1145
|
params.args.map((arg, index) => {
|
|
@@ -1162,7 +1165,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1162
1165
|
return toolResult(details).text(value).done();
|
|
1163
1166
|
}
|
|
1164
1167
|
case "get_html": {
|
|
1165
|
-
const page = await this
|
|
1168
|
+
const page = await this.#ensurePage(params);
|
|
1166
1169
|
if (params.args?.length) {
|
|
1167
1170
|
const values = (await Promise.all(
|
|
1168
1171
|
params.args.map((arg, index) => {
|
|
@@ -1188,7 +1191,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1188
1191
|
return toolResult(details).text(value).done();
|
|
1189
1192
|
}
|
|
1190
1193
|
case "get_attribute": {
|
|
1191
|
-
const page = await this
|
|
1194
|
+
const page = await this.#ensurePage(params);
|
|
1192
1195
|
if (params.args?.length) {
|
|
1193
1196
|
const values = (await Promise.all(
|
|
1194
1197
|
params.args.map((arg, index) => {
|
|
@@ -1226,7 +1229,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1226
1229
|
return toolResult(details).text(output).done();
|
|
1227
1230
|
}
|
|
1228
1231
|
case "extract_readable": {
|
|
1229
|
-
const page = await this
|
|
1232
|
+
const page = await this.#ensurePage(params);
|
|
1230
1233
|
const format = params.format ?? "markdown";
|
|
1231
1234
|
const html = (await untilAborted(signal, () => page.content())) as string;
|
|
1232
1235
|
const url = page.url();
|
|
@@ -1263,7 +1266,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1263
1266
|
.done();
|
|
1264
1267
|
}
|
|
1265
1268
|
case "screenshot": {
|
|
1266
|
-
const page = await this
|
|
1269
|
+
const page = await this.#ensurePage(params);
|
|
1267
1270
|
const fullPage = params.selector ? false : (params.full_page ?? false);
|
|
1268
1271
|
let buffer: Buffer;
|
|
1269
1272
|
|
|
@@ -1289,7 +1292,7 @@ export class BrowserTool implements AgentTool<typeof browserSchema, BrowserToolD
|
|
|
1289
1292
|
{ maxBytes: 0.75 * 1024 * 1024 },
|
|
1290
1293
|
);
|
|
1291
1294
|
const dimensionNote = formatDimensionNote(resized);
|
|
1292
|
-
const tempFile = path.join(tmpdir(), `omp-sshots-${Snowflake.next()}.png`);
|
|
1295
|
+
const tempFile = path.join(os.tmpdir(), `omp-sshots-${Snowflake.next()}.png`);
|
|
1293
1296
|
await Bun.write(tempFile, resized.buffer);
|
|
1294
1297
|
details.screenshotPath = tempFile;
|
|
1295
1298
|
details.mimeType = resized.mimeType;
|