agiagent-dev 2026.1.33 → 2026.1.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/bash-tools.exec.js +6 -1
- package/dist/agents/tools/browser-tool.js +64 -9
- package/dist/agents/tools/browser-tool.schema.d.ts +3 -1
- package/dist/agents/tools/browser-tool.schema.js +5 -0
- package/dist/agents/tools/nodes-tool.js +7 -0
- package/dist/browser/chrome.js +1 -1
- package/dist/browser/control-service.js +2 -2
- package/dist/browser/pw-ai.d.ts +1 -0
- package/dist/browser/pw-ai.js +1 -0
- package/dist/browser/pw-tools-core.uploads.d.ts +51 -0
- package/dist/browser/pw-tools-core.uploads.js +225 -0
- package/dist/browser/routes/agent.act.js +55 -0
- package/dist/browser/server.js +2 -2
- package/dist/build-info.json +3 -3
- package/dist/canvas-host/a2ui/.bundle.hash +1 -1
- package/dist/cli/banner.d.ts +0 -1
- package/dist/cli/banner.js +4 -7
- package/dist/cli/connect-cli/connect-ui.d.ts +20 -0
- package/dist/cli/connect-cli/connect-ui.js +221 -0
- package/dist/cli/connect-cli/register.js +56 -6
- package/dist/gateway/auth.d.ts +1 -0
- package/dist/gateway/hosted-agent.d.ts +11 -0
- package/dist/gateway/hosted-agent.js +140 -4
- package/dist/gateway/hosted-db.d.ts +6 -0
- package/dist/gateway/hosted-db.js +40 -0
- package/dist/gateway/hosted-telegram.d.ts +2 -0
- package/dist/gateway/hosted-telegram.js +70 -2
- package/dist/gateway/server/ws-connection/message-handler.js +3 -1
- package/dist/gateway/server-constants.js +1 -1
- package/dist/node-host/runner.d.ts +3 -0
- package/dist/node-host/runner.js +9 -7
- package/package.json +1 -1
|
@@ -551,10 +551,15 @@ export function createExecTool(defaults) {
|
|
|
551
551
|
const parsedAgentSession = parseAgentSessionKey(defaults?.sessionKey);
|
|
552
552
|
const agentId = defaults?.agentId ??
|
|
553
553
|
(parsedAgentSession ? resolveAgentIdFromSessionKey(defaults?.sessionKey) : undefined);
|
|
554
|
+
// In hosted mode, update description to clarify commands run on user's device
|
|
555
|
+
const isHostedMode = process.env.AGIAGENT_HOSTED_MODE === "1";
|
|
556
|
+
const execDescription = isHostedMode
|
|
557
|
+
? "Execute shell commands on the user's connected device. Commands run on their Mac/PC, not on this server. Use yieldMs/background for long-running commands. Use pty=true for TTY-required commands."
|
|
558
|
+
: "Execute shell commands with background continuation. Use yieldMs/background to continue later via process tool. Use pty=true for TTY-required commands (terminal UIs, coding agents).";
|
|
554
559
|
return {
|
|
555
560
|
name: "exec",
|
|
556
561
|
label: "exec",
|
|
557
|
-
description:
|
|
562
|
+
description: execDescription,
|
|
558
563
|
parameters: execSchema,
|
|
559
564
|
execute: async (_toolCallId, args, signal, onUpdate) => {
|
|
560
565
|
const params = args;
|
|
@@ -19,19 +19,34 @@ async function resolveBrowserNodeTarget(params) {
|
|
|
19
19
|
const cfg = loadConfig();
|
|
20
20
|
const policy = cfg.gateway?.nodes?.browser;
|
|
21
21
|
const mode = policy?.mode ?? "auto";
|
|
22
|
+
// In hosted mode, prefer node browser since there's no local browser on the gateway
|
|
23
|
+
const isHostedMode = process.env.AGIAGENT_HOSTED_MODE === "1";
|
|
22
24
|
if (mode === "off") {
|
|
23
25
|
if (params.target === "node" || params.requestedNode) {
|
|
24
26
|
throw new Error("Node browser proxy is disabled (gateway.nodes.browser.mode=off).");
|
|
25
27
|
}
|
|
26
|
-
|
|
28
|
+
// In hosted mode, we still need a node even if mode is off (no local browser)
|
|
29
|
+
if (!isHostedMode) {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
27
32
|
}
|
|
28
33
|
if (params.sandboxBridgeUrl?.trim() && params.target !== "node" && !params.requestedNode) {
|
|
29
|
-
|
|
34
|
+
// Sandbox browser available, but in hosted mode prefer node if no explicit target
|
|
35
|
+
if (!isHostedMode) {
|
|
36
|
+
return null;
|
|
37
|
+
}
|
|
30
38
|
}
|
|
31
39
|
if (params.target && params.target !== "node") {
|
|
32
|
-
|
|
40
|
+
// Explicit non-node target requested
|
|
41
|
+
// In hosted mode with target=host, fall through to check for nodes since host won't work
|
|
42
|
+
if (params.target === "host" && isHostedMode) {
|
|
43
|
+
// Let it fall through to try node browser
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
return null;
|
|
47
|
+
}
|
|
33
48
|
}
|
|
34
|
-
if (mode === "manual" && params.target !== "node" && !params.requestedNode) {
|
|
49
|
+
if (mode === "manual" && params.target !== "node" && !params.requestedNode && !isHostedMode) {
|
|
35
50
|
return null;
|
|
36
51
|
}
|
|
37
52
|
const nodes = await listNodes({});
|
|
@@ -40,6 +55,10 @@ async function resolveBrowserNodeTarget(params) {
|
|
|
40
55
|
if (params.target === "node" || params.requestedNode) {
|
|
41
56
|
throw new Error("No connected browser-capable nodes.");
|
|
42
57
|
}
|
|
58
|
+
// In hosted mode, we need a node browser but none available
|
|
59
|
+
if (isHostedMode) {
|
|
60
|
+
throw new Error("No connected browser-capable nodes. Connect a device with browser support to use browser features.");
|
|
61
|
+
}
|
|
43
62
|
return null;
|
|
44
63
|
}
|
|
45
64
|
const requested = params.requestedNode?.trim() || policy?.node?.trim();
|
|
@@ -48,7 +67,7 @@ async function resolveBrowserNodeTarget(params) {
|
|
|
48
67
|
const node = browserNodes.find((entry) => entry.nodeId === nodeId);
|
|
49
68
|
return { nodeId, label: node?.displayName ?? node?.remoteIp ?? nodeId };
|
|
50
69
|
}
|
|
51
|
-
if (params.target === "node") {
|
|
70
|
+
if (params.target === "node" || isHostedMode) {
|
|
52
71
|
if (browserNodes.length === 1) {
|
|
53
72
|
const node = browserNodes[0];
|
|
54
73
|
return { nodeId: node.nodeId, label: node.displayName ?? node.remoteIp ?? node.nodeId };
|
|
@@ -147,11 +166,9 @@ export function createBrowserTool(opts) {
|
|
|
147
166
|
label: "Browser",
|
|
148
167
|
name: "browser",
|
|
149
168
|
description: [
|
|
150
|
-
"Control the browser via AGIAgent's browser control server (status/start/stop/profiles/tabs/open/snapshot/screenshot/actions).",
|
|
151
|
-
'
|
|
152
|
-
'If the user mentions the Chrome extension / Browser Relay / toolbar button / “attach tab”, ALWAYS use profile="chrome" (do not ask which profile).',
|
|
169
|
+
"Control the browser via AGIAgent's browser control server (status/start/stop/profiles/tabs/open/snapshot/screenshot/actions/list_upload_inputs/attach_file).",
|
|
170
|
+
'ALWAYS use profile="agiagent". This is the isolated agiagent-managed browser. Do NOT use any other profile. Never use profile="chrome".',
|
|
153
171
|
'When a node-hosted browser proxy is available, the tool may auto-route to it. Pin a node with node=<id|name> or target="node".',
|
|
154
|
-
"Chrome extension relay needs an attached tab: user must click the AGIAgent Browser Relay toolbar icon on the tab (badge ON). If no tab is connected, ask them to attach it.",
|
|
155
172
|
"When using refs from snapshot (e.g. e12), keep the same tab: prefer passing targetId from the snapshot response into subsequent actions (act/click/type/etc).",
|
|
156
173
|
'For stable, self-resolving refs across calls, use snapshot with refs="aria" (Playwright aria-ref ids). Default refs="role" are role+name-based.',
|
|
157
174
|
"Use snapshot+act for UI automation. Avoid act:wait by default; use only in exceptional cases when no reliable UI state exists.",
|
|
@@ -595,6 +612,44 @@ export function createBrowserTool(opts) {
|
|
|
595
612
|
throw err;
|
|
596
613
|
}
|
|
597
614
|
}
|
|
615
|
+
case "list_upload_inputs": {
|
|
616
|
+
const targetId = typeof params.targetId === "string" ? params.targetId.trim() : undefined;
|
|
617
|
+
if (proxyRequest) {
|
|
618
|
+
const result = await proxyRequest({
|
|
619
|
+
method: "GET",
|
|
620
|
+
path: "/upload-inputs",
|
|
621
|
+
profile,
|
|
622
|
+
query: { targetId },
|
|
623
|
+
});
|
|
624
|
+
return jsonResult(result);
|
|
625
|
+
}
|
|
626
|
+
// Local execution not yet implemented - requires browser control server
|
|
627
|
+
throw new Error("list_upload_inputs requires a connected browser node");
|
|
628
|
+
}
|
|
629
|
+
case "attach_file": {
|
|
630
|
+
const targetId = typeof params.targetId === "string" ? params.targetId.trim() : undefined;
|
|
631
|
+
const inputIndex = typeof params.inputIndex === "number" && Number.isFinite(params.inputIndex)
|
|
632
|
+
? params.inputIndex
|
|
633
|
+
: undefined;
|
|
634
|
+
const filePath = readStringParam(params, "filePath");
|
|
635
|
+
if (inputIndex === undefined) {
|
|
636
|
+
throw new Error("inputIndex is required");
|
|
637
|
+
}
|
|
638
|
+
if (!filePath) {
|
|
639
|
+
throw new Error("filePath is required");
|
|
640
|
+
}
|
|
641
|
+
if (proxyRequest) {
|
|
642
|
+
const result = await proxyRequest({
|
|
643
|
+
method: "POST",
|
|
644
|
+
path: "/attach-file",
|
|
645
|
+
profile,
|
|
646
|
+
body: { targetId, inputIndex, filePath },
|
|
647
|
+
});
|
|
648
|
+
return jsonResult(result);
|
|
649
|
+
}
|
|
650
|
+
// Local execution not yet implemented - requires browser control server
|
|
651
|
+
throw new Error("attach_file requires a connected browser node");
|
|
652
|
+
}
|
|
598
653
|
default:
|
|
599
654
|
throw new Error(`Unknown action: ${action}`);
|
|
600
655
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export declare const BrowserToolSchema: import("@sinclair/typebox").TObject<{
|
|
2
|
-
action: import("@sinclair/typebox").TUnsafe<"close" | "status" | "start" | "open" | "navigate" | "profiles" | "upload" | "snapshot" | "stop" | "tabs" | "focus" | "screenshot" | "console" | "pdf" | "dialog" | "act">;
|
|
2
|
+
action: import("@sinclair/typebox").TUnsafe<"close" | "status" | "start" | "open" | "navigate" | "profiles" | "upload" | "snapshot" | "stop" | "tabs" | "focus" | "screenshot" | "console" | "pdf" | "dialog" | "act" | "list_upload_inputs" | "attach_file">;
|
|
3
3
|
target: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TUnsafe<"sandbox" | "node" | "host">>;
|
|
4
4
|
node: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
5
5
|
profile: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
@@ -47,4 +47,6 @@ export declare const BrowserToolSchema: import("@sinclair/typebox").TObject<{
|
|
|
47
47
|
textGone: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
48
48
|
fn: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
49
49
|
}>>;
|
|
50
|
+
inputIndex: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TNumber>;
|
|
51
|
+
filePath: import("@sinclair/typebox").TOptional<import("@sinclair/typebox").TString>;
|
|
50
52
|
}>;
|
|
@@ -30,6 +30,8 @@ const BROWSER_TOOL_ACTIONS = [
|
|
|
30
30
|
"upload",
|
|
31
31
|
"dialog",
|
|
32
32
|
"act",
|
|
33
|
+
"list_upload_inputs",
|
|
34
|
+
"attach_file",
|
|
33
35
|
];
|
|
34
36
|
const BROWSER_TARGETS = ["sandbox", "host", "node"];
|
|
35
37
|
const BROWSER_SNAPSHOT_FORMATS = ["aria", "ai"];
|
|
@@ -102,4 +104,7 @@ export const BrowserToolSchema = Type.Object({
|
|
|
102
104
|
accept: Type.Optional(Type.Boolean()),
|
|
103
105
|
promptText: Type.Optional(Type.String()),
|
|
104
106
|
request: Type.Optional(BrowserActSchema),
|
|
107
|
+
// attach_file action
|
|
108
|
+
inputIndex: Type.Optional(Type.Number()),
|
|
109
|
+
filePath: Type.Optional(Type.String()),
|
|
105
110
|
});
|
|
@@ -364,6 +364,8 @@ export function createNodesTool(options) {
|
|
|
364
364
|
const needsScreenRecording = typeof params.needsScreenRecording === "boolean"
|
|
365
365
|
? params.needsScreenRecording
|
|
366
366
|
: undefined;
|
|
367
|
+
// In hosted mode, auto-approve commands - they're pre-authorized by the gateway
|
|
368
|
+
const isHostedMode = process.env.AGIAGENT_HOSTED_MODE === "1";
|
|
367
369
|
const raw = await callGatewayTool("node.invoke", gatewayOpts, {
|
|
368
370
|
nodeId,
|
|
369
371
|
command: "system.run",
|
|
@@ -375,6 +377,11 @@ export function createNodesTool(options) {
|
|
|
375
377
|
needsScreenRecording,
|
|
376
378
|
agentId,
|
|
377
379
|
sessionKey,
|
|
380
|
+
// Hosted mode: pass approval flags so node skips its approval check
|
|
381
|
+
approved: isHostedMode ? true : undefined,
|
|
382
|
+
approvalDecision: isHostedMode ? "allow-once" : undefined,
|
|
383
|
+
// Tell node to skip macOS app exec host and use direct spawn
|
|
384
|
+
hostedMode: isHostedMode ? true : undefined,
|
|
378
385
|
},
|
|
379
386
|
timeoutMs: invokeTimeoutMs,
|
|
380
387
|
idempotencyKey: crypto.randomUUID(),
|
package/dist/browser/chrome.js
CHANGED
|
@@ -223,7 +223,7 @@ export async function launchAGIAgentChrome(resolved, profile) {
|
|
|
223
223
|
throw new Error(`Failed to start Chrome CDP on port ${profile.cdpPort} for profile "${profile.name}".`);
|
|
224
224
|
}
|
|
225
225
|
const pid = proc.pid ?? -1;
|
|
226
|
-
log.
|
|
226
|
+
log.debug(`agiagent browser started (${exe.kind}) profile "${profile.name}" on 127.0.0.1:${profile.cdpPort} (pid ${pid})`);
|
|
227
227
|
return {
|
|
228
228
|
pid,
|
|
229
229
|
exe,
|
|
@@ -37,10 +37,10 @@ export async function startBrowserControlServiceFromConfig() {
|
|
|
37
37
|
continue;
|
|
38
38
|
}
|
|
39
39
|
await ensureChromeExtensionRelayServer({ cdpUrl: profile.cdpUrl }).catch((err) => {
|
|
40
|
-
logService.
|
|
40
|
+
logService.debug(`Chrome extension relay init failed for profile "${name}": ${String(err)}`);
|
|
41
41
|
});
|
|
42
42
|
}
|
|
43
|
-
logService.
|
|
43
|
+
logService.debug(`Browser control service ready (profiles=${Object.keys(resolved.profiles).length})`);
|
|
44
44
|
return state;
|
|
45
45
|
}
|
|
46
46
|
export async function stopBrowserControlService() {
|
package/dist/browser/pw-ai.d.ts
CHANGED
|
@@ -1,2 +1,3 @@
|
|
|
1
1
|
export { type BrowserConsoleMessage, closePageByTargetIdViaPlaywright, closePlaywrightBrowserConnection, createPageViaPlaywright, ensurePageState, focusPageByTargetIdViaPlaywright, getPageForTargetId, listPagesViaPlaywright, refLocator, type WithSnapshotForAI, } from "./pw-session.js";
|
|
2
2
|
export { armDialogViaPlaywright, armFileUploadViaPlaywright, clickViaPlaywright, closePageViaPlaywright, cookiesClearViaPlaywright, cookiesGetViaPlaywright, cookiesSetViaPlaywright, downloadViaPlaywright, dragViaPlaywright, emulateMediaViaPlaywright, evaluateViaPlaywright, fillFormViaPlaywright, getConsoleMessagesViaPlaywright, getNetworkRequestsViaPlaywright, getPageErrorsViaPlaywright, highlightViaPlaywright, hoverViaPlaywright, navigateViaPlaywright, pdfViaPlaywright, pressKeyViaPlaywright, resizeViewportViaPlaywright, responseBodyViaPlaywright, scrollIntoViewViaPlaywright, selectOptionViaPlaywright, setDeviceViaPlaywright, setExtraHTTPHeadersViaPlaywright, setGeolocationViaPlaywright, setHttpCredentialsViaPlaywright, setInputFilesViaPlaywright, setLocaleViaPlaywright, setOfflineViaPlaywright, setTimezoneViaPlaywright, snapshotAiViaPlaywright, snapshotAriaViaPlaywright, snapshotRoleViaPlaywright, screenshotWithLabelsViaPlaywright, storageClearViaPlaywright, storageGetViaPlaywright, storageSetViaPlaywright, takeScreenshotViaPlaywright, traceStartViaPlaywright, traceStopViaPlaywright, typeViaPlaywright, waitForDownloadViaPlaywright, waitForViaPlaywright, } from "./pw-tools-core.js";
|
|
3
|
+
export { type FileInputInfo, type ListFileInputsResult, type AttachFileResult, listFileInputsViaPlaywright, attachFileToInputViaPlaywright, isFileTypeAccepted, } from "./pw-tools-core.uploads.js";
|
package/dist/browser/pw-ai.js
CHANGED
|
@@ -1,2 +1,3 @@
|
|
|
1
1
|
export { closePageByTargetIdViaPlaywright, closePlaywrightBrowserConnection, createPageViaPlaywright, ensurePageState, focusPageByTargetIdViaPlaywright, getPageForTargetId, listPagesViaPlaywright, refLocator, } from "./pw-session.js";
|
|
2
2
|
export { armDialogViaPlaywright, armFileUploadViaPlaywright, clickViaPlaywright, closePageViaPlaywright, cookiesClearViaPlaywright, cookiesGetViaPlaywright, cookiesSetViaPlaywright, downloadViaPlaywright, dragViaPlaywright, emulateMediaViaPlaywright, evaluateViaPlaywright, fillFormViaPlaywright, getConsoleMessagesViaPlaywright, getNetworkRequestsViaPlaywright, getPageErrorsViaPlaywright, highlightViaPlaywright, hoverViaPlaywright, navigateViaPlaywright, pdfViaPlaywright, pressKeyViaPlaywright, resizeViewportViaPlaywright, responseBodyViaPlaywright, scrollIntoViewViaPlaywright, selectOptionViaPlaywright, setDeviceViaPlaywright, setExtraHTTPHeadersViaPlaywright, setGeolocationViaPlaywright, setHttpCredentialsViaPlaywright, setInputFilesViaPlaywright, setLocaleViaPlaywright, setOfflineViaPlaywright, setTimezoneViaPlaywright, snapshotAiViaPlaywright, snapshotAriaViaPlaywright, snapshotRoleViaPlaywright, screenshotWithLabelsViaPlaywright, storageClearViaPlaywright, storageGetViaPlaywright, storageSetViaPlaywright, takeScreenshotViaPlaywright, traceStartViaPlaywright, traceStopViaPlaywright, typeViaPlaywright, waitForDownloadViaPlaywright, waitForViaPlaywright, } from "./pw-tools-core.js";
|
|
3
|
+
export { listFileInputsViaPlaywright, attachFileToInputViaPlaywright, isFileTypeAccepted, } from "./pw-tools-core.uploads.js";
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File upload discovery and attachment functions for browser automation.
|
|
3
|
+
*
|
|
4
|
+
* These functions support the LLM-driven file upload workflow:
|
|
5
|
+
* 1. listFileInputsViaPlaywright() - discovers all file inputs on page + iframes
|
|
6
|
+
* 2. attachFileToInputViaPlaywright() - uploads file to a specific input by index
|
|
7
|
+
*/
|
|
8
|
+
export type FileInputInfo = {
|
|
9
|
+
index: number;
|
|
10
|
+
id: string;
|
|
11
|
+
name: string;
|
|
12
|
+
accept: string;
|
|
13
|
+
multiple: boolean;
|
|
14
|
+
frameUrl: string;
|
|
15
|
+
nearbyText: string;
|
|
16
|
+
ariaLabel: string | null;
|
|
17
|
+
};
|
|
18
|
+
export type ListFileInputsResult = {
|
|
19
|
+
inputs: FileInputInfo[];
|
|
20
|
+
targetId?: string;
|
|
21
|
+
};
|
|
22
|
+
export type AttachFileResult = {
|
|
23
|
+
success: boolean;
|
|
24
|
+
inputIndex: number;
|
|
25
|
+
inputId: string;
|
|
26
|
+
fileName: string;
|
|
27
|
+
frameUrl: string;
|
|
28
|
+
};
|
|
29
|
+
/**
|
|
30
|
+
* Discovers all file inputs on the page and all iframes.
|
|
31
|
+
* Returns structured metadata that the LLM can use to decide which input to target.
|
|
32
|
+
*/
|
|
33
|
+
export declare function listFileInputsViaPlaywright(opts: {
|
|
34
|
+
cdpUrl: string;
|
|
35
|
+
targetId?: string;
|
|
36
|
+
}): Promise<ListFileInputsResult>;
|
|
37
|
+
/**
|
|
38
|
+
* Attaches a file to a specific file input by index.
|
|
39
|
+
* The index corresponds to the index returned by listFileInputsViaPlaywright.
|
|
40
|
+
*/
|
|
41
|
+
export declare function attachFileToInputViaPlaywright(opts: {
|
|
42
|
+
cdpUrl: string;
|
|
43
|
+
targetId?: string;
|
|
44
|
+
inputIndex: number;
|
|
45
|
+
filePath: string;
|
|
46
|
+
}): Promise<AttachFileResult>;
|
|
47
|
+
/**
|
|
48
|
+
* Validates that a file extension matches an accept attribute.
|
|
49
|
+
* Returns true if the file is allowed, false otherwise.
|
|
50
|
+
*/
|
|
51
|
+
export declare function isFileTypeAccepted(filePath: string, accept: string): boolean;
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File upload discovery and attachment functions for browser automation.
|
|
3
|
+
*
|
|
4
|
+
* These functions support the LLM-driven file upload workflow:
|
|
5
|
+
* 1. listFileInputsViaPlaywright() - discovers all file inputs on page + iframes
|
|
6
|
+
* 2. attachFileToInputViaPlaywright() - uploads file to a specific input by index
|
|
7
|
+
*/
|
|
8
|
+
import { existsSync } from "node:fs";
|
|
9
|
+
import { basename, extname } from "node:path";
|
|
10
|
+
import { getPageForTargetId, ensurePageState } from "./pw-session.js";
|
|
11
|
+
/**
|
|
12
|
+
* Extracts metadata about file inputs from a page or frame.
|
|
13
|
+
*/
|
|
14
|
+
async function extractFileInputsFromFrame(frame, frameUrl, startIndex) {
|
|
15
|
+
try {
|
|
16
|
+
const inputs = await frame.$$("input[type='file']");
|
|
17
|
+
const results = [];
|
|
18
|
+
for (let i = 0; i < inputs.length; i++) {
|
|
19
|
+
try {
|
|
20
|
+
const info = await inputs[i].evaluate((el) => {
|
|
21
|
+
// Get nearby text for context (parent, siblings, labels)
|
|
22
|
+
let nearbyText = "";
|
|
23
|
+
// Check for associated label
|
|
24
|
+
const labelFor = el.id ? document.querySelector(`label[for="${el.id}"]`) : null;
|
|
25
|
+
if (labelFor) {
|
|
26
|
+
nearbyText = labelFor.textContent?.trim().substring(0, 100) || "";
|
|
27
|
+
}
|
|
28
|
+
// Check parent for text
|
|
29
|
+
if (!nearbyText && el.parentElement) {
|
|
30
|
+
const parentText = el.parentElement.textContent?.trim() || "";
|
|
31
|
+
// Only use if not too long (avoid grabbing entire page text)
|
|
32
|
+
if (parentText.length < 200) {
|
|
33
|
+
nearbyText = parentText.substring(0, 100);
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
// Check previous sibling
|
|
37
|
+
if (!nearbyText && el.previousElementSibling) {
|
|
38
|
+
nearbyText = el.previousElementSibling.textContent?.trim().substring(0, 100) || "";
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
id: el.id || "",
|
|
42
|
+
name: el.name || "",
|
|
43
|
+
accept: el.accept || "",
|
|
44
|
+
multiple: el.multiple,
|
|
45
|
+
ariaLabel: el.getAttribute("aria-label"),
|
|
46
|
+
nearbyText,
|
|
47
|
+
};
|
|
48
|
+
});
|
|
49
|
+
results.push({
|
|
50
|
+
index: startIndex + i,
|
|
51
|
+
id: info.id,
|
|
52
|
+
name: info.name,
|
|
53
|
+
accept: info.accept,
|
|
54
|
+
multiple: info.multiple,
|
|
55
|
+
frameUrl,
|
|
56
|
+
nearbyText: info.nearbyText,
|
|
57
|
+
ariaLabel: info.ariaLabel,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
// Skip inputs that can't be accessed
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
return results;
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
return [];
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Discovers all file inputs on the page and all iframes.
|
|
72
|
+
* Returns structured metadata that the LLM can use to decide which input to target.
|
|
73
|
+
*/
|
|
74
|
+
export async function listFileInputsViaPlaywright(opts) {
|
|
75
|
+
const page = await getPageForTargetId(opts);
|
|
76
|
+
ensurePageState(page);
|
|
77
|
+
const allInputs = [];
|
|
78
|
+
// Get inputs from main page
|
|
79
|
+
const mainInputs = await extractFileInputsFromFrame(page, page.url(), 0);
|
|
80
|
+
allInputs.push(...mainInputs);
|
|
81
|
+
// Get inputs from all frames
|
|
82
|
+
const frames = page.frames();
|
|
83
|
+
for (const frame of frames) {
|
|
84
|
+
// Skip main frame (already processed) and empty frames
|
|
85
|
+
if (frame === page.mainFrame()) {
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
const frameUrl = frame.url();
|
|
89
|
+
if (!frameUrl || frameUrl === "about:blank") {
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
92
|
+
const frameInputs = await extractFileInputsFromFrame(frame, frameUrl, allInputs.length);
|
|
93
|
+
allInputs.push(...frameInputs);
|
|
94
|
+
}
|
|
95
|
+
return {
|
|
96
|
+
inputs: allInputs,
|
|
97
|
+
targetId: opts.targetId,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Internal: find the file input element by index across page and iframes.
|
|
102
|
+
*/
|
|
103
|
+
async function findInputByIndex(page, targetIndex) {
|
|
104
|
+
let currentIndex = 0;
|
|
105
|
+
// Check main page first
|
|
106
|
+
const mainInputs = await page.$$("input[type='file']");
|
|
107
|
+
if (targetIndex < currentIndex + mainInputs.length) {
|
|
108
|
+
const localIndex = targetIndex - currentIndex;
|
|
109
|
+
return { element: mainInputs[localIndex], frame: page };
|
|
110
|
+
}
|
|
111
|
+
currentIndex += mainInputs.length;
|
|
112
|
+
// Check frames
|
|
113
|
+
const frames = page.frames();
|
|
114
|
+
for (const frame of frames) {
|
|
115
|
+
if (frame === page.mainFrame()) {
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
const frameUrl = frame.url();
|
|
119
|
+
if (!frameUrl || frameUrl === "about:blank") {
|
|
120
|
+
continue;
|
|
121
|
+
}
|
|
122
|
+
try {
|
|
123
|
+
const frameInputs = await frame.$$("input[type='file']");
|
|
124
|
+
if (targetIndex < currentIndex + frameInputs.length) {
|
|
125
|
+
const localIndex = targetIndex - currentIndex;
|
|
126
|
+
return { element: frameInputs[localIndex], frame };
|
|
127
|
+
}
|
|
128
|
+
currentIndex += frameInputs.length;
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
// Frame might be detached, continue
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Attaches a file to a specific file input by index.
|
|
138
|
+
* The index corresponds to the index returned by listFileInputsViaPlaywright.
|
|
139
|
+
*/
|
|
140
|
+
export async function attachFileToInputViaPlaywright(opts) {
|
|
141
|
+
const page = await getPageForTargetId(opts);
|
|
142
|
+
ensurePageState(page);
|
|
143
|
+
// Validate file exists
|
|
144
|
+
if (!existsSync(opts.filePath)) {
|
|
145
|
+
throw new Error(`File not found: ${opts.filePath}`);
|
|
146
|
+
}
|
|
147
|
+
// Find the input by index
|
|
148
|
+
const found = await findInputByIndex(page, opts.inputIndex);
|
|
149
|
+
if (!found || !found.element) {
|
|
150
|
+
throw new Error(`No file input found at index ${opts.inputIndex}. Run list_upload_inputs first.`);
|
|
151
|
+
}
|
|
152
|
+
const { element, frame } = found;
|
|
153
|
+
// Get input ID for the response
|
|
154
|
+
const inputId = await element.evaluate((el) => el.id || "(no id)");
|
|
155
|
+
// Set the file
|
|
156
|
+
await element.setInputFiles(opts.filePath);
|
|
157
|
+
// Dispatch change/input events for sites that need them
|
|
158
|
+
try {
|
|
159
|
+
await element.evaluate((el) => {
|
|
160
|
+
el.dispatchEvent(new Event("input", { bubbles: true }));
|
|
161
|
+
el.dispatchEvent(new Event("change", { bubbles: true }));
|
|
162
|
+
});
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
// Best-effort event dispatch
|
|
166
|
+
}
|
|
167
|
+
return {
|
|
168
|
+
success: true,
|
|
169
|
+
inputIndex: opts.inputIndex,
|
|
170
|
+
inputId,
|
|
171
|
+
fileName: basename(opts.filePath),
|
|
172
|
+
frameUrl: "url" in frame && typeof frame.url === "function" ? frame.url() : page.url(),
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Validates that a file extension matches an accept attribute.
|
|
177
|
+
* Returns true if the file is allowed, false otherwise.
|
|
178
|
+
*/
|
|
179
|
+
export function isFileTypeAccepted(filePath, accept) {
|
|
180
|
+
if (!accept || accept.trim() === "") {
|
|
181
|
+
return true; // No restriction
|
|
182
|
+
}
|
|
183
|
+
const fileExt = extname(filePath).toLowerCase();
|
|
184
|
+
const acceptedTypes = accept.split(",").map((t) => t.trim().toLowerCase());
|
|
185
|
+
for (const accepted of acceptedTypes) {
|
|
186
|
+
// Handle extension matches (.pdf, .doc, etc)
|
|
187
|
+
if (accepted.startsWith(".") && fileExt === accepted) {
|
|
188
|
+
return true;
|
|
189
|
+
}
|
|
190
|
+
// Handle MIME type matches (application/pdf, image/*, etc)
|
|
191
|
+
if (accepted.includes("/")) {
|
|
192
|
+
// Map common extensions to MIME types for basic matching
|
|
193
|
+
const mimeMap = {
|
|
194
|
+
".pdf": ["application/pdf"],
|
|
195
|
+
".doc": ["application/msword"],
|
|
196
|
+
".docx": ["application/vnd.openxmlformats-officedocument.wordprocessingml.document"],
|
|
197
|
+
".xls": ["application/vnd.ms-excel"],
|
|
198
|
+
".xlsx": ["application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"],
|
|
199
|
+
".png": ["image/png"],
|
|
200
|
+
".jpg": ["image/jpeg"],
|
|
201
|
+
".jpeg": ["image/jpeg"],
|
|
202
|
+
".gif": ["image/gif"],
|
|
203
|
+
".webp": ["image/webp"],
|
|
204
|
+
".txt": ["text/plain"],
|
|
205
|
+
".md": ["text/markdown", "text/plain"],
|
|
206
|
+
".csv": ["text/csv"],
|
|
207
|
+
};
|
|
208
|
+
const fileMimes = mimeMap[fileExt] || [];
|
|
209
|
+
if (accepted.endsWith("/*")) {
|
|
210
|
+
// Wildcard like image/*
|
|
211
|
+
const prefix = accepted.replace("/*", "/");
|
|
212
|
+
if (fileMimes.some((m) => m.startsWith(prefix))) {
|
|
213
|
+
return true;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
else {
|
|
217
|
+
// Exact MIME match
|
|
218
|
+
if (fileMimes.includes(accepted)) {
|
|
219
|
+
return true;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
return false;
|
|
225
|
+
}
|
|
@@ -474,6 +474,61 @@ export function registerBrowserAgentActRoutes(app, ctx) {
|
|
|
474
474
|
handleRouteError(ctx, res, err);
|
|
475
475
|
}
|
|
476
476
|
});
|
|
477
|
+
app.get("/upload-inputs", async (req, res) => {
|
|
478
|
+
const profileCtx = resolveProfileContext(req, res, ctx);
|
|
479
|
+
if (!profileCtx) {
|
|
480
|
+
return;
|
|
481
|
+
}
|
|
482
|
+
const targetId = toStringOrEmpty(req.query.targetId) || undefined;
|
|
483
|
+
try {
|
|
484
|
+
const tab = await profileCtx.ensureTabAvailable(targetId);
|
|
485
|
+
const pw = await requirePwAi(res, "list upload inputs");
|
|
486
|
+
if (!pw) {
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
const result = await pw.listFileInputsViaPlaywright({
|
|
490
|
+
cdpUrl: profileCtx.profile.cdpUrl,
|
|
491
|
+
targetId: tab.targetId,
|
|
492
|
+
});
|
|
493
|
+
res.json({ ok: true, targetId: tab.targetId, ...result });
|
|
494
|
+
}
|
|
495
|
+
catch (err) {
|
|
496
|
+
handleRouteError(ctx, res, err);
|
|
497
|
+
}
|
|
498
|
+
});
|
|
499
|
+
app.post("/attach-file", async (req, res) => {
|
|
500
|
+
const profileCtx = resolveProfileContext(req, res, ctx);
|
|
501
|
+
if (!profileCtx) {
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
const body = readBody(req);
|
|
505
|
+
const targetId = toStringOrEmpty(body.targetId) || undefined;
|
|
506
|
+
const inputIndex = toNumber(body.inputIndex);
|
|
507
|
+
const filePath = toStringOrEmpty(body.filePath);
|
|
508
|
+
if (inputIndex === undefined || inputIndex === null) {
|
|
509
|
+
return jsonError(res, 400, "inputIndex is required");
|
|
510
|
+
}
|
|
511
|
+
if (!filePath) {
|
|
512
|
+
return jsonError(res, 400, "filePath is required");
|
|
513
|
+
}
|
|
514
|
+
try {
|
|
515
|
+
const tab = await profileCtx.ensureTabAvailable(targetId);
|
|
516
|
+
const pw = await requirePwAi(res, "attach file");
|
|
517
|
+
if (!pw) {
|
|
518
|
+
return;
|
|
519
|
+
}
|
|
520
|
+
const result = await pw.attachFileToInputViaPlaywright({
|
|
521
|
+
cdpUrl: profileCtx.profile.cdpUrl,
|
|
522
|
+
targetId: tab.targetId,
|
|
523
|
+
inputIndex,
|
|
524
|
+
filePath,
|
|
525
|
+
});
|
|
526
|
+
res.json({ ok: true, targetId: tab.targetId, ...result });
|
|
527
|
+
}
|
|
528
|
+
catch (err) {
|
|
529
|
+
handleRouteError(ctx, res, err);
|
|
530
|
+
}
|
|
531
|
+
});
|
|
477
532
|
app.post("/highlight", async (req, res) => {
|
|
478
533
|
const profileCtx = resolveProfileContext(req, res, ctx);
|
|
479
534
|
if (!profileCtx) {
|
package/dist/browser/server.js
CHANGED
|
@@ -48,10 +48,10 @@ export async function startBrowserControlServerFromConfig() {
|
|
|
48
48
|
continue;
|
|
49
49
|
}
|
|
50
50
|
await ensureChromeExtensionRelayServer({ cdpUrl: profile.cdpUrl }).catch((err) => {
|
|
51
|
-
logServer.
|
|
51
|
+
logServer.debug(`Chrome extension relay init failed for profile "${name}": ${String(err)}`);
|
|
52
52
|
});
|
|
53
53
|
}
|
|
54
|
-
logServer.
|
|
54
|
+
logServer.debug(`Browser control listening on http://127.0.0.1:${port}/`);
|
|
55
55
|
return state;
|
|
56
56
|
}
|
|
57
57
|
export async function stopBrowserControlServer() {
|
package/dist/build-info.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
6501b8ff92b04fbf4088b742a68e2e4f6339f3f5ee75fe23c1e2dc6c8ea9fb70
|
package/dist/cli/banner.d.ts
CHANGED
package/dist/cli/banner.js
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import { resolveCommitHash } from "../infra/git-commit.js";
|
|
2
1
|
import { visibleWidth } from "../terminal/ansi.js";
|
|
3
2
|
import { isRich, theme } from "../terminal/theme.js";
|
|
4
3
|
import { pickTagline } from "./tagline.js";
|
|
@@ -20,27 +19,25 @@ function splitGraphemes(value) {
|
|
|
20
19
|
const hasJsonFlag = (argv) => argv.some((arg) => arg === "--json" || arg.startsWith("--json="));
|
|
21
20
|
const hasVersionFlag = (argv) => argv.some((arg) => arg === "--version" || arg === "-V" || arg === "-v");
|
|
22
21
|
export function formatCliBannerLine(version, options = {}) {
|
|
23
|
-
const commit = options.commit ?? resolveCommitHash({ env: options.env });
|
|
24
|
-
const commitLabel = commit ?? "unknown";
|
|
25
22
|
const tagline = pickTagline(options);
|
|
26
23
|
const rich = options.richTty ?? isRich();
|
|
27
24
|
const title = "🦞 AGIAgent";
|
|
28
25
|
const prefix = "🦞 ";
|
|
29
26
|
const columns = options.columns ?? process.stdout.columns ?? 120;
|
|
30
|
-
const plainFullLine = `${title} ${version}
|
|
27
|
+
const plainFullLine = `${title} ${version} — ${tagline}`;
|
|
31
28
|
const fitsOnOneLine = visibleWidth(plainFullLine) <= columns;
|
|
32
29
|
if (rich) {
|
|
33
30
|
if (fitsOnOneLine) {
|
|
34
|
-
return `${theme.heading(title)} ${theme.info(version)} ${theme.muted(
|
|
31
|
+
return `${theme.heading(title)} ${theme.info(version)} ${theme.muted("—")} ${theme.accentDim(tagline)}`;
|
|
35
32
|
}
|
|
36
|
-
const line1 = `${theme.heading(title)} ${theme.info(version)}
|
|
33
|
+
const line1 = `${theme.heading(title)} ${theme.info(version)}`;
|
|
37
34
|
const line2 = `${" ".repeat(prefix.length)}${theme.accentDim(tagline)}`;
|
|
38
35
|
return `${line1}\n${line2}`;
|
|
39
36
|
}
|
|
40
37
|
if (fitsOnOneLine) {
|
|
41
38
|
return plainFullLine;
|
|
42
39
|
}
|
|
43
|
-
const line1 = `${title} ${version}
|
|
40
|
+
const line1 = `${title} ${version}`;
|
|
44
41
|
const line2 = `${" ".repeat(prefix.length)}${tagline}`;
|
|
45
42
|
return `${line1}\n${line2}`;
|
|
46
43
|
}
|