bangonit 0.4.3 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/app/desktopapp/dist/main/index.js +10 -4
- package/app/desktopapp/dist/main/ipc.js +9 -24
- package/app/desktopapp/dist/main/preload.js +0 -7
- package/app/desktopapp/dist/main/tabs.js +10 -5
- package/app/desktopapp/package.json +0 -1
- package/app/replay/dist/replay.js +2 -2
- package/app/webapp/.next/standalone/app/webapp/.next/BUILD_ID +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/app-build-manifest.json +11 -11
- package/app/webapp/.next/standalone/app/webapp/.next/app-path-routes-manifest.json +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/build-manifest.json +3 -3
- package/app/webapp/.next/standalone/app/webapp/.next/prerender-manifest.json +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/required-server-files.json +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/_not-found/page.js +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/_not-found/page_client-reference-manifest.js +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/_not-found.html +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/_not-found.rsc +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/api/chat/route.js +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/api/screenshot/route.js +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/app/page.js +6 -6
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/app/page_client-reference-manifest.js +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/app.html +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/app.rsc +2 -2
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/index.html +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/index.rsc +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/page.js +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app/page_client-reference-manifest.js +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/app-paths-manifest.json +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/chunks/679.js +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/chunks/708.js +3 -3
- package/app/webapp/.next/standalone/app/webapp/.next/server/middleware-build-manifest.js +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/pages/404.html +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/pages/500.html +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/server/server-reference-manifest.json +1 -1
- package/app/webapp/.next/standalone/app/webapp/.next/static/chunks/app/app/page-533a30559a8f39fa.js +1 -0
- package/app/webapp/.next/standalone/app/webapp/.next/static/chunks/app/layout-40f50d9380154ecf.js +1 -0
- package/app/webapp/.next/{static/chunks/main-app-106dd83f859b9dfa.js → standalone/app/webapp/.next/static/chunks/main-app-76384b941f0b51cb.js} +1 -1
- package/app/webapp/.next/standalone/app/webapp/package.json +2 -6
- package/app/webapp/.next/standalone/app/webapp/server.js +1 -1
- package/app/webapp/.next/standalone/package.json +18 -6
- package/app/webapp/.next/static/chunks/app/app/page-533a30559a8f39fa.js +1 -0
- package/app/webapp/.next/static/chunks/app/layout-40f50d9380154ecf.js +1 -0
- package/app/webapp/.next/{standalone/app/webapp/.next/static/chunks/main-app-106dd83f859b9dfa.js → static/chunks/main-app-76384b941f0b51cb.js} +1 -1
- package/app/webapp/package.json +2 -6
- package/app/webapp/skills/document-review.md +1 -0
- package/app/webapp/skills/gmail.md +2 -0
- package/app/webapp/src/app/globals.css +8 -3
- package/app/webapp/src/app/layout.tsx +2 -8
- package/app/webapp/src/shared/api/chat.ts +49 -25
- package/app/webapp/src/shared/api/screenshot.ts +11 -10
- package/app/webapp/src/shared/components/AppShell.tsx +80 -109
- package/app/webapp/src/shared/components/SessionView.tsx +335 -248
- package/app/webapp/src/shared/components/VirtualCursor.tsx +13 -14
- package/app/webapp/src/shared/lib/browser/cursor.ts +2 -7
- package/app/webapp/src/shared/lib/browser/index.ts +56 -36
- package/app/webapp/src/shared/lib/browser/mouse.ts +86 -21
- package/app/webapp/src/shared/lib/browser/navigate.ts +1 -4
- package/app/webapp/src/shared/lib/browser/recorder.ts +12 -5
- package/app/webapp/src/shared/lib/browser/screenshot.ts +4 -4
- package/app/webapp/src/shared/lib/browser/snapshot.ts +9 -5
- package/app/webapp/src/shared/lib/browser/tabs.ts +1 -1
- package/app/webapp/src/shared/lib/browser/types.ts +3 -2
- package/app/webapp/src/shared/lib/browser/wait.ts +1 -1
- package/app/webapp/src/shared/lib/recorder/session-recorder.ts +1 -1
- package/app/webapp/src/shared/types/global.d.ts +8 -19
- package/app/webapp/tailwind.config.js +1 -3
- package/bin/src/cli/bangonit.js +270 -177
- package/package.json +18 -6
- package/app/webapp/.next/standalone/app/webapp/.next/static/chunks/app/app/page-03dbc2fc67c26b74.js +0 -1
- package/app/webapp/.next/standalone/app/webapp/.next/static/chunks/app/layout-57acb80d8da0067a.js +0 -1
- package/app/webapp/.next/static/chunks/app/app/page-03dbc2fc67c26b74.js +0 -1
- package/app/webapp/.next/static/chunks/app/layout-57acb80d8da0067a.js +0 -1
- /package/app/webapp/.next/standalone/app/webapp/.next/static/{z2gRF0NKwztPLZ9d7ok06 → kz1a_SRPtSly3Fe8wHKDq}/_buildManifest.js +0 -0
- /package/app/webapp/.next/standalone/app/webapp/.next/static/{z2gRF0NKwztPLZ9d7ok06 → kz1a_SRPtSly3Fe8wHKDq}/_ssgManifest.js +0 -0
- /package/app/webapp/.next/static/{z2gRF0NKwztPLZ9d7ok06 → kz1a_SRPtSly3Fe8wHKDq}/_buildManifest.js +0 -0
- /package/app/webapp/.next/static/{z2gRF0NKwztPLZ9d7ok06 → kz1a_SRPtSly3Fe8wHKDq}/_ssgManifest.js +0 -0
|
@@ -70,10 +70,11 @@ export default function VirtualCursor({ agentId }: { agentId: string }) {
|
|
|
70
70
|
|
|
71
71
|
stateRef.current.rafId = requestAnimationFrame(render);
|
|
72
72
|
};
|
|
73
|
-
stateRef.current
|
|
73
|
+
const state = stateRef.current;
|
|
74
|
+
state.rafId = requestAnimationFrame(render);
|
|
74
75
|
|
|
75
76
|
return () => {
|
|
76
|
-
cancelAnimationFrame(
|
|
77
|
+
cancelAnimationFrame(state.rafId);
|
|
77
78
|
ro.disconnect();
|
|
78
79
|
};
|
|
79
80
|
}, []);
|
|
@@ -83,15 +84,16 @@ export default function VirtualCursor({ agentId }: { agentId: string }) {
|
|
|
83
84
|
const cleanups: (() => void)[] = [];
|
|
84
85
|
const s = stateRef.current;
|
|
85
86
|
|
|
86
|
-
|
|
87
|
+
const moveCb = window.bangonit?.onCursorMove((data) => {
|
|
87
88
|
if (data.agentId !== agentId) return;
|
|
88
89
|
s.x = data.x;
|
|
89
90
|
s.y = data.y;
|
|
90
91
|
s.hasPos = true;
|
|
91
92
|
markActive();
|
|
92
|
-
})
|
|
93
|
+
});
|
|
94
|
+
if (moveCb) cleanups.push(moveCb);
|
|
93
95
|
|
|
94
|
-
|
|
96
|
+
const downCb = window.bangonit?.onCursorDown((data) => {
|
|
95
97
|
if (data.agentId !== agentId) return;
|
|
96
98
|
s.x = data.x;
|
|
97
99
|
s.y = data.y;
|
|
@@ -99,23 +101,20 @@ export default function VirtualCursor({ agentId }: { agentId: string }) {
|
|
|
99
101
|
s.clicking = true;
|
|
100
102
|
s.ripples.push({ x: data.x, y: data.y, startTs: Date.now() });
|
|
101
103
|
markActive();
|
|
102
|
-
})
|
|
104
|
+
});
|
|
105
|
+
if (downCb) cleanups.push(downCb);
|
|
103
106
|
|
|
104
|
-
|
|
107
|
+
const upCb = window.bangonit?.onCursorUp((data) => {
|
|
105
108
|
if (data.agentId !== agentId) return;
|
|
106
109
|
s.clicking = false;
|
|
107
110
|
markActive();
|
|
108
|
-
})
|
|
111
|
+
});
|
|
112
|
+
if (upCb) cleanups.push(upCb);
|
|
109
113
|
|
|
110
114
|
return () => {
|
|
111
115
|
cleanups.forEach((fn) => fn?.());
|
|
112
116
|
};
|
|
113
117
|
}, [agentId, markActive]);
|
|
114
118
|
|
|
115
|
-
return
|
|
116
|
-
<canvas
|
|
117
|
-
ref={canvasRef}
|
|
118
|
-
className="pointer-events-none absolute inset-0 z-20"
|
|
119
|
-
/>
|
|
120
|
-
);
|
|
119
|
+
return <canvas ref={canvasRef} className="pointer-events-none absolute inset-0 z-20" />;
|
|
121
120
|
}
|
|
@@ -18,7 +18,7 @@ export function drawCursor(
|
|
|
18
18
|
x: number,
|
|
19
19
|
y: number,
|
|
20
20
|
clicking: boolean,
|
|
21
|
-
opacity: number = 1
|
|
21
|
+
opacity: number = 1,
|
|
22
22
|
): void {
|
|
23
23
|
if (opacity <= 0) return;
|
|
24
24
|
ctx.save();
|
|
@@ -48,12 +48,7 @@ export function drawCursor(
|
|
|
48
48
|
ctx.restore();
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
-
export function drawRipple(
|
|
52
|
-
ctx: CanvasRenderingContext2D,
|
|
53
|
-
x: number,
|
|
54
|
-
y: number,
|
|
55
|
-
progress: number
|
|
56
|
-
): void {
|
|
51
|
+
export function drawRipple(ctx: CanvasRenderingContext2D, x: number, y: number, progress: number): void {
|
|
57
52
|
if (progress >= 1) return;
|
|
58
53
|
|
|
59
54
|
// Filled circle
|
|
@@ -9,7 +9,7 @@ import type { ScreenshotResult, BrowserToolOutput } from "./schemas";
|
|
|
9
9
|
import { navigate, goBack, goForward } from "./navigate";
|
|
10
10
|
import { tabAction, getTabSummary } from "./tabs";
|
|
11
11
|
import { waitFor, NetworkIdleTracker } from "./wait";
|
|
12
|
-
import type { BrowserAction, BrowserToolInput
|
|
12
|
+
import type { BrowserAction, BrowserToolInput } from "./types";
|
|
13
13
|
import { ACTION_TO_TOOL } from "./types";
|
|
14
14
|
import type { BrowserRecorder } from "./recorder";
|
|
15
15
|
|
|
@@ -19,7 +19,9 @@ let globalLock: Promise<void> = Promise.resolve();
|
|
|
19
19
|
|
|
20
20
|
function acquireLock(): { promise: Promise<void>; release: () => void } {
|
|
21
21
|
let release!: () => void;
|
|
22
|
-
const next = new Promise<void>((resolve) => {
|
|
22
|
+
const next = new Promise<void>((resolve) => {
|
|
23
|
+
release = resolve;
|
|
24
|
+
});
|
|
23
25
|
const promise = globalLock;
|
|
24
26
|
globalLock = globalLock.then(() => next);
|
|
25
27
|
return { promise, release };
|
|
@@ -81,7 +83,7 @@ export class BrowserTools {
|
|
|
81
83
|
args.steps,
|
|
82
84
|
this.mouseX,
|
|
83
85
|
this.mouseY,
|
|
84
|
-
this.cursorCallbacks()
|
|
86
|
+
this.cursorCallbacks(),
|
|
85
87
|
);
|
|
86
88
|
this.mouseX = mouseX;
|
|
87
89
|
this.mouseY = mouseY;
|
|
@@ -162,23 +164,21 @@ export class BrowserTools {
|
|
|
162
164
|
}
|
|
163
165
|
|
|
164
166
|
// Run observation after all actions complete
|
|
165
|
-
if
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
}
|
|
167
|
+
// Wait for network idle before observing if there were preceding actions
|
|
168
|
+
if (actions.length > 0) {
|
|
169
|
+
await this.networkTracker.waitForIdle(500, 2000);
|
|
170
|
+
}
|
|
170
171
|
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
172
|
+
try {
|
|
173
|
+
const tabSummary = await getTabSummary(this.agentId);
|
|
174
|
+
const snapshotResult = await takeSnapshot(this.agentId, this.mouseX, this.mouseY, tabSummary);
|
|
175
|
+
results.push(snapshotResult);
|
|
175
176
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
}
|
|
179
|
-
} catch (err: any) {
|
|
180
|
-
results.push(`observe: Error — ${err.message}`);
|
|
177
|
+
if (observe === "snapshot_and_screenshot") {
|
|
178
|
+
imageOutput = await takeScreenshot(this.agentId);
|
|
181
179
|
}
|
|
180
|
+
} catch (err: any) {
|
|
181
|
+
results.push(`observe: Error — ${err.message}`);
|
|
182
182
|
}
|
|
183
183
|
|
|
184
184
|
// Stop recording clip (base64 data is stored in the clip meta)
|
|
@@ -195,8 +195,12 @@ export class BrowserTools {
|
|
|
195
195
|
|
|
196
196
|
private actionToToolArgs(act: BrowserAction): any {
|
|
197
197
|
switch (act.action) {
|
|
198
|
-
case "navigate":
|
|
199
|
-
|
|
198
|
+
case "navigate":
|
|
199
|
+
return { url: act.url };
|
|
200
|
+
case "back":
|
|
201
|
+
case "forward":
|
|
202
|
+
case "close":
|
|
203
|
+
return {};
|
|
200
204
|
case "mouse": {
|
|
201
205
|
const steps = (act.mouseActions || []).map((s) => {
|
|
202
206
|
if (s.action === "wait") return { action: "wait", ms: s.ms || 100 };
|
|
@@ -206,33 +210,49 @@ export class BrowserTools {
|
|
|
206
210
|
});
|
|
207
211
|
return { steps };
|
|
208
212
|
}
|
|
209
|
-
case "type":
|
|
210
|
-
|
|
211
|
-
case "
|
|
212
|
-
|
|
213
|
-
case "
|
|
214
|
-
|
|
213
|
+
case "type":
|
|
214
|
+
return { text: act.text };
|
|
215
|
+
case "press":
|
|
216
|
+
return { key: act.key };
|
|
217
|
+
case "upload":
|
|
218
|
+
return { paths: act.paths, cancel: act.cancel };
|
|
219
|
+
case "tabs":
|
|
220
|
+
return { action: act.tabAction, tabId: act.tabId, url: act.url };
|
|
221
|
+
case "wait":
|
|
222
|
+
return { timeout: act.timeout };
|
|
223
|
+
default:
|
|
224
|
+
return {};
|
|
215
225
|
}
|
|
216
226
|
}
|
|
217
227
|
|
|
218
228
|
private actionLabel(act: BrowserAction): string {
|
|
219
229
|
switch (act.action) {
|
|
220
|
-
case "navigate":
|
|
230
|
+
case "navigate":
|
|
231
|
+
return `navigate ${act.url}`;
|
|
221
232
|
case "mouse": {
|
|
222
233
|
const summary = (act.mouseActions || [])
|
|
223
234
|
.map((s) =>
|
|
224
|
-
s.action === "wait"
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
235
|
+
s.action === "wait"
|
|
236
|
+
? `wait ${s.ms}ms`
|
|
237
|
+
: s.action === "wheel"
|
|
238
|
+
? `wheel ${s.x},${s.y} ${s.dx},${s.dy}`
|
|
239
|
+
: `${s.action} ${s.x},${s.y}`,
|
|
240
|
+
)
|
|
241
|
+
.join(" → ");
|
|
228
242
|
return `mouse ${summary}`;
|
|
229
243
|
}
|
|
230
|
-
case "type":
|
|
231
|
-
|
|
232
|
-
case "
|
|
233
|
-
|
|
234
|
-
case "
|
|
235
|
-
|
|
244
|
+
case "type":
|
|
245
|
+
return `type "${(act.text || "").slice(0, 30)}"`;
|
|
246
|
+
case "press":
|
|
247
|
+
return `press ${act.key}`;
|
|
248
|
+
case "upload":
|
|
249
|
+
return act.cancel ? "upload cancel" : `upload ${(act.paths || []).join(", ")}`;
|
|
250
|
+
case "tabs":
|
|
251
|
+
return `tabs ${act.tabAction}${act.tabId ? " " + act.tabId : ""}`;
|
|
252
|
+
case "wait":
|
|
253
|
+
return `wait ${act.timeout || 10}s`;
|
|
254
|
+
default:
|
|
255
|
+
return act.action;
|
|
236
256
|
}
|
|
237
257
|
}
|
|
238
258
|
}
|
|
@@ -11,7 +11,7 @@ export interface ResolvedPos {
|
|
|
11
11
|
export async function resolveTarget(
|
|
12
12
|
agentId: string,
|
|
13
13
|
target: { ref?: string; x?: number; y?: number },
|
|
14
|
-
autoScroll: boolean
|
|
14
|
+
autoScroll: boolean,
|
|
15
15
|
): Promise<ResolvedPos> {
|
|
16
16
|
const dmj = window.bangonit!;
|
|
17
17
|
|
|
@@ -60,8 +60,13 @@ export async function resolveTarget(
|
|
|
60
60
|
const r = rectResult.value;
|
|
61
61
|
if (r) {
|
|
62
62
|
const dy = r.top < 0 ? Math.round(r.top - 100) : r.bottom > r.vh ? Math.round(r.bottom - r.vh + 100) : 0;
|
|
63
|
-
const hint =
|
|
64
|
-
|
|
63
|
+
const hint =
|
|
64
|
+
dy !== 0
|
|
65
|
+
? ` Scroll ${dy > 0 ? "down" : "up"} ~${Math.abs(dy)}px (wheel dy=${dy} at current mouse position)`
|
|
66
|
+
: "";
|
|
67
|
+
throw new Error(
|
|
68
|
+
`Element ref "${target.ref}" is not in the viewport and cannot auto-scroll because this batch contains x,y coordinates that would be invalidated by scrolling.${hint}`,
|
|
69
|
+
);
|
|
65
70
|
}
|
|
66
71
|
}
|
|
67
72
|
|
|
@@ -99,13 +104,20 @@ export async function resolveTarget(
|
|
|
99
104
|
const r = rectResult.value;
|
|
100
105
|
if (r) {
|
|
101
106
|
const dy = r.top < 0 ? Math.round(r.top - 100) : r.bottom > r.vh ? Math.round(r.bottom - r.vh + 100) : 0;
|
|
102
|
-
const hint =
|
|
103
|
-
|
|
107
|
+
const hint =
|
|
108
|
+
dy !== 0
|
|
109
|
+
? ` Scroll ${dy > 0 ? "down" : "up"} ~${Math.abs(dy)}px (wheel dy=${dy} at current mouse position)`
|
|
110
|
+
: "";
|
|
111
|
+
throw new Error(
|
|
112
|
+
`Element ref "${target.ref}" is not in the viewport and cannot auto-scroll because this batch contains x,y coordinates that would be invalidated by scrolling.${hint}`,
|
|
113
|
+
);
|
|
104
114
|
}
|
|
105
115
|
} catch (scrollErr: any) {
|
|
106
116
|
if (scrollErr.message.includes("not in the viewport")) throw scrollErr;
|
|
107
117
|
}
|
|
108
|
-
throw new Error(
|
|
118
|
+
throw new Error(
|
|
119
|
+
`Element ref "${target.ref}" is not visible in the viewport — use only refs or scroll manually first`,
|
|
120
|
+
);
|
|
109
121
|
} finally {
|
|
110
122
|
dmj.cdpSend(agentId, "Runtime.releaseObject", { objectId: result.objectId }).catch((e) => console.error(e));
|
|
111
123
|
}
|
|
@@ -121,7 +133,7 @@ export async function moveTo(
|
|
|
121
133
|
sy: number,
|
|
122
134
|
tx: number,
|
|
123
135
|
ty: number,
|
|
124
|
-
onCursorMove?: (x: number, y: number) => void
|
|
136
|
+
onCursorMove?: (x: number, y: number) => void,
|
|
125
137
|
): Promise<void> {
|
|
126
138
|
const dmj = window.bangonit!;
|
|
127
139
|
const dist = Math.hypot(tx - sx, ty - sy);
|
|
@@ -156,7 +168,7 @@ export async function mouseSequence(
|
|
|
156
168
|
onCursorUp?: (x: number, y: number) => void;
|
|
157
169
|
onMouseLock?: () => void;
|
|
158
170
|
onMouseUnlock?: () => void;
|
|
159
|
-
}
|
|
171
|
+
},
|
|
160
172
|
): Promise<{ result: string; mouseX: number; mouseY: number }> {
|
|
161
173
|
const dmj = window.bangonit!;
|
|
162
174
|
callbacks.onMouseLock?.();
|
|
@@ -184,44 +196,97 @@ export async function mouseSequence(
|
|
|
184
196
|
switch (step.action) {
|
|
185
197
|
case "down":
|
|
186
198
|
await moveTo(agentId, mx, my, x, y, callbacks.onCursorMove);
|
|
187
|
-
mx = x;
|
|
199
|
+
mx = x;
|
|
200
|
+
my = y;
|
|
188
201
|
callbacks.onCursorDown?.(x, y);
|
|
189
|
-
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", {
|
|
202
|
+
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", {
|
|
203
|
+
type: "mousePressed",
|
|
204
|
+
x,
|
|
205
|
+
y,
|
|
206
|
+
button: "left",
|
|
207
|
+
clickCount: 1,
|
|
208
|
+
});
|
|
190
209
|
log.push(`down(${label})`);
|
|
191
210
|
break;
|
|
192
211
|
case "move":
|
|
193
212
|
await moveTo(agentId, mx, my, x, y, callbacks.onCursorMove);
|
|
194
|
-
mx = x;
|
|
213
|
+
mx = x;
|
|
214
|
+
my = y;
|
|
195
215
|
log.push(`move(${label})`);
|
|
196
216
|
break;
|
|
197
217
|
case "up":
|
|
198
|
-
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", {
|
|
218
|
+
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", {
|
|
219
|
+
type: "mouseReleased",
|
|
220
|
+
x,
|
|
221
|
+
y,
|
|
222
|
+
button: "left",
|
|
223
|
+
clickCount: 1,
|
|
224
|
+
});
|
|
199
225
|
callbacks.onCursorUp?.(x, y);
|
|
200
226
|
log.push(`up(${label})`);
|
|
201
227
|
break;
|
|
202
228
|
case "click":
|
|
203
229
|
await moveTo(agentId, mx, my, x, y, callbacks.onCursorMove);
|
|
204
|
-
mx = x;
|
|
230
|
+
mx = x;
|
|
231
|
+
my = y;
|
|
205
232
|
callbacks.onCursorDown?.(x, y);
|
|
206
|
-
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", {
|
|
207
|
-
|
|
233
|
+
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", {
|
|
234
|
+
type: "mousePressed",
|
|
235
|
+
x,
|
|
236
|
+
y,
|
|
237
|
+
button: "left",
|
|
238
|
+
clickCount: 1,
|
|
239
|
+
});
|
|
240
|
+
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", {
|
|
241
|
+
type: "mouseReleased",
|
|
242
|
+
x,
|
|
243
|
+
y,
|
|
244
|
+
button: "left",
|
|
245
|
+
clickCount: 1,
|
|
246
|
+
});
|
|
208
247
|
callbacks.onCursorUp?.(x, y);
|
|
209
248
|
log.push(`click(${label})`);
|
|
210
249
|
break;
|
|
211
250
|
case "dblclick":
|
|
212
251
|
await moveTo(agentId, mx, my, x, y, callbacks.onCursorMove);
|
|
213
|
-
mx = x;
|
|
252
|
+
mx = x;
|
|
253
|
+
my = y;
|
|
214
254
|
callbacks.onCursorDown?.(x, y);
|
|
215
|
-
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", {
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
255
|
+
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", {
|
|
256
|
+
type: "mousePressed",
|
|
257
|
+
x,
|
|
258
|
+
y,
|
|
259
|
+
button: "left",
|
|
260
|
+
clickCount: 1,
|
|
261
|
+
});
|
|
262
|
+
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", {
|
|
263
|
+
type: "mouseReleased",
|
|
264
|
+
x,
|
|
265
|
+
y,
|
|
266
|
+
button: "left",
|
|
267
|
+
clickCount: 1,
|
|
268
|
+
});
|
|
269
|
+
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", {
|
|
270
|
+
type: "mousePressed",
|
|
271
|
+
x,
|
|
272
|
+
y,
|
|
273
|
+
button: "left",
|
|
274
|
+
clickCount: 2,
|
|
275
|
+
});
|
|
276
|
+
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", {
|
|
277
|
+
type: "mouseReleased",
|
|
278
|
+
x,
|
|
279
|
+
y,
|
|
280
|
+
button: "left",
|
|
281
|
+
clickCount: 2,
|
|
282
|
+
});
|
|
219
283
|
callbacks.onCursorUp?.(x, y);
|
|
220
284
|
log.push(`dblclick(${label})`);
|
|
221
285
|
break;
|
|
222
286
|
case "wheel": {
|
|
223
287
|
await moveTo(agentId, mx, my, x, y, callbacks.onCursorMove);
|
|
224
|
-
mx = x;
|
|
288
|
+
mx = x;
|
|
289
|
+
my = y;
|
|
225
290
|
const dx = step.dx || 0;
|
|
226
291
|
const dy = step.dy || 0;
|
|
227
292
|
await dmj.cdpSend(agentId, "Input.dispatchMouseEvent", { type: "mouseWheel", x, y, deltaX: dx, deltaY: dy });
|
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
// Navigation: loadURL
|
|
2
2
|
|
|
3
|
-
export async function navigate(
|
|
4
|
-
agentId: string,
|
|
5
|
-
url: string
|
|
6
|
-
): Promise<string> {
|
|
3
|
+
export async function navigate(agentId: string, url: string): Promise<string> {
|
|
7
4
|
const dmj = window.bangonit!;
|
|
8
5
|
if (!url.startsWith("http://") && !url.startsWith("https://")) url = "https://" + url;
|
|
9
6
|
|
|
@@ -26,8 +26,6 @@ interface CapturedFrame {
|
|
|
26
26
|
scrollOffsetY: number;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
-
|
|
30
|
-
|
|
31
29
|
export class BrowserRecorder {
|
|
32
30
|
private agentId: string;
|
|
33
31
|
private clips: ClipMeta[] = [];
|
|
@@ -218,13 +216,22 @@ export class BrowserRecorder {
|
|
|
218
216
|
|
|
219
217
|
const emitFrame = (
|
|
220
218
|
img: ImageBitmap,
|
|
221
|
-
cx: number,
|
|
222
|
-
|
|
219
|
+
cx: number,
|
|
220
|
+
cy: number,
|
|
221
|
+
clicking: boolean,
|
|
222
|
+
ts: number,
|
|
223
|
+
scaleX: number,
|
|
224
|
+
scaleY: number,
|
|
223
225
|
) => {
|
|
224
226
|
const sx = cx * scaleX;
|
|
225
227
|
const sy = cy * scaleY;
|
|
226
228
|
|
|
227
|
-
if (
|
|
229
|
+
if (
|
|
230
|
+
clicking &&
|
|
231
|
+
(!activeRipples.length ||
|
|
232
|
+
activeRipples[activeRipples.length - 1].x !== sx ||
|
|
233
|
+
activeRipples[activeRipples.length - 1].y !== sy)
|
|
234
|
+
) {
|
|
228
235
|
activeRipples.push({ x: sx, y: sy, startTs: ts });
|
|
229
236
|
}
|
|
230
237
|
|
|
@@ -12,12 +12,12 @@ function cacheKey(agentId: string, tabId: number | null): string {
|
|
|
12
12
|
async function sha256(data: string): Promise<string> {
|
|
13
13
|
const buf = new TextEncoder().encode(data);
|
|
14
14
|
const hash = await crypto.subtle.digest("SHA-256", buf);
|
|
15
|
-
return Array.from(new Uint8Array(hash))
|
|
15
|
+
return Array.from(new Uint8Array(hash))
|
|
16
|
+
.map((b) => b.toString(16).padStart(2, "0"))
|
|
17
|
+
.join("");
|
|
16
18
|
}
|
|
17
19
|
|
|
18
|
-
export async function takeScreenshot(
|
|
19
|
-
agentId: string,
|
|
20
|
-
): Promise<ScreenshotResult> {
|
|
20
|
+
export async function takeScreenshot(agentId: string): Promise<ScreenshotResult> {
|
|
21
21
|
const dmj = window.bangonit!;
|
|
22
22
|
let base64: string;
|
|
23
23
|
try {
|
|
@@ -122,7 +122,7 @@ export async function takeSnapshot(
|
|
|
122
122
|
agentId: string,
|
|
123
123
|
mouseX: number,
|
|
124
124
|
mouseY: number,
|
|
125
|
-
tabSummary: string
|
|
125
|
+
tabSummary: string,
|
|
126
126
|
): Promise<string> {
|
|
127
127
|
const [{ result }, downloads, fileChooser, { result: vpResult }] = await Promise.all([
|
|
128
128
|
window.bangonit!.cdpSend(agentId, "Runtime.evaluate", {
|
|
@@ -136,16 +136,20 @@ export async function takeSnapshot(
|
|
|
136
136
|
returnByValue: true,
|
|
137
137
|
}),
|
|
138
138
|
]);
|
|
139
|
-
const vp = (() => {
|
|
139
|
+
const vp = (() => {
|
|
140
|
+
try {
|
|
141
|
+
return JSON.parse(vpResult.value);
|
|
142
|
+
} catch {
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
})();
|
|
140
146
|
const vpStr = vp ? `\nViewport: ${vp.w}x${vp.h}` : "";
|
|
141
147
|
let out = (result.value || "Empty page") + `\n\nMouse position: ${mouseX},${mouseY}${vpStr}` + tabSummary;
|
|
142
148
|
if (fileChooser.open) {
|
|
143
149
|
out += `\n\n[File picker is open — use upload action with paths to select files, or cancel: true to dismiss]`;
|
|
144
150
|
}
|
|
145
151
|
if (downloads.length > 0) {
|
|
146
|
-
const lines = downloads.map((d: any) =>
|
|
147
|
-
`- ${d.filename} (${d.bytes} bytes) [${d.state}]`
|
|
148
|
-
);
|
|
152
|
+
const lines = downloads.map((d: any) => `- ${d.filename} (${d.bytes} bytes) [${d.state}]`);
|
|
149
153
|
out += `\n\nDownloads:\n${lines.join("\n")}`;
|
|
150
154
|
}
|
|
151
155
|
return out;
|
|
@@ -28,7 +28,7 @@ export interface BrowserAction {
|
|
|
28
28
|
|
|
29
29
|
export interface BrowserToolInput {
|
|
30
30
|
actions: BrowserAction[];
|
|
31
|
-
observe
|
|
31
|
+
observe: ObserveMode;
|
|
32
32
|
prompts?: string[];
|
|
33
33
|
}
|
|
34
34
|
|
|
@@ -85,7 +85,8 @@ export const TOOL_DEFS = [
|
|
|
85
85
|
},
|
|
86
86
|
{
|
|
87
87
|
name: "browser_upload",
|
|
88
|
-
description:
|
|
88
|
+
description:
|
|
89
|
+
"Upload files to an open file picker, or cancel it. The snapshot will show when a file picker is open.",
|
|
89
90
|
parameters: {
|
|
90
91
|
type: "object",
|
|
91
92
|
properties: {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
export async function waitFor(
|
|
4
4
|
agentId: string,
|
|
5
|
-
args: { text?: string; state?: string; timeout?: number }
|
|
5
|
+
args: { text?: string; state?: string; timeout?: number },
|
|
6
6
|
): Promise<string> {
|
|
7
7
|
const dmj = window.bangonit!;
|
|
8
8
|
const timeout = Math.min(Math.max(args?.timeout || 10, 0.1), 30);
|
|
@@ -5,14 +5,6 @@ interface TabInfo {
|
|
|
5
5
|
title?: string;
|
|
6
6
|
}
|
|
7
7
|
|
|
8
|
-
interface AgentSessionData {
|
|
9
|
-
messages: any[];
|
|
10
|
-
initialPrompt: string;
|
|
11
|
-
tabs?: { id: number; url: string; title: string }[];
|
|
12
|
-
activeTabId?: number;
|
|
13
|
-
todos?: { content: string; status: "pending" | "in_progress" | "completed" }[];
|
|
14
|
-
}
|
|
15
|
-
|
|
16
8
|
interface TestResult {
|
|
17
9
|
agentId: string;
|
|
18
10
|
status: "pass" | "fail";
|
|
@@ -20,15 +12,6 @@ interface TestResult {
|
|
|
20
12
|
}
|
|
21
13
|
|
|
22
14
|
interface BangOnIt {
|
|
23
|
-
// Agent CRUD
|
|
24
|
-
getAgents: () => Promise<any[]>;
|
|
25
|
-
setAgents: (agents: any[]) => Promise<void>;
|
|
26
|
-
|
|
27
|
-
// Agent session persistence
|
|
28
|
-
getAgentSession: (agentId: string) => Promise<AgentSessionData | null>;
|
|
29
|
-
setAgentSession: (agentId: string, session: AgentSessionData) => Promise<void>;
|
|
30
|
-
deleteAgentSession: (agentId: string) => Promise<void>;
|
|
31
|
-
|
|
32
15
|
// --- Low-level browser primitives ---
|
|
33
16
|
|
|
34
17
|
// CDP: send a raw CDP command to the active tab
|
|
@@ -58,7 +41,11 @@ interface BangOnIt {
|
|
|
58
41
|
getDownloads: (agentId: string) => Promise<any[]>;
|
|
59
42
|
|
|
60
43
|
// File chooser
|
|
61
|
-
handleFileChooser: (
|
|
44
|
+
handleFileChooser: (
|
|
45
|
+
agentId: string,
|
|
46
|
+
action: "accept" | "cancel",
|
|
47
|
+
files?: string[],
|
|
48
|
+
) => Promise<{ ok: boolean; error?: string }>;
|
|
62
49
|
getFileChooserState: (agentId: string) => Promise<{ open: boolean; mode?: string }>;
|
|
63
50
|
|
|
64
51
|
// Events — data includes agentId
|
|
@@ -84,7 +71,9 @@ interface BangOnIt {
|
|
|
84
71
|
emitMouseUnlock: (agentId: string) => void;
|
|
85
72
|
|
|
86
73
|
// IPC
|
|
87
|
-
onTestPlan?: (
|
|
74
|
+
onTestPlan?: (
|
|
75
|
+
cb: (data: { agentIndex: number; testPlan: string; name?: string; retries?: number; extraPrompt?: string }) => void,
|
|
76
|
+
) => () => void;
|
|
88
77
|
onFocusAgent?: (cb: (data: { agentId: string }) => void) => () => void;
|
|
89
78
|
reportTestResult?: (result: TestResult) => void;
|
|
90
79
|
emitAgentOutput?: (data: { agentId: string; type: string; text: string; name?: string }) => void;
|