donobu 5.60.5 → 5.60.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/esm/lib/ai/locate/locateElement.js +12 -2
- package/dist/esm/lib/page/extendPage.js +1 -1
- package/dist/esm/managers/DonobuFlowsManager.js +1 -1
- package/dist/esm/managers/InteractionVisualizer.d.ts +14 -21
- package/dist/esm/managers/InteractionVisualizer.js +28 -78
- package/dist/esm/tools/ReplayableInteraction.js +1 -1
- package/dist/esm/utils/MiscUtils.js +11 -15
- package/dist/lib/ai/locate/locateElement.js +12 -2
- package/dist/lib/page/extendPage.js +1 -1
- package/dist/managers/DonobuFlowsManager.js +1 -1
- package/dist/managers/InteractionVisualizer.d.ts +14 -21
- package/dist/managers/InteractionVisualizer.js +28 -78
- package/dist/tools/ReplayableInteraction.js +1 -1
- package/dist/utils/MiscUtils.js +11 -15
- package/package.json +1 -1
|
@@ -13,6 +13,14 @@ const locateSchema_1 = require("./locateSchema");
|
|
|
13
13
|
const DISAMBIGUATE_THRESHOLD = 5;
|
|
14
14
|
/** Maximum outer HTML length per snippet shown during disambiguation. */
|
|
15
15
|
const SNIPPET_MAX_CHARS = 200;
|
|
16
|
+
/**
|
|
17
|
+
* Per-candidate read budget during disambiguation. Locator `evaluate`/
|
|
18
|
+
* `boundingBox` default to *no* timeout, so a candidate that detaches mid-loop
|
|
19
|
+
* would wait forever (the surrounding try/catch only traps errors, not hangs,
|
|
20
|
+
* and the AbortSignal isn't plumbed into these ops). Cap it and let a miss fall
|
|
21
|
+
* through to the placeholder snippet below.
|
|
22
|
+
*/
|
|
23
|
+
const CANDIDATE_READ_TIMEOUT_MILLIS = 1000;
|
|
16
24
|
/**
|
|
17
25
|
* Resolve a natural-language element description to a Playwright {@link Locator}.
|
|
18
26
|
*
|
|
@@ -87,8 +95,10 @@ async function disambiguate(page, description, gptClient, locator, locateResult,
|
|
|
87
95
|
const html = await nth.evaluate((el, max) => {
|
|
88
96
|
const raw = el.outerHTML;
|
|
89
97
|
return raw.length > max ? raw.slice(0, max) + '…' : raw;
|
|
90
|
-
}, SNIPPET_MAX_CHARS);
|
|
91
|
-
const box = await nth.boundingBox(
|
|
98
|
+
}, SNIPPET_MAX_CHARS, { timeout: CANDIDATE_READ_TIMEOUT_MILLIS });
|
|
99
|
+
const box = await nth.boundingBox({
|
|
100
|
+
timeout: CANDIDATE_READ_TIMEOUT_MILLIS,
|
|
101
|
+
});
|
|
92
102
|
const pos = box
|
|
93
103
|
? `(${Math.round(box.x)}, ${Math.round(box.y)})`
|
|
94
104
|
: '(unknown)';
|
|
@@ -166,7 +166,7 @@ async function extendPage(page, options) {
|
|
|
166
166
|
aiInvocations: [],
|
|
167
167
|
};
|
|
168
168
|
const showMouse = async (p) => {
|
|
169
|
-
if (interactionVisualizer.
|
|
169
|
+
if (interactionVisualizer.defaultCursorDurationMillis > 0) {
|
|
170
170
|
await interactionVisualizer.showMouse(p);
|
|
171
171
|
}
|
|
172
172
|
};
|
|
@@ -174,7 +174,7 @@ class DonobuFlowsManager {
|
|
|
174
174
|
gptConfigName: gptClientData.gptConfigName,
|
|
175
175
|
hasGptConfigNameOverride: !gptClientData.agentName,
|
|
176
176
|
customTools: flowParams.customTools ?? null,
|
|
177
|
-
defaultMessageDuration: interactionVisualizer.
|
|
177
|
+
defaultMessageDuration: interactionVisualizer.defaultCursorDurationMillis,
|
|
178
178
|
callbackUrl: flowParams.callbackUrl || null,
|
|
179
179
|
overallObjective: flowParams.overallObjective ?? null,
|
|
180
180
|
allowedTools: allowedTools.map((tool) => tool.name),
|
|
@@ -1,37 +1,38 @@
|
|
|
1
1
|
import type { Locator, Page } from 'playwright';
|
|
2
2
|
export declare class InteractionVisualizer {
|
|
3
|
-
readonly
|
|
3
|
+
readonly defaultCursorDurationMillis: number;
|
|
4
4
|
private static readonly RAW_MOUSE_D;
|
|
5
5
|
private static readonly TIP_X;
|
|
6
6
|
private static readonly TIP_Y;
|
|
7
7
|
private static readonly SVG_MOUSE;
|
|
8
|
-
/**
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Upper bound on resolving a target's bounding box. `Locator.boundingBox`
|
|
10
|
+
* defaults to *no* timeout, so a locator that resolves to zero elements
|
|
11
|
+
* (e.g. detached mid-flow) would otherwise wait forever and hang the flow.
|
|
12
|
+
* The cursor is cosmetic, so we cap the wait and treat a miss as a no-op.
|
|
13
|
+
*/
|
|
14
|
+
private static readonly BOUNDING_BOX_TIMEOUT_MILLIS;
|
|
11
15
|
/**
|
|
12
16
|
* Per-page cursor state. Keyed weakly so closed/GC'd pages drop out without
|
|
13
17
|
* us tracking page lifecycles.
|
|
14
18
|
*/
|
|
15
19
|
private readonly states;
|
|
16
|
-
constructor(
|
|
17
|
-
private static escapeHtml;
|
|
20
|
+
constructor(defaultCursorDurationMillis: number);
|
|
18
21
|
/**
|
|
19
|
-
* Moves the virtual cursor to the center of the specified element
|
|
22
|
+
* Moves the virtual cursor to the center of the specified element.
|
|
20
23
|
*
|
|
21
24
|
* @param page - The Playwright page instance where the cursor will be displayed.
|
|
22
25
|
* @param locator - Optional target element to point at. If omitted, cursor remains at current position.
|
|
23
|
-
* @param
|
|
24
|
-
* @param duration - Duration in milliseconds for the cursor animation and message display.
|
|
26
|
+
* @param duration - Duration in milliseconds for the cursor animation.
|
|
25
27
|
* Defaults to the instance's configured duration. If ≤ 0, no action is taken
|
|
26
28
|
*
|
|
27
|
-
* @returns Promise that resolves
|
|
29
|
+
* @returns Promise that resolves once the cursor has reached the target
|
|
28
30
|
*
|
|
29
31
|
* @remarks
|
|
30
32
|
* - The cursor animates smoothly to the element's center point
|
|
31
|
-
* -
|
|
32
|
-
* - Overlays are `pointer-events: none`, so they never intercept real interactions
|
|
33
|
+
* - The overlay is `pointer-events: none`, so it never intercepts real interactions
|
|
33
34
|
*/
|
|
34
|
-
pointAt(page: Page, locator?: Pick<Locator, 'boundingBox'>,
|
|
35
|
+
pointAt(page: Page, locator?: Pick<Locator, 'boundingBox'>, duration?: number): Promise<void>;
|
|
35
36
|
/**
|
|
36
37
|
* Shows the virtual mouse cursor on the page at its current position.
|
|
37
38
|
*
|
|
@@ -54,18 +55,10 @@ export declare class InteractionVisualizer {
|
|
|
54
55
|
* (e.g. cleared by a navigation) by swallowing its dispose error.
|
|
55
56
|
*/
|
|
56
57
|
private renderCursor;
|
|
57
|
-
private showMessage;
|
|
58
58
|
/**
|
|
59
59
|
* Builds the cursor overlay markup: the arrow SVG plus a `@keyframes` glide
|
|
60
60
|
* from the previous position and a one-shot ripple at the arrow tip.
|
|
61
61
|
*/
|
|
62
62
|
private buildCursorHtml;
|
|
63
|
-
/**
|
|
64
|
-
* Builds the message tooltip markup, positioned near `target` and clamped
|
|
65
|
-
* inside the viewport. Without being able to measure the rendered box, we
|
|
66
|
-
* keep it on-screen with a conservative max-width clamp and anchor it above
|
|
67
|
-
* the cursor (via `translateY(-100%)`) when the cursor sits low in the page.
|
|
68
|
-
*/
|
|
69
|
-
private buildMessageHtml;
|
|
70
63
|
}
|
|
71
64
|
//# sourceMappingURL=InteractionVisualizer.d.ts.map
|
|
@@ -4,8 +4,8 @@ exports.InteractionVisualizer = void 0;
|
|
|
4
4
|
const Logger_1 = require("../utils/Logger");
|
|
5
5
|
const PlaywrightUtils_1 = require("../utils/PlaywrightUtils");
|
|
6
6
|
class InteractionVisualizer {
|
|
7
|
-
constructor(
|
|
8
|
-
this.
|
|
7
|
+
constructor(defaultCursorDurationMillis) {
|
|
8
|
+
this.defaultCursorDurationMillis = defaultCursorDurationMillis;
|
|
9
9
|
/**
|
|
10
10
|
* Per-page cursor state. Keyed weakly so closed/GC'd pages drop out without
|
|
11
11
|
* us tracking page lifecycles.
|
|
@@ -15,30 +15,21 @@ class InteractionVisualizer {
|
|
|
15
15
|
/* ------------------------------------------------------------------ */
|
|
16
16
|
/* Public API */
|
|
17
17
|
/* ------------------------------------------------------------------ */
|
|
18
|
-
static escapeHtml(text) {
|
|
19
|
-
return text
|
|
20
|
-
.replace(/&/g, '&')
|
|
21
|
-
.replace(/</g, '<')
|
|
22
|
-
.replace(/>/g, '>')
|
|
23
|
-
.replace(/"/g, '"');
|
|
24
|
-
}
|
|
25
18
|
/**
|
|
26
|
-
* Moves the virtual cursor to the center of the specified element
|
|
19
|
+
* Moves the virtual cursor to the center of the specified element.
|
|
27
20
|
*
|
|
28
21
|
* @param page - The Playwright page instance where the cursor will be displayed.
|
|
29
22
|
* @param locator - Optional target element to point at. If omitted, cursor remains at current position.
|
|
30
|
-
* @param
|
|
31
|
-
* @param duration - Duration in milliseconds for the cursor animation and message display.
|
|
23
|
+
* @param duration - Duration in milliseconds for the cursor animation.
|
|
32
24
|
* Defaults to the instance's configured duration. If ≤ 0, no action is taken
|
|
33
25
|
*
|
|
34
|
-
* @returns Promise that resolves
|
|
26
|
+
* @returns Promise that resolves once the cursor has reached the target
|
|
35
27
|
*
|
|
36
28
|
* @remarks
|
|
37
29
|
* - The cursor animates smoothly to the element's center point
|
|
38
|
-
* -
|
|
39
|
-
* - Overlays are `pointer-events: none`, so they never intercept real interactions
|
|
30
|
+
* - The overlay is `pointer-events: none`, so it never intercepts real interactions
|
|
40
31
|
*/
|
|
41
|
-
async pointAt(page, locator,
|
|
32
|
+
async pointAt(page, locator, duration = this.defaultCursorDurationMillis) {
|
|
42
33
|
if (!duration || duration <= 0) {
|
|
43
34
|
return;
|
|
44
35
|
}
|
|
@@ -47,20 +38,21 @@ class InteractionVisualizer {
|
|
|
47
38
|
const from = { ...state.pos };
|
|
48
39
|
let target = from;
|
|
49
40
|
if (locator) {
|
|
50
|
-
|
|
41
|
+
// Cap the wait and swallow a timeout: an unresolvable locator must
|
|
42
|
+
// leave the (cosmetic) cursor in place, never block the flow.
|
|
43
|
+
const box = await locator
|
|
44
|
+
.boundingBox({
|
|
45
|
+
timeout: InteractionVisualizer.BOUNDING_BOX_TIMEOUT_MILLIS,
|
|
46
|
+
})
|
|
47
|
+
.catch(() => null);
|
|
51
48
|
if (box) {
|
|
52
49
|
target = { x: box.x + box.width / 2, y: box.y + box.height / 2 };
|
|
53
50
|
}
|
|
54
51
|
}
|
|
55
52
|
state.pos = target;
|
|
56
|
-
|
|
57
|
-
state
|
|
58
|
-
|
|
59
|
-
: Promise.resolve(),
|
|
60
|
-
message?.trim()
|
|
61
|
-
? this.showMessage(page, target, message.trim(), duration)
|
|
62
|
-
: Promise.resolve(),
|
|
63
|
-
]);
|
|
53
|
+
if (state.visible) {
|
|
54
|
+
await this.renderCursor(page, state, from, target, duration / 2);
|
|
55
|
+
}
|
|
64
56
|
}
|
|
65
57
|
catch (error) {
|
|
66
58
|
if (!PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
@@ -137,26 +129,15 @@ class InteractionVisualizer {
|
|
|
137
129
|
// showOverlay resolves the moment the overlay is added, before the CSS
|
|
138
130
|
// glide plays. Callers (e.g. ClickTool) await pointAt expecting the cursor
|
|
139
131
|
// to reach the element *before* they act, so block for the animation here.
|
|
140
|
-
|
|
132
|
+
// Only the moving case animates (see buildCursorHtml), so a no-op move
|
|
133
|
+
// (e.g. an unresolved target) shouldn't incur the wait.
|
|
134
|
+
const animating = animMs > 0 && (from.x !== to.x || from.y !== to.y);
|
|
135
|
+
if (animating) {
|
|
141
136
|
await new Promise((resolve) => {
|
|
142
137
|
setTimeout(resolve, animMs);
|
|
143
138
|
});
|
|
144
139
|
}
|
|
145
140
|
}
|
|
146
|
-
async showMessage(page, target, text, duration) {
|
|
147
|
-
// Sticky overlay + self-managed removal. The `duration` option on
|
|
148
|
-
// showOverlay only composites into the recorded screencast stream, not the
|
|
149
|
-
// live page render layer (so it would be invisible in headed/screenshots);
|
|
150
|
-
// a no-duration overlay renders everywhere, matching the cursor.
|
|
151
|
-
const handle = await page.screencast.showOverlay(this.buildMessageHtml(page, target, text));
|
|
152
|
-
const timer = setTimeout(() => {
|
|
153
|
-
handle.dispose().catch(() => {
|
|
154
|
-
// Page navigated/closed before the message expired — nothing to clean up.
|
|
155
|
-
});
|
|
156
|
-
}, duration);
|
|
157
|
-
// Don't let a pending message removal keep the process alive on shutdown.
|
|
158
|
-
timer.unref?.();
|
|
159
|
-
}
|
|
160
141
|
/**
|
|
161
142
|
* Builds the cursor overlay markup: the arrow SVG plus a `@keyframes` glide
|
|
162
143
|
* from the previous position and a one-shot ripple at the arrow tip.
|
|
@@ -179,41 +160,6 @@ class InteractionVisualizer {
|
|
|
179
160
|
</style>
|
|
180
161
|
<div class="donobu-cursor">${InteractionVisualizer.SVG_MOUSE}</div>`;
|
|
181
162
|
}
|
|
182
|
-
/**
|
|
183
|
-
* Builds the message tooltip markup, positioned near `target` and clamped
|
|
184
|
-
* inside the viewport. Without being able to measure the rendered box, we
|
|
185
|
-
* keep it on-screen with a conservative max-width clamp and anchor it above
|
|
186
|
-
* the cursor (via `translateY(-100%)`) when the cursor sits low in the page.
|
|
187
|
-
*/
|
|
188
|
-
buildMessageHtml(page, target, text) {
|
|
189
|
-
const maxW = InteractionVisualizer.MESSAGE_MAX_WIDTH;
|
|
190
|
-
const margin = InteractionVisualizer.MESSAGE_MARGIN;
|
|
191
|
-
const vp = page.viewportSize() ?? { width: 1280, height: 720 };
|
|
192
|
-
// Clamp the (centered) anchor so a max-width box can't overflow either edge.
|
|
193
|
-
const centerX = Math.max(maxW / 2 + margin, Math.min(vp.width - maxW / 2 - margin, target.x));
|
|
194
|
-
const placeAbove = target.y > vp.height * 0.6;
|
|
195
|
-
const top = placeAbove ? Math.max(margin, target.y - 24) : target.y + 24;
|
|
196
|
-
const translate = placeAbove
|
|
197
|
-
? 'translate(-50%, -100%)'
|
|
198
|
-
: 'translate(-50%, 0)';
|
|
199
|
-
const style = [
|
|
200
|
-
'position:fixed',
|
|
201
|
-
`left:${centerX}px`,
|
|
202
|
-
`top:${top}px`,
|
|
203
|
-
`transform:${translate}`,
|
|
204
|
-
`max-width:${maxW}px`,
|
|
205
|
-
'z-index:2147483647',
|
|
206
|
-
'pointer-events:none',
|
|
207
|
-
'background:#000',
|
|
208
|
-
'color:#fff',
|
|
209
|
-
'padding:8px 10px',
|
|
210
|
-
'border-radius:5px',
|
|
211
|
-
"font:12px/1 'Source Sans Pro','Helvetica Neue',Helvetica,Arial,sans-serif",
|
|
212
|
-
'white-space:pre-wrap',
|
|
213
|
-
'box-shadow:2px 2px 10px rgba(0,0,0,.2)',
|
|
214
|
-
].join(';');
|
|
215
|
-
return `<div style="${style}">${InteractionVisualizer.escapeHtml(text)}</div>`;
|
|
216
|
-
}
|
|
217
163
|
}
|
|
218
164
|
exports.InteractionVisualizer = InteractionVisualizer;
|
|
219
165
|
/* ------------------------------------------------------------------ */
|
|
@@ -235,7 +181,11 @@ InteractionVisualizer.SVG_MOUSE = `
|
|
|
235
181
|
stroke-linejoin="round">
|
|
236
182
|
<path d="${InteractionVisualizer.RAW_MOUSE_D}" />
|
|
237
183
|
</svg>`;
|
|
238
|
-
/**
|
|
239
|
-
|
|
240
|
-
|
|
184
|
+
/**
|
|
185
|
+
* Upper bound on resolving a target's bounding box. `Locator.boundingBox`
|
|
186
|
+
* defaults to *no* timeout, so a locator that resolves to zero elements
|
|
187
|
+
* (e.g. detached mid-flow) would otherwise wait forever and hang the flow.
|
|
188
|
+
* The cursor is cosmetic, so we cap the wait and treat a miss as a no-op.
|
|
189
|
+
*/
|
|
190
|
+
InteractionVisualizer.BOUNDING_BOX_TIMEOUT_MILLIS = 1000;
|
|
241
191
|
//# sourceMappingURL=InteractionVisualizer.js.map
|
|
@@ -432,7 +432,7 @@ class ReplayableInteraction extends Tool_1.Tool {
|
|
|
432
432
|
// Only reveal the cursor now that we have a real target to point at, so a
|
|
433
433
|
// non-interactive proposal never pops a stationary cursor.
|
|
434
434
|
await (0, TargetUtils_1.webInspector)(context).showInteractionCursor();
|
|
435
|
-
await context.interactionVisualizer.pointAt(page, pointTarget.first(),
|
|
435
|
+
await context.interactionVisualizer.pointAt(page, pointTarget.first(), ReplayableInteraction.PREVIEW_CURSOR_DURATION_MILLIS);
|
|
436
436
|
}
|
|
437
437
|
async callFromGpt(context, parameters) {
|
|
438
438
|
const page = (0, TargetUtils_1.webPage)(context);
|
|
@@ -204,25 +204,21 @@ class MiscUtils {
|
|
|
204
204
|
* Merges adjacent user messages (some LLMs like Gemini's or Anthropic's require this).
|
|
205
205
|
*/
|
|
206
206
|
static mergeAdjacentUserMessages(messages) {
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
const
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
const mergedMessage = {
|
|
207
|
+
const updatedMessages = [];
|
|
208
|
+
// Fold each user message into the previously emitted one when that is also
|
|
209
|
+
// a user message. Folding into the accumulated result (rather than
|
|
210
|
+
// consuming pairs) collapses runs of any length — pair-consuming would
|
|
211
|
+
// leave the 3rd, 5th, ... message in a run un-merged.
|
|
212
|
+
for (const message of messages) {
|
|
213
|
+
const last = updatedMessages[updatedMessages.length - 1];
|
|
214
|
+
if (message.type === 'user' && last?.type === 'user') {
|
|
215
|
+
updatedMessages[updatedMessages.length - 1] = {
|
|
217
216
|
type: 'user',
|
|
218
|
-
items: [...
|
|
217
|
+
items: [...last.items, ...message.items],
|
|
219
218
|
};
|
|
220
|
-
updatedMessages.push(mergedMessage);
|
|
221
|
-
// Skip the next message.
|
|
222
|
-
++i;
|
|
223
219
|
}
|
|
224
220
|
else {
|
|
225
|
-
updatedMessages.push(
|
|
221
|
+
updatedMessages.push(message);
|
|
226
222
|
}
|
|
227
223
|
}
|
|
228
224
|
return updatedMessages;
|
|
@@ -13,6 +13,14 @@ const locateSchema_1 = require("./locateSchema");
|
|
|
13
13
|
const DISAMBIGUATE_THRESHOLD = 5;
|
|
14
14
|
/** Maximum outer HTML length per snippet shown during disambiguation. */
|
|
15
15
|
const SNIPPET_MAX_CHARS = 200;
|
|
16
|
+
/**
|
|
17
|
+
* Per-candidate read budget during disambiguation. Locator `evaluate`/
|
|
18
|
+
* `boundingBox` default to *no* timeout, so a candidate that detaches mid-loop
|
|
19
|
+
* would wait forever (the surrounding try/catch only traps errors, not hangs,
|
|
20
|
+
* and the AbortSignal isn't plumbed into these ops). Cap it and let a miss fall
|
|
21
|
+
* through to the placeholder snippet below.
|
|
22
|
+
*/
|
|
23
|
+
const CANDIDATE_READ_TIMEOUT_MILLIS = 1000;
|
|
16
24
|
/**
|
|
17
25
|
* Resolve a natural-language element description to a Playwright {@link Locator}.
|
|
18
26
|
*
|
|
@@ -87,8 +95,10 @@ async function disambiguate(page, description, gptClient, locator, locateResult,
|
|
|
87
95
|
const html = await nth.evaluate((el, max) => {
|
|
88
96
|
const raw = el.outerHTML;
|
|
89
97
|
return raw.length > max ? raw.slice(0, max) + '…' : raw;
|
|
90
|
-
}, SNIPPET_MAX_CHARS);
|
|
91
|
-
const box = await nth.boundingBox(
|
|
98
|
+
}, SNIPPET_MAX_CHARS, { timeout: CANDIDATE_READ_TIMEOUT_MILLIS });
|
|
99
|
+
const box = await nth.boundingBox({
|
|
100
|
+
timeout: CANDIDATE_READ_TIMEOUT_MILLIS,
|
|
101
|
+
});
|
|
92
102
|
const pos = box
|
|
93
103
|
? `(${Math.round(box.x)}, ${Math.round(box.y)})`
|
|
94
104
|
: '(unknown)';
|
|
@@ -166,7 +166,7 @@ async function extendPage(page, options) {
|
|
|
166
166
|
aiInvocations: [],
|
|
167
167
|
};
|
|
168
168
|
const showMouse = async (p) => {
|
|
169
|
-
if (interactionVisualizer.
|
|
169
|
+
if (interactionVisualizer.defaultCursorDurationMillis > 0) {
|
|
170
170
|
await interactionVisualizer.showMouse(p);
|
|
171
171
|
}
|
|
172
172
|
};
|
|
@@ -174,7 +174,7 @@ class DonobuFlowsManager {
|
|
|
174
174
|
gptConfigName: gptClientData.gptConfigName,
|
|
175
175
|
hasGptConfigNameOverride: !gptClientData.agentName,
|
|
176
176
|
customTools: flowParams.customTools ?? null,
|
|
177
|
-
defaultMessageDuration: interactionVisualizer.
|
|
177
|
+
defaultMessageDuration: interactionVisualizer.defaultCursorDurationMillis,
|
|
178
178
|
callbackUrl: flowParams.callbackUrl || null,
|
|
179
179
|
overallObjective: flowParams.overallObjective ?? null,
|
|
180
180
|
allowedTools: allowedTools.map((tool) => tool.name),
|
|
@@ -1,37 +1,38 @@
|
|
|
1
1
|
import type { Locator, Page } from 'playwright';
|
|
2
2
|
export declare class InteractionVisualizer {
|
|
3
|
-
readonly
|
|
3
|
+
readonly defaultCursorDurationMillis: number;
|
|
4
4
|
private static readonly RAW_MOUSE_D;
|
|
5
5
|
private static readonly TIP_X;
|
|
6
6
|
private static readonly TIP_Y;
|
|
7
7
|
private static readonly SVG_MOUSE;
|
|
8
|
-
/**
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Upper bound on resolving a target's bounding box. `Locator.boundingBox`
|
|
10
|
+
* defaults to *no* timeout, so a locator that resolves to zero elements
|
|
11
|
+
* (e.g. detached mid-flow) would otherwise wait forever and hang the flow.
|
|
12
|
+
* The cursor is cosmetic, so we cap the wait and treat a miss as a no-op.
|
|
13
|
+
*/
|
|
14
|
+
private static readonly BOUNDING_BOX_TIMEOUT_MILLIS;
|
|
11
15
|
/**
|
|
12
16
|
* Per-page cursor state. Keyed weakly so closed/GC'd pages drop out without
|
|
13
17
|
* us tracking page lifecycles.
|
|
14
18
|
*/
|
|
15
19
|
private readonly states;
|
|
16
|
-
constructor(
|
|
17
|
-
private static escapeHtml;
|
|
20
|
+
constructor(defaultCursorDurationMillis: number);
|
|
18
21
|
/**
|
|
19
|
-
* Moves the virtual cursor to the center of the specified element
|
|
22
|
+
* Moves the virtual cursor to the center of the specified element.
|
|
20
23
|
*
|
|
21
24
|
* @param page - The Playwright page instance where the cursor will be displayed.
|
|
22
25
|
* @param locator - Optional target element to point at. If omitted, cursor remains at current position.
|
|
23
|
-
* @param
|
|
24
|
-
* @param duration - Duration in milliseconds for the cursor animation and message display.
|
|
26
|
+
* @param duration - Duration in milliseconds for the cursor animation.
|
|
25
27
|
* Defaults to the instance's configured duration. If ≤ 0, no action is taken
|
|
26
28
|
*
|
|
27
|
-
* @returns Promise that resolves
|
|
29
|
+
* @returns Promise that resolves once the cursor has reached the target
|
|
28
30
|
*
|
|
29
31
|
* @remarks
|
|
30
32
|
* - The cursor animates smoothly to the element's center point
|
|
31
|
-
* -
|
|
32
|
-
* - Overlays are `pointer-events: none`, so they never intercept real interactions
|
|
33
|
+
* - The overlay is `pointer-events: none`, so it never intercepts real interactions
|
|
33
34
|
*/
|
|
34
|
-
pointAt(page: Page, locator?: Pick<Locator, 'boundingBox'>,
|
|
35
|
+
pointAt(page: Page, locator?: Pick<Locator, 'boundingBox'>, duration?: number): Promise<void>;
|
|
35
36
|
/**
|
|
36
37
|
* Shows the virtual mouse cursor on the page at its current position.
|
|
37
38
|
*
|
|
@@ -54,18 +55,10 @@ export declare class InteractionVisualizer {
|
|
|
54
55
|
* (e.g. cleared by a navigation) by swallowing its dispose error.
|
|
55
56
|
*/
|
|
56
57
|
private renderCursor;
|
|
57
|
-
private showMessage;
|
|
58
58
|
/**
|
|
59
59
|
* Builds the cursor overlay markup: the arrow SVG plus a `@keyframes` glide
|
|
60
60
|
* from the previous position and a one-shot ripple at the arrow tip.
|
|
61
61
|
*/
|
|
62
62
|
private buildCursorHtml;
|
|
63
|
-
/**
|
|
64
|
-
* Builds the message tooltip markup, positioned near `target` and clamped
|
|
65
|
-
* inside the viewport. Without being able to measure the rendered box, we
|
|
66
|
-
* keep it on-screen with a conservative max-width clamp and anchor it above
|
|
67
|
-
* the cursor (via `translateY(-100%)`) when the cursor sits low in the page.
|
|
68
|
-
*/
|
|
69
|
-
private buildMessageHtml;
|
|
70
63
|
}
|
|
71
64
|
//# sourceMappingURL=InteractionVisualizer.d.ts.map
|
|
@@ -4,8 +4,8 @@ exports.InteractionVisualizer = void 0;
|
|
|
4
4
|
const Logger_1 = require("../utils/Logger");
|
|
5
5
|
const PlaywrightUtils_1 = require("../utils/PlaywrightUtils");
|
|
6
6
|
class InteractionVisualizer {
|
|
7
|
-
constructor(
|
|
8
|
-
this.
|
|
7
|
+
constructor(defaultCursorDurationMillis) {
|
|
8
|
+
this.defaultCursorDurationMillis = defaultCursorDurationMillis;
|
|
9
9
|
/**
|
|
10
10
|
* Per-page cursor state. Keyed weakly so closed/GC'd pages drop out without
|
|
11
11
|
* us tracking page lifecycles.
|
|
@@ -15,30 +15,21 @@ class InteractionVisualizer {
|
|
|
15
15
|
/* ------------------------------------------------------------------ */
|
|
16
16
|
/* Public API */
|
|
17
17
|
/* ------------------------------------------------------------------ */
|
|
18
|
-
static escapeHtml(text) {
|
|
19
|
-
return text
|
|
20
|
-
.replace(/&/g, '&')
|
|
21
|
-
.replace(/</g, '<')
|
|
22
|
-
.replace(/>/g, '>')
|
|
23
|
-
.replace(/"/g, '"');
|
|
24
|
-
}
|
|
25
18
|
/**
|
|
26
|
-
* Moves the virtual cursor to the center of the specified element
|
|
19
|
+
* Moves the virtual cursor to the center of the specified element.
|
|
27
20
|
*
|
|
28
21
|
* @param page - The Playwright page instance where the cursor will be displayed.
|
|
29
22
|
* @param locator - Optional target element to point at. If omitted, cursor remains at current position.
|
|
30
|
-
* @param
|
|
31
|
-
* @param duration - Duration in milliseconds for the cursor animation and message display.
|
|
23
|
+
* @param duration - Duration in milliseconds for the cursor animation.
|
|
32
24
|
* Defaults to the instance's configured duration. If ≤ 0, no action is taken
|
|
33
25
|
*
|
|
34
|
-
* @returns Promise that resolves
|
|
26
|
+
* @returns Promise that resolves once the cursor has reached the target
|
|
35
27
|
*
|
|
36
28
|
* @remarks
|
|
37
29
|
* - The cursor animates smoothly to the element's center point
|
|
38
|
-
* -
|
|
39
|
-
* - Overlays are `pointer-events: none`, so they never intercept real interactions
|
|
30
|
+
* - The overlay is `pointer-events: none`, so it never intercepts real interactions
|
|
40
31
|
*/
|
|
41
|
-
async pointAt(page, locator,
|
|
32
|
+
async pointAt(page, locator, duration = this.defaultCursorDurationMillis) {
|
|
42
33
|
if (!duration || duration <= 0) {
|
|
43
34
|
return;
|
|
44
35
|
}
|
|
@@ -47,20 +38,21 @@ class InteractionVisualizer {
|
|
|
47
38
|
const from = { ...state.pos };
|
|
48
39
|
let target = from;
|
|
49
40
|
if (locator) {
|
|
50
|
-
|
|
41
|
+
// Cap the wait and swallow a timeout: an unresolvable locator must
|
|
42
|
+
// leave the (cosmetic) cursor in place, never block the flow.
|
|
43
|
+
const box = await locator
|
|
44
|
+
.boundingBox({
|
|
45
|
+
timeout: InteractionVisualizer.BOUNDING_BOX_TIMEOUT_MILLIS,
|
|
46
|
+
})
|
|
47
|
+
.catch(() => null);
|
|
51
48
|
if (box) {
|
|
52
49
|
target = { x: box.x + box.width / 2, y: box.y + box.height / 2 };
|
|
53
50
|
}
|
|
54
51
|
}
|
|
55
52
|
state.pos = target;
|
|
56
|
-
|
|
57
|
-
state
|
|
58
|
-
|
|
59
|
-
: Promise.resolve(),
|
|
60
|
-
message?.trim()
|
|
61
|
-
? this.showMessage(page, target, message.trim(), duration)
|
|
62
|
-
: Promise.resolve(),
|
|
63
|
-
]);
|
|
53
|
+
if (state.visible) {
|
|
54
|
+
await this.renderCursor(page, state, from, target, duration / 2);
|
|
55
|
+
}
|
|
64
56
|
}
|
|
65
57
|
catch (error) {
|
|
66
58
|
if (!PlaywrightUtils_1.PlaywrightUtils.isPageClosedError(error)) {
|
|
@@ -137,26 +129,15 @@ class InteractionVisualizer {
|
|
|
137
129
|
// showOverlay resolves the moment the overlay is added, before the CSS
|
|
138
130
|
// glide plays. Callers (e.g. ClickTool) await pointAt expecting the cursor
|
|
139
131
|
// to reach the element *before* they act, so block for the animation here.
|
|
140
|
-
|
|
132
|
+
// Only the moving case animates (see buildCursorHtml), so a no-op move
|
|
133
|
+
// (e.g. an unresolved target) shouldn't incur the wait.
|
|
134
|
+
const animating = animMs > 0 && (from.x !== to.x || from.y !== to.y);
|
|
135
|
+
if (animating) {
|
|
141
136
|
await new Promise((resolve) => {
|
|
142
137
|
setTimeout(resolve, animMs);
|
|
143
138
|
});
|
|
144
139
|
}
|
|
145
140
|
}
|
|
146
|
-
async showMessage(page, target, text, duration) {
|
|
147
|
-
// Sticky overlay + self-managed removal. The `duration` option on
|
|
148
|
-
// showOverlay only composites into the recorded screencast stream, not the
|
|
149
|
-
// live page render layer (so it would be invisible in headed/screenshots);
|
|
150
|
-
// a no-duration overlay renders everywhere, matching the cursor.
|
|
151
|
-
const handle = await page.screencast.showOverlay(this.buildMessageHtml(page, target, text));
|
|
152
|
-
const timer = setTimeout(() => {
|
|
153
|
-
handle.dispose().catch(() => {
|
|
154
|
-
// Page navigated/closed before the message expired — nothing to clean up.
|
|
155
|
-
});
|
|
156
|
-
}, duration);
|
|
157
|
-
// Don't let a pending message removal keep the process alive on shutdown.
|
|
158
|
-
timer.unref?.();
|
|
159
|
-
}
|
|
160
141
|
/**
|
|
161
142
|
* Builds the cursor overlay markup: the arrow SVG plus a `@keyframes` glide
|
|
162
143
|
* from the previous position and a one-shot ripple at the arrow tip.
|
|
@@ -179,41 +160,6 @@ class InteractionVisualizer {
|
|
|
179
160
|
</style>
|
|
180
161
|
<div class="donobu-cursor">${InteractionVisualizer.SVG_MOUSE}</div>`;
|
|
181
162
|
}
|
|
182
|
-
/**
|
|
183
|
-
* Builds the message tooltip markup, positioned near `target` and clamped
|
|
184
|
-
* inside the viewport. Without being able to measure the rendered box, we
|
|
185
|
-
* keep it on-screen with a conservative max-width clamp and anchor it above
|
|
186
|
-
* the cursor (via `translateY(-100%)`) when the cursor sits low in the page.
|
|
187
|
-
*/
|
|
188
|
-
buildMessageHtml(page, target, text) {
|
|
189
|
-
const maxW = InteractionVisualizer.MESSAGE_MAX_WIDTH;
|
|
190
|
-
const margin = InteractionVisualizer.MESSAGE_MARGIN;
|
|
191
|
-
const vp = page.viewportSize() ?? { width: 1280, height: 720 };
|
|
192
|
-
// Clamp the (centered) anchor so a max-width box can't overflow either edge.
|
|
193
|
-
const centerX = Math.max(maxW / 2 + margin, Math.min(vp.width - maxW / 2 - margin, target.x));
|
|
194
|
-
const placeAbove = target.y > vp.height * 0.6;
|
|
195
|
-
const top = placeAbove ? Math.max(margin, target.y - 24) : target.y + 24;
|
|
196
|
-
const translate = placeAbove
|
|
197
|
-
? 'translate(-50%, -100%)'
|
|
198
|
-
: 'translate(-50%, 0)';
|
|
199
|
-
const style = [
|
|
200
|
-
'position:fixed',
|
|
201
|
-
`left:${centerX}px`,
|
|
202
|
-
`top:${top}px`,
|
|
203
|
-
`transform:${translate}`,
|
|
204
|
-
`max-width:${maxW}px`,
|
|
205
|
-
'z-index:2147483647',
|
|
206
|
-
'pointer-events:none',
|
|
207
|
-
'background:#000',
|
|
208
|
-
'color:#fff',
|
|
209
|
-
'padding:8px 10px',
|
|
210
|
-
'border-radius:5px',
|
|
211
|
-
"font:12px/1 'Source Sans Pro','Helvetica Neue',Helvetica,Arial,sans-serif",
|
|
212
|
-
'white-space:pre-wrap',
|
|
213
|
-
'box-shadow:2px 2px 10px rgba(0,0,0,.2)',
|
|
214
|
-
].join(';');
|
|
215
|
-
return `<div style="${style}">${InteractionVisualizer.escapeHtml(text)}</div>`;
|
|
216
|
-
}
|
|
217
163
|
}
|
|
218
164
|
exports.InteractionVisualizer = InteractionVisualizer;
|
|
219
165
|
/* ------------------------------------------------------------------ */
|
|
@@ -235,7 +181,11 @@ InteractionVisualizer.SVG_MOUSE = `
|
|
|
235
181
|
stroke-linejoin="round">
|
|
236
182
|
<path d="${InteractionVisualizer.RAW_MOUSE_D}" />
|
|
237
183
|
</svg>`;
|
|
238
|
-
/**
|
|
239
|
-
|
|
240
|
-
|
|
184
|
+
/**
|
|
185
|
+
* Upper bound on resolving a target's bounding box. `Locator.boundingBox`
|
|
186
|
+
* defaults to *no* timeout, so a locator that resolves to zero elements
|
|
187
|
+
* (e.g. detached mid-flow) would otherwise wait forever and hang the flow.
|
|
188
|
+
* The cursor is cosmetic, so we cap the wait and treat a miss as a no-op.
|
|
189
|
+
*/
|
|
190
|
+
InteractionVisualizer.BOUNDING_BOX_TIMEOUT_MILLIS = 1000;
|
|
241
191
|
//# sourceMappingURL=InteractionVisualizer.js.map
|
|
@@ -432,7 +432,7 @@ class ReplayableInteraction extends Tool_1.Tool {
|
|
|
432
432
|
// Only reveal the cursor now that we have a real target to point at, so a
|
|
433
433
|
// non-interactive proposal never pops a stationary cursor.
|
|
434
434
|
await (0, TargetUtils_1.webInspector)(context).showInteractionCursor();
|
|
435
|
-
await context.interactionVisualizer.pointAt(page, pointTarget.first(),
|
|
435
|
+
await context.interactionVisualizer.pointAt(page, pointTarget.first(), ReplayableInteraction.PREVIEW_CURSOR_DURATION_MILLIS);
|
|
436
436
|
}
|
|
437
437
|
async callFromGpt(context, parameters) {
|
|
438
438
|
const page = (0, TargetUtils_1.webPage)(context);
|
package/dist/utils/MiscUtils.js
CHANGED
|
@@ -204,25 +204,21 @@ class MiscUtils {
|
|
|
204
204
|
* Merges adjacent user messages (some LLMs like Gemini's or Anthropic's require this).
|
|
205
205
|
*/
|
|
206
206
|
static mergeAdjacentUserMessages(messages) {
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
const
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
const mergedMessage = {
|
|
207
|
+
const updatedMessages = [];
|
|
208
|
+
// Fold each user message into the previously emitted one when that is also
|
|
209
|
+
// a user message. Folding into the accumulated result (rather than
|
|
210
|
+
// consuming pairs) collapses runs of any length — pair-consuming would
|
|
211
|
+
// leave the 3rd, 5th, ... message in a run un-merged.
|
|
212
|
+
for (const message of messages) {
|
|
213
|
+
const last = updatedMessages[updatedMessages.length - 1];
|
|
214
|
+
if (message.type === 'user' && last?.type === 'user') {
|
|
215
|
+
updatedMessages[updatedMessages.length - 1] = {
|
|
217
216
|
type: 'user',
|
|
218
|
-
items: [...
|
|
217
|
+
items: [...last.items, ...message.items],
|
|
219
218
|
};
|
|
220
|
-
updatedMessages.push(mergedMessage);
|
|
221
|
-
// Skip the next message.
|
|
222
|
-
++i;
|
|
223
219
|
}
|
|
224
220
|
else {
|
|
225
|
-
updatedMessages.push(
|
|
221
|
+
updatedMessages.push(message);
|
|
226
222
|
}
|
|
227
223
|
}
|
|
228
224
|
return updatedMessages;
|