@empiricalrun/test-gen 0.81.0 → 0.81.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/package.json +7 -7
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/agent/browsing/index.d.ts +0 -15
- package/dist/agent/browsing/index.d.ts.map +0 -1
- package/dist/agent/browsing/index.js +0 -60
- package/dist/agent/master/action-tool-calls.d.ts +0 -42
- package/dist/agent/master/action-tool-calls.d.ts.map +0 -1
- package/dist/agent/master/action-tool-calls.js +0 -87
- package/dist/agent/master/element-annotation.d.ts +0 -30
- package/dist/agent/master/element-annotation.d.ts.map +0 -1
- package/dist/agent/master/element-annotation.js +0 -175
- package/dist/agent/master/execute-browser-action.d.ts +0 -24
- package/dist/agent/master/execute-browser-action.d.ts.map +0 -1
- package/dist/agent/master/execute-browser-action.js +0 -121
- package/dist/agent/master/next-action.d.ts +0 -22
- package/dist/agent/master/next-action.d.ts.map +0 -1
- package/dist/agent/master/next-action.js +0 -97
- package/dist/agent/master/planner.d.ts +0 -15
- package/dist/agent/master/planner.d.ts.map +0 -1
- package/dist/agent/master/planner.js +0 -142
- package/dist/agent/master/run.d.ts +0 -17
- package/dist/agent/master/run.d.ts.map +0 -1
- package/dist/agent/master/run.js +0 -156
- package/dist/agent/master/scroller.d.ts +0 -15
- package/dist/agent/master/scroller.d.ts.map +0 -1
- package/dist/agent/master/scroller.js +0 -369
- package/dist/agent/master/with-hints.d.ts +0 -17
- package/dist/agent/master/with-hints.d.ts.map +0 -1
- package/dist/agent/master/with-hints.js +0 -103
- package/dist/agent/planner/run-time-planner.d.ts +0 -15
- package/dist/agent/planner/run-time-planner.d.ts.map +0 -1
- package/dist/agent/planner/run-time-planner.js +0 -98
- package/dist/agent/planner/run.d.ts +0 -7
- package/dist/agent/planner/run.d.ts.map +0 -1
- package/dist/agent/planner/run.js +0 -128
- package/dist/browser-injected-scripts/annotate-elements.js +0 -612
- package/dist/browser-injected-scripts/annotate-elements.spec.d.ts +0 -2
- package/dist/browser-injected-scripts/annotate-elements.spec.d.ts.map +0 -1
- package/dist/browser-injected-scripts/annotate-elements.spec.js +0 -202
- package/dist/browser-injected-scripts/annotate-elements.spec.ts +0 -327
- package/dist/generate-summary/frame-sampling.d.ts +0 -12
- package/dist/generate-summary/frame-sampling.d.ts.map +0 -1
- package/dist/generate-summary/frame-sampling.js +0 -72
- package/dist/generate-summary/generate-error-stack-summary.d.ts +0 -11
- package/dist/generate-summary/generate-error-stack-summary.d.ts.map +0 -1
- package/dist/generate-summary/generate-error-stack-summary.js +0 -41
- package/dist/generate-summary/generate-failed-step-screenshot-diff-summary.d.ts +0 -58
- package/dist/generate-summary/generate-failed-step-screenshot-diff-summary.d.ts.map +0 -1
- package/dist/generate-summary/generate-failed-step-screenshot-diff-summary.js +0 -460
- package/dist/generate-summary/generate-grouped-summary.d.ts +0 -18
- package/dist/generate-summary/generate-grouped-summary.d.ts.map +0 -1
- package/dist/generate-summary/generate-grouped-summary.js +0 -88
- package/dist/generate-summary/merge-summary.d.ts +0 -16
- package/dist/generate-summary/merge-summary.d.ts.map +0 -1
- package/dist/generate-summary/merge-summary.js +0 -43
- package/dist/generate-summary/pick-videos-for-comparison.d.ts +0 -9
- package/dist/generate-summary/pick-videos-for-comparison.d.ts.map +0 -1
- package/dist/generate-summary/pick-videos-for-comparison.js +0 -54
- package/dist/utils/env.d.ts +0 -2
- package/dist/utils/env.d.ts.map +0 -1
- package/dist/utils/env.js +0 -7
|
@@ -1,369 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.scroller = scroller;
|
|
4
|
-
const llm_1 = require("@empiricalrun/llm");
|
|
5
|
-
const vision_1 = require("@empiricalrun/llm/vision");
|
|
6
|
-
const constants_1 = require("../../constants");
|
|
7
|
-
const action_tool_calls_1 = require("./action-tool-calls");
|
|
8
|
-
const element_annotation_1 = require("./element-annotation");
|
|
9
|
-
let usedAnnotations = [];
|
|
10
|
-
// This checks whether scroll is possible or not
|
|
11
|
-
// If the div annotation is undefined, we check the scrollability of the page
|
|
12
|
-
// else we check for the element
|
|
13
|
-
async function isScrollable({ scrollDirection, page, divAnnotation, }) {
|
|
14
|
-
if (divAnnotation) {
|
|
15
|
-
return await page.evaluate(({ scrollDirection, divAnnotation }) => {
|
|
16
|
-
if (
|
|
17
|
-
// @ts-ignore
|
|
18
|
-
window?.annotationInstance?.annotations?.[divAnnotation]) {
|
|
19
|
-
let element = // @ts-ignore
|
|
20
|
-
window?.annotationInstance?.annotations?.[divAnnotation]?.node;
|
|
21
|
-
if (scrollDirection === "up") {
|
|
22
|
-
return element.scrollTop > 0;
|
|
23
|
-
}
|
|
24
|
-
else {
|
|
25
|
-
return (element.scrollHeight > element.clientHeight + element.scrollTop);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
return false;
|
|
29
|
-
}, { scrollDirection, divAnnotation });
|
|
30
|
-
}
|
|
31
|
-
return await page.evaluate(({ scrollDirection }) => {
|
|
32
|
-
if (scrollDirection === "up") {
|
|
33
|
-
return window.scrollY > 0;
|
|
34
|
-
}
|
|
35
|
-
else {
|
|
36
|
-
return (document.documentElement.scrollHeight >
|
|
37
|
-
window.innerHeight + window.scrollY);
|
|
38
|
-
}
|
|
39
|
-
}, { scrollDirection });
|
|
40
|
-
}
|
|
41
|
-
// Performs scroll on page or the element
|
|
42
|
-
// If the div annotation is undefined, we scroll the page
|
|
43
|
-
// else we scroll the element
|
|
44
|
-
async function scroll({ scrollBy, page, direction, divAnnotation, logger, }) {
|
|
45
|
-
if (divAnnotation) {
|
|
46
|
-
logger?.log("Scrolling the div since element is not in view");
|
|
47
|
-
return await page.evaluate(({ scrollBy, direction, divAnnotation }) => {
|
|
48
|
-
if (
|
|
49
|
-
// @ts-ignore
|
|
50
|
-
window?.annotationInstance?.annotations?.[divAnnotation]) {
|
|
51
|
-
let element = // @ts-ignore
|
|
52
|
-
window?.annotationInstance?.annotations?.[divAnnotation]?.node;
|
|
53
|
-
let scrollHeight = scrollBy || element.clientHeight;
|
|
54
|
-
element.scrollBy(0, scrollHeight * (direction === "up" ? -1 : 1));
|
|
55
|
-
return scrollHeight;
|
|
56
|
-
}
|
|
57
|
-
return 0;
|
|
58
|
-
}, { scrollBy, direction, divAnnotation });
|
|
59
|
-
}
|
|
60
|
-
logger?.log("Scrolling the page since element is not in view");
|
|
61
|
-
return await page.evaluate(({ scrollBy, direction }) => {
|
|
62
|
-
let scrollHeight = scrollBy || window.innerHeight;
|
|
63
|
-
window.scrollBy(0, scrollHeight * (direction === "up" ? -1 : 1));
|
|
64
|
-
return scrollHeight;
|
|
65
|
-
}, { scrollBy, direction });
|
|
66
|
-
}
|
|
67
|
-
// Scrolls the page to top
|
|
68
|
-
async function scrollToTop(page) {
|
|
69
|
-
await page.evaluate(() => {
|
|
70
|
-
window.scrollTo({ top: 0 });
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
// Checks if the element is visible in the current frame
|
|
74
|
-
async function isElementVisibleInFrame({ elementDescription, page, trace, logger, }) {
|
|
75
|
-
const buffer = await page.screenshot();
|
|
76
|
-
const frameScreenshot = buffer.toString("base64");
|
|
77
|
-
const systemMessage = {
|
|
78
|
-
role: "system",
|
|
79
|
-
content: `
|
|
80
|
-
You are a web automation tool having extraordinary capabilities of going through the webpage screenshots. You are given a task to identify whether the element with given element description is present on the screen.
|
|
81
|
-
You need to make a decision whether the element is visible or not. Only consider an element to be visible if it's completely visible, otherwise respond false.`,
|
|
82
|
-
};
|
|
83
|
-
const userMessage = {
|
|
84
|
-
role: "user",
|
|
85
|
-
content: [
|
|
86
|
-
{
|
|
87
|
-
type: "text",
|
|
88
|
-
text: `
|
|
89
|
-
Element description:
|
|
90
|
-
${elementDescription}
|
|
91
|
-
|
|
92
|
-
----
|
|
93
|
-
|
|
94
|
-
Follow the instructions before responding:
|
|
95
|
-
- Scan through the content in the screenshot
|
|
96
|
-
- While scanning check whether there is any element for which the given element description matches.
|
|
97
|
-
- If it matches set is_visible as true
|
|
98
|
-
- else set is_visible as false
|
|
99
|
-
`,
|
|
100
|
-
},
|
|
101
|
-
{
|
|
102
|
-
type: "text",
|
|
103
|
-
text: "Screenshot",
|
|
104
|
-
},
|
|
105
|
-
{
|
|
106
|
-
type: "image_url",
|
|
107
|
-
image_url: {
|
|
108
|
-
url: (0, vision_1.imageFormatForProvider)(constants_1.DEFAULT_MODEL_PROVIDER, frameScreenshot),
|
|
109
|
-
},
|
|
110
|
-
},
|
|
111
|
-
],
|
|
112
|
-
};
|
|
113
|
-
const tool = {
|
|
114
|
-
type: "function",
|
|
115
|
-
function: {
|
|
116
|
-
name: "is-element-visible",
|
|
117
|
-
description: "Is the element with given element description present in the screenshot",
|
|
118
|
-
parameters: {
|
|
119
|
-
type: "object",
|
|
120
|
-
properties: {
|
|
121
|
-
reason: {
|
|
122
|
-
type: "string",
|
|
123
|
-
description: "Explain why the element is marked as visible and its location. The reason should be clear and concise.",
|
|
124
|
-
},
|
|
125
|
-
is_visible: {
|
|
126
|
-
type: "boolean",
|
|
127
|
-
description: "Boolean value for whether the element is completely visible in the screenshot.",
|
|
128
|
-
},
|
|
129
|
-
},
|
|
130
|
-
required: ["reason", "is_visible"],
|
|
131
|
-
},
|
|
132
|
-
},
|
|
133
|
-
};
|
|
134
|
-
const messages = [
|
|
135
|
-
systemMessage,
|
|
136
|
-
userMessage,
|
|
137
|
-
];
|
|
138
|
-
const scrollSpan = trace?.span({
|
|
139
|
-
name: "is-element-visible-after-scroll",
|
|
140
|
-
input: {
|
|
141
|
-
elementDescription,
|
|
142
|
-
messages,
|
|
143
|
-
},
|
|
144
|
-
});
|
|
145
|
-
const llm = new llm_1.LLM({
|
|
146
|
-
provider: constants_1.DEFAULT_MODEL_PROVIDER,
|
|
147
|
-
defaultModel: constants_1.DEFAULT_MODEL,
|
|
148
|
-
});
|
|
149
|
-
const completion = await llm.createChatCompletion({
|
|
150
|
-
messages,
|
|
151
|
-
modelParameters: {
|
|
152
|
-
...constants_1.DEFAULT_MODEL_PARAMETERS,
|
|
153
|
-
tool_choice: "required",
|
|
154
|
-
temperature: 1,
|
|
155
|
-
},
|
|
156
|
-
trace: scrollSpan,
|
|
157
|
-
tools: [tool],
|
|
158
|
-
});
|
|
159
|
-
let isVisible = false;
|
|
160
|
-
const toolCall = completion?.tool_calls?.[0];
|
|
161
|
-
scrollSpan?.end({ output: toolCall });
|
|
162
|
-
if (toolCall && "function" in toolCall) {
|
|
163
|
-
const args = JSON.parse(toolCall.function.arguments);
|
|
164
|
-
isVisible = args.is_visible || false;
|
|
165
|
-
}
|
|
166
|
-
else {
|
|
167
|
-
logger?.error(`No tool call found in completion. [Trace](${trace?.getTraceUrl()})`);
|
|
168
|
-
}
|
|
169
|
-
return {
|
|
170
|
-
isVisible,
|
|
171
|
-
frameScreenshot,
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
// Returns the element annotation to scroll on
|
|
175
|
-
// if there is no element matching the description we return "NA"
|
|
176
|
-
async function getDivAnnotationToScrollOn({ elementDescription, page, trace, logger, }) {
|
|
177
|
-
const preference = {
|
|
178
|
-
actionType: action_tool_calls_1.ActionType.SCROLL,
|
|
179
|
-
};
|
|
180
|
-
let { annotationKeys, annotatedPageScreenshot } = await (0, element_annotation_1.getAnnotationKeys)({
|
|
181
|
-
page,
|
|
182
|
-
preference: {
|
|
183
|
-
actionType: action_tool_calls_1.ActionType.SCROLL,
|
|
184
|
-
},
|
|
185
|
-
trace,
|
|
186
|
-
});
|
|
187
|
-
// Remove the used annotations from the list
|
|
188
|
-
annotationKeys = annotationKeys.filter((key) => !usedAnnotations.includes(key.elementID));
|
|
189
|
-
if (annotationKeys.length === 0) {
|
|
190
|
-
return;
|
|
191
|
-
}
|
|
192
|
-
const annotationKeysString = annotationKeys
|
|
193
|
-
?.map((a) => `${a.elementID}:${a.text}`)
|
|
194
|
-
.join("\n");
|
|
195
|
-
const annotationsSpan = trace?.span({
|
|
196
|
-
name: "get-div-annotation",
|
|
197
|
-
input: {
|
|
198
|
-
elementDescription,
|
|
199
|
-
annotationKeys,
|
|
200
|
-
annotatedPageScreenshot,
|
|
201
|
-
preference,
|
|
202
|
-
},
|
|
203
|
-
});
|
|
204
|
-
const systemMessage = {
|
|
205
|
-
role: "system",
|
|
206
|
-
content: `
|
|
207
|
-
You are an expert in describing the images and it's content. You will be provided with an annotated screenshot where scrollable divs are annotated.
|
|
208
|
-
|
|
209
|
-
The annotation is done by drawing a red box around the element and a small yellow box on it which contains unique element id.
|
|
210
|
-
|
|
211
|
-
You are given a "Annotations" which contains list of unique element Id and description of the element separated by ":".
|
|
212
|
-
|
|
213
|
-
You are also given the description of the element which can be present inside the annotated element. The description includes information about how the element looks, it's position etc.
|
|
214
|
-
|
|
215
|
-
Your task is to provide the annotation of the div on which the scroll action needs to be performed based on whether the provided element is inside the div. You can use the description of the annotated element as well. E.g. text in the div contains the text as described in the element description.
|
|
216
|
-
|
|
217
|
-
You also need to extract the relevant information like the element text or position from the provided element description, this can be used to match the div content with the element.
|
|
218
|
-
|
|
219
|
-
If there is a match provide the annotation for the div else respond with "NA".
|
|
220
|
-
|
|
221
|
-
Follow steps to fulfil your task:
|
|
222
|
-
- If the provided "Annotations" is empty, respond with "NA"
|
|
223
|
-
- For each element Id, read the description for the div element
|
|
224
|
-
- If the description contains the element description or anything on similar lines
|
|
225
|
-
- Respond with the element Id
|
|
226
|
-
- If none of the description contains the element description respond with "NA"
|
|
227
|
-
- If the specified element Id is not found in the "Annotations" section, the response is invalid.
|
|
228
|
-
`,
|
|
229
|
-
};
|
|
230
|
-
const userMessage = {
|
|
231
|
-
role: "user",
|
|
232
|
-
content: [
|
|
233
|
-
{
|
|
234
|
-
type: "text",
|
|
235
|
-
text: `
|
|
236
|
-
Element description:
|
|
237
|
-
${elementDescription}
|
|
238
|
-
|
|
239
|
-
Annotations:
|
|
240
|
-
${annotationKeysString}`,
|
|
241
|
-
},
|
|
242
|
-
{
|
|
243
|
-
type: "image_url",
|
|
244
|
-
image_url: {
|
|
245
|
-
url: (0, vision_1.imageFormatForProvider)(constants_1.DEFAULT_MODEL_PROVIDER, annotatedPageScreenshot),
|
|
246
|
-
},
|
|
247
|
-
},
|
|
248
|
-
],
|
|
249
|
-
};
|
|
250
|
-
const messages = [
|
|
251
|
-
systemMessage,
|
|
252
|
-
userMessage,
|
|
253
|
-
];
|
|
254
|
-
const annotationToolAction = {
|
|
255
|
-
name: "element_annotation",
|
|
256
|
-
schema: {
|
|
257
|
-
type: "function",
|
|
258
|
-
function: {
|
|
259
|
-
name: "element_annotation",
|
|
260
|
-
description: "Handles annotations for elements",
|
|
261
|
-
parameters: {
|
|
262
|
-
type: "object",
|
|
263
|
-
properties: {
|
|
264
|
-
element: {
|
|
265
|
-
type: "string",
|
|
266
|
-
description: "Relevant information from the provided element description.",
|
|
267
|
-
},
|
|
268
|
-
reason: {
|
|
269
|
-
type: "string",
|
|
270
|
-
description: "Explain why this element is selected. The reason should be clear and align with the task or purpose.",
|
|
271
|
-
},
|
|
272
|
-
element_annotation: {
|
|
273
|
-
type: "string",
|
|
274
|
-
description: "Return the unique element ID for the element on which the action needs to be performed.",
|
|
275
|
-
},
|
|
276
|
-
},
|
|
277
|
-
required: ["element", "reason", "element_annotation"],
|
|
278
|
-
},
|
|
279
|
-
},
|
|
280
|
-
},
|
|
281
|
-
};
|
|
282
|
-
const llm = new llm_1.LLM({
|
|
283
|
-
provider: constants_1.DEFAULT_MODEL_PROVIDER,
|
|
284
|
-
defaultModel: constants_1.DEFAULT_MODEL,
|
|
285
|
-
});
|
|
286
|
-
const completion = await llm.createChatCompletion({
|
|
287
|
-
messages,
|
|
288
|
-
modelParameters: {
|
|
289
|
-
...constants_1.DEFAULT_MODEL_PARAMETERS,
|
|
290
|
-
tool_choice: "required",
|
|
291
|
-
temperature: 1,
|
|
292
|
-
},
|
|
293
|
-
trace: annotationsSpan,
|
|
294
|
-
traceName: "get-element-from-action",
|
|
295
|
-
//@ts-ignore
|
|
296
|
-
tools: [annotationToolAction.schema],
|
|
297
|
-
});
|
|
298
|
-
const toolCall = completion?.tool_calls?.[0];
|
|
299
|
-
annotationsSpan?.end({ output: toolCall });
|
|
300
|
-
if (toolCall && "function" in toolCall) {
|
|
301
|
-
const args = JSON.parse(toolCall.function.arguments);
|
|
302
|
-
const isAnnotationPresentInKeys = annotationKeys.some((annotation) => annotation.elementID === args.element_annotation);
|
|
303
|
-
if (args.element_annotation !== "NA" && isAnnotationPresentInKeys) {
|
|
304
|
-
usedAnnotations.push(args.element_annotation);
|
|
305
|
-
return args.element_annotation;
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
else {
|
|
309
|
-
logger?.error(`No tool call found in completion. [Trace](${trace?.getTraceUrl()})`);
|
|
310
|
-
}
|
|
311
|
-
return;
|
|
312
|
-
}
|
|
313
|
-
// Scrolls the page and returns the reference to the frame in which the element is visible
|
|
314
|
-
async function scroller({ elementDescription, page, trace, frameReference, logger, }) {
|
|
315
|
-
await scrollToTop(page);
|
|
316
|
-
if (frameReference) {
|
|
317
|
-
await scroll({
|
|
318
|
-
scrollBy: frameReference.scrollPosition,
|
|
319
|
-
page,
|
|
320
|
-
direction: "down",
|
|
321
|
-
});
|
|
322
|
-
return [frameReference];
|
|
323
|
-
}
|
|
324
|
-
let referenceImages = [];
|
|
325
|
-
let scrolledHeight = 0;
|
|
326
|
-
let isScrollAvailable = true;
|
|
327
|
-
const divAnnotation = await getDivAnnotationToScrollOn({
|
|
328
|
-
elementDescription,
|
|
329
|
-
page,
|
|
330
|
-
trace,
|
|
331
|
-
logger,
|
|
332
|
-
});
|
|
333
|
-
while (isScrollAvailable) {
|
|
334
|
-
// Perform action to check for visibility
|
|
335
|
-
const { isVisible, frameScreenshot } = await isElementVisibleInFrame({
|
|
336
|
-
elementDescription,
|
|
337
|
-
page,
|
|
338
|
-
trace,
|
|
339
|
-
logger,
|
|
340
|
-
});
|
|
341
|
-
if (isVisible) {
|
|
342
|
-
usedAnnotations = [];
|
|
343
|
-
referenceImages.push({
|
|
344
|
-
scrollPosition: scrolledHeight,
|
|
345
|
-
frameScreenshot,
|
|
346
|
-
});
|
|
347
|
-
// TODO: remove this to support multiple images
|
|
348
|
-
break;
|
|
349
|
-
}
|
|
350
|
-
isScrollAvailable = await isScrollable({
|
|
351
|
-
scrollDirection: "down",
|
|
352
|
-
page,
|
|
353
|
-
divAnnotation,
|
|
354
|
-
});
|
|
355
|
-
if (isScrollAvailable) {
|
|
356
|
-
scrolledHeight += await scroll({
|
|
357
|
-
page,
|
|
358
|
-
divAnnotation,
|
|
359
|
-
direction: "down",
|
|
360
|
-
logger,
|
|
361
|
-
});
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
if (referenceImages.length === 0) {
|
|
365
|
-
logger?.log("Element not found in the current scroll");
|
|
366
|
-
}
|
|
367
|
-
await (0, llm_1.flushAllTraces)();
|
|
368
|
-
return referenceImages;
|
|
369
|
-
}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import { LLM, TraceClient } from "@empiricalrun/llm";
|
|
2
|
-
import OpenAI from "openai";
|
|
3
|
-
import { PlaywrightActions } from "../../actions";
|
|
4
|
-
export declare const triggerHintsFlow: ({ outputFromGetNextAction, generatedAnnotations, actions, llm, trace, }: {
|
|
5
|
-
outputFromGetNextAction: {
|
|
6
|
-
action: string;
|
|
7
|
-
elementAnnotation?: string;
|
|
8
|
-
};
|
|
9
|
-
generatedAnnotations: Record<string, any>;
|
|
10
|
-
actions: PlaywrightActions;
|
|
11
|
-
llm: LLM;
|
|
12
|
-
trace?: TraceClient;
|
|
13
|
-
}) => Promise<{
|
|
14
|
-
shouldTriggerHintsFlow: boolean;
|
|
15
|
-
hintsExecutionCompletion: OpenAI.ChatCompletionMessage | undefined;
|
|
16
|
-
}>;
|
|
17
|
-
//# sourceMappingURL=with-hints.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"with-hints.d.ts","sourceRoot":"","sources":["../../../src/agent/master/with-hints.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,MAAM,MAAM,QAAQ,CAAC;AAE5B,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAOlD,eAAO,MAAM,gBAAgB,GAAU,yEAMpC;IACD,uBAAuB,EAAE;QACvB,MAAM,EAAE,MAAM,CAAC;QACf,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,oBAAoB,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1C,OAAO,EAAE,iBAAiB,CAAC;IAC3B,GAAG,EAAE,GAAG,CAAC;IACT,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,KAAG,OAAO,CAAC;IACV,sBAAsB,EAAE,OAAO,CAAC;IAChC,wBAAwB,EAAE,MAAM,CAAC,qBAAqB,GAAG,SAAS,CAAC;CACpE,CA0GA,CAAC"}
|
|
@@ -1,103 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.triggerHintsFlow = void 0;
|
|
4
|
-
const assert_1 = require("../../actions/assert");
|
|
5
|
-
const click_1 = require("../../actions/click");
|
|
6
|
-
const fill_1 = require("../../actions/fill");
|
|
7
|
-
const hover_1 = require("../../actions/hover");
|
|
8
|
-
const constants_1 = require("../../constants");
|
|
9
|
-
const triggerHintsFlow = async ({ outputFromGetNextAction, generatedAnnotations, actions, llm, trace, }) => {
|
|
10
|
-
try {
|
|
11
|
-
const hasElementAnnotation = outputFromGetNextAction?.elementAnnotation?.length &&
|
|
12
|
-
outputFromGetNextAction?.elementAnnotation?.trim()?.length &&
|
|
13
|
-
generatedAnnotations.some(
|
|
14
|
-
//@ts-ignore
|
|
15
|
-
(annotation) => annotation.elementID === outputFromGetNextAction?.elementAnnotation);
|
|
16
|
-
trace?.event({
|
|
17
|
-
name: "has-element-annotation",
|
|
18
|
-
output: {
|
|
19
|
-
hasElementAnnotation,
|
|
20
|
-
generatedAnnotations,
|
|
21
|
-
},
|
|
22
|
-
});
|
|
23
|
-
if (!hasElementAnnotation) {
|
|
24
|
-
return {
|
|
25
|
-
shouldTriggerHintsFlow: false,
|
|
26
|
-
hintsExecutionCompletion: undefined,
|
|
27
|
-
};
|
|
28
|
-
}
|
|
29
|
-
const completion = await llm.createChatCompletion({
|
|
30
|
-
messages: [
|
|
31
|
-
{
|
|
32
|
-
role: "system",
|
|
33
|
-
content: `You are a browser automation agent who is given a task to generate code for navigation and assertion. This task is your goal and you must achieve it.
|
|
34
|
-
|
|
35
|
-
You need to pick the next step to achieve the task. Remember that the goal must be achieved.
|
|
36
|
-
|
|
37
|
-
Based on the goal and available tool calls you need to pick the appropriate tool call.
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
Instructions:
|
|
41
|
-
- Take actions one at a time. Do not try to take multiple actions
|
|
42
|
-
- You can respond with multiple assertions in one shot
|
|
43
|
-
- Stick to the task provided to you and mark the task done once the task is complete
|
|
44
|
-
- Do not execute any action which is not mentioned in the task`,
|
|
45
|
-
},
|
|
46
|
-
{
|
|
47
|
-
role: "user",
|
|
48
|
-
content: [
|
|
49
|
-
{
|
|
50
|
-
type: "text",
|
|
51
|
-
text: `Task: ${outputFromGetNextAction?.action}`,
|
|
52
|
-
},
|
|
53
|
-
],
|
|
54
|
-
},
|
|
55
|
-
],
|
|
56
|
-
tools: actions.getBrowsingActionSchemas(),
|
|
57
|
-
modelParameters: {
|
|
58
|
-
temperature: constants_1.DEFAULT_MODEL_PARAMETERS.temperature,
|
|
59
|
-
max_completion_tokens: constants_1.DEFAULT_MODEL_PARAMETERS.max_completion_tokens,
|
|
60
|
-
tool_choice: "required",
|
|
61
|
-
},
|
|
62
|
-
trace,
|
|
63
|
-
});
|
|
64
|
-
const canTriggerHintsFlow = completion?.tool_calls?.some((currentToolCall) => "function" in currentToolCall &&
|
|
65
|
-
(currentToolCall.function.name === fill_1.PLAYWRIGHT_FILL_ACTION_NAME ||
|
|
66
|
-
currentToolCall.function.name === click_1.PLAYWRIGHT_CLICK_ACTION_NAME ||
|
|
67
|
-
currentToolCall.function.name === hover_1.PLAYWRIGHT_HOVER_ACTION_NAME ||
|
|
68
|
-
currentToolCall.function.name ===
|
|
69
|
-
assert_1.PLAYWRIGHT_ASSERT_TEXT_VISIBILITY_ACTION_NAME));
|
|
70
|
-
trace?.event({
|
|
71
|
-
name: "can-trigger-hints-flow",
|
|
72
|
-
output: {
|
|
73
|
-
canTriggerHintsFlow,
|
|
74
|
-
},
|
|
75
|
-
});
|
|
76
|
-
if (!canTriggerHintsFlow) {
|
|
77
|
-
return {
|
|
78
|
-
shouldTriggerHintsFlow: false,
|
|
79
|
-
hintsExecutionCompletion: undefined,
|
|
80
|
-
};
|
|
81
|
-
}
|
|
82
|
-
return {
|
|
83
|
-
shouldTriggerHintsFlow: true,
|
|
84
|
-
// @ts-ignore
|
|
85
|
-
hintsExecutionCompletion: completion,
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
catch (e) {
|
|
89
|
-
console.error("Error in triggerHintsFlow", e);
|
|
90
|
-
trace?.event({
|
|
91
|
-
name: "trigger-hints-flow-error",
|
|
92
|
-
output: {
|
|
93
|
-
errorMessage: e?.message,
|
|
94
|
-
errorStack: e?.stack,
|
|
95
|
-
},
|
|
96
|
-
});
|
|
97
|
-
return {
|
|
98
|
-
shouldTriggerHintsFlow: false,
|
|
99
|
-
hintsExecutionCompletion: undefined,
|
|
100
|
-
};
|
|
101
|
-
}
|
|
102
|
-
};
|
|
103
|
-
exports.triggerHintsFlow = triggerHintsFlow;
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
import { TraceClient } from "@empiricalrun/llm";
|
|
2
|
-
import type { Page } from "playwright";
|
|
3
|
-
import { TestGenPage } from "../../types";
|
|
4
|
-
export declare function runtimePlanner({ trace, task, successfulActions, pages, currentPage, }: {
|
|
5
|
-
trace?: TraceClient;
|
|
6
|
-
successfulActions: string[];
|
|
7
|
-
task: string;
|
|
8
|
-
pages: Record<string, Page>;
|
|
9
|
-
currentPage: TestGenPage;
|
|
10
|
-
}): Promise<{
|
|
11
|
-
pageName: string;
|
|
12
|
-
isDone: boolean;
|
|
13
|
-
reason: string;
|
|
14
|
-
}>;
|
|
15
|
-
//# sourceMappingURL=run-time-planner.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"run-time-planner.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run-time-planner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACpE,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,wBAAsB,cAAc,CAAC,EACnC,KAAK,EACL,IAAI,EACJ,iBAAiB,EACjB,KAAK,EACL,WAAW,GACZ,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;CAC1B;cA6EoC,MAAM;YACV,OAAO;YACP,MAAM;GAgBtC"}
|
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.runtimePlanner = runtimePlanner;
|
|
4
|
-
const llm_1 = require("@empiricalrun/llm");
|
|
5
|
-
const promptTemplate_0 = "{{#section \"system\"}}\nYou are given a list of successfully executed actions that are done towards completing a task (which\nis also provided to you). Your goal is to analyse the list and determine if the task is completed.\n\nIf the task is not fully completed, identify which specific actions are missing\nand suggest next steps to complete the task. Assume that the conversation provided\nis entirely truthful and no additional actions were performed beyond those listed.\n\nThese actions were executed by AI agents using Playwright on a browser. These agents\nalready have access to browser tabs to execute actions. If there is a pending action,\none of the agents will execute it in the browser. However, they need your help to\nchoose which browser tab (= page) to use for the next action.\n\nTo fulfil your goal, follow these steps:\n- Divide the task into individual actions.\n- Compare each task action against the actions listed in the successfully executed actions list.\n- Identify which actions have been executed and which have not.\n- If all actions are executed, respond with the task as done.\n- If any actions are missing, respond with the task as not done, listing all actions\n and specifying which are complete and which are missing.\n- If provided with list of pages, based on the next pending action and previously executed\n action, identify the page on which next action needs to be taken\n{{/section}}\n\n{{#section \"user\"}}\nTask:\n{{task}}\n\n----\n\nSuccessfully executed actions:\n{{successfulActions}}\n\n----\n\nList of pages with their current URLs:\n{{pagesSummary}}\n\n\n{{/section}}\n";
|
|
6
|
-
async function runtimePlanner({ trace, task, successfulActions, pages, currentPage, }) {
|
|
7
|
-
const runTimePlannerSpan = trace?.span({
|
|
8
|
-
name: "runtime-planner",
|
|
9
|
-
input: {
|
|
10
|
-
task,
|
|
11
|
-
successfulActions,
|
|
12
|
-
currentPage,
|
|
13
|
-
},
|
|
14
|
-
});
|
|
15
|
-
const llm = new llm_1.LLM({ provider: "openai" });
|
|
16
|
-
// Add the current page to the page state variables
|
|
17
|
-
pages[currentPage.name] = currentPage.pwPageInstance;
|
|
18
|
-
const pagesSummary = pages
|
|
19
|
-
? Object.entries(pages).map(([pageName, page]) => `${pageName}: Currently on ${page.url()}`)
|
|
20
|
-
: [];
|
|
21
|
-
const pageNamesEnum = Object.keys(pages);
|
|
22
|
-
const response = await llm.createChatCompletion({
|
|
23
|
-
trace: runTimePlannerSpan,
|
|
24
|
-
traceName: "runtime-planner-llm",
|
|
25
|
-
model: "gpt-4o",
|
|
26
|
-
messages: (0, llm_1.compilePrompt)(promptTemplate_0, {
|
|
27
|
-
task,
|
|
28
|
-
successfulActions: successfulActions.join("\n"),
|
|
29
|
-
pagesSummary: pagesSummary.join("\n"),
|
|
30
|
-
}),
|
|
31
|
-
tools: [
|
|
32
|
-
{
|
|
33
|
-
type: "function",
|
|
34
|
-
function: {
|
|
35
|
-
name: "task_done",
|
|
36
|
-
description: "end the task by calling this method",
|
|
37
|
-
parameters: {
|
|
38
|
-
type: "object",
|
|
39
|
-
properties: {
|
|
40
|
-
actions: {
|
|
41
|
-
type: "string",
|
|
42
|
-
description: "actions extracted from task",
|
|
43
|
-
},
|
|
44
|
-
successful_actions: {
|
|
45
|
-
type: "string",
|
|
46
|
-
description: "successful actions mentioned in the conversation",
|
|
47
|
-
},
|
|
48
|
-
reason: {
|
|
49
|
-
type: "string",
|
|
50
|
-
description: "reasoning for identification of task status",
|
|
51
|
-
},
|
|
52
|
-
nextAction: {
|
|
53
|
-
type: "string",
|
|
54
|
-
description: "next action to be taken",
|
|
55
|
-
},
|
|
56
|
-
pageName: {
|
|
57
|
-
type: "string",
|
|
58
|
-
enum: pageNamesEnum,
|
|
59
|
-
description: "page name for the next action.",
|
|
60
|
-
},
|
|
61
|
-
isDone: {
|
|
62
|
-
type: "boolean",
|
|
63
|
-
description: "whether the task is done",
|
|
64
|
-
},
|
|
65
|
-
},
|
|
66
|
-
required: ["isDone", "reason", "pageName", "nextAction"],
|
|
67
|
-
},
|
|
68
|
-
},
|
|
69
|
-
},
|
|
70
|
-
],
|
|
71
|
-
modelParameters: {
|
|
72
|
-
tool_choice: "required",
|
|
73
|
-
temperature: 0.5,
|
|
74
|
-
},
|
|
75
|
-
});
|
|
76
|
-
const toolCallResp = (response?.tool_calls || [])[0];
|
|
77
|
-
if (toolCallResp && "function" in toolCallResp) {
|
|
78
|
-
const toolCall = JSON.parse(toolCallResp.function.arguments);
|
|
79
|
-
const output = {
|
|
80
|
-
pageName: toolCall.pageName,
|
|
81
|
-
isDone: toolCall.isDone,
|
|
82
|
-
reason: toolCall.reason,
|
|
83
|
-
};
|
|
84
|
-
runTimePlannerSpan?.end({
|
|
85
|
-
output,
|
|
86
|
-
});
|
|
87
|
-
return output;
|
|
88
|
-
}
|
|
89
|
-
const output = {
|
|
90
|
-
pageName: "",
|
|
91
|
-
isDone: false,
|
|
92
|
-
reason: "LLM failed to generate a valid response",
|
|
93
|
-
};
|
|
94
|
-
runTimePlannerSpan?.end({
|
|
95
|
-
output,
|
|
96
|
-
});
|
|
97
|
-
return output;
|
|
98
|
-
}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/planner/run.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAcpE,wBAAsB,QAAQ,CAAC,EAC7B,IAAI,EACJ,QAAQ,EACR,KAAK,GACN,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,gBA6HA"}
|