@empiricalrun/test-gen 0.42.8 → 0.42.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +20 -0
- package/dist/agent/browsing/utils.d.ts.map +1 -1
- package/dist/agent/browsing/utils.js +1 -0
- package/dist/agent/codegen/create-test-block.d.ts.map +1 -1
- package/dist/agent/codegen/create-test-block.js +4 -2
- package/dist/agent/codegen/lexical-scoped-vars.d.ts.map +1 -1
- package/dist/agent/codegen/lexical-scoped-vars.js +6 -9
- package/dist/agent/codegen/promptBuilder.d.ts +3 -0
- package/dist/agent/codegen/promptBuilder.d.ts.map +1 -0
- package/dist/agent/codegen/promptBuilder.js +44 -0
- package/dist/agent/master/action-tool-calls.d.ts +6 -1
- package/dist/agent/master/action-tool-calls.d.ts.map +1 -1
- package/dist/agent/master/action-tool-calls.js +14 -1
- package/dist/agent/master/element-annotation.d.ts +1 -1
- package/dist/agent/master/element-annotation.d.ts.map +1 -1
- package/dist/agent/master/element-annotation.js +12 -3
- package/dist/agent/master/next-action.d.ts +10 -5
- package/dist/agent/master/next-action.d.ts.map +1 -1
- package/dist/agent/master/next-action.js +59 -11
- package/dist/agent/master/run.d.ts.map +1 -1
- package/dist/agent/master/run.js +16 -16
- package/dist/agent/master/scroller.d.ts +15 -0
- package/dist/agent/master/scroller.d.ts.map +1 -0
- package/dist/agent/master/scroller.js +375 -0
- package/dist/agent/utils.d.ts +2 -0
- package/dist/agent/utils.d.ts.map +1 -0
- package/dist/agent/utils.js +12 -0
- package/dist/browser-injected-scripts/annotate-elements.js +71 -48
- package/dist/browser-injected-scripts/annotate-elements.spec.js +5 -20
- package/dist/browser-injected-scripts/annotate-elements.spec.ts +4 -19
- package/dist/evals/master-agent.evals.d.ts.map +1 -1
- package/dist/evals/master-agent.evals.js +3 -4
- package/dist/prompts/lib/ts-transformer.d.ts +4 -0
- package/dist/prompts/lib/ts-transformer.d.ts.map +1 -0
- package/dist/prompts/lib/ts-transformer.js +90 -0
- package/dist/prompts/lib/vitest-plugin.d.ts +8 -0
- package/dist/prompts/lib/vitest-plugin.d.ts.map +1 -0
- package/dist/prompts/lib/vitest-plugin.js +20 -0
- package/dist/session/index.d.ts.map +1 -1
- package/dist/session/index.js +4 -0
- package/package.json +11 -9
- package/vitest.config.ts +5 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
# @empiricalrun/test-gen
|
|
2
2
|
|
|
3
|
+
## 0.42.10
|
|
4
|
+
|
|
5
|
+
### Patch Changes
|
|
6
|
+
|
|
7
|
+
- ad2e2c5: fix: annotations for small elements should be outside the container
|
|
8
|
+
- 6471bf9: test: updated timeouts
|
|
9
|
+
- fabdfd6: fix: moved lexical vars check to o3-mini and increased timeout
|
|
10
|
+
- 61bc738: fix: added message for scroll on dashboard
|
|
11
|
+
- Updated dependencies [fabdfd6]
|
|
12
|
+
- @empiricalrun/llm@0.9.31
|
|
13
|
+
|
|
14
|
+
## 0.42.9
|
|
15
|
+
|
|
16
|
+
### Patch Changes
|
|
17
|
+
|
|
18
|
+
- 7560d20: chore: fix test gen configs in tests
|
|
19
|
+
- 13eb978: fix: tests for create-test-block and move prompt to hbs
|
|
20
|
+
- 128123f: fix: added scroller method to master agent
|
|
21
|
+
- 347e20f: feat: prompt builder with handlebars templates
|
|
22
|
+
|
|
3
23
|
## 0.42.8
|
|
4
24
|
|
|
5
25
|
### Patch Changes
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAI3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAiBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAMzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../../src/agent/browsing/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAa,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAI3D,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAClC,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAiBvD,OAAO,EAAe,aAAa,EAAE,MAAM,aAAa,CAAC;AAMzD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,GAAG,IAAI,MAAM,CAKhD;AAED,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,MAAM,EAAE,UAIvD;AA8FD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,aAAa,EACxB,KAAK,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,MAAM,CAAC,CA0DjB;AAyBD,wBAAsB,wBAAwB,CAAC,IAAI,EAAE,IAAI,iBAuHxD;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,QAIjD;AAED;;;GAGG;AACH,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAM1E;AAWD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,YAAY,EAAE,MAAM,EACpB,gBAAgB,EAAE,oBAAoB,EACtC,gBAAgB,GAAE,MAAM,EAAU,GACjC,OAAO,CAAC,MAAM,CAAC,CA+CjB;AAED,wBAAsB,sBAAsB,CAAC,EAC3C,YAAiB,EACjB,IAAS,EACT,eAAoB,EACpB,gBAAqB,EACrB,UAAyC,GAC1C,EAAE;IACD,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,8EASA;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,SAAS;gBAAT,SAAS,EAAE,MAAM;IACrC,OAAO,CAAC,aAAa,CAAqB;YAE5B,mBAAmB;YAUnB,gBAAgB;IAsBjB,OAAO;IAuBb,SAAS;CAKjB"}
|
|
@@ -61,6 +61,7 @@ async function prepareFileForUpdateScenario(genConfig, trace) {
|
|
|
61
61
|
const scopeVariables = await (0, lexical_scoped_vars_1.getLexicalScopedVars)({
|
|
62
62
|
file: await fs_extra_1.default.readFile(createTestFilePath, "utf-8"),
|
|
63
63
|
referencePoint: "await createTest",
|
|
64
|
+
trace: fetchScopeVariablesSpan,
|
|
64
65
|
});
|
|
65
66
|
fetchScopeVariablesSpan?.end({
|
|
66
67
|
name: "fetch-scope-variables",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"create-test-block.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/create-test-block.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"create-test-block.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/create-test-block.ts"],"names":[],"mappings":"AAAA,OAAO,EAAyB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAavE,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAG7D,wBAAsB,wBAAwB,CAAC,EAC7C,QAAQ,EACR,IAAI,EACJ,OAAO,EACP,KAAK,GACN,EAAE;IACD,QAAQ,EAAE,QAAQ,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,KAAK,CAAC,EAAE,WAAW,CAAC;CACrB,+BAqDA"}
|
|
@@ -6,7 +6,9 @@ const logger_1 = require("../../bin/logger");
|
|
|
6
6
|
const context_1 = require("../../bin/utils/context");
|
|
7
7
|
const web_1 = require("../../bin/utils/platform/web");
|
|
8
8
|
const constants_1 = require("../../constants");
|
|
9
|
+
const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software test engineer who is given a task to write an empty test block.\nBased on the inputs you need to create an empty playwright test block with correctly imported fixture.\n\nThe test will contain a test name which you will need to use to build the empty test case block.\n\nYou will be provided with current tests, fixtures and page object models for you to use and create test case block as\nper the task provided to you.\n\nBefore responding you need to ensure that the code change is minimal and the change is reusable across tests. You need\nto ensure the code follows DRY principle.\n\nHere is the list of current tests and fixtures:\n\n{{testFiles}}\n\nHere is the list of current page object models:\n\n{{pageFiles}}\n{{/section}}\n\n{{#section \"user\"}}\nFollowing is the test scenario for which you need to write the empty test case block:\ntest name:\n{{scenarioName}}\n\ntask:\ncreate an empty test case block for the following test steps:\n{{scenario}}\n\ntest file path: {{scenarioFile}}\n\n------\n\nYou also need to ensure that the empty test case block has a starting page to begin test.\n\nIn order to identify the right page with which the test should start, follow the steps:\n- based on the similarities with other test cases mentioned in the file, identify the right page fixture to be imported\n- Read the page fixture methods step by step. Identify whether the fixture handles navigating to a page.\n- Identify whether other tests using the page fixture had to add separate steps for navigation or not\n- Based on the above analysis there will be following cases and choose either for the given test scenario:\n-- Case 1: if the test case scenario provided inside the task mentions about page navigation, then use that page\nnavigation. skip other cases if this case is satisfied.\n-- Case 2: refer other test cases which import similar fixtures and infer the first page navigation of this test case.\nYou should prefer tests which are in the same file. Tests within same file have higher overlaps in first page\nnavigation.\n- Once the page fixture is decided, look for userContext fixture in files. If its available then add \"userContext\" to\nthe test case block\n\n\n\nFollow these instructions before responding with output:\n- Read the code line by line and achieve the task provided to you\n- Read the dependencies of the code block by scanning through file paths and file provided to you. refer the same file\npath while responding with update\n- Focus only on the test case provided and associated JS methods called from the test case.\n- Respond only with the new empty test case block to be created and nothing else.\n- DO NOT respond with any backticks or markdown syntax\n- If \"userContext\" fixture is available in fixtures file, ensure importing that fixture in the test case block.\n- Provide a reason based on the test steps provided to you on why you chose the fixture or page.goto statement. The\nreason should be one of the list steps provided to you and mention why the case was chosen\n{{/section}}";
|
|
9
10
|
const session_1 = require("../../session");
|
|
11
|
+
const promptBuilder_1 = require("./promptBuilder");
|
|
10
12
|
async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
|
|
11
13
|
const logger = new logger_1.CustomLogger({ useReporter: false });
|
|
12
14
|
logger.log("Creating new test block");
|
|
@@ -27,13 +29,13 @@ async function createEmptyTestCaseBlock({ testCase, file, options, trace, }) {
|
|
|
27
29
|
const promptSpan = trace?.span({
|
|
28
30
|
name: "build-create-empty-test-case-prompt",
|
|
29
31
|
});
|
|
30
|
-
const prompt = await (0,
|
|
32
|
+
const prompt = await (0, promptBuilder_1.compilePrompt)(promptTemplate_0, {
|
|
31
33
|
testFiles: context.codePrompt,
|
|
32
34
|
pageFiles: context.pomPrompt,
|
|
33
35
|
scenarioName: testCase.name,
|
|
34
36
|
scenario: testCase.steps.join("\n"),
|
|
35
37
|
scenarioFile: file,
|
|
36
|
-
}
|
|
38
|
+
});
|
|
37
39
|
promptSpan?.end({ output: { prompt } });
|
|
38
40
|
const llm = new llm_1.LLM({
|
|
39
41
|
trace,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lexical-scoped-vars.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/lexical-scoped-vars.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"lexical-scoped-vars.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/lexical-scoped-vars.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAQrD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAGnD,wBAAsB,oBAAoB,CAAC,EACzC,KAAK,EACL,IAAI,EACJ,cAAc,EACd,OAAO,GACR,EAAE;IACD,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,EAAE,oBAAoB,CAAC;CAChC,qBAoDA"}
|
|
@@ -3,29 +3,26 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.getLexicalScopedVars = void 0;
|
|
4
4
|
const llm_1 = require("@empiricalrun/llm");
|
|
5
5
|
const constants_1 = require("../../constants");
|
|
6
|
+
const promptTemplate_0 = "{{#section \"system\"}}\nYou are a software engineer tasked with analysing Typescript code to identify all variables available in the lexical\nscope at a specific reference point within a file. You will be given a file that contains multiple Playwright tests or\npage object models, along with a reference point inside the file. Your goal is to evaluate the list of all variables\navailable in the lexical scope at that reference point.\n\nTo accomplish this, you need to evaluate the Abstract Syntax Tree (AST) and accumulate all variables that are in the\nlexical scope, which includes:\n1. Variables declared within the test before the reference point.\n2. Arguments of the function.\n3. Variables defined in the parent scope. Identify all variables available in the lexical scope at a specific execution\nreference point within a file, considering only those variables that have been declared and assigned prior to the\nexecution of this point in the code.\n4. Global variables defined in the file.\n\nBefore responding:\n- Ignore variables imported from the `\"./pages\"` path.\n- keep in mind temporal dead zone phenomenon before responding with variables\n{{/section}}\n\n{{#section \"user\"}}\nFile:\n{{testFile}}\n\nReference point:\n{{referencePoint}}\n{{/section}}";
|
|
7
|
+
const promptBuilder_1 = require("./promptBuilder");
|
|
6
8
|
async function getLexicalScopedVars({ trace, file, referencePoint, options, }) {
|
|
7
9
|
const fetchLexicalScopedVarsSpan = trace?.span({
|
|
8
10
|
name: "lexical-scoped-vars",
|
|
9
11
|
});
|
|
10
|
-
const
|
|
11
|
-
name: "lexical-scoped-vars-prompt",
|
|
12
|
-
});
|
|
13
|
-
const instruction = await (0, llm_1.getPrompt)("lexical-scope-variables", {
|
|
12
|
+
const messages = await (0, promptBuilder_1.compilePrompt)(promptTemplate_0, {
|
|
14
13
|
testFile: file || "",
|
|
15
14
|
referencePoint: referencePoint || "",
|
|
16
15
|
});
|
|
17
|
-
promptSpan?.end({ output: { instruction } });
|
|
18
16
|
const llm = new llm_1.LLM({
|
|
19
17
|
trace: fetchLexicalScopedVarsSpan,
|
|
20
18
|
provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
|
|
21
|
-
defaultModel: options?.model ||
|
|
19
|
+
defaultModel: options?.model || "o3-mini-2025-01-31",
|
|
22
20
|
providerApiKey: constants_1.MODEL_API_KEYS[options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER],
|
|
23
21
|
});
|
|
24
22
|
const message = await llm.createChatCompletion({
|
|
25
|
-
messages
|
|
23
|
+
messages,
|
|
26
24
|
modelParameters: {
|
|
27
|
-
|
|
28
|
-
...options?.modelParameters,
|
|
25
|
+
max_completion_tokens: constants_1.DEFAULT_O1_MODEL_PARAMETERS.max_completion_tokens,
|
|
29
26
|
},
|
|
30
27
|
responseFormat: {
|
|
31
28
|
type: "json_schema",
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promptBuilder.d.ts","sourceRoot":"","sources":["../../../src/agent/codegen/promptBuilder.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,QAAQ,CAAC;AA6B5B,wBAAsB,aAAa,CAAC,CAAC,SAAS,MAAM,EAClD,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,CAAC,GACR,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,0BAA0B,EAAE,CAAC,CAe/D"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.compilePrompt = void 0;
|
|
7
|
+
const handlebars_1 = __importDefault(require("handlebars"));
|
|
8
|
+
class SectionManager {
|
|
9
|
+
sections = {};
|
|
10
|
+
getSection(name) {
|
|
11
|
+
return this.sections[name] || "";
|
|
12
|
+
}
|
|
13
|
+
setSection(name, content) {
|
|
14
|
+
this.sections[name] = content;
|
|
15
|
+
}
|
|
16
|
+
getAllSections() {
|
|
17
|
+
return this.sections;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
function createHandlebarsEnv() {
|
|
21
|
+
const HandlebarsEnv = handlebars_1.default.create();
|
|
22
|
+
const sectionManager = new SectionManager();
|
|
23
|
+
HandlebarsEnv.registerHelper("section", function (name, options) {
|
|
24
|
+
const content = options.fn(this);
|
|
25
|
+
sectionManager.setSection(name, content);
|
|
26
|
+
return ""; // Don't output anything in place
|
|
27
|
+
});
|
|
28
|
+
return { HandlebarsEnv, sectionManager };
|
|
29
|
+
}
|
|
30
|
+
async function compilePrompt(promptTemplate, params) {
|
|
31
|
+
const { HandlebarsEnv, sectionManager } = createHandlebarsEnv();
|
|
32
|
+
const template = HandlebarsEnv.compile(promptTemplate, { noEscape: true });
|
|
33
|
+
template(params);
|
|
34
|
+
const { system, user } = sectionManager.getAllSections();
|
|
35
|
+
if (!system || !user) {
|
|
36
|
+
// TODO: support templates that have only one section
|
|
37
|
+
throw new Error("Both system and user sections must be defined in the template");
|
|
38
|
+
}
|
|
39
|
+
return [
|
|
40
|
+
{ role: "system", content: system },
|
|
41
|
+
{ role: "user", content: user },
|
|
42
|
+
];
|
|
43
|
+
}
|
|
44
|
+
exports.compilePrompt = compilePrompt;
|
|
@@ -4,7 +4,8 @@ export declare enum ActionType {
|
|
|
4
4
|
CLICK = "click",
|
|
5
5
|
PRESS_ACTION = "keyboard_press_on_element",
|
|
6
6
|
ASSERT_TEXT = "assert_text",
|
|
7
|
-
HOVER = "hover_element"
|
|
7
|
+
HOVER = "hover_element",
|
|
8
|
+
SCROLL = "scroll"
|
|
8
9
|
}
|
|
9
10
|
export declare function isValidActionType(value: string): value is ActionType;
|
|
10
11
|
export declare function getActionToolCalls(): {
|
|
@@ -27,6 +28,10 @@ export declare function getActionToolCalls(): {
|
|
|
27
28
|
type: string;
|
|
28
29
|
description: string;
|
|
29
30
|
};
|
|
31
|
+
is_element_visible: {
|
|
32
|
+
type: string;
|
|
33
|
+
description: string;
|
|
34
|
+
};
|
|
30
35
|
};
|
|
31
36
|
required: string[];
|
|
32
37
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"action-tool-calls.d.ts","sourceRoot":"","sources":["../../../src/agent/master/action-tool-calls.ts"],"names":[],"mappings":"AAAA,oBAAY,UAAU;IACpB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,YAAY,8BAA8B;IAC1C,WAAW,gBAAgB;IAC3B,KAAK,kBAAkB;
|
|
1
|
+
{"version":3,"file":"action-tool-calls.d.ts","sourceRoot":"","sources":["../../../src/agent/master/action-tool-calls.ts"],"names":[],"mappings":"AAAA,oBAAY,UAAU;IACpB,IAAI,SAAS;IACb,SAAS,cAAc;IACvB,KAAK,UAAU;IACf,YAAY,8BAA8B;IAC1C,WAAW,gBAAgB;IAC3B,KAAK,kBAAkB;IACvB,MAAM,WAAW;CAClB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,KAAK,IAAI,UAAU,CAEpE;AAyDD,wBAAgB,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAoCjC"}
|
|
@@ -9,6 +9,7 @@ var ActionType;
|
|
|
9
9
|
ActionType["PRESS_ACTION"] = "keyboard_press_on_element";
|
|
10
10
|
ActionType["ASSERT_TEXT"] = "assert_text";
|
|
11
11
|
ActionType["HOVER"] = "hover_element";
|
|
12
|
+
ActionType["SCROLL"] = "scroll";
|
|
12
13
|
})(ActionType || (exports.ActionType = ActionType = {}));
|
|
13
14
|
function isValidActionType(value) {
|
|
14
15
|
return Object.values(ActionType).includes(value);
|
|
@@ -34,18 +35,24 @@ const createActionCall = (name, description, additionalProperties = {}) => ({
|
|
|
34
35
|
description: `explain the next action in natural language.
|
|
35
36
|
The next action should be as atomic as possible, precise and should contain enough details about the action to be performed.
|
|
36
37
|
E.g. each click, key press, input, assert should be a separate action.
|
|
37
|
-
Each action should take the task to completion, if not the action is invalid
|
|
38
|
+
Each action should take the task to completion, if not the action is invalid.
|
|
39
|
+
If the element is not visible then action should be scroll.`,
|
|
38
40
|
},
|
|
39
41
|
element_description: {
|
|
40
42
|
type: "string",
|
|
41
43
|
description: "The description of the element on which action needs to be taken, including its position, appearance, etc.",
|
|
42
44
|
},
|
|
45
|
+
is_element_visible: {
|
|
46
|
+
type: "boolean",
|
|
47
|
+
description: "A boolean to indicate whether the concerned element is visible in the screenshot.",
|
|
48
|
+
},
|
|
43
49
|
...additionalProperties,
|
|
44
50
|
},
|
|
45
51
|
required: [
|
|
46
52
|
"reason",
|
|
47
53
|
"action",
|
|
48
54
|
"element_description",
|
|
55
|
+
"is_element_visible",
|
|
49
56
|
...Object.keys(additionalProperties),
|
|
50
57
|
],
|
|
51
58
|
},
|
|
@@ -65,6 +72,12 @@ function getActionToolCalls() {
|
|
|
65
72
|
createActionCall(ActionType.PAGE_GOTO, "navigate the page to a new URL"),
|
|
66
73
|
createActionCall(ActionType.PRESS_ACTION, "Focuses the matching element and presses a combination of the keys"),
|
|
67
74
|
createActionCall(ActionType.HOVER, "hover over an element on the page"),
|
|
75
|
+
createActionCall(ActionType.SCROLL, "Scroll the page since element is not visible in the screenshot", {
|
|
76
|
+
element: {
|
|
77
|
+
type: "string",
|
|
78
|
+
description: "Element to look for after scrolling to complete the task.",
|
|
79
|
+
},
|
|
80
|
+
}),
|
|
68
81
|
].map((call) => call.schema);
|
|
69
82
|
}
|
|
70
83
|
exports.getActionToolCalls = getActionToolCalls;
|
|
@@ -12,7 +12,7 @@ export declare function getElementAnnotation({ elementDescription, annotations,
|
|
|
12
12
|
preference: AnnotationPreference;
|
|
13
13
|
}): Promise<string | undefined>;
|
|
14
14
|
export type AnnotationPreference = {
|
|
15
|
-
actionType: "all" | ActionType.FILL | ActionType.ASSERT_TEXT;
|
|
15
|
+
actionType: "all" | ActionType.FILL | ActionType.ASSERT_TEXT | ActionType.SCROLL;
|
|
16
16
|
assertionText?: string | undefined;
|
|
17
17
|
};
|
|
18
18
|
export declare function getAnnotationKeys({ page, preference, options, }: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"element-annotation.d.ts","sourceRoot":"","sources":["../../../src/agent/master/element-annotation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAOlC,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AA0DjD,wBAAsB,oBAAoB,CAAC,EACzC,kBAAkB,EAClB,WAAW,EACX,mBAAmB,EACnB,KAAK,EACL,GAAG,EACH,OAAO,EACP,UAAU,GACX,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,UAAU,EAAE,oBAAoB,CAAC;CAClC,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CA6F9B;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EACN,KAAK,GACL,UAAU,CAAC,IAAI,GACf,UAAU,CAAC,WAAW,GACtB,UAAU,CAAC,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACpC,CAAC;AAEF,wBAAsB,iBAAiB,CAAC,EACtC,IAAI,EACJ,UAAU,EACV,OAAO,GACR,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,oBAAoB,CAAC;IACjC,OAAO,EAAE,oBAAoB,CAAC;CAC/B,GAAG,OAAO,CAAC;IACV,cAAc,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IACtD,gBAAgB,EAAE,MAAM,CAAC;IACzB,uBAAuB,EAAE,MAAM,CAAC;CACjC,CAAC,CAqDD"}
|
|
@@ -36,12 +36,21 @@ const annotationToolAction = {
|
|
|
36
36
|
type: "string",
|
|
37
37
|
description: "Explain why this element is selected. The reason should be clear and align with the task or purpose.",
|
|
38
38
|
},
|
|
39
|
+
element: {
|
|
40
|
+
type: "string",
|
|
41
|
+
description: "Detailed description of the DOM element's visual characteristics and position.",
|
|
42
|
+
},
|
|
39
43
|
element_annotation: {
|
|
40
44
|
type: "string",
|
|
41
45
|
description: "Return the unique element ID for the element on which the action needs to be performed.",
|
|
42
46
|
},
|
|
43
47
|
},
|
|
44
|
-
required: [
|
|
48
|
+
required: [
|
|
49
|
+
"enriched_annotations",
|
|
50
|
+
"reason",
|
|
51
|
+
"element",
|
|
52
|
+
"element_annotation",
|
|
53
|
+
],
|
|
45
54
|
},
|
|
46
55
|
},
|
|
47
56
|
},
|
|
@@ -120,7 +129,7 @@ async function getElementAnnotation({ elementDescription, annotations, annotated
|
|
|
120
129
|
...constants_1.DEFAULT_MODEL_PARAMETERS,
|
|
121
130
|
...options?.modelParameters,
|
|
122
131
|
tool_choice: "required",
|
|
123
|
-
temperature:
|
|
132
|
+
temperature: 0.5,
|
|
124
133
|
},
|
|
125
134
|
trace: annotationsSpan,
|
|
126
135
|
traceName: "get-element-from-action",
|
|
@@ -137,7 +146,7 @@ async function getAnnotationKeys({ page, preference, options, }) {
|
|
|
137
146
|
const annotationKeys = await page.evaluate(({ preference, options }) => {
|
|
138
147
|
// @ts-ignore
|
|
139
148
|
// eslint-disable-next-line no-undef
|
|
140
|
-
window.annotationInstance =
|
|
149
|
+
window.annotationInstance = annotateElementsWithPreference({
|
|
141
150
|
options: options,
|
|
142
151
|
preference: preference,
|
|
143
152
|
});
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { LLM, TraceClient } from "@empiricalrun/llm";
|
|
2
|
+
import { Page } from "playwright";
|
|
2
3
|
import { PlaywrightActions } from "../../actions";
|
|
4
|
+
import { CustomLogger } from "../../bin/logger";
|
|
3
5
|
import { BrowsingAgentOptions } from "../browsing";
|
|
4
6
|
import { ActionType } from "./action-tool-calls";
|
|
5
|
-
export declare function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, actions, actionTypes, disableSkills, }: {
|
|
7
|
+
export declare function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, actions, actionTypes, disableSkills, page, logger, }: {
|
|
6
8
|
task: string;
|
|
7
9
|
executedActions: string[];
|
|
8
10
|
failedActions: any[];
|
|
@@ -10,11 +12,14 @@ export declare function getNextAction({ task, executedActions, failedActions, pa
|
|
|
10
12
|
trace?: TraceClient;
|
|
11
13
|
llm?: LLM;
|
|
12
14
|
options?: BrowsingAgentOptions;
|
|
13
|
-
pageScreenshot: string;
|
|
14
|
-
annotatedPageScreenshot?: string;
|
|
15
|
+
pageScreenshot: string[];
|
|
15
16
|
actions: PlaywrightActions;
|
|
16
17
|
actionTypes: typeof ActionType;
|
|
17
18
|
disableSkills: boolean;
|
|
18
|
-
|
|
19
|
-
|
|
19
|
+
page: Page;
|
|
20
|
+
logger?: CustomLogger;
|
|
21
|
+
}): Promise<{
|
|
22
|
+
actionType: string;
|
|
23
|
+
toolCallArgs: string;
|
|
24
|
+
} | undefined>;
|
|
20
25
|
//# sourceMappingURL=next-action.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"next-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/next-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"next-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/next-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAElD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAMhD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAsB,MAAM,qBAAqB,CAAC;AAGrE,wBAAsB,aAAa,CAAC,EAClC,IAAI,EACJ,eAAe,EACf,aAAa,EACb,OAAO,EACP,KAAK,EACL,GAAG,EACH,OAAO,EACP,cAAc,EACd,OAAO,EACP,WAAW,EACX,aAAa,EACb,IAAI,EACJ,MAAM,GACP,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,GAAG,CAAC;IACV,OAAO,CAAC,EAAE,oBAAoB,CAAC;IAC/B,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,OAAO,EAAE,iBAAiB,CAAC;IAC3B,WAAW,EAAE,OAAO,UAAU,CAAC;IAC/B,aAAa,EAAE,OAAO,CAAC;IACvB,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CACP;IACE,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;CACtB,GACD,SAAS,CACZ,CAsLA"}
|
|
@@ -5,8 +5,10 @@ const llm_1 = require("@empiricalrun/llm");
|
|
|
5
5
|
const vision_1 = require("@empiricalrun/llm/vision");
|
|
6
6
|
const skill_1 = require("../../actions/skill");
|
|
7
7
|
const constants_1 = require("../../constants");
|
|
8
|
+
const utils_1 = require("../utils");
|
|
8
9
|
const action_tool_calls_1 = require("./action-tool-calls");
|
|
9
|
-
|
|
10
|
+
const scroller_1 = require("./scroller");
|
|
11
|
+
async function getNextAction({ task, executedActions, failedActions, pageUrl, trace, llm, options, pageScreenshot, actions, actionTypes, disableSkills, page, logger, }) {
|
|
10
12
|
const nextActionSpan = trace?.span({
|
|
11
13
|
name: "master-agent-next-action",
|
|
12
14
|
input: {
|
|
@@ -23,9 +25,17 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
|
|
|
23
25
|
const promptSpan = nextActionSpan?.span({
|
|
24
26
|
name: "master-agent-prompt",
|
|
25
27
|
});
|
|
28
|
+
const screenshotsForPrompt = pageScreenshot.map((screenshot) => {
|
|
29
|
+
return {
|
|
30
|
+
type: "image_url",
|
|
31
|
+
image_url: {
|
|
32
|
+
url: (0, vision_1.imageFormatForProvider)(options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, screenshot),
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
});
|
|
26
36
|
const systemMessage = {
|
|
27
37
|
role: "system",
|
|
28
|
-
content: `You are
|
|
38
|
+
content: `You are a web automation tool which is given a task to complete. You need to execute the task provided to you with the help of web page screenshot, a browser automation tool or skills which are learnt while writing previous tests.
|
|
29
39
|
|
|
30
40
|
Browser automation tool is a tool which uses Playwright and browser to execute action using next_action tool call.
|
|
31
41
|
Skill usage is a tool which helps to execute previously known pieces of code to achieve a task.
|
|
@@ -37,7 +47,7 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
|
|
|
37
47
|
You will also be provided with failed next action predicted by you, so that you can avoid suggesting the same action again - which failed.
|
|
38
48
|
|
|
39
49
|
The next action should be as atomic as possible.
|
|
40
|
-
e.g: click on an element, fill an input element, assert, extract text from an element are valid next action as they are atomic in nature.
|
|
50
|
+
e.g: scroll, click on an element, fill an input element, assert, extract text from an element are valid next action as they are atomic in nature.
|
|
41
51
|
|
|
42
52
|
You also need to provide the action type using the list below, action type which is not present in the list is invalid.
|
|
43
53
|
${Object.values(actionTypes)}
|
|
@@ -88,14 +98,9 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
|
|
|
88
98
|
},
|
|
89
99
|
{
|
|
90
100
|
type: "text",
|
|
91
|
-
text:
|
|
92
|
-
},
|
|
93
|
-
{
|
|
94
|
-
type: "image_url",
|
|
95
|
-
image_url: {
|
|
96
|
-
url: (0, vision_1.imageFormatForProvider)(options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER, pageScreenshot),
|
|
97
|
-
},
|
|
101
|
+
text: pageScreenshot.length > 1 ? "Screenshots:" : "Screenshot:",
|
|
98
102
|
},
|
|
103
|
+
...screenshotsForPrompt,
|
|
99
104
|
],
|
|
100
105
|
};
|
|
101
106
|
const messages = [
|
|
@@ -128,6 +133,49 @@ async function getNextAction({ task, executedActions, failedActions, pageUrl, tr
|
|
|
128
133
|
});
|
|
129
134
|
const toolCall = completion?.tool_calls?.[0];
|
|
130
135
|
nextActionSpan?.end({ output: toolCall });
|
|
131
|
-
|
|
136
|
+
if (toolCall) {
|
|
137
|
+
const toolCallArgs = (0, utils_1.parseJson)(toolCall.function.arguments);
|
|
138
|
+
const actionType = toolCall.function.name;
|
|
139
|
+
// If the action type is scroll, we need to scroll the page and get the reference to the frame in which the element is visible
|
|
140
|
+
// else we return the next action
|
|
141
|
+
// For scroll we have at max 2 retries
|
|
142
|
+
// If the element is not visible after 2 retries, we throw an error
|
|
143
|
+
if (actionType === "scroll" && toolCallArgs) {
|
|
144
|
+
let maxScrollRetries = 2;
|
|
145
|
+
while (maxScrollRetries--) {
|
|
146
|
+
const frames = await (0, scroller_1.scroller)({
|
|
147
|
+
elementDescription: toolCallArgs.element,
|
|
148
|
+
page,
|
|
149
|
+
trace: nextActionSpan,
|
|
150
|
+
logger,
|
|
151
|
+
});
|
|
152
|
+
if (frames.length > 0) {
|
|
153
|
+
return getNextAction({
|
|
154
|
+
task,
|
|
155
|
+
executedActions,
|
|
156
|
+
failedActions,
|
|
157
|
+
pageUrl,
|
|
158
|
+
trace,
|
|
159
|
+
llm,
|
|
160
|
+
options,
|
|
161
|
+
pageScreenshot: frames.map((frame) => frame.frameScreenshot),
|
|
162
|
+
actions,
|
|
163
|
+
actionTypes,
|
|
164
|
+
disableSkills,
|
|
165
|
+
page,
|
|
166
|
+
logger,
|
|
167
|
+
});
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
if (maxScrollRetries === 0) {
|
|
171
|
+
throw new Error("Agent is not able to figure out the next action, since element is not visible on screen.");
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
return {
|
|
175
|
+
actionType,
|
|
176
|
+
toolCallArgs: toolCall?.function.arguments,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
return;
|
|
132
180
|
}
|
|
133
181
|
exports.getNextAction = getNextAction;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/agent/master/run.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAclC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EACL,oBAAoB,EAErB,MAAM,aAAa,CAAC;AA4BrB,wBAAsB,0BAA0B,CAAC,EAC/C,IAAI,EACJ,IAAI,EACJ,QAAQ,EACR,OAAO,EACP,SAAS,GACV,EAAE;IACD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,IAAI,CAAC;IACX,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,OAAO,EAAE,oBAAoB,CAAC;IAC9B,SAAS,CAAC,EAAE,SAAS,CAAC;CACvB;;;GA+UA"}
|
package/dist/agent/master/run.js
CHANGED
|
@@ -14,6 +14,7 @@ const browsing_1 = require("../browsing");
|
|
|
14
14
|
const utils_2 = require("../browsing/utils");
|
|
15
15
|
const skills_retriever_1 = require("../codegen/skills-retriever");
|
|
16
16
|
const run_time_planner_1 = require("../planner/run-time-planner");
|
|
17
|
+
const utils_3 = require("../utils");
|
|
17
18
|
const action_tool_calls_1 = require("./action-tool-calls");
|
|
18
19
|
const element_annotation_1 = require("./element-annotation");
|
|
19
20
|
const next_action_1 = require("./next-action");
|
|
@@ -134,29 +135,30 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
|
|
|
134
135
|
trace: masterAgentSpan,
|
|
135
136
|
llm,
|
|
136
137
|
options,
|
|
137
|
-
pageScreenshot,
|
|
138
|
+
pageScreenshot: [pageScreenshot],
|
|
138
139
|
actions,
|
|
139
140
|
actionTypes: action_tool_calls_1.ActionType,
|
|
140
141
|
disableSkills,
|
|
142
|
+
page,
|
|
143
|
+
logger,
|
|
141
144
|
});
|
|
142
145
|
if (await (0, session_1.shouldStopSession)()) {
|
|
143
146
|
break;
|
|
144
147
|
}
|
|
145
148
|
if (toolCall) {
|
|
146
|
-
const args =
|
|
149
|
+
const args = (0, utils_3.parseJson)(toolCall.toolCallArgs);
|
|
147
150
|
const masterAgentActionSpan = masterAgentSpan?.span({
|
|
148
151
|
name: "master-agent-execute-action",
|
|
149
152
|
});
|
|
150
153
|
try {
|
|
151
154
|
output = {
|
|
152
|
-
action:
|
|
153
|
-
|
|
154
|
-
reason: JSON.parse(toolCall.function.arguments).reason,
|
|
155
|
+
action: args.action || args.skill,
|
|
156
|
+
reason: args.reason,
|
|
155
157
|
};
|
|
156
158
|
void testGenReporter.sendMessage(output.reason);
|
|
157
159
|
logger.log(`Next Action: ${output.action}`);
|
|
158
|
-
if (toolCall.
|
|
159
|
-
const code = await actions.executeAction(toolCall.
|
|
160
|
+
if (toolCall.actionType === skill_1.SKILL_USAGE) {
|
|
161
|
+
const code = await actions.executeAction(toolCall.actionType, args, masterAgentActionSpan);
|
|
160
162
|
if (code) {
|
|
161
163
|
generatedCodeSteps.push(code);
|
|
162
164
|
}
|
|
@@ -166,8 +168,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
|
|
|
166
168
|
let hintsExecutionCompletion;
|
|
167
169
|
let elementAnnotation;
|
|
168
170
|
await page.waitForTimeout(2000);
|
|
169
|
-
const
|
|
170
|
-
const actionType = toolCall.function.name;
|
|
171
|
+
const actionType = toolCall.actionType;
|
|
171
172
|
let preference = {
|
|
172
173
|
actionType: "all",
|
|
173
174
|
};
|
|
@@ -181,7 +182,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
|
|
|
181
182
|
case action_tool_calls_1.ActionType.ASSERT_TEXT:
|
|
182
183
|
preference = {
|
|
183
184
|
actionType: action_tool_calls_1.ActionType.ASSERT_TEXT,
|
|
184
|
-
assertionText:
|
|
185
|
+
assertionText: args.assertion_text,
|
|
185
186
|
};
|
|
186
187
|
break;
|
|
187
188
|
default:
|
|
@@ -197,13 +198,12 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
|
|
|
197
198
|
});
|
|
198
199
|
if (annotationKeys.length > 0) {
|
|
199
200
|
const annotationMapString = annotationKeys
|
|
200
|
-
?.map((a) => `${a.elementID}
|
|
201
|
+
?.map((a) => `${a.elementID}: ${a.text}`)
|
|
201
202
|
.join("\n");
|
|
202
203
|
// Provides the annotations for all the element present on screen
|
|
203
204
|
// Also provides the annotation of element on which the action needs to be taken
|
|
204
205
|
elementAnnotation = await (0, element_annotation_1.getElementAnnotation)({
|
|
205
|
-
elementDescription:
|
|
206
|
-
.element_description,
|
|
206
|
+
elementDescription: args.element_description,
|
|
207
207
|
annotations: annotationMapString,
|
|
208
208
|
annotatedScreenshot: annotatedPageScreenshot,
|
|
209
209
|
trace: masterAgentActionSpan,
|
|
@@ -269,7 +269,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
|
|
|
269
269
|
input: {
|
|
270
270
|
action: output.action,
|
|
271
271
|
reason: output.reason,
|
|
272
|
-
type: toolCall.
|
|
272
|
+
type: toolCall.actionType,
|
|
273
273
|
},
|
|
274
274
|
output: {
|
|
275
275
|
success: true,
|
|
@@ -279,7 +279,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
|
|
|
279
279
|
disableSkills = false;
|
|
280
280
|
}
|
|
281
281
|
catch (e) {
|
|
282
|
-
if (toolCall.
|
|
282
|
+
if (toolCall.actionType === skill_1.SKILL_USAGE) {
|
|
283
283
|
logger.log("Disabling skill usage for next retry");
|
|
284
284
|
disableSkills = true;
|
|
285
285
|
}
|
|
@@ -287,7 +287,7 @@ async function createTestUsingMasterAgent({ task, page, testCase, options, scope
|
|
|
287
287
|
input: {
|
|
288
288
|
action: output.action,
|
|
289
289
|
reason: output.reason,
|
|
290
|
-
type: toolCall.
|
|
290
|
+
type: toolCall.actionType,
|
|
291
291
|
},
|
|
292
292
|
output: {
|
|
293
293
|
error: true,
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { TraceClient } from "@empiricalrun/llm";
|
|
2
|
+
import { Page } from "playwright";
|
|
3
|
+
import { CustomLogger } from "../../bin/logger";
|
|
4
|
+
export type FrameReference = {
|
|
5
|
+
scrollPosition: number;
|
|
6
|
+
frameScreenshot: string;
|
|
7
|
+
};
|
|
8
|
+
export declare function scroller({ elementDescription, page, trace, frameReference, logger, }: {
|
|
9
|
+
elementDescription: string;
|
|
10
|
+
page: Page;
|
|
11
|
+
trace?: TraceClient;
|
|
12
|
+
frameReference?: FrameReference;
|
|
13
|
+
logger?: CustomLogger;
|
|
14
|
+
}): Promise<FrameReference[]>;
|
|
15
|
+
//# sourceMappingURL=scroller.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"scroller.d.ts","sourceRoot":"","sources":["../../../src/agent/master/scroller.ts"],"names":[],"mappings":"AAAA,OAAO,EAAuB,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGrE,OAAO,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAElC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAchD,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;CACzB,CAAC;AA2ZF,wBAAsB,QAAQ,CAAC,EAC7B,kBAAkB,EAClB,IAAI,EACJ,KAAK,EACL,cAAc,EACd,MAAM,GACP,EAAE;IACD,kBAAkB,EAAE,MAAM,CAAC;IAC3B,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC,MAAM,CAAC,EAAE,YAAY,CAAC;CACvB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA6D5B"}
|