@empiricalrun/playwright-utils 0.23.0 → 0.23.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/package.json +2 -2
- package/bundled/test-gen/actions/assert.d.ts +0 -4
- package/bundled/test-gen/actions/assert.d.ts.map +0 -1
- package/bundled/test-gen/actions/assert.js +0 -50
- package/bundled/test-gen/actions/click.d.ts +0 -4
- package/bundled/test-gen/actions/click.d.ts.map +0 -1
- package/bundled/test-gen/actions/click.js +0 -51
- package/bundled/test-gen/actions/constants/index.d.ts +0 -2
- package/bundled/test-gen/actions/constants/index.d.ts.map +0 -1
- package/bundled/test-gen/actions/constants/index.js +0 -4
- package/bundled/test-gen/actions/done.d.ts +0 -4
- package/bundled/test-gen/actions/done.d.ts.map +0 -1
- package/bundled/test-gen/actions/done.js +0 -30
- package/bundled/test-gen/actions/fill.d.ts +0 -4
- package/bundled/test-gen/actions/fill.d.ts.map +0 -1
- package/bundled/test-gen/actions/fill.js +0 -82
- package/bundled/test-gen/actions/goto.d.ts +0 -4
- package/bundled/test-gen/actions/goto.d.ts.map +0 -1
- package/bundled/test-gen/actions/goto.js +0 -47
- package/bundled/test-gen/actions/hover.d.ts +0 -4
- package/bundled/test-gen/actions/hover.d.ts.map +0 -1
- package/bundled/test-gen/actions/hover.js +0 -51
- package/bundled/test-gen/actions/index.d.ts +0 -30
- package/bundled/test-gen/actions/index.d.ts.map +0 -1
- package/bundled/test-gen/actions/index.js +0 -159
- package/bundled/test-gen/actions/next-task.d.ts +0 -53
- package/bundled/test-gen/actions/next-task.d.ts.map +0 -1
- package/bundled/test-gen/actions/next-task.js +0 -58
- package/bundled/test-gen/actions/press.d.ts +0 -4
- package/bundled/test-gen/actions/press.d.ts.map +0 -1
- package/bundled/test-gen/actions/press.js +0 -56
- package/bundled/test-gen/actions/skill.d.ts +0 -21
- package/bundled/test-gen/actions/skill.d.ts.map +0 -1
- package/bundled/test-gen/actions/skill.js +0 -127
- package/bundled/test-gen/actions/text-content.d.ts +0 -4
- package/bundled/test-gen/actions/text-content.d.ts.map +0 -1
- package/bundled/test-gen/actions/text-content.js +0 -58
- package/bundled/test-gen/actions/utils/index.d.ts +0 -10
- package/bundled/test-gen/actions/utils/index.d.ts.map +0 -1
- package/bundled/test-gen/actions/utils/index.js +0 -109
- package/bundled/test-gen/agent/browsing/index.d.ts +0 -15
- package/bundled/test-gen/agent/browsing/index.d.ts.map +0 -1
- package/bundled/test-gen/agent/browsing/index.js +0 -68
- package/bundled/test-gen/agent/browsing/run.d.ts +0 -19
- package/bundled/test-gen/agent/browsing/run.d.ts.map +0 -1
- package/bundled/test-gen/agent/browsing/run.js +0 -88
- package/bundled/test-gen/agent/browsing/utils.d.ts +0 -41
- package/bundled/test-gen/agent/browsing/utils.d.ts.map +0 -1
- package/bundled/test-gen/agent/browsing/utils.js +0 -406
- package/bundled/test-gen/agent/codegen/create-test-block.d.ts +0 -9
- package/bundled/test-gen/agent/codegen/create-test-block.d.ts.map +0 -1
- package/bundled/test-gen/agent/codegen/create-test-block.js +0 -63
- package/bundled/test-gen/agent/codegen/fix-ts-errors.d.ts +0 -13
- package/bundled/test-gen/agent/codegen/fix-ts-errors.d.ts.map +0 -1
- package/bundled/test-gen/agent/codegen/fix-ts-errors.js +0 -80
- package/bundled/test-gen/agent/codegen/generate-code-apply-changes.d.ts +0 -13
- package/bundled/test-gen/agent/codegen/generate-code-apply-changes.d.ts.map +0 -1
- package/bundled/test-gen/agent/codegen/generate-code-apply-changes.js +0 -381
- package/bundled/test-gen/agent/codegen/lexical-scoped-vars.d.ts +0 -9
- package/bundled/test-gen/agent/codegen/lexical-scoped-vars.d.ts.map +0 -1
- package/bundled/test-gen/agent/codegen/lexical-scoped-vars.js +0 -56
- package/bundled/test-gen/agent/codegen/repo-edit.d.ts +0 -23
- package/bundled/test-gen/agent/codegen/repo-edit.d.ts.map +0 -1
- package/bundled/test-gen/agent/codegen/repo-edit.js +0 -86
- package/bundled/test-gen/agent/codegen/run.d.ts +0 -16
- package/bundled/test-gen/agent/codegen/run.d.ts.map +0 -1
- package/bundled/test-gen/agent/codegen/run.js +0 -119
- package/bundled/test-gen/agent/codegen/skills-retriever.d.ts +0 -26
- package/bundled/test-gen/agent/codegen/skills-retriever.d.ts.map +0 -1
- package/bundled/test-gen/agent/codegen/skills-retriever.js +0 -93
- package/bundled/test-gen/agent/codegen/test-update-feedback.d.ts +0 -12
- package/bundled/test-gen/agent/codegen/test-update-feedback.d.ts.map +0 -1
- package/bundled/test-gen/agent/codegen/test-update-feedback.js +0 -50
- package/bundled/test-gen/agent/codegen/types.d.ts +0 -25
- package/bundled/test-gen/agent/codegen/types.d.ts.map +0 -1
- package/bundled/test-gen/agent/codegen/types.js +0 -8
- package/bundled/test-gen/agent/codegen/update-flow.d.ts +0 -34
- package/bundled/test-gen/agent/codegen/update-flow.d.ts.map +0 -1
- package/bundled/test-gen/agent/codegen/update-flow.js +0 -300
- package/bundled/test-gen/agent/codegen/use-skill.d.ts +0 -11
- package/bundled/test-gen/agent/codegen/use-skill.d.ts.map +0 -1
- package/bundled/test-gen/agent/codegen/use-skill.js +0 -54
- package/bundled/test-gen/agent/codegen/utils.d.ts +0 -126
- package/bundled/test-gen/agent/codegen/utils.d.ts.map +0 -1
- package/bundled/test-gen/agent/codegen/utils.js +0 -416
- package/bundled/test-gen/agent/diagnosis-agent/index.d.ts +0 -18
- package/bundled/test-gen/agent/diagnosis-agent/index.d.ts.map +0 -1
- package/bundled/test-gen/agent/diagnosis-agent/index.js +0 -105
- package/bundled/test-gen/agent/diagnosis-agent/strict-mode-violation.d.ts +0 -9
- package/bundled/test-gen/agent/diagnosis-agent/strict-mode-violation.d.ts.map +0 -1
- package/bundled/test-gen/agent/diagnosis-agent/strict-mode-violation.js +0 -31
- package/bundled/test-gen/agent/enrich-prompt/index.d.ts +0 -12
- package/bundled/test-gen/agent/enrich-prompt/index.d.ts.map +0 -1
- package/bundled/test-gen/agent/enrich-prompt/index.js +0 -81
- package/bundled/test-gen/agent/enrich-prompt/utils.d.ts +0 -6
- package/bundled/test-gen/agent/enrich-prompt/utils.d.ts.map +0 -1
- package/bundled/test-gen/agent/enrich-prompt/utils.js +0 -12
- package/bundled/test-gen/agent/infer-agent/index.d.ts +0 -10
- package/bundled/test-gen/agent/infer-agent/index.d.ts.map +0 -1
- package/bundled/test-gen/agent/infer-agent/index.js +0 -70
- package/bundled/test-gen/agent/master/action-tool-calls.d.ts +0 -42
- package/bundled/test-gen/agent/master/action-tool-calls.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/action-tool-calls.js +0 -87
- package/bundled/test-gen/agent/master/browser-tests/fixtures.d.ts +0 -9
- package/bundled/test-gen/agent/master/browser-tests/fixtures.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/browser-tests/fixtures.js +0 -33
- package/bundled/test-gen/agent/master/browser-tests/index.spec.d.ts +0 -2
- package/bundled/test-gen/agent/master/browser-tests/index.spec.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/browser-tests/index.spec.js +0 -113
- package/bundled/test-gen/agent/master/browser-tests/skills.spec.d.ts +0 -2
- package/bundled/test-gen/agent/master/browser-tests/skills.spec.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/browser-tests/skills.spec.js +0 -109
- package/bundled/test-gen/agent/master/element-annotation.d.ts +0 -30
- package/bundled/test-gen/agent/master/element-annotation.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/element-annotation.js +0 -195
- package/bundled/test-gen/agent/master/execute-browser-action.d.ts +0 -24
- package/bundled/test-gen/agent/master/execute-browser-action.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/execute-browser-action.js +0 -124
- package/bundled/test-gen/agent/master/execute-skill-action.d.ts +0 -11
- package/bundled/test-gen/agent/master/execute-skill-action.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/execute-skill-action.js +0 -25
- package/bundled/test-gen/agent/master/icon-descriptor/index.d.ts +0 -22
- package/bundled/test-gen/agent/master/icon-descriptor/index.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/icon-descriptor/index.js +0 -250
- package/bundled/test-gen/agent/master/icon-descriptor/normalize-svg.d.ts +0 -2
- package/bundled/test-gen/agent/master/icon-descriptor/normalize-svg.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/icon-descriptor/normalize-svg.js +0 -248
- package/bundled/test-gen/agent/master/next-action.d.ts +0 -22
- package/bundled/test-gen/agent/master/next-action.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/next-action.js +0 -104
- package/bundled/test-gen/agent/master/planner.d.ts +0 -15
- package/bundled/test-gen/agent/master/planner.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/planner.js +0 -144
- package/bundled/test-gen/agent/master/run.d.ts +0 -15
- package/bundled/test-gen/agent/master/run.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/run.js +0 -274
- package/bundled/test-gen/agent/master/scroller.d.ts +0 -15
- package/bundled/test-gen/agent/master/scroller.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/scroller.js +0 -375
- package/bundled/test-gen/agent/master/with-hints.d.ts +0 -17
- package/bundled/test-gen/agent/master/with-hints.d.ts.map +0 -1
- package/bundled/test-gen/agent/master/with-hints.js +0 -102
- package/bundled/test-gen/agent/planner/run-time-planner.d.ts +0 -15
- package/bundled/test-gen/agent/planner/run-time-planner.d.ts.map +0 -1
- package/bundled/test-gen/agent/planner/run-time-planner.js +0 -100
- package/bundled/test-gen/agent/planner/run.d.ts +0 -7
- package/bundled/test-gen/agent/planner/run.d.ts.map +0 -1
- package/bundled/test-gen/agent/planner/run.js +0 -127
- package/bundled/test-gen/agent/utils.d.ts +0 -2
- package/bundled/test-gen/agent/utils.d.ts.map +0 -1
- package/bundled/test-gen/agent/utils.js +0 -12
- package/bundled/test-gen/bin/index.d.ts +0 -3
- package/bundled/test-gen/bin/index.d.ts.map +0 -1
- package/bundled/test-gen/bin/index.js +0 -212
- package/bundled/test-gen/bin/logger/index.d.ts +0 -14
- package/bundled/test-gen/bin/logger/index.d.ts.map +0 -1
- package/bundled/test-gen/bin/logger/index.js +0 -57
- package/bundled/test-gen/bin/utils/context.d.ts +0 -13
- package/bundled/test-gen/bin/utils/context.d.ts.map +0 -1
- package/bundled/test-gen/bin/utils/context.js +0 -67
- package/bundled/test-gen/bin/utils/fs/index.d.ts +0 -6
- package/bundled/test-gen/bin/utils/fs/index.d.ts.map +0 -1
- package/bundled/test-gen/bin/utils/fs/index.js +0 -63
- package/bundled/test-gen/bin/utils/index.d.ts +0 -9
- package/bundled/test-gen/bin/utils/index.d.ts.map +0 -1
- package/bundled/test-gen/bin/utils/index.js +0 -64
- package/bundled/test-gen/bin/utils/platform/web/index.d.ts +0 -78
- package/bundled/test-gen/bin/utils/platform/web/index.d.ts.map +0 -1
- package/bundled/test-gen/bin/utils/platform/web/index.js +0 -544
- package/bundled/test-gen/bin/utils/platform/web/test-files/ts-path-import-validate.d.ts +0 -2
- package/bundled/test-gen/bin/utils/platform/web/test-files/ts-path-import-validate.d.ts.map +0 -1
- package/bundled/test-gen/bin/utils/platform/web/test-files/ts-path-import-validate.js +0 -7
- package/bundled/test-gen/bin/utils/scenarios/index.d.ts +0 -6
- package/bundled/test-gen/bin/utils/scenarios/index.d.ts.map +0 -1
- package/bundled/test-gen/bin/utils/scenarios/index.js +0 -57
- package/bundled/test-gen/browser-injected-scripts/annotate-elements.js +0 -615
- package/bundled/test-gen/browser-injected-scripts/annotate-elements.spec.d.ts +0 -2
- package/bundled/test-gen/browser-injected-scripts/annotate-elements.spec.d.ts.map +0 -1
- package/bundled/test-gen/browser-injected-scripts/annotate-elements.spec.js +0 -207
- package/bundled/test-gen/browser-injected-scripts/annotate-elements.spec.ts +0 -332
- package/bundled/test-gen/constants/index.d.ts +0 -7
- package/bundled/test-gen/constants/index.d.ts.map +0 -1
- package/bundled/test-gen/constants/index.js +0 -18
- package/bundled/test-gen/errors/index.d.ts +0 -5
- package/bundled/test-gen/errors/index.d.ts.map +0 -1
- package/bundled/test-gen/errors/index.js +0 -9
- package/bundled/test-gen/evals/add-scenario-agent.evals.d.ts +0 -4
- package/bundled/test-gen/evals/add-scenario-agent.evals.d.ts.map +0 -1
- package/bundled/test-gen/evals/add-scenario-agent.evals.js +0 -44
- package/bundled/test-gen/evals/append-create-test-agent.evals.d.ts +0 -4
- package/bundled/test-gen/evals/append-create-test-agent.evals.d.ts.map +0 -1
- package/bundled/test-gen/evals/append-create-test-agent.evals.js +0 -117
- package/bundled/test-gen/evals/fetch-pom-skills-agent.evals.d.ts +0 -4
- package/bundled/test-gen/evals/fetch-pom-skills-agent.evals.d.ts.map +0 -1
- package/bundled/test-gen/evals/fetch-pom-skills-agent.evals.js +0 -36
- package/bundled/test-gen/evals/infer-master-or-code-agent.evals.d.ts +0 -4
- package/bundled/test-gen/evals/infer-master-or-code-agent.evals.d.ts.map +0 -1
- package/bundled/test-gen/evals/infer-master-or-code-agent.evals.js +0 -22
- package/bundled/test-gen/evals/master-agent.evals.d.ts +0 -4
- package/bundled/test-gen/evals/master-agent.evals.d.ts.map +0 -1
- package/bundled/test-gen/evals/master-agent.evals.js +0 -35
- package/bundled/test-gen/evals/type.d.ts +0 -12
- package/bundled/test-gen/evals/type.d.ts.map +0 -1
- package/bundled/test-gen/evals/type.js +0 -2
- package/bundled/test-gen/evals/update-scenario-agent.evals.d.ts +0 -4
- package/bundled/test-gen/evals/update-scenario-agent.evals.d.ts.map +0 -1
- package/bundled/test-gen/evals/update-scenario-agent.evals.js +0 -47
- package/bundled/test-gen/file/client.d.ts +0 -14
- package/bundled/test-gen/file/client.d.ts.map +0 -1
- package/bundled/test-gen/file/client.js +0 -48
- package/bundled/test-gen/file/server.d.ts +0 -13
- package/bundled/test-gen/file/server.d.ts.map +0 -1
- package/bundled/test-gen/file/server.js +0 -52
- package/bundled/test-gen/human-in-the-loop/cli.d.ts +0 -2
- package/bundled/test-gen/human-in-the-loop/cli.d.ts.map +0 -1
- package/bundled/test-gen/human-in-the-loop/cli.js +0 -24
- package/bundled/test-gen/human-in-the-loop/index.d.ts +0 -12
- package/bundled/test-gen/human-in-the-loop/index.d.ts.map +0 -1
- package/bundled/test-gen/human-in-the-loop/index.js +0 -30
- package/bundled/test-gen/human-in-the-loop/ipc.d.ts +0 -4
- package/bundled/test-gen/human-in-the-loop/ipc.d.ts.map +0 -1
- package/bundled/test-gen/human-in-the-loop/ipc.js +0 -47
- package/bundled/test-gen/index.d.ts +0 -4
- package/bundled/test-gen/index.d.ts.map +0 -1
- package/bundled/test-gen/index.js +0 -55
- package/bundled/test-gen/package.json +0 -106
- package/bundled/test-gen/page/index.d.ts +0 -11
- package/bundled/test-gen/page/index.d.ts.map +0 -1
- package/bundled/test-gen/page/index.js +0 -16
- package/bundled/test-gen/prompts/lib/ts-transformer.d.ts +0 -4
- package/bundled/test-gen/prompts/lib/ts-transformer.d.ts.map +0 -1
- package/bundled/test-gen/prompts/lib/ts-transformer.js +0 -92
- package/bundled/test-gen/reporter/index.d.ts +0 -33
- package/bundled/test-gen/reporter/index.d.ts.map +0 -1
- package/bundled/test-gen/reporter/index.js +0 -161
- package/bundled/test-gen/session/index.d.ts +0 -20
- package/bundled/test-gen/session/index.d.ts.map +0 -1
- package/bundled/test-gen/session/index.js +0 -105
- package/bundled/test-gen/test-build/index.d.ts +0 -10
- package/bundled/test-gen/test-build/index.d.ts.map +0 -1
- package/bundled/test-gen/test-build/index.js +0 -30
- package/bundled/test-gen/types/index.d.ts +0 -69
- package/bundled/test-gen/types/index.d.ts.map +0 -1
- package/bundled/test-gen/types/index.js +0 -2
- package/bundled/test-gen/uploader/index.d.ts +0 -26
- package/bundled/test-gen/uploader/index.d.ts.map +0 -1
- package/bundled/test-gen/uploader/index.js +0 -102
- package/bundled/test-gen/utils/env.d.ts +0 -2
- package/bundled/test-gen/utils/env.d.ts.map +0 -1
- package/bundled/test-gen/utils/env.js +0 -9
- package/bundled/test-gen/utils/exec.d.ts +0 -4
- package/bundled/test-gen/utils/exec.d.ts.map +0 -1
- package/bundled/test-gen/utils/exec.js +0 -45
- package/bundled/test-gen/utils/file.d.ts +0 -2
- package/bundled/test-gen/utils/file.d.ts.map +0 -1
- package/bundled/test-gen/utils/file.js +0 -25
- package/bundled/test-gen/utils/html.d.ts +0 -4
- package/bundled/test-gen/utils/html.d.ts.map +0 -1
- package/bundled/test-gen/utils/html.js +0 -46
- package/bundled/test-gen/utils/index.d.ts +0 -2
- package/bundled/test-gen/utils/index.d.ts.map +0 -1
- package/bundled/test-gen/utils/index.js +0 -5
- package/bundled/test-gen/utils/pw-test.d.ts +0 -3
- package/bundled/test-gen/utils/pw-test.d.ts.map +0 -1
- package/bundled/test-gen/utils/pw-test.js +0 -26
- package/bundled/test-gen/utils/slug.d.ts +0 -2
- package/bundled/test-gen/utils/slug.d.ts.map +0 -1
- package/bundled/test-gen/utils/slug.js +0 -18
- package/bundled/test-gen/utils/string.d.ts +0 -2
- package/bundled/test-gen/utils/string.d.ts.map +0 -1
- package/bundled/test-gen/utils/string.js +0 -9
- package/scripts/prepare-publish.js +0 -42
|
@@ -1,195 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.getAnnotationKeys = exports.getElementAnnotation = void 0;
|
|
4
|
-
const llm_1 = require("@empiricalrun/llm");
|
|
5
|
-
const constants_1 = require("../../constants");
|
|
6
|
-
const promptTemplate_0 = "{{#section \"system\"}}\nYou are an expert in describing the images and it's content. You need to provide the descriptions of annotated elements\npresent in the image.\n\nYou will be provided with an annotated screenshot where interact-able / clickable elements are annotated. The annotation\nis done by drawing a red box around the element and a small yellow box on it which contains unique element id.\n\nYou are given a Annotations which contains list of unique element id and description of the element separated by \":\".\n\nYou are also given the description of the element on which the action needs to be taken. The description includes\ninformation about how the element looks, it's position etc.\n\nYour task is to provide the annotation of the element on which the action needs to be performed based on the element\ndescription.\n\nFollow steps to fulfil your task:\n- Using the list of all element Ids provided to you, map all the element Ids on the annotated screen and describe each\nelement.\n- For describing each element Id\n-- iterate over each element Id in annotation list\n-- check if the description is already present for the element Id in the Annotation provided to you. If present skip\ndescribing it and use it as is.\n-- if the description is NA, then identify the element in the annotated screenshot and describe it using the image or\nicon enclosed in the element.\n- Respond with the mapped element Ids as \"enriched_annotations\"\n- Based on the description provided to you and the enriched annotations, first identify the element Id whose description\nmatches the task provided\n\nNote:\n- Ensure providing the description of all the elements in the list.\n- Don't update the description if its already present in the given annotations\n- Replace all the \"NA\" with description of the element. Its position, how does it look like etc.\n- There should be no \"NA\" present in any of the element description\n{{/section}}\n\n{{#section \"user\"}}\nElement description:\n{{elementDescription}}\n\nAnnotations:\n{{annotations}}\n\n{{image annotatedScreenshot}}\n{{/section}}";
|
|
7
|
-
const utils_1 = require("../utils");
|
|
8
|
-
const icon_descriptor_1 = require("./icon-descriptor");
|
|
9
|
-
const annotationToolAction = {
|
|
10
|
-
name: "element_annotation",
|
|
11
|
-
schema: {
|
|
12
|
-
type: "function",
|
|
13
|
-
function: {
|
|
14
|
-
name: "element_annotation",
|
|
15
|
-
description: "Handles annotations for elements and enriched annotations.",
|
|
16
|
-
parameters: {
|
|
17
|
-
type: "object",
|
|
18
|
-
properties: {
|
|
19
|
-
enriched_annotations: {
|
|
20
|
-
type: "array",
|
|
21
|
-
description: "List of enriched annotations for elements.",
|
|
22
|
-
items: {
|
|
23
|
-
type: "object",
|
|
24
|
-
properties: {
|
|
25
|
-
element_id: {
|
|
26
|
-
type: "string",
|
|
27
|
-
description: "Unique element ID for the element.",
|
|
28
|
-
},
|
|
29
|
-
description: {
|
|
30
|
-
type: "string",
|
|
31
|
-
description: "Detailed description for the element. Must not be NA",
|
|
32
|
-
},
|
|
33
|
-
},
|
|
34
|
-
required: ["element_id", "description"],
|
|
35
|
-
},
|
|
36
|
-
},
|
|
37
|
-
reason: {
|
|
38
|
-
type: "string",
|
|
39
|
-
description: "Explain why this element is selected. The reason should be clear and align with the task or purpose.",
|
|
40
|
-
},
|
|
41
|
-
element: {
|
|
42
|
-
type: "string",
|
|
43
|
-
description: "Detailed description of the DOM element's visual characteristics and position.",
|
|
44
|
-
},
|
|
45
|
-
element_annotation: {
|
|
46
|
-
type: "string",
|
|
47
|
-
description: "Return the unique element ID for the element on which the action needs to be performed.",
|
|
48
|
-
},
|
|
49
|
-
},
|
|
50
|
-
required: [
|
|
51
|
-
"enriched_annotations",
|
|
52
|
-
"reason",
|
|
53
|
-
"element",
|
|
54
|
-
"element_annotation",
|
|
55
|
-
],
|
|
56
|
-
},
|
|
57
|
-
},
|
|
58
|
-
},
|
|
59
|
-
};
|
|
60
|
-
async function getElementAnnotation({ elementDescription, annotations, annotatedScreenshot, trace, llm, options, preference, }) {
|
|
61
|
-
const annotationsSpan = trace?.span({
|
|
62
|
-
name: "get-element-annotation",
|
|
63
|
-
input: {
|
|
64
|
-
elementDescription,
|
|
65
|
-
annotations,
|
|
66
|
-
annotatedScreenshot,
|
|
67
|
-
preference,
|
|
68
|
-
},
|
|
69
|
-
});
|
|
70
|
-
const messages = (0, llm_1.compilePrompt)(promptTemplate_0, {
|
|
71
|
-
elementDescription,
|
|
72
|
-
annotations,
|
|
73
|
-
annotatedScreenshot,
|
|
74
|
-
}, options);
|
|
75
|
-
llm =
|
|
76
|
-
llm ||
|
|
77
|
-
new llm_1.LLM({
|
|
78
|
-
provider: options?.modelProvider || constants_1.DEFAULT_MODEL_PROVIDER,
|
|
79
|
-
defaultModel: options?.model || constants_1.DEFAULT_MODEL,
|
|
80
|
-
});
|
|
81
|
-
const completion = await llm.createChatCompletion({
|
|
82
|
-
messages,
|
|
83
|
-
modelParameters: {
|
|
84
|
-
...constants_1.DEFAULT_MODEL_PARAMETERS,
|
|
85
|
-
...options?.modelParameters,
|
|
86
|
-
tool_choice: "required",
|
|
87
|
-
temperature: 0.5,
|
|
88
|
-
},
|
|
89
|
-
trace: annotationsSpan,
|
|
90
|
-
traceName: "get-element-from-action",
|
|
91
|
-
//@ts-ignore
|
|
92
|
-
tools: [annotationToolAction.schema],
|
|
93
|
-
});
|
|
94
|
-
const toolCall = completion?.tool_calls?.[0];
|
|
95
|
-
annotationsSpan?.end({ output: toolCall });
|
|
96
|
-
if (toolCall) {
|
|
97
|
-
const args = (0, utils_1.parseJson)(toolCall.function.arguments);
|
|
98
|
-
return args.element_annotation;
|
|
99
|
-
}
|
|
100
|
-
return;
|
|
101
|
-
}
|
|
102
|
-
exports.getElementAnnotation = getElementAnnotation;
|
|
103
|
-
async function getAnnotationKeys({ page, preference, trace, }) {
|
|
104
|
-
const annotatedElements = await page.evaluate(({ preference, options }) => {
|
|
105
|
-
// @ts-ignore
|
|
106
|
-
// eslint-disable-next-line no-undef
|
|
107
|
-
window.annotationInstance = annotateElementsWithPreference({
|
|
108
|
-
options: options,
|
|
109
|
-
preference: preference,
|
|
110
|
-
});
|
|
111
|
-
const annotations = Object.entries(
|
|
112
|
-
// @ts-ignore
|
|
113
|
-
window.annotationInstance.annotations).map(([key, value]) => {
|
|
114
|
-
// Helper function to safely get text content
|
|
115
|
-
const getTextContent = (node) => {
|
|
116
|
-
try {
|
|
117
|
-
return node?.innerText?.trim() || null;
|
|
118
|
-
}
|
|
119
|
-
catch {
|
|
120
|
-
return null;
|
|
121
|
-
}
|
|
122
|
-
};
|
|
123
|
-
// Helper function to safely get placeholder
|
|
124
|
-
const getPlaceholder = (node) => {
|
|
125
|
-
try {
|
|
126
|
-
return node?.placeholder?.trim() || null;
|
|
127
|
-
}
|
|
128
|
-
catch {
|
|
129
|
-
return null;
|
|
130
|
-
}
|
|
131
|
-
};
|
|
132
|
-
return {
|
|
133
|
-
elementID: key,
|
|
134
|
-
innerText: getTextContent(value.node),
|
|
135
|
-
innerHTML: value.node.innerHTML,
|
|
136
|
-
outerHTML: value.node.outerHTML,
|
|
137
|
-
placeholder: getPlaceholder(value.node),
|
|
138
|
-
};
|
|
139
|
-
});
|
|
140
|
-
const fullPageHTML = document.documentElement.outerHTML;
|
|
141
|
-
return { annotations, fullPageHTML };
|
|
142
|
-
}, { preference, options: {} });
|
|
143
|
-
const annotationKeys = await enrichAnnotations(annotatedElements.annotations, annotatedElements.fullPageHTML, trace);
|
|
144
|
-
const annotationBuffer = await page.screenshot({
|
|
145
|
-
// path: `screenshots/screenshot-${screenshotIndex++}.png`,
|
|
146
|
-
});
|
|
147
|
-
const annotatedPageScreenshot = annotationBuffer.toString("base64");
|
|
148
|
-
await page.evaluate(() => {
|
|
149
|
-
console.log({
|
|
150
|
-
// @ts-ignore
|
|
151
|
-
disable: window?.annotationInstance?.disable,
|
|
152
|
-
});
|
|
153
|
-
if (
|
|
154
|
-
// @ts-ignore
|
|
155
|
-
window?.annotationInstance &&
|
|
156
|
-
// @ts-ignore
|
|
157
|
-
window?.annotationInstance?.destroy) {
|
|
158
|
-
// @ts-ignore
|
|
159
|
-
window?.annotationInstance?.destroy();
|
|
160
|
-
}
|
|
161
|
-
});
|
|
162
|
-
return {
|
|
163
|
-
annotationKeys,
|
|
164
|
-
annotationBuffer,
|
|
165
|
-
annotatedPageScreenshot,
|
|
166
|
-
};
|
|
167
|
-
}
|
|
168
|
-
exports.getAnnotationKeys = getAnnotationKeys;
|
|
169
|
-
async function enrichAnnotations(annotatedElements, pageHtml, trace) {
|
|
170
|
-
// create icon descriptor span
|
|
171
|
-
const iconDescriptionSpan = trace?.span({
|
|
172
|
-
name: "describe-icons",
|
|
173
|
-
});
|
|
174
|
-
const results = [];
|
|
175
|
-
for (const element of annotatedElements) {
|
|
176
|
-
const text = element.innerText?.trim() || element.placeholder?.trim();
|
|
177
|
-
if (text) {
|
|
178
|
-
results.push({
|
|
179
|
-
elementID: element.elementID,
|
|
180
|
-
text,
|
|
181
|
-
});
|
|
182
|
-
continue;
|
|
183
|
-
}
|
|
184
|
-
const description = await (0, icon_descriptor_1.getIconDescription)({
|
|
185
|
-
htmlString: element.outerHTML,
|
|
186
|
-
pageHtml: pageHtml,
|
|
187
|
-
trace: iconDescriptionSpan,
|
|
188
|
-
});
|
|
189
|
-
results.push({
|
|
190
|
-
elementID: element.elementID,
|
|
191
|
-
text: description || "NA",
|
|
192
|
-
});
|
|
193
|
-
}
|
|
194
|
-
return results;
|
|
195
|
-
}
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import { LLM, TraceClient } from "@empiricalrun/llm";
|
|
2
|
-
import type { Page } from "@playwright/test";
|
|
3
|
-
import { PlaywrightActions } from "../../actions";
|
|
4
|
-
export declare function executeBrowserAction({ page, nextAction, flags, actions, llm, trace, }: {
|
|
5
|
-
page: Page;
|
|
6
|
-
nextAction: {
|
|
7
|
-
actionType: string;
|
|
8
|
-
toolCallArgs: string;
|
|
9
|
-
};
|
|
10
|
-
flags: {
|
|
11
|
-
useActionSpecificAnnotations: boolean;
|
|
12
|
-
};
|
|
13
|
-
actions: PlaywrightActions;
|
|
14
|
-
trace: TraceClient | undefined;
|
|
15
|
-
llm: LLM;
|
|
16
|
-
}): Promise<{
|
|
17
|
-
generatedCodeSteps: string[];
|
|
18
|
-
output: {
|
|
19
|
-
action: string;
|
|
20
|
-
reason: string;
|
|
21
|
-
elementAnnotation?: string | undefined;
|
|
22
|
-
};
|
|
23
|
-
}>;
|
|
24
|
-
//# sourceMappingURL=execute-browser-action.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"execute-browser-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-browser-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAYlD,wBAAsB,oBAAoB,CAAC,EACzC,IAAI,EACJ,UAAU,EACV,KAAK,EACL,OAAO,EACP,GAAG,EACH,KAAK,GACN,EAAE;IACD,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE;QACL,4BAA4B,EAAE,OAAO,CAAC;KACvC,CAAC;IACF,OAAO,EAAE,iBAAiB,CAAC;IAC3B,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;IAC/B,GAAG,EAAE,GAAG,CAAC;CACV;;;gBAIW,MAAM;gBACN,MAAM;;;GAkIjB"}
|
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.executeBrowserAction = void 0;
|
|
4
|
-
const reporter_1 = require("../../reporter");
|
|
5
|
-
const browsing_1 = require("../browsing");
|
|
6
|
-
const utils_1 = require("../utils");
|
|
7
|
-
const action_tool_calls_1 = require("./action-tool-calls");
|
|
8
|
-
const element_annotation_1 = require("./element-annotation");
|
|
9
|
-
const with_hints_1 = require("./with-hints");
|
|
10
|
-
async function executeBrowserAction({ page, nextAction, flags, actions, llm, trace, }) {
|
|
11
|
-
const args = (0, utils_1.parseJson)(nextAction.toolCallArgs);
|
|
12
|
-
let generatedCodeSteps = [];
|
|
13
|
-
let output = {
|
|
14
|
-
action: args.action,
|
|
15
|
-
reason: args.reason,
|
|
16
|
-
};
|
|
17
|
-
let shouldTriggerHintsFlow;
|
|
18
|
-
let hintsExecutionCompletion;
|
|
19
|
-
let elementAnnotation;
|
|
20
|
-
const actionType = nextAction.actionType;
|
|
21
|
-
let preference = {
|
|
22
|
-
actionType: "all",
|
|
23
|
-
};
|
|
24
|
-
if (flags.useActionSpecificAnnotations && (0, action_tool_calls_1.isValidActionType)(actionType)) {
|
|
25
|
-
switch (actionType) {
|
|
26
|
-
case action_tool_calls_1.ActionType.FILL:
|
|
27
|
-
preference = {
|
|
28
|
-
actionType: action_tool_calls_1.ActionType.FILL,
|
|
29
|
-
};
|
|
30
|
-
break;
|
|
31
|
-
case action_tool_calls_1.ActionType.ASSERT_TEXT:
|
|
32
|
-
preference = {
|
|
33
|
-
actionType: action_tool_calls_1.ActionType.ASSERT_TEXT,
|
|
34
|
-
assertionText: args.assertion_text,
|
|
35
|
-
};
|
|
36
|
-
break;
|
|
37
|
-
default:
|
|
38
|
-
preference = {
|
|
39
|
-
actionType: "all",
|
|
40
|
-
};
|
|
41
|
-
}
|
|
42
|
-
}
|
|
43
|
-
let { annotationKeys, annotatedPageScreenshot, annotationBuffer } = await (0, element_annotation_1.getAnnotationKeys)({
|
|
44
|
-
page,
|
|
45
|
-
preference,
|
|
46
|
-
trace,
|
|
47
|
-
});
|
|
48
|
-
if (annotationKeys.length > 0) {
|
|
49
|
-
// TODO: this string has newline characters that makes it harder to read
|
|
50
|
-
const annotationMapString = annotationKeys
|
|
51
|
-
?.map((a) => `${a.elementID}: ${a.text}`)
|
|
52
|
-
.join("\n");
|
|
53
|
-
// Provides the annotations for all the element present on screen
|
|
54
|
-
// Also provides the annotation of element on which the action needs to be taken
|
|
55
|
-
elementAnnotation = await (0, element_annotation_1.getElementAnnotation)({
|
|
56
|
-
elementDescription: args.element_description,
|
|
57
|
-
annotations: annotationMapString,
|
|
58
|
-
annotatedScreenshot: annotatedPageScreenshot,
|
|
59
|
-
trace,
|
|
60
|
-
llm,
|
|
61
|
-
preference,
|
|
62
|
-
});
|
|
63
|
-
output.elementAnnotation = elementAnnotation;
|
|
64
|
-
console.log("Output: ", output);
|
|
65
|
-
const testGenReporter = new reporter_1.TestGenUpdatesReporter();
|
|
66
|
-
await testGenReporter.sendCurrentView(annotationBuffer);
|
|
67
|
-
const triggerHintsFlowSpan = trace?.span({
|
|
68
|
-
name: "trigger-hints-flow",
|
|
69
|
-
input: {
|
|
70
|
-
outputFromGetNextAction: output,
|
|
71
|
-
generatedAnnotations: annotationKeys,
|
|
72
|
-
},
|
|
73
|
-
});
|
|
74
|
-
// Provides the action whether its a click, fill etc.
|
|
75
|
-
const result = await (0, with_hints_1.triggerHintsFlow)({
|
|
76
|
-
outputFromGetNextAction: output,
|
|
77
|
-
generatedAnnotations: annotationKeys,
|
|
78
|
-
actions,
|
|
79
|
-
llm,
|
|
80
|
-
trace: triggerHintsFlowSpan,
|
|
81
|
-
});
|
|
82
|
-
shouldTriggerHintsFlow = result.shouldTriggerHintsFlow;
|
|
83
|
-
hintsExecutionCompletion = result.hintsExecutionCompletion;
|
|
84
|
-
triggerHintsFlowSpan?.end({
|
|
85
|
-
output: result,
|
|
86
|
-
});
|
|
87
|
-
}
|
|
88
|
-
if (shouldTriggerHintsFlow && hintsExecutionCompletion) {
|
|
89
|
-
const toolCalls = hintsExecutionCompletion?.tool_calls || [];
|
|
90
|
-
for (const i in toolCalls) {
|
|
91
|
-
const currentToolCall = toolCalls[i];
|
|
92
|
-
const code = await actions.executeAction(currentToolCall.function.name, {
|
|
93
|
-
...JSON.parse(currentToolCall.function.arguments),
|
|
94
|
-
elementAnnotation,
|
|
95
|
-
}, trace);
|
|
96
|
-
if (code) {
|
|
97
|
-
generatedCodeSteps.push(code);
|
|
98
|
-
}
|
|
99
|
-
}
|
|
100
|
-
if (actions.isStuckInLoop()) {
|
|
101
|
-
throw new Error("Agent is not able to figure out next action when using hints");
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
else {
|
|
105
|
-
const browserAction = await (0, browsing_1.executeTaskUsingBrowsingAgent)({
|
|
106
|
-
trace,
|
|
107
|
-
action: output.action,
|
|
108
|
-
page,
|
|
109
|
-
llm,
|
|
110
|
-
actions,
|
|
111
|
-
});
|
|
112
|
-
if (browserAction) {
|
|
113
|
-
output.action = browserAction.action;
|
|
114
|
-
if (browserAction.code) {
|
|
115
|
-
generatedCodeSteps.push(browserAction.code);
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
return {
|
|
120
|
-
generatedCodeSteps,
|
|
121
|
-
output,
|
|
122
|
-
};
|
|
123
|
-
}
|
|
124
|
-
exports.executeBrowserAction = executeBrowserAction;
|
|
@@ -1,11 +0,0 @@
|
|
|
1
|
-
import { TraceClient } from "@empiricalrun/llm";
|
|
2
|
-
import { PlaywrightActions } from "../../actions";
|
|
3
|
-
export declare function executeSkillAction({ actions, nextAction, trace, }: {
|
|
4
|
-
actions: PlaywrightActions;
|
|
5
|
-
nextAction: {
|
|
6
|
-
actionType: string;
|
|
7
|
-
toolCallArgs: string;
|
|
8
|
-
};
|
|
9
|
-
trace: TraceClient | undefined;
|
|
10
|
-
}): Promise<string | undefined>;
|
|
11
|
-
//# sourceMappingURL=execute-skill-action.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"execute-skill-action.d.ts","sourceRoot":"","sources":["../../../src/agent/master/execute-skill-action.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAOlD,wBAAsB,kBAAkB,CAAC,EACvC,OAAO,EACP,UAAU,EACV,KAAK,GACN,EAAE;IACD,OAAO,EAAE,iBAAiB,CAAC;IAC3B,UAAU,EAAE;QACV,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,KAAK,EAAE,WAAW,GAAG,SAAS,CAAC;CAChC,+BAeA"}
|
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.executeSkillAction = void 0;
|
|
4
|
-
const skill_1 = require("../../actions/skill");
|
|
5
|
-
const errors_1 = require("../../errors");
|
|
6
|
-
const human_in_the_loop_1 = require("../../human-in-the-loop");
|
|
7
|
-
const env_1 = require("../../utils/env");
|
|
8
|
-
const utils_1 = require("../utils");
|
|
9
|
-
async function executeSkillAction({ actions, nextAction, trace, }) {
|
|
10
|
-
const args = (0, utils_1.parseJson)(nextAction.toolCallArgs);
|
|
11
|
-
if (!(0, env_1.isRunningOnCloud)()) {
|
|
12
|
-
const { skillDetails } = (0, skill_1.extractSkillFromArgs)(args);
|
|
13
|
-
const feedback = await human_in_the_loop_1.humanLoop.getFeedback({
|
|
14
|
-
message: `Should I use this skill?\n\n${skillDetails?.usageExample}\n\nfrom: ${skillDetails?.filePath}\n\nAnswer with "yes" or "no"`,
|
|
15
|
-
});
|
|
16
|
-
if (feedback.toLowerCase().trim().includes("no")) {
|
|
17
|
-
throw new errors_1.HumanApprovalDenied("Skill usage approval not provided");
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
const code = await actions.executeAction(nextAction.actionType, args, trace);
|
|
21
|
-
if (code) {
|
|
22
|
-
return code;
|
|
23
|
-
}
|
|
24
|
-
}
|
|
25
|
-
exports.executeSkillAction = executeSkillAction;
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
import { TraceClient } from "@empiricalrun/llm";
|
|
2
|
-
type IconKnowledge = {
|
|
3
|
-
description: string;
|
|
4
|
-
key: string;
|
|
5
|
-
normalized: string;
|
|
6
|
-
createdAt: Date;
|
|
7
|
-
};
|
|
8
|
-
export declare function loadIconsKnowledge(): Array<IconKnowledge>;
|
|
9
|
-
export declare function saveIconsKnowledge(iconsData: Array<IconKnowledge>): Promise<void>;
|
|
10
|
-
export declare function generateKey(htmlString: string): string;
|
|
11
|
-
export declare function reverseKey(hash: string): string;
|
|
12
|
-
export declare function createNodeFromHTML(htmlString: string): {
|
|
13
|
-
node: Element | null;
|
|
14
|
-
descendant: Element[];
|
|
15
|
-
};
|
|
16
|
-
export declare function getIconDescription({ htmlString, pageHtml, trace, }: {
|
|
17
|
-
htmlString: string;
|
|
18
|
-
trace?: TraceClient;
|
|
19
|
-
pageHtml: string;
|
|
20
|
-
}): Promise<string | undefined>;
|
|
21
|
-
export {};
|
|
22
|
-
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/agent/master/icon-descriptor/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAO,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAarD,KAAK,aAAa,GAAG;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,IAAI,CAAC;CACjB,CAAC;AAEF,wBAAgB,kBAAkB,IAAI,KAAK,CAAC,aAAa,CAAC,CAoDzD;AAED,wBAAsB,kBAAkB,CACtC,SAAS,EAAE,KAAK,CAAC,aAAa,CAAC,GAC9B,OAAO,CAAC,IAAI,CAAC,CAaf;AAED,wBAAgB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAG/C;AAED,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM;;;EAUpD;AAqKD,wBAAsB,kBAAkB,CAAC,EACvC,UAAU,EACV,QAAQ,EACR,KAAK,GACN,EAAE;IACD,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAkC9B"}
|
|
@@ -1,250 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
-
};
|
|
5
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
-
exports.getIconDescription = exports.createNodeFromHTML = exports.reverseKey = exports.generateKey = exports.saveIconsKnowledge = exports.loadIconsKnowledge = void 0;
|
|
7
|
-
const llm_1 = require("@empiricalrun/llm");
|
|
8
|
-
const fs_1 = __importDefault(require("fs"));
|
|
9
|
-
const jsdom_1 = require("jsdom");
|
|
10
|
-
const path_1 = __importDefault(require("path"));
|
|
11
|
-
const constants_1 = require("../../../constants");
|
|
12
|
-
const normalize_svg_1 = require("./normalize-svg");
|
|
13
|
-
const OLD_ICONS_KNOWLEDGE_PATH = path_1.default.join(process.cwd(), "icons.json");
|
|
14
|
-
const EMPIRICAL_RUN_DIR = path_1.default.join(process.cwd(), ".empiricalrun");
|
|
15
|
-
const ICONS_KNOWLEDGE_PATH = path_1.default.join(EMPIRICAL_RUN_DIR, "icons.json");
|
|
16
|
-
function loadIconsKnowledge() {
|
|
17
|
-
// Check if file exists in the new location
|
|
18
|
-
if (fs_1.default.existsSync(ICONS_KNOWLEDGE_PATH)) {
|
|
19
|
-
const raw = fs_1.default.readFileSync(ICONS_KNOWLEDGE_PATH, "utf-8");
|
|
20
|
-
if (raw) {
|
|
21
|
-
try {
|
|
22
|
-
return JSON.parse(raw);
|
|
23
|
-
}
|
|
24
|
-
catch (err) {
|
|
25
|
-
console.log("error parsing iconsKnowledge json");
|
|
26
|
-
return [];
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
return [];
|
|
30
|
-
}
|
|
31
|
-
// Check if file exists in the old location and move it if it does
|
|
32
|
-
if (fs_1.default.existsSync(OLD_ICONS_KNOWLEDGE_PATH)) {
|
|
33
|
-
try {
|
|
34
|
-
const raw = fs_1.default.readFileSync(OLD_ICONS_KNOWLEDGE_PATH, "utf-8");
|
|
35
|
-
let iconData = [];
|
|
36
|
-
if (raw) {
|
|
37
|
-
try {
|
|
38
|
-
iconData = JSON.parse(raw);
|
|
39
|
-
}
|
|
40
|
-
catch (err) {
|
|
41
|
-
console.log("error parsing iconsKnowledge json from old location");
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
// Create the directory and move the file
|
|
45
|
-
if (!fs_1.default.existsSync(EMPIRICAL_RUN_DIR)) {
|
|
46
|
-
fs_1.default.mkdirSync(EMPIRICAL_RUN_DIR, { recursive: true });
|
|
47
|
-
}
|
|
48
|
-
// Save to new location
|
|
49
|
-
fs_1.default.writeFileSync(ICONS_KNOWLEDGE_PATH, JSON.stringify(iconData, null, 2), "utf-8");
|
|
50
|
-
// Remove the old file
|
|
51
|
-
fs_1.default.unlinkSync(OLD_ICONS_KNOWLEDGE_PATH);
|
|
52
|
-
return iconData;
|
|
53
|
-
}
|
|
54
|
-
catch (err) {
|
|
55
|
-
console.log("Error moving icons knowledge file:", err);
|
|
56
|
-
return [];
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
return [];
|
|
60
|
-
}
|
|
61
|
-
exports.loadIconsKnowledge = loadIconsKnowledge;
|
|
62
|
-
async function saveIconsKnowledge(iconsData) {
|
|
63
|
-
// Ensure the directory exists
|
|
64
|
-
if (!fs_1.default.existsSync(EMPIRICAL_RUN_DIR)) {
|
|
65
|
-
fs_1.default.mkdirSync(EMPIRICAL_RUN_DIR, { recursive: true });
|
|
66
|
-
}
|
|
67
|
-
const content = JSON.stringify(iconsData, null, 2);
|
|
68
|
-
fs_1.default.writeFileSync(ICONS_KNOWLEDGE_PATH, content, "utf-8");
|
|
69
|
-
// If old file exists, remove it
|
|
70
|
-
if (fs_1.default.existsSync(OLD_ICONS_KNOWLEDGE_PATH)) {
|
|
71
|
-
fs_1.default.unlinkSync(OLD_ICONS_KNOWLEDGE_PATH);
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
|
-
exports.saveIconsKnowledge = saveIconsKnowledge;
|
|
75
|
-
function generateKey(htmlString) {
|
|
76
|
-
const normalized = (0, normalize_svg_1.normalizeSVG)(htmlString);
|
|
77
|
-
// generate base64 string
|
|
78
|
-
const encoded = Buffer.from(normalized).toString("base64");
|
|
79
|
-
return `icon_${encoded}`;
|
|
80
|
-
}
|
|
81
|
-
exports.generateKey = generateKey;
|
|
82
|
-
function reverseKey(hash) {
|
|
83
|
-
const encoded = hash.replace(/^icon_/, "");
|
|
84
|
-
return Buffer.from(encoded, "base64").toString();
|
|
85
|
-
}
|
|
86
|
-
exports.reverseKey = reverseKey;
|
|
87
|
-
function createNodeFromHTML(htmlString) {
|
|
88
|
-
const dom = new jsdom_1.JSDOM(htmlString);
|
|
89
|
-
const document = dom.window.document;
|
|
90
|
-
const node = document.body.firstElementChild;
|
|
91
|
-
return {
|
|
92
|
-
node, // Return the first node
|
|
93
|
-
descendant: node ? Array.from(node.querySelectorAll("*")) : [], // Convert HTMLCollection to array
|
|
94
|
-
};
|
|
95
|
-
}
|
|
96
|
-
exports.createNodeFromHTML = createNodeFromHTML;
|
|
97
|
-
function processSvgWithUseElements(svgElement, document) {
|
|
98
|
-
try {
|
|
99
|
-
const useElements = svgElement.querySelectorAll("use");
|
|
100
|
-
for (const useEl of Array.from(useElements)) {
|
|
101
|
-
const href = useEl.getAttribute("href") || useEl.getAttribute("xlink:href");
|
|
102
|
-
if (!href)
|
|
103
|
-
continue;
|
|
104
|
-
const id = href.startsWith("#") ? href.substring(1) : href;
|
|
105
|
-
const referencedElement = document.getElementById(id);
|
|
106
|
-
if (referencedElement) {
|
|
107
|
-
if (referencedElement.tagName.toLowerCase() === "symbol" ||
|
|
108
|
-
referencedElement.tagName.toLowerCase() === "svg") {
|
|
109
|
-
const group = document.createElement("g");
|
|
110
|
-
Array.from(useEl.attributes).forEach((attr) => {
|
|
111
|
-
if (attr.name !== "href" && attr.name !== "xlink:href") {
|
|
112
|
-
group.setAttribute(attr.name, attr.value);
|
|
113
|
-
}
|
|
114
|
-
});
|
|
115
|
-
if (referencedElement.hasAttribute("viewBox")) {
|
|
116
|
-
group.setAttribute("viewBox", referencedElement.getAttribute("viewBox") || "");
|
|
117
|
-
}
|
|
118
|
-
group.innerHTML = referencedElement.innerHTML;
|
|
119
|
-
useEl.parentNode?.replaceChild(group, useEl);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
}
|
|
124
|
-
catch (error) {
|
|
125
|
-
console.error("Error processing SVG with use elements:", error);
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
/**
|
|
129
|
-
* Extracts and returns an SVG element from a given HTML string.
|
|
130
|
-
*
|
|
131
|
-
* This function checks whether the provided HTML string contains an SVG element
|
|
132
|
-
* either as the root node or as a child node.
|
|
133
|
-
*/
|
|
134
|
-
function getHtmlForDescription(elementHtml, pageHtml) {
|
|
135
|
-
const dom = new jsdom_1.JSDOM(pageHtml);
|
|
136
|
-
const page = dom.window.document;
|
|
137
|
-
const { node, descendant } = createNodeFromHTML(elementHtml);
|
|
138
|
-
if (!node) {
|
|
139
|
-
return undefined;
|
|
140
|
-
}
|
|
141
|
-
if (node.tagName.toLowerCase() === "svg") {
|
|
142
|
-
processSvgWithUseElements(node, page);
|
|
143
|
-
return node.outerHTML;
|
|
144
|
-
}
|
|
145
|
-
const svgChildren = descendant?.filter((child) => child?.tagName?.toLowerCase() === "svg");
|
|
146
|
-
if (svgChildren && svgChildren.length > 0) {
|
|
147
|
-
for (const svgChild of svgChildren) {
|
|
148
|
-
processSvgWithUseElements(svgChild, page);
|
|
149
|
-
}
|
|
150
|
-
return node.outerHTML;
|
|
151
|
-
}
|
|
152
|
-
return undefined;
|
|
153
|
-
}
|
|
154
|
-
async function describeSVGElementWithLLM({ element, trace, }) {
|
|
155
|
-
try {
|
|
156
|
-
const svgDescriptionSpan = trace?.span({
|
|
157
|
-
name: "get-svg-description",
|
|
158
|
-
input: {
|
|
159
|
-
element,
|
|
160
|
-
},
|
|
161
|
-
});
|
|
162
|
-
const messages = [
|
|
163
|
-
{
|
|
164
|
-
role: "system",
|
|
165
|
-
content: `
|
|
166
|
-
You are an expert at reading svg icons. You are given a task to provide the description of an icon based on the given HTML element string.
|
|
167
|
-
If the string does not contain any svg element, return NA.
|
|
168
|
-
`,
|
|
169
|
-
},
|
|
170
|
-
{
|
|
171
|
-
role: "user",
|
|
172
|
-
content: `
|
|
173
|
-
Provide the description of below element:
|
|
174
|
-
${element}
|
|
175
|
-
|
|
176
|
-
Follow below steps to provide the description:
|
|
177
|
-
- Identify the svg element in the given HTML string
|
|
178
|
-
- If the string does not contain any svg element, return NA
|
|
179
|
-
- Else, read the svg element and provide a single line description for the element
|
|
180
|
-
- The description should be precise and readable
|
|
181
|
-
- Only provide the element description and nothing else
|
|
182
|
-
`,
|
|
183
|
-
},
|
|
184
|
-
];
|
|
185
|
-
const llm = new llm_1.LLM({
|
|
186
|
-
trace: svgDescriptionSpan,
|
|
187
|
-
provider: "anthropic",
|
|
188
|
-
defaultModel: "claude-3-5-sonnet-latest",
|
|
189
|
-
providerApiKey: constants_1.MODEL_API_KEYS["anthropic"],
|
|
190
|
-
});
|
|
191
|
-
const svgDescription = await llm.createChatCompletion({
|
|
192
|
-
messages,
|
|
193
|
-
modelParameters: {
|
|
194
|
-
temperature: 0.3,
|
|
195
|
-
},
|
|
196
|
-
responseFormat: {
|
|
197
|
-
type: "json_schema",
|
|
198
|
-
json_schema: {
|
|
199
|
-
name: "describe-svg-element",
|
|
200
|
-
strict: true,
|
|
201
|
-
schema: {
|
|
202
|
-
type: "object",
|
|
203
|
-
properties: {
|
|
204
|
-
description: {
|
|
205
|
-
type: "string",
|
|
206
|
-
description: "A clear description of what the SVG visually represents",
|
|
207
|
-
},
|
|
208
|
-
},
|
|
209
|
-
additionalProperties: false,
|
|
210
|
-
required: ["description"],
|
|
211
|
-
},
|
|
212
|
-
},
|
|
213
|
-
},
|
|
214
|
-
});
|
|
215
|
-
svgDescriptionSpan?.end({ output: svgDescription });
|
|
216
|
-
return svgDescription?.content ?? undefined;
|
|
217
|
-
}
|
|
218
|
-
catch (err) {
|
|
219
|
-
console.error(`Error generating svg description ${err}`);
|
|
220
|
-
return undefined;
|
|
221
|
-
}
|
|
222
|
-
}
|
|
223
|
-
async function getIconDescription({ htmlString, pageHtml, trace, }) {
|
|
224
|
-
const elementStringForDescription = getHtmlForDescription(htmlString, pageHtml);
|
|
225
|
-
if (!elementStringForDescription) {
|
|
226
|
-
return undefined;
|
|
227
|
-
}
|
|
228
|
-
const key = generateKey(elementStringForDescription);
|
|
229
|
-
const iconsData = loadIconsKnowledge();
|
|
230
|
-
const cachedIcon = iconsData.find((icon) => icon.key === key);
|
|
231
|
-
if (cachedIcon) {
|
|
232
|
-
return cachedIcon.description;
|
|
233
|
-
}
|
|
234
|
-
const description = await describeSVGElementWithLLM({
|
|
235
|
-
element: elementStringForDescription,
|
|
236
|
-
trace,
|
|
237
|
-
});
|
|
238
|
-
if (description) {
|
|
239
|
-
const newIconEntry = {
|
|
240
|
-
description,
|
|
241
|
-
key,
|
|
242
|
-
normalized: htmlString,
|
|
243
|
-
createdAt: new Date(),
|
|
244
|
-
};
|
|
245
|
-
iconsData.push(newIconEntry);
|
|
246
|
-
await saveIconsKnowledge(iconsData);
|
|
247
|
-
}
|
|
248
|
-
return description;
|
|
249
|
-
}
|
|
250
|
-
exports.getIconDescription = getIconDescription;
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"normalize-svg.d.ts","sourceRoot":"","sources":["../../../../src/agent/master/icon-descriptor/normalize-svg.ts"],"names":[],"mappings":"AAEA,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CA8BpD"}
|