libretto 0.5.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +109 -35
- package/dist/cli/cli.js +22 -97
- package/dist/cli/commands/browser.js +86 -59
- package/dist/cli/commands/execution.js +199 -86
- package/dist/cli/commands/init.js +34 -29
- package/dist/cli/commands/logs.js +4 -5
- package/dist/cli/commands/shared.js +30 -29
- package/dist/cli/commands/snapshot.js +26 -39
- package/dist/cli/core/ai-config.js +21 -4
- package/dist/cli/core/api-snapshot-analyzer.js +15 -5
- package/dist/cli/core/browser.js +207 -37
- package/dist/cli/core/context.js +4 -1
- package/dist/cli/core/session-telemetry.js +434 -174
- package/dist/cli/core/session.js +21 -8
- package/dist/cli/core/snapshot-analyzer.js +14 -31
- package/dist/cli/core/snapshot-api-config.js +2 -6
- package/dist/cli/core/telemetry.js +20 -4
- package/dist/cli/framework/simple-cli.js +45 -25
- package/dist/cli/router.js +14 -21
- package/dist/cli/workers/run-integration-runtime.js +24 -5
- package/dist/cli/workers/run-integration-worker-protocol.js +3 -1
- package/dist/cli/workers/run-integration-worker.js +1 -4
- package/dist/index.d.ts +1 -2
- package/dist/index.js +7 -10
- package/dist/runtime/download/download.js +5 -1
- package/dist/runtime/extract/extract.js +11 -2
- package/dist/runtime/network/network.js +8 -1
- package/dist/runtime/recovery/agent.js +6 -2
- package/dist/runtime/recovery/errors.js +3 -1
- package/dist/runtime/recovery/recovery.js +3 -1
- package/dist/shared/condense-dom/condense-dom.js +17 -69
- package/dist/shared/config/config.d.ts +1 -9
- package/dist/shared/config/config.js +0 -18
- package/dist/shared/config/index.d.ts +2 -1
- package/dist/shared/config/index.js +0 -10
- package/dist/shared/debug/pause.js +9 -3
- package/dist/shared/dom-semantics.d.ts +8 -0
- package/dist/shared/dom-semantics.js +69 -0
- package/dist/shared/instrumentation/instrument.js +101 -5
- package/dist/shared/llm/ai-sdk-adapter.js +3 -1
- package/dist/shared/llm/client.js +3 -1
- package/dist/shared/logger/index.js +4 -1
- package/dist/shared/run/api.js +3 -1
- package/dist/shared/run/browser.js +47 -3
- package/dist/shared/state/session-state.d.ts +2 -1
- package/dist/shared/state/session-state.js +5 -2
- package/dist/shared/visualization/ghost-cursor.js +36 -14
- package/dist/shared/visualization/highlight.js +9 -6
- package/dist/shared/workflow/workflow.d.ts +4 -5
- package/dist/shared/workflow/workflow.js +3 -5
- package/package.json +6 -2
- package/scripts/check-skills-sync.mjs +25 -0
- package/scripts/compare-eval-summary.mjs +47 -0
- package/scripts/postinstall.mjs +15 -15
- package/scripts/prepare-release.sh +97 -0
- package/scripts/skills-libretto.mjs +103 -0
- package/scripts/summarize-evals.mjs +135 -0
- package/scripts/sync-skills.mjs +12 -0
- package/skills/libretto/SKILL.md +132 -54
- package/skills/libretto/references/action-logs.md +101 -0
- package/skills/libretto/references/auth-profiles.md +1 -2
- package/skills/libretto/references/code-generation-rules.md +210 -0
- package/skills/libretto/references/configuration-file-reference.md +53 -0
- package/skills/libretto/references/pages-and-page-targeting.md +1 -1
- package/skills/libretto/references/site-security-review.md +143 -0
- package/src/cli/cli.ts +23 -110
- package/src/cli/commands/browser.ts +94 -70
- package/src/cli/commands/execution.ts +233 -102
- package/src/cli/commands/init.ts +37 -33
- package/src/cli/commands/logs.ts +7 -7
- package/src/cli/commands/shared.ts +36 -37
- package/src/cli/commands/snapshot.ts +44 -59
- package/src/cli/core/ai-config.ts +24 -4
- package/src/cli/core/api-snapshot-analyzer.ts +17 -6
- package/src/cli/core/browser.ts +260 -49
- package/src/cli/core/context.ts +7 -2
- package/src/cli/core/session-telemetry.ts +449 -197
- package/src/cli/core/session.ts +21 -7
- package/src/cli/core/snapshot-analyzer.ts +26 -46
- package/src/cli/core/snapshot-api-config.ts +170 -175
- package/src/cli/core/telemetry.ts +39 -4
- package/src/cli/framework/simple-cli.ts +144 -77
- package/src/cli/router.ts +13 -21
- package/src/cli/workers/run-integration-runtime.ts +36 -9
- package/src/cli/workers/run-integration-worker-protocol.ts +2 -0
- package/src/cli/workers/run-integration-worker.ts +1 -4
- package/src/index.ts +73 -66
- package/src/runtime/download/download.ts +62 -58
- package/src/runtime/download/index.ts +5 -5
- package/src/runtime/extract/extract.ts +71 -61
- package/src/runtime/network/index.ts +3 -3
- package/src/runtime/network/network.ts +99 -93
- package/src/runtime/recovery/agent.ts +217 -212
- package/src/runtime/recovery/errors.ts +107 -104
- package/src/runtime/recovery/index.ts +3 -3
- package/src/runtime/recovery/recovery.ts +38 -35
- package/src/shared/condense-dom/condense-dom.ts +27 -82
- package/src/shared/config/config.ts +0 -19
- package/src/shared/config/index.ts +0 -5
- package/src/shared/debug/pause.ts +57 -51
- package/src/shared/dom-semantics.ts +68 -0
- package/src/shared/instrumentation/errors.ts +64 -62
- package/src/shared/instrumentation/index.ts +5 -5
- package/src/shared/instrumentation/instrument.ts +339 -209
- package/src/shared/llm/ai-sdk-adapter.ts +58 -55
- package/src/shared/llm/client.ts +181 -174
- package/src/shared/llm/types.ts +39 -39
- package/src/shared/logger/index.ts +11 -4
- package/src/shared/logger/logger.ts +312 -306
- package/src/shared/logger/sinks.ts +118 -114
- package/src/shared/paths/paths.ts +50 -49
- package/src/shared/paths/repo-root.ts +17 -17
- package/src/shared/run/api.ts +5 -1
- package/src/shared/run/browser.ts +65 -3
- package/src/shared/state/index.ts +9 -9
- package/src/shared/state/session-state.ts +46 -43
- package/src/shared/visualization/ghost-cursor.ts +180 -149
- package/src/shared/visualization/highlight.ts +89 -86
- package/src/shared/visualization/index.ts +13 -13
- package/src/shared/workflow/workflow.ts +19 -25
- package/skills/libretto/references/reverse-engineering-network-requests.md +0 -39
- package/skills/libretto/references/user-action-log.md +0 -31
|
@@ -1,22 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
"alt",
|
|
11
|
-
"type",
|
|
12
|
-
"value",
|
|
13
|
-
"placeholder",
|
|
14
|
-
"autocomplete",
|
|
15
|
-
"href",
|
|
16
|
-
"action",
|
|
17
|
-
"method",
|
|
18
|
-
"src"
|
|
19
|
-
]);
|
|
1
|
+
import {
|
|
2
|
+
filterSemanticClasses,
|
|
3
|
+
INTERACTIVE_ROLE_NAMES,
|
|
4
|
+
INTERACTIVE_TAG_NAMES,
|
|
5
|
+
TEST_ATTRIBUTE_NAMES,
|
|
6
|
+
TRUSTED_ATTRIBUTE_NAMES
|
|
7
|
+
} from "../dom-semantics.js";
|
|
8
|
+
const TEST_ATTRS = new Set(TEST_ATTRIBUTE_NAMES);
|
|
9
|
+
const TRUSTED_ATTRS = new Set(TRUSTED_ATTRIBUTE_NAMES);
|
|
20
10
|
const STATE_ATTRS = /* @__PURE__ */ new Set([
|
|
21
11
|
"disabled",
|
|
22
12
|
"hidden",
|
|
@@ -28,12 +18,7 @@ const STATE_ATTRS = /* @__PURE__ */ new Set([
|
|
|
28
18
|
"open",
|
|
29
19
|
"multiple"
|
|
30
20
|
]);
|
|
31
|
-
const BOOLEAN_ATTRS = /* @__PURE__ */ new Set([
|
|
32
|
-
...STATE_ATTRS,
|
|
33
|
-
"async",
|
|
34
|
-
"defer",
|
|
35
|
-
"nomodule"
|
|
36
|
-
]);
|
|
21
|
+
const BOOLEAN_ATTRS = /* @__PURE__ */ new Set([...STATE_ATTRS, "async", "defer", "nomodule"]);
|
|
37
22
|
const EMPTY_VALUE_DROP_ATTRS = /* @__PURE__ */ new Set([
|
|
38
23
|
"alt",
|
|
39
24
|
"autocomplete",
|
|
@@ -60,28 +45,8 @@ const SCRIPT_ATTRS = /* @__PURE__ */ new Set([
|
|
|
60
45
|
"referrerpolicy"
|
|
61
46
|
]);
|
|
62
47
|
const STYLE_TAG_ATTRS = /* @__PURE__ */ new Set(["media", "type", "nonce", "title"]);
|
|
63
|
-
const INTERACTIVE_TAGS =
|
|
64
|
-
|
|
65
|
-
"button",
|
|
66
|
-
"input",
|
|
67
|
-
"select",
|
|
68
|
-
"textarea",
|
|
69
|
-
"form",
|
|
70
|
-
"details",
|
|
71
|
-
"dialog",
|
|
72
|
-
"label"
|
|
73
|
-
]);
|
|
74
|
-
const INTERACTIVE_ROLES = /* @__PURE__ */ new Set([
|
|
75
|
-
"button",
|
|
76
|
-
"link",
|
|
77
|
-
"tab",
|
|
78
|
-
"menuitem",
|
|
79
|
-
"checkbox",
|
|
80
|
-
"radio",
|
|
81
|
-
"switch",
|
|
82
|
-
"slider",
|
|
83
|
-
"combobox"
|
|
84
|
-
]);
|
|
48
|
+
const INTERACTIVE_TAGS = new Set(INTERACTIVE_TAG_NAMES);
|
|
49
|
+
const INTERACTIVE_ROLES = new Set(INTERACTIVE_ROLE_NAMES);
|
|
85
50
|
const OPEN_TAG_PATTERN = /<([a-zA-Z][\w:-]*)(\s(?:[^"'<>/]|"[^"]*"|'[^']*')*)?\s*(\/?)>/g;
|
|
86
51
|
function condenseDom(html) {
|
|
87
52
|
const originalLength = html.length;
|
|
@@ -170,12 +135,8 @@ function condenseDom(html) {
|
|
|
170
135
|
}
|
|
171
136
|
const hasAriaLabel = /aria-label\s*=/i.test(attrs);
|
|
172
137
|
if (!hasAriaLabel) {
|
|
173
|
-
const titleMatch = inner.match(
|
|
174
|
-
|
|
175
|
-
);
|
|
176
|
-
const descMatch = inner.match(
|
|
177
|
-
/<desc[^>]*>([^<]+)<\/desc>/i
|
|
178
|
-
);
|
|
138
|
+
const titleMatch = inner.match(/<title[^>]*>([^<]+)<\/title>/i);
|
|
139
|
+
const descMatch = inner.match(/<desc[^>]*>([^<]+)<\/desc>/i);
|
|
179
140
|
const labelText = titleMatch?.[1]?.trim() || descMatch?.[1]?.trim();
|
|
180
141
|
if (labelText) {
|
|
181
142
|
keepAttrs.push(
|
|
@@ -346,21 +307,6 @@ function normalizeUrlValue(value) {
|
|
|
346
307
|
return `${value.slice(0, 96)}[omitted]`;
|
|
347
308
|
}
|
|
348
309
|
}
|
|
349
|
-
function filterSemanticClasses(value) {
|
|
350
|
-
const classes = value.split(/\s+/).filter(Boolean);
|
|
351
|
-
const kept = classes.filter((cls) => !isObfuscatedClass(cls));
|
|
352
|
-
return kept.join(" ");
|
|
353
|
-
}
|
|
354
|
-
function isObfuscatedClass(cls) {
|
|
355
|
-
if (cls.length > 80) return true;
|
|
356
|
-
if (/^_?[0-9a-f]{6,}$/i.test(cls)) return true;
|
|
357
|
-
if (/^[a-z]+_[0-9a-f]{4,}$/i.test(cls)) return true;
|
|
358
|
-
if (/^[a-z]{1,2}[0-9]{2,}$/i.test(cls)) return true;
|
|
359
|
-
const digits = (cls.match(/[0-9]/g) || []).length;
|
|
360
|
-
const letters = (cls.match(/[a-zA-Z]/g) || []).length;
|
|
361
|
-
if (cls.length >= 6 && digits >= letters * 0.5 && digits >= 2) return true;
|
|
362
|
-
return false;
|
|
363
|
-
}
|
|
364
310
|
function parseAttributes(rawAttrs) {
|
|
365
311
|
const attrs = [];
|
|
366
312
|
const attrPattern = /([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
|
|
@@ -404,7 +350,9 @@ function shouldKeepCustomDataAttribute(tagName, attrName, value, interactive) {
|
|
|
404
350
|
function looksMeaningfulToken(value) {
|
|
405
351
|
if (!/^[a-z][a-z0-9-]{1,40}$/i.test(value)) return false;
|
|
406
352
|
if (!/[a-z]{3}/i.test(value)) return false;
|
|
407
|
-
if (/(track|metric|telemetry|analytics|component|display|loaded|token|dps|color|screen|strict|rehydr|fetch)/i.test(
|
|
353
|
+
if (/(track|metric|telemetry|analytics|component|display|loaded|token|dps|color|screen|strict|rehydr|fetch)/i.test(
|
|
354
|
+
value
|
|
355
|
+
)) {
|
|
408
356
|
return false;
|
|
409
357
|
}
|
|
410
358
|
return true;
|
|
@@ -1,10 +1,2 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Runtime configuration for libretto.
|
|
3
|
-
*
|
|
4
|
-
* Values are derived from environment variables only.
|
|
5
|
-
*/
|
|
6
|
-
declare function isDebugMode(): boolean;
|
|
7
|
-
declare function isDryRun(): boolean;
|
|
8
|
-
declare function shouldPauseBeforeMutation(): boolean;
|
|
9
1
|
|
|
10
|
-
export {
|
|
2
|
+
export { }
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
function isDebugMode() {
|
|
2
|
-
return process.env.LIBRETTO_DEBUG === "true";
|
|
3
|
-
}
|
|
4
|
-
function isDryRun() {
|
|
5
|
-
const explicit = process.env.LIBRETTO_DRY_RUN;
|
|
6
|
-
if (explicit !== void 0) {
|
|
7
|
-
return explicit === "true";
|
|
8
|
-
}
|
|
9
|
-
return process.env.NODE_ENV === "development";
|
|
10
|
-
}
|
|
11
|
-
function shouldPauseBeforeMutation() {
|
|
12
|
-
return isDryRun() && isDebugMode();
|
|
13
|
-
}
|
|
14
|
-
export {
|
|
15
|
-
isDebugMode,
|
|
16
|
-
isDryRun,
|
|
17
|
-
shouldPauseBeforeMutation
|
|
18
|
-
};
|
|
@@ -1 +1,2 @@
|
|
|
1
|
-
|
|
1
|
+
|
|
2
|
+
export { }
|
|
@@ -1,8 +1,14 @@
|
|
|
1
1
|
import { existsSync } from "node:fs";
|
|
2
2
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
3
3
|
import { getSessionDir } from "../../cli/core/context.js";
|
|
4
|
-
import {
|
|
5
|
-
|
|
4
|
+
import {
|
|
5
|
+
getPauseSignalPaths,
|
|
6
|
+
removeSignalIfExists
|
|
7
|
+
} from "../../cli/core/pause-signals.js";
|
|
8
|
+
import {
|
|
9
|
+
listSessionsWithStateFile,
|
|
10
|
+
readSessionState
|
|
11
|
+
} from "../../cli/core/session.js";
|
|
6
12
|
function isPidRunning(pid) {
|
|
7
13
|
try {
|
|
8
14
|
process.kill(pid, 0);
|
|
@@ -14,7 +20,7 @@ function isPidRunning(pid) {
|
|
|
14
20
|
function getRunningSessions() {
|
|
15
21
|
return listSessionsWithStateFile().filter((candidate) => {
|
|
16
22
|
const state = readSessionState(candidate);
|
|
17
|
-
return state !== null && isPidRunning(state.pid);
|
|
23
|
+
return state !== null && state.pid != null && isPidRunning(state.pid);
|
|
18
24
|
});
|
|
19
25
|
}
|
|
20
26
|
function throwMissingSessionError() {
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
declare const TEST_ATTRIBUTE_NAMES: readonly ["data-testid", "data-test", "data-qa", "data-cy"];
|
|
2
|
+
declare const TRUSTED_ATTRIBUTE_NAMES: readonly ["id", "name", "for", "tabindex", "contenteditable", "role", "title", "alt", "type", "value", "placeholder", "autocomplete", "href", "action", "method", "src"];
|
|
3
|
+
declare const INTERACTIVE_TAG_NAMES: readonly ["a", "button", "input", "select", "textarea", "form", "details", "dialog", "label"];
|
|
4
|
+
declare const INTERACTIVE_ROLE_NAMES: readonly ["button", "link", "tab", "menuitem", "checkbox", "radio", "switch", "slider", "combobox"];
|
|
5
|
+
declare function filterSemanticClasses(value: string): string;
|
|
6
|
+
declare function isObfuscatedClass(cls: string): boolean;
|
|
7
|
+
|
|
8
|
+
export { INTERACTIVE_ROLE_NAMES, INTERACTIVE_TAG_NAMES, TEST_ATTRIBUTE_NAMES, TRUSTED_ATTRIBUTE_NAMES, filterSemanticClasses, isObfuscatedClass };
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
const TEST_ATTRIBUTE_NAMES = [
|
|
2
|
+
"data-testid",
|
|
3
|
+
"data-test",
|
|
4
|
+
"data-qa",
|
|
5
|
+
"data-cy"
|
|
6
|
+
];
|
|
7
|
+
const TRUSTED_ATTRIBUTE_NAMES = [
|
|
8
|
+
"id",
|
|
9
|
+
"name",
|
|
10
|
+
"for",
|
|
11
|
+
"tabindex",
|
|
12
|
+
"contenteditable",
|
|
13
|
+
"role",
|
|
14
|
+
"title",
|
|
15
|
+
"alt",
|
|
16
|
+
"type",
|
|
17
|
+
"value",
|
|
18
|
+
"placeholder",
|
|
19
|
+
"autocomplete",
|
|
20
|
+
"href",
|
|
21
|
+
"action",
|
|
22
|
+
"method",
|
|
23
|
+
"src"
|
|
24
|
+
];
|
|
25
|
+
const INTERACTIVE_TAG_NAMES = [
|
|
26
|
+
"a",
|
|
27
|
+
"button",
|
|
28
|
+
"input",
|
|
29
|
+
"select",
|
|
30
|
+
"textarea",
|
|
31
|
+
"form",
|
|
32
|
+
"details",
|
|
33
|
+
"dialog",
|
|
34
|
+
"label"
|
|
35
|
+
];
|
|
36
|
+
const INTERACTIVE_ROLE_NAMES = [
|
|
37
|
+
"button",
|
|
38
|
+
"link",
|
|
39
|
+
"tab",
|
|
40
|
+
"menuitem",
|
|
41
|
+
"checkbox",
|
|
42
|
+
"radio",
|
|
43
|
+
"switch",
|
|
44
|
+
"slider",
|
|
45
|
+
"combobox"
|
|
46
|
+
];
|
|
47
|
+
function filterSemanticClasses(value) {
|
|
48
|
+
const classes = value.split(/\s+/).filter(Boolean);
|
|
49
|
+
const kept = classes.filter((cls) => !isObfuscatedClass(cls));
|
|
50
|
+
return kept.join(" ");
|
|
51
|
+
}
|
|
52
|
+
function isObfuscatedClass(cls) {
|
|
53
|
+
if (cls.length > 80) return true;
|
|
54
|
+
if (/^_?[0-9a-f]{6,}$/i.test(cls)) return true;
|
|
55
|
+
if (/^[a-z]+_[0-9a-f]{4,}$/i.test(cls)) return true;
|
|
56
|
+
if (/^[a-z]{1,2}[0-9]{2,}$/i.test(cls)) return true;
|
|
57
|
+
const digits = (cls.match(/[0-9]/g) || []).length;
|
|
58
|
+
const letters = (cls.match(/[a-zA-Z]/g) || []).length;
|
|
59
|
+
if (cls.length >= 6 && digits >= letters * 0.5 && digits >= 2) return true;
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
export {
|
|
63
|
+
INTERACTIVE_ROLE_NAMES,
|
|
64
|
+
INTERACTIVE_TAG_NAMES,
|
|
65
|
+
TEST_ATTRIBUTE_NAMES,
|
|
66
|
+
TRUSTED_ATTRIBUTE_NAMES,
|
|
67
|
+
filterSemanticClasses,
|
|
68
|
+
isObfuscatedClass
|
|
69
|
+
};
|
|
@@ -25,6 +25,7 @@ const LOCATOR_ACTIONS = [
|
|
|
25
25
|
];
|
|
26
26
|
const NAV_ACTIONS = ["goto", "reload", "goBack", "goForward"];
|
|
27
27
|
const POINTER_ACTIONS = /* @__PURE__ */ new Set(["click", "dblclick", "hover"]);
|
|
28
|
+
const instrumentedTargets = /* @__PURE__ */ new WeakSet();
|
|
28
29
|
const pageQueues = /* @__PURE__ */ new WeakMap();
|
|
29
30
|
function enqueue(page, fn) {
|
|
30
31
|
const prev = pageQueues.get(page) ?? Promise.resolve();
|
|
@@ -96,11 +97,93 @@ function wrapLocatorActions(locator, page, opts) {
|
|
|
96
97
|
};
|
|
97
98
|
}
|
|
98
99
|
}
|
|
100
|
+
const LOCATOR_FACTORY_METHODS = [
|
|
101
|
+
"locator",
|
|
102
|
+
"getByRole",
|
|
103
|
+
"getByText",
|
|
104
|
+
"getByLabel",
|
|
105
|
+
"getByPlaceholder",
|
|
106
|
+
"getByAltText",
|
|
107
|
+
"getByTitle",
|
|
108
|
+
"getByTestId",
|
|
109
|
+
"filter",
|
|
110
|
+
"and",
|
|
111
|
+
"or",
|
|
112
|
+
"first",
|
|
113
|
+
"last",
|
|
114
|
+
"nth"
|
|
115
|
+
];
|
|
116
|
+
const FRAME_LOCATOR_FACTORY_METHODS = [
|
|
117
|
+
"locator",
|
|
118
|
+
"getByRole",
|
|
119
|
+
"getByText",
|
|
120
|
+
"getByLabel",
|
|
121
|
+
"getByPlaceholder",
|
|
122
|
+
"getByAltText",
|
|
123
|
+
"getByTitle",
|
|
124
|
+
"getByTestId",
|
|
125
|
+
"owner",
|
|
126
|
+
"first",
|
|
127
|
+
"last",
|
|
128
|
+
"nth"
|
|
129
|
+
];
|
|
130
|
+
function instrumentLocator(locator, page, opts) {
|
|
131
|
+
const target = locator;
|
|
132
|
+
if (instrumentedTargets.has(target)) {
|
|
133
|
+
return locator;
|
|
134
|
+
}
|
|
135
|
+
instrumentedTargets.add(target);
|
|
136
|
+
wrapLocatorActions(locator, page, opts);
|
|
137
|
+
for (const method of LOCATOR_FACTORY_METHODS) {
|
|
138
|
+
if (typeof locator[method] !== "function") continue;
|
|
139
|
+
const orig = locator[method].bind(locator);
|
|
140
|
+
locator[method] = (...args) => {
|
|
141
|
+
const nextLocator = orig(...args);
|
|
142
|
+
return instrumentLocator(nextLocator, page, opts);
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
if (typeof locator.contentFrame === "function") {
|
|
146
|
+
const origContentFrame = locator.contentFrame.bind(locator);
|
|
147
|
+
locator.contentFrame = (...args) => {
|
|
148
|
+
const frameLocator = origContentFrame(...args);
|
|
149
|
+
return instrumentFrameLocator(frameLocator, page, opts);
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
return locator;
|
|
153
|
+
}
|
|
154
|
+
function instrumentFrameLocator(frameLocator, page, opts) {
|
|
155
|
+
const target = frameLocator;
|
|
156
|
+
if (instrumentedTargets.has(target)) {
|
|
157
|
+
return frameLocator;
|
|
158
|
+
}
|
|
159
|
+
instrumentedTargets.add(target);
|
|
160
|
+
for (const method of FRAME_LOCATOR_FACTORY_METHODS) {
|
|
161
|
+
if (typeof frameLocator[method] !== "function") continue;
|
|
162
|
+
const orig = frameLocator[method].bind(frameLocator);
|
|
163
|
+
frameLocator[method] = (...args) => {
|
|
164
|
+
const result = orig(...args);
|
|
165
|
+
if (method === "first" || method === "last" || method === "nth") {
|
|
166
|
+
return instrumentFrameLocator(result, page, opts);
|
|
167
|
+
}
|
|
168
|
+
return instrumentLocator(result, page, opts);
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
if (typeof frameLocator.frameLocator === "function") {
|
|
172
|
+
const origFrameLocator = frameLocator.frameLocator.bind(
|
|
173
|
+
frameLocator
|
|
174
|
+
);
|
|
175
|
+
frameLocator.frameLocator = (...args) => {
|
|
176
|
+
const nestedFrameLocator = origFrameLocator(...args);
|
|
177
|
+
return instrumentFrameLocator(nestedFrameLocator, page, opts);
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
return frameLocator;
|
|
181
|
+
}
|
|
99
182
|
function isTimeoutError(err) {
|
|
100
183
|
if (!err || typeof err.message !== "string") return false;
|
|
101
184
|
return err.message.includes("Timeout") || err.message.includes("timeout") || err.name === "TimeoutError";
|
|
102
185
|
}
|
|
103
|
-
const
|
|
186
|
+
const PAGE_LOCATOR_FACTORIES = [
|
|
104
187
|
"locator",
|
|
105
188
|
"getByRole",
|
|
106
189
|
"getByText",
|
|
@@ -110,6 +193,7 @@ const LOCATOR_FACTORIES = [
|
|
|
110
193
|
"getByTitle",
|
|
111
194
|
"getByTestId"
|
|
112
195
|
];
|
|
196
|
+
const PAGE_FRAME_LOCATOR_FACTORIES = ["frameLocator"];
|
|
113
197
|
async function installInstrumentation(page, options) {
|
|
114
198
|
if (page.__librettoInstrumented) return;
|
|
115
199
|
page.__librettoInstrumented = true;
|
|
@@ -129,7 +213,12 @@ async function installInstrumentation(page, options) {
|
|
|
129
213
|
try {
|
|
130
214
|
const loc = page.locator(args[0]);
|
|
131
215
|
const box = await loc.boundingBox();
|
|
132
|
-
await visualizeBeforeAction(
|
|
216
|
+
await visualizeBeforeAction(
|
|
217
|
+
page,
|
|
218
|
+
box,
|
|
219
|
+
method,
|
|
220
|
+
highlightBeforeActionMs
|
|
221
|
+
);
|
|
133
222
|
} catch {
|
|
134
223
|
}
|
|
135
224
|
});
|
|
@@ -161,13 +250,20 @@ async function installInstrumentation(page, options) {
|
|
|
161
250
|
return orig(...args);
|
|
162
251
|
};
|
|
163
252
|
}
|
|
164
|
-
for (const factory of
|
|
253
|
+
for (const factory of PAGE_LOCATOR_FACTORIES) {
|
|
165
254
|
if (typeof page[factory] !== "function") continue;
|
|
166
255
|
const origFactory = page[factory].bind(page);
|
|
167
256
|
page[factory] = (...factoryArgs) => {
|
|
168
257
|
const locator = origFactory(...factoryArgs);
|
|
169
|
-
|
|
170
|
-
|
|
258
|
+
return instrumentLocator(locator, page, mergedOpts);
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
for (const factory of PAGE_FRAME_LOCATOR_FACTORIES) {
|
|
262
|
+
if (typeof page[factory] !== "function") continue;
|
|
263
|
+
const origFactory = page[factory].bind(page);
|
|
264
|
+
page[factory] = (...factoryArgs) => {
|
|
265
|
+
const frameLocator = origFactory(...factoryArgs);
|
|
266
|
+
return instrumentFrameLocator(frameLocator, page, mergedOpts);
|
|
171
267
|
};
|
|
172
268
|
}
|
|
173
269
|
}
|
|
@@ -18,7 +18,9 @@ function createLLMClientFromModel(model) {
|
|
|
18
18
|
if (msg.role === "assistant") {
|
|
19
19
|
return {
|
|
20
20
|
role: "assistant",
|
|
21
|
-
content: msg.content.filter(
|
|
21
|
+
content: msg.content.filter(
|
|
22
|
+
(part) => part.type === "text"
|
|
23
|
+
).map((part) => ({ type: "text", text: part.text }))
|
|
22
24
|
};
|
|
23
25
|
}
|
|
24
26
|
return {
|
|
@@ -121,7 +121,9 @@ function convertUserContentParts(parts) {
|
|
|
121
121
|
});
|
|
122
122
|
}
|
|
123
123
|
function convertAssistantContentParts(parts) {
|
|
124
|
-
return parts.filter(
|
|
124
|
+
return parts.filter(
|
|
125
|
+
(part) => part.type === "text"
|
|
126
|
+
).map((part) => ({ type: "text", text: part.text }));
|
|
125
127
|
}
|
|
126
128
|
function convertMessages(messages) {
|
|
127
129
|
return messages.map((msg) => {
|
package/dist/shared/run/api.js
CHANGED
|
@@ -1,8 +1,14 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
chromium
|
|
3
|
+
} from "playwright";
|
|
2
4
|
import { createServer } from "node:net";
|
|
3
5
|
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
4
6
|
import { ensureLibrettoSessionStatePath } from "../paths/paths.js";
|
|
5
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
SESSION_STATE_VERSION,
|
|
9
|
+
SessionStateFileSchema
|
|
10
|
+
} from "../state/session-state.js";
|
|
11
|
+
import { readLibrettoConfig } from "../../cli/core/ai-config.js";
|
|
6
12
|
async function pickFreePort() {
|
|
7
13
|
return await new Promise((resolve, reject) => {
|
|
8
14
|
const server = createServer();
|
|
@@ -18,6 +24,41 @@ async function pickFreePort() {
|
|
|
18
24
|
});
|
|
19
25
|
});
|
|
20
26
|
}
|
|
27
|
+
function resolveWindowPosition() {
|
|
28
|
+
return readLibrettoConfig().windowPosition;
|
|
29
|
+
}
|
|
30
|
+
async function applyWindowPosition(browser, context, page, windowPosition) {
|
|
31
|
+
if (!windowPosition) {
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
const requestedBounds = {
|
|
35
|
+
left: windowPosition.x,
|
|
36
|
+
top: windowPosition.y,
|
|
37
|
+
windowState: "normal"
|
|
38
|
+
};
|
|
39
|
+
const pageCdp = await context.newCDPSession(page);
|
|
40
|
+
let browserCdp;
|
|
41
|
+
try {
|
|
42
|
+
const targetInfo = await pageCdp.send("Target.getTargetInfo");
|
|
43
|
+
const targetId = targetInfo.targetInfo?.targetId;
|
|
44
|
+
browserCdp = await browser.newBrowserCDPSession();
|
|
45
|
+
const windowResult = await browserCdp.send(
|
|
46
|
+
"Browser.getWindowForTarget",
|
|
47
|
+
targetId ? { targetId } : {}
|
|
48
|
+
);
|
|
49
|
+
await browserCdp.send("Browser.setWindowBounds", {
|
|
50
|
+
windowId: windowResult.windowId,
|
|
51
|
+
bounds: requestedBounds
|
|
52
|
+
});
|
|
53
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
54
|
+
} catch {
|
|
55
|
+
} finally {
|
|
56
|
+
await pageCdp.detach().catch(() => {
|
|
57
|
+
});
|
|
58
|
+
await browserCdp?.detach().catch(() => {
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
21
62
|
async function launchBrowser({
|
|
22
63
|
sessionName,
|
|
23
64
|
headless = false,
|
|
@@ -25,12 +66,14 @@ async function launchBrowser({
|
|
|
25
66
|
storageStatePath
|
|
26
67
|
}) {
|
|
27
68
|
const debugPort = await pickFreePort();
|
|
69
|
+
const windowPosition = headless ? void 0 : resolveWindowPosition();
|
|
28
70
|
const browser = await chromium.launch({
|
|
29
71
|
headless,
|
|
30
72
|
args: [
|
|
31
73
|
"--disable-blink-features=AutomationControlled",
|
|
32
74
|
`--remote-debugging-port=${debugPort}`,
|
|
33
|
-
"--no-focus-on-check"
|
|
75
|
+
"--no-focus-on-check",
|
|
76
|
+
...windowPosition ? [`--window-position=${windowPosition.x},${windowPosition.y}`] : []
|
|
34
77
|
]
|
|
35
78
|
});
|
|
36
79
|
const context = await browser.newContext({
|
|
@@ -38,6 +81,7 @@ async function launchBrowser({
|
|
|
38
81
|
...storageStatePath ? { storageState: storageStatePath } : {}
|
|
39
82
|
});
|
|
40
83
|
const page = await context.newPage();
|
|
84
|
+
await applyWindowPosition(browser, context, page, windowPosition);
|
|
41
85
|
page.setDefaultTimeout(3e4);
|
|
42
86
|
page.setDefaultNavigationTimeout(45e3);
|
|
43
87
|
const metadataPath = ensureLibrettoSessionStatePath(sessionName);
|
|
@@ -15,7 +15,8 @@ declare const SessionViewportSchema: z.ZodObject<{
|
|
|
15
15
|
declare const SessionStateFileSchema: z.ZodObject<{
|
|
16
16
|
version: z.ZodLiteral<1>;
|
|
17
17
|
port: z.ZodNumber;
|
|
18
|
-
pid: z.ZodNumber
|
|
18
|
+
pid: z.ZodOptional<z.ZodNumber>;
|
|
19
|
+
cdpEndpoint: z.ZodOptional<z.ZodString>;
|
|
19
20
|
session: z.ZodString;
|
|
20
21
|
startedAt: z.ZodString;
|
|
21
22
|
status: z.ZodOptional<z.ZodEnum<{
|
|
@@ -14,7 +14,8 @@ const SessionViewportSchema = z.object({
|
|
|
14
14
|
const SessionStateFileSchema = z.object({
|
|
15
15
|
version: z.literal(SESSION_STATE_VERSION),
|
|
16
16
|
port: z.number().int().min(0).max(65535),
|
|
17
|
-
pid: z.number().int(),
|
|
17
|
+
pid: z.number().int().optional(),
|
|
18
|
+
cdpEndpoint: z.string().url().optional(),
|
|
18
19
|
session: z.string().min(1),
|
|
19
20
|
startedAt: z.string().datetime({ offset: true }),
|
|
20
21
|
status: SessionStatusSchema.optional(),
|
|
@@ -29,7 +30,9 @@ function formatIssues(error) {
|
|
|
29
30
|
function parseSessionStateData(rawState, source) {
|
|
30
31
|
const parsed = SessionStateFileSchema.safeParse(rawState);
|
|
31
32
|
if (!parsed.success) {
|
|
32
|
-
throw new Error(
|
|
33
|
+
throw new Error(
|
|
34
|
+
`Session state at ${source} is invalid: ${formatIssues(parsed.error)}`
|
|
35
|
+
);
|
|
33
36
|
}
|
|
34
37
|
const { version: _version, ...state } = parsed.data;
|
|
35
38
|
return state;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
const DEFAULTS = {
|
|
2
2
|
style: "minimal",
|
|
3
3
|
color: "rgba(255, 70, 70, 0.9)",
|
|
4
|
-
size:
|
|
4
|
+
size: 23,
|
|
5
5
|
zIndex: 2147483646,
|
|
6
6
|
easing: "cubic-bezier(0.16, 1, 0.3, 1)",
|
|
7
7
|
minDurationMs: 100,
|
|
@@ -16,19 +16,32 @@ function buildCursorSvg(style, color, size) {
|
|
|
16
16
|
if (style === "screenstudio") {
|
|
17
17
|
return `<div style="width:${size * 1.4}px;height:${size * 1.4}px;border-radius:50%;background:${color};box-shadow:0 0 ${size * 0.6}px ${color};opacity:0.7;"></div>`;
|
|
18
18
|
}
|
|
19
|
-
return `<svg width="${size}" height="${size}" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
|
|
19
|
+
return `<svg width="${size}" height="${size}" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg" style="display:block;filter:drop-shadow(0 2px 6px rgba(15,23,42,0.22));">
|
|
20
20
|
<path d="M5 3L19 12L12 13L9 20L5 3Z" fill="${color}" stroke="rgba(0,0,0,0.3)" stroke-width="1"/>
|
|
21
21
|
</svg>`;
|
|
22
22
|
}
|
|
23
|
+
function buildCursorMarkup(style, color, size) {
|
|
24
|
+
const cursor = buildCursorSvg(style, color, size);
|
|
25
|
+
const badgeHeight = Math.max(12, Math.round(size * 0.54));
|
|
26
|
+
const fontSize = Math.max(8, Math.round(size * 0.28));
|
|
27
|
+
const minWidth = Math.max(28, Math.round(size * 1.28));
|
|
28
|
+
const paddingX = Math.max(5, Math.round(size * 0.2));
|
|
29
|
+
const left = Math.round(size * 0.84);
|
|
30
|
+
const top = Math.round(size * 0.74);
|
|
31
|
+
const width = Math.round(size * 2.4);
|
|
32
|
+
const height = Math.round(size * 1.95);
|
|
33
|
+
const badge = `<div aria-hidden="true" style="position:absolute;left:${left}px;top:${top}px;display:flex;align-items:center;justify-content:center;min-width:${minWidth}px;height:${badgeHeight}px;padding:0 ${paddingX}px;border-radius:${badgeHeight}px;background:${color};color:rgba(255,255,255,0.96);font:700 ${fontSize}px/1 ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,'Segoe UI',sans-serif;letter-spacing:0.02em;white-space:nowrap;border:1px solid rgba(0,0,0,0.16);box-shadow:0 4px 12px rgba(0,0,0,0.14);transform-origin:left center;">Agent</div>`;
|
|
34
|
+
return `<div style="position:relative;width:${width}px;height:${height}px;overflow:visible;">${cursor}${badge}</div>`;
|
|
35
|
+
}
|
|
23
36
|
function buildInitScript(opts) {
|
|
24
|
-
const
|
|
37
|
+
const markup = buildCursorMarkup(opts.style, opts.color, opts.size);
|
|
25
38
|
return `
|
|
26
39
|
(function() {
|
|
27
40
|
if (document.getElementById("${CURSOR_ID}")) return;
|
|
28
41
|
var el = document.createElement("div");
|
|
29
42
|
el.id = "${CURSOR_ID}";
|
|
30
43
|
el.style.cssText = "position:fixed;top:0;left:0;z-index:${opts.zIndex};pointer-events:none;transform:translate3d(-100px,-100px,0);transition:none;will-change:transform,opacity;opacity:0;";
|
|
31
|
-
el.innerHTML = ${JSON.stringify(
|
|
44
|
+
el.innerHTML = ${JSON.stringify(markup)};
|
|
32
45
|
document.documentElement.appendChild(el);
|
|
33
46
|
})();
|
|
34
47
|
`;
|
|
@@ -64,7 +77,13 @@ async function moveGhostCursor(page, target) {
|
|
|
64
77
|
el.style.transition = `transform ${duration}ms ${easing}`;
|
|
65
78
|
el.style.transform = `translate3d(${x}px, ${y}px, 0)`;
|
|
66
79
|
},
|
|
67
|
-
{
|
|
80
|
+
{
|
|
81
|
+
id: CURSOR_ID,
|
|
82
|
+
x: target.x,
|
|
83
|
+
y: target.y,
|
|
84
|
+
duration: durationMs,
|
|
85
|
+
easing: opts.easing
|
|
86
|
+
}
|
|
68
87
|
);
|
|
69
88
|
await page.waitForTimeout(durationMs);
|
|
70
89
|
} catch {
|
|
@@ -122,15 +141,18 @@ async function hideGhostCursor(page) {
|
|
|
122
141
|
}
|
|
123
142
|
async function getGhostCursorPosition(page) {
|
|
124
143
|
try {
|
|
125
|
-
return await page.evaluate(
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
144
|
+
return await page.evaluate(
|
|
145
|
+
({ id }) => {
|
|
146
|
+
const el = document.getElementById(id);
|
|
147
|
+
if (!el) return null;
|
|
148
|
+
const match = el.style.transform.match(
|
|
149
|
+
/translate3d\(\s*([\d.-]+)px\s*,\s*([\d.-]+)px/
|
|
150
|
+
);
|
|
151
|
+
if (!match) return null;
|
|
152
|
+
return { x: parseFloat(match[1]), y: parseFloat(match[2]) };
|
|
153
|
+
},
|
|
154
|
+
{ id: CURSOR_ID }
|
|
155
|
+
);
|
|
134
156
|
} catch {
|
|
135
157
|
return null;
|
|
136
158
|
}
|