libretto 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +128 -126
- package/dist/cli/cli.js +2 -0
- package/dist/cli/commands/browser.js +4 -1
- package/dist/cli/commands/execution.js +21 -6
- package/dist/cli/commands/logs.js +36 -8
- package/dist/cli/commands/snapshot.js +14 -7
- package/dist/cli/core/browser.js +89 -253
- package/dist/cli/core/session-telemetry.js +491 -0
- package/dist/cli/core/telemetry.js +18 -6
- package/dist/cli/workers/run-integration-runtime.js +19 -1
- package/dist/index.cjs +2 -6
- package/dist/index.d.cts +2 -5
- package/dist/index.d.ts +2 -5
- package/dist/index.js +2 -5
- package/dist/runtime/download/download.d.cts +2 -2
- package/dist/runtime/download/download.d.ts +2 -2
- package/dist/runtime/extract/extract.cjs +2 -1
- package/dist/runtime/extract/extract.d.cts +5 -5
- package/dist/runtime/extract/extract.d.ts +5 -5
- package/dist/runtime/extract/extract.js +2 -1
- package/dist/runtime/network/network.d.cts +6 -6
- package/dist/runtime/network/network.d.ts +6 -6
- package/dist/runtime/recovery/agent.cjs +12 -7
- package/dist/runtime/recovery/agent.d.cts +2 -2
- package/dist/runtime/recovery/agent.d.ts +2 -2
- package/dist/runtime/recovery/agent.js +12 -7
- package/dist/runtime/recovery/errors.cjs +8 -6
- package/dist/runtime/recovery/errors.d.cts +2 -2
- package/dist/runtime/recovery/errors.d.ts +2 -2
- package/dist/runtime/recovery/errors.js +8 -6
- package/dist/runtime/recovery/recovery.cjs +5 -3
- package/dist/runtime/recovery/recovery.d.cts +2 -2
- package/dist/runtime/recovery/recovery.d.ts +2 -2
- package/dist/runtime/recovery/recovery.js +5 -3
- package/dist/shared/instrumentation/instrument.d.cts +2 -2
- package/dist/shared/instrumentation/instrument.d.ts +2 -2
- package/dist/shared/llm/types.d.cts +5 -5
- package/dist/shared/llm/types.d.ts +5 -5
- package/dist/shared/logger/index.cjs +2 -0
- package/dist/shared/logger/index.d.cts +1 -1
- package/dist/shared/logger/index.d.ts +1 -1
- package/dist/shared/logger/index.js +2 -1
- package/dist/shared/logger/logger.cjs +15 -2
- package/dist/shared/logger/logger.d.cts +13 -1
- package/dist/shared/logger/logger.d.ts +13 -1
- package/dist/shared/logger/logger.js +13 -1
- package/dist/shared/state/session-state.d.cts +2 -2
- package/dist/shared/state/session-state.d.ts +2 -2
- package/package.json +15 -11
- package/scripts/postinstall.mjs +48 -0
- package/skill/SKILL.md +438 -0
- package/skill/code-generation-rules.md +190 -0
- package/skill/integration-approach-selection.md +174 -0
- package/dist/runtime/step/index.cjs +0 -31
- package/dist/runtime/step/index.d.cts +0 -7
- package/dist/runtime/step/index.d.ts +0 -7
- package/dist/runtime/step/index.js +0 -6
- package/dist/runtime/step/runner.cjs +0 -208
- package/dist/runtime/step/runner.d.cts +0 -16
- package/dist/runtime/step/runner.d.ts +0 -16
- package/dist/runtime/step/runner.js +0 -187
- package/dist/runtime/step/step.cjs +0 -67
- package/dist/runtime/step/step.d.cts +0 -23
- package/dist/runtime/step/step.d.ts +0 -23
- package/dist/runtime/step/step.js +0 -43
- package/dist/runtime/step/types.cjs +0 -16
- package/dist/runtime/step/types.d.cts +0 -72
- package/dist/runtime/step/types.d.ts +0 -72
- package/dist/runtime/step/types.js +0 -0
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,4 @@
|
|
|
1
|
-
export {
|
|
2
|
-
export { Runner, createRunner } from './runtime/step/runner.js';
|
|
3
|
-
export { DebugBundle, RecoveryHandler, RunnerConfig, Step, StepContext, StepHandler, StepHistoryEntry, StepOptions } from './runtime/step/types.js';
|
|
4
|
-
export { LogOptions, Logger, LoggerApi, LoggerSink } from './shared/logger/logger.js';
|
|
1
|
+
export { LogOptions, Logger, LoggerApi, LoggerSink, MinimalLogger, defaultLogger } from './shared/logger/logger.js';
|
|
5
2
|
export { createFileLogSink, jsonlConsoleSink, prettyConsoleSink } from './shared/logger/sinks.js';
|
|
6
3
|
export { LLMClient, Message, MessageContentPart } from './shared/llm/types.js';
|
|
7
4
|
export { SESSION_STATE_VERSION, SessionState, SessionStateFile, SessionStateFileSchema, SessionStatus, SessionStatusSchema, parseSessionStateContent, parseSessionStateData, serializeSessionState } from './shared/state/session-state.js';
|
|
@@ -18,5 +15,5 @@ export { GhostCursorOptions, ensureGhostCursor, ghostClick, hideGhostCursor, mov
|
|
|
18
15
|
export { HighlightOptions, clearHighlights, ensureHighlightLayer, showHighlight } from './shared/visualization/highlight.js';
|
|
19
16
|
export { BrowserSession, LaunchBrowserArgs, launchBrowser } from './shared/run/browser.js';
|
|
20
17
|
export { LIBRETTO_WORKFLOW_BRAND, LibrettoAuthProfile, LibrettoWorkflow, LibrettoWorkflowContext, LibrettoWorkflowHandler, LibrettoWorkflowMetadata, workflow } from './shared/workflow/workflow.js';
|
|
21
|
-
import 'playwright';
|
|
22
18
|
import 'zod';
|
|
19
|
+
import 'playwright';
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { createRunner } from "./runtime/step/runner.js";
|
|
3
|
-
import { Logger } from "./shared/logger/logger.js";
|
|
1
|
+
import { Logger, defaultLogger } from "./shared/logger/logger.js";
|
|
4
2
|
import {
|
|
5
3
|
createFileLogSink,
|
|
6
4
|
prettyConsoleSink,
|
|
@@ -76,8 +74,8 @@ export {
|
|
|
76
74
|
attemptWithRecovery,
|
|
77
75
|
clearHighlights,
|
|
78
76
|
createFileLogSink,
|
|
79
|
-
createRunner,
|
|
80
77
|
debugPause,
|
|
78
|
+
defaultLogger,
|
|
81
79
|
detectSubmissionError,
|
|
82
80
|
downloadAndSave,
|
|
83
81
|
downloadViaClick,
|
|
@@ -105,6 +103,5 @@ export {
|
|
|
105
103
|
serializeSessionState,
|
|
106
104
|
shouldPauseBeforeMutation,
|
|
107
105
|
showHighlight,
|
|
108
|
-
step,
|
|
109
106
|
workflow
|
|
110
107
|
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
|
-
import {
|
|
2
|
+
import { MinimalLogger } from '../../shared/logger/logger.cjs';
|
|
3
3
|
|
|
4
4
|
type DownloadResult = {
|
|
5
5
|
/** The raw file contents. */
|
|
@@ -8,7 +8,7 @@ type DownloadResult = {
|
|
|
8
8
|
filename: string;
|
|
9
9
|
};
|
|
10
10
|
type DownloadViaClickOptions = {
|
|
11
|
-
logger?:
|
|
11
|
+
logger?: MinimalLogger;
|
|
12
12
|
/** Timeout in milliseconds for waiting on the download event. Defaults to 30 000. */
|
|
13
13
|
timeout?: number;
|
|
14
14
|
};
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
|
-
import {
|
|
2
|
+
import { MinimalLogger } from '../../shared/logger/logger.js';
|
|
3
3
|
|
|
4
4
|
type DownloadResult = {
|
|
5
5
|
/** The raw file contents. */
|
|
@@ -8,7 +8,7 @@ type DownloadResult = {
|
|
|
8
8
|
filename: string;
|
|
9
9
|
};
|
|
10
10
|
type DownloadViaClickOptions = {
|
|
11
|
-
logger?:
|
|
11
|
+
logger?: MinimalLogger;
|
|
12
12
|
/** Timeout in milliseconds for waiting on the download event. Defaults to 30 000. */
|
|
13
13
|
timeout?: number;
|
|
14
14
|
};
|
|
@@ -21,8 +21,9 @@ __export(extract_exports, {
|
|
|
21
21
|
extractFromPage: () => extractFromPage
|
|
22
22
|
});
|
|
23
23
|
module.exports = __toCommonJS(extract_exports);
|
|
24
|
+
var import_logger = require("../../shared/logger/logger.js");
|
|
24
25
|
async function extractFromPage(options) {
|
|
25
|
-
const { page, instruction, schema, selector, logger, llmClient } = options;
|
|
26
|
+
const { page, instruction, schema, selector, logger = import_logger.defaultLogger, llmClient } = options;
|
|
26
27
|
let screenshot;
|
|
27
28
|
let domContent;
|
|
28
29
|
if (selector) {
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
2
|
+
import z from 'zod';
|
|
3
|
+
import { MinimalLogger } from '../../shared/logger/logger.cjs';
|
|
4
4
|
import { LLMClient } from '../../shared/llm/types.cjs';
|
|
5
5
|
|
|
6
|
-
type ExtractOptions<T extends ZodType> = {
|
|
6
|
+
type ExtractOptions<T extends z.ZodType> = {
|
|
7
7
|
page: Page;
|
|
8
8
|
instruction: string;
|
|
9
9
|
schema: T;
|
|
10
10
|
llmClient: LLMClient;
|
|
11
|
-
logger
|
|
11
|
+
logger?: MinimalLogger;
|
|
12
12
|
/** Optional CSS selector to scope extraction to a specific element. */
|
|
13
13
|
selector?: string;
|
|
14
14
|
};
|
|
@@ -18,6 +18,6 @@ type ExtractOptions<T extends ZodType> = {
|
|
|
18
18
|
* captures DOM content, and uses an LLM to extract structured data
|
|
19
19
|
* matching the provided Zod schema.
|
|
20
20
|
*/
|
|
21
|
-
declare function extractFromPage<T extends ZodType>(options: ExtractOptions<T>): Promise<infer<T>>;
|
|
21
|
+
declare function extractFromPage<T extends z.ZodType>(options: ExtractOptions<T>): Promise<z.infer<T>>;
|
|
22
22
|
|
|
23
23
|
export { type ExtractOptions, extractFromPage };
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
2
|
+
import z from 'zod';
|
|
3
|
+
import { MinimalLogger } from '../../shared/logger/logger.js';
|
|
4
4
|
import { LLMClient } from '../../shared/llm/types.js';
|
|
5
5
|
|
|
6
|
-
type ExtractOptions<T extends ZodType> = {
|
|
6
|
+
type ExtractOptions<T extends z.ZodType> = {
|
|
7
7
|
page: Page;
|
|
8
8
|
instruction: string;
|
|
9
9
|
schema: T;
|
|
10
10
|
llmClient: LLMClient;
|
|
11
|
-
logger
|
|
11
|
+
logger?: MinimalLogger;
|
|
12
12
|
/** Optional CSS selector to scope extraction to a specific element. */
|
|
13
13
|
selector?: string;
|
|
14
14
|
};
|
|
@@ -18,6 +18,6 @@ type ExtractOptions<T extends ZodType> = {
|
|
|
18
18
|
* captures DOM content, and uses an LLM to extract structured data
|
|
19
19
|
* matching the provided Zod schema.
|
|
20
20
|
*/
|
|
21
|
-
declare function extractFromPage<T extends ZodType>(options: ExtractOptions<T>): Promise<infer<T>>;
|
|
21
|
+
declare function extractFromPage<T extends z.ZodType>(options: ExtractOptions<T>): Promise<z.infer<T>>;
|
|
22
22
|
|
|
23
23
|
export { type ExtractOptions, extractFromPage };
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { defaultLogger } from "../../shared/logger/logger.js";
|
|
1
2
|
async function extractFromPage(options) {
|
|
2
|
-
const { page, instruction, schema, selector, logger, llmClient } = options;
|
|
3
|
+
const { page, instruction, schema, selector, logger = defaultLogger, llmClient } = options;
|
|
3
4
|
let screenshot;
|
|
4
5
|
let domContent;
|
|
5
6
|
if (selector) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
2
|
+
import z from 'zod';
|
|
3
|
+
import { MinimalLogger } from '../../shared/logger/logger.cjs';
|
|
4
4
|
|
|
5
5
|
type RequestConfig = {
|
|
6
6
|
url: string;
|
|
@@ -12,17 +12,17 @@ type RequestConfig = {
|
|
|
12
12
|
/** How to parse the response. Defaults to "json". */
|
|
13
13
|
responseType?: "json" | "text" | "xml";
|
|
14
14
|
};
|
|
15
|
-
type PageRequestOptions<T extends ZodType | undefined = undefined> = {
|
|
16
|
-
logger?:
|
|
15
|
+
type PageRequestOptions<T extends z.ZodType | undefined = undefined> = {
|
|
16
|
+
logger?: MinimalLogger;
|
|
17
17
|
/** Optional Zod schema to validate the response body. */
|
|
18
18
|
schema?: T;
|
|
19
19
|
};
|
|
20
|
-
type PageRequestResult<T extends ZodType | undefined> = T extends ZodType ? infer<T> : any;
|
|
20
|
+
type PageRequestResult<T extends z.ZodType | undefined> = T extends z.ZodType ? z.infer<T> : any;
|
|
21
21
|
/**
|
|
22
22
|
* Executes a fetch() call inside the browser context via page.evaluate().
|
|
23
23
|
* Provides typed request config, automatic response parsing, optional Zod
|
|
24
24
|
* validation, and logging.
|
|
25
25
|
*/
|
|
26
|
-
declare function pageRequest<T extends ZodType | undefined = undefined>(page: Page, config: RequestConfig, options?: PageRequestOptions<T>): Promise<PageRequestResult<T>>;
|
|
26
|
+
declare function pageRequest<T extends z.ZodType | undefined = undefined>(page: Page, config: RequestConfig, options?: PageRequestOptions<T>): Promise<PageRequestResult<T>>;
|
|
27
27
|
|
|
28
28
|
export { type PageRequestOptions, type RequestConfig, pageRequest };
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
|
-
import
|
|
3
|
-
import {
|
|
2
|
+
import z from 'zod';
|
|
3
|
+
import { MinimalLogger } from '../../shared/logger/logger.js';
|
|
4
4
|
|
|
5
5
|
type RequestConfig = {
|
|
6
6
|
url: string;
|
|
@@ -12,17 +12,17 @@ type RequestConfig = {
|
|
|
12
12
|
/** How to parse the response. Defaults to "json". */
|
|
13
13
|
responseType?: "json" | "text" | "xml";
|
|
14
14
|
};
|
|
15
|
-
type PageRequestOptions<T extends ZodType | undefined = undefined> = {
|
|
16
|
-
logger?:
|
|
15
|
+
type PageRequestOptions<T extends z.ZodType | undefined = undefined> = {
|
|
16
|
+
logger?: MinimalLogger;
|
|
17
17
|
/** Optional Zod schema to validate the response body. */
|
|
18
18
|
schema?: T;
|
|
19
19
|
};
|
|
20
|
-
type PageRequestResult<T extends ZodType | undefined> = T extends ZodType ? infer<T> : any;
|
|
20
|
+
type PageRequestResult<T extends z.ZodType | undefined> = T extends z.ZodType ? z.infer<T> : any;
|
|
21
21
|
/**
|
|
22
22
|
* Executes a fetch() call inside the browser context via page.evaluate().
|
|
23
23
|
* Provides typed request config, automatic response parsing, optional Zod
|
|
24
24
|
* validation, and logging.
|
|
25
25
|
*/
|
|
26
|
-
declare function pageRequest<T extends ZodType | undefined = undefined>(page: Page, config: RequestConfig, options?: PageRequestOptions<T>): Promise<PageRequestResult<T>>;
|
|
26
|
+
declare function pageRequest<T extends z.ZodType | undefined = undefined>(page: Page, config: RequestConfig, options?: PageRequestOptions<T>): Promise<PageRequestResult<T>>;
|
|
27
27
|
|
|
28
28
|
export { type PageRequestOptions, type RequestConfig, pageRequest };
|
|
@@ -21,6 +21,7 @@ __export(agent_exports, {
|
|
|
21
21
|
executeRecoveryAgent: () => executeRecoveryAgent
|
|
22
22
|
});
|
|
23
23
|
module.exports = __toCommonJS(agent_exports);
|
|
24
|
+
var import_logger = require("../../shared/logger/logger.js");
|
|
24
25
|
var import_zod = require("zod");
|
|
25
26
|
function delay(ms) {
|
|
26
27
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
@@ -53,7 +54,7 @@ const KEY_MAPPINGS = {
|
|
|
53
54
|
function mapKeyName(key) {
|
|
54
55
|
return KEY_MAPPINGS[key.toUpperCase()] ?? key;
|
|
55
56
|
}
|
|
56
|
-
async function executeBrowserAction(page, action, logger) {
|
|
57
|
+
async function executeBrowserAction(page, action, logger = import_logger.defaultLogger) {
|
|
57
58
|
switch (action.type) {
|
|
58
59
|
case "click": {
|
|
59
60
|
const { x, y, button = "left" } = action;
|
|
@@ -156,7 +157,11 @@ const recoveryActionSchema = import_zod.z.object({
|
|
|
156
157
|
])
|
|
157
158
|
});
|
|
158
159
|
async function executeRecoveryAgent(page, instruction, logger, llmClient) {
|
|
159
|
-
|
|
160
|
+
if (!llmClient) {
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
const log = logger ?? import_logger.defaultLogger;
|
|
164
|
+
log.info("Executing vision-based recovery agent", { instruction });
|
|
160
165
|
const viewport = page.viewportSize();
|
|
161
166
|
if (!viewport) {
|
|
162
167
|
throw new Error("Viewport size not found");
|
|
@@ -165,7 +170,7 @@ async function executeRecoveryAgent(page, instruction, logger, llmClient) {
|
|
|
165
170
|
try {
|
|
166
171
|
screenshot = (await page.screenshot({ fullPage: false, timeout: 1e4 })).toString("base64");
|
|
167
172
|
} catch (screenshotError) {
|
|
168
|
-
|
|
173
|
+
log.warn("Failed to take screenshot for recovery agent, skipping", {
|
|
169
174
|
screenshotError: screenshotError instanceof Error ? screenshotError.message : String(screenshotError)
|
|
170
175
|
});
|
|
171
176
|
throw new Error("Failed to take screenshot for recovery agent");
|
|
@@ -196,21 +201,21 @@ Analyze the screenshot and decide what action to take. If the task is complete o
|
|
|
196
201
|
],
|
|
197
202
|
temperature: 0
|
|
198
203
|
});
|
|
199
|
-
|
|
204
|
+
log.info(`Recovery step ${step}/${maxSteps}`, {
|
|
200
205
|
reasoning: result.reasoning,
|
|
201
206
|
action: result.action
|
|
202
207
|
});
|
|
203
208
|
if (result.action.type === "done") {
|
|
204
|
-
|
|
209
|
+
log.info("Recovery agent completed - no more actions needed");
|
|
205
210
|
break;
|
|
206
211
|
}
|
|
207
|
-
await executeBrowserAction(page, result.action,
|
|
212
|
+
await executeBrowserAction(page, result.action, log);
|
|
208
213
|
await delay(2e3);
|
|
209
214
|
screenshot = (await page.screenshot({ fullPage: false })).toString(
|
|
210
215
|
"base64"
|
|
211
216
|
);
|
|
212
217
|
}
|
|
213
|
-
|
|
218
|
+
log.info("Recovery agent execution completed");
|
|
214
219
|
}
|
|
215
220
|
// Annotate the CommonJS export names for ESM import in node:
|
|
216
221
|
0 && (module.exports = {
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
|
-
import {
|
|
2
|
+
import { MinimalLogger } from '../../shared/logger/logger.cjs';
|
|
3
3
|
import { LLMClient } from '../../shared/llm/types.cjs';
|
|
4
4
|
import 'zod';
|
|
5
5
|
|
|
@@ -8,6 +8,6 @@ import 'zod';
|
|
|
8
8
|
* Takes a screenshot, sends it to the LLM with the instruction, and executes
|
|
9
9
|
* the LLM's suggested browser actions.
|
|
10
10
|
*/
|
|
11
|
-
declare function executeRecoveryAgent(page: Page, instruction: string, logger
|
|
11
|
+
declare function executeRecoveryAgent(page: Page, instruction: string, logger?: MinimalLogger, llmClient?: LLMClient): Promise<void>;
|
|
12
12
|
|
|
13
13
|
export { executeRecoveryAgent };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
|
-
import {
|
|
2
|
+
import { MinimalLogger } from '../../shared/logger/logger.js';
|
|
3
3
|
import { LLMClient } from '../../shared/llm/types.js';
|
|
4
4
|
import 'zod';
|
|
5
5
|
|
|
@@ -8,6 +8,6 @@ import 'zod';
|
|
|
8
8
|
* Takes a screenshot, sends it to the LLM with the instruction, and executes
|
|
9
9
|
* the LLM's suggested browser actions.
|
|
10
10
|
*/
|
|
11
|
-
declare function executeRecoveryAgent(page: Page, instruction: string, logger
|
|
11
|
+
declare function executeRecoveryAgent(page: Page, instruction: string, logger?: MinimalLogger, llmClient?: LLMClient): Promise<void>;
|
|
12
12
|
|
|
13
13
|
export { executeRecoveryAgent };
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { defaultLogger } from "../../shared/logger/logger.js";
|
|
1
2
|
function delay(ms) {
|
|
2
3
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
3
4
|
}
|
|
@@ -29,7 +30,7 @@ const KEY_MAPPINGS = {
|
|
|
29
30
|
function mapKeyName(key) {
|
|
30
31
|
return KEY_MAPPINGS[key.toUpperCase()] ?? key;
|
|
31
32
|
}
|
|
32
|
-
async function executeBrowserAction(page, action, logger) {
|
|
33
|
+
async function executeBrowserAction(page, action, logger = defaultLogger) {
|
|
33
34
|
switch (action.type) {
|
|
34
35
|
case "click": {
|
|
35
36
|
const { x, y, button = "left" } = action;
|
|
@@ -133,7 +134,11 @@ const recoveryActionSchema = z.object({
|
|
|
133
134
|
])
|
|
134
135
|
});
|
|
135
136
|
async function executeRecoveryAgent(page, instruction, logger, llmClient) {
|
|
136
|
-
|
|
137
|
+
if (!llmClient) {
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
const log = logger ?? defaultLogger;
|
|
141
|
+
log.info("Executing vision-based recovery agent", { instruction });
|
|
137
142
|
const viewport = page.viewportSize();
|
|
138
143
|
if (!viewport) {
|
|
139
144
|
throw new Error("Viewport size not found");
|
|
@@ -142,7 +147,7 @@ async function executeRecoveryAgent(page, instruction, logger, llmClient) {
|
|
|
142
147
|
try {
|
|
143
148
|
screenshot = (await page.screenshot({ fullPage: false, timeout: 1e4 })).toString("base64");
|
|
144
149
|
} catch (screenshotError) {
|
|
145
|
-
|
|
150
|
+
log.warn("Failed to take screenshot for recovery agent, skipping", {
|
|
146
151
|
screenshotError: screenshotError instanceof Error ? screenshotError.message : String(screenshotError)
|
|
147
152
|
});
|
|
148
153
|
throw new Error("Failed to take screenshot for recovery agent");
|
|
@@ -173,21 +178,21 @@ Analyze the screenshot and decide what action to take. If the task is complete o
|
|
|
173
178
|
],
|
|
174
179
|
temperature: 0
|
|
175
180
|
});
|
|
176
|
-
|
|
181
|
+
log.info(`Recovery step ${step}/${maxSteps}`, {
|
|
177
182
|
reasoning: result.reasoning,
|
|
178
183
|
action: result.action
|
|
179
184
|
});
|
|
180
185
|
if (result.action.type === "done") {
|
|
181
|
-
|
|
186
|
+
log.info("Recovery agent completed - no more actions needed");
|
|
182
187
|
break;
|
|
183
188
|
}
|
|
184
|
-
await executeBrowserAction(page, result.action,
|
|
189
|
+
await executeBrowserAction(page, result.action, log);
|
|
185
190
|
await delay(2e3);
|
|
186
191
|
screenshot = (await page.screenshot({ fullPage: false })).toString(
|
|
187
192
|
"base64"
|
|
188
193
|
);
|
|
189
194
|
}
|
|
190
|
-
|
|
195
|
+
log.info("Recovery agent execution completed");
|
|
191
196
|
}
|
|
192
197
|
export {
|
|
193
198
|
executeRecoveryAgent
|
|
@@ -21,13 +21,15 @@ __export(errors_exports, {
|
|
|
21
21
|
detectSubmissionError: () => detectSubmissionError
|
|
22
22
|
});
|
|
23
23
|
module.exports = __toCommonJS(errors_exports);
|
|
24
|
+
var import_logger = require("../../shared/logger/logger.js");
|
|
24
25
|
var import_zod = require("zod");
|
|
25
26
|
const detectSubmissionErrorSchema = import_zod.z.object({
|
|
26
27
|
hasError: import_zod.z.boolean().describe("Whether an error is visible on the page"),
|
|
27
28
|
matchedKnownErrorId: import_zod.z.string().nullable().describe("The ID of the matched known error, or null if no match"),
|
|
28
29
|
errorMessage: import_zod.z.string().nullable().describe("The error message visible on screen, or null if no error")
|
|
29
30
|
});
|
|
30
|
-
async function detectSubmissionError(page,
|
|
31
|
+
async function detectSubmissionError(page, error, logContext, llmClient, knownErrors = [], logger) {
|
|
32
|
+
const log = logger ?? import_logger.defaultLogger;
|
|
31
33
|
let screenshot;
|
|
32
34
|
let domSnapshot;
|
|
33
35
|
try {
|
|
@@ -38,7 +40,7 @@ async function detectSubmissionError(page, logger, error, logContext, llmClient,
|
|
|
38
40
|
});
|
|
39
41
|
screenshot = data;
|
|
40
42
|
} catch (screenshotError) {
|
|
41
|
-
|
|
43
|
+
log.warn(
|
|
42
44
|
"Failed to take screenshot via CDP for error detection, skipping LLM analysis",
|
|
43
45
|
{ screenshotError, originalError: error }
|
|
44
46
|
);
|
|
@@ -48,7 +50,7 @@ async function detectSubmissionError(page, logger, error, logContext, llmClient,
|
|
|
48
50
|
const htmlContent = await page.content();
|
|
49
51
|
domSnapshot = htmlContent.length > 5e4 ? htmlContent.slice(0, 5e4) + "\n... [truncated]" : htmlContent;
|
|
50
52
|
} catch (domError) {
|
|
51
|
-
|
|
53
|
+
log.warn("Failed to capture DOM snapshot", {
|
|
52
54
|
domError: domError instanceof Error ? domError.message : String(domError)
|
|
53
55
|
});
|
|
54
56
|
}
|
|
@@ -91,14 +93,14 @@ ${domSnapshot}
|
|
|
91
93
|
temperature: 0
|
|
92
94
|
});
|
|
93
95
|
if (!result.hasError) {
|
|
94
|
-
|
|
96
|
+
log.info("No error detected by LLM", { result });
|
|
95
97
|
}
|
|
96
98
|
if (result.matchedKnownErrorId) {
|
|
97
99
|
const knownError = knownErrors.find(
|
|
98
100
|
(e) => e.id === result.matchedKnownErrorId
|
|
99
101
|
);
|
|
100
102
|
if (knownError) {
|
|
101
|
-
|
|
103
|
+
log.warn(logContext, {
|
|
102
104
|
error,
|
|
103
105
|
browserError: result.errorMessage,
|
|
104
106
|
knownErrorId: result.matchedKnownErrorId
|
|
@@ -110,7 +112,7 @@ ${domSnapshot}
|
|
|
110
112
|
};
|
|
111
113
|
}
|
|
112
114
|
}
|
|
113
|
-
|
|
115
|
+
log.warn(logContext, {
|
|
114
116
|
error,
|
|
115
117
|
browserError: result.errorMessage
|
|
116
118
|
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
|
-
import {
|
|
2
|
+
import { MinimalLogger } from '../../shared/logger/logger.cjs';
|
|
3
3
|
import { LLMClient } from '../../shared/llm/types.cjs';
|
|
4
4
|
import 'zod';
|
|
5
5
|
|
|
@@ -26,6 +26,6 @@ type DetectedSubmissionError = {
|
|
|
26
26
|
* @returns DetectedSubmissionError if a known error is matched
|
|
27
27
|
* @throws The original error if no known error matches
|
|
28
28
|
*/
|
|
29
|
-
declare function detectSubmissionError(page: Page,
|
|
29
|
+
declare function detectSubmissionError(page: Page, error: unknown, logContext: string, llmClient: LLMClient, knownErrors?: KnownSubmissionError[], logger?: MinimalLogger): Promise<DetectedSubmissionError>;
|
|
30
30
|
|
|
31
31
|
export { type DetectedSubmissionError, type KnownSubmissionError, detectSubmissionError };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
|
-
import {
|
|
2
|
+
import { MinimalLogger } from '../../shared/logger/logger.js';
|
|
3
3
|
import { LLMClient } from '../../shared/llm/types.js';
|
|
4
4
|
import 'zod';
|
|
5
5
|
|
|
@@ -26,6 +26,6 @@ type DetectedSubmissionError = {
|
|
|
26
26
|
* @returns DetectedSubmissionError if a known error is matched
|
|
27
27
|
* @throws The original error if no known error matches
|
|
28
28
|
*/
|
|
29
|
-
declare function detectSubmissionError(page: Page,
|
|
29
|
+
declare function detectSubmissionError(page: Page, error: unknown, logContext: string, llmClient: LLMClient, knownErrors?: KnownSubmissionError[], logger?: MinimalLogger): Promise<DetectedSubmissionError>;
|
|
30
30
|
|
|
31
31
|
export { type DetectedSubmissionError, type KnownSubmissionError, detectSubmissionError };
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
+
import { defaultLogger } from "../../shared/logger/logger.js";
|
|
1
2
|
import { z } from "zod";
|
|
2
3
|
const detectSubmissionErrorSchema = z.object({
|
|
3
4
|
hasError: z.boolean().describe("Whether an error is visible on the page"),
|
|
4
5
|
matchedKnownErrorId: z.string().nullable().describe("The ID of the matched known error, or null if no match"),
|
|
5
6
|
errorMessage: z.string().nullable().describe("The error message visible on screen, or null if no error")
|
|
6
7
|
});
|
|
7
|
-
async function detectSubmissionError(page,
|
|
8
|
+
async function detectSubmissionError(page, error, logContext, llmClient, knownErrors = [], logger) {
|
|
9
|
+
const log = logger ?? defaultLogger;
|
|
8
10
|
let screenshot;
|
|
9
11
|
let domSnapshot;
|
|
10
12
|
try {
|
|
@@ -15,7 +17,7 @@ async function detectSubmissionError(page, logger, error, logContext, llmClient,
|
|
|
15
17
|
});
|
|
16
18
|
screenshot = data;
|
|
17
19
|
} catch (screenshotError) {
|
|
18
|
-
|
|
20
|
+
log.warn(
|
|
19
21
|
"Failed to take screenshot via CDP for error detection, skipping LLM analysis",
|
|
20
22
|
{ screenshotError, originalError: error }
|
|
21
23
|
);
|
|
@@ -25,7 +27,7 @@ async function detectSubmissionError(page, logger, error, logContext, llmClient,
|
|
|
25
27
|
const htmlContent = await page.content();
|
|
26
28
|
domSnapshot = htmlContent.length > 5e4 ? htmlContent.slice(0, 5e4) + "\n... [truncated]" : htmlContent;
|
|
27
29
|
} catch (domError) {
|
|
28
|
-
|
|
30
|
+
log.warn("Failed to capture DOM snapshot", {
|
|
29
31
|
domError: domError instanceof Error ? domError.message : String(domError)
|
|
30
32
|
});
|
|
31
33
|
}
|
|
@@ -68,14 +70,14 @@ ${domSnapshot}
|
|
|
68
70
|
temperature: 0
|
|
69
71
|
});
|
|
70
72
|
if (!result.hasError) {
|
|
71
|
-
|
|
73
|
+
log.info("No error detected by LLM", { result });
|
|
72
74
|
}
|
|
73
75
|
if (result.matchedKnownErrorId) {
|
|
74
76
|
const knownError = knownErrors.find(
|
|
75
77
|
(e) => e.id === result.matchedKnownErrorId
|
|
76
78
|
);
|
|
77
79
|
if (knownError) {
|
|
78
|
-
|
|
80
|
+
log.warn(logContext, {
|
|
79
81
|
error,
|
|
80
82
|
browserError: result.errorMessage,
|
|
81
83
|
knownErrorId: result.matchedKnownErrorId
|
|
@@ -87,7 +89,7 @@ ${domSnapshot}
|
|
|
87
89
|
};
|
|
88
90
|
}
|
|
89
91
|
}
|
|
90
|
-
|
|
92
|
+
log.warn(logContext, {
|
|
91
93
|
error,
|
|
92
94
|
browserError: result.errorMessage
|
|
93
95
|
});
|
|
@@ -21,13 +21,15 @@ __export(recovery_exports, {
|
|
|
21
21
|
attemptWithRecovery: () => attemptWithRecovery
|
|
22
22
|
});
|
|
23
23
|
module.exports = __toCommonJS(recovery_exports);
|
|
24
|
+
var import_logger = require("../../shared/logger/logger.js");
|
|
24
25
|
var import_agent = require("./agent.js");
|
|
25
26
|
async function attemptWithRecovery(page, fn, logger, llmClient) {
|
|
27
|
+
const log = logger ?? import_logger.defaultLogger;
|
|
26
28
|
try {
|
|
27
29
|
return await fn();
|
|
28
30
|
} catch (error) {
|
|
29
31
|
if (error instanceof Error && (error.message.includes("Target closed") || error.message.includes("browser has been closed") || error.message.includes("context or browser has been closed"))) {
|
|
30
|
-
|
|
32
|
+
log.warn("Page/browser has been closed, cannot recover", {
|
|
31
33
|
error: error.message
|
|
32
34
|
});
|
|
33
35
|
throw error;
|
|
@@ -35,13 +37,13 @@ async function attemptWithRecovery(page, fn, logger, llmClient) {
|
|
|
35
37
|
if (!llmClient) {
|
|
36
38
|
throw error;
|
|
37
39
|
}
|
|
38
|
-
|
|
40
|
+
log.info("Action failed, attempting popup recovery", {
|
|
39
41
|
error: error instanceof Error ? error.message : String(error)
|
|
40
42
|
});
|
|
41
43
|
await (0, import_agent.executeRecoveryAgent)(
|
|
42
44
|
page,
|
|
43
45
|
"Look at the page to see if there is a popup blocking the screen. If so, close the popup.",
|
|
44
|
-
|
|
46
|
+
log,
|
|
45
47
|
llmClient
|
|
46
48
|
);
|
|
47
49
|
return await fn();
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
|
-
import {
|
|
2
|
+
import { MinimalLogger } from '../../shared/logger/logger.cjs';
|
|
3
3
|
import { LLMClient } from '../../shared/llm/types.cjs';
|
|
4
4
|
import 'zod';
|
|
5
5
|
|
|
@@ -7,6 +7,6 @@ import 'zod';
|
|
|
7
7
|
* Attempts to execute a function, and if it fails, runs popup recovery
|
|
8
8
|
* (if an LLM client is provided) and retries the function once.
|
|
9
9
|
*/
|
|
10
|
-
declare function attemptWithRecovery<T>(page: Page, fn: () => Promise<T>, logger
|
|
10
|
+
declare function attemptWithRecovery<T>(page: Page, fn: () => Promise<T>, logger?: MinimalLogger, llmClient?: LLMClient): Promise<T>;
|
|
11
11
|
|
|
12
12
|
export { attemptWithRecovery };
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
|
-
import {
|
|
2
|
+
import { MinimalLogger } from '../../shared/logger/logger.js';
|
|
3
3
|
import { LLMClient } from '../../shared/llm/types.js';
|
|
4
4
|
import 'zod';
|
|
5
5
|
|
|
@@ -7,6 +7,6 @@ import 'zod';
|
|
|
7
7
|
* Attempts to execute a function, and if it fails, runs popup recovery
|
|
8
8
|
* (if an LLM client is provided) and retries the function once.
|
|
9
9
|
*/
|
|
10
|
-
declare function attemptWithRecovery<T>(page: Page, fn: () => Promise<T>, logger
|
|
10
|
+
declare function attemptWithRecovery<T>(page: Page, fn: () => Promise<T>, logger?: MinimalLogger, llmClient?: LLMClient): Promise<T>;
|
|
11
11
|
|
|
12
12
|
export { attemptWithRecovery };
|
|
@@ -1,10 +1,12 @@
|
|
|
1
|
+
import { defaultLogger } from "../../shared/logger/logger.js";
|
|
1
2
|
import { executeRecoveryAgent } from "./agent.js";
|
|
2
3
|
async function attemptWithRecovery(page, fn, logger, llmClient) {
|
|
4
|
+
const log = logger ?? defaultLogger;
|
|
3
5
|
try {
|
|
4
6
|
return await fn();
|
|
5
7
|
} catch (error) {
|
|
6
8
|
if (error instanceof Error && (error.message.includes("Target closed") || error.message.includes("browser has been closed") || error.message.includes("context or browser has been closed"))) {
|
|
7
|
-
|
|
9
|
+
log.warn("Page/browser has been closed, cannot recover", {
|
|
8
10
|
error: error.message
|
|
9
11
|
});
|
|
10
12
|
throw error;
|
|
@@ -12,13 +14,13 @@ async function attemptWithRecovery(page, fn, logger, llmClient) {
|
|
|
12
14
|
if (!llmClient) {
|
|
13
15
|
throw error;
|
|
14
16
|
}
|
|
15
|
-
|
|
17
|
+
log.info("Action failed, attempting popup recovery", {
|
|
16
18
|
error: error instanceof Error ? error.message : String(error)
|
|
17
19
|
});
|
|
18
20
|
await executeRecoveryAgent(
|
|
19
21
|
page,
|
|
20
22
|
"Look at the page to see if there is a popup blocking the screen. If so, close the popup.",
|
|
21
|
-
|
|
23
|
+
log,
|
|
22
24
|
llmClient
|
|
23
25
|
);
|
|
24
26
|
return await fn();
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { Page, BrowserContext } from 'playwright';
|
|
2
|
-
import {
|
|
2
|
+
import { MinimalLogger } from '../logger/logger.cjs';
|
|
3
3
|
import { GhostCursorOptions } from '../visualization/ghost-cursor.cjs';
|
|
4
4
|
import { HighlightOptions } from '../visualization/highlight.cjs';
|
|
5
5
|
|
|
6
6
|
type InstrumentationOptions = {
|
|
7
7
|
visualize?: boolean;
|
|
8
|
-
logger?:
|
|
8
|
+
logger?: MinimalLogger;
|
|
9
9
|
highlightBeforeActionMs?: number;
|
|
10
10
|
ghostCursor?: GhostCursorOptions;
|
|
11
11
|
highlight?: HighlightOptions;
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { Page, BrowserContext } from 'playwright';
|
|
2
|
-
import {
|
|
2
|
+
import { MinimalLogger } from '../logger/logger.js';
|
|
3
3
|
import { GhostCursorOptions } from '../visualization/ghost-cursor.js';
|
|
4
4
|
import { HighlightOptions } from '../visualization/highlight.js';
|
|
5
5
|
|
|
6
6
|
type InstrumentationOptions = {
|
|
7
7
|
visualize?: boolean;
|
|
8
|
-
logger?:
|
|
8
|
+
logger?: MinimalLogger;
|
|
9
9
|
highlightBeforeActionMs?: number;
|
|
10
10
|
ghostCursor?: GhostCursorOptions;
|
|
11
11
|
highlight?: HighlightOptions;
|