libretto 0.6.21 → 0.6.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/README.template.md +5 -1
- package/dist/cli/commands/execution.js +8 -1
- package/dist/cli/core/browser.js +8 -3
- package/dist/cli/core/daemon/daemon.js +8 -6
- package/dist/cli/core/providers/kernel.js +107 -29
- package/dist/cli/core/providers/steel.js +10 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.js +15 -1
- package/dist/runtime/recovery/agent.d.ts +50 -2
- package/dist/runtime/recovery/agent.js +159 -45
- package/dist/runtime/recovery/index.d.ts +2 -1
- package/dist/runtime/recovery/index.js +16 -2
- package/dist/runtime/recovery/page-fallbacks.d.ts +45 -0
- package/dist/runtime/recovery/page-fallbacks.js +389 -0
- package/dist/shared/state/index.d.ts +1 -1
- package/dist/shared/state/session-state.d.ts +4 -1
- package/dist/shared/state/session-state.js +2 -1
- package/dist/shared/workflow/workflow.d.ts +19 -6
- package/dist/shared/workflow/workflow.js +38 -9
- package/docs/reference/runtime/page-fallbacks.mdx +85 -0
- package/docs/understand-libretto/error-handling-and-recovery.mdx +45 -0
- package/package.json +4 -12
- package/skills/libretto/SKILL.md +8 -2
- package/skills/libretto/references/code-generation-rules.md +23 -6
- package/skills/libretto-readonly/SKILL.md +1 -1
- package/src/cli/commands/execution.ts +8 -1
- package/src/cli/core/browser.ts +7 -2
- package/src/cli/core/daemon/daemon.ts +9 -4
- package/src/cli/core/daemon/ipc.ts +1 -0
- package/src/cli/core/providers/kernel.ts +153 -29
- package/src/cli/core/providers/steel.ts +11 -1
- package/src/cli/core/providers/types.ts +3 -0
- package/src/index.ts +22 -2
- package/src/runtime/recovery/agent.ts +227 -50
- package/src/runtime/recovery/index.ts +21 -1
- package/src/runtime/recovery/page-fallbacks.ts +527 -0
- package/src/shared/state/index.ts +1 -0
- package/src/shared/state/session-state.ts +2 -0
- package/src/shared/workflow/workflow.ts +90 -20
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
import { executeRecoveryAgent } from "./agent.js";
|
|
2
|
+
import { defaultLogger } from "../../shared/logger/logger.js";
|
|
3
|
+
const POPUP_RECOVERY_INSTRUCTION = [
|
|
4
|
+
"Look at the page for any popup, modal, cookie banner, overlay, dialog, or interstitial that blocks interaction.",
|
|
5
|
+
"If any blocking popup is visible, close it before returning done.",
|
|
6
|
+
"Prefer obvious close, dismiss, continue, accept, or X buttons.",
|
|
7
|
+
"Do not return done while a blocking overlay or dialog is still visible."
|
|
8
|
+
].join(" ");
|
|
9
|
+
const COMPUTER_USE_RECOVERY_MODELS = {
|
|
10
|
+
anthropic: "claude-sonnet-4-6",
|
|
11
|
+
openai: "gpt-5.5"
|
|
12
|
+
};
|
|
13
|
+
const PAGE_UI_METHODS = /* @__PURE__ */ new Set([
|
|
14
|
+
"click",
|
|
15
|
+
"dblclick",
|
|
16
|
+
"tap",
|
|
17
|
+
"hover",
|
|
18
|
+
"fill",
|
|
19
|
+
"type",
|
|
20
|
+
"press",
|
|
21
|
+
"pressSequentially",
|
|
22
|
+
"check",
|
|
23
|
+
"uncheck",
|
|
24
|
+
"setChecked",
|
|
25
|
+
"selectOption",
|
|
26
|
+
"setInputFiles",
|
|
27
|
+
"selectText",
|
|
28
|
+
"dispatchEvent",
|
|
29
|
+
"focus",
|
|
30
|
+
"blur",
|
|
31
|
+
"dragAndDrop"
|
|
32
|
+
]);
|
|
33
|
+
const PAGE_READ_METHODS = /* @__PURE__ */ new Set([
|
|
34
|
+
"title",
|
|
35
|
+
"content",
|
|
36
|
+
"screenshot",
|
|
37
|
+
"waitForLoadState",
|
|
38
|
+
"waitForRequest",
|
|
39
|
+
"waitForResponse",
|
|
40
|
+
"waitForURL"
|
|
41
|
+
]);
|
|
42
|
+
const LOCATOR_UI_METHODS = /* @__PURE__ */ new Set([
|
|
43
|
+
"click",
|
|
44
|
+
"dblclick",
|
|
45
|
+
"tap",
|
|
46
|
+
"hover",
|
|
47
|
+
"fill",
|
|
48
|
+
"type",
|
|
49
|
+
"press",
|
|
50
|
+
"pressSequentially",
|
|
51
|
+
"check",
|
|
52
|
+
"uncheck",
|
|
53
|
+
"setChecked",
|
|
54
|
+
"selectOption",
|
|
55
|
+
"setInputFiles",
|
|
56
|
+
"selectText",
|
|
57
|
+
"dispatchEvent",
|
|
58
|
+
"focus",
|
|
59
|
+
"blur",
|
|
60
|
+
"clear",
|
|
61
|
+
"dragTo",
|
|
62
|
+
"scrollIntoViewIfNeeded"
|
|
63
|
+
]);
|
|
64
|
+
const LOCATOR_READ_METHODS = /* @__PURE__ */ new Set([
|
|
65
|
+
"textContent",
|
|
66
|
+
"innerText",
|
|
67
|
+
"innerHTML",
|
|
68
|
+
"allTextContents",
|
|
69
|
+
"allInnerTexts",
|
|
70
|
+
"ariaSnapshot",
|
|
71
|
+
"boundingBox",
|
|
72
|
+
"count",
|
|
73
|
+
"getAttribute",
|
|
74
|
+
"inputValue",
|
|
75
|
+
"isChecked",
|
|
76
|
+
"isDisabled",
|
|
77
|
+
"isEditable",
|
|
78
|
+
"isEnabled",
|
|
79
|
+
"isVisible",
|
|
80
|
+
"isHidden",
|
|
81
|
+
"screenshot",
|
|
82
|
+
"waitFor"
|
|
83
|
+
]);
|
|
84
|
+
const PAGE_LOCATOR_FACTORY_METHODS = /* @__PURE__ */ new Set([
|
|
85
|
+
"locator",
|
|
86
|
+
"getByRole",
|
|
87
|
+
"getByText",
|
|
88
|
+
"getByLabel",
|
|
89
|
+
"getByPlaceholder",
|
|
90
|
+
"getByAltText",
|
|
91
|
+
"getByTitle",
|
|
92
|
+
"getByTestId"
|
|
93
|
+
]);
|
|
94
|
+
const LOCATOR_FACTORY_METHODS = /* @__PURE__ */ new Set([
|
|
95
|
+
"locator",
|
|
96
|
+
"getByRole",
|
|
97
|
+
"getByText",
|
|
98
|
+
"getByLabel",
|
|
99
|
+
"getByPlaceholder",
|
|
100
|
+
"getByAltText",
|
|
101
|
+
"getByTitle",
|
|
102
|
+
"getByTestId",
|
|
103
|
+
"filter",
|
|
104
|
+
"and",
|
|
105
|
+
"or",
|
|
106
|
+
"first",
|
|
107
|
+
"last",
|
|
108
|
+
"nth"
|
|
109
|
+
]);
|
|
110
|
+
const FRAME_LOCATOR_FACTORY_METHODS = /* @__PURE__ */ new Set([
|
|
111
|
+
"locator",
|
|
112
|
+
"getByRole",
|
|
113
|
+
"getByText",
|
|
114
|
+
"getByLabel",
|
|
115
|
+
"getByPlaceholder",
|
|
116
|
+
"getByAltText",
|
|
117
|
+
"getByTitle",
|
|
118
|
+
"getByTestId",
|
|
119
|
+
"owner",
|
|
120
|
+
"first",
|
|
121
|
+
"last",
|
|
122
|
+
"nth",
|
|
123
|
+
"frameLocator"
|
|
124
|
+
]);
|
|
125
|
+
function isUiMethod(targetType, method) {
|
|
126
|
+
return targetType === "page" ? PAGE_UI_METHODS.has(method) : LOCATOR_UI_METHODS.has(method);
|
|
127
|
+
}
|
|
128
|
+
function isReadMethod(targetType, method) {
|
|
129
|
+
return targetType === "page" ? PAGE_READ_METHODS.has(method) : LOCATOR_READ_METHODS.has(method);
|
|
130
|
+
}
|
|
131
|
+
function isSupportedMethod(targetType, method) {
|
|
132
|
+
return isUiMethod(targetType, method) || isReadMethod(targetType, method);
|
|
133
|
+
}
|
|
134
|
+
async function runWithFallback(args) {
|
|
135
|
+
try {
|
|
136
|
+
return await args.invoke();
|
|
137
|
+
} catch (originalError) {
|
|
138
|
+
const baseContext = {
|
|
139
|
+
page: args.page,
|
|
140
|
+
targetType: args.targetType,
|
|
141
|
+
method: args.method,
|
|
142
|
+
args: args.methodArgs
|
|
143
|
+
};
|
|
144
|
+
if (!isSupportedMethod(baseContext.targetType, baseContext.method)) {
|
|
145
|
+
throw originalError;
|
|
146
|
+
}
|
|
147
|
+
defaultLogger.info("Action failed, attempting recovery", {
|
|
148
|
+
targetType: baseContext.targetType,
|
|
149
|
+
method: baseContext.method,
|
|
150
|
+
argsCount: baseContext.args.length,
|
|
151
|
+
error: formatErrorForLog(originalError)
|
|
152
|
+
});
|
|
153
|
+
let recoveryResult;
|
|
154
|
+
try {
|
|
155
|
+
recoveryResult = await args.options.recoveryAction({
|
|
156
|
+
...baseContext,
|
|
157
|
+
error: originalError
|
|
158
|
+
});
|
|
159
|
+
} catch (recoveryError) {
|
|
160
|
+
defaultLogger.warn("Recovery action failed", {
|
|
161
|
+
targetType: baseContext.targetType,
|
|
162
|
+
method: baseContext.method,
|
|
163
|
+
originalError: formatErrorForLog(originalError),
|
|
164
|
+
recoveryError: formatErrorForLog(recoveryError)
|
|
165
|
+
});
|
|
166
|
+
throw new AggregateError(
|
|
167
|
+
[originalError, recoveryError],
|
|
168
|
+
"Recovery action failed after the original action failed."
|
|
169
|
+
);
|
|
170
|
+
}
|
|
171
|
+
defaultLogger.info("Recovery action completed, retrying original action", {
|
|
172
|
+
targetType: baseContext.targetType,
|
|
173
|
+
method: baseContext.method,
|
|
174
|
+
recoveryResult
|
|
175
|
+
});
|
|
176
|
+
try {
|
|
177
|
+
const result = await args.invoke();
|
|
178
|
+
defaultLogger.info("Recovered action retry succeeded", {
|
|
179
|
+
targetType: baseContext.targetType,
|
|
180
|
+
method: baseContext.method
|
|
181
|
+
});
|
|
182
|
+
return result;
|
|
183
|
+
} catch (retryError) {
|
|
184
|
+
defaultLogger.warn("Recovered action retry failed", {
|
|
185
|
+
targetType: baseContext.targetType,
|
|
186
|
+
method: baseContext.method,
|
|
187
|
+
originalError: formatErrorForLog(originalError),
|
|
188
|
+
retryError: formatErrorForLog(retryError)
|
|
189
|
+
});
|
|
190
|
+
throw originalError;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
function formatErrorForLog(error) {
|
|
195
|
+
if (error instanceof Error) {
|
|
196
|
+
return {
|
|
197
|
+
name: error.name,
|
|
198
|
+
message: error.message,
|
|
199
|
+
stack: error.stack
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
return { value: String(error) };
|
|
203
|
+
}
|
|
204
|
+
function bindOrWrapLocatorMethod(locator, rawPage, method, value, options, caches) {
|
|
205
|
+
if (typeof value !== "function") return value;
|
|
206
|
+
if (LOCATOR_FACTORY_METHODS.has(method)) {
|
|
207
|
+
return (...args) => {
|
|
208
|
+
const nextLocator = value.apply(locator, args);
|
|
209
|
+
return createFallbackLocator(nextLocator, rawPage, options, caches);
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
if (method === "all") {
|
|
213
|
+
return async (...args) => {
|
|
214
|
+
const locators = await value.apply(locator, args);
|
|
215
|
+
return locators.map(
|
|
216
|
+
(nextLocator) => createFallbackLocator(nextLocator, rawPage, options, caches)
|
|
217
|
+
);
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
if (method === "contentFrame") {
|
|
221
|
+
return (...args) => {
|
|
222
|
+
const frameLocator = value.apply(locator, args);
|
|
223
|
+
return createFallbackFrameLocator(frameLocator, rawPage, options, caches);
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
if (!isSupportedMethod("locator", method)) {
|
|
227
|
+
return value.bind(locator);
|
|
228
|
+
}
|
|
229
|
+
return (...args) => runWithFallback({
|
|
230
|
+
page: rawPage,
|
|
231
|
+
targetType: "locator",
|
|
232
|
+
method,
|
|
233
|
+
methodArgs: args,
|
|
234
|
+
invoke: () => value.apply(locator, args),
|
|
235
|
+
options
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
function createFallbackLocator(locator, rawPage, options, caches) {
|
|
239
|
+
const cached = caches.locators.get(locator);
|
|
240
|
+
if (cached) return cached;
|
|
241
|
+
const proxy = new Proxy(locator, {
|
|
242
|
+
get(target, prop, receiver) {
|
|
243
|
+
if (typeof prop !== "string") {
|
|
244
|
+
return Reflect.get(target, prop, receiver);
|
|
245
|
+
}
|
|
246
|
+
return bindOrWrapLocatorMethod(
|
|
247
|
+
target,
|
|
248
|
+
rawPage,
|
|
249
|
+
prop,
|
|
250
|
+
Reflect.get(target, prop, target),
|
|
251
|
+
options,
|
|
252
|
+
caches
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
});
|
|
256
|
+
caches.locators.set(locator, proxy);
|
|
257
|
+
return proxy;
|
|
258
|
+
}
|
|
259
|
+
function createFallbackFrameLocator(frameLocator, rawPage, options, caches) {
|
|
260
|
+
const cached = caches.frameLocators.get(frameLocator);
|
|
261
|
+
if (cached) return cached;
|
|
262
|
+
const proxy = new Proxy(frameLocator, {
|
|
263
|
+
get(target, prop, receiver) {
|
|
264
|
+
if (typeof prop !== "string") {
|
|
265
|
+
return Reflect.get(target, prop, receiver);
|
|
266
|
+
}
|
|
267
|
+
const value = Reflect.get(target, prop, target);
|
|
268
|
+
if (typeof value !== "function") return value;
|
|
269
|
+
if (FRAME_LOCATOR_FACTORY_METHODS.has(prop)) {
|
|
270
|
+
return (...args) => {
|
|
271
|
+
const result = value.apply(target, args);
|
|
272
|
+
if (prop === "first" || prop === "last" || prop === "nth") {
|
|
273
|
+
return createFallbackFrameLocator(
|
|
274
|
+
result,
|
|
275
|
+
rawPage,
|
|
276
|
+
options,
|
|
277
|
+
caches
|
|
278
|
+
);
|
|
279
|
+
}
|
|
280
|
+
if (prop === "frameLocator") {
|
|
281
|
+
return createFallbackFrameLocator(
|
|
282
|
+
result,
|
|
283
|
+
rawPage,
|
|
284
|
+
options,
|
|
285
|
+
caches
|
|
286
|
+
);
|
|
287
|
+
}
|
|
288
|
+
return createFallbackLocator(
|
|
289
|
+
result,
|
|
290
|
+
rawPage,
|
|
291
|
+
options,
|
|
292
|
+
caches
|
|
293
|
+
);
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
return value.bind(target);
|
|
297
|
+
}
|
|
298
|
+
});
|
|
299
|
+
caches.frameLocators.set(frameLocator, proxy);
|
|
300
|
+
return proxy;
|
|
301
|
+
}
|
|
302
|
+
function createRecoveryPage(page, options) {
|
|
303
|
+
const caches = {
|
|
304
|
+
locators: /* @__PURE__ */ new WeakMap(),
|
|
305
|
+
frameLocators: /* @__PURE__ */ new WeakMap()
|
|
306
|
+
};
|
|
307
|
+
return new Proxy(page, {
|
|
308
|
+
get(target, prop, receiver) {
|
|
309
|
+
if (typeof prop !== "string") {
|
|
310
|
+
return Reflect.get(target, prop, receiver);
|
|
311
|
+
}
|
|
312
|
+
const value = Reflect.get(target, prop, target);
|
|
313
|
+
if (typeof value !== "function") return value;
|
|
314
|
+
if (PAGE_LOCATOR_FACTORY_METHODS.has(prop)) {
|
|
315
|
+
return (...args) => {
|
|
316
|
+
const locator = value.apply(target, args);
|
|
317
|
+
return createFallbackLocator(locator, page, options, caches);
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
if (prop === "frameLocator") {
|
|
321
|
+
return (...args) => {
|
|
322
|
+
const frameLocator = value.apply(target, args);
|
|
323
|
+
return createFallbackFrameLocator(frameLocator, page, options, caches);
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
if (!isSupportedMethod("page", prop)) {
|
|
327
|
+
return value.bind(target);
|
|
328
|
+
}
|
|
329
|
+
return (...args) => runWithFallback({
|
|
330
|
+
page,
|
|
331
|
+
targetType: "page",
|
|
332
|
+
method: prop,
|
|
333
|
+
methodArgs: args,
|
|
334
|
+
invoke: () => value.apply(target, args),
|
|
335
|
+
options
|
|
336
|
+
});
|
|
337
|
+
}
|
|
338
|
+
});
|
|
339
|
+
}
|
|
340
|
+
async function resolveComputerUseRecoveryModel(options) {
|
|
341
|
+
if ("provider" in options) {
|
|
342
|
+
if (options.provider === "openai") {
|
|
343
|
+
const model2 = options.model ?? COMPUTER_USE_RECOVERY_MODELS.openai;
|
|
344
|
+
if (model2 !== COMPUTER_USE_RECOVERY_MODELS.openai) {
|
|
345
|
+
throw new Error(
|
|
346
|
+
`Unsupported OpenAI computer use recovery model "${model2}". Supported model: ${COMPUTER_USE_RECOVERY_MODELS.openai}.`
|
|
347
|
+
);
|
|
348
|
+
}
|
|
349
|
+
return import("@ai-sdk/openai").then(
|
|
350
|
+
({ createOpenAI }) => createOpenAI({ apiKey: options.apiKey })(model2)
|
|
351
|
+
);
|
|
352
|
+
}
|
|
353
|
+
const model = options.model ?? COMPUTER_USE_RECOVERY_MODELS.anthropic;
|
|
354
|
+
if (model !== COMPUTER_USE_RECOVERY_MODELS.anthropic) {
|
|
355
|
+
throw new Error(
|
|
356
|
+
`Unsupported Anthropic computer use recovery model "${model}". Supported model: ${COMPUTER_USE_RECOVERY_MODELS.anthropic}.`
|
|
357
|
+
);
|
|
358
|
+
}
|
|
359
|
+
return import("@ai-sdk/anthropic").then(
|
|
360
|
+
({ createAnthropic }) => createAnthropic({ apiKey: options.apiKey })(model)
|
|
361
|
+
);
|
|
362
|
+
}
|
|
363
|
+
return options.languageModel;
|
|
364
|
+
}
|
|
365
|
+
function computerUseRecoveryAction(options) {
|
|
366
|
+
return async ({ page }) => {
|
|
367
|
+
const model = await resolveComputerUseRecoveryModel(options);
|
|
368
|
+
return executeRecoveryAgent(
|
|
369
|
+
page,
|
|
370
|
+
options.instruction,
|
|
371
|
+
void 0,
|
|
372
|
+
model,
|
|
373
|
+
options.maxSteps
|
|
374
|
+
);
|
|
375
|
+
};
|
|
376
|
+
}
|
|
377
|
+
function popupRecoveryAction(options) {
|
|
378
|
+
return computerUseRecoveryAction({
|
|
379
|
+
...options,
|
|
380
|
+
instruction: POPUP_RECOVERY_INSTRUCTION
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
export {
|
|
384
|
+
COMPUTER_USE_RECOVERY_MODELS,
|
|
385
|
+
POPUP_RECOVERY_INSTRUCTION,
|
|
386
|
+
computerUseRecoveryAction,
|
|
387
|
+
createRecoveryPage,
|
|
388
|
+
popupRecoveryAction
|
|
389
|
+
};
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export { SESSION_STATE_VERSION, SessionAccessMode, SessionAccessModeSchema, SessionState, SessionStateFile, SessionStateFileSchema, SessionStatus, SessionStatusSchema, parseSessionStateContent, parseSessionStateData, serializeSessionState } from './session-state.js';
|
|
1
|
+
export { ProviderState, SESSION_STATE_VERSION, SessionAccessMode, SessionAccessModeSchema, SessionState, SessionStateFile, SessionStateFileSchema, SessionStatus, SessionStatusSchema, parseSessionStateContent, parseSessionStateData, serializeSessionState } from './session-state.js';
|
|
2
2
|
import 'zod';
|
|
@@ -20,6 +20,7 @@ declare const SessionViewportSchema: z.ZodObject<{
|
|
|
20
20
|
declare const ProviderStateSchema: z.ZodObject<{
|
|
21
21
|
name: z.ZodString;
|
|
22
22
|
sessionId: z.ZodString;
|
|
23
|
+
recordingUrl: z.ZodOptional<z.ZodString>;
|
|
23
24
|
}, z.core.$strip>;
|
|
24
25
|
declare const SessionStateFileSchema: z.ZodObject<{
|
|
25
26
|
version: z.ZodLiteral<1>;
|
|
@@ -48,15 +49,17 @@ declare const SessionStateFileSchema: z.ZodObject<{
|
|
|
48
49
|
provider: z.ZodOptional<z.ZodObject<{
|
|
49
50
|
name: z.ZodString;
|
|
50
51
|
sessionId: z.ZodString;
|
|
52
|
+
recordingUrl: z.ZodOptional<z.ZodString>;
|
|
51
53
|
}, z.core.$strip>>;
|
|
52
54
|
daemonSocketPath: z.ZodOptional<z.ZodString>;
|
|
53
55
|
}, z.core.$strip>;
|
|
54
56
|
type SessionStatus = z.infer<typeof SessionStatusSchema>;
|
|
55
57
|
type SessionAccessMode = z.infer<typeof SessionAccessModeSchema>;
|
|
58
|
+
type ProviderState = z.infer<typeof ProviderStateSchema>;
|
|
56
59
|
type SessionStateFile = z.infer<typeof SessionStateFileSchema>;
|
|
57
60
|
type SessionState = Omit<SessionStateFile, "version">;
|
|
58
61
|
declare function parseSessionStateData(rawState: unknown, source: string): SessionState;
|
|
59
62
|
declare function parseSessionStateContent(content: string, source: string): SessionState;
|
|
60
63
|
declare function serializeSessionState(state: SessionState): SessionStateFile;
|
|
61
64
|
|
|
62
|
-
export { ProviderStateSchema, SESSION_STATE_VERSION, type SessionAccessMode, SessionAccessModeSchema, type SessionState, type SessionStateFile, SessionStateFileSchema, type SessionStatus, SessionStatusSchema, SessionViewportSchema, parseSessionStateContent, parseSessionStateData, serializeSessionState };
|
|
65
|
+
export { type ProviderState, ProviderStateSchema, SESSION_STATE_VERSION, type SessionAccessMode, SessionAccessModeSchema, type SessionState, type SessionStateFile, SessionStateFileSchema, type SessionStatus, SessionStatusSchema, SessionViewportSchema, parseSessionStateContent, parseSessionStateData, serializeSessionState };
|
|
@@ -15,7 +15,8 @@ const SessionViewportSchema = z.object({
|
|
|
15
15
|
});
|
|
16
16
|
const ProviderStateSchema = z.object({
|
|
17
17
|
name: z.string(),
|
|
18
|
-
sessionId: z.string()
|
|
18
|
+
sessionId: z.string(),
|
|
19
|
+
recordingUrl: z.string().url().optional()
|
|
19
20
|
});
|
|
20
21
|
const SessionStateFileSchema = z.object({
|
|
21
22
|
version: z.literal(SESSION_STATE_VERSION),
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { Page } from 'playwright';
|
|
2
2
|
import { z } from 'zod';
|
|
3
|
+
import { RecoveryAction } from '../../runtime/recovery/page-fallbacks.js';
|
|
4
|
+
import 'ai';
|
|
3
5
|
|
|
4
6
|
declare const LIBRETTO_WORKFLOW_BRAND: unique symbol;
|
|
5
7
|
type LibrettoWorkflowContext = {
|
|
@@ -7,9 +9,13 @@ type LibrettoWorkflowContext = {
|
|
|
7
9
|
page: Page;
|
|
8
10
|
};
|
|
9
11
|
type LibrettoWorkflowHandler<Input = unknown, Output = unknown> = (ctx: LibrettoWorkflowContext, input: Input) => Promise<Output>;
|
|
10
|
-
type
|
|
11
|
-
input
|
|
12
|
-
output
|
|
12
|
+
type LibrettoWorkflowDefinition<InputSchema extends z.ZodType = z.ZodType<unknown>, OutputSchema extends z.ZodType = z.ZodType<unknown>> = {
|
|
13
|
+
input?: InputSchema;
|
|
14
|
+
output?: OutputSchema;
|
|
15
|
+
recoveryAction?: RecoveryAction;
|
|
16
|
+
};
|
|
17
|
+
type LibrettoWorkflowOptions<InputSchema extends z.ZodType = z.ZodType<unknown>, OutputSchema extends z.ZodType = z.ZodType<unknown>> = LibrettoWorkflowDefinition<InputSchema, OutputSchema> & {
|
|
18
|
+
handler: LibrettoWorkflowHandler<z.infer<InputSchema>, z.infer<OutputSchema>>;
|
|
13
19
|
};
|
|
14
20
|
declare class LibrettoWorkflowInputError extends Error {
|
|
15
21
|
readonly workflowName: string;
|
|
@@ -26,8 +32,13 @@ declare class LibrettoWorkflow<InputSchema extends z.ZodType = z.ZodType<unknown
|
|
|
26
32
|
readonly name: string;
|
|
27
33
|
readonly inputSchema?: InputSchema;
|
|
28
34
|
readonly outputSchema?: OutputSchema;
|
|
35
|
+
readonly recoveryAction?: RecoveryAction;
|
|
29
36
|
private readonly handler;
|
|
30
|
-
constructor(name: string,
|
|
37
|
+
constructor(name: string, options: {
|
|
38
|
+
inputSchema?: InputSchema;
|
|
39
|
+
outputSchema?: OutputSchema;
|
|
40
|
+
recoveryAction?: RecoveryAction;
|
|
41
|
+
} | undefined, handler: LibrettoWorkflowHandler<z.infer<InputSchema>, z.infer<OutputSchema>>);
|
|
31
42
|
run(ctx: LibrettoWorkflowContext, input: unknown): Promise<z.infer<OutputSchema>>;
|
|
32
43
|
}
|
|
33
44
|
type ExportedLibrettoWorkflow = {
|
|
@@ -35,6 +46,7 @@ type ExportedLibrettoWorkflow = {
|
|
|
35
46
|
readonly name: string;
|
|
36
47
|
readonly inputSchema?: z.ZodType;
|
|
37
48
|
readonly outputSchema?: z.ZodType;
|
|
49
|
+
readonly recoveryAction?: RecoveryAction;
|
|
38
50
|
run: (ctx: LibrettoWorkflowContext, input: unknown) => Promise<unknown>;
|
|
39
51
|
};
|
|
40
52
|
type WorkflowModuleExports = Record<string, unknown>;
|
|
@@ -42,7 +54,8 @@ declare function isLibrettoWorkflow(value: unknown): value is ExportedLibrettoWo
|
|
|
42
54
|
declare function getWorkflowsFromModuleExports(moduleExports: WorkflowModuleExports): ExportedLibrettoWorkflow[];
|
|
43
55
|
declare function getDefaultWorkflowFromModuleExports(moduleExports: WorkflowModuleExports): ExportedLibrettoWorkflow | null;
|
|
44
56
|
declare function getWorkflowFromModuleExports(moduleExports: WorkflowModuleExports, workflowName: string): ExportedLibrettoWorkflow | null;
|
|
45
|
-
declare function workflow<InputSchema extends z.ZodType
|
|
57
|
+
declare function workflow<InputSchema extends z.ZodType = z.ZodType<unknown>, OutputSchema extends z.ZodType = z.ZodType<unknown>>(name: string, definition: LibrettoWorkflowDefinition<InputSchema, OutputSchema>, handler: LibrettoWorkflowHandler<z.infer<InputSchema>, z.infer<OutputSchema>>): LibrettoWorkflow<InputSchema, OutputSchema>;
|
|
58
|
+
declare function workflow<InputSchema extends z.ZodType = z.ZodType<unknown>, OutputSchema extends z.ZodType = z.ZodType<unknown>>(name: string, options: LibrettoWorkflowOptions<InputSchema, OutputSchema>): LibrettoWorkflow<InputSchema, OutputSchema>;
|
|
46
59
|
declare function workflow<Input = unknown, Output = unknown>(name: string, handler: LibrettoWorkflowHandler<Input, Output>): LibrettoWorkflow<z.ZodType<Input>, z.ZodType<Output>>;
|
|
47
60
|
|
|
48
|
-
export { type ExportedLibrettoWorkflow, LIBRETTO_WORKFLOW_BRAND, LibrettoWorkflow, type LibrettoWorkflowContext, type LibrettoWorkflowHandler, LibrettoWorkflowInputError, type
|
|
61
|
+
export { type ExportedLibrettoWorkflow, LIBRETTO_WORKFLOW_BRAND, LibrettoWorkflow, type LibrettoWorkflowContext, type LibrettoWorkflowDefinition, type LibrettoWorkflowHandler, LibrettoWorkflowInputError, type LibrettoWorkflowOptions, type WorkflowInputValidator, getDefaultWorkflowFromModuleExports, getWorkflowFromModuleExports, getWorkflowsFromModuleExports, isLibrettoWorkflow, validateWorkflowInput, workflow };
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import {
|
|
2
|
+
createRecoveryPage
|
|
3
|
+
} from "../../runtime/recovery/page-fallbacks.js";
|
|
1
4
|
const LIBRETTO_WORKFLOW_BRAND = /* @__PURE__ */ Symbol.for("libretto.workflow");
|
|
2
5
|
class LibrettoWorkflowInputError extends Error {
|
|
3
6
|
workflowName;
|
|
@@ -41,16 +44,24 @@ class LibrettoWorkflow {
|
|
|
41
44
|
// this schema to JSON Schema at build time and exposes it via
|
|
42
45
|
// /v1/workflows/get so API consumers know the workflow's output shape.
|
|
43
46
|
outputSchema;
|
|
47
|
+
recoveryAction;
|
|
44
48
|
handler;
|
|
45
|
-
constructor(name,
|
|
49
|
+
constructor(name, options, handler) {
|
|
46
50
|
this.name = name;
|
|
47
|
-
this.inputSchema =
|
|
48
|
-
this.outputSchema =
|
|
51
|
+
this.inputSchema = options?.inputSchema;
|
|
52
|
+
this.outputSchema = options?.outputSchema;
|
|
53
|
+
this.recoveryAction = options?.recoveryAction;
|
|
49
54
|
this.handler = handler;
|
|
50
55
|
}
|
|
51
56
|
async run(ctx, input) {
|
|
52
57
|
const parsed = parseWorkflowInput(this.name, this.inputSchema, input);
|
|
53
|
-
|
|
58
|
+
const workflowContext = !this.recoveryAction ? ctx : {
|
|
59
|
+
...ctx,
|
|
60
|
+
page: createRecoveryPage(ctx.page, {
|
|
61
|
+
recoveryAction: this.recoveryAction
|
|
62
|
+
})
|
|
63
|
+
};
|
|
64
|
+
return this.handler(workflowContext, parsed);
|
|
54
65
|
}
|
|
55
66
|
}
|
|
56
67
|
function isLibrettoWorkflow(value) {
|
|
@@ -101,16 +112,34 @@ function getWorkflowFromModuleExports(moduleExports, workflowName) {
|
|
|
101
112
|
}
|
|
102
113
|
return null;
|
|
103
114
|
}
|
|
104
|
-
function
|
|
105
|
-
|
|
106
|
-
|
|
115
|
+
function getWorkflowConstructorOptions(options) {
|
|
116
|
+
return {
|
|
117
|
+
inputSchema: options.input,
|
|
118
|
+
outputSchema: options.output,
|
|
119
|
+
recoveryAction: options.recoveryAction
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
function workflow(name, definitionOrHandler, maybeHandler) {
|
|
123
|
+
if (typeof definitionOrHandler === "function") {
|
|
124
|
+
return new LibrettoWorkflow(name, void 0, definitionOrHandler);
|
|
125
|
+
}
|
|
126
|
+
if ("handler" in definitionOrHandler) {
|
|
127
|
+
return new LibrettoWorkflow(
|
|
128
|
+
name,
|
|
129
|
+
getWorkflowConstructorOptions(definitionOrHandler),
|
|
130
|
+
definitionOrHandler.handler
|
|
131
|
+
);
|
|
107
132
|
}
|
|
108
133
|
if (!maybeHandler) {
|
|
109
134
|
throw new Error(
|
|
110
|
-
`workflow("${name}") called
|
|
135
|
+
`workflow("${name}") called without a handler. Pass the handler as the third argument or in the options object.`
|
|
111
136
|
);
|
|
112
137
|
}
|
|
113
|
-
return new LibrettoWorkflow(
|
|
138
|
+
return new LibrettoWorkflow(
|
|
139
|
+
name,
|
|
140
|
+
getWorkflowConstructorOptions(definitionOrHandler),
|
|
141
|
+
maybeHandler
|
|
142
|
+
);
|
|
114
143
|
}
|
|
115
144
|
export {
|
|
116
145
|
LIBRETTO_WORKFLOW_BRAND,
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# Runtime recovery actions
|
|
2
|
+
|
|
3
|
+
`recoveryAction` lets a workflow recover from a supported Playwright Page or Locator failure. When a supported action fails, Libretto runs the recovery action and retries the original action once.
|
|
4
|
+
|
|
5
|
+
```typescript
|
|
6
|
+
export default workflow("reviewOrder", {
|
|
7
|
+
input,
|
|
8
|
+
output,
|
|
9
|
+
recoveryAction,
|
|
10
|
+
handler: async ({ page }, params) => {
|
|
11
|
+
await page.goto(params.url);
|
|
12
|
+
await page.locator("#review").click();
|
|
13
|
+
return { complete: true };
|
|
14
|
+
},
|
|
15
|
+
});
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## computerUseRecoveryAction
|
|
19
|
+
|
|
20
|
+
`computerUseRecoveryAction()` runs a small vision agent with your instruction. It screenshots the viewport, asks the model for a browser action, executes it with Playwright coordinates, and stops when the model returns `done` or `maxSteps` is reached.
|
|
21
|
+
|
|
22
|
+
One step is one screenshot, one model decision, and one browser action. The default `maxSteps` is `3`, which covers the common popup flow of close, confirm, then done.
|
|
23
|
+
|
|
24
|
+
```typescript
|
|
25
|
+
import { computerUseRecoveryAction } from "libretto";
|
|
26
|
+
|
|
27
|
+
const recoveryAction = computerUseRecoveryAction({
|
|
28
|
+
provider: "openai",
|
|
29
|
+
apiKey: process.env.OPENAI_API_KEY!,
|
|
30
|
+
model: "gpt-5.5",
|
|
31
|
+
instruction: "Close any visible blocker that prevents submitting the form.",
|
|
32
|
+
maxSteps: 3,
|
|
33
|
+
});
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Supported provider shortcuts:
|
|
37
|
+
|
|
38
|
+
```typescript
|
|
39
|
+
{ provider: "openai", apiKey, model?: "gpt-5.5" }
|
|
40
|
+
{ provider: "anthropic", apiKey, model?: "claude-sonnet-4-6" }
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Advanced callers can pass a preconfigured AI SDK `LanguageModel` with `languageModel`.
|
|
44
|
+
|
|
45
|
+
## popupRecoveryAction
|
|
46
|
+
|
|
47
|
+
`popupRecoveryAction()` is a preset for the common popup case. It uses Libretto's default instruction for closing popups, cookie banners, modals, overlays, and similar blockers.
|
|
48
|
+
|
|
49
|
+
```typescript
|
|
50
|
+
import { popupRecoveryAction } from "libretto";
|
|
51
|
+
|
|
52
|
+
const recoveryAction = popupRecoveryAction({
|
|
53
|
+
provider: "anthropic",
|
|
54
|
+
apiKey: process.env.ANTHROPIC_API_KEY!,
|
|
55
|
+
model: "claude-sonnet-4-6",
|
|
56
|
+
maxSteps: 3,
|
|
57
|
+
});
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Custom recovery logic
|
|
61
|
+
|
|
62
|
+
Use a custom `RecoveryAction` when the site needs deterministic recovery logic, or when it needs to combine `popupRecoveryAction()` with other steps such as reloading the page.
|
|
63
|
+
|
|
64
|
+
```typescript
|
|
65
|
+
import {
|
|
66
|
+
popupRecoveryAction,
|
|
67
|
+
type RecoveryAction,
|
|
68
|
+
} from "libretto";
|
|
69
|
+
|
|
70
|
+
const popupRecoveryAgent = popupRecoveryAction({
|
|
71
|
+
provider: "openai",
|
|
72
|
+
apiKey: process.env.OPENAI_API_KEY!,
|
|
73
|
+
model: "gpt-5.5",
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
const recoveryAction: RecoveryAction = async (context) => {
|
|
77
|
+
const result = await popupRecoveryAgent(context);
|
|
78
|
+
if (result.status === "incomplete") {
|
|
79
|
+
await context.page.reload();
|
|
80
|
+
}
|
|
81
|
+
return result;
|
|
82
|
+
};
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
If the recovery action returns without throwing, Libretto retries the original failed method once.
|