momentic 0.0.5 → 0.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +3052 -0
- package/dist/index.js +1369 -1341
- package/package.json +17 -14
- package/dist/index.mjs +0 -2289
package/bin/cli.js
ADDED
|
@@ -0,0 +1,3052 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/cli.ts
|
|
4
|
+
import chalk from "chalk";
|
|
5
|
+
import { Command as Command4, Option } from "commander";
|
|
6
|
+
import { execa } from "execa";
|
|
7
|
+
import waitOnFn from "wait-on";
|
|
8
|
+
|
|
9
|
+
// ../../packages/types/src/commands.ts
|
|
10
|
+
import dedent from "dedent";
|
|
11
|
+
import * as z2 from "zod";
|
|
12
|
+
|
|
13
|
+
// ../../packages/types/src/a11y-targets.ts
|
|
14
|
+
import * as z from "zod";
|
|
15
|
+
var A11yTargetWithCacheSchema = z.object({
|
|
16
|
+
// a11y ID
|
|
17
|
+
id: z.number().int(),
|
|
18
|
+
// additional metadata stored after the action is executed
|
|
19
|
+
// to assist in re-execution
|
|
20
|
+
role: z.string().optional(),
|
|
21
|
+
name: z.string().optional(),
|
|
22
|
+
content: z.string().optional(),
|
|
23
|
+
pathFromRoot: z.string().optional(),
|
|
24
|
+
serializedForm: z.string().optional()
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
// ../../packages/types/src/commands.ts
|
|
28
|
+
var CommandType = /* @__PURE__ */ ((CommandType2) => {
|
|
29
|
+
CommandType2["AI_ASSERTION"] = "AI_ASSERTION";
|
|
30
|
+
CommandType2["CLICK"] = "CLICK";
|
|
31
|
+
CommandType2["SELECT_OPTION"] = "SELECT_OPTION";
|
|
32
|
+
CommandType2["TYPE"] = "TYPE";
|
|
33
|
+
CommandType2["PRESS"] = "PRESS";
|
|
34
|
+
CommandType2["NAVIGATE"] = "NAVIGATE";
|
|
35
|
+
CommandType2["SCROLL_UP"] = "SCROLL_UP";
|
|
36
|
+
CommandType2["SCROLL_DOWN"] = "SCROLL_DOWN";
|
|
37
|
+
CommandType2["GO_BACK"] = "GO_BACK";
|
|
38
|
+
CommandType2["GO_FORWARD"] = "GO_FORWARD";
|
|
39
|
+
CommandType2["WAIT"] = "WAIT";
|
|
40
|
+
CommandType2["REFRESH"] = "REFRESH";
|
|
41
|
+
CommandType2["TAB"] = "TAB";
|
|
42
|
+
CommandType2["COOKIE"] = "COOKIE";
|
|
43
|
+
CommandType2["SUCCESS"] = "SUCCESS";
|
|
44
|
+
return CommandType2;
|
|
45
|
+
})(CommandType || {});
|
|
46
|
+
var ElementDescriptorSchema = z2.object({
|
|
47
|
+
// natural language passed to LLM
|
|
48
|
+
elementDescriptor: z2.string(),
|
|
49
|
+
// Cached A11y target - when a user creates a preset action, this will not exist
|
|
50
|
+
a11yData: A11yTargetWithCacheSchema.optional()
|
|
51
|
+
});
|
|
52
|
+
var CommonCommandSchema = z2.object({
|
|
53
|
+
// If the command is suggested by AI, why it did so
|
|
54
|
+
thoughts: z2.string().optional()
|
|
55
|
+
});
|
|
56
|
+
var NavigateCommandSchema = CommonCommandSchema.merge(
|
|
57
|
+
z2.object({
|
|
58
|
+
type: z2.literal("NAVIGATE" /* NAVIGATE */),
|
|
59
|
+
url: z2.string()
|
|
60
|
+
})
|
|
61
|
+
).describe("NAVIGATE <url> - Go to the specified url");
|
|
62
|
+
var ScrollUpCommandSchema = CommonCommandSchema.merge(
|
|
63
|
+
z2.object({
|
|
64
|
+
type: z2.literal("SCROLL_UP" /* SCROLL_UP */)
|
|
65
|
+
})
|
|
66
|
+
).describe("SCROLL_UP - Scroll up one page");
|
|
67
|
+
var ScrollDownCommandSchema = CommonCommandSchema.merge(
|
|
68
|
+
z2.object({
|
|
69
|
+
type: z2.literal("SCROLL_DOWN" /* SCROLL_DOWN */)
|
|
70
|
+
})
|
|
71
|
+
).describe("SCROLL_DOWN - Scroll down one page");
|
|
72
|
+
var WaitCommandSchema = CommonCommandSchema.merge(
|
|
73
|
+
z2.object({
|
|
74
|
+
type: z2.literal("WAIT" /* WAIT */),
|
|
75
|
+
delay: z2.number()
|
|
76
|
+
// seconds
|
|
77
|
+
})
|
|
78
|
+
);
|
|
79
|
+
var RefreshCommandSchema = CommonCommandSchema.merge(
|
|
80
|
+
z2.object({
|
|
81
|
+
type: z2.literal("REFRESH" /* REFRESH */)
|
|
82
|
+
})
|
|
83
|
+
);
|
|
84
|
+
var GoBackCommandSchema = CommonCommandSchema.merge(
|
|
85
|
+
z2.object({
|
|
86
|
+
type: z2.literal("GO_BACK" /* GO_BACK */)
|
|
87
|
+
})
|
|
88
|
+
);
|
|
89
|
+
var GoForwardCommandSchema = CommonCommandSchema.merge(
|
|
90
|
+
z2.object({
|
|
91
|
+
type: z2.literal("GO_FORWARD" /* GO_FORWARD */)
|
|
92
|
+
})
|
|
93
|
+
);
|
|
94
|
+
var ClickCommandSchema = CommonCommandSchema.merge(
|
|
95
|
+
z2.object({
|
|
96
|
+
type: z2.literal("CLICK" /* CLICK */),
|
|
97
|
+
target: ElementDescriptorSchema,
|
|
98
|
+
doubleClick: z2.boolean().default(false),
|
|
99
|
+
rightClick: z2.boolean().default(false)
|
|
100
|
+
})
|
|
101
|
+
).describe(
|
|
102
|
+
dedent`CLICK <id> - click on the element that has the specified id.
|
|
103
|
+
You are NOT allowed to click on disabled, hidden or StaticText elements.
|
|
104
|
+
Only click on elements on the Current Page.
|
|
105
|
+
Only click on elements with the following tag names: button, input, link, image, generic.
|
|
106
|
+
`.replace("\n", " ")
|
|
107
|
+
);
|
|
108
|
+
var SelectOptionCommandSchema = CommonCommandSchema.merge(
|
|
109
|
+
z2.object({
|
|
110
|
+
type: z2.literal("SELECT_OPTION" /* SELECT_OPTION */),
|
|
111
|
+
target: ElementDescriptorSchema,
|
|
112
|
+
option: z2.string()
|
|
113
|
+
})
|
|
114
|
+
).describe(
|
|
115
|
+
// TODO: if we move to a non-mutative way of selecting elements (e.g. by selector), we should update this description
|
|
116
|
+
`SELECT_OPTION <id> "<option>" - select the specified item from the select with the specified id. The item should exist on the page. Use the name of the item instead of the id. Make sure to include quotes around the option.`
|
|
117
|
+
);
|
|
118
|
+
var AIAssertionCommandSchema = CommonCommandSchema.merge(
|
|
119
|
+
z2.object({
|
|
120
|
+
type: z2.literal("AI_ASSERTION" /* AI_ASSERTION */),
|
|
121
|
+
assertion: z2.string(),
|
|
122
|
+
useVision: z2.boolean().default(false),
|
|
123
|
+
disableCache: z2.boolean().default(false)
|
|
124
|
+
})
|
|
125
|
+
);
|
|
126
|
+
var TypeOptionsSchema = z2.object({
|
|
127
|
+
clearContent: z2.boolean().default(true),
|
|
128
|
+
pressKeysSequentially: z2.boolean().default(false)
|
|
129
|
+
});
|
|
130
|
+
var TypeCommandSchema = CommonCommandSchema.merge(
|
|
131
|
+
z2.object({
|
|
132
|
+
type: z2.literal("TYPE" /* TYPE */),
|
|
133
|
+
target: ElementDescriptorSchema,
|
|
134
|
+
value: z2.string(),
|
|
135
|
+
pressEnter: z2.boolean().default(false)
|
|
136
|
+
})
|
|
137
|
+
).merge(TypeOptionsSchema).describe(
|
|
138
|
+
`TYPE <id> "<text>" - type the specified text into the input with the specified id. The text should be specified by the user - do not use text from the EXAMPLES or generate text yourself. Make sure to include quotes around the text.`
|
|
139
|
+
);
|
|
140
|
+
var PressCommandSchema = CommonCommandSchema.merge(
|
|
141
|
+
z2.object({
|
|
142
|
+
type: z2.literal("PRESS" /* PRESS */),
|
|
143
|
+
value: z2.string()
|
|
144
|
+
})
|
|
145
|
+
).describe(
|
|
146
|
+
`PRESS <key> - press the specified key, such as "ArrowLeft", "Enter", or "a". You must specify at least one key.`
|
|
147
|
+
);
|
|
148
|
+
var TabCommandSchema = CommonCommandSchema.merge(
|
|
149
|
+
z2.object({
|
|
150
|
+
type: z2.literal("TAB" /* TAB */),
|
|
151
|
+
url: z2.string()
|
|
152
|
+
})
|
|
153
|
+
);
|
|
154
|
+
var CookieCommandSchema = CommonCommandSchema.merge(
|
|
155
|
+
z2.object({
|
|
156
|
+
type: z2.literal("COOKIE" /* COOKIE */),
|
|
157
|
+
value: z2.string()
|
|
158
|
+
})
|
|
159
|
+
);
|
|
160
|
+
var SuccessCommandSchema = CommonCommandSchema.merge(
|
|
161
|
+
z2.object({
|
|
162
|
+
type: z2.literal("SUCCESS" /* SUCCESS */),
|
|
163
|
+
condition: AIAssertionCommandSchema.optional()
|
|
164
|
+
})
|
|
165
|
+
).describe("SUCCESS - the user goal has been successfully achieved");
|
|
166
|
+
var UserEditableAICommandSchema = z2.discriminatedUnion("type", [
|
|
167
|
+
ClickCommandSchema,
|
|
168
|
+
TypeCommandSchema,
|
|
169
|
+
PressCommandSchema,
|
|
170
|
+
SelectOptionCommandSchema,
|
|
171
|
+
NavigateCommandSchema,
|
|
172
|
+
ScrollDownCommandSchema,
|
|
173
|
+
ScrollUpCommandSchema,
|
|
174
|
+
SuccessCommandSchema
|
|
175
|
+
]);
|
|
176
|
+
var UserEditablePresetCommandSchema = z2.discriminatedUnion("type", [
|
|
177
|
+
GoBackCommandSchema,
|
|
178
|
+
GoForwardCommandSchema,
|
|
179
|
+
RefreshCommandSchema,
|
|
180
|
+
AIAssertionCommandSchema,
|
|
181
|
+
WaitCommandSchema,
|
|
182
|
+
TabCommandSchema,
|
|
183
|
+
CookieCommandSchema
|
|
184
|
+
]);
|
|
185
|
+
var CommandSchema = z2.discriminatedUnion("type", [
|
|
186
|
+
// Commands that can be either specified manually or auto-created by AI in an AI step
|
|
187
|
+
...UserEditableAICommandSchema.options,
|
|
188
|
+
// Commands that can only be specified manually ("preset commands")
|
|
189
|
+
...UserEditablePresetCommandSchema.options
|
|
190
|
+
]);
|
|
191
|
+
var FailureCommandSchema = CommonCommandSchema.merge(
|
|
192
|
+
z2.object({
|
|
193
|
+
type: z2.literal("FAILURE")
|
|
194
|
+
})
|
|
195
|
+
).describe(
|
|
196
|
+
"FAILURE - there are no commands to suggest that could make progress that have not already been tried before"
|
|
197
|
+
);
|
|
198
|
+
var AICommandSchema = z2.discriminatedUnion("type", [
|
|
199
|
+
...UserEditableAICommandSchema.options,
|
|
200
|
+
FailureCommandSchema
|
|
201
|
+
]);
|
|
202
|
+
|
|
203
|
+
// ../../packages/types/src/steps.ts
|
|
204
|
+
import * as z3 from "zod";
|
|
205
|
+
var StepType = /* @__PURE__ */ ((StepType2) => {
|
|
206
|
+
StepType2["AI_ACTION"] = "AI_ACTION";
|
|
207
|
+
StepType2["PRESET_ACTION"] = "PRESET_ACTION";
|
|
208
|
+
StepType2["MODULE"] = "MODULE";
|
|
209
|
+
return StepType2;
|
|
210
|
+
})(StepType || {});
|
|
211
|
+
var AIActionSchema = z3.object({
|
|
212
|
+
type: z3.literal("AI_ACTION" /* AI_ACTION */),
|
|
213
|
+
text: z3.string(),
|
|
214
|
+
// Cached commands for this step
|
|
215
|
+
commands: z3.array(UserEditableAICommandSchema).optional()
|
|
216
|
+
});
|
|
217
|
+
var PresetActionSchema = z3.object({
|
|
218
|
+
type: z3.literal("PRESET_ACTION" /* PRESET_ACTION */),
|
|
219
|
+
command: CommandSchema
|
|
220
|
+
});
|
|
221
|
+
var ModuleStepSchema = z3.object({
|
|
222
|
+
type: z3.literal("MODULE" /* MODULE */),
|
|
223
|
+
moduleId: z3.string().uuid()
|
|
224
|
+
});
|
|
225
|
+
var AllowedModuleStepSchema = z3.union([
|
|
226
|
+
AIActionSchema,
|
|
227
|
+
PresetActionSchema
|
|
228
|
+
]);
|
|
229
|
+
var ResolvedModuleStepSchema = z3.object({
|
|
230
|
+
type: z3.literal("RESOLVED_MODULE"),
|
|
231
|
+
moduleId: z3.string().uuid(),
|
|
232
|
+
name: z3.string(),
|
|
233
|
+
steps: AllowedModuleStepSchema.array()
|
|
234
|
+
});
|
|
235
|
+
var StepSchema = z3.union([
|
|
236
|
+
AIActionSchema,
|
|
237
|
+
PresetActionSchema,
|
|
238
|
+
ModuleStepSchema
|
|
239
|
+
]);
|
|
240
|
+
var ResolvedStepSchema = z3.union([
|
|
241
|
+
AIActionSchema,
|
|
242
|
+
PresetActionSchema,
|
|
243
|
+
ResolvedModuleStepSchema
|
|
244
|
+
]);
|
|
245
|
+
|
|
246
|
+
// ../../packages/web-agent/src/browsers/chrome.ts
|
|
247
|
+
import {
|
|
248
|
+
chromium,
|
|
249
|
+
devices
|
|
250
|
+
} from "playwright";
|
|
251
|
+
|
|
252
|
+
// ../../packages/types/src/assertions.ts
|
|
253
|
+
import { z as z4 } from "zod";
|
|
254
|
+
var AIAssertionResultSchema = z4.object({
|
|
255
|
+
thoughts: z4.string(),
|
|
256
|
+
result: z4.boolean(),
|
|
257
|
+
relevantElements: z4.array(z4.number()).optional()
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
// ../../packages/types/src/ai-command-generation.ts
|
|
261
|
+
import { z as z5 } from "zod";
|
|
262
|
+
|
|
263
|
+
// ../../packages/types/src/errors.ts
|
|
264
|
+
var BrowserExecutionError = class extends Error {
|
|
265
|
+
constructor(message, options = {}) {
|
|
266
|
+
super(message, options);
|
|
267
|
+
this.name = "BrowserExecutionError";
|
|
268
|
+
}
|
|
269
|
+
};
|
|
270
|
+
var EmptyA11yTreeError = class extends Error {
|
|
271
|
+
constructor(options = {}) {
|
|
272
|
+
super("Got empty a11y tree", options);
|
|
273
|
+
this.name = "EmptyA11yTreeError";
|
|
274
|
+
}
|
|
275
|
+
};
|
|
276
|
+
|
|
277
|
+
// ../../packages/types/src/ai-command-generation.ts
|
|
278
|
+
var LLMOutputSchema = z5.object({
|
|
279
|
+
command: z5.string(),
|
|
280
|
+
thoughts: z5.string()
|
|
281
|
+
});
|
|
282
|
+
var NumericStringSchema = z5.string().pipe(z5.coerce.number());
|
|
283
|
+
|
|
284
|
+
// ../../packages/types/src/command-results.ts
|
|
285
|
+
import * as z6 from "zod";
|
|
286
|
+
var ResultStatus = /* @__PURE__ */ ((ResultStatus2) => {
|
|
287
|
+
ResultStatus2["SUCCESS"] = "SUCCESS";
|
|
288
|
+
ResultStatus2["FAILED"] = "FAILED";
|
|
289
|
+
ResultStatus2["RUNNING"] = "RUNNING";
|
|
290
|
+
ResultStatus2["IDLE"] = "IDLE";
|
|
291
|
+
ResultStatus2["CANCELLED"] = "CANCELLED";
|
|
292
|
+
return ResultStatus2;
|
|
293
|
+
})(ResultStatus || {});
|
|
294
|
+
var CommandStatus = /* @__PURE__ */ ((CommandStatus2) => {
|
|
295
|
+
CommandStatus2["SUCCESS"] = "SUCCESS";
|
|
296
|
+
CommandStatus2["FAILED"] = "FAILED";
|
|
297
|
+
return CommandStatus2;
|
|
298
|
+
})(CommandStatus || {});
|
|
299
|
+
var CommandMetadataSchema = z6.object({
|
|
300
|
+
beforeUrl: z6.string(),
|
|
301
|
+
// FIXME: this should be a discriminated union of string | Buffer
|
|
302
|
+
// but to avoid too much schema wranging we leave this for now
|
|
303
|
+
// https://github.com/colinhacks/zod/issues/153
|
|
304
|
+
beforeScreenshot: z6.string().or(z6.instanceof(Buffer)),
|
|
305
|
+
afterUrl: z6.string().optional(),
|
|
306
|
+
afterScreenshot: z6.string().or(z6.instanceof(Buffer)).optional(),
|
|
307
|
+
startedAt: z6.coerce.date(),
|
|
308
|
+
finishedAt: z6.coerce.date(),
|
|
309
|
+
viewport: z6.object({
|
|
310
|
+
height: z6.number(),
|
|
311
|
+
width: z6.number()
|
|
312
|
+
}),
|
|
313
|
+
status: z6.nativeEnum(CommandStatus),
|
|
314
|
+
// used for error message and thoughts
|
|
315
|
+
message: z6.string().optional(),
|
|
316
|
+
elementInteracted: z6.string().optional()
|
|
317
|
+
});
|
|
318
|
+
var StepResultMetadataSchema = z6.object({
|
|
319
|
+
startedAt: z6.coerce.date(),
|
|
320
|
+
finishedAt: z6.coerce.date(),
|
|
321
|
+
status: z6.nativeEnum(ResultStatus),
|
|
322
|
+
// used for error message and thoughts
|
|
323
|
+
message: z6.string().optional(),
|
|
324
|
+
// browser info
|
|
325
|
+
userAgent: z6.string().optional()
|
|
326
|
+
});
|
|
327
|
+
var PresetActionResultSchema = PresetActionSchema.merge(
|
|
328
|
+
StepResultMetadataSchema
|
|
329
|
+
).merge(
|
|
330
|
+
z6.object({
|
|
331
|
+
// Array just for consistency with other result types, should only ever be one for preset.
|
|
332
|
+
results: CommandMetadataSchema.array()
|
|
333
|
+
})
|
|
334
|
+
);
|
|
335
|
+
var AIActionResultSchema = AIActionSchema.merge(
|
|
336
|
+
StepResultMetadataSchema
|
|
337
|
+
).merge(
|
|
338
|
+
z6.object({
|
|
339
|
+
results: PresetActionResultSchema.array()
|
|
340
|
+
})
|
|
341
|
+
);
|
|
342
|
+
var ModuleResultSchema = ModuleStepSchema.merge(
|
|
343
|
+
StepResultMetadataSchema
|
|
344
|
+
).merge(
|
|
345
|
+
z6.object({
|
|
346
|
+
// nested results
|
|
347
|
+
results: z6.union([AIActionResultSchema, PresetActionResultSchema]).array()
|
|
348
|
+
})
|
|
349
|
+
);
|
|
350
|
+
var ResultSchema = z6.discriminatedUnion("type", [
|
|
351
|
+
AIActionResultSchema,
|
|
352
|
+
PresetActionResultSchema,
|
|
353
|
+
ModuleResultSchema
|
|
354
|
+
]);
|
|
355
|
+
|
|
356
|
+
// ../../packages/types/src/cookies.ts
|
|
357
|
+
import { parseString } from "set-cookie-parser";
|
|
358
|
+
function parseCookieString(cookie) {
|
|
359
|
+
const parsedCookie = parseString(cookie);
|
|
360
|
+
if (!parsedCookie.name) {
|
|
361
|
+
throw new Error("Name missing from cookie");
|
|
362
|
+
}
|
|
363
|
+
if (!parsedCookie.value) {
|
|
364
|
+
throw new Error("Value missing from cookie");
|
|
365
|
+
}
|
|
366
|
+
let sameSite;
|
|
367
|
+
if (parsedCookie.sameSite) {
|
|
368
|
+
const sameSiteSetting = parsedCookie.sameSite.trim().toLowerCase();
|
|
369
|
+
if (sameSiteSetting === "strict") {
|
|
370
|
+
sameSite = "Strict";
|
|
371
|
+
} else if (sameSiteSetting === "lax") {
|
|
372
|
+
sameSite = "Lax";
|
|
373
|
+
} else if (sameSiteSetting === "none") {
|
|
374
|
+
sameSite = "None";
|
|
375
|
+
} else {
|
|
376
|
+
throw new Error(`Invalid sameSite setting in cookie: ${sameSiteSetting}`);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
if (!parsedCookie.path && parsedCookie.domain) {
|
|
380
|
+
parsedCookie.path = "/";
|
|
381
|
+
}
|
|
382
|
+
const result = {
|
|
383
|
+
...parsedCookie,
|
|
384
|
+
expires: parsedCookie.expires ? parsedCookie.expires.getTime() / 1e3 : void 0,
|
|
385
|
+
sameSite
|
|
386
|
+
};
|
|
387
|
+
return result;
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
// ../../packages/types/src/execute-results.ts
|
|
391
|
+
import * as z7 from "zod";
|
|
392
|
+
var ExecuteCommandHistoryEntrySchema = z7.object({
|
|
393
|
+
// type of command executed
|
|
394
|
+
type: z7.nativeEnum(StepType),
|
|
395
|
+
// if AI step type, what command was executed
|
|
396
|
+
generatedStep: UserEditableAICommandSchema.optional(),
|
|
397
|
+
// human readable descriptor for action taken, including element interacted with
|
|
398
|
+
serializedCommand: z7.string().optional(),
|
|
399
|
+
// human readable descriptor for element interacted with
|
|
400
|
+
elementInteracted: z7.string().optional()
|
|
401
|
+
});
|
|
402
|
+
|
|
403
|
+
// ../../packages/types/src/goal-splitter.ts
|
|
404
|
+
import { z as z8 } from "zod";
|
|
405
|
+
var InstructionsSchema = z8.string().array();
|
|
406
|
+
|
|
407
|
+
// ../../packages/types/src/locator.ts
|
|
408
|
+
import * as z9 from "zod";
|
|
409
|
+
var AILocatorSchema = z9.object({
|
|
410
|
+
thoughts: z9.string(),
|
|
411
|
+
// a11y id
|
|
412
|
+
id: z9.number().int(),
|
|
413
|
+
// dropdowns should have options
|
|
414
|
+
options: z9.array(z9.string()).optional()
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
// ../../packages/types/src/modules.ts
|
|
418
|
+
import { z as z10 } from "zod";
|
|
419
|
+
var ModuleMetadataSchema = z10.object({
|
|
420
|
+
id: z10.string(),
|
|
421
|
+
createdAt: z10.coerce.date(),
|
|
422
|
+
createdBy: z10.string(),
|
|
423
|
+
organizationId: z10.string().or(z10.null()),
|
|
424
|
+
name: z10.string(),
|
|
425
|
+
schemaVersion: z10.string(),
|
|
426
|
+
// this is only used in the client and is not stored in the db
|
|
427
|
+
numSteps: z10.number()
|
|
428
|
+
});
|
|
429
|
+
var ModuleSchema = z10.object({
|
|
430
|
+
steps: AllowedModuleStepSchema.array()
|
|
431
|
+
}).merge(ModuleMetadataSchema.omit({ numSteps: true }));
|
|
432
|
+
|
|
433
|
+
// ../../packages/types/src/runs.ts
|
|
434
|
+
import { z as z11 } from "zod";
|
|
435
|
+
var RunTrigger = {
|
|
436
|
+
WEBHOOK: "WEBHOOK",
|
|
437
|
+
CRON: "CRON",
|
|
438
|
+
MANUAL: "MANUAL"
|
|
439
|
+
};
|
|
440
|
+
var RunStatusEnum = {
|
|
441
|
+
PENDING: "PENDING",
|
|
442
|
+
RUNNING: "RUNNING",
|
|
443
|
+
PASSED: "PASSED",
|
|
444
|
+
FAILED: "FAILED",
|
|
445
|
+
CANCELLED: "CANCELLED"
|
|
446
|
+
};
|
|
447
|
+
var DateOrStringSchema = z11.string().pipe(z11.coerce.date()).or(z11.date());
|
|
448
|
+
var RunMetadataSchema = z11.object({
|
|
449
|
+
id: z11.string(),
|
|
450
|
+
createdAt: DateOrStringSchema,
|
|
451
|
+
createdBy: z11.string(),
|
|
452
|
+
organizationId: z11.string().or(z11.null()),
|
|
453
|
+
scheduledAt: DateOrStringSchema.or(z11.null()),
|
|
454
|
+
startedAt: DateOrStringSchema.or(z11.null()),
|
|
455
|
+
finishedAt: DateOrStringSchema.or(z11.null()),
|
|
456
|
+
testId: z11.string().or(z11.null()),
|
|
457
|
+
status: z11.nativeEnum(RunStatusEnum),
|
|
458
|
+
trigger: z11.nativeEnum(RunTrigger),
|
|
459
|
+
test: z11.object({
|
|
460
|
+
name: z11.string(),
|
|
461
|
+
id: z11.string()
|
|
462
|
+
}).or(z11.null())
|
|
463
|
+
});
|
|
464
|
+
var RunWithTestSchema = RunMetadataSchema.merge(
|
|
465
|
+
z11.object({
|
|
466
|
+
results: ResultSchema.array(),
|
|
467
|
+
test: z11.object({
|
|
468
|
+
name: z11.string(),
|
|
469
|
+
id: z11.string(),
|
|
470
|
+
baseUrl: z11.string()
|
|
471
|
+
}).or(z11.null())
|
|
472
|
+
})
|
|
473
|
+
);
|
|
474
|
+
|
|
475
|
+
// ../../packages/types/src/serialization.ts
|
|
476
|
+
function clampText(text, length) {
|
|
477
|
+
if (text.length < length) {
|
|
478
|
+
return text;
|
|
479
|
+
}
|
|
480
|
+
return text.slice(0, length - 3) + "[...]";
|
|
481
|
+
}
|
|
482
|
+
function serializeCommand(command) {
|
|
483
|
+
var _a, _b;
|
|
484
|
+
switch (command.type) {
|
|
485
|
+
case "SUCCESS" /* SUCCESS */:
|
|
486
|
+
if ((_a = command.condition) == null ? void 0 : _a.assertion) {
|
|
487
|
+
return `Check success condition: ${command.condition.assertion}`;
|
|
488
|
+
}
|
|
489
|
+
return `All commands completed`;
|
|
490
|
+
case "NAVIGATE" /* NAVIGATE */:
|
|
491
|
+
return `Go to URL: ${clampText(command.url, 30)}`;
|
|
492
|
+
case "GO_BACK" /* GO_BACK */:
|
|
493
|
+
return `Go back to the previous page`;
|
|
494
|
+
case "GO_FORWARD" /* GO_FORWARD */:
|
|
495
|
+
return `Go forward to the next page`;
|
|
496
|
+
case "SCROLL_DOWN" /* SCROLL_DOWN */:
|
|
497
|
+
return `Scroll down one page`;
|
|
498
|
+
case "SCROLL_UP" /* SCROLL_UP */:
|
|
499
|
+
return `Scroll up one page`;
|
|
500
|
+
case "WAIT" /* WAIT */:
|
|
501
|
+
return `Wait for ${command.delay} seconds`;
|
|
502
|
+
case "REFRESH" /* REFRESH */:
|
|
503
|
+
return `Refresh the page`;
|
|
504
|
+
case "CLICK" /* CLICK */:
|
|
505
|
+
return `Click on '${command.target.elementDescriptor}'`;
|
|
506
|
+
case "TYPE" /* TYPE */:
|
|
507
|
+
let serializedTarget = "";
|
|
508
|
+
if ((_b = command.target.a11yData) == null ? void 0 : _b.serializedForm) {
|
|
509
|
+
serializedTarget = ` in element ${command.target.a11yData.serializedForm}`;
|
|
510
|
+
} else if (command.target.elementDescriptor.length > 0) {
|
|
511
|
+
serializedTarget = ` in element ${command.target.elementDescriptor}`;
|
|
512
|
+
}
|
|
513
|
+
return `Type${serializedTarget}: '${command.value}'`;
|
|
514
|
+
case "PRESS" /* PRESS */:
|
|
515
|
+
return `Press '${command.value}'`;
|
|
516
|
+
case "SELECT_OPTION" /* SELECT_OPTION */:
|
|
517
|
+
return `Select option '${command.option}' in '${command.target.elementDescriptor}'`;
|
|
518
|
+
case "TAB" /* TAB */:
|
|
519
|
+
return `Switch to tab: ${command.url}`;
|
|
520
|
+
case "COOKIE" /* COOKIE */:
|
|
521
|
+
return `Set cookie: ${command.value}`;
|
|
522
|
+
case "AI_ASSERTION" /* AI_ASSERTION */:
|
|
523
|
+
return `${command.useVision ? "Visual assertion" : "Assertion"}: '${command.assertion}'`;
|
|
524
|
+
default:
|
|
525
|
+
const assertUnreachable = (_x) => {
|
|
526
|
+
throw "If Typescript complains about the line below, you missed a case or break in the switch above";
|
|
527
|
+
};
|
|
528
|
+
return assertUnreachable(command);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// ../../packages/types/src/card-display.ts
|
|
533
|
+
var SELECTABLE_PRESET_COMMAND_OPTIONS_SET = new Set(
|
|
534
|
+
Object.values(CommandType)
|
|
535
|
+
);
|
|
536
|
+
var CARD_DISPLAY_NAMES = {
|
|
537
|
+
["AI_ACTION" /* AI_ACTION */]: "AI action",
|
|
538
|
+
["MODULE" /* MODULE */]: "Module",
|
|
539
|
+
["AI_ASSERTION" /* AI_ASSERTION */]: "AI check",
|
|
540
|
+
["CLICK" /* CLICK */]: "Click",
|
|
541
|
+
["SELECT_OPTION" /* SELECT_OPTION */]: "Select",
|
|
542
|
+
["TYPE" /* TYPE */]: "Type",
|
|
543
|
+
["PRESS" /* PRESS */]: "Press",
|
|
544
|
+
["NAVIGATE" /* NAVIGATE */]: "Navigate",
|
|
545
|
+
["SCROLL_UP" /* SCROLL_UP */]: "Scroll up",
|
|
546
|
+
["SCROLL_DOWN" /* SCROLL_DOWN */]: "Scroll down",
|
|
547
|
+
["GO_BACK" /* GO_BACK */]: "Go back",
|
|
548
|
+
["GO_FORWARD" /* GO_FORWARD */]: "Go forward",
|
|
549
|
+
["WAIT" /* WAIT */]: "Wait",
|
|
550
|
+
["REFRESH" /* REFRESH */]: "Refresh",
|
|
551
|
+
["TAB" /* TAB */]: "Switch tab",
|
|
552
|
+
["COOKIE" /* COOKIE */]: "Set cookie",
|
|
553
|
+
["SUCCESS" /* SUCCESS */]: "Done"
|
|
554
|
+
};
|
|
555
|
+
var CARD_DESCRIPTIONS = {
|
|
556
|
+
["AI_ACTION" /* AI_ACTION */]: "Ask AI to plan and execute something on the page.",
|
|
557
|
+
["MODULE" /* MODULE */]: "A list of steps that can be reused in multiple tests.",
|
|
558
|
+
["AI_ASSERTION" /* AI_ASSERTION */]: "Ask AI whether something is true on the page.",
|
|
559
|
+
["CLICK" /* CLICK */]: "Click on an element on the page based on a description.",
|
|
560
|
+
["SELECT_OPTION" /* SELECT_OPTION */]: "Select an option from a dropdown based on a description.",
|
|
561
|
+
["TYPE" /* TYPE */]: "Type the specified text into an element.",
|
|
562
|
+
["PRESS" /* PRESS */]: "Press the specified keys using the keyboard. (e.g. Ctrl+A)",
|
|
563
|
+
["NAVIGATE" /* NAVIGATE */]: "Navigate to the specified URL.",
|
|
564
|
+
["SCROLL_UP" /* SCROLL_UP */]: "Scroll up one page.",
|
|
565
|
+
["SCROLL_DOWN" /* SCROLL_DOWN */]: "Scroll down one page.",
|
|
566
|
+
["GO_BACK" /* GO_BACK */]: "Go back in browser history.",
|
|
567
|
+
["GO_FORWARD" /* GO_FORWARD */]: "Go forward in browser history.",
|
|
568
|
+
["WAIT" /* WAIT */]: "Wait for the specified number of seconds.",
|
|
569
|
+
["REFRESH" /* REFRESH */]: "Refresh the page. This will not clear cookies or session data.",
|
|
570
|
+
["TAB" /* TAB */]: "Switch to different tab in the browser.",
|
|
571
|
+
["COOKIE" /* COOKIE */]: "Set a cookie that will persist throughout the browser session",
|
|
572
|
+
["SUCCESS" /* SUCCESS */]: "Indicate the entire AI action has succeeded, optionally based on a condition."
|
|
573
|
+
};
|
|
574
|
+
|
|
575
|
+
// ../../packages/types/src/test.ts
|
|
576
|
+
import { z as z13 } from "zod";
|
|
577
|
+
|
|
578
|
+
// ../../packages/types/src/test-settings.ts
|
|
579
|
+
import { isValidCron } from "cron-validator";
|
|
580
|
+
import { z as z12 } from "zod";
|
|
581
|
+
var TestAdvancedSettingsSchema = z12.object({
|
|
582
|
+
availableAsModule: z12.boolean().default(false),
|
|
583
|
+
disableAICaching: z12.boolean().default(false)
|
|
584
|
+
});
|
|
585
|
+
var ScheduleSettingsSchema = z12.object({
|
|
586
|
+
cron: z12.string().refine(
|
|
587
|
+
(v) => {
|
|
588
|
+
return isValidCron(v);
|
|
589
|
+
},
|
|
590
|
+
{ message: "Invalid cron expression." }
|
|
591
|
+
).default("0 0 */1 * *"),
|
|
592
|
+
enabled: z12.boolean().default(false),
|
|
593
|
+
timeZone: z12.string().default("America/Los_Angeles"),
|
|
594
|
+
// this is used for removing repeatable jobs (not set by user)
|
|
595
|
+
jobKey: z12.string().optional()
|
|
596
|
+
});
|
|
597
|
+
var WebhookSchema = z12.object({
|
|
598
|
+
lastStatus: z12.number().optional(),
|
|
599
|
+
url: z12.string().url()
|
|
600
|
+
});
|
|
601
|
+
var WebhookSettingsSchema = z12.array(WebhookSchema).default([]);
|
|
602
|
+
var TestSettingsSchema = z12.object({
|
|
603
|
+
name: z12.string().min(1),
|
|
604
|
+
baseUrl: z12.string().url(),
|
|
605
|
+
advanced: TestAdvancedSettingsSchema
|
|
606
|
+
});
|
|
607
|
+
|
|
608
|
+
// ../../packages/types/src/test.ts
|
|
609
|
+
var ResolvedTestSchema = z13.object({
|
|
610
|
+
id: z13.string(),
|
|
611
|
+
name: z13.string(),
|
|
612
|
+
baseUrl: z13.string(),
|
|
613
|
+
steps: z13.array(ResolvedStepSchema),
|
|
614
|
+
createdAt: z13.coerce.date(),
|
|
615
|
+
updatedAt: z13.coerce.date(),
|
|
616
|
+
createdBy: z13.string(),
|
|
617
|
+
organizationId: z13.string().or(z13.null()),
|
|
618
|
+
schemaVersion: z13.string(),
|
|
619
|
+
advanced: TestAdvancedSettingsSchema,
|
|
620
|
+
schedule: ScheduleSettingsSchema,
|
|
621
|
+
webhooks: WebhookSettingsSchema
|
|
622
|
+
});
|
|
623
|
+
|
|
624
|
+
// ../../packages/types/src/context.ts
|
|
625
|
+
import * as z14 from "zod";
|
|
626
|
+
var DynamicContextSchema = z14.object({
|
|
627
|
+
// user goal or instruction
|
|
628
|
+
goal: z14.string(),
|
|
629
|
+
// current url of the browser
|
|
630
|
+
url: z14.string(),
|
|
631
|
+
// serialized page state
|
|
632
|
+
browserState: z14.string(),
|
|
633
|
+
// serialized history of previous commands
|
|
634
|
+
history: z14.string(),
|
|
635
|
+
// number of previously executed commands
|
|
636
|
+
numPrevious: z14.number(),
|
|
637
|
+
// last executed command, if any
|
|
638
|
+
lastCommand: ExecuteCommandHistoryEntrySchema.or(z14.null())
|
|
639
|
+
});
|
|
640
|
+
|
|
641
|
+
// ../../packages/types/src/public-api.ts
|
|
642
|
+
import * as z15 from "zod";
|
|
643
|
+
var GeneratorOptionsSchema = z15.object({
|
|
644
|
+
disableCache: z15.boolean()
|
|
645
|
+
});
|
|
646
|
+
var GetNextCommandBodySchema = DynamicContextSchema.merge(
|
|
647
|
+
GeneratorOptionsSchema
|
|
648
|
+
);
|
|
649
|
+
var GetNextCommandResponseSchema = AICommandSchema;
|
|
650
|
+
var GetAssertionResultBodySchema = z15.discriminatedUnion("vision", [
|
|
651
|
+
DynamicContextSchema.merge(GeneratorOptionsSchema).merge(
|
|
652
|
+
z15.object({
|
|
653
|
+
vision: z15.literal(false)
|
|
654
|
+
})
|
|
655
|
+
),
|
|
656
|
+
DynamicContextSchema.pick({
|
|
657
|
+
goal: true,
|
|
658
|
+
url: true
|
|
659
|
+
}).merge(GeneratorOptionsSchema).merge(
|
|
660
|
+
z15.object({
|
|
661
|
+
// base64 encoded image
|
|
662
|
+
screenshot: z15.string(),
|
|
663
|
+
vision: z15.literal(true)
|
|
664
|
+
})
|
|
665
|
+
)
|
|
666
|
+
]);
|
|
667
|
+
var GetAssertionResponseSchema = AIAssertionResultSchema;
|
|
668
|
+
var LocateBodySchema = DynamicContextSchema.pick({
|
|
669
|
+
browserState: true,
|
|
670
|
+
goal: true
|
|
671
|
+
}).merge(GeneratorOptionsSchema);
|
|
672
|
+
var LocateResponseSchema = AILocatorSchema;
|
|
673
|
+
var SplitGoalBodySchema = DynamicContextSchema.pick({
|
|
674
|
+
goal: true,
|
|
675
|
+
url: true
|
|
676
|
+
}).merge(GeneratorOptionsSchema);
|
|
677
|
+
var SplitGoalResponseSchema = z15.string().array();
|
|
678
|
+
var QueueBodySchema = z15.object({
|
|
679
|
+
testIds: z15.string().array()
|
|
680
|
+
});
|
|
681
|
+
var GetTestResponseSchema = ResolvedTestSchema;
|
|
682
|
+
var CreateRunBodySchema = z15.object({
|
|
683
|
+
testId: z15.string()
|
|
684
|
+
});
|
|
685
|
+
var CreateRunResponseSchema = RunWithTestSchema;
|
|
686
|
+
var GetRunResponseSchema = RunWithTestSchema;
|
|
687
|
+
var UpdateRunBodySchema = z15.object({
|
|
688
|
+
finishedAt: z15.coerce.date(),
|
|
689
|
+
results: ResultSchema.array(),
|
|
690
|
+
status: z15.nativeEnum(RunStatusEnum)
|
|
691
|
+
}).partial();
|
|
692
|
+
var CreateScreenshotBodySchema = z15.object({
|
|
693
|
+
// base64 string
|
|
694
|
+
screenshot: z15.string()
|
|
695
|
+
});
|
|
696
|
+
var CreateScreenshotResponseSchema = z15.object({
|
|
697
|
+
key: z15.string()
|
|
698
|
+
});
|
|
699
|
+
|
|
700
|
+
// ../../packages/web-agent/src/utils/url.ts
|
|
701
|
+
var urlChanged = (url1, url2) => {
|
|
702
|
+
const { hostname, pathname } = new URL(url1);
|
|
703
|
+
const { hostname: hostname2, pathname: pathname2 } = new URL(url2);
|
|
704
|
+
return hostname !== hostname2 || pathname !== pathname2;
|
|
705
|
+
};
|
|
706
|
+
|
|
707
|
+
// ../../packages/web-agent/src/browsers/a11y.ts
|
|
708
|
+
var bannedProperties = /* @__PURE__ */ new Set(["focusable"]);
|
|
709
|
+
var alwaysInterestingRoles = /* @__PURE__ */ new Set([
|
|
710
|
+
"textbox",
|
|
711
|
+
"checkbox",
|
|
712
|
+
"button",
|
|
713
|
+
"link"
|
|
714
|
+
]);
|
|
715
|
+
var rolesToOmitID = /* @__PURE__ */ new Set(["paragraph", "menuitem", "option"]);
|
|
716
|
+
var defaultA11yNodeSerializeParams = {
|
|
717
|
+
indentLevel: 0,
|
|
718
|
+
noID: false,
|
|
719
|
+
noChildren: false,
|
|
720
|
+
noProperties: false
|
|
721
|
+
};
|
|
722
|
+
var ProcessedA11yNode = class {
|
|
723
|
+
id;
|
|
724
|
+
role;
|
|
725
|
+
name;
|
|
726
|
+
content;
|
|
727
|
+
properties;
|
|
728
|
+
// css-like selector from the root of the tree to the current node
|
|
729
|
+
pathFromRoot;
|
|
730
|
+
parent;
|
|
731
|
+
// md5 hash - set lazily in most cases (not used at the moment)
|
|
732
|
+
// md5Sum: string;
|
|
733
|
+
children;
|
|
734
|
+
backendNodeID;
|
|
735
|
+
constructor(params) {
|
|
736
|
+
this.id = params.id;
|
|
737
|
+
this.role = params.role;
|
|
738
|
+
this.name = params.name;
|
|
739
|
+
this.content = params.content;
|
|
740
|
+
this.properties = params.properties;
|
|
741
|
+
this.pathFromRoot = params.pathFromRoot;
|
|
742
|
+
this.children = params.children;
|
|
743
|
+
this.backendNodeID = params.backendNodeID;
|
|
744
|
+
}
|
|
745
|
+
getLogForm() {
|
|
746
|
+
return JSON.stringify({
|
|
747
|
+
id: this.id,
|
|
748
|
+
name: this.name ?? "",
|
|
749
|
+
role: this.role ?? "",
|
|
750
|
+
backendNodeId: this.backendNodeID
|
|
751
|
+
});
|
|
752
|
+
}
|
|
753
|
+
/**
|
|
754
|
+
* Returns true if the current node contains interesting properties.
|
|
755
|
+
* Does not go through children.
|
|
756
|
+
*/
|
|
757
|
+
isInteresting() {
|
|
758
|
+
if (alwaysInterestingRoles.has(this.role))
|
|
759
|
+
return true;
|
|
760
|
+
if (this.children.some((child) => child.role === "StaticText"))
|
|
761
|
+
return true;
|
|
762
|
+
return !!this.name.trim() || !!this.content;
|
|
763
|
+
}
|
|
764
|
+
serialize(opts = defaultA11yNodeSerializeParams) {
|
|
765
|
+
const { indentLevel, noChildren, noProperties, noID } = Object.assign(
|
|
766
|
+
{},
|
|
767
|
+
defaultA11yNodeSerializeParams,
|
|
768
|
+
opts
|
|
769
|
+
);
|
|
770
|
+
const indent = " ".repeat(indentLevel);
|
|
771
|
+
if (this.role === "StaticText") {
|
|
772
|
+
return `${indent}${this.name}
|
|
773
|
+
`;
|
|
774
|
+
}
|
|
775
|
+
let s = `${indent}<${this.role}`;
|
|
776
|
+
if (!noID && !rolesToOmitID.has(this.role)) {
|
|
777
|
+
s += ` id="${this.id}"`;
|
|
778
|
+
}
|
|
779
|
+
if (this.name) {
|
|
780
|
+
s += ` name="${this.name}"`;
|
|
781
|
+
}
|
|
782
|
+
if (this.content) {
|
|
783
|
+
s += ` content="${this.content}"`;
|
|
784
|
+
}
|
|
785
|
+
if (Object.keys(this.properties).length > 0 && !noProperties) {
|
|
786
|
+
Object.entries(this.properties).forEach(([k, v]) => {
|
|
787
|
+
if (bannedProperties.has(k)) {
|
|
788
|
+
return;
|
|
789
|
+
} else if (typeof v === "string") {
|
|
790
|
+
s += ` ${k}="${v}"`;
|
|
791
|
+
} else if (typeof v === "boolean") {
|
|
792
|
+
if (v) {
|
|
793
|
+
s += ` ${k}`;
|
|
794
|
+
} else {
|
|
795
|
+
s += ` ${k}={false}`;
|
|
796
|
+
}
|
|
797
|
+
} else if (typeof v !== "undefined") {
|
|
798
|
+
s += ` ${k}={${JSON.stringify(v)}}`;
|
|
799
|
+
}
|
|
800
|
+
});
|
|
801
|
+
}
|
|
802
|
+
if (this.children.length === 0 || noChildren) {
|
|
803
|
+
s += " />\n";
|
|
804
|
+
return s;
|
|
805
|
+
} else {
|
|
806
|
+
s += ">\n";
|
|
807
|
+
}
|
|
808
|
+
for (const child of this.children) {
|
|
809
|
+
s += child.serialize({ indentLevel: indentLevel + 2 });
|
|
810
|
+
}
|
|
811
|
+
s += `${indent}</${this.role}>
|
|
812
|
+
`;
|
|
813
|
+
return s;
|
|
814
|
+
}
|
|
815
|
+
};
|
|
816
|
+
var ProcessedA11yTree = class {
|
|
817
|
+
constructor(root, nodeMap) {
|
|
818
|
+
this.root = root;
|
|
819
|
+
this.nodeMap = nodeMap;
|
|
820
|
+
}
|
|
821
|
+
serialize() {
|
|
822
|
+
if (!this.root) {
|
|
823
|
+
return "";
|
|
824
|
+
}
|
|
825
|
+
return this.root.serialize();
|
|
826
|
+
}
|
|
827
|
+
// public diff(other: ProcessedA11yTree): string[] {
|
|
828
|
+
// const results: string[] = [];
|
|
829
|
+
// }
|
|
830
|
+
};
|
|
831
|
+
function getNodePathIdentifier(node) {
|
|
832
|
+
var _a, _b;
|
|
833
|
+
if ((_a = node.name) == null ? void 0 : _a.value) {
|
|
834
|
+
return `"${node.name.value}"`;
|
|
835
|
+
}
|
|
836
|
+
if (((_b = node.role) == null ? void 0 : _b.value) && node.role.value !== "none" && node.role.value !== "generic") {
|
|
837
|
+
return `"${node.role.value}"`;
|
|
838
|
+
}
|
|
839
|
+
return `"${node.nodeId}"`;
|
|
840
|
+
}
|
|
841
|
+
function processA11yTreeDFS(node, parent, inputNodeMap, outputNodeMap) {
|
|
842
|
+
var _a, _b, _c, _d, _e, _f;
|
|
843
|
+
if (!parent && node.parentId) {
|
|
844
|
+
throw new Error(
|
|
845
|
+
`Got no parent for accessibility node ${node.nodeId}: ${JSON.stringify(
|
|
846
|
+
node
|
|
847
|
+
)}`
|
|
848
|
+
);
|
|
849
|
+
}
|
|
850
|
+
const processedNode = new ProcessedA11yNode({
|
|
851
|
+
id: node.nodeId,
|
|
852
|
+
role: ((_a = node.role) == null ? void 0 : _a.value) || "",
|
|
853
|
+
name: ((_b = node.name) == null ? void 0 : _b.value) || "",
|
|
854
|
+
content: ((_c = node.value) == null ? void 0 : _c.value) || "",
|
|
855
|
+
properties: {},
|
|
856
|
+
children: [],
|
|
857
|
+
pathFromRoot: (parent ? `${parent.pathFromRoot} ` : "") + getNodePathIdentifier(node),
|
|
858
|
+
backendNodeID: node.backendDOMNodeId
|
|
859
|
+
// md5Sum: "",
|
|
860
|
+
});
|
|
861
|
+
if ((_d = node.value) == null ? void 0 : _d.value) {
|
|
862
|
+
processedNode.content = `${(_e = node.value) == null ? void 0 : _e.value}`;
|
|
863
|
+
}
|
|
864
|
+
if (node.properties) {
|
|
865
|
+
node.properties.forEach((prop) => {
|
|
866
|
+
processedNode.properties[prop.name] = prop.value.value;
|
|
867
|
+
});
|
|
868
|
+
}
|
|
869
|
+
outputNodeMap.set(processedNode.id, processedNode);
|
|
870
|
+
const children = node.childIds ?? [];
|
|
871
|
+
for (const childId of children) {
|
|
872
|
+
if (!childId) {
|
|
873
|
+
continue;
|
|
874
|
+
}
|
|
875
|
+
const child = inputNodeMap.get(childId);
|
|
876
|
+
if (!child) {
|
|
877
|
+
continue;
|
|
878
|
+
}
|
|
879
|
+
const processedChildren = processA11yTreeDFS(
|
|
880
|
+
child,
|
|
881
|
+
processedNode,
|
|
882
|
+
inputNodeMap,
|
|
883
|
+
outputNodeMap
|
|
884
|
+
);
|
|
885
|
+
if (!processedChildren.length) {
|
|
886
|
+
continue;
|
|
887
|
+
}
|
|
888
|
+
processedNode.children = processedNode.children.concat(processedChildren);
|
|
889
|
+
}
|
|
890
|
+
if (processedNode.role === "StaticText") {
|
|
891
|
+
processedNode.children = [];
|
|
892
|
+
}
|
|
893
|
+
if (processedNode.children.length === 1 && processedNode.children[0].role === "StaticText") {
|
|
894
|
+
const currentName = processedNode.name;
|
|
895
|
+
const childName = (_f = processedNode.children[0]) == null ? void 0 : _f.name;
|
|
896
|
+
if (currentName === childName || !childName) {
|
|
897
|
+
processedNode.children = [];
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
const staticTextGroupedChildren = [];
|
|
901
|
+
for (let i = processedNode.children.length - 1; i >= 0; i--) {
|
|
902
|
+
const node2 = processedNode.children[i];
|
|
903
|
+
if (node2.role !== "StaticText") {
|
|
904
|
+
staticTextGroupedChildren.push(node2);
|
|
905
|
+
continue;
|
|
906
|
+
}
|
|
907
|
+
if (i === 0 || processedNode.children[i - 1].role !== "StaticText") {
|
|
908
|
+
staticTextGroupedChildren.push(node2);
|
|
909
|
+
continue;
|
|
910
|
+
}
|
|
911
|
+
processedNode.children[i - 1].name += ` ${node2.name}`;
|
|
912
|
+
}
|
|
913
|
+
processedNode.children = staticTextGroupedChildren.reverse();
|
|
914
|
+
for (const child of processedNode.children) {
|
|
915
|
+
child.parent = processedNode;
|
|
916
|
+
}
|
|
917
|
+
const interesting = processedNode.isInteresting();
|
|
918
|
+
if (!interesting) {
|
|
919
|
+
if (processedNode.children.length === 0) {
|
|
920
|
+
return [];
|
|
921
|
+
} else if (processedNode.children.length === 1) {
|
|
922
|
+
return [processedNode.children[0]];
|
|
923
|
+
} else if (node.parentId) {
|
|
924
|
+
return processedNode.children;
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
return [processedNode];
|
|
928
|
+
}
|
|
929
|
+
function processA11yTree(graph) {
|
|
930
|
+
if (!graph.root) {
|
|
931
|
+
throw new Error("a11y tree has null root");
|
|
932
|
+
}
|
|
933
|
+
graph.allNodes = graph.allNodes.filter((node) => {
|
|
934
|
+
var _a;
|
|
935
|
+
if (!node.ignored) {
|
|
936
|
+
return true;
|
|
937
|
+
}
|
|
938
|
+
return !((_a = node.ignoredReasons) == null ? void 0 : _a.find(
|
|
939
|
+
(reason) => {
|
|
940
|
+
var _a2;
|
|
941
|
+
return reason.name === "notRendered" && ((_a2 = reason.value) == null ? void 0 : _a2.value);
|
|
942
|
+
}
|
|
943
|
+
));
|
|
944
|
+
});
|
|
945
|
+
const nodeMap = /* @__PURE__ */ new Map();
|
|
946
|
+
for (const node of graph.allNodes) {
|
|
947
|
+
nodeMap.set(node.nodeId, node);
|
|
948
|
+
}
|
|
949
|
+
const outputNodeMap = /* @__PURE__ */ new Map();
|
|
950
|
+
const processedRoot = processA11yTreeDFS(
|
|
951
|
+
graph.root,
|
|
952
|
+
null,
|
|
953
|
+
nodeMap,
|
|
954
|
+
outputNodeMap
|
|
955
|
+
);
|
|
956
|
+
if (processedRoot.length > 1) {
|
|
957
|
+
throw new Error(
|
|
958
|
+
`Something went horribly wrong processing the a11y tree, we got: ${JSON.stringify(
|
|
959
|
+
processedRoot
|
|
960
|
+
)}`
|
|
961
|
+
);
|
|
962
|
+
} else if (processedRoot.length === 0) {
|
|
963
|
+
throw new EmptyA11yTreeError();
|
|
964
|
+
}
|
|
965
|
+
return new ProcessedA11yTree(processedRoot[0], outputNodeMap);
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
// ../../packages/web-agent/src/browsers/cdp.ts
|
|
969
|
+
var GREEN = { r: 147, g: 196, b: 125, a: 0.55 };
|
|
970
|
+
var NODE_HIGHLIGHT_CONFIG = {
|
|
971
|
+
showInfo: false,
|
|
972
|
+
showRulers: false,
|
|
973
|
+
showStyles: false,
|
|
974
|
+
showAccessibilityInfo: false,
|
|
975
|
+
showExtensionLines: false,
|
|
976
|
+
contrastAlgorithm: "aa",
|
|
977
|
+
contentColor: GREEN,
|
|
978
|
+
paddingColor: GREEN,
|
|
979
|
+
borderColor: GREEN,
|
|
980
|
+
marginColor: GREEN,
|
|
981
|
+
eventTargetColor: GREEN,
|
|
982
|
+
shapeColor: GREEN,
|
|
983
|
+
shapeMarginColor: GREEN
|
|
984
|
+
};
|
|
985
|
+
|
|
986
|
+
// ../../packages/web-agent/src/browsers/constants.ts
|
|
987
|
+
var RETINA_WINDOW_SCALE_FACTOR = 2;
|
|
988
|
+
var MAX_LOAD_TIMEOUT_MS = 8e3;
|
|
989
|
+
var NETWORK_STABLE_DURATION_MS = 1250;
|
|
990
|
+
var NETWORK_IDLE_TIMEOUT_MS = 3e3;
|
|
991
|
+
var CHECK_INTERVAL_MS = 250;
|
|
992
|
+
var A11Y_LOAD_TIMEOUT_MS = 1e3;
|
|
993
|
+
var A11Y_STABLE_TIMEOUT_MS = NETWORK_IDLE_TIMEOUT_MS;
|
|
994
|
+
var A11Y_STABLE_DURATION_MS = NETWORK_STABLE_DURATION_MS;
|
|
995
|
+
var BROWSER_ACTION_TIMEOUT_MS = MAX_LOAD_TIMEOUT_MS;
|
|
996
|
+
var COMPLICATED_BROWSER_ACTION_TIMEOUT_MS = MAX_LOAD_TIMEOUT_MS;
|
|
997
|
+
var HIGHLIGHT_DURATION_MS = 3e3;
|
|
998
|
+
var CHROME_INTERNAL_URLS = /* @__PURE__ */ new Set([
|
|
999
|
+
"about:blank",
|
|
1000
|
+
"chrome-error://chromewebdata/"
|
|
1001
|
+
]);
|
|
1002
|
+
var MAX_BROWSER_ACTION_ATTEMPTS = 2;
|
|
1003
|
+
|
|
1004
|
+
// ../../packages/web-agent/src/browsers/utils/time.ts
|
|
1005
|
+
var sleep = (ms = 1e3) => {
|
|
1006
|
+
return new Promise((resolve) => setTimeout(() => resolve(), ms));
|
|
1007
|
+
};
|
|
1008
|
+
|
|
1009
|
+
// ../../packages/web-agent/src/browsers/utils/scripts/cursor.ts
|
|
1010
|
+
function addCursorScript() {
|
|
1011
|
+
cursor = document.createElement("img");
|
|
1012
|
+
cursor.setAttribute(
|
|
1013
|
+
"src",
|
|
1014
|
+
"data:image/svg+xml;base64,PHN2ZyBoZWlnaHQ9IjMyIiB2aWV3Qm94PSIwIDAgMzIgMzIiIHdpZHRoPSIzMiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48ZyBmaWxsPSJub25lIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiIHRyYW5zZm9ybT0idHJhbnNsYXRlKDEwIDcpIj48cGF0aCBkPSJtNi4xNDggMTguNDczIDEuODYzLTEuMDAzIDEuNjE1LS44MzktMi41NjgtNC44MTZoNC4zMzJsLTExLjM3OS0xMS40MDh2MTYuMDE1bDMuMzE2LTMuMjIxeiIgZmlsbD0iI2ZmZiIvPjxwYXRoIGQ9Im02LjQzMSAxNyAxLjc2NS0uOTQxLTIuNzc1LTUuMjAyaDMuNjA0bC04LjAyNS04LjA0M3YxMS4xODhsMi41My0yLjQ0MnoiIGZpbGw9IiMwMDAiLz48L2c+PC9zdmc+"
|
|
1015
|
+
);
|
|
1016
|
+
cursor.setAttribute("id", "selenium_cursor");
|
|
1017
|
+
cursor.setAttribute(
|
|
1018
|
+
"style",
|
|
1019
|
+
"position: absolute; z-index: 99999999999; pointer-events: none; left:0; top:0"
|
|
1020
|
+
);
|
|
1021
|
+
cursor.style.filter = "invert(0%) sepia(6%) saturate(24%) hue-rotate(315deg) brightness(89%) contrast(110%)";
|
|
1022
|
+
document.body.appendChild(cursor);
|
|
1023
|
+
document.onmousemove = function(e) {
|
|
1024
|
+
e = e || window.event;
|
|
1025
|
+
document.getElementById("selenium_cursor").style.left = e.pageX + "px";
|
|
1026
|
+
document.getElementById("selenium_cursor").style.top = e.pageY + "px";
|
|
1027
|
+
};
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
// ../../packages/web-agent/src/browsers/utils/scripts/addIDs.ts
|
|
1031
|
+
function addIDsScript() {
|
|
1032
|
+
const allElements = document.getElementsByTagName("*");
|
|
1033
|
+
let currentID = 1;
|
|
1034
|
+
for (let i = 0; i < allElements.length; i++) {
|
|
1035
|
+
const element = allElements[i];
|
|
1036
|
+
element == null ? void 0 : element.setAttribute("data-momentic-id", currentID);
|
|
1037
|
+
currentID++;
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
|
|
1041
|
+
// ../../packages/web-agent/src/browsers/utils/playwright.ts
|
|
1042
|
+
var sometimesRelevantResourceTypes = /* @__PURE__ */ new Set([
|
|
1043
|
+
"document",
|
|
1044
|
+
"script",
|
|
1045
|
+
"XMLHttpRequest",
|
|
1046
|
+
"fetch",
|
|
1047
|
+
"xhr"
|
|
1048
|
+
]);
|
|
1049
|
+
var alwaysRelevantResourceTypes = /* @__PURE__ */ new Set(["script", "document"]);
|
|
1050
|
+
var bannedDomains = [
|
|
1051
|
+
"intercom.io",
|
|
1052
|
+
"googletagmanager.com",
|
|
1053
|
+
"google-analytics.com",
|
|
1054
|
+
"www.gstatic.com",
|
|
1055
|
+
"apis.google.com",
|
|
1056
|
+
"sentry.io",
|
|
1057
|
+
"newrelic.com",
|
|
1058
|
+
"p.retool.com",
|
|
1059
|
+
"m.stripe.com",
|
|
1060
|
+
"m.stripe.network",
|
|
1061
|
+
"js.stripe.com",
|
|
1062
|
+
"assets.trybento.co",
|
|
1063
|
+
"udon.trybento.co",
|
|
1064
|
+
"cdn.lr-in-prod.com",
|
|
1065
|
+
"r.lr-in-prod.com",
|
|
1066
|
+
"content.product-usage.assembledhq.com",
|
|
1067
|
+
"data.product-usage.assembledhq.com",
|
|
1068
|
+
"static.zdassets.com"
|
|
1069
|
+
];
|
|
1070
|
+
function serializeRequest(request) {
|
|
1071
|
+
return `${request.resourceType()} ${request.method()} ${request.url()}`;
|
|
1072
|
+
}
|
|
1073
|
+
function stripWWWPrefix(url) {
|
|
1074
|
+
url = url.replace(/^www\./, "");
|
|
1075
|
+
return url;
|
|
1076
|
+
}
|
|
1077
|
+
function isRequestRelevantForPageLoad(request, currentURL) {
|
|
1078
|
+
if (!sometimesRelevantResourceTypes.has(request.resourceType())) {
|
|
1079
|
+
return false;
|
|
1080
|
+
}
|
|
1081
|
+
const parsedCurrentURL = new URL(currentURL);
|
|
1082
|
+
const parsedRequestURL = new URL(request.url());
|
|
1083
|
+
if (bannedDomains.some((domain) => parsedRequestURL.hostname.includes(domain))) {
|
|
1084
|
+
return false;
|
|
1085
|
+
}
|
|
1086
|
+
if (alwaysRelevantResourceTypes.has(request.resourceType())) {
|
|
1087
|
+
return true;
|
|
1088
|
+
}
|
|
1089
|
+
if (request.method() !== "GET") {
|
|
1090
|
+
return true;
|
|
1091
|
+
}
|
|
1092
|
+
return stripWWWPrefix(parsedRequestURL.hostname).includes(
|
|
1093
|
+
stripWWWPrefix(parsedCurrentURL.hostname)
|
|
1094
|
+
);
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
// ../../packages/web-agent/src/browsers/chrome.ts
|
|
1098
|
+
async function initCDPSession(cdpClient) {
|
|
1099
|
+
await cdpClient.send("Accessibility.enable");
|
|
1100
|
+
await cdpClient.send("DOM.enable");
|
|
1101
|
+
await cdpClient.send("Overlay.enable");
|
|
1102
|
+
}
|
|
1103
|
+
var ChromeBrowser = class _ChromeBrowser {
|
|
1104
|
+
browser;
|
|
1105
|
+
context;
|
|
1106
|
+
page;
|
|
1107
|
+
// key is nodeId, according to the a11y tree
|
|
1108
|
+
nodeMap = /* @__PURE__ */ new Map();
|
|
1109
|
+
cdpClient;
|
|
1110
|
+
logger;
|
|
1111
|
+
baseURL;
|
|
1112
|
+
constructor({
|
|
1113
|
+
browser,
|
|
1114
|
+
context,
|
|
1115
|
+
page,
|
|
1116
|
+
baseURL,
|
|
1117
|
+
cdpClient,
|
|
1118
|
+
logger
|
|
1119
|
+
}) {
|
|
1120
|
+
this.browser = browser;
|
|
1121
|
+
this.context = context;
|
|
1122
|
+
this.page = page;
|
|
1123
|
+
this.baseURL = baseURL;
|
|
1124
|
+
this.cdpClient = cdpClient;
|
|
1125
|
+
this.logger = logger;
|
|
1126
|
+
}
|
|
1127
|
+
static USER_AGENT = devices["Desktop Chrome"].userAgent;
|
|
1128
|
+
/**
|
|
1129
|
+
* Creates a new browser and waits for navigation to the given test URL.
|
|
1130
|
+
*/
|
|
1131
|
+
static async init(baseURL, logger, onScreenshot, timeout = MAX_LOAD_TIMEOUT_MS) {
|
|
1132
|
+
const browser = await chromium.launch({ headless: true });
|
|
1133
|
+
const context = await browser.newContext({
|
|
1134
|
+
viewport: {
|
|
1135
|
+
width: 1920,
|
|
1136
|
+
height: 1080
|
|
1137
|
+
},
|
|
1138
|
+
// comment out the below if you are on Mac OS but you're using a monitor
|
|
1139
|
+
deviceScaleFactor: process.platform === "darwin" ? RETINA_WINDOW_SCALE_FACTOR : 1,
|
|
1140
|
+
userAgent: devices["Desktop Chrome"].userAgent,
|
|
1141
|
+
geolocation: { latitude: 37.7749, longitude: -122.4194 },
|
|
1142
|
+
// san francisco
|
|
1143
|
+
locale: "en-US",
|
|
1144
|
+
timezoneId: "America/Los_Angeles"
|
|
1145
|
+
});
|
|
1146
|
+
const page = await context.newPage();
|
|
1147
|
+
const cdpClient = await context.newCDPSession(page);
|
|
1148
|
+
const chrome = new _ChromeBrowser({
|
|
1149
|
+
browser,
|
|
1150
|
+
context,
|
|
1151
|
+
page,
|
|
1152
|
+
baseURL,
|
|
1153
|
+
cdpClient,
|
|
1154
|
+
logger
|
|
1155
|
+
});
|
|
1156
|
+
let completed = false;
|
|
1157
|
+
const navigateAndInitCDP = async () => {
|
|
1158
|
+
try {
|
|
1159
|
+
await chrome.navigate(baseURL, false);
|
|
1160
|
+
await initCDPSession(cdpClient);
|
|
1161
|
+
} catch (err) {
|
|
1162
|
+
logger.error({ err }, "Failed to initialize chrome browser");
|
|
1163
|
+
} finally {
|
|
1164
|
+
completed = true;
|
|
1165
|
+
}
|
|
1166
|
+
};
|
|
1167
|
+
void navigateAndInitCDP();
|
|
1168
|
+
const sendScreenshot = async () => {
|
|
1169
|
+
if (!onScreenshot) {
|
|
1170
|
+
return;
|
|
1171
|
+
}
|
|
1172
|
+
try {
|
|
1173
|
+
onScreenshot({
|
|
1174
|
+
viewport: chrome.viewport,
|
|
1175
|
+
buffer: await chrome.screenshot()
|
|
1176
|
+
});
|
|
1177
|
+
} catch (err) {
|
|
1178
|
+
logger.error({ err }, "Failed to take screenshot");
|
|
1179
|
+
}
|
|
1180
|
+
};
|
|
1181
|
+
void sendScreenshot();
|
|
1182
|
+
const screenshotInterval = setInterval(() => {
|
|
1183
|
+
void sendScreenshot();
|
|
1184
|
+
}, 250);
|
|
1185
|
+
const startTime = Date.now();
|
|
1186
|
+
while (!completed && Date.now() - startTime < timeout) {
|
|
1187
|
+
await sleep(CHECK_INTERVAL_MS);
|
|
1188
|
+
}
|
|
1189
|
+
clearInterval(screenshotInterval);
|
|
1190
|
+
if (!completed) {
|
|
1191
|
+
logger.warn(
|
|
1192
|
+
"Timeout elapsed waiting for browser to initialize - are you sure this page is accessible?"
|
|
1193
|
+
);
|
|
1194
|
+
}
|
|
1195
|
+
return chrome;
|
|
1196
|
+
}
|
|
1197
|
+
// Things to do on every page load
|
|
1198
|
+
async pageSetup() {
|
|
1199
|
+
await this.page.evaluate(addCursorScript);
|
|
1200
|
+
await this.page.evaluate(addIDsScript);
|
|
1201
|
+
}
|
|
1202
|
+
async wait(timeoutMs) {
|
|
1203
|
+
await this.page.waitForTimeout(timeoutMs);
|
|
1204
|
+
}
|
|
1205
|
+
async cleanup() {
|
|
1206
|
+
await this.page.close();
|
|
1207
|
+
await this.context.close();
|
|
1208
|
+
await this.browser.close();
|
|
1209
|
+
}
|
|
1210
|
+
get closed() {
|
|
1211
|
+
return this.page.isClosed() || !this.browser.isConnected();
|
|
1212
|
+
}
|
|
1213
|
+
async html() {
|
|
1214
|
+
return await this.page.content();
|
|
1215
|
+
}
|
|
1216
|
+
get url() {
|
|
1217
|
+
return this.page.url();
|
|
1218
|
+
}
|
|
1219
|
+
async screenshot(quality = 100, scale = "device") {
|
|
1220
|
+
return await this.page.screenshot({
|
|
1221
|
+
fullPage: false,
|
|
1222
|
+
quality,
|
|
1223
|
+
scale,
|
|
1224
|
+
type: "jpeg",
|
|
1225
|
+
// allow the blinking text cursor thing to remain there
|
|
1226
|
+
caret: "initial"
|
|
1227
|
+
});
|
|
1228
|
+
}
|
|
1229
|
+
get viewport() {
|
|
1230
|
+
const viewport = this.page.viewportSize();
|
|
1231
|
+
if (!viewport) {
|
|
1232
|
+
throw new Error("failed to get viewport");
|
|
1233
|
+
}
|
|
1234
|
+
return viewport;
|
|
1235
|
+
}
|
|
1236
|
+
async navigate(url, wrapPossibleNavigation = true) {
|
|
1237
|
+
this.logger.debug(`Navigating to ${url}`);
|
|
1238
|
+
const startTime = Date.now();
|
|
1239
|
+
const doNav = async () => {
|
|
1240
|
+
try {
|
|
1241
|
+
await this.page.goto(url, {
|
|
1242
|
+
timeout: MAX_LOAD_TIMEOUT_MS
|
|
1243
|
+
});
|
|
1244
|
+
this.logger.debug(
|
|
1245
|
+
{ url },
|
|
1246
|
+
`Got load event in ${Math.floor(Date.now() - startTime)}ms`
|
|
1247
|
+
);
|
|
1248
|
+
} catch (e) {
|
|
1249
|
+
this.logger.warn(
|
|
1250
|
+
{ url, type: "navigate", err: e },
|
|
1251
|
+
"Timeout elapsed waiting for page to load, continuing anyways..."
|
|
1252
|
+
);
|
|
1253
|
+
}
|
|
1254
|
+
};
|
|
1255
|
+
if (wrapPossibleNavigation) {
|
|
1256
|
+
await this.wrapPossibleNavigation(doNav);
|
|
1257
|
+
} else {
|
|
1258
|
+
await doNav();
|
|
1259
|
+
}
|
|
1260
|
+
if (CHROME_INTERNAL_URLS.has(this.url) && process.env.NODE_ENV === "production") {
|
|
1261
|
+
throw new Error(
|
|
1262
|
+
`${url} took too long to load \u{1F61E}. Please ensure the site and your internet are working.`
|
|
1263
|
+
);
|
|
1264
|
+
}
|
|
1265
|
+
await this.pageSetup();
|
|
1266
|
+
this.logger.debug({ url }, "Navigation complete");
|
|
1267
|
+
}
|
|
1268
|
+
async fill(target, text, options = {}) {
|
|
1269
|
+
const element = await this.click(target, {
|
|
1270
|
+
doubleClick: false,
|
|
1271
|
+
rightClick: false
|
|
1272
|
+
});
|
|
1273
|
+
await this.type(text, options);
|
|
1274
|
+
return element;
|
|
1275
|
+
}
|
|
1276
|
+
async type(text, options = {}) {
|
|
1277
|
+
const { clearContent = true, pressKeysSequentially = false } = options;
|
|
1278
|
+
if (clearContent) {
|
|
1279
|
+
await this.page.keyboard.press("Meta+A");
|
|
1280
|
+
await this.page.keyboard.press("Backspace");
|
|
1281
|
+
}
|
|
1282
|
+
if (pressKeysSequentially) {
|
|
1283
|
+
await this.page.keyboard.type(text);
|
|
1284
|
+
} else {
|
|
1285
|
+
await this.page.keyboard.insertText(text);
|
|
1286
|
+
}
|
|
1287
|
+
}
|
|
1288
|
+
async clickByA11yID(index, options = {}) {
|
|
1289
|
+
const node = this.nodeMap.get(`${index}`);
|
|
1290
|
+
if (!node) {
|
|
1291
|
+
throw new Error(`Could not find node in DOM with index: ${index}`);
|
|
1292
|
+
}
|
|
1293
|
+
const nodeClicked = await this.clickUsingCDP(node, options);
|
|
1294
|
+
await this.highlightNode(nodeClicked);
|
|
1295
|
+
return node.serialize({ noChildren: true, noProperties: true, noID: true });
|
|
1296
|
+
}
|
|
1297
|
+
async selectOptionByA11yID(index, option) {
|
|
1298
|
+
const node = this.nodeMap.get(`${index}`);
|
|
1299
|
+
if (!node) {
|
|
1300
|
+
throw new Error(`Could not find node in DOM with index: ${index}`);
|
|
1301
|
+
}
|
|
1302
|
+
if (!node.backendNodeID) {
|
|
1303
|
+
throw new Error(
|
|
1304
|
+
`Select target missing backend node id: ${node.getLogForm()}`
|
|
1305
|
+
);
|
|
1306
|
+
}
|
|
1307
|
+
const locator = await this.getLocatorFromBackendID(node.backendNodeID);
|
|
1308
|
+
await locator.selectOption(option, {
|
|
1309
|
+
timeout: COMPLICATED_BROWSER_ACTION_TIMEOUT_MS
|
|
1310
|
+
});
|
|
1311
|
+
await this.highlightNode(node);
|
|
1312
|
+
return node.serialize({ noChildren: true, noProperties: true, noID: true });
|
|
1313
|
+
}
|
|
1314
|
+
async highlight(target) {
|
|
1315
|
+
try {
|
|
1316
|
+
await this.highlightByA11yID(target.id);
|
|
1317
|
+
} catch (err) {
|
|
1318
|
+
this.logger.warn({ err, target }, "Failed to highlight target");
|
|
1319
|
+
}
|
|
1320
|
+
}
|
|
1321
|
+
async highlightByA11yID(index) {
|
|
1322
|
+
const node = this.nodeMap.get(`${index}`);
|
|
1323
|
+
if (!node) {
|
|
1324
|
+
throw new Error(`Could not find node in DOM with index: ${index}`);
|
|
1325
|
+
}
|
|
1326
|
+
if (!node.backendNodeID) {
|
|
1327
|
+
throw new Error(
|
|
1328
|
+
`Select target missing backend node id: ${node.getLogForm()}`
|
|
1329
|
+
);
|
|
1330
|
+
}
|
|
1331
|
+
await this.highlightNode(node);
|
|
1332
|
+
}
|
|
1333
|
+
async highlightNode(node) {
|
|
1334
|
+
try {
|
|
1335
|
+
await this.cdpClient.send("Overlay.highlightNode", {
|
|
1336
|
+
highlightConfig: NODE_HIGHLIGHT_CONFIG,
|
|
1337
|
+
backendNodeId: node.backendNodeID
|
|
1338
|
+
});
|
|
1339
|
+
} catch (err) {
|
|
1340
|
+
this.logger.warn({ err }, "Failed to add node highlight");
|
|
1341
|
+
}
|
|
1342
|
+
const hideHighlight = async () => {
|
|
1343
|
+
try {
|
|
1344
|
+
await this.cdpClient.send("Overlay.hideHighlight", {
|
|
1345
|
+
backendNodeId: node.backendNodeID
|
|
1346
|
+
});
|
|
1347
|
+
} catch (err) {
|
|
1348
|
+
this.logger.debug({ err }, "Failed to remove node highlight");
|
|
1349
|
+
}
|
|
1350
|
+
};
|
|
1351
|
+
setTimeout(() => {
|
|
1352
|
+
void hideHighlight();
|
|
1353
|
+
}, HIGHLIGHT_DURATION_MS);
|
|
1354
|
+
}
|
|
1355
|
+
async wrapPossibleNavigation(fn, timeoutMS = MAX_LOAD_TIMEOUT_MS) {
|
|
1356
|
+
const startTime = Date.now();
|
|
1357
|
+
const startURL = this.url;
|
|
1358
|
+
let lastRequestReceived = Date.now();
|
|
1359
|
+
const firedRequests = /* @__PURE__ */ new Map();
|
|
1360
|
+
const finishedRequests = /* @__PURE__ */ new Map();
|
|
1361
|
+
const requestFinishedListener = (request) => {
|
|
1362
|
+
const key = serializeRequest(request);
|
|
1363
|
+
finishedRequests.set(key, (finishedRequests.get(key) ?? 0) + 1);
|
|
1364
|
+
};
|
|
1365
|
+
const requestFiredListener = (request) => {
|
|
1366
|
+
if (!isRequestRelevantForPageLoad(request, this.url)) {
|
|
1367
|
+
this.logger.debug(
|
|
1368
|
+
{
|
|
1369
|
+
uri: serializeRequest(request)
|
|
1370
|
+
},
|
|
1371
|
+
"Ignoring request for page load network stability"
|
|
1372
|
+
);
|
|
1373
|
+
return;
|
|
1374
|
+
}
|
|
1375
|
+
const key = serializeRequest(request);
|
|
1376
|
+
this.logger.debug(
|
|
1377
|
+
{
|
|
1378
|
+
uri: key
|
|
1379
|
+
},
|
|
1380
|
+
"Request fired on page load, delaying network stability"
|
|
1381
|
+
);
|
|
1382
|
+
firedRequests.set(key, (firedRequests.get(key) ?? 0) + 1);
|
|
1383
|
+
lastRequestReceived = Date.now();
|
|
1384
|
+
};
|
|
1385
|
+
this.page.on("requestfinished", requestFinishedListener);
|
|
1386
|
+
this.page.on("request", requestFiredListener);
|
|
1387
|
+
let rejected = false;
|
|
1388
|
+
const retPromise = fn().catch((e) => {
|
|
1389
|
+
rejected = true;
|
|
1390
|
+
if (e instanceof Error)
|
|
1391
|
+
return e;
|
|
1392
|
+
return new Error(`${e}`);
|
|
1393
|
+
});
|
|
1394
|
+
await sleep(CHECK_INTERVAL_MS);
|
|
1395
|
+
const unwrapAndThrowError = async (p) => {
|
|
1396
|
+
const v = await p;
|
|
1397
|
+
if (v instanceof Error) {
|
|
1398
|
+
throw v;
|
|
1399
|
+
}
|
|
1400
|
+
return v;
|
|
1401
|
+
};
|
|
1402
|
+
let unfinishedRequests = /* @__PURE__ */ new Set();
|
|
1403
|
+
const waitForNetworkIdle = async () => {
|
|
1404
|
+
while (!rejected && Date.now() - startTime < timeoutMS) {
|
|
1405
|
+
unfinishedRequests = /* @__PURE__ */ new Set();
|
|
1406
|
+
await sleep(CHECK_INTERVAL_MS);
|
|
1407
|
+
if (Date.now() - lastRequestReceived <= NETWORK_STABLE_DURATION_MS) {
|
|
1408
|
+
continue;
|
|
1409
|
+
}
|
|
1410
|
+
let anyDifference = false;
|
|
1411
|
+
for (const key of firedRequests.keys()) {
|
|
1412
|
+
if (firedRequests.get(key) !== finishedRequests.get(key)) {
|
|
1413
|
+
this.logger.debug({ uri: key }, "Waiting on request to finish");
|
|
1414
|
+
anyDifference = true;
|
|
1415
|
+
unfinishedRequests.add(key);
|
|
1416
|
+
}
|
|
1417
|
+
}
|
|
1418
|
+
if (!anyDifference) {
|
|
1419
|
+
this.logger.debug(
|
|
1420
|
+
{
|
|
1421
|
+
url: this.url,
|
|
1422
|
+
requests: JSON.stringify(Array.from(firedRequests.entries()))
|
|
1423
|
+
},
|
|
1424
|
+
`Network idle in ${Math.floor(Date.now() - startTime)}ms`
|
|
1425
|
+
);
|
|
1426
|
+
return true;
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
if (!rejected) {
|
|
1430
|
+
this.logger.warn(
|
|
1431
|
+
{
|
|
1432
|
+
url: this.url,
|
|
1433
|
+
requests: JSON.stringify(Array.from(unfinishedRequests.entries()))
|
|
1434
|
+
},
|
|
1435
|
+
"Timeout elapsed waiting for network idle, continuing anyways..."
|
|
1436
|
+
);
|
|
1437
|
+
}
|
|
1438
|
+
return false;
|
|
1439
|
+
};
|
|
1440
|
+
const waitResult = await waitForNetworkIdle();
|
|
1441
|
+
this.page.off("requestfinished", requestFinishedListener);
|
|
1442
|
+
this.page.off("request", requestFiredListener);
|
|
1443
|
+
if (!waitResult) {
|
|
1444
|
+
return unwrapAndThrowError(retPromise);
|
|
1445
|
+
}
|
|
1446
|
+
if (!rejected && urlChanged(this.url, startURL)) {
|
|
1447
|
+
this.logger.debug(
|
|
1448
|
+
`Detected url change in wrapPossibleNavigation, waiting for load state`
|
|
1449
|
+
);
|
|
1450
|
+
try {
|
|
1451
|
+
await this.page.waitForLoadState("load", {
|
|
1452
|
+
timeout: timeoutMS - (Date.now() - startTime)
|
|
1453
|
+
});
|
|
1454
|
+
} catch (e) {
|
|
1455
|
+
this.logger.warn(
|
|
1456
|
+
{ url: this.url },
|
|
1457
|
+
"Timeout elapsed waiting for load state to fire, continuing anyways..."
|
|
1458
|
+
);
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
return unwrapAndThrowError(retPromise);
|
|
1462
|
+
}
|
|
1463
|
+
async click(target, options = {}) {
|
|
1464
|
+
const elementInteracted = await this.wrapPossibleNavigation(
|
|
1465
|
+
() => this.clickByA11yID(target.id, options)
|
|
1466
|
+
);
|
|
1467
|
+
return elementInteracted;
|
|
1468
|
+
}
|
|
1469
|
+
async selectOption(target, option) {
|
|
1470
|
+
return this.selectOptionByA11yID(target.id, option);
|
|
1471
|
+
}
|
|
1472
|
+
async press(key) {
|
|
1473
|
+
await this.wrapPossibleNavigation(() => this.page.keyboard.press(key));
|
|
1474
|
+
}
|
|
1475
|
+
async refresh() {
|
|
1476
|
+
await this.page.reload();
|
|
1477
|
+
await this.pageSetup();
|
|
1478
|
+
}
|
|
1479
|
+
async getA11yTree() {
|
|
1480
|
+
let processedTree = null;
|
|
1481
|
+
let attempt = 0;
|
|
1482
|
+
const url = this.url;
|
|
1483
|
+
while (!processedTree) {
|
|
1484
|
+
try {
|
|
1485
|
+
this.logger.debug(`Getting a11y tree at ${url}`);
|
|
1486
|
+
const graph = await this.getRawA11yTree();
|
|
1487
|
+
if (!graph.root || graph.allNodes.length === 0) {
|
|
1488
|
+
throw new Error("No a11y tree found on page");
|
|
1489
|
+
}
|
|
1490
|
+
processedTree = processA11yTree(graph);
|
|
1491
|
+
} catch (e) {
|
|
1492
|
+
this.logger.error({ err: e, url }, "Error fetching a11y tree");
|
|
1493
|
+
if (attempt === 0) {
|
|
1494
|
+
await sleep(1e3);
|
|
1495
|
+
attempt++;
|
|
1496
|
+
} else {
|
|
1497
|
+
throw new Error(`Max retries exceeded fetching a11y tree: ${e}`);
|
|
1498
|
+
}
|
|
1499
|
+
}
|
|
1500
|
+
}
|
|
1501
|
+
if (!processedTree.root) {
|
|
1502
|
+
this.logger.warn("A11y tree was pruned entirely");
|
|
1503
|
+
}
|
|
1504
|
+
this.nodeMap = processedTree.nodeMap;
|
|
1505
|
+
return processedTree;
|
|
1506
|
+
}
|
|
1507
|
+
async getRawA11yTree() {
|
|
1508
|
+
const url = this.page.url();
|
|
1509
|
+
let lastTreeUpdateTimestamp = Date.now();
|
|
1510
|
+
const treeUpdateListener = () => {
|
|
1511
|
+
lastTreeUpdateTimestamp = Date.now();
|
|
1512
|
+
};
|
|
1513
|
+
this.cdpClient.addListener(
|
|
1514
|
+
"Accessibility.nodesUpdated",
|
|
1515
|
+
treeUpdateListener
|
|
1516
|
+
);
|
|
1517
|
+
let accessibilityTreeLoadFired = false;
|
|
1518
|
+
const accessibilityLoadListener = () => {
|
|
1519
|
+
this.logger.info({ url }, `A11y tree load event fired`);
|
|
1520
|
+
accessibilityTreeLoadFired = true;
|
|
1521
|
+
};
|
|
1522
|
+
this.cdpClient.addListener(
|
|
1523
|
+
"Accessibility.loadComplete",
|
|
1524
|
+
accessibilityLoadListener
|
|
1525
|
+
);
|
|
1526
|
+
const a11yLoadStart = Date.now();
|
|
1527
|
+
let timeoutTriggered = true;
|
|
1528
|
+
while (Date.now() - a11yLoadStart < A11Y_STABLE_TIMEOUT_MS) {
|
|
1529
|
+
await sleep(CHECK_INTERVAL_MS);
|
|
1530
|
+
if (!accessibilityTreeLoadFired && Date.now() - a11yLoadStart < A11Y_LOAD_TIMEOUT_MS) {
|
|
1531
|
+
this.logger.debug({ url }, `A11y tree not loaded yet, waiting...`);
|
|
1532
|
+
continue;
|
|
1533
|
+
}
|
|
1534
|
+
if (Date.now() - lastTreeUpdateTimestamp >= A11Y_STABLE_DURATION_MS) {
|
|
1535
|
+
this.logger.debug({ url }, `A11y tree not stable yet, waiting...`);
|
|
1536
|
+
continue;
|
|
1537
|
+
}
|
|
1538
|
+
timeoutTriggered = false;
|
|
1539
|
+
break;
|
|
1540
|
+
}
|
|
1541
|
+
this.logger.debug(
|
|
1542
|
+
{
|
|
1543
|
+
duration: Date.now() - a11yLoadStart,
|
|
1544
|
+
eventReceived: accessibilityTreeLoadFired,
|
|
1545
|
+
timeoutTriggered
|
|
1546
|
+
},
|
|
1547
|
+
"A11y wait phase completed"
|
|
1548
|
+
);
|
|
1549
|
+
const { node: root } = await this.cdpClient.send(
|
|
1550
|
+
"Accessibility.getRootAXNode"
|
|
1551
|
+
);
|
|
1552
|
+
const { nodes } = await this.cdpClient.send("Accessibility.queryAXTree", {
|
|
1553
|
+
backendNodeId: root.backendDOMNodeId
|
|
1554
|
+
});
|
|
1555
|
+
this.cdpClient.removeListener(
|
|
1556
|
+
"Accessibility.loadComplete",
|
|
1557
|
+
accessibilityLoadListener
|
|
1558
|
+
);
|
|
1559
|
+
this.cdpClient.removeListener(
|
|
1560
|
+
"Accessibility.nodesUpdated",
|
|
1561
|
+
treeUpdateListener
|
|
1562
|
+
);
|
|
1563
|
+
return {
|
|
1564
|
+
root,
|
|
1565
|
+
allNodes: nodes
|
|
1566
|
+
};
|
|
1567
|
+
}
|
|
1568
|
+
async clickUsingVisualCoordinates(backendNodeId) {
|
|
1569
|
+
const location = await this.getElementLocation(backendNodeId);
|
|
1570
|
+
if (!location) {
|
|
1571
|
+
throw new Error(
|
|
1572
|
+
`Could not find element location with backend node id: ${backendNodeId}`
|
|
1573
|
+
);
|
|
1574
|
+
}
|
|
1575
|
+
this.logger.debug({ location }, "Executing mouse click");
|
|
1576
|
+
await this.page.mouse.click(location.centerX, location.centerY);
|
|
1577
|
+
}
|
|
1578
|
+
// Get the "id" attribute value from an HTML element.
|
|
1579
|
+
async getIDAttributeUsingCDP(objectId) {
|
|
1580
|
+
await this.cdpClient.send("DOM.getDocument", { depth: 0 });
|
|
1581
|
+
const cdpNodeResult = await this.cdpClient.send("DOM.requestNode", {
|
|
1582
|
+
objectId
|
|
1583
|
+
});
|
|
1584
|
+
const attrResult = await this.cdpClient.send("DOM.getAttributes", {
|
|
1585
|
+
nodeId: cdpNodeResult.nodeId
|
|
1586
|
+
});
|
|
1587
|
+
const attributes = attrResult.attributes;
|
|
1588
|
+
const indexAttr = attributes.findIndex((s) => s === "data-momentic-id");
|
|
1589
|
+
if (indexAttr === -1) {
|
|
1590
|
+
return "";
|
|
1591
|
+
}
|
|
1592
|
+
return attributes[indexAttr + 1] || "";
|
|
1593
|
+
}
|
|
1594
|
+
async getLocatorFromBackendID(backendNodeId) {
|
|
1595
|
+
await this.page.evaluate(addIDsScript);
|
|
1596
|
+
const cdpResolveResult = await this.cdpClient.send("DOM.resolveNode", {
|
|
1597
|
+
backendNodeId
|
|
1598
|
+
});
|
|
1599
|
+
if (!cdpResolveResult || !cdpResolveResult.object.objectId) {
|
|
1600
|
+
throw new Error(`Could not resolve backend node ${backendNodeId}`);
|
|
1601
|
+
}
|
|
1602
|
+
try {
|
|
1603
|
+
const id = await this.getIDAttributeUsingCDP(
|
|
1604
|
+
cdpResolveResult.object.objectId
|
|
1605
|
+
);
|
|
1606
|
+
if (!id) {
|
|
1607
|
+
throw new Error("Failed getting data-momentic-id attribute using CDP");
|
|
1608
|
+
}
|
|
1609
|
+
return this.page.locator(`[data-momentic-id="${id}"]`);
|
|
1610
|
+
} catch (err) {
|
|
1611
|
+
this.logger.error(
|
|
1612
|
+
{
|
|
1613
|
+
err
|
|
1614
|
+
},
|
|
1615
|
+
"Failed to get ID attribute"
|
|
1616
|
+
);
|
|
1617
|
+
throw err;
|
|
1618
|
+
}
|
|
1619
|
+
}
|
|
1620
|
+
async clickUsingCDP(originalNode, options = {}) {
|
|
1621
|
+
let clickAttempts = 0;
|
|
1622
|
+
let candidateNode = originalNode;
|
|
1623
|
+
while (clickAttempts < MAX_BROWSER_ACTION_ATTEMPTS) {
|
|
1624
|
+
if (!candidateNode || candidateNode.role === "RootWebArea") {
|
|
1625
|
+
throw new Error(
|
|
1626
|
+
`Attempted to click node with no clickable surrounding elements: ${originalNode.getLogForm()}`
|
|
1627
|
+
);
|
|
1628
|
+
}
|
|
1629
|
+
if (candidateNode.role === "StaticText") {
|
|
1630
|
+
candidateNode = candidateNode.parent;
|
|
1631
|
+
continue;
|
|
1632
|
+
}
|
|
1633
|
+
const candidateNodeID = candidateNode.backendNodeID;
|
|
1634
|
+
if (!candidateNodeID) {
|
|
1635
|
+
this.logger.warn(
|
|
1636
|
+
{ node: candidateNode.getLogForm() },
|
|
1637
|
+
"Click candidate had no backend node ID"
|
|
1638
|
+
);
|
|
1639
|
+
candidateNode = candidateNode.parent;
|
|
1640
|
+
continue;
|
|
1641
|
+
}
|
|
1642
|
+
try {
|
|
1643
|
+
const locator = await this.getLocatorFromBackendID(candidateNodeID);
|
|
1644
|
+
if (options.doubleClick) {
|
|
1645
|
+
await locator.dblclick({
|
|
1646
|
+
timeout: BROWSER_ACTION_TIMEOUT_MS
|
|
1647
|
+
});
|
|
1648
|
+
} else {
|
|
1649
|
+
await locator.click({
|
|
1650
|
+
timeout: BROWSER_ACTION_TIMEOUT_MS,
|
|
1651
|
+
button: options.rightClick ? "right" : "left"
|
|
1652
|
+
});
|
|
1653
|
+
}
|
|
1654
|
+
if (candidateNode.id !== originalNode.id) {
|
|
1655
|
+
this.logger.info(
|
|
1656
|
+
{
|
|
1657
|
+
oldNode: originalNode.getLogForm(),
|
|
1658
|
+
newNode: candidateNode.getLogForm()
|
|
1659
|
+
},
|
|
1660
|
+
`Redirected click successfully to new element`
|
|
1661
|
+
);
|
|
1662
|
+
}
|
|
1663
|
+
return candidateNode;
|
|
1664
|
+
} catch (err) {
|
|
1665
|
+
this.logger.error(
|
|
1666
|
+
{ err, node: candidateNode.getLogForm() },
|
|
1667
|
+
"Failed click or click timed out"
|
|
1668
|
+
);
|
|
1669
|
+
clickAttempts++;
|
|
1670
|
+
candidateNode = candidateNode.parent;
|
|
1671
|
+
}
|
|
1672
|
+
}
|
|
1673
|
+
throw new Error(
|
|
1674
|
+
`Max click redirection attempts exhausted on original element: ${originalNode.getLogForm()}`
|
|
1675
|
+
);
|
|
1676
|
+
}
|
|
1677
|
+
/**
|
|
1678
|
+
* Currently unused, but could be useful for vision model integration.
|
|
1679
|
+
* Gets x/y position of an a11y node.
|
|
1680
|
+
*/
|
|
1681
|
+
async getElementLocation(backendNodeId) {
|
|
1682
|
+
const tree = await this.cdpClient.send("DOMSnapshot.captureSnapshot", {
|
|
1683
|
+
computedStyles: [],
|
|
1684
|
+
includeDOMRects: true,
|
|
1685
|
+
includePaintOrder: true
|
|
1686
|
+
});
|
|
1687
|
+
let devicePixelRatio = await this.page.evaluate(
|
|
1688
|
+
() => window.devicePixelRatio
|
|
1689
|
+
);
|
|
1690
|
+
if (process.platform === "darwin" && devicePixelRatio === 1) {
|
|
1691
|
+
devicePixelRatio = RETINA_WINDOW_SCALE_FACTOR;
|
|
1692
|
+
}
|
|
1693
|
+
const document2 = tree["documents"][0];
|
|
1694
|
+
const layout = document2["layout"];
|
|
1695
|
+
const nodes = document2["nodes"];
|
|
1696
|
+
const nodeNames = nodes["nodeName"] || [];
|
|
1697
|
+
const backendNodeIds = nodes["backendNodeId"] || [];
|
|
1698
|
+
const layoutNodeIndex = layout["nodeIndex"];
|
|
1699
|
+
const bounds = layout["bounds"];
|
|
1700
|
+
let cursor2 = -1;
|
|
1701
|
+
for (let i = 0; i < nodeNames.length; i++) {
|
|
1702
|
+
if (backendNodeIds[i] === backendNodeId) {
|
|
1703
|
+
cursor2 = layoutNodeIndex.indexOf(i);
|
|
1704
|
+
break;
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1707
|
+
if (cursor2 === -1) {
|
|
1708
|
+
throw new Error(
|
|
1709
|
+
`Could not find any backend node with ID ${backendNodeId}`
|
|
1710
|
+
);
|
|
1711
|
+
}
|
|
1712
|
+
let [x = 0, y = 0, width = 0, height = 0] = bounds[cursor2];
|
|
1713
|
+
x /= devicePixelRatio;
|
|
1714
|
+
y /= devicePixelRatio;
|
|
1715
|
+
width /= devicePixelRatio;
|
|
1716
|
+
height /= devicePixelRatio;
|
|
1717
|
+
const centerX = x + width / 2;
|
|
1718
|
+
const centerY = y + height / 2;
|
|
1719
|
+
return { centerX, centerY };
|
|
1720
|
+
}
|
|
1721
|
+
async scrollUp() {
|
|
1722
|
+
await this.page.evaluate(() => {
|
|
1723
|
+
(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop - window.innerHeight;
|
|
1724
|
+
});
|
|
1725
|
+
await this.page.evaluate(() => {
|
|
1726
|
+
(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop + window.innerHeight;
|
|
1727
|
+
});
|
|
1728
|
+
}
|
|
1729
|
+
async scrollDown() {
|
|
1730
|
+
await this.page.evaluate(() => {
|
|
1731
|
+
(document.scrollingElement || document.body).scrollTop = (document.scrollingElement || document.body).scrollTop + window.innerHeight;
|
|
1732
|
+
});
|
|
1733
|
+
}
|
|
1734
|
+
async goForward() {
|
|
1735
|
+
await this.wrapPossibleNavigation(
|
|
1736
|
+
() => this.page.goForward({ timeout: MAX_LOAD_TIMEOUT_MS })
|
|
1737
|
+
);
|
|
1738
|
+
await this.pageSetup();
|
|
1739
|
+
}
|
|
1740
|
+
async goBack() {
|
|
1741
|
+
await this.wrapPossibleNavigation(
|
|
1742
|
+
() => this.page.goBack({ timeout: MAX_LOAD_TIMEOUT_MS })
|
|
1743
|
+
);
|
|
1744
|
+
await this.pageSetup();
|
|
1745
|
+
}
|
|
1746
|
+
async switchToPage(urlSubstring) {
|
|
1747
|
+
const allPages = await this.context.pages();
|
|
1748
|
+
for (let i = 0; i < allPages.length; i++) {
|
|
1749
|
+
const page = allPages[i];
|
|
1750
|
+
if (page.url().includes(urlSubstring)) {
|
|
1751
|
+
this.page = page;
|
|
1752
|
+
await page.waitForLoadState("load", {
|
|
1753
|
+
timeout: MAX_LOAD_TIMEOUT_MS
|
|
1754
|
+
});
|
|
1755
|
+
await this.pageSetup();
|
|
1756
|
+
this.cdpClient = await this.context.newCDPSession(page);
|
|
1757
|
+
await initCDPSession(this.cdpClient);
|
|
1758
|
+
this.logger.info(`Switching to tab ${i} with url ${page.url()}`);
|
|
1759
|
+
return;
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
throw new Error(`Could not find page with url containing ${urlSubstring}`);
|
|
1763
|
+
}
|
|
1764
|
+
async setCookie(cookie) {
|
|
1765
|
+
const cookieSettings = parseCookieString(cookie);
|
|
1766
|
+
await this.context.addCookies([cookieSettings]);
|
|
1767
|
+
}
|
|
1768
|
+
};
|
|
1769
|
+
|
|
1770
|
+
// ../../packages/web-agent/src/configs/controller.ts
|
|
1771
|
+
var A11Y_CONTROLLER_CONFIG = {
|
|
1772
|
+
type: "a11y",
|
|
1773
|
+
version: "1.0.0",
|
|
1774
|
+
useHistory: "diff",
|
|
1775
|
+
useGoalSplitter: true
|
|
1776
|
+
};
|
|
1777
|
+
var DEFAULT_CONTROLLER_CONFIG = A11Y_CONTROLLER_CONFIG;
|
|
1778
|
+
|
|
1779
|
+
// ../../packages/web-agent/src/controller.ts
|
|
1780
|
+
import dedent2 from "dedent";
|
|
1781
|
+
import diffLines from "diff-lines";
|
|
1782
|
+
var MAX_HISTORY_CHAR_LENGTH = 1e4;
|
|
1783
|
+
var AgentController = class {
|
|
1784
|
+
// Instance of browser to interact with
|
|
1785
|
+
browser;
|
|
1786
|
+
// Stack of queued-up instructions
|
|
1787
|
+
pendingInstructions;
|
|
1788
|
+
// manager for all AI generation
|
|
1789
|
+
generator;
|
|
1790
|
+
// Stack of commands previously executed.
|
|
1791
|
+
// Top of stack can be a pending command that hasn't been executed yet.
|
|
1792
|
+
// Should not contain intermediate successes due to granular commands.
|
|
1793
|
+
commandHistory;
|
|
1794
|
+
config;
|
|
1795
|
+
logger;
|
|
1796
|
+
constructor({ browser, config, generator, logger }) {
|
|
1797
|
+
this.browser = browser;
|
|
1798
|
+
this.generator = generator;
|
|
1799
|
+
this.config = config;
|
|
1800
|
+
this.logger = logger;
|
|
1801
|
+
this.pendingInstructions = [];
|
|
1802
|
+
this.commandHistory = [];
|
|
1803
|
+
}
|
|
1804
|
+
/**
|
|
1805
|
+
* Get copy of executed commands in human readable form. Most recent is last.
|
|
1806
|
+
* Only commands that have completed execution are returned.
|
|
1807
|
+
*/
|
|
1808
|
+
get history() {
|
|
1809
|
+
return this.commandHistory.filter((cmd) => cmd.state === "DONE");
|
|
1810
|
+
}
|
|
1811
|
+
get lastExecutedCommand() {
|
|
1812
|
+
const history = this.history;
|
|
1813
|
+
if (history.length === 0)
|
|
1814
|
+
return null;
|
|
1815
|
+
const lastEntry = history[history.length - 1];
|
|
1816
|
+
return lastEntry;
|
|
1817
|
+
}
|
|
1818
|
+
/**
|
|
1819
|
+
* Reset the command history provided to agents.
|
|
1820
|
+
* Should be called due to a logical break between commands
|
|
1821
|
+
* such as a SUCCESS being issued.
|
|
1822
|
+
*/
|
|
1823
|
+
resetHistory() {
|
|
1824
|
+
this.commandHistory = [];
|
|
1825
|
+
this.pendingInstructions = [];
|
|
1826
|
+
}
|
|
1827
|
+
/**
|
|
1828
|
+
* Reset controller and browser state.
|
|
1829
|
+
*/
|
|
1830
|
+
async resetState() {
|
|
1831
|
+
this.resetHistory();
|
|
1832
|
+
await this.browser.navigate(this.browser.baseURL);
|
|
1833
|
+
}
|
|
1834
|
+
/**
|
|
1835
|
+
* Get the browser state as a string
|
|
1836
|
+
*/
|
|
1837
|
+
async getBrowserState() {
|
|
1838
|
+
const a11yTree = await this.browser.getA11yTree();
|
|
1839
|
+
return a11yTree.serialize();
|
|
1840
|
+
}
|
|
1841
|
+
getSerializedHistory(url, currentBrowserState) {
|
|
1842
|
+
let history;
|
|
1843
|
+
if (this.config.useHistory === "diff") {
|
|
1844
|
+
history = this.getDiffHistory(url, currentBrowserState);
|
|
1845
|
+
} else {
|
|
1846
|
+
history = this.getListHistory();
|
|
1847
|
+
}
|
|
1848
|
+
return history;
|
|
1849
|
+
}
|
|
1850
|
+
async splitUserGoal(type, goal, disableCache) {
|
|
1851
|
+
if (type === "AI_ACTION" /* AI_ACTION */ && goal.match(/[,!;.]|(?:and)|(?:then)/) && this.config.useGoalSplitter) {
|
|
1852
|
+
const granularInstructions = await this.generator.getGranularGoals(
|
|
1853
|
+
{ goal, url: this.browser.url },
|
|
1854
|
+
disableCache
|
|
1855
|
+
);
|
|
1856
|
+
this.pendingInstructions = granularInstructions.reverse();
|
|
1857
|
+
} else {
|
|
1858
|
+
this.pendingInstructions = [goal];
|
|
1859
|
+
}
|
|
1860
|
+
}
|
|
1861
|
+
/**
|
|
1862
|
+
* Given previously executed commands, generate command for the current prompt.
|
|
1863
|
+
* Should only be used for AI action.
|
|
1864
|
+
*/
|
|
1865
|
+
async promptToCommand(type, goal, disableCache) {
|
|
1866
|
+
if (this.pendingInstructions.length === 0) {
|
|
1867
|
+
await this.splitUserGoal(type, goal, disableCache);
|
|
1868
|
+
}
|
|
1869
|
+
const currInstruction = this.pendingInstructions[this.pendingInstructions.length - 1];
|
|
1870
|
+
this.logger.info({ goal: currInstruction }, "Starting prompt translation");
|
|
1871
|
+
const getBrowserStateStart = Date.now();
|
|
1872
|
+
const url = this.browser.url;
|
|
1873
|
+
const browserState = await this.getBrowserState();
|
|
1874
|
+
this.logger.info(
|
|
1875
|
+
{
|
|
1876
|
+
duration: Date.now() - getBrowserStateStart,
|
|
1877
|
+
url
|
|
1878
|
+
},
|
|
1879
|
+
"Got browser state"
|
|
1880
|
+
);
|
|
1881
|
+
const numPrevious = this.commandHistory.length;
|
|
1882
|
+
this.commandHistory.push({
|
|
1883
|
+
state: "PENDING",
|
|
1884
|
+
browserStateBeforeCommand: browserState,
|
|
1885
|
+
urlBeforeCommand: url,
|
|
1886
|
+
type
|
|
1887
|
+
});
|
|
1888
|
+
const history = this.getSerializedHistory(url, browserState);
|
|
1889
|
+
const getCommandProposalStart = Date.now();
|
|
1890
|
+
const proposedCommand = await this.generator.getProposedCommand(
|
|
1891
|
+
{
|
|
1892
|
+
url,
|
|
1893
|
+
numPrevious,
|
|
1894
|
+
browserState,
|
|
1895
|
+
history,
|
|
1896
|
+
goal: currInstruction,
|
|
1897
|
+
lastCommand: this.lastExecutedCommand
|
|
1898
|
+
},
|
|
1899
|
+
disableCache
|
|
1900
|
+
);
|
|
1901
|
+
this.logger.info(
|
|
1902
|
+
{ duration: Date.now() - getCommandProposalStart },
|
|
1903
|
+
"Got proposed command"
|
|
1904
|
+
);
|
|
1905
|
+
if (proposedCommand.type === "SUCCESS" /* SUCCESS */) {
|
|
1906
|
+
const finishedInstruction = this.pendingInstructions.pop();
|
|
1907
|
+
this.logger.info(
|
|
1908
|
+
{
|
|
1909
|
+
finishedInstruction,
|
|
1910
|
+
remainingInstructions: this.pendingInstructions
|
|
1911
|
+
},
|
|
1912
|
+
"Removing pending instruction due to SUCCESS"
|
|
1913
|
+
);
|
|
1914
|
+
if (this.pendingInstructions.length !== 0) {
|
|
1915
|
+
this.commandHistory.pop();
|
|
1916
|
+
return this.promptToCommand(type, "", disableCache);
|
|
1917
|
+
}
|
|
1918
|
+
} else if (
|
|
1919
|
+
// on failure, we don't continue to execute
|
|
1920
|
+
proposedCommand.type === "FAILURE"
|
|
1921
|
+
) {
|
|
1922
|
+
this.logger.info(
|
|
1923
|
+
{
|
|
1924
|
+
remainingInstructions: this.pendingInstructions
|
|
1925
|
+
},
|
|
1926
|
+
"Removing pending instructions due to FAILURE"
|
|
1927
|
+
);
|
|
1928
|
+
this.pendingInstructions = [];
|
|
1929
|
+
}
|
|
1930
|
+
return proposedCommand;
|
|
1931
|
+
}
|
|
1932
|
+
async locateElement(description, disableCache) {
|
|
1933
|
+
const locator = await this.generator.getElementLocation(
|
|
1934
|
+
{ browserState: await this.getBrowserState(), goal: description },
|
|
1935
|
+
disableCache
|
|
1936
|
+
);
|
|
1937
|
+
if (locator.id < 0) {
|
|
1938
|
+
throw new Error(
|
|
1939
|
+
`Unable to locate element with description: ${description}`
|
|
1940
|
+
);
|
|
1941
|
+
}
|
|
1942
|
+
return locator;
|
|
1943
|
+
}
|
|
1944
|
+
/**
|
|
1945
|
+
* Construct a detailed history that can be passed to the LLM.
|
|
1946
|
+
* History includes commands executed as well as browser state changes that occurred
|
|
1947
|
+
* at each step.
|
|
1948
|
+
*/
|
|
1949
|
+
getDiffHistory(currentURL, currentPageState) {
|
|
1950
|
+
const doneCommands = this.history.filter(
|
|
1951
|
+
(h) => h.type === "AI_ACTION" /* AI_ACTION */
|
|
1952
|
+
);
|
|
1953
|
+
if (doneCommands.length === 0)
|
|
1954
|
+
return "<NONE/>";
|
|
1955
|
+
const historyLines = [
|
|
1956
|
+
"\nYou have already executed the following commands successfully (most recent listed first)",
|
|
1957
|
+
"-".repeat(10)
|
|
1958
|
+
];
|
|
1959
|
+
doneCommands.reverse().forEach((log, i) => {
|
|
1960
|
+
historyLines.push(
|
|
1961
|
+
`COMMAND ${doneCommands.length - i}${i === 0 ? " (command just executed)" : ""}: ${log.serializedCommand}`
|
|
1962
|
+
);
|
|
1963
|
+
if (i === 0) {
|
|
1964
|
+
if (urlChanged(log.urlBeforeCommand, currentURL)) {
|
|
1965
|
+
historyLines.push(
|
|
1966
|
+
` URL CHANGE: '${log.urlBeforeCommand}' -> '${currentURL}'`
|
|
1967
|
+
);
|
|
1968
|
+
} else {
|
|
1969
|
+
const browserStateDiff = diffLines(
|
|
1970
|
+
log.browserStateBeforeCommand,
|
|
1971
|
+
currentPageState,
|
|
1972
|
+
{
|
|
1973
|
+
n_surrounding: 1
|
|
1974
|
+
}
|
|
1975
|
+
);
|
|
1976
|
+
if (!browserStateDiff) {
|
|
1977
|
+
historyLines.push("PAGE CONTENT CHANGE: <NONE/>");
|
|
1978
|
+
} else if (browserStateDiff.length < MAX_HISTORY_CHAR_LENGTH) {
|
|
1979
|
+
historyLines.push("PAGE CONTENT CHANGE:");
|
|
1980
|
+
browserStateDiff.split("\n").forEach((l) => historyLines.push(` ${l}`));
|
|
1981
|
+
} else {
|
|
1982
|
+
historyLines.push("PAGE CONTENT CHANGE: <TOO_LONG_TO_DISPLAY/>");
|
|
1983
|
+
}
|
|
1984
|
+
}
|
|
1985
|
+
}
|
|
1986
|
+
historyLines.push("-".repeat(10));
|
|
1987
|
+
});
|
|
1988
|
+
historyLines.push(`STARTING URL: ${this.browser.baseURL}`);
|
|
1989
|
+
return historyLines.join("\n");
|
|
1990
|
+
}
|
|
1991
|
+
getListHistory() {
|
|
1992
|
+
return dedent2`Here are the commands that you have successfully executed:
|
|
1993
|
+
${this.commandHistory.filter((cmd) => cmd.type === "AI_ACTION" /* AI_ACTION */).map((cmd) => `- ${cmd.serializedCommand}`).join("\n")}`;
|
|
1994
|
+
}
|
|
1995
|
+
/**
|
|
1996
|
+
* Given a command, interact with the chromium browser to actually execute the actions
|
|
1997
|
+
* @param [stateless=false] Execute this command in a stateless fashion, without modifying any controller state such as
|
|
1998
|
+
* pending instructions. Useful when executing cached instructions.
|
|
1999
|
+
*/
|
|
2000
|
+
async executeCommand(command, disableCache, stateless = false) {
|
|
2001
|
+
const pendingHistory = this.commandHistory[this.commandHistory.length - 1];
|
|
2002
|
+
if (!stateless) {
|
|
2003
|
+
if (!pendingHistory || pendingHistory.state !== "PENDING") {
|
|
2004
|
+
throw new Error(
|
|
2005
|
+
"Executing command but there is no pending entry in the history"
|
|
2006
|
+
);
|
|
2007
|
+
}
|
|
2008
|
+
} else {
|
|
2009
|
+
await this.browser.getA11yTree();
|
|
2010
|
+
}
|
|
2011
|
+
let result;
|
|
2012
|
+
try {
|
|
2013
|
+
const executionStart = Date.now();
|
|
2014
|
+
result = await this.executePresetStep(
|
|
2015
|
+
command,
|
|
2016
|
+
disableCache
|
|
2017
|
+
);
|
|
2018
|
+
this.logger.info(
|
|
2019
|
+
{ result, duration: Date.now() - executionStart },
|
|
2020
|
+
"Got execution result"
|
|
2021
|
+
);
|
|
2022
|
+
} catch (e) {
|
|
2023
|
+
if (e instanceof Error) {
|
|
2024
|
+
throw new BrowserExecutionError(`Failed to execute command: ${e}`, {
|
|
2025
|
+
cause: e
|
|
2026
|
+
});
|
|
2027
|
+
}
|
|
2028
|
+
throw new BrowserExecutionError(
|
|
2029
|
+
`Unexpected throw from executing command`,
|
|
2030
|
+
{
|
|
2031
|
+
cause: new Error(`${e}`)
|
|
2032
|
+
}
|
|
2033
|
+
);
|
|
2034
|
+
}
|
|
2035
|
+
if (result.succeedImmediately && !stateless) {
|
|
2036
|
+
this.pendingInstructions.pop();
|
|
2037
|
+
if (this.pendingInstructions.length > 0) {
|
|
2038
|
+
result.succeedImmediately = false;
|
|
2039
|
+
}
|
|
2040
|
+
}
|
|
2041
|
+
if (result.elementInteracted && "target" in command && !command.target.elementDescriptor) {
|
|
2042
|
+
command.target.elementDescriptor = result.elementInteracted.trim();
|
|
2043
|
+
}
|
|
2044
|
+
if (!stateless) {
|
|
2045
|
+
pendingHistory.generatedStep = command;
|
|
2046
|
+
pendingHistory.serializedCommand = serializeCommand(command);
|
|
2047
|
+
pendingHistory.state = "DONE";
|
|
2048
|
+
}
|
|
2049
|
+
return result;
|
|
2050
|
+
}
|
|
2051
|
+
async executeAssertion(urlBeforeCommand, command) {
|
|
2052
|
+
let params;
|
|
2053
|
+
if (command.useVision) {
|
|
2054
|
+
params = {
|
|
2055
|
+
goal: command.assertion,
|
|
2056
|
+
url: urlBeforeCommand,
|
|
2057
|
+
// used for vision only
|
|
2058
|
+
screenshot: await this.browser.screenshot(),
|
|
2059
|
+
// unused for visual assertion
|
|
2060
|
+
browserState: "",
|
|
2061
|
+
history: "",
|
|
2062
|
+
numPrevious: -1,
|
|
2063
|
+
lastCommand: null
|
|
2064
|
+
};
|
|
2065
|
+
} else {
|
|
2066
|
+
const browserState = await this.getBrowserState();
|
|
2067
|
+
const history = this.getSerializedHistory(urlBeforeCommand, browserState);
|
|
2068
|
+
params = {
|
|
2069
|
+
goal: command.assertion,
|
|
2070
|
+
url: urlBeforeCommand,
|
|
2071
|
+
// used for text only
|
|
2072
|
+
browserState,
|
|
2073
|
+
history,
|
|
2074
|
+
lastCommand: this.lastExecutedCommand,
|
|
2075
|
+
numPrevious: this.commandHistory.length
|
|
2076
|
+
};
|
|
2077
|
+
}
|
|
2078
|
+
const assertionEval = await this.generator.getAssertionResult(
|
|
2079
|
+
params,
|
|
2080
|
+
command.useVision,
|
|
2081
|
+
command.disableCache
|
|
2082
|
+
);
|
|
2083
|
+
if (assertionEval.relevantElements) {
|
|
2084
|
+
void Promise.all(
|
|
2085
|
+
assertionEval.relevantElements.map(
|
|
2086
|
+
(id) => this.browser.highlight({ id })
|
|
2087
|
+
)
|
|
2088
|
+
);
|
|
2089
|
+
}
|
|
2090
|
+
if (!assertionEval.result) {
|
|
2091
|
+
throw new Error(assertionEval.thoughts);
|
|
2092
|
+
}
|
|
2093
|
+
return {
|
|
2094
|
+
succeedImmediately: false,
|
|
2095
|
+
thoughts: assertionEval.thoughts,
|
|
2096
|
+
urlAfterCommand: urlBeforeCommand
|
|
2097
|
+
};
|
|
2098
|
+
}
|
|
2099
|
+
/**
|
|
2100
|
+
* Executes a preset command.
|
|
2101
|
+
* For most cases, the execution result contains metadata about the command executed.
|
|
2102
|
+
* For assertions, an AssertionResult with thoughts is returned.
|
|
2103
|
+
* Throws on failure.
|
|
2104
|
+
*/
|
|
2105
|
+
async executePresetStep(command, disableCache) {
|
|
2106
|
+
var _a, _b, _c;
|
|
2107
|
+
const urlBeforeCommand = this.browser.url;
|
|
2108
|
+
switch (command.type) {
|
|
2109
|
+
case "SUCCESS" /* SUCCESS */:
|
|
2110
|
+
if ((_a = command.condition) == null ? void 0 : _a.assertion.trim()) {
|
|
2111
|
+
return this.executeAssertion(urlBeforeCommand, command.condition);
|
|
2112
|
+
}
|
|
2113
|
+
return {
|
|
2114
|
+
succeedImmediately: false,
|
|
2115
|
+
urlAfterCommand: this.browser.url
|
|
2116
|
+
};
|
|
2117
|
+
case "AI_ASSERTION" /* AI_ASSERTION */: {
|
|
2118
|
+
return this.executeAssertion(urlBeforeCommand, command);
|
|
2119
|
+
}
|
|
2120
|
+
case "NAVIGATE" /* NAVIGATE */:
|
|
2121
|
+
await this.browser.navigate(command.url);
|
|
2122
|
+
break;
|
|
2123
|
+
case "GO_BACK" /* GO_BACK */:
|
|
2124
|
+
await this.browser.goBack();
|
|
2125
|
+
break;
|
|
2126
|
+
case "GO_FORWARD" /* GO_FORWARD */:
|
|
2127
|
+
await this.browser.goForward();
|
|
2128
|
+
break;
|
|
2129
|
+
case "SCROLL_DOWN" /* SCROLL_DOWN */:
|
|
2130
|
+
await this.browser.scrollDown();
|
|
2131
|
+
break;
|
|
2132
|
+
case "SCROLL_UP" /* SCROLL_UP */:
|
|
2133
|
+
await this.browser.scrollUp();
|
|
2134
|
+
break;
|
|
2135
|
+
case "WAIT" /* WAIT */:
|
|
2136
|
+
await this.browser.wait(command.delay * 1e3);
|
|
2137
|
+
break;
|
|
2138
|
+
case "REFRESH" /* REFRESH */:
|
|
2139
|
+
await this.browser.refresh();
|
|
2140
|
+
break;
|
|
2141
|
+
case "CLICK" /* CLICK */: {
|
|
2142
|
+
let id;
|
|
2143
|
+
if (command.target.a11yData) {
|
|
2144
|
+
id = (_b = command.target.a11yData) == null ? void 0 : _b.id;
|
|
2145
|
+
} else {
|
|
2146
|
+
const locator = await this.locateElement(
|
|
2147
|
+
command.target.elementDescriptor,
|
|
2148
|
+
disableCache
|
|
2149
|
+
);
|
|
2150
|
+
id = locator.id;
|
|
2151
|
+
}
|
|
2152
|
+
const elementInteracted = await this.browser.click(
|
|
2153
|
+
{
|
|
2154
|
+
id
|
|
2155
|
+
},
|
|
2156
|
+
{
|
|
2157
|
+
doubleClick: command.doubleClick,
|
|
2158
|
+
rightClick: command.rightClick
|
|
2159
|
+
}
|
|
2160
|
+
);
|
|
2161
|
+
const result2 = {
|
|
2162
|
+
urlAfterCommand: this.browser.url,
|
|
2163
|
+
succeedImmediately: false,
|
|
2164
|
+
elementInteracted
|
|
2165
|
+
};
|
|
2166
|
+
if (urlChanged(urlBeforeCommand, result2.urlAfterCommand)) {
|
|
2167
|
+
result2.succeedImmediately = true;
|
|
2168
|
+
result2.succeedImmediatelyReason = "URL changed";
|
|
2169
|
+
}
|
|
2170
|
+
return result2;
|
|
2171
|
+
}
|
|
2172
|
+
case "SELECT_OPTION" /* SELECT_OPTION */: {
|
|
2173
|
+
let id;
|
|
2174
|
+
if (command.target.a11yData) {
|
|
2175
|
+
id = (_c = command.target.a11yData) == null ? void 0 : _c.id;
|
|
2176
|
+
} else {
|
|
2177
|
+
const locator = await this.locateElement(
|
|
2178
|
+
command.target.elementDescriptor,
|
|
2179
|
+
disableCache
|
|
2180
|
+
);
|
|
2181
|
+
id = locator.id;
|
|
2182
|
+
}
|
|
2183
|
+
const elementInteracted = await this.browser.selectOption(
|
|
2184
|
+
{
|
|
2185
|
+
id
|
|
2186
|
+
},
|
|
2187
|
+
command.option
|
|
2188
|
+
);
|
|
2189
|
+
return {
|
|
2190
|
+
succeedImmediately: false,
|
|
2191
|
+
urlAfterCommand: this.browser.url,
|
|
2192
|
+
elementInteracted
|
|
2193
|
+
};
|
|
2194
|
+
}
|
|
2195
|
+
case "TAB" /* TAB */:
|
|
2196
|
+
await this.browser.switchToPage(command.url);
|
|
2197
|
+
break;
|
|
2198
|
+
case "COOKIE" /* COOKIE */:
|
|
2199
|
+
await this.browser.setCookie(command.value);
|
|
2200
|
+
break;
|
|
2201
|
+
case "TYPE" /* TYPE */: {
|
|
2202
|
+
let elementInteracted;
|
|
2203
|
+
const target = command.target;
|
|
2204
|
+
if (target.a11yData) {
|
|
2205
|
+
elementInteracted = await this.browser.click({
|
|
2206
|
+
id: target.a11yData.id
|
|
2207
|
+
});
|
|
2208
|
+
} else if (target.elementDescriptor.length > 0) {
|
|
2209
|
+
const locator = await this.locateElement(
|
|
2210
|
+
command.target.elementDescriptor,
|
|
2211
|
+
disableCache
|
|
2212
|
+
);
|
|
2213
|
+
elementInteracted = await this.browser.click({
|
|
2214
|
+
id: locator.id
|
|
2215
|
+
});
|
|
2216
|
+
}
|
|
2217
|
+
await this.browser.type(command.value, {
|
|
2218
|
+
clearContent: command.clearContent,
|
|
2219
|
+
pressKeysSequentially: command.pressKeysSequentially
|
|
2220
|
+
});
|
|
2221
|
+
if (command.pressEnter) {
|
|
2222
|
+
await this.browser.press("Enter");
|
|
2223
|
+
}
|
|
2224
|
+
const result2 = {
|
|
2225
|
+
urlAfterCommand: this.browser.url,
|
|
2226
|
+
succeedImmediately: false,
|
|
2227
|
+
elementInteracted
|
|
2228
|
+
};
|
|
2229
|
+
if (urlChanged(urlBeforeCommand, result2.urlAfterCommand)) {
|
|
2230
|
+
result2.succeedImmediately = true;
|
|
2231
|
+
result2.succeedImmediatelyReason = "URL changed";
|
|
2232
|
+
}
|
|
2233
|
+
return result2;
|
|
2234
|
+
}
|
|
2235
|
+
case "PRESS" /* PRESS */:
|
|
2236
|
+
await this.browser.press(command.value);
|
|
2237
|
+
const result = {
|
|
2238
|
+
urlAfterCommand: this.browser.url,
|
|
2239
|
+
succeedImmediately: false
|
|
2240
|
+
};
|
|
2241
|
+
if (urlChanged(urlBeforeCommand, result.urlAfterCommand)) {
|
|
2242
|
+
result.succeedImmediately = true;
|
|
2243
|
+
result.succeedImmediatelyReason = "URL changed";
|
|
2244
|
+
}
|
|
2245
|
+
return result;
|
|
2246
|
+
default:
|
|
2247
|
+
const assertUnreachable = (_x) => {
|
|
2248
|
+
throw "If Typescript complains about the line below, you missed a case or break in the switch above";
|
|
2249
|
+
};
|
|
2250
|
+
return assertUnreachable(command);
|
|
2251
|
+
}
|
|
2252
|
+
return {
|
|
2253
|
+
succeedImmediately: false,
|
|
2254
|
+
urlAfterCommand: this.browser.url
|
|
2255
|
+
};
|
|
2256
|
+
}
|
|
2257
|
+
};
|
|
2258
|
+
|
|
2259
|
+
// ../../packages/web-agent/src/generators/api-generator.ts
|
|
2260
|
+
import fetchRetry from "fetch-retry";
|
|
2261
|
+
var fetch2 = fetchRetry(global.fetch);
|
|
2262
|
+
var API_VERSION = "v1";
|
|
2263
|
+
var APIGenerator = class {
|
|
2264
|
+
baseURL;
|
|
2265
|
+
apiKey;
|
|
2266
|
+
constructor(params) {
|
|
2267
|
+
this.baseURL = params.baseURL;
|
|
2268
|
+
this.apiKey = params.apiKey;
|
|
2269
|
+
}
|
|
2270
|
+
async getElementLocation(context, disableCache) {
|
|
2271
|
+
const result = await this.sendRequest(
|
|
2272
|
+
`/${API_VERSION}/web-agent/locate-element`,
|
|
2273
|
+
{
|
|
2274
|
+
browserState: context.browserState,
|
|
2275
|
+
goal: context.goal,
|
|
2276
|
+
disableCache
|
|
2277
|
+
}
|
|
2278
|
+
);
|
|
2279
|
+
return LocateResponseSchema.parse(result);
|
|
2280
|
+
}
|
|
2281
|
+
async getAssertionResult(context, useVision, disableCache) {
|
|
2282
|
+
var _a;
|
|
2283
|
+
if (useVision) {
|
|
2284
|
+
const result2 = await this.sendRequest(
|
|
2285
|
+
`/${API_VERSION}/web-agent/assertion`,
|
|
2286
|
+
{
|
|
2287
|
+
url: context.url,
|
|
2288
|
+
goal: context.goal,
|
|
2289
|
+
screenshot: (_a = context.screenshot) == null ? void 0 : _a.toString("base64"),
|
|
2290
|
+
disableCache,
|
|
2291
|
+
vision: true
|
|
2292
|
+
}
|
|
2293
|
+
);
|
|
2294
|
+
return GetAssertionResponseSchema.parse(result2);
|
|
2295
|
+
}
|
|
2296
|
+
const result = await this.sendRequest(
|
|
2297
|
+
`/${API_VERSION}/web-agent/assertion`,
|
|
2298
|
+
{
|
|
2299
|
+
url: context.url,
|
|
2300
|
+
browserState: context.browserState,
|
|
2301
|
+
goal: context.goal,
|
|
2302
|
+
history: context.history,
|
|
2303
|
+
numPrevious: context.numPrevious,
|
|
2304
|
+
lastCommand: context.lastCommand,
|
|
2305
|
+
disableCache,
|
|
2306
|
+
vision: false
|
|
2307
|
+
}
|
|
2308
|
+
);
|
|
2309
|
+
return GetAssertionResponseSchema.parse(result);
|
|
2310
|
+
}
|
|
2311
|
+
async getProposedCommand(context, disableCache) {
|
|
2312
|
+
const result = await this.sendRequest(
|
|
2313
|
+
`/${API_VERSION}/web-agent/next-command`,
|
|
2314
|
+
{
|
|
2315
|
+
url: context.url,
|
|
2316
|
+
browserState: context.browserState,
|
|
2317
|
+
goal: context.goal,
|
|
2318
|
+
history: context.history,
|
|
2319
|
+
numPrevious: context.numPrevious,
|
|
2320
|
+
lastCommand: context.lastCommand,
|
|
2321
|
+
disableCache
|
|
2322
|
+
}
|
|
2323
|
+
);
|
|
2324
|
+
return GetNextCommandResponseSchema.parse(result);
|
|
2325
|
+
}
|
|
2326
|
+
async getGranularGoals(context, disableCache) {
|
|
2327
|
+
const result = await this.sendRequest(
|
|
2328
|
+
`/${API_VERSION}/web-agent/split-goal`,
|
|
2329
|
+
{
|
|
2330
|
+
url: context.url,
|
|
2331
|
+
goal: context.goal,
|
|
2332
|
+
disableCache
|
|
2333
|
+
}
|
|
2334
|
+
);
|
|
2335
|
+
return SplitGoalResponseSchema.parse(result);
|
|
2336
|
+
}
|
|
2337
|
+
async sendRequest(path, body) {
|
|
2338
|
+
const response = await fetch2(`${this.baseURL}${path}`, {
|
|
2339
|
+
retries: 3,
|
|
2340
|
+
retryDelay: 1e3,
|
|
2341
|
+
method: "POST",
|
|
2342
|
+
body: JSON.stringify(body),
|
|
2343
|
+
headers: {
|
|
2344
|
+
"Content-Type": "application/json",
|
|
2345
|
+
Authorization: `Bearer ${this.apiKey}`
|
|
2346
|
+
}
|
|
2347
|
+
});
|
|
2348
|
+
if (!response.ok) {
|
|
2349
|
+
throw new Error(
|
|
2350
|
+
`Request to ${path} failed with status ${response.status}: ${await response.text()}`
|
|
2351
|
+
);
|
|
2352
|
+
}
|
|
2353
|
+
return response.json();
|
|
2354
|
+
}
|
|
2355
|
+
};
|
|
2356
|
+
|
|
2357
|
+
// package.json
|
|
2358
|
+
var version = "1.0.0";
|
|
2359
|
+
|
|
2360
|
+
// src/api-client.ts
|
|
2361
|
+
var API_VERSION2 = "v1";
|
|
2362
|
+
var APIClient = class {
|
|
2363
|
+
baseURL;
|
|
2364
|
+
apiKey;
|
|
2365
|
+
constructor(params) {
|
|
2366
|
+
this.baseURL = params.baseURL;
|
|
2367
|
+
this.apiKey = params.apiKey;
|
|
2368
|
+
}
|
|
2369
|
+
async getRun(runId) {
|
|
2370
|
+
const result = await this.sendRequest(`/${API_VERSION2}/runs/${runId}`, {
|
|
2371
|
+
method: "GET"
|
|
2372
|
+
});
|
|
2373
|
+
return GetRunResponseSchema.parse(result);
|
|
2374
|
+
}
|
|
2375
|
+
async createRun(body) {
|
|
2376
|
+
const result = await this.sendRequest(`/${API_VERSION2}/runs`, {
|
|
2377
|
+
method: "POST",
|
|
2378
|
+
body
|
|
2379
|
+
});
|
|
2380
|
+
return CreateRunResponseSchema.parse(result);
|
|
2381
|
+
}
|
|
2382
|
+
async updateRun(runId, body) {
|
|
2383
|
+
await this.sendRequest(`/${API_VERSION2}/runs/${runId}`, {
|
|
2384
|
+
method: "PATCH",
|
|
2385
|
+
body
|
|
2386
|
+
});
|
|
2387
|
+
}
|
|
2388
|
+
async getTest(testId) {
|
|
2389
|
+
const result = await this.sendRequest(`/${API_VERSION2}/tests/${testId}`, {
|
|
2390
|
+
method: "GET"
|
|
2391
|
+
});
|
|
2392
|
+
return GetTestResponseSchema.parse(result);
|
|
2393
|
+
}
|
|
2394
|
+
async uploadScreenshot(body) {
|
|
2395
|
+
const result = await this.sendRequest(`/${API_VERSION2}/screenshots`, {
|
|
2396
|
+
method: "POST",
|
|
2397
|
+
body
|
|
2398
|
+
});
|
|
2399
|
+
return CreateScreenshotResponseSchema.parse(result);
|
|
2400
|
+
}
|
|
2401
|
+
async sendRequest(path, options) {
|
|
2402
|
+
const response = await fetch(`${this.baseURL}${path}`, {
|
|
2403
|
+
method: options.method,
|
|
2404
|
+
body: options.body ? JSON.stringify(options.body) : void 0,
|
|
2405
|
+
headers: {
|
|
2406
|
+
"Content-Type": "application/json",
|
|
2407
|
+
Authorization: `Bearer ${this.apiKey}`
|
|
2408
|
+
}
|
|
2409
|
+
});
|
|
2410
|
+
if (!response.ok) {
|
|
2411
|
+
throw new Error(
|
|
2412
|
+
`Request to ${path} failed with status ${response.status}: ${await response.text()}`
|
|
2413
|
+
);
|
|
2414
|
+
}
|
|
2415
|
+
if (response.status === 204) {
|
|
2416
|
+
return response.text();
|
|
2417
|
+
}
|
|
2418
|
+
return response.json();
|
|
2419
|
+
}
|
|
2420
|
+
};
|
|
2421
|
+
|
|
2422
|
+
// ../../packages/execute/src/constants.ts
|
|
2423
|
+
var MAX_COMMANDS_PER_STEP = 20;
|
|
2424
|
+
|
|
2425
|
+
// ../../packages/execute/src/steps/ai.ts
|
|
2426
|
+
var executeAIStep = async ({
|
|
2427
|
+
controller,
|
|
2428
|
+
step,
|
|
2429
|
+
logger,
|
|
2430
|
+
advanced,
|
|
2431
|
+
...callbacks
|
|
2432
|
+
}) => {
|
|
2433
|
+
var _a, _b, _c, _d, _e, _f;
|
|
2434
|
+
(_a = callbacks.onStarted) == null ? void 0 : _a.call(callbacks);
|
|
2435
|
+
controller.resetHistory();
|
|
2436
|
+
const result = {
|
|
2437
|
+
...step,
|
|
2438
|
+
startedAt: /* @__PURE__ */ new Date(),
|
|
2439
|
+
userAgent: ChromeBrowser.USER_AGENT,
|
|
2440
|
+
// placeholder values
|
|
2441
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
2442
|
+
results: [],
|
|
2443
|
+
status: "SUCCESS" /* SUCCESS */
|
|
2444
|
+
};
|
|
2445
|
+
try {
|
|
2446
|
+
let commandIndex = 0;
|
|
2447
|
+
let useSavedCommands = step.commands && step.commands.length > 0;
|
|
2448
|
+
while (true) {
|
|
2449
|
+
if (commandIndex > MAX_COMMANDS_PER_STEP) {
|
|
2450
|
+
throw new Error(
|
|
2451
|
+
`Exceeded max number of commands per step (${MAX_COMMANDS_PER_STEP})`
|
|
2452
|
+
);
|
|
2453
|
+
}
|
|
2454
|
+
let command;
|
|
2455
|
+
const startedAt = /* @__PURE__ */ new Date();
|
|
2456
|
+
const beforeScreenshotBuffer = await controller.browser.screenshot();
|
|
2457
|
+
const beforeScreenshot = await callbacks.onSaveScreenshot(
|
|
2458
|
+
beforeScreenshotBuffer
|
|
2459
|
+
);
|
|
2460
|
+
if (useSavedCommands) {
|
|
2461
|
+
command = step.commands[commandIndex];
|
|
2462
|
+
if (!command) {
|
|
2463
|
+
throw new Error(
|
|
2464
|
+
`Saved command at index ${commandIndex} is undefined.`
|
|
2465
|
+
);
|
|
2466
|
+
}
|
|
2467
|
+
} else {
|
|
2468
|
+
command = await controller.promptToCommand(
|
|
2469
|
+
step.type,
|
|
2470
|
+
step.text,
|
|
2471
|
+
advanced.disableAICaching
|
|
2472
|
+
);
|
|
2473
|
+
}
|
|
2474
|
+
if (command.type === "FAILURE") {
|
|
2475
|
+
result.finishedAt = /* @__PURE__ */ new Date();
|
|
2476
|
+
result.status = "FAILED" /* FAILED */;
|
|
2477
|
+
result.message = command.thoughts;
|
|
2478
|
+
break;
|
|
2479
|
+
}
|
|
2480
|
+
(_b = callbacks.onCommandGenerated) == null ? void 0 : _b.call(callbacks, {
|
|
2481
|
+
commandIndex,
|
|
2482
|
+
message: CARD_DISPLAY_NAMES[command.type] || `Unknown command (${command.type})`
|
|
2483
|
+
});
|
|
2484
|
+
const cmdResult = {
|
|
2485
|
+
beforeScreenshot,
|
|
2486
|
+
beforeUrl: controller.browser.url,
|
|
2487
|
+
startedAt,
|
|
2488
|
+
viewport: controller.browser.viewport,
|
|
2489
|
+
// placeholder values
|
|
2490
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
2491
|
+
status: "SUCCESS" /* SUCCESS */
|
|
2492
|
+
};
|
|
2493
|
+
logger.info(
|
|
2494
|
+
`Executing command ${commandIndex}: ${serializeCommand(command)}`
|
|
2495
|
+
);
|
|
2496
|
+
try {
|
|
2497
|
+
const executionResult = await controller.executeCommand(
|
|
2498
|
+
command,
|
|
2499
|
+
advanced.disableAICaching,
|
|
2500
|
+
useSavedCommands
|
|
2501
|
+
);
|
|
2502
|
+
cmdResult.elementInteracted = executionResult.elementInteracted;
|
|
2503
|
+
(_c = callbacks.onCommandExecuted) == null ? void 0 : _c.call(callbacks, {
|
|
2504
|
+
commandIndex,
|
|
2505
|
+
message: serializeCommand(command),
|
|
2506
|
+
command
|
|
2507
|
+
});
|
|
2508
|
+
const afterScreenshotBuffer = await controller.browser.screenshot();
|
|
2509
|
+
const afterScreenshot = await callbacks.onSaveScreenshot(
|
|
2510
|
+
afterScreenshotBuffer
|
|
2511
|
+
);
|
|
2512
|
+
cmdResult.afterScreenshot = afterScreenshot;
|
|
2513
|
+
cmdResult.afterUrl = controller.browser.url;
|
|
2514
|
+
cmdResult.finishedAt = /* @__PURE__ */ new Date();
|
|
2515
|
+
const presetActionResult = {
|
|
2516
|
+
status: "SUCCESS" /* SUCCESS */,
|
|
2517
|
+
startedAt: cmdResult.startedAt,
|
|
2518
|
+
finishedAt: cmdResult.finishedAt,
|
|
2519
|
+
type: "PRESET_ACTION" /* PRESET_ACTION */,
|
|
2520
|
+
command,
|
|
2521
|
+
results: [cmdResult]
|
|
2522
|
+
};
|
|
2523
|
+
result.results.push(presetActionResult);
|
|
2524
|
+
if (command.type === "SUCCESS" /* SUCCESS */) {
|
|
2525
|
+
result.finishedAt = /* @__PURE__ */ new Date();
|
|
2526
|
+
result.status = "SUCCESS" /* SUCCESS */;
|
|
2527
|
+
result.message = executionResult.thoughts ?? "All commands completed.";
|
|
2528
|
+
break;
|
|
2529
|
+
}
|
|
2530
|
+
if (executionResult.succeedImmediately && !useSavedCommands) {
|
|
2531
|
+
result.finishedAt = /* @__PURE__ */ new Date();
|
|
2532
|
+
result.status = "SUCCESS" /* SUCCESS */;
|
|
2533
|
+
result.message = executionResult.succeedImmediatelyReason;
|
|
2534
|
+
command = {
|
|
2535
|
+
type: "SUCCESS" /* SUCCESS */
|
|
2536
|
+
};
|
|
2537
|
+
(_d = callbacks.onCommandExecuted) == null ? void 0 : _d.call(callbacks, {
|
|
2538
|
+
commandIndex: commandIndex + 1,
|
|
2539
|
+
message: serializeCommand(command),
|
|
2540
|
+
command
|
|
2541
|
+
});
|
|
2542
|
+
result.results.push({
|
|
2543
|
+
...presetActionResult,
|
|
2544
|
+
command
|
|
2545
|
+
});
|
|
2546
|
+
break;
|
|
2547
|
+
}
|
|
2548
|
+
} catch (err) {
|
|
2549
|
+
if (useSavedCommands) {
|
|
2550
|
+
useSavedCommands = false;
|
|
2551
|
+
commandIndex = 0;
|
|
2552
|
+
result.results = [];
|
|
2553
|
+
continue;
|
|
2554
|
+
}
|
|
2555
|
+
cmdResult.status = "FAILED" /* FAILED */;
|
|
2556
|
+
cmdResult.message = `${err}`;
|
|
2557
|
+
cmdResult.finishedAt = /* @__PURE__ */ new Date();
|
|
2558
|
+
cmdResult.afterScreenshot = void 0;
|
|
2559
|
+
cmdResult.afterUrl = controller.browser.url;
|
|
2560
|
+
result.results.push({
|
|
2561
|
+
status: "FAILED" /* FAILED */,
|
|
2562
|
+
startedAt: cmdResult.startedAt,
|
|
2563
|
+
finishedAt: cmdResult.finishedAt,
|
|
2564
|
+
type: "PRESET_ACTION" /* PRESET_ACTION */,
|
|
2565
|
+
command,
|
|
2566
|
+
results: [cmdResult],
|
|
2567
|
+
message: `${err}`
|
|
2568
|
+
});
|
|
2569
|
+
result.status = "FAILED" /* FAILED */;
|
|
2570
|
+
result.finishedAt = /* @__PURE__ */ new Date();
|
|
2571
|
+
result.message = `${err}`;
|
|
2572
|
+
break;
|
|
2573
|
+
}
|
|
2574
|
+
commandIndex++;
|
|
2575
|
+
}
|
|
2576
|
+
} catch (err) {
|
|
2577
|
+
result.message = `${err}`;
|
|
2578
|
+
result.finishedAt = /* @__PURE__ */ new Date();
|
|
2579
|
+
result.status = "FAILED" /* FAILED */;
|
|
2580
|
+
}
|
|
2581
|
+
if (result.status === "SUCCESS" /* SUCCESS */) {
|
|
2582
|
+
(_e = callbacks.onSuccess) == null ? void 0 : _e.call(callbacks, {
|
|
2583
|
+
message: result.message || "AI step succeeded.",
|
|
2584
|
+
startedAt: result.startedAt.getTime(),
|
|
2585
|
+
durationMs: result.finishedAt.getTime() - result.startedAt.getTime()
|
|
2586
|
+
});
|
|
2587
|
+
} else {
|
|
2588
|
+
(_f = callbacks.onFailure) == null ? void 0 : _f.call(callbacks, {
|
|
2589
|
+
message: result.message || "AI step errored.",
|
|
2590
|
+
startedAt: result.startedAt.getTime(),
|
|
2591
|
+
durationMs: result.finishedAt.getTime() - result.startedAt.getTime()
|
|
2592
|
+
});
|
|
2593
|
+
}
|
|
2594
|
+
return result;
|
|
2595
|
+
};
|
|
2596
|
+
|
|
2597
|
+
// ../../packages/execute/src/steps/preset.ts
|
|
2598
|
+
var executePresetStep = async ({
|
|
2599
|
+
controller,
|
|
2600
|
+
step,
|
|
2601
|
+
advanced,
|
|
2602
|
+
...callbacks
|
|
2603
|
+
}) => {
|
|
2604
|
+
var _a, _b, _c;
|
|
2605
|
+
(_a = callbacks.onStarted) == null ? void 0 : _a.call(callbacks);
|
|
2606
|
+
const startedAt = /* @__PURE__ */ new Date();
|
|
2607
|
+
const beforeUrl = controller.browser.url;
|
|
2608
|
+
const beforeScreenshotBuffer = await controller.browser.screenshot();
|
|
2609
|
+
const beforeScreenshot = await callbacks.onSaveScreenshot(
|
|
2610
|
+
beforeScreenshotBuffer
|
|
2611
|
+
);
|
|
2612
|
+
try {
|
|
2613
|
+
const execResult = await controller.executePresetStep(
|
|
2614
|
+
step.command,
|
|
2615
|
+
advanced.disableAICaching
|
|
2616
|
+
);
|
|
2617
|
+
const afterScreenshotBuffer = await controller.browser.screenshot();
|
|
2618
|
+
const afterScreenshot = await callbacks.onSaveScreenshot(
|
|
2619
|
+
afterScreenshotBuffer
|
|
2620
|
+
);
|
|
2621
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
2622
|
+
const result = {
|
|
2623
|
+
...step,
|
|
2624
|
+
startedAt,
|
|
2625
|
+
finishedAt,
|
|
2626
|
+
// placeholder values
|
|
2627
|
+
status: "SUCCESS" /* SUCCESS */,
|
|
2628
|
+
results: []
|
|
2629
|
+
};
|
|
2630
|
+
let message = "Successfully executed preset action.";
|
|
2631
|
+
if (step.command.type === "AI_ASSERTION" /* AI_ASSERTION */) {
|
|
2632
|
+
message = execResult.thoughts || "Assertion passed.";
|
|
2633
|
+
}
|
|
2634
|
+
const cmdMetadata = {
|
|
2635
|
+
beforeUrl,
|
|
2636
|
+
beforeScreenshot,
|
|
2637
|
+
afterUrl: controller.browser.url,
|
|
2638
|
+
afterScreenshot,
|
|
2639
|
+
startedAt,
|
|
2640
|
+
finishedAt,
|
|
2641
|
+
viewport: controller.browser.viewport,
|
|
2642
|
+
status: "SUCCESS" /* SUCCESS */
|
|
2643
|
+
};
|
|
2644
|
+
result.status = "SUCCESS" /* SUCCESS */;
|
|
2645
|
+
result.results = [cmdMetadata];
|
|
2646
|
+
result.message = message;
|
|
2647
|
+
(_b = callbacks.onSuccess) == null ? void 0 : _b.call(callbacks, {
|
|
2648
|
+
message,
|
|
2649
|
+
startedAt: startedAt.getTime(),
|
|
2650
|
+
durationMs: finishedAt.getTime() - startedAt.getTime()
|
|
2651
|
+
});
|
|
2652
|
+
return result;
|
|
2653
|
+
} catch (err) {
|
|
2654
|
+
const finishedAt = /* @__PURE__ */ new Date();
|
|
2655
|
+
const result = {
|
|
2656
|
+
...step,
|
|
2657
|
+
startedAt,
|
|
2658
|
+
finishedAt,
|
|
2659
|
+
status: "FAILED" /* FAILED */,
|
|
2660
|
+
message: `${err}`,
|
|
2661
|
+
results: [
|
|
2662
|
+
{
|
|
2663
|
+
beforeUrl,
|
|
2664
|
+
beforeScreenshot,
|
|
2665
|
+
afterUrl: controller.browser.url,
|
|
2666
|
+
afterScreenshot: void 0,
|
|
2667
|
+
startedAt,
|
|
2668
|
+
finishedAt,
|
|
2669
|
+
viewport: controller.browser.viewport,
|
|
2670
|
+
status: "FAILED" /* FAILED */,
|
|
2671
|
+
message: `${err}`
|
|
2672
|
+
}
|
|
2673
|
+
]
|
|
2674
|
+
};
|
|
2675
|
+
(_c = callbacks.onFailure) == null ? void 0 : _c.call(callbacks, {
|
|
2676
|
+
message: `${err}`,
|
|
2677
|
+
startedAt: startedAt.getTime(),
|
|
2678
|
+
durationMs: finishedAt.getTime() - startedAt.getTime()
|
|
2679
|
+
});
|
|
2680
|
+
return result;
|
|
2681
|
+
}
|
|
2682
|
+
};
|
|
2683
|
+
|
|
2684
|
+
// ../../packages/execute/src/steps/module.ts
|
|
2685
|
+
var executeModuleStep = async ({
|
|
2686
|
+
controller,
|
|
2687
|
+
step,
|
|
2688
|
+
advanced,
|
|
2689
|
+
logger,
|
|
2690
|
+
...callbacks
|
|
2691
|
+
}) => {
|
|
2692
|
+
var _a, _b, _c;
|
|
2693
|
+
(_a = callbacks.onStarted) == null ? void 0 : _a.call(callbacks);
|
|
2694
|
+
const result = {
|
|
2695
|
+
type: "MODULE" /* MODULE */,
|
|
2696
|
+
moduleId: step.moduleId,
|
|
2697
|
+
startedAt: /* @__PURE__ */ new Date(),
|
|
2698
|
+
userAgent: ChromeBrowser.USER_AGENT,
|
|
2699
|
+
// placeholder values
|
|
2700
|
+
results: [],
|
|
2701
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
2702
|
+
status: "SUCCESS" /* SUCCESS */
|
|
2703
|
+
};
|
|
2704
|
+
for (let i = 0; i < step.steps.length; i++) {
|
|
2705
|
+
const moduleStep = step.steps[i];
|
|
2706
|
+
logger.info({ i, moduleStep }, `Starting module step`);
|
|
2707
|
+
let moduleStepResult;
|
|
2708
|
+
switch (moduleStep.type) {
|
|
2709
|
+
case "PRESET_ACTION" /* PRESET_ACTION */:
|
|
2710
|
+
moduleStepResult = await executePresetStep({
|
|
2711
|
+
controller,
|
|
2712
|
+
step: moduleStep,
|
|
2713
|
+
advanced,
|
|
2714
|
+
logger,
|
|
2715
|
+
onSaveScreenshot: callbacks.onSaveScreenshot,
|
|
2716
|
+
onStarted() {
|
|
2717
|
+
var _a2;
|
|
2718
|
+
(_a2 = callbacks.onStepStarted) == null ? void 0 : _a2.call(callbacks, { index: i });
|
|
2719
|
+
},
|
|
2720
|
+
onSuccess({ message, startedAt, durationMs }) {
|
|
2721
|
+
var _a2;
|
|
2722
|
+
(_a2 = callbacks.onStepSuccess) == null ? void 0 : _a2.call(callbacks, {
|
|
2723
|
+
index: i,
|
|
2724
|
+
message,
|
|
2725
|
+
startedAt,
|
|
2726
|
+
durationMs
|
|
2727
|
+
});
|
|
2728
|
+
},
|
|
2729
|
+
onFailure({ message, startedAt, durationMs }) {
|
|
2730
|
+
var _a2;
|
|
2731
|
+
(_a2 = callbacks.onStepFailure) == null ? void 0 : _a2.call(callbacks, {
|
|
2732
|
+
index: i,
|
|
2733
|
+
message,
|
|
2734
|
+
startedAt,
|
|
2735
|
+
durationMs
|
|
2736
|
+
});
|
|
2737
|
+
}
|
|
2738
|
+
});
|
|
2739
|
+
break;
|
|
2740
|
+
case "AI_ACTION" /* AI_ACTION */:
|
|
2741
|
+
moduleStepResult = await executeAIStep({
|
|
2742
|
+
controller,
|
|
2743
|
+
step: moduleStep,
|
|
2744
|
+
advanced,
|
|
2745
|
+
logger,
|
|
2746
|
+
onSaveScreenshot: callbacks.onSaveScreenshot,
|
|
2747
|
+
onStarted() {
|
|
2748
|
+
var _a2;
|
|
2749
|
+
(_a2 = callbacks.onStepStarted) == null ? void 0 : _a2.call(callbacks, { index: i });
|
|
2750
|
+
},
|
|
2751
|
+
onSuccess({ message, startedAt, durationMs }) {
|
|
2752
|
+
var _a2;
|
|
2753
|
+
(_a2 = callbacks.onStepSuccess) == null ? void 0 : _a2.call(callbacks, {
|
|
2754
|
+
index: i,
|
|
2755
|
+
message,
|
|
2756
|
+
startedAt,
|
|
2757
|
+
durationMs
|
|
2758
|
+
});
|
|
2759
|
+
},
|
|
2760
|
+
onFailure({ message, startedAt, durationMs }) {
|
|
2761
|
+
var _a2;
|
|
2762
|
+
(_a2 = callbacks.onStepFailure) == null ? void 0 : _a2.call(callbacks, {
|
|
2763
|
+
index: i,
|
|
2764
|
+
message,
|
|
2765
|
+
startedAt,
|
|
2766
|
+
durationMs
|
|
2767
|
+
});
|
|
2768
|
+
},
|
|
2769
|
+
onCommandGenerated({ commandIndex, message }) {
|
|
2770
|
+
var _a2;
|
|
2771
|
+
(_a2 = callbacks.onCommandGenerated) == null ? void 0 : _a2.call(callbacks, { index: i, commandIndex, message });
|
|
2772
|
+
},
|
|
2773
|
+
onCommandExecuted({ commandIndex, message, command }) {
|
|
2774
|
+
var _a2;
|
|
2775
|
+
(_a2 = callbacks.onCommandExecuted) == null ? void 0 : _a2.call(callbacks, {
|
|
2776
|
+
index: i,
|
|
2777
|
+
commandIndex,
|
|
2778
|
+
message,
|
|
2779
|
+
command
|
|
2780
|
+
});
|
|
2781
|
+
}
|
|
2782
|
+
});
|
|
2783
|
+
break;
|
|
2784
|
+
default:
|
|
2785
|
+
const assertUnreachable = (_x) => {
|
|
2786
|
+
throw "If Typescript complains about the line below, you missed a case or break in the switch above";
|
|
2787
|
+
};
|
|
2788
|
+
return assertUnreachable(moduleStep);
|
|
2789
|
+
}
|
|
2790
|
+
result.results.push(moduleStepResult);
|
|
2791
|
+
if (moduleStepResult.status === "FAILED" /* FAILED */) {
|
|
2792
|
+
result.status = "FAILED" /* FAILED */;
|
|
2793
|
+
result.finishedAt = /* @__PURE__ */ new Date();
|
|
2794
|
+
for (let j = i + 1; j < step.steps.length; j++) {
|
|
2795
|
+
const skippedStep = step.steps[j];
|
|
2796
|
+
const skippedResult = {
|
|
2797
|
+
...skippedStep,
|
|
2798
|
+
status: "CANCELLED" /* CANCELLED */,
|
|
2799
|
+
startedAt: /* @__PURE__ */ new Date(),
|
|
2800
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
2801
|
+
userAgent: ChromeBrowser.USER_AGENT,
|
|
2802
|
+
results: [],
|
|
2803
|
+
message: "Cancelled due to previous failure."
|
|
2804
|
+
};
|
|
2805
|
+
result.results.push(skippedResult);
|
|
2806
|
+
}
|
|
2807
|
+
break;
|
|
2808
|
+
}
|
|
2809
|
+
}
|
|
2810
|
+
if (result.status === "SUCCESS" /* SUCCESS */) {
|
|
2811
|
+
(_b = callbacks.onSuccess) == null ? void 0 : _b.call(callbacks, {
|
|
2812
|
+
message: "Executed module step.",
|
|
2813
|
+
startedAt: result.startedAt.getTime(),
|
|
2814
|
+
durationMs: result.finishedAt.getTime() - result.startedAt.getTime()
|
|
2815
|
+
});
|
|
2816
|
+
} else {
|
|
2817
|
+
(_c = callbacks.onFailure) == null ? void 0 : _c.call(callbacks, {
|
|
2818
|
+
message: "Failed to execute module step.",
|
|
2819
|
+
startedAt: result.startedAt.getTime(),
|
|
2820
|
+
durationMs: result.finishedAt.getTime() - result.startedAt.getTime()
|
|
2821
|
+
});
|
|
2822
|
+
}
|
|
2823
|
+
return result;
|
|
2824
|
+
};
|
|
2825
|
+
|
|
2826
|
+
// ../../packages/execute/src/test.ts
|
|
2827
|
+
var executeTest = async ({
|
|
2828
|
+
test,
|
|
2829
|
+
runId,
|
|
2830
|
+
controller,
|
|
2831
|
+
logger,
|
|
2832
|
+
onUpdateRun,
|
|
2833
|
+
onSaveScreenshot
|
|
2834
|
+
}) => {
|
|
2835
|
+
const advanced = TestAdvancedSettingsSchema.parse(test.advanced);
|
|
2836
|
+
logger.info(`Starting run ${runId} for test ${test.id}`);
|
|
2837
|
+
await onUpdateRun({
|
|
2838
|
+
status: "RUNNING",
|
|
2839
|
+
startedAt: /* @__PURE__ */ new Date()
|
|
2840
|
+
});
|
|
2841
|
+
let failed = false;
|
|
2842
|
+
const results = [];
|
|
2843
|
+
for (let i = 0; i < test.steps.length; i++) {
|
|
2844
|
+
const step = test.steps[i];
|
|
2845
|
+
let result;
|
|
2846
|
+
switch (step.type) {
|
|
2847
|
+
case "PRESET_ACTION" /* PRESET_ACTION */:
|
|
2848
|
+
result = await executePresetStep({
|
|
2849
|
+
controller,
|
|
2850
|
+
step,
|
|
2851
|
+
advanced,
|
|
2852
|
+
logger,
|
|
2853
|
+
onSaveScreenshot
|
|
2854
|
+
});
|
|
2855
|
+
break;
|
|
2856
|
+
case "AI_ACTION" /* AI_ACTION */:
|
|
2857
|
+
result = await executeAIStep({
|
|
2858
|
+
controller,
|
|
2859
|
+
step,
|
|
2860
|
+
advanced,
|
|
2861
|
+
logger,
|
|
2862
|
+
onSaveScreenshot
|
|
2863
|
+
});
|
|
2864
|
+
break;
|
|
2865
|
+
case "RESOLVED_MODULE":
|
|
2866
|
+
result = await executeModuleStep({
|
|
2867
|
+
controller,
|
|
2868
|
+
step,
|
|
2869
|
+
advanced,
|
|
2870
|
+
logger,
|
|
2871
|
+
onSaveScreenshot
|
|
2872
|
+
});
|
|
2873
|
+
break;
|
|
2874
|
+
default:
|
|
2875
|
+
const assertUnreachable = (_x) => {
|
|
2876
|
+
throw "If Typescript complains about the line below, you missed a case or break in the switch above";
|
|
2877
|
+
};
|
|
2878
|
+
return assertUnreachable(step);
|
|
2879
|
+
}
|
|
2880
|
+
results.push(result);
|
|
2881
|
+
await onUpdateRun({
|
|
2882
|
+
results
|
|
2883
|
+
});
|
|
2884
|
+
if (result.status === "FAILED" /* FAILED */) {
|
|
2885
|
+
failed = true;
|
|
2886
|
+
for (let j = i + 1; j < test.steps.length; j++) {
|
|
2887
|
+
const skippedStep = test.steps[j];
|
|
2888
|
+
if (skippedStep.type === "RESOLVED_MODULE") {
|
|
2889
|
+
const skippedResult = {
|
|
2890
|
+
type: "MODULE" /* MODULE */,
|
|
2891
|
+
moduleId: skippedStep.moduleId,
|
|
2892
|
+
startedAt: /* @__PURE__ */ new Date(),
|
|
2893
|
+
userAgent: ChromeBrowser.USER_AGENT,
|
|
2894
|
+
results: skippedStep.steps.map((s) => {
|
|
2895
|
+
return {
|
|
2896
|
+
...s,
|
|
2897
|
+
status: "CANCELLED" /* CANCELLED */,
|
|
2898
|
+
startedAt: /* @__PURE__ */ new Date(),
|
|
2899
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
2900
|
+
userAgent: ChromeBrowser.USER_AGENT,
|
|
2901
|
+
results: []
|
|
2902
|
+
};
|
|
2903
|
+
}),
|
|
2904
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
2905
|
+
status: "CANCELLED" /* CANCELLED */
|
|
2906
|
+
};
|
|
2907
|
+
results.push(skippedResult);
|
|
2908
|
+
} else {
|
|
2909
|
+
const skippedResult = {
|
|
2910
|
+
...skippedStep,
|
|
2911
|
+
status: "CANCELLED" /* CANCELLED */,
|
|
2912
|
+
startedAt: /* @__PURE__ */ new Date(),
|
|
2913
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
2914
|
+
userAgent: ChromeBrowser.USER_AGENT,
|
|
2915
|
+
results: []
|
|
2916
|
+
};
|
|
2917
|
+
results.push(skippedResult);
|
|
2918
|
+
}
|
|
2919
|
+
}
|
|
2920
|
+
}
|
|
2921
|
+
if (failed) {
|
|
2922
|
+
break;
|
|
2923
|
+
}
|
|
2924
|
+
}
|
|
2925
|
+
await onUpdateRun({
|
|
2926
|
+
status: failed ? "FAILED" : "PASSED",
|
|
2927
|
+
finishedAt: /* @__PURE__ */ new Date(),
|
|
2928
|
+
results
|
|
2929
|
+
});
|
|
2930
|
+
await controller.browser.cleanup();
|
|
2931
|
+
return failed;
|
|
2932
|
+
};
|
|
2933
|
+
|
|
2934
|
+
// src/run-test.ts
|
|
2935
|
+
var consoleLogger = {
|
|
2936
|
+
info: console.log,
|
|
2937
|
+
error: console.error,
|
|
2938
|
+
debug: console.debug,
|
|
2939
|
+
warn: console.warn,
|
|
2940
|
+
child: () => consoleLogger,
|
|
2941
|
+
flush: () => {
|
|
2942
|
+
}
|
|
2943
|
+
};
|
|
2944
|
+
async function runTest({
|
|
2945
|
+
testId,
|
|
2946
|
+
apiClient,
|
|
2947
|
+
generator
|
|
2948
|
+
}) {
|
|
2949
|
+
const test = await apiClient.getTest(testId);
|
|
2950
|
+
const browser = await ChromeBrowser.init(test.baseUrl, consoleLogger);
|
|
2951
|
+
const controller = new AgentController({
|
|
2952
|
+
browser,
|
|
2953
|
+
generator,
|
|
2954
|
+
config: DEFAULT_CONTROLLER_CONFIG,
|
|
2955
|
+
logger: consoleLogger
|
|
2956
|
+
});
|
|
2957
|
+
const run = await apiClient.createRun({
|
|
2958
|
+
testId
|
|
2959
|
+
});
|
|
2960
|
+
let failed = true;
|
|
2961
|
+
try {
|
|
2962
|
+
failed = await executeTest({
|
|
2963
|
+
test,
|
|
2964
|
+
runId: run.id,
|
|
2965
|
+
controller,
|
|
2966
|
+
logger: consoleLogger,
|
|
2967
|
+
onSaveScreenshot: async (buffer) => {
|
|
2968
|
+
const { key } = await apiClient.uploadScreenshot({
|
|
2969
|
+
screenshot: buffer.toString("base64")
|
|
2970
|
+
});
|
|
2971
|
+
return key;
|
|
2972
|
+
},
|
|
2973
|
+
onUpdateRun: async (data) => {
|
|
2974
|
+
await apiClient.updateRun(run.id, data);
|
|
2975
|
+
}
|
|
2976
|
+
});
|
|
2977
|
+
} catch (err) {
|
|
2978
|
+
await apiClient.updateRun(run.id, {
|
|
2979
|
+
status: "FAILED",
|
|
2980
|
+
finishedAt: /* @__PURE__ */ new Date()
|
|
2981
|
+
});
|
|
2982
|
+
}
|
|
2983
|
+
return failed;
|
|
2984
|
+
}
|
|
2985
|
+
|
|
2986
|
+
// src/cli.ts
|
|
2987
|
+
var program = new Command4();
|
|
2988
|
+
program.name("momentic").description("Momentic CLI").version(version);
|
|
2989
|
+
program.command("run-tests").addOption(
|
|
2990
|
+
new Option(
|
|
2991
|
+
"--tests <tests...>",
|
|
2992
|
+
"specify tests to run"
|
|
2993
|
+
).makeOptionMandatory(true)
|
|
2994
|
+
).addOption(
|
|
2995
|
+
new Option(
|
|
2996
|
+
"--start <command>",
|
|
2997
|
+
"specify start command"
|
|
2998
|
+
).makeOptionMandatory(true)
|
|
2999
|
+
).addOption(
|
|
3000
|
+
new Option("--wait-on <url>", "specify url to wait on").makeOptionMandatory(
|
|
3001
|
+
true
|
|
3002
|
+
)
|
|
3003
|
+
).addOption(
|
|
3004
|
+
new Option(
|
|
3005
|
+
"--wait-on-timeout <timeout>",
|
|
3006
|
+
"specify how long to wait on url"
|
|
3007
|
+
).default(60, "one minute")
|
|
3008
|
+
).addOption(
|
|
3009
|
+
new Option("--api-key <key>", "API key for authenticating").env("MOMENTIC_API_KEY").makeOptionMandatory(true)
|
|
3010
|
+
).action(async (options) => {
|
|
3011
|
+
const { tests, start, waitOn, waitOnTimeout, apiKey } = options;
|
|
3012
|
+
console.log({ tests, start, waitOn, waitOnTimeout, apiKey });
|
|
3013
|
+
void execa(start);
|
|
3014
|
+
await waitOnFn({
|
|
3015
|
+
resources: [waitOn],
|
|
3016
|
+
timeout: waitOnTimeout * 1e3
|
|
3017
|
+
});
|
|
3018
|
+
const apiClient = new APIClient({
|
|
3019
|
+
baseURL: "https://api.momentic.ai",
|
|
3020
|
+
apiKey
|
|
3021
|
+
});
|
|
3022
|
+
const apiGenerator = new APIGenerator({
|
|
3023
|
+
baseURL: "https://api.momentic.ai",
|
|
3024
|
+
apiKey
|
|
3025
|
+
});
|
|
3026
|
+
const promises = tests.map((testId) => {
|
|
3027
|
+
const failed = runTest({
|
|
3028
|
+
testId,
|
|
3029
|
+
apiClient,
|
|
3030
|
+
generator: apiGenerator
|
|
3031
|
+
});
|
|
3032
|
+
return { failed, testId };
|
|
3033
|
+
});
|
|
3034
|
+
const results = await Promise.all(promises);
|
|
3035
|
+
const failedResults = results.filter((result) => result.failed);
|
|
3036
|
+
if (failedResults.length > 0) {
|
|
3037
|
+
console.log(
|
|
3038
|
+
chalk.red(
|
|
3039
|
+
`Failed ${failedResults.length} out of ${results.length} tests`
|
|
3040
|
+
)
|
|
3041
|
+
);
|
|
3042
|
+
failedResults.forEach((result) => {
|
|
3043
|
+
console.log(chalk.red(`- ${result.testId}`));
|
|
3044
|
+
});
|
|
3045
|
+
process.exit(1);
|
|
3046
|
+
}
|
|
3047
|
+
console.log(chalk.green(`All ${results.length} tests passed!`));
|
|
3048
|
+
});
|
|
3049
|
+
async function main() {
|
|
3050
|
+
await program.parseAsync(process.argv);
|
|
3051
|
+
}
|
|
3052
|
+
void main();
|