e2e-testing-agent 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,74 @@
1
+ import OpenAI from "openai";
2
+ import { faker } from "@faker-js/faker";
3
+
4
+ export type ToolDefinition = {
5
+ name: string;
6
+ description: string;
7
+ parameters?: Record<string, any>;
8
+ handleCall: () => string;
9
+ };
10
+
11
+ const fakerTools: ToolDefinition[] = [
12
+ {
13
+ name: "get_email",
14
+ description: "Generates an email address.",
15
+ handleCall: () => {
16
+ return faker.internet.email();
17
+ },
18
+ },
19
+ {
20
+ name: "get_name",
21
+ description: "Generates a person's full name.",
22
+ handleCall: () => {
23
+ return faker.person.fullName();
24
+ },
25
+ },
26
+ ];
27
+
28
+ export function toolsForModelCall(
29
+ availableTools: ToolDefinition[]
30
+ ): OpenAI.Responses.FunctionTool[] {
31
+ return availableTools.map((tool) => ({
32
+ type: "function",
33
+ name: tool.name,
34
+ strict: false,
35
+ description: tool.description,
36
+ parameters: tool.parameters ?? {},
37
+ }));
38
+ }
39
+
40
+ // TODO: support all Faker functions
41
+ export function handleFunctionCalls(
42
+ response: OpenAI.Responses.Response,
43
+ availableTools: ToolDefinition[]
44
+ ): OpenAI.Responses.ResponseFunctionToolCallOutputItem[] {
45
+ const toolCalls = response.output.filter(
46
+ (item) => item.type === "function_call"
47
+ );
48
+ const functionCallOutputs = [];
49
+
50
+ const toolMap = new Map<string, ToolDefinition>();
51
+ for (const tool of availableTools) {
52
+ toolMap.set(tool.name, tool);
53
+ }
54
+
55
+ for (const toolCall of toolCalls) {
56
+ if (toolCall.type === "function_call") {
57
+ functionCallOutputs.push({
58
+ id: toolCall.call_id.replace(/^call_/, "fc_"),
59
+ call_id: toolCall.call_id,
60
+ type: "function_call_output",
61
+ output: (() => {
62
+ return (
63
+ toolMap.get(toolCall.name)?.handleCall() ??
64
+ `Unknown function: ${toolCall.name}`
65
+ );
66
+ })(),
67
+ });
68
+ }
69
+ }
70
+
71
+ return functionCallOutputs;
72
+ }
73
+
74
+ export const builtinTools = [...fakerTools];
@@ -0,0 +1,26 @@
1
+ import { test, expect } from "bun:test";
2
+ import { e2e_test } from "./e2e_agent";
3
+
4
+ test("supports email signup", async () => {
5
+ const success = await e2e_test(
6
+ "https://scalingfastbook.com",
7
+ "Sign up for a preview of the Scaling Fast book"
8
+ );
9
+ expect(success).toBe(true);
10
+ });
11
+
12
+ test("shows testimonials", async () => {
13
+ const success = await e2e_test(
14
+ "https://scalingfastbook.com",
15
+ "Verify the page has testimonials from readers who have benefited from the Scaling Fast book"
16
+ );
17
+ expect(success).toBe(true);
18
+ });
19
+
20
+ test("shows pitch video", async () => {
21
+ const success = await e2e_test(
22
+ "https://scalingfastbook.com",
23
+ "Verify the page has pitch video from the author"
24
+ );
25
+ expect(success).toBe(true);
26
+ });
package/package.json ADDED
@@ -0,0 +1,19 @@
1
+ {
2
+ "name": "e2e-testing-agent",
3
+ "module": "e2e_agent/index.ts",
4
+ "version": "0.1.0",
5
+ "type": "module",
6
+ "scripts": {},
7
+ "devDependencies": {
8
+ "@types/bun": "latest"
9
+ },
10
+ "peerDependencies": {
11
+ "typescript": "^5"
12
+ },
13
+ "dependencies": {
14
+ "@faker-js/faker": "^10.1.0",
15
+ "openai": "^6.15.0",
16
+ "playwright": "^1.57.0",
17
+ "uuid": "^13.0.0"
18
+ }
19
+ }
@@ -0,0 +1,39 @@
1
+ import { test, expect } from "bun:test";
2
+ import { e2e_test, continue_from } from "./e2e_agent";
3
+
4
+ test("can login", async () => {
5
+ const passed = await e2e_test(
6
+ "https://dev.plasmidsaurus.com",
7
+ `Login as ${process.env.TEST_USER_EMAIL}. You'll see a welcome message upon successful login.`,
8
+ [
9
+ {
10
+ name: "get_password",
11
+ description: "Returns the password for the test user.",
12
+ handleCall: () => process.env.TEST_USER_PASSWORD || "",
13
+ },
14
+ ]
15
+ );
16
+
17
+ expect(passed).toBe(true);
18
+ });
19
+
20
+ test("can purchase a plasmid", async () => {
21
+ const page = await continue_from(
22
+ "https://dev.plasmidsaurus.com",
23
+ `Login as ${process.env.TEST_USER_EMAIL}. You'll see a welcome message upon successful login.`
24
+ );
25
+
26
+ const passed = await e2e_test(
27
+ page,
28
+ "Purchase 3 standard high concentration plasmids using a purchase order. Use the 2 Tower Place, San Francisco dropbox as the shipping option",
29
+ [
30
+ {
31
+ name: "get_purchase_order",
32
+ description: "Returns a valid purchase order PO number.",
33
+ handleCall: () => "PO34238124",
34
+ },
35
+ ]
36
+ );
37
+
38
+ expect(passed).toBe(true);
39
+ });
@@ -0,0 +1,100 @@
1
+ {
2
+ "url": "https://dev.plasmidsaurus.com/",
3
+ "goal": "Login as swizec.teller+test_user@plasmidsaurus.com. You'll see a welcome message upon successful login.",
4
+ "computerCallStack": [
5
+ {
6
+ "id": "cu_079260ebdb70ac6000695b135da9488193942d379c9b79191f",
7
+ "type": "computer_call",
8
+ "status": "completed",
9
+ "action": {
10
+ "type": "click",
11
+ "button": "left",
12
+ "x": 1001,
13
+ "y": 114
14
+ },
15
+ "call_id": "call_lsUIvXmwxABKXEkAtyaI7uc2",
16
+ "pending_safety_checks": []
17
+ },
18
+ {
19
+ "id": "cu_079260ebdb70ac6000695b13624f6881939a8f799e5fae6106",
20
+ "type": "computer_call",
21
+ "status": "completed",
22
+ "action": {
23
+ "type": "wait"
24
+ },
25
+ "call_id": "call_k4FYmTVdsCrqQTlo2t3Yy5Tk",
26
+ "pending_safety_checks": []
27
+ },
28
+ {
29
+ "id": "cu_079260ebdb70ac6000695b1368e2608193a542423b23a49905",
30
+ "type": "computer_call",
31
+ "status": "completed",
32
+ "action": {
33
+ "type": "click",
34
+ "button": "left",
35
+ "x": 586,
36
+ "y": 280
37
+ },
38
+ "call_id": "call_KmAu9y4uB4fbCHfBDzwsJzoo",
39
+ "pending_safety_checks": []
40
+ },
41
+ {
42
+ "id": "cu_079260ebdb70ac6000695b136d0ce0819385f59a835653ccfb",
43
+ "type": "computer_call",
44
+ "status": "completed",
45
+ "action": {
46
+ "type": "type",
47
+ "text": "swizec.teller+test_user@plasmidsaurus.com"
48
+ },
49
+ "call_id": "call_46LjSJCUIDv1SFPywSmHxtZA",
50
+ "pending_safety_checks": []
51
+ },
52
+ {
53
+ "id": "cu_079260ebdb70ac6000695b1372a7608193947baff1ff55e87d",
54
+ "type": "computer_call",
55
+ "status": "completed",
56
+ "action": {
57
+ "type": "click",
58
+ "button": "left",
59
+ "x": 594,
60
+ "y": 357
61
+ },
62
+ "call_id": "call_XXPk9tgl3Z7Oi9tjXUkguiEn",
63
+ "pending_safety_checks": []
64
+ },
65
+ {
66
+ "id": "cu_079260ebdb70ac6000695b137dffcc81939cbfa50f84621508",
67
+ "type": "computer_call",
68
+ "status": "completed",
69
+ "action": {
70
+ "type": "type",
71
+ "text": "Zr%6tjyhHq*H*!W0"
72
+ },
73
+ "call_id": "call_uWeHqhCio4T4sdcH8ITxZ22H",
74
+ "pending_safety_checks": []
75
+ },
76
+ {
77
+ "id": "cu_079260ebdb70ac6000695b1383bab48193b54414c2c15be30c",
78
+ "type": "computer_call",
79
+ "status": "completed",
80
+ "action": {
81
+ "type": "click",
82
+ "button": "left",
83
+ "x": 613,
84
+ "y": 459
85
+ },
86
+ "call_id": "call_VmLBspwj03D7Tp3PqHPBanc5",
87
+ "pending_safety_checks": []
88
+ },
89
+ {
90
+ "id": "cu_079260ebdb70ac6000695b13881da8819381669a1e50992630",
91
+ "type": "computer_call",
92
+ "status": "completed",
93
+ "action": {
94
+ "type": "wait"
95
+ },
96
+ "call_id": "call_MKiOoFoO25a7lkknvNLVY29p",
97
+ "pending_safety_checks": []
98
+ }
99
+ ]
100
+ }
@@ -0,0 +1,315 @@
1
+ {
2
+ "url": "https://app.dev.plasmidsaurus.com/home",
3
+ "goal": "Purchase 3 standard high concentration plasmids using a purchase order. Use the 2 Tower Place, San Francisco dropbox as the shipping option",
4
+ "computerCallStack": [
5
+ {
6
+ "id": "cu_06865ed27262ec0400695b14c4aadc81979b6cbf581e7ca596",
7
+ "type": "computer_call",
8
+ "status": "completed",
9
+ "action": {
10
+ "type": "click",
11
+ "button": "left",
12
+ "x": 176,
13
+ "y": 412
14
+ },
15
+ "call_id": "call_itkbl2uWUtHHEdFYk7N18GCw",
16
+ "pending_safety_checks": []
17
+ },
18
+ {
19
+ "id": "cu_06865ed27262ec0400695b14cfa7d88197bd35a3dbd505d1d8",
20
+ "type": "computer_call",
21
+ "status": "completed",
22
+ "action": {
23
+ "type": "click",
24
+ "button": "left",
25
+ "x": 323,
26
+ "y": 678
27
+ },
28
+ "call_id": "call_UnhfBynU2xXG8mfzEbxepblp",
29
+ "pending_safety_checks": []
30
+ },
31
+ {
32
+ "id": "cu_06865ed27262ec0400695b14d9d3dc8197a55ade6e5e5e6396",
33
+ "type": "computer_call",
34
+ "status": "completed",
35
+ "action": {
36
+ "type": "scroll",
37
+ "scroll_x": 0,
38
+ "scroll_y": 409,
39
+ "x": 248,
40
+ "y": 702
41
+ },
42
+ "call_id": "call_eEObI958nKM1NGYLrUJX9fIC",
43
+ "pending_safety_checks": []
44
+ },
45
+ {
46
+ "id": "cu_06865ed27262ec0400695b14e4256481979d39995f20c06ea1",
47
+ "type": "computer_call",
48
+ "status": "completed",
49
+ "action": {
50
+ "type": "click",
51
+ "button": "left",
52
+ "x": 128,
53
+ "y": 393
54
+ },
55
+ "call_id": "call_EHzDg9MTDu6qOaOHUiWCl8hU",
56
+ "pending_safety_checks": []
57
+ },
58
+ {
59
+ "id": "cu_06865ed27262ec0400695b14f1423c819797069638097b7192",
60
+ "type": "computer_call",
61
+ "status": "completed",
62
+ "action": {
63
+ "type": "scroll",
64
+ "scroll_x": 0,
65
+ "scroll_y": 429,
66
+ "x": 224,
67
+ "y": 689
68
+ },
69
+ "call_id": "call_omRtE3Hafz4dly6C44SNScMe",
70
+ "pending_safety_checks": []
71
+ },
72
+ {
73
+ "id": "cu_06865ed27262ec0400695b14fec23881978a0afa49ff996077",
74
+ "type": "computer_call",
75
+ "status": "completed",
76
+ "action": {
77
+ "type": "click",
78
+ "button": "left",
79
+ "x": 200,
80
+ "y": 535
81
+ },
82
+ "call_id": "call_aLjiWXmaagywKICdQO9i6NAd",
83
+ "pending_safety_checks": []
84
+ },
85
+ {
86
+ "id": "cu_06865ed27262ec0400695b1505bb608197bca5ba6daea7047c",
87
+ "type": "computer_call",
88
+ "status": "completed",
89
+ "action": {
90
+ "type": "type",
91
+ "text": "Standard High Concentration Plasmids"
92
+ },
93
+ "call_id": "call_x8qnnmZjqtRKAOzJUNjSvIVI",
94
+ "pending_safety_checks": []
95
+ },
96
+ {
97
+ "id": "cu_06865ed27262ec0400695b1510beec8197a42eccba7f39c09b",
98
+ "type": "computer_call",
99
+ "status": "completed",
100
+ "action": {
101
+ "type": "scroll",
102
+ "scroll_x": 0,
103
+ "scroll_y": 472,
104
+ "x": 454,
105
+ "y": 645
106
+ },
107
+ "call_id": "call_RjBLCasxvNBEwxWj51Wxw0bG",
108
+ "pending_safety_checks": []
109
+ },
110
+ {
111
+ "id": "cu_06865ed27262ec0400695b151ac624819796e199831a669bf0",
112
+ "type": "computer_call",
113
+ "status": "completed",
114
+ "action": {
115
+ "type": "click",
116
+ "button": "left",
117
+ "x": 193,
118
+ "y": 370
119
+ },
120
+ "call_id": "call_I8s2BGNnaR8BwJb5DAqEbBD5",
121
+ "pending_safety_checks": []
122
+ },
123
+ {
124
+ "id": "cu_06865ed27262ec0400695b1520abf48197ad7cb74ad52773cb",
125
+ "type": "computer_call",
126
+ "status": "completed",
127
+ "action": {
128
+ "type": "type",
129
+ "text": "Plasmid 1\nPlasmid 2\nPlasmid 3"
130
+ },
131
+ "call_id": "call_JdRA26RTdr8OmkcckzY9z0e6",
132
+ "pending_safety_checks": []
133
+ },
134
+ {
135
+ "id": "cu_06865ed27262ec0400695b152b5c248197ae29fadbc69ae5e5",
136
+ "type": "computer_call",
137
+ "status": "completed",
138
+ "action": {
139
+ "type": "scroll",
140
+ "scroll_x": 0,
141
+ "scroll_y": 444,
142
+ "x": 791,
143
+ "y": 669
144
+ },
145
+ "call_id": "call_0VxxAyjMbxEQmPFQwRLFSBgs",
146
+ "pending_safety_checks": []
147
+ },
148
+ {
149
+ "id": "cu_06865ed27262ec0400695b153925848197a845c5ad2491b3f6",
150
+ "type": "computer_call",
151
+ "status": "completed",
152
+ "action": {
153
+ "type": "click",
154
+ "button": "left",
155
+ "x": 1096,
156
+ "y": 272
157
+ },
158
+ "call_id": "call_sbOVnfCMG03maFC3q45978aJ",
159
+ "pending_safety_checks": []
160
+ },
161
+ {
162
+ "id": "cu_06865ed27262ec0400695b1540623c81979640afc8c0f2fded",
163
+ "type": "computer_call",
164
+ "status": "completed",
165
+ "action": {
166
+ "type": "wait"
167
+ },
168
+ "call_id": "call_NGcCoWJPzVVxdr65DCMmZsgm",
169
+ "pending_safety_checks": []
170
+ },
171
+ {
172
+ "id": "cu_06865ed27262ec0400695b154c2fec8197b115842041b8e2ca",
173
+ "type": "computer_call",
174
+ "status": "completed",
175
+ "action": {
176
+ "type": "click",
177
+ "button": "left",
178
+ "x": 253,
179
+ "y": 680
180
+ },
181
+ "call_id": "call_EO756KjNLWyaxhdk6s9oKkD8",
182
+ "pending_safety_checks": []
183
+ },
184
+ {
185
+ "id": "cu_06865ed27262ec0400695b1551ade48197800e71844fbe1b41",
186
+ "type": "computer_call",
187
+ "status": "completed",
188
+ "action": {
189
+ "type": "type",
190
+ "text": "2 Tower Place, San Francisco"
191
+ },
192
+ "call_id": "call_EiXiJDeqiT4CRtWjHAdg1P2z",
193
+ "pending_safety_checks": []
194
+ },
195
+ {
196
+ "id": "cu_06865ed27262ec0400695b155d07ec8197b6828441af33ec4e",
197
+ "type": "computer_call",
198
+ "status": "completed",
199
+ "action": {
200
+ "type": "wait"
201
+ },
202
+ "call_id": "call_8shbiIESUVWq2vs7xAIS5NhD",
203
+ "pending_safety_checks": []
204
+ },
205
+ {
206
+ "id": "cu_06865ed27262ec0400695b15696c048197bd89b0ebe2f771ef",
207
+ "type": "computer_call",
208
+ "status": "completed",
209
+ "action": {
210
+ "type": "scroll",
211
+ "scroll_x": 0,
212
+ "scroll_y": 242,
213
+ "x": 694,
214
+ "y": 617
215
+ },
216
+ "call_id": "call_XaHMpy0scRJnMtkcHuen1YN9",
217
+ "pending_safety_checks": []
218
+ },
219
+ {
220
+ "id": "cu_06865ed27262ec0400695b1574b90881978ba8d0a3a5e9bb93",
221
+ "type": "computer_call",
222
+ "status": "completed",
223
+ "action": {
224
+ "type": "click",
225
+ "button": "left",
226
+ "x": 280,
227
+ "y": 576
228
+ },
229
+ "call_id": "call_HHZXHJR7Sxaiu1aiY0SaxrZN",
230
+ "pending_safety_checks": []
231
+ },
232
+ {
233
+ "id": "cu_06865ed27262ec0400695b157b59c4819797f8ed97d49f14fb",
234
+ "type": "computer_call",
235
+ "status": "completed",
236
+ "action": {
237
+ "type": "wait"
238
+ },
239
+ "call_id": "call_20suVBoDMvIrdy2Jk84Kj3v4",
240
+ "pending_safety_checks": []
241
+ },
242
+ {
243
+ "id": "cu_06865ed27262ec0400695b15895558819795dad7b87bc75812",
244
+ "type": "computer_call",
245
+ "status": "completed",
246
+ "action": {
247
+ "type": "scroll",
248
+ "scroll_x": 0,
249
+ "scroll_y": 486,
250
+ "x": 729,
251
+ "y": 705
252
+ },
253
+ "call_id": "call_ozu6HIeGEoxHJHMfm03rXppA",
254
+ "pending_safety_checks": []
255
+ },
256
+ {
257
+ "id": "cu_06865ed27262ec0400695b1595745481979f740393d5671b25",
258
+ "type": "computer_call",
259
+ "status": "completed",
260
+ "action": {
261
+ "type": "scroll",
262
+ "scroll_x": 0,
263
+ "scroll_y": 517,
264
+ "x": 863,
265
+ "y": 695
266
+ },
267
+ "call_id": "call_pU6BBYPhe4r6zcAxIRX5L6Hw",
268
+ "pending_safety_checks": []
269
+ },
270
+ {
271
+ "id": "cu_06865ed27262ec0400695b15ac74508197aea771ad75cd9cdb",
272
+ "type": "computer_call",
273
+ "status": "completed",
274
+ "action": {
275
+ "type": "click",
276
+ "button": "left",
277
+ "x": 234,
278
+ "y": 276
279
+ },
280
+ "call_id": "call_i4mFNei4gVfyaMdmLiJsrunc",
281
+ "pending_safety_checks": []
282
+ },
283
+ {
284
+ "id": "cu_06865ed27262ec0400695b15b2fa5881978c2256e5b67ffc08",
285
+ "type": "computer_call",
286
+ "status": "completed",
287
+ "action": {
288
+ "type": "type",
289
+ "text": "PO34238124"
290
+ },
291
+ "call_id": "call_SiZekoedPjIcBWFIHNFX0gQ4",
292
+ "pending_safety_checks": []
293
+ },
294
+ {
295
+ "id": "cu_06865ed27262ec0400695b15be1b988197b627b3ac8ac6dfee",
296
+ "type": "computer_call",
297
+ "status": "completed",
298
+ "action": {
299
+ "type": "scroll",
300
+ "scroll_x": 0,
301
+ "scroll_y": 510,
302
+ "x": 795,
303
+ "y": 675
304
+ },
305
+ "call_id": "call_KuMWEzzecAdG87EQZg7JKt0P",
306
+ "pending_safety_checks": [
307
+ {
308
+ "id": "cu_sc_06865ed27262ec0400695b15be52008197bda5a918d2bca859",
309
+ "code": "malicious_instructions",
310
+ "message": "We've detected instructions that may cause your application to perform malicious or unauthorized actions. Please acknowledge this warning if you'd like to proceed."
311
+ }
312
+ ]
313
+ }
314
+ ]
315
+ }