e2e-testing-agent 0.1.0

package/.env.example

@@ -0,0 +1,3 @@
+OPENAI_API_KEY=
+DISPLAY_WIDTH=1280
+DISPLAY_HEIGHT=720

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Swizec Teller
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,89 @@
+# e2e-testing-agent
+
+Here’s the idea:
+1. You write plain language goal oriented specs
+2. Agent looks at your UI and tries to follow the spec
+3. Writes cheap-to-run playwright based tests
+
+Goal is to make e2e testing ergonomic to use and cheap to update when things change.
+
+## What it looks like
+
+Here's an example of verifying a checkout flow. The agent figured it out from scratch on first run and recorded every action. Subsequent runs replay those actions on your page and check everything still works.
+
+[![](https://s3.amazonaws.com/techletter.app/screenshot-1767579154979.png)](https://x.com/Swizec/status/2006748633425985776)
+
+You specify the test in plain language. No fiddly Playwright commands, cumbersome testing code to write, or messing around with the DOM – your tests are fully declarative and goal oriented.
+
+```typescript
+test("can purchase a plasmid", async () => {
+    const page = await continue_from(
+        "https://dev.plasmidsaurus.com",
+        `Login as ${process.env.TEST_USER_EMAIL}. You'll see a welcome message upon successful login.`
+    );
+
+    const passed = await e2e_test(
+        page,
+        "Purchase 3 standard high concentration plasmids using a purchase order. Use the 2 Tower Place, San Francisco dropbox as the shipping option",
+        [
+            {
+                name: "get_purchase_order",
+                description: "Returns a valid purchase order PO number.",
+                handleCall: () => "PO34238124",
+            },
+        ]
+    );
+
+    expect(passed).toBe(true);
+});
+```
+
+## How to use
+
+`e2e-testing-agent` is extremely early software. Use with caution and please report sharp edges. I have tested with `bun test` but the library should work with every test runner.
+
+The core API consists of 2 functions:
+
+### e2e_test
+
+```typescript
+e2e_test(
+    start_location: string | Page,
+    goal: string,
+    tools: ToolDefinition[] = []
+): Promise<boolean>
+```
+
+Accepts a URL or open Playwright page as the starting point. The `goal` specifies what you want the agent to achieve.
+
+Returns pass/fail as a boolean.
+
+Uses agentic mode on first run then blindly repeats those steps on subsequent runs. 
+
+### continue_from
+
+```typescript
+continue_from(url: string, goal: string): Promise<Page>
+```
+
+Accepts url and goal that should match an existing test with a stored replay. Returns a page with those steps replayed.
+
+This lets you reuse existing verified steps so your tests are cheaper to run. Similar idea as fixtures.
+
+### Provide tools to the agent
+
+The agent can use tools (function calls) to do work on the page. You can use this to provide custom values for certain actions or execute custom code that interacts with your system.
+
+You pass tools as an array of objects to `e2e_test`:
+
+```typescript
+[
+    {
+        name: "get_password",
+        description: "Returns the password for the test user.",
+        handleCall: () => process.env.TEST_USER_PASSWORD || "",
+    },
+]
+```
+
+Every tool needs a `name` and `description` that the agent can use to figure out which tools to use, and `handleCall` which is the function to execute. 

package/e2e_agent/agent.ts

@@ -0,0 +1,417 @@
+import OpenAI from "openai";
+import { type Page } from "playwright";
+import {
+    builtinTools,
+    handleFunctionCalls,
+    toolsForModelCall,
+    type ToolDefinition,
+} from "./tools";
+import { getReplay, storeReplay } from "./memory";
+
+const openai = new OpenAI();
+
+async function handleModelAction(
+    page: Page,
+    action:
+        | OpenAI.Responses.ResponseComputerToolCall.Click
+        | OpenAI.Responses.ResponseComputerToolCall.DoubleClick
+        | OpenAI.Responses.ResponseComputerToolCall.Drag
+        | OpenAI.Responses.ResponseComputerToolCall.Keypress
+        | OpenAI.Responses.ResponseComputerToolCall.Move
+        | OpenAI.Responses.ResponseComputerToolCall.Screenshot
+        | OpenAI.Responses.ResponseComputerToolCall.Scroll
+        | OpenAI.Responses.ResponseComputerToolCall.Type
+        | OpenAI.Responses.ResponseComputerToolCall.Wait
+) {
+    // Given a computer action (e.g., click, double_click, scroll, etc.),
+    // execute the corresponding operation on the Playwright page.
+
+    const actionType = action.type;
+
+    try {
+        switch (actionType) {
+            case "click": {
+                const { x, y, button = "left" } = action;
+                console.log(
+                    `Action: click at (${x}, ${y}) with button '${button}'`
+                );
+                await page.mouse.click(x, y, { button });
+                break;
+            }
+
+            case "scroll": {
+                const { x, y, scroll_x, scroll_y } = action;
+                console.log(
+                    `Action: scroll at (${x}, ${y}) with offsets (scrollX=${scroll_x}, scrollY=${scroll_y})`
+                );
+                await page.mouse.move(x, y);
+                await page.evaluate(
+                    `window.scrollBy(${scroll_x}, ${scroll_y})`
+                );
+                break;
+            }
+
+            case "keypress": {
+                const { keys } = action;
+                for (const k of keys) {
+                    console.log(`Action: keypress '${k}'`);
+                    // A simple mapping for common keys; expand as needed.
+                    if (k.includes("ENTER")) {
+                        await page.keyboard.press("Enter");
+                    } else if (k.includes("SPACE")) {
+                        await page.keyboard.press(" ");
+                    } else if (k.includes("CMD")) {
+                        await page.keyboard.press("ControlOrMeta");
+                    } else {
+                        await page.keyboard.press(k);
+                    }
+                }
+                break;
+            }
+
+            case "type": {
+                const { text } = action;
+                console.log(`Action: type text '${text}'`);
+                await page.keyboard.type(text);
+                break;
+            }
+
+            case "wait": {
+                console.log(`Action: wait`);
+                await page.waitForTimeout(3000);
+                break;
+            }
+
+            case "screenshot": {
+                // Nothing to do as screenshot is taken at each turn
+                console.log(`Action: screenshot`);
+                break;
+            }
+
+            // Handle other actions here
+
+            default:
+                console.log("Unrecognized action:", action);
+        }
+    } catch (e) {
+        console.error("Error handling action", action, ":", e);
+    }
+}
+
+async function computerUseLoop(
+    page: Page,
+    response: OpenAI.Responses.Response,
+    availableTools: ToolDefinition[]
+): Promise<[OpenAI.Responses.Response, any[]]> {
+    /**
+     * Run the loop that executes computer actions and tool calls until no 'computer_call' or 'function_call' is found.
+     */
+    const computerCallStack = [];
+
+    while (true) {
+        console.debug(response.output);
+
+        const functionCallOutputs = handleFunctionCalls(
+            response,
+            availableTools
+        );
+
+        const computerCalls = response.output.filter(
+            (item) => item.type === "computer_call"
+        );
+
+        if (computerCalls.length === 0 && functionCallOutputs.length === 0) {
+            console.debug("No computer or tool calls found. Final output:");
+            console.debug(JSON.stringify(response.output, null, 2));
+
+            break; // Exit when no computer calls are issued.
+        }
+
+        let hasNewAction = false;
+
+        if (computerCalls[0]) {
+            computerCallStack.push(computerCalls[0]);
+            hasNewAction = true;
+        }
+
+        // We expect at most one computer call per response.
+        const lastComputerCall =
+            computerCallStack[computerCallStack.length - 1];
+        const lastCallId = lastComputerCall?.call_id;
+        const action = lastComputerCall?.action;
+        const safetyChecks = lastComputerCall?.pending_safety_checks || [];
+
+        console.debug("safety checks", safetyChecks);
+
+        if (hasNewAction && action) {
+            // Execute the action (function defined in step 3)
+            handleModelAction(page, action);
+            await new Promise((resolve) => setTimeout(resolve, 200)); // Allow time for changes to take effect.
+        }
+
+        // Take a screenshot after the action
+        const screenshotBytes = await page.screenshot();
+        const screenshotBase64 =
+            Buffer.from(screenshotBytes).toString("base64");
+
+        console.debug("Calling model with screenshot and tool outputs...");
+
+        // Send the screenshot back as a computer_call_output
+        response = await openai.responses.create({
+            model: "computer-use-preview",
+            previous_response_id: response.id,
+            tools: [
+                {
+                    type: "computer_use_preview",
+                    display_width: Number(process.env.DISPLAY_WIDTH),
+                    display_height: Number(process.env.DISPLAY_HEIGHT),
+                    environment: "browser",
+                },
+                ...toolsForModelCall(availableTools),
+            ],
+            input: [
+                {
+                    call_id: lastCallId,
+                    type: "computer_call_output",
+                    acknowledged_safety_checks: safetyChecks,
+                    output: {
+                        type: "input_image",
+                        image_url: `data:image/png;base64,${screenshotBase64}`,
+                    },
+                },
+                ...functionCallOutputs,
+            ],
+            truncation: "auto",
+        });
+    }
+
+    return [response, computerCallStack];
+}
+
+async function verifyGoalAchieved(
+    response: OpenAI.Responses.Response,
+    goal: string
+): Promise<boolean> {
+    /**
+     * Verify if the goal has been achieved based on the model's final response.
+     */
+
+    console.debug(
+        "Verifying response against goal",
+        JSON.stringify(
+            [
+                {
+                    role: "user",
+                    content: `Based on the following information, did the agent successfully accomplish the goal: "${goal}"? Respond with "yes" or "no" only.`,
+                },
+                {
+                    role: "user",
+                    content: `Final agent response: ${JSON.stringify(
+                        response.output,
+                        null,
+                        2
+                    )}`,
+                },
+            ],
+            null,
+            2
+        )
+    );
+
+    const verificationResponse = await openai.responses.create({
+        model: "gpt-5-nano",
+        input: [
+            {
+                role: "user",
+                content: `Based on the following information, did the agent successfully accomplish the goal: "${goal}"? Respond with "yes" or "no" only.`,
+            },
+            {
+                role: "user",
+                content: `Final agent response: ${JSON.stringify(
+                    response.output,
+                    null,
+                    2
+                )}`,
+            },
+        ],
+    });
+
+    console.debug(JSON.stringify(verificationResponse.output, null, 2));
+
+    const answer = verificationResponse.output
+        .map((item) => {
+            if (item.type === "message") {
+                return item.content
+                    .filter((c) => c.type === "output_text")
+                    .map((c) => c.text)
+                    .join(" ")
+                    .toString()
+                    .toLowerCase();
+            }
+            return "";
+        })
+        .join(" ");
+
+    console.debug("Verification answer:", answer);
+    return answer.includes("yes") && !answer.includes("no");
+}
+
+async function verifyGoalAchievedFromScreenshot(
+    screenshotBase64: string,
+    goal: string
+): Promise<boolean> {
+    /**
+     * Verify if the goal has been achieved based on the model's final response.
+     */
+
+    const verificationResponse = await openai.responses.create({
+        model: "gpt-5-nano",
+        input: [
+            {
+                role: "user",
+                content: [
+                    {
+                        type: "input_text",
+                        text: `Based on the following information, did the agent successfully accomplish the goal: "${goal}"? Respond with "yes" or "no" only.`,
+                    },
+                    {
+                        type: "input_image",
+                        image_url: `data:image/png;base64,${screenshotBase64}`,
+                        detail: "high",
+                    },
+                ],
+            },
+        ],
+    });
+
+    console.debug(JSON.stringify(verificationResponse.output, null, 2));
+
+    const answer = verificationResponse.output
+        .map((item) => {
+            if (item.type === "message") {
+                return item.content
+                    .filter((c) => c.type === "output_text")
+                    .map((c) => c.text)
+                    .join(" ")
+                    .toString()
+                    .toLowerCase();
+            }
+            return "";
+        })
+        .join(" ");
+
+    console.debug("Verification answer:", answer);
+    return answer.includes("yes") && !answer.includes("no");
+}
+
+export async function replayActions(
+    url: string,
+    goal: string,
+    page: Page
+): Promise<boolean> {
+    const computerCallStack = await getReplay(url, goal);
+
+    if (computerCallStack) {
+        console.debug(
+            `Restoring previous computer call stack with ${computerCallStack.length} actions...`
+        );
+
+        for (const call of computerCallStack) {
+            const action = call.action;
+            if (action) {
+                await handleModelAction(page, action);
+                await new Promise((resolve) => setTimeout(resolve, 500)); // Allow time for changes to take effect.
+            }
+        }
+    }
+
+    return !!computerCallStack;
+}
+
+async function takeScreenshotAsBase64(page: Page): Promise<string> {
+    const screenshotBytes = await page.screenshot();
+    const screenshotBase64 = Buffer.from(screenshotBytes).toString("base64");
+    return screenshotBase64;
+}
+
+export async function testFromReplay(
+    url: string,
+    goal: string,
+    page: Page
+): Promise<boolean> {
+    await replayActions(url, goal, page);
+    const screenshotBase64 = await takeScreenshotAsBase64(page);
+
+    const passed = await verifyGoalAchievedFromScreenshot(
+        screenshotBase64,
+        goal
+    );
+    // browser.close();
+
+    return passed;
+}
+
+export async function testFromScratch(
+    url: string,
+    goal: string,
+    page: Page,
+    tools: ToolDefinition[]
+): Promise<boolean> {
+    const openai = new OpenAI();
+    const availableTools: ToolDefinition[] = [...builtinTools, ...tools];
+
+    const screenshotBase64 = await takeScreenshotAsBase64(page);
+
+    const response = await openai.responses.create({
+        model: "computer-use-preview",
+        tools: [
+            {
+                type: "computer_use_preview",
+                display_width: Number(process.env.DISPLAY_WIDTH),
+                display_height: Number(process.env.DISPLAY_HEIGHT),
+                environment: "browser",
+            },
+            ...toolsForModelCall(availableTools),
+        ],
+        input: [
+            {
+                role: "system",
+                content:
+                    "You are an autonomous agent running in a sandbox environment designed to test web applications. Use the browser and provided tools to accomplish the user's goal. Submit forms without asking the user for confirmation.",
+            },
+            {
+                role: "user",
+                content: [
+                    {
+                        type: "input_text",
+                        text: `You are in a browser navigated to ${url}. ${goal}.`,
+                    },
+                    {
+                        type: "input_image",
+                        image_url: `data:image/png;base64,${screenshotBase64}`,
+                        detail: "high",
+                    },
+                ],
+            },
+        ],
+        reasoning: {
+            summary: "concise",
+        },
+        truncation: "auto",
+    });
+
+    const [finalResponse, computerCallStack] = await computerUseLoop(
+        page,
+        response,
+        availableTools
+    );
+
+    await storeReplay(url, goal, computerCallStack);
+
+    // browser.close();
+
+    console.debug("Verifying goal achieved");
+    const passed = await verifyGoalAchieved(finalResponse, goal);
+
+    return passed;
+}

package/e2e_agent/index.ts

@@ -0,0 +1,60 @@
+import { chromium, type Page } from "playwright";
+import { type ToolDefinition } from "./tools";
+import { canReplayActions } from "./memory";
+import { replayActions, testFromReplay, testFromScratch } from "./agent";
+
+async function openBrowser(url: string): Promise<Page> {
+    const browser = await chromium.launch({
+        headless: false,
+        chromiumSandbox: true,
+        env: {},
+        args: ["--disable-extensions", "--disable-filesystem"],
+    });
+
+    const displayWidth = Number(process.env.DISPLAY_WIDTH);
+    const displayHeight = Number(process.env.DISPLAY_HEIGHT);
+
+    const page = await browser.newPage();
+    await page.setViewportSize({
+        width: displayWidth,
+        height: displayHeight,
+    });
+    await page.goto(url);
+
+    return page;
+}
+
+export async function continue_from(url: string, goal: string): Promise<Page> {
+    const page = await openBrowser(url);
+    const replayed = await replayActions(url, goal, page);
+
+    if (!replayed) {
+        throw new Error(
+            "No saved computer call stack found for the given URL and goal."
+        );
+    }
+
+    return page;
+}
+
+export async function e2e_test(
+    start_location: string | Page,
+    goal: string,
+    tools: ToolDefinition[] = []
+): Promise<boolean> {
+    let page: Page;
+
+    if (typeof start_location === "string") {
+        page = await openBrowser(start_location);
+    } else {
+        page = start_location;
+    }
+
+    const startUrl = page.url();
+
+    if (await canReplayActions(startUrl, goal)) {
+        return await testFromReplay(startUrl, goal, page);
+    } else {
+        return await testFromScratch(startUrl, goal, page, tools);
+    }
+}

package/e2e_agent/memory.ts

@@ -0,0 +1,110 @@
+import fs from "fs/promises";
+import path from "path";
+import crypto from "crypto";
+
+function getCallingTestFilename() {
+    const err = new Error();
+    const stack = err.stack?.split("\n") ?? [];
+
+    // Skip internal frames until we leave node_modules / your framework
+    const frame = stack.find(
+        (line) =>
+            // prefer test/spec files but skip node_modules
+            !line.includes("node_modules") &&
+            (line.includes(".test") || line.includes(".spec"))
+    );
+
+    const callingTestInfo = frame?.trim();
+
+    // Examples of frames we expect:
+    // at async <anonymous> (/path/to/file.test.ts:5:26)
+    // at /path/to/file.test.ts:5:26
+    const m =
+        callingTestInfo?.match(/\((.*?):(\d+):(\d+)\)$/) ||
+        callingTestInfo?.match(/at (.*?):(\d+):(\d+)$/);
+
+    const filename = m?.[1] ?? null;
+
+    // Return only the basename (file name with extension), not the full path
+    const base = filename ? path.basename(filename) : null;
+
+    return base;
+}
+
+function getFilename(url: string, goal: string): string {
+    const hash = crypto
+        .createHash("md5")
+        .update(url.replace(/\/$/, "") + goal)
+        .digest("hex");
+
+    // const testFilename = getCallingTestFilename();
+    // TODO: use more human readable filenames
+    const filename = `test_replays/${hash}.json`;
+
+    return filename;
+}
+
+export async function storeReplay(
+    url: string,
+    goal: string,
+    computerCallStack: any[]
+) {
+    /**
+     * Store the computer call stack for future reference.
+     */
+    const filename = getFilename(url, goal);
+
+    // Ensure the directory for the filename exists before writing.
+    const dir = path.dirname(filename);
+    await fs.mkdir(dir, { recursive: true });
+
+    await fs.writeFile(
+        filename,
+        JSON.stringify(
+            {
+                url,
+                goal,
+                computerCallStack,
+            },
+            null,
+            2
+        )
+    );
+}
+
+export async function getReplay(
+    url: string,
+    goal: string
+): Promise<any[] | null> {
+    /**
+     * Restore the computer call stack if it exists.
+     */
+    const filename = getFilename(url, goal);
+
+    console.debug({ url, goal, filename });
+
+    try {
+        const data = await fs.readFile(filename, "utf-8");
+        const parsed = JSON.parse(data);
+        return parsed.computerCallStack;
+    } catch (e) {
+        return null;
+    }
+}
+
+export async function canReplayActions(
+    url: string,
+    goal: string
+): Promise<boolean> {
+    /**
+     * Check if a replay exists for the given URL and goal.
+     */
+    const filename = getFilename(url, goal);
+
+    try {
+        await fs.access(filename);
+        return true;
+    } catch (e) {
+        return false;
+    }
+}