expect-sdk 0.0.24

package/src/test-run.ts

@@ -0,0 +1,19 @@
+import type { TestResult, TestEvent, TestRun } from "./types";
+
+interface TestRunCallbacks {
+  readonly promise: Promise<TestResult>;
+  readonly subscribe: () => AsyncIterableIterator<TestEvent>;
+}
+
+export const createTestRun = (callbacks: TestRunCallbacks): TestRun => ({
+  then<TResult1 = TestResult, TResult2 = never>(
+    onfulfilled?: ((value: TestResult) => TResult1 | PromiseLike<TResult1>) | undefined | null,
+    onrejected?: ((reason: unknown) => TResult2 | PromiseLike<TResult2>) | undefined | null,
+  ): PromiseLike<TResult1 | TResult2> {
+    return callbacks.promise.then(onfulfilled, onrejected);
+  },
+
+  [Symbol.asyncIterator](): AsyncIterator<TestEvent> {
+    return callbacks.subscribe();
+  },
+});

package/src/tool.ts

@@ -0,0 +1,26 @@
+import type { StandardJSONSchemaV1 } from "@standard-schema/spec";
+import type { Tool } from "./types";
+
+type SchemaInput = StandardJSONSchemaV1 | Record<string, unknown>;
+
+const isStandardSchema = (schema: SchemaInput): schema is StandardJSONSchemaV1 =>
+  "~standard" in schema && typeof (schema as Record<string, unknown>)["~standard"] === "object";
+
+const toInputSchema = (schema: SchemaInput): Tool["inputSchema"] => {
+  if (isStandardSchema(schema)) {
+    return schema["~standard"].jsonSchema.input({ target: "draft-07" }) as Tool["inputSchema"];
+  }
+  return schema as Tool["inputSchema"];
+};
+
+export const tool = (
+  name: string,
+  description: string,
+  schema: SchemaInput,
+  handler: (input: Record<string, unknown>) => Promise<string>,
+): Tool => ({
+  name,
+  description,
+  inputSchema: toInputSchema(schema),
+  handler,
+});

package/src/types.ts

@@ -0,0 +1,98 @@
+import type { Page, BrowserContext, Cookie } from "playwright";
+import type { Tool as McpTool } from "@modelcontextprotocol/sdk/types.js";
+
+export type { Cookie };
+
+export type Action = string | ((page: Page) => Promise<void | string>);
+
+export type BrowserName = "chrome" | "firefox" | "safari" | "edge" | "brave" | "arc";
+
+export type CookieInput = true | BrowserName | BrowserName[] | Cookie[];
+
+export type Test = string | { title?: string; prompt: string };
+
+export type Status = "pending" | "passed" | "failed";
+
+export interface Tool extends Pick<McpTool, "name" | "description" | "inputSchema"> {
+  readonly handler: (input: Record<string, unknown>) => Promise<string>;
+}
+
+export interface StepResult {
+  readonly title: string;
+  readonly status: Status;
+  readonly summary: string;
+  readonly screenshotPath?: string;
+  readonly duration: number;
+}
+
+export interface TestResult {
+  readonly status: Status;
+  readonly url: string;
+  readonly duration: number;
+  readonly recordingPath?: string;
+  readonly steps: StepResult[];
+  readonly errors: StepResult[];
+}
+
+export type TestEvent =
+  | { readonly type: "run:started"; readonly title: string; readonly baseUrl?: string }
+  | { readonly type: "step:started"; readonly title: string }
+  | { readonly type: "step:passed"; readonly step: StepResult }
+  | { readonly type: "step:failed"; readonly step: StepResult }
+  | { readonly type: "step:skipped"; readonly title: string; readonly reason: string }
+  | { readonly type: "screenshot"; readonly title: string; readonly path: string }
+  | { readonly type: "completed"; readonly result: TestResult };
+
+export interface TestRun extends PromiseLike<TestResult> {
+  [Symbol.asyncIterator](): AsyncIterator<TestEvent>;
+}
+
+export interface TestInput {
+  readonly url?: string;
+  readonly page?: Page;
+  readonly cookies?: CookieInput;
+  readonly tools?: Tool[];
+  readonly tests: Test[];
+  readonly before?: Action;
+  readonly after?: Action;
+  readonly mode?: "headed" | "headless";
+  readonly timeout?: number;
+  readonly isRecording?: boolean;
+}
+
+export interface SessionConfig {
+  readonly url?: string;
+  readonly browserContext?: BrowserContext;
+  readonly cookies?: CookieInput;
+  readonly tools?: Tool[];
+  readonly mode?: "headed" | "headless";
+  readonly timeout?: number;
+  readonly isRecording?: boolean;
+}
+
+export interface SessionTestInput {
+  readonly url?: string;
+  readonly tests: Test[];
+  readonly before?: Action;
+  readonly after?: Action;
+  readonly mode?: "headed" | "headless";
+  readonly timeout?: number;
+  readonly isRecording?: boolean;
+}
+
+export interface ExpectSession {
+  test(input: SessionTestInput): TestRun;
+  close(): Promise<void>;
+  [Symbol.asyncDispose](): Promise<void>;
+}
+
+export interface ExpectConfig {
+  readonly baseUrl?: string;
+  readonly browser?: BrowserName;
+  readonly mode?: "headed" | "headless";
+  readonly cookies?: CookieInput;
+  readonly timeout?: number;
+  readonly model?: string;
+  readonly apiKey?: string;
+  readonly rootDir?: string;
+}

package/tests/build-instruction.test.ts

@@ -0,0 +1,62 @@
+import { describe, it, expect } from "vite-plus/test";
+import { resolveUrl, buildInstruction } from "../src/build-instruction";
+
+describe("resolveUrl", () => {
+  it("passes through absolute http URLs", () => {
+    expect(resolveUrl("http://localhost:3000/login", undefined)).toBe(
+      "http://localhost:3000/login",
+    );
+  });
+
+  it("passes through absolute https URLs", () => {
+    expect(resolveUrl("https://example.com/page", undefined)).toBe("https://example.com/page");
+  });
+
+  it("resolves relative URL against baseUrl", () => {
+    expect(resolveUrl("/login", "http://localhost:3000")).toBe("http://localhost:3000/login");
+  });
+
+  it("resolves relative URL without leading slash", () => {
+    expect(resolveUrl("login", "http://localhost:3000")).toBe("http://localhost:3000/login");
+  });
+
+  it("strips trailing slash from baseUrl", () => {
+    expect(resolveUrl("/login", "http://localhost:3000/")).toBe("http://localhost:3000/login");
+  });
+
+  it("ignores baseUrl for absolute URLs", () => {
+    expect(resolveUrl("http://other.com/page", "http://localhost:3000")).toBe(
+      "http://other.com/page",
+    );
+  });
+
+  it("throws for non-string URL", () => {
+    expect(() => resolveUrl(undefined, undefined)).toThrow("Expected a URL string");
+  });
+
+  it("throws for relative URL without baseUrl", () => {
+    expect(() => resolveUrl("/login", undefined)).toThrow("No baseUrl configured");
+  });
+
+  it("includes Fix line in error", () => {
+    try {
+      resolveUrl("/login", undefined);
+    } catch (error) {
+      expect((error as Error).message).toContain("Fix:");
+    }
+  });
+});
+
+describe("buildInstruction", () => {
+  it("builds single test", () => {
+    const result = buildInstruction("http://localhost:3000/login", ["login works"]);
+    expect(result).toContain("1. login works");
+    expect(result).toContain("Navigate to http://localhost:3000/login");
+  });
+
+  it("numbers multiple tests", () => {
+    const result = buildInstruction("http://localhost:3000", ["test one", "test two"]);
+    expect(result).toContain("1. test one");
+    expect(result).toContain("2. test two");
+  });
+});

package/tests/config.test.ts

@@ -0,0 +1,44 @@
+import { describe, it, expect, beforeEach } from "vite-plus/test";
+import { defineConfig, configure, getGlobalConfig, resetGlobalConfig } from "../src/config";
+
+describe("defineConfig", () => {
+  it("returns the same reference", () => {
+    const config = { baseUrl: "http://localhost:3000" };
+    expect(defineConfig(config)).toBe(config);
+  });
+});
+
+describe("configure", () => {
+  beforeEach(() => {
+    resetGlobalConfig();
+  });
+
+  it("merges into global config", () => {
+    configure({ baseUrl: "http://localhost:3000" });
+    expect(getGlobalConfig().baseUrl).toBe("http://localhost:3000");
+  });
+
+  it("merges successive calls", () => {
+    configure({ baseUrl: "http://localhost:3000" });
+    configure({ mode: "headed" });
+    const config = getGlobalConfig();
+    expect(config.baseUrl).toBe("http://localhost:3000");
+    expect(config.mode).toBe("headed");
+  });
+
+  it("later values override earlier ones", () => {
+    configure({ baseUrl: "http://localhost:3000" });
+    configure({ baseUrl: "http://localhost:4000" });
+    expect(getGlobalConfig().baseUrl).toBe("http://localhost:4000");
+  });
+});
+
+describe("resetGlobalConfig", () => {
+  it("clears all values", () => {
+    configure({ baseUrl: "http://localhost:3000", mode: "headed" });
+    resetGlobalConfig();
+    const config = getGlobalConfig();
+    expect(config.baseUrl).toBeUndefined();
+    expect(config.mode).toBeUndefined();
+  });
+});

package/tests/e2e.ts

@@ -0,0 +1,81 @@
+import { Expect } from "../src/expect";
+import { configure, resetGlobalConfig } from "../src/config";
+import { startFixtureServer } from "./fixtures/fixture-server";
+import type { TestEvent } from "../src/types";
+
+const server = await startFixtureServer();
+const fixtureUrl = server.url;
+
+console.log(`Fixture server started at ${fixtureUrl}`);
+
+try {
+  console.log("\n--- test: page loads correctly ---");
+  const basic = await Expect.test({
+    url: fixtureUrl,
+    tests: ["the page loads and shows a heading"],
+  });
+  console.log(`Status: ${basic.status} (${basic.steps.length} steps)`);
+  for (const step of basic.steps) {
+    console.log(`  ${step.status === "passed" ? "✅" : "❌"} ${step.title}`);
+  }
+  if (basic.status !== "passed") throw new Error("basic test failed");
+
+  console.log("\n--- test: streaming events ---");
+  const run = Expect.test({
+    url: fixtureUrl,
+    tests: ["the main heading says Welcome"],
+  });
+  const events: TestEvent[] = [];
+  for await (const event of run) {
+    events.push(event);
+    if (event.type === "step:passed") console.log(`  ✅ ${event.step.title}`);
+    if (event.type === "step:failed") console.log(`  ❌ ${event.step.title}`);
+  }
+  const streamResult = await run;
+  console.log(`Status: ${streamResult.status} (${events.length} events)`);
+  if (events.length === 0) throw new Error("no events received");
+
+  console.log("\n--- test: login page ---");
+  const loginResult = await Expect.test({
+    url: `${fixtureUrl}/login`,
+    tests: ["the login page has an email field and a password field"],
+  });
+  console.log(`Status: ${loginResult.status}`);
+
+  console.log("\n--- test: configure baseUrl ---");
+  configure({ baseUrl: fixtureUrl });
+  const signupResult = await Expect.test({
+    url: "/signup",
+    tests: ["the signup page has email, password, and confirm password fields"],
+  });
+  console.log(`Status: ${signupResult.status}`);
+  resetGlobalConfig();
+
+  console.log("\n--- test: session ---");
+  const session = Expect.session({ url: fixtureUrl });
+  const homeResult = await session.test({
+    url: fixtureUrl,
+    tests: ["the homepage has navigation links"],
+  });
+  console.log(`Home: ${homeResult.status}`);
+  const sessionLoginResult = await session.test({
+    url: `${fixtureUrl}/login`,
+    tests: ["the login page has a sign-in form"],
+  });
+  console.log(`Login: ${sessionLoginResult.status}`);
+  await session.close();
+
+  const allPassed = [
+    basic,
+    streamResult,
+    loginResult,
+    signupResult,
+    homeResult,
+    sessionLoginResult,
+  ].every((result) => result.status === "passed");
+
+  console.log(`\nAll passed: ${allPassed}`);
+  process.exit(allPassed ? 0 : 1);
+} finally {
+  await server.close();
+}

package/tests/examples.test.ts

@@ -0,0 +1,44 @@
+import { describe, it, expect, beforeEach } from "vite-plus/test";
+import { configure, resetGlobalConfig } from "../src/config";
+import { resolveUrl, buildInstruction } from "../src/build-instruction";
+import { Expect } from "../src/expect";
+
+describe("example patterns — validation only", () => {
+  beforeEach(() => {
+    resetGlobalConfig();
+  });
+
+  it("basic: resolves absolute url", () => {
+    const url = resolveUrl("http://localhost:3000", undefined);
+    expect(url).toBe("http://localhost:3000/");
+  });
+
+  it("auth: configure baseUrl resolves relative urls", () => {
+    configure({ baseUrl: "http://localhost:3000" });
+    const loginUrl = resolveUrl("/login", "http://localhost:3000");
+    const signupUrl = resolveUrl("/signup", "http://localhost:3000");
+    expect(loginUrl).toBe("http://localhost:3000/login");
+    expect(signupUrl).toBe("http://localhost:3000/signup");
+  });
+
+  it("session: creates session with test and close methods", () => {
+    const session = Expect.session({ url: "http://localhost:3000" });
+    expect(typeof session.test).toBe("function");
+    expect(typeof session.close).toBe("function");
+    expect(typeof session[Symbol.asyncDispose]).toBe("function");
+  });
+
+  it("instruction: builds numbered test list", () => {
+    const instruction = buildInstruction("http://localhost:3000/login", [
+      "signing in with valid credentials redirects to the dashboard",
+      "the dashboard shows the user's name",
+    ]);
+    expect(instruction).toContain("1. signing in");
+    expect(instruction).toContain("2. the dashboard");
+    expect(instruction).toContain("http://localhost:3000/login");
+  });
+
+  it("cookies: Expect.cookies is callable", () => {
+    expect(typeof Expect.cookies).toBe("function");
+  });
+});

package/tests/expect.test.ts

@@ -0,0 +1,143 @@
+import { describe, it, expect, beforeEach } from "vite-plus/test";
+import { resetGlobalConfig, configure } from "../src/config";
+import { Expect } from "../src/expect";
+import { default as ExpectDefault } from "../src/index";
+
+describe("Expect.test input validation", () => {
+  beforeEach(() => {
+    resetGlobalConfig();
+  });
+
+  it("throws for empty tests array", () => {
+    expect(() => Expect.test({ url: "http://localhost:3000", tests: [] })).toThrow(
+      "tests array is empty",
+    );
+  });
+
+  it("throws when no URL and no baseUrl", () => {
+    expect(() => Expect.test({ tests: ["something"] })).toThrow("No URL provided");
+  });
+
+  it("throws for relative URL without baseUrl", () => {
+    expect(() => Expect.test({ url: "/login", tests: ["something"] })).toThrow(
+      "No baseUrl configured",
+    );
+  });
+
+  it("does not throw when baseUrl is configured", () => {
+    configure({ baseUrl: "http://localhost:3000" });
+    const run = Expect.test({ url: "/login", tests: ["something"] });
+    run.then(
+      () => {},
+      () => {},
+    );
+    expect(run).toBeDefined();
+  });
+
+  it("uses baseUrl when no url provided", () => {
+    configure({ baseUrl: "http://localhost:3000" });
+    const run = Expect.test({ tests: ["something"] });
+    run.then(
+      () => {},
+      () => {},
+    );
+    expect(run).toBeDefined();
+  });
+
+  it("throws when function before has no page", () => {
+    expect(() =>
+      Expect.test({
+        url: "http://localhost:3000",
+        tests: ["something"],
+        before: async () => {},
+      }),
+    ).toThrow("Function before requires a page");
+  });
+
+  it("throws when function after has no page", () => {
+    expect(() =>
+      Expect.test({
+        url: "http://localhost:3000",
+        tests: ["something"],
+        after: async () => {},
+      }),
+    ).toThrow("Function after requires a page");
+  });
+
+  it("throws when tools are provided", () => {
+    expect(() =>
+      Expect.test({
+        url: "http://localhost:3000",
+        tests: ["something"],
+        tools: [
+          {
+            name: "t",
+            description: "d",
+            inputSchema: { type: "object" as const },
+            handler: async () => "",
+          },
+        ],
+      }),
+    ).toThrow("Custom tools are not yet supported");
+  });
+});
+
+describe("Expect.session", () => {
+  beforeEach(() => {
+    resetGlobalConfig();
+  });
+
+  it("creates a session with test method", () => {
+    const session = Expect.session({ url: "http://localhost:3000" });
+    expect(typeof session.test).toBe("function");
+    expect(typeof session.close).toBe("function");
+    expect(typeof session[Symbol.asyncDispose]).toBe("function");
+  });
+
+  it("session.test throws for empty tests", () => {
+    const session = Expect.session({ url: "http://localhost:3000" });
+    expect(() => session.test({ tests: [] })).toThrow("tests array is empty");
+  });
+
+  it("session.close resolves", async () => {
+    const session = Expect.session({ url: "http://localhost:3000" });
+    await expect(session.close()).resolves.toBeUndefined();
+  });
+
+  it("throws when browserContext is provided", () => {
+    expect(() =>
+      Expect.session({
+        url: "http://localhost:3000",
+        browserContext: {} as never,
+      }),
+    ).toThrow("External browserContext is not yet supported");
+  });
+
+  it("throws when tools are provided", () => {
+    expect(() =>
+      Expect.session({
+        url: "http://localhost:3000",
+        tools: [
+          {
+            name: "t",
+            description: "d",
+            inputSchema: { type: "object" as const },
+            handler: async () => "",
+          },
+        ],
+      }),
+    ).toThrow("Custom tools are not yet supported");
+  });
+});
+
+describe("Expect.cookies", () => {
+  it("is a function", () => {
+    expect(typeof Expect.cookies).toBe("function");
+  });
+});
+
+describe("default export", () => {
+  it("Expect is available as default export", () => {
+    expect(ExpectDefault).toBe(Expect);
+  });
+});

package/tests/fixtures/fixture-server.ts

@@ -0,0 +1,111 @@
+import http from "node:http";
+import type { AddressInfo } from "node:net";
+
+const INDEX_HTML = `<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="UTF-8"><title>Acme App</title></head>
+<body>
+  <h1>Welcome to Acme</h1>
+  <p>Your dashboard for everything.</p>
+  <nav>
+    <a href="/login">Login</a>
+    <a href="/signup">Signup</a>
+  </nav>
+</body>
+</html>`;
+
+const LOGIN_HTML = `<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="UTF-8"><title>Login - Acme</title></head>
+<body>
+  <h1>Sign in to Acme</h1>
+  <form id="login-form">
+    <div>
+      <label for="email">Email</label>
+      <input id="email" type="email" required placeholder="you@example.com" />
+      <span class="error" id="email-error" style="display:none;color:red">Email is required</span>
+    </div>
+    <div>
+      <label for="password">Password</label>
+      <input id="password" type="password" required placeholder="••••••••" />
+      <span class="error" id="password-error" style="display:none;color:red">Password is required</span>
+    </div>
+    <button type="submit">Sign in</button>
+  </form>
+  <div id="server-error" style="display:none;color:red">Invalid email or password</div>
+  <p>Don't have an account? <a href="/signup">Sign up</a></p>
+  <script>
+    document.getElementById('login-form').addEventListener('submit', function(e) {
+      e.preventDefault();
+      var email = document.getElementById('email').value;
+      var password = document.getElementById('password').value;
+      var hasError = false;
+      if (!email) { document.getElementById('email-error').style.display = 'block'; hasError = true; }
+      else { document.getElementById('email-error').style.display = 'none'; }
+      if (!password) { document.getElementById('password-error').style.display = 'block'; hasError = true; }
+      else { document.getElementById('password-error').style.display = 'none'; }
+      if (hasError) return;
+      document.getElementById('server-error').style.display = 'block';
+    });
+  </script>
+</body>
+</html>`;
+
+const SIGNUP_HTML = `<!DOCTYPE html>
+<html lang="en">
+<head><meta charset="UTF-8"><title>Signup - Acme</title></head>
+<body>
+  <h1>Create an account</h1>
+  <form id="signup-form">
+    <div>
+      <label for="email">Email</label>
+      <input id="email" type="email" required />
+    </div>
+    <div>
+      <label for="password">Password</label>
+      <input id="password" type="password" required />
+    </div>
+    <div>
+      <label for="confirm">Confirm Password</label>
+      <input id="confirm" type="password" required />
+    </div>
+    <button type="submit">Create account</button>
+    <span id="match-error" style="display:none;color:red">Passwords do not match</span>
+  </form>
+  <p>Already have an account? <a href="/login">Sign in</a></p>
+  <script>
+    document.getElementById('signup-form').addEventListener('submit', function(e) {
+      e.preventDefault();
+      var pw = document.getElementById('password').value;
+      var confirm = document.getElementById('confirm').value;
+      if (pw !== confirm) {
+        document.getElementById('match-error').style.display = 'block';
+      } else {
+        document.getElementById('match-error').style.display = 'none';
+      }
+    });
+  </script>
+</body>
+</html>`;
+
+const PAGES: Record<string, string> = {
+  "/": INDEX_HTML,
+  "/login": LOGIN_HTML,
+  "/signup": SIGNUP_HTML,
+};
+
+export const startFixtureServer = (): Promise<{ url: string; close: () => Promise<void> }> =>
+  new Promise((resolve) => {
+    const server = http.createServer((req, res) => {
+      const page = PAGES[req.url ?? "/"] ?? INDEX_HTML;
+      res.writeHead(200, { "Content-Type": "text/html" });
+      res.end(page);
+    });
+
+    server.listen(0, "127.0.0.1", () => {
+      const { port } = server.address() as AddressInfo;
+      const url = `http://127.0.0.1:${port}`;
+      const close = () => new Promise<void>((done) => server.close(() => done()));
+      resolve({ url, close });
+    });
+  });