@browserbasehq/convex-stagehand 0.0.1

package/src/client/index.ts

@@ -0,0 +1,348 @@
+/**
+ * Stagehand Client
+ *
+ * Type-safe wrapper for the Stagehand Convex component.
+ * Uses Zod schemas for extraction typing.
+ */
+
+import { zodToJsonSchema } from "zod-to-json-schema";
+import type { z } from "zod";
+import type {
+  GenericActionCtx,
+} from "convex/server";
+
+// Re-export component type
+export type { ComponentApi } from "../component/_generated/component.js";
+import type { ComponentApi } from "../component/_generated/component.js";
+
+type ActionCtx = GenericActionCtx<any>;
+
+export interface StagehandConfig {
+  browserbaseApiKey: string;
+  browserbaseProjectId: string;
+  modelApiKey: string;
+  modelName?: string;
+}
+
+export interface SessionInfo {
+  sessionId: string;
+  browserbaseSessionId?: string;
+  cdpUrl?: string;
+}
+
+export interface StartSessionOptions {
+  timeout?: number;
+  waitUntil?: "load" | "domcontentloaded" | "networkidle";
+  domSettleTimeoutMs?: number;
+  selfHeal?: boolean;
+  systemPrompt?: string;
+}
+
+export interface ExtractOptions {
+  timeout?: number;
+  waitUntil?: "load" | "domcontentloaded" | "networkidle";
+}
+
+export interface ActOptions {
+  timeout?: number;
+  waitUntil?: "load" | "domcontentloaded" | "networkidle";
+}
+
+export interface ObserveOptions {
+  timeout?: number;
+  waitUntil?: "load" | "domcontentloaded" | "networkidle";
+}
+
+export interface AgentOptions {
+  cua?: boolean;
+  maxSteps?: number;
+  systemPrompt?: string;
+  timeout?: number;
+  waitUntil?: "load" | "domcontentloaded" | "networkidle";
+}
+
+export interface ObservedAction {
+  description: string;
+  selector: string;
+  method: string;
+  arguments?: string[];
+}
+
+export interface ActResult {
+  success: boolean;
+  message: string;
+  actionDescription: string;
+}
+
+export interface AgentAction {
+  type: string;
+  action?: string;
+  reasoning?: string;
+  timeMs?: number;
+}
+
+export interface AgentResult {
+  actions: AgentAction[];
+  completed: boolean;
+  message: string;
+  success: boolean;
+}
+
+/**
+ * Stagehand client for AI-powered browser automation.
+ *
+ * @example
+ * ```typescript
+ * import { Stagehand } from "convex-stagehand";
+ * import { components } from "./_generated/api";
+ *
+ * const stagehand = new Stagehand(components.stagehand, {
+ *   browserbaseApiKey: process.env.BROWSERBASE_API_KEY!,
+ *   browserbaseProjectId: process.env.BROWSERBASE_PROJECT_ID!,
+ *   modelApiKey: process.env.MODEL_API_KEY!,
+ * });
+ *
+ * export const scrape = action({
+ *   handler: async (ctx) => {
+ *     return await stagehand.extract(ctx, {
+ *       url: "https://example.com",
+ *       instruction: "Extract all product names",
+ *       schema: z.object({ products: z.array(z.string()) }),
+ *     });
+ *   },
+ * });
+ * ```
+ */
+export class Stagehand {
+  constructor(
+    private component: ComponentApi,
+    private config: StagehandConfig,
+  ) {}
+
+  /**
+   * Start a new browser session.
+   * Returns session info including cdpUrl for direct Playwright/Puppeteer connection.
+   *
+   * @param ctx - Convex action context
+   * @param args - Session parameters
+   * @returns Session info with sessionId, browserbaseSessionId, and cdpUrl
+   *
+   * @example
+   * ```typescript
+   * const session = await stagehand.startSession(ctx, {
+   *   url: "https://example.com",
+   * });
+   * // Use session.sessionId with other operations
+   * // Or connect Playwright: puppeteer.connect({ browserWSEndpoint: session.cdpUrl })
+   * ```
+   */
+  async startSession(
+    ctx: ActionCtx,
+    args: {
+      url: string;
+      browserbaseSessionId?: string;
+      options?: StartSessionOptions;
+    },
+  ): Promise<SessionInfo> {
+    return ctx.runAction(this.component.lib.startSession as any, {
+      ...this.config,
+      url: args.url,
+      browserbaseSessionId: args.browserbaseSessionId,
+      options: args.options,
+    });
+  }
+
+  /**
+   * End a browser session.
+   *
+   * @param ctx - Convex action context
+   * @param args - Session to end
+   * @returns Success status
+   *
+   * @example
+   * ```typescript
+   * await stagehand.endSession(ctx, { sessionId: session.sessionId });
+   * ```
+   */
+  async endSession(
+    ctx: ActionCtx,
+    args: {
+      sessionId: string;
+    },
+  ): Promise<{ success: boolean }> {
+    return ctx.runAction(this.component.lib.endSession as any, {
+      ...this.config,
+      sessionId: args.sessionId,
+    });
+  }
+
+  /**
+   * Extract structured data from a web page using AI.
+   *
+   * @param ctx - Convex action context
+   * @param args - Extraction parameters
+   * @returns Extracted data matching the provided schema
+   *
+   * @example
+   * ```typescript
+   * // Without session (creates and destroys its own)
+   * const data = await stagehand.extract(ctx, {
+   *   url: "https://news.ycombinator.com",
+   *   instruction: "Extract the top 5 stories with title and score",
+   *   schema: z.object({
+   *     stories: z.array(z.object({
+   *       title: z.string(),
+   *       score: z.string(),
+   *     }))
+   *   }),
+   * });
+   *
+   * // With existing session (reuses session)
+   * const data = await stagehand.extract(ctx, {
+   *   sessionId: session.sessionId,
+   *   instruction: "Extract the top 5 stories",
+   *   schema: z.object({ ... }),
+   * });
+   * ```
+   */
+  async extract<T extends z.ZodType>(
+    ctx: ActionCtx,
+    args: {
+      sessionId?: string;
+      url?: string;
+      instruction: string;
+      schema: T;
+      options?: ExtractOptions;
+    },
+  ): Promise<z.infer<T>> {
+    const jsonSchema: any = zodToJsonSchema(args.schema);
+    // Remove $schema field as it's reserved in Convex
+    delete jsonSchema.$schema;
+    return ctx.runAction(this.component.lib.extract as any, {
+      ...this.config,
+      sessionId: args.sessionId,
+      url: args.url,
+      instruction: args.instruction,
+      schema: jsonSchema,
+      options: args.options,
+    });
+  }
+
+  /**
+   * Execute a browser action using natural language.
+   *
+   * @param ctx - Convex action context
+   * @param args - Action parameters
+   * @returns Result of the action
+   *
+   * @example
+   * ```typescript
+   * // Without session
+   * const result = await stagehand.act(ctx, {
+   *   url: "https://example.com/login",
+   *   action: "Click the login button",
+   * });
+   *
+   * // With existing session
+   * const result = await stagehand.act(ctx, {
+   *   sessionId: session.sessionId,
+   *   action: "Click the submit button",
+   * });
+   * ```
+   */
+  async act(
+    ctx: ActionCtx,
+    args: {
+      sessionId?: string;
+      url?: string;
+      action: string;
+      options?: ActOptions;
+    },
+  ): Promise<ActResult> {
+    return ctx.runAction(this.component.lib.act as any, {
+      ...this.config,
+      sessionId: args.sessionId,
+      url: args.url,
+      action: args.action,
+      options: args.options,
+    });
+  }
+
+  /**
+   * Find available actions on a web page.
+   *
+   * @param ctx - Convex action context
+   * @param args - Observe parameters
+   * @returns List of available actions
+   *
+   * @example
+   * ```typescript
+   * const actions = await stagehand.observe(ctx, {
+   *   url: "https://example.com",
+   *   instruction: "Find all navigation links",
+   * });
+   * ```
+   */
+  async observe(
+    ctx: ActionCtx,
+    args: {
+      sessionId?: string;
+      url?: string;
+      instruction: string;
+      options?: ObserveOptions;
+    },
+  ): Promise<ObservedAction[]> {
+    return ctx.runAction(this.component.lib.observe as any, {
+      ...this.config,
+      sessionId: args.sessionId,
+      url: args.url,
+      instruction: args.instruction,
+      options: args.options,
+    });
+  }
+
+  /**
+   * Execute autonomous multi-step browser automation using an AI agent.
+   * The agent interprets the instruction and decides what actions to take.
+   *
+   * @param ctx - Convex action context
+   * @param args - Agent parameters
+   * @returns Agent execution result with actions taken
+   *
+   * @example
+   * ```typescript
+   * // Agent creates its own session
+   * const result = await stagehand.agent(ctx, {
+   *   url: "https://google.com",
+   *   instruction: "Search for 'convex database' and extract the top 3 results",
+   *   options: { maxSteps: 10 },
+   * });
+   *
+   * // Agent with existing session
+   * const result = await stagehand.agent(ctx, {
+   *   sessionId: session.sessionId,
+   *   instruction: "Fill out the form and submit",
+   *   options: { maxSteps: 5 },
+   * });
+   * ```
+   */
+  async agent(
+    ctx: ActionCtx,
+    args: {
+      sessionId?: string;
+      url?: string;
+      instruction: string;
+      options?: AgentOptions;
+    },
+  ): Promise<AgentResult> {
+    return ctx.runAction(this.component.lib.agent as any, {
+      ...this.config,
+      sessionId: args.sessionId,
+      url: args.url,
+      instruction: args.instruction,
+      options: args.options,
+    });
+  }
+}
+
+export default Stagehand;

package/src/component/README.md

@@ -0,0 +1,90 @@
+# Welcome to your Convex functions directory!
+
+Write your Convex functions here.
+See https://docs.convex.dev/functions for more.
+
+A query function that takes two arguments looks like:
+
+```ts
+// convex/myFunctions.ts
+import { query } from "./_generated/server";
+import { v } from "convex/values";
+
+export const myQueryFunction = query({
+  // Validators for arguments.
+  args: {
+    first: v.number(),
+    second: v.string(),
+  },
+
+  // Function implementation.
+  handler: async (ctx, args) => {
+    // Read the database as many times as you need here.
+    // See https://docs.convex.dev/database/reading-data.
+    const documents = await ctx.db.query("tablename").collect();
+
+    // Arguments passed from the client are properties of the args object.
+    console.log(args.first, args.second);
+
+    // Write arbitrary JavaScript here: filter, aggregate, build derived data,
+    // remove non-public properties, or create new objects.
+    return documents;
+  },
+});
+```
+
+Using this query function in a React component looks like:
+
+```ts
+const data = useQuery(api.myFunctions.myQueryFunction, {
+  first: 10,
+  second: "hello",
+});
+```
+
+A mutation function looks like:
+
+```ts
+// convex/myFunctions.ts
+import { mutation } from "./_generated/server";
+import { v } from "convex/values";
+
+export const myMutationFunction = mutation({
+  // Validators for arguments.
+  args: {
+    first: v.string(),
+    second: v.string(),
+  },
+
+  // Function implementation.
+  handler: async (ctx, args) => {
+    // Insert or modify documents in the database here.
+    // Mutations can also read from the database like queries.
+    // See https://docs.convex.dev/database/writing-data.
+    const message = { body: args.first, author: args.second };
+    const id = await ctx.db.insert("messages", message);
+
+    // Optionally, return a value from your mutation.
+    return await ctx.db.get("messages", id);
+  },
+});
+```
+
+Using this mutation function in a React component looks like:
+
+```ts
+const mutation = useMutation(api.myFunctions.myMutationFunction);
+function handleButtonPress() {
+  // fire and forget, the most common way to use mutations
+  mutation({ first: "Hello!", second: "me" });
+  // OR
+  // use the result once the mutation has completed
+  mutation({ first: "Hello!", second: "me" }).then((result) =>
+    console.log(result),
+  );
+}
+```
+
+Use the Convex CLI to push your functions to a deployment. See everything
+the Convex CLI can do by running `npx convex -h` in your project root
+directory. To learn more, launch the docs with `npx convex docs`.

package/src/component/_generated/api.ts

@@ -0,0 +1,52 @@
+/* eslint-disable */
+/**
+ * Generated `api` utility.
+ *
+ * THIS CODE IS AUTOMATICALLY GENERATED.
+ *
+ * To regenerate, run `npx convex dev`.
+ * @module
+ */
+
+import type * as api_ from "../api.js";
+import type * as lib from "../lib.js";
+
+import type {
+  ApiFromModules,
+  FilterApi,
+  FunctionReference,
+} from "convex/server";
+import { anyApi, componentsGeneric } from "convex/server";
+
+const fullApi: ApiFromModules<{
+  api: typeof api_;
+  lib: typeof lib;
+}> = anyApi as any;
+
+/**
+ * A utility for referencing Convex functions in your app's public API.
+ *
+ * Usage:
+ * ```js
+ * const myFunctionReference = api.myModule.myFunction;
+ * ```
+ */
+export const api: FilterApi<
+  typeof fullApi,
+  FunctionReference<any, "public">
+> = anyApi as any;
+
+/**
+ * A utility for referencing Convex functions in your app's internal API.
+ *
+ * Usage:
+ * ```js
+ * const myFunctionReference = internal.myModule.myFunction;
+ * ```
+ */
+export const internal: FilterApi<
+  typeof fullApi,
+  FunctionReference<any, "internal">
+> = anyApi as any;
+
+export const components = componentsGeneric() as unknown as {};

package/src/component/_generated/component.ts

@@ -0,0 +1,26 @@
+/* eslint-disable */
+/**
+ * Generated component type.
+ *
+ * THIS CODE IS AUTOMATICALLY GENERATED.
+ *
+ * To regenerate, run `npx convex dev`.
+ * @module
+ */
+
+import type { FunctionReference } from "convex/server";
+
+/**
+ * Component API type for the Stagehand component.
+ * Used by the client to call component actions.
+ */
+export type ComponentApi = {
+  lib: {
+    startSession: FunctionReference<"action", "public", any, any>;
+    endSession: FunctionReference<"action", "public", any, any>;
+    extract: FunctionReference<"action", "public", any, any>;
+    act: FunctionReference<"action", "public", any, any>;
+    observe: FunctionReference<"action", "public", any, any>;
+    agent: FunctionReference<"action", "public", any, any>;
+  };
+};

package/src/component/_generated/dataModel.ts

@@ -0,0 +1,17 @@
+/* eslint-disable */
+/**
+ * Generated data model types.
+ *
+ * THIS CODE IS AUTOMATICALLY GENERATED.
+ *
+ * To regenerate, run `npx convex dev`.
+ * @module
+ */
+
+import type { DataModelFromSchemaDefinition } from "convex/server";
+import type schema from "../schema.js";
+
+/**
+ * The type of a document stored in Convex.
+ */
+export type DataModel = DataModelFromSchemaDefinition<typeof schema>;

package/src/component/_generated/server.ts

@@ -0,0 +1,156 @@
+/* eslint-disable */
+/**
+ * Generated utilities for implementing server-side Convex query and mutation functions.
+ *
+ * THIS CODE IS AUTOMATICALLY GENERATED.
+ *
+ * To regenerate, run `npx convex dev`.
+ * @module
+ */
+
+import type {
+  ActionBuilder,
+  HttpActionBuilder,
+  MutationBuilder,
+  QueryBuilder,
+  GenericActionCtx,
+  GenericMutationCtx,
+  GenericQueryCtx,
+  GenericDatabaseReader,
+  GenericDatabaseWriter,
+} from "convex/server";
+import {
+  actionGeneric,
+  httpActionGeneric,
+  queryGeneric,
+  mutationGeneric,
+  internalActionGeneric,
+  internalMutationGeneric,
+  internalQueryGeneric,
+} from "convex/server";
+import type { DataModel } from "./dataModel.js";
+
+/**
+ * Define a query in this Convex app's public API.
+ *
+ * This function will be allowed to read your Convex database and will be accessible from the client.
+ *
+ * @param func - The query function. It receives a {@link QueryCtx} as its first argument.
+ * @returns The wrapped query. Include this as an `export` to name it and make it accessible.
+ */
+export const query: QueryBuilder<DataModel, "public"> = queryGeneric;
+
+/**
+ * Define a query that is only accessible from other Convex functions (but not from the client).
+ *
+ * This function will be allowed to read from your Convex database. It will not be accessible from the client.
+ *
+ * @param func - The query function. It receives a {@link QueryCtx} as its first argument.
+ * @returns The wrapped query. Include this as an `export` to name it and make it accessible.
+ */
+export const internalQuery: QueryBuilder<DataModel, "internal"> =
+  internalQueryGeneric;
+
+/**
+ * Define a mutation in this Convex app's public API.
+ *
+ * This function will be allowed to modify your Convex database and will be accessible from the client.
+ *
+ * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument.
+ * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible.
+ */
+export const mutation: MutationBuilder<DataModel, "public"> = mutationGeneric;
+
+/**
+ * Define a mutation that is only accessible from other Convex functions (but not from the client).
+ *
+ * This function will be allowed to modify your Convex database. It will not be accessible from the client.
+ *
+ * @param func - The mutation function. It receives a {@link MutationCtx} as its first argument.
+ * @returns The wrapped mutation. Include this as an `export` to name it and make it accessible.
+ */
+export const internalMutation: MutationBuilder<DataModel, "internal"> =
+  internalMutationGeneric;
+
+/**
+ * Define an action in this Convex app's public API.
+ *
+ * An action is a function which can execute any JavaScript code, including non-deterministic
+ * code and code with side-effects, like calling third-party services.
+ * They can be run in Convex's JavaScript environment or in Node.js using the "use node" directive.
+ * They can interact with the database indirectly by calling queries and mutations using the {@link ActionCtx}.
+ *
+ * @param func - The action. It receives an {@link ActionCtx} as its first argument.
+ * @returns The wrapped action. Include this as an `export` to name it and make it accessible.
+ */
+export const action: ActionBuilder<DataModel, "public"> = actionGeneric;
+
+/**
+ * Define an action that is only accessible from other Convex functions (but not from the client).
+ *
+ * @param func - The function. It receives an {@link ActionCtx} as its first argument.
+ * @returns The wrapped function. Include this as an `export` to name it and make it accessible.
+ */
+export const internalAction: ActionBuilder<DataModel, "internal"> =
+  internalActionGeneric;
+
+/**
+ * Define an HTTP action.
+ *
+ * The wrapped function will be used to respond to HTTP requests received
+ * by a Convex deployment if the requests matches the path and method where
+ * this action is routed. Be sure to route your httpAction in `convex/http.js`.
+ *
+ * @param func - The function. It receives an {@link ActionCtx} as its first argument
+ * and a Fetch API `Request` object as its second.
+ * @returns The wrapped function. Import this function from `convex/http.js` and route it to hook it up.
+ */
+export const httpAction: HttpActionBuilder = httpActionGeneric;
+
+/**
+ * A set of services for use within Convex query functions.
+ *
+ * The query context is passed as the first argument to any Convex query
+ * function run on the server.
+ *
+ * If you're using code generation, use the `QueryCtx` type in `convex/_generated/server.d.ts` instead.
+ */
+export type QueryCtx = GenericQueryCtx<DataModel>;
+
+/**
+ * A set of services for use within Convex mutation functions.
+ *
+ * The mutation context is passed as the first argument to any Convex mutation
+ * function run on the server.
+ *
+ * If you're using code generation, use the `MutationCtx` type in `convex/_generated/server.d.ts` instead.
+ */
+export type MutationCtx = GenericMutationCtx<DataModel>;
+
+/**
+ * A set of services for use within Convex action functions.
+ *
+ * The action context is passed as the first argument to any Convex action
+ * function run on the server.
+ */
+export type ActionCtx = GenericActionCtx<DataModel>;
+
+/**
+ * An interface to read from the database within Convex query functions.
+ *
+ * The two entry points are {@link DatabaseReader.get}, which fetches a single
+ * document by its {@link Id}, or {@link DatabaseReader.query}, which starts
+ * building a query.
+ */
+export type DatabaseReader = GenericDatabaseReader<DataModel>;
+
+/**
+ * An interface to read from and write to the database within Convex mutation
+ * functions.
+ *
+ * Convex guarantees that all writes within a single mutation are
+ * executed atomically, so you never have to worry about partial writes leaving
+ * your data in an inconsistent state. See [the Convex Guide](https://docs.convex.dev/understanding/convex-fundamentals/functions#atomicity-and-optimistic-concurrency-control)
+ * for the guarantees Convex provides your functions.
+ */
+export type DatabaseWriter = GenericDatabaseWriter<DataModel>;