@browserbasehq/convex-stagehand 0.0.1

package/src/component/api.ts

@@ -0,0 +1,243 @@
+/**
+ * Stagehand REST API Client
+ *
+ * Wraps the Stagehand API at https://api.stagehand.browserbase.com
+ */
+
+const API_BASE = "https://api.stagehand.browserbase.com/v1";
+
+export interface ApiConfig {
+  browserbaseApiKey: string;
+  browserbaseProjectId: string;
+  modelApiKey: string;
+  modelName?: string;
+}
+
+export interface SessionData {
+  sessionId: string;
+  browserbaseSessionId?: string;
+  cdpUrl?: string;
+  available: boolean;
+}
+
+export interface StartSessionOptions {
+  browserbaseSessionId?: string;
+  domSettleTimeoutMs?: number;
+  selfHeal?: boolean;
+  systemPrompt?: string;
+}
+
+export interface ApiResponse<T> {
+  data: T;
+  success: boolean;
+}
+
+function getHeaders(config: ApiConfig): Record<string, string> {
+  return {
+    "Content-Type": "application/json",
+    "x-bb-api-key": config.browserbaseApiKey,
+    "x-bb-project-id": config.browserbaseProjectId,
+    "x-model-api-key": config.modelApiKey,
+  };
+}
+
+async function handleResponse<T>(response: Response): Promise<T> {
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(
+      `Stagehand API error (${response.status}): ${errorText}`,
+    );
+  }
+  const json = (await response.json()) as ApiResponse<T>;
+  if (!json.success) {
+    throw new Error(`Stagehand API returned success: false`);
+  }
+  return json.data;
+}
+
+export async function startSession(
+  config: ApiConfig,
+  options?: StartSessionOptions,
+): Promise<SessionData> {
+  const response = await fetch(`${API_BASE}/sessions/start`, {
+    method: "POST",
+    headers: getHeaders(config),
+    body: JSON.stringify({
+      modelName: config.modelName || "openai/gpt-4o",
+      browserbaseSessionId: options?.browserbaseSessionId,
+      domSettleTimeoutMs: options?.domSettleTimeoutMs,
+      selfHeal: options?.selfHeal,
+      systemPrompt: options?.systemPrompt,
+    }),
+  });
+  return handleResponse<SessionData>(response);
+}
+
+export async function endSession(
+  sessionId: string,
+  config: ApiConfig,
+): Promise<void> {
+  try {
+    await fetch(`${API_BASE}/sessions/${sessionId}/end`, {
+      method: "POST",
+      headers: getHeaders(config),
+    });
+  } catch {
+    // Ignore errors when ending session - best effort cleanup
+  }
+}
+
+export interface NavigateOptions {
+  waitUntil?: "load" | "domcontentloaded" | "networkidle";
+  timeout?: number;
+}
+
+export async function navigate(
+  sessionId: string,
+  url: string,
+  config: ApiConfig,
+  options?: NavigateOptions,
+): Promise<void> {
+  const response = await fetch(`${API_BASE}/sessions/${sessionId}/navigate`, {
+    method: "POST",
+    headers: getHeaders(config),
+    body: JSON.stringify({
+      url,
+      options: {
+        waitUntil: options?.waitUntil || "networkidle",
+        timeout: options?.timeout,
+      },
+    }),
+  });
+  await handleResponse(response);
+}
+
+export interface ExtractResult<T = any> {
+  result: T;
+  actionId: string;
+}
+
+export async function extract(
+  sessionId: string,
+  instruction: string,
+  schema: any,
+  config: ApiConfig,
+): Promise<ExtractResult> {
+  const response = await fetch(`${API_BASE}/sessions/${sessionId}/extract`, {
+    method: "POST",
+    headers: getHeaders(config),
+    body: JSON.stringify({
+      instruction,
+      schema,
+    }),
+  });
+  return handleResponse<ExtractResult>(response);
+}
+
+export interface ActResult {
+  result: {
+    actionDescription: string;
+    actions: Array<{
+      description: string;
+      selector: string;
+      arguments?: string[];
+      method: string;
+    }>;
+    message: string;
+    success: boolean;
+  };
+  actionId: string;
+}
+
+export async function act(
+  sessionId: string,
+  action: string,
+  config: ApiConfig,
+): Promise<ActResult> {
+  const response = await fetch(`${API_BASE}/sessions/${sessionId}/act`, {
+    method: "POST",
+    headers: getHeaders(config),
+    body: JSON.stringify({
+      input: action,
+    }),
+  });
+  return handleResponse<ActResult>(response);
+}
+
+export interface ObserveResult {
+  result: Array<{
+    description: string;
+    selector: string;
+    arguments?: string[];
+    backendNodeId?: number;
+    method: string;
+  }>;
+  actionId: string;
+}
+
+export async function observe(
+  sessionId: string,
+  instruction: string,
+  config: ApiConfig,
+): Promise<ObserveResult> {
+  const response = await fetch(`${API_BASE}/sessions/${sessionId}/observe`, {
+    method: "POST",
+    headers: getHeaders(config),
+    body: JSON.stringify({
+      instruction,
+    }),
+  });
+  return handleResponse<ObserveResult>(response);
+}
+
+export interface AgentConfig {
+  cua?: boolean;
+  model?: string;
+  systemPrompt?: string;
+}
+
+export interface AgentExecuteOptions {
+  instruction: string;
+  maxSteps?: number;
+}
+
+export interface AgentAction {
+  type: string;
+  action?: string;
+  reasoning?: string;
+  timeMs?: number;
+}
+
+export interface AgentExecuteResult {
+  result: {
+    actions: AgentAction[];
+    completed: boolean;
+    message: string;
+    success: boolean;
+  };
+}
+
+export async function agentExecute(
+  sessionId: string,
+  agentConfig: AgentConfig,
+  executeOptions: AgentExecuteOptions,
+  config: ApiConfig,
+): Promise<AgentExecuteResult> {
+  const response = await fetch(
+    `${API_BASE}/sessions/${sessionId}/agent/execute`,
+    {
+      method: "POST",
+      headers: getHeaders(config),
+      body: JSON.stringify({
+        agentConfig: {
+          cua: agentConfig.cua,
+          model: agentConfig.model,
+          systemPrompt: agentConfig.systemPrompt,
+        },
+        instruction: executeOptions.instruction,
+        maxSteps: executeOptions.maxSteps,
+      }),
+    },
+  );
+  return handleResponse<AgentExecuteResult>(response);
+}

package/src/component/convex.config.ts

@@ -0,0 +1,3 @@
+import { defineComponent } from "convex/server";
+
+export default defineComponent("stagehand");

package/src/component/lib.ts

@@ -0,0 +1,416 @@
+/**
+ * Stagehand Component Library
+ *
+ * AI-powered browser automation actions using the Stagehand REST API.
+ * Supports both automatic session management and manual session control.
+ */
+
+import { action } from "./_generated/server.js";
+import { v } from "convex/values";
+import * as api from "./api.js";
+
+const observedActionValidator = v.object({
+  description: v.string(),
+  selector: v.string(),
+  method: v.string(),
+  arguments: v.optional(v.array(v.string())),
+});
+
+const waitUntilValidator = v.union(
+  v.literal("load"),
+  v.literal("domcontentloaded"),
+  v.literal("networkidle"),
+);
+
+const agentActionValidator = v.object({
+  type: v.string(),
+  action: v.optional(v.string()),
+  reasoning: v.optional(v.string()),
+  timeMs: v.optional(v.number()),
+});
+
+/**
+ * Start a new browser session.
+ * Returns session info including cdpUrl for direct Playwright/Puppeteer connection.
+ */
+export const startSession = action({
+  args: {
+    browserbaseApiKey: v.string(),
+    browserbaseProjectId: v.string(),
+    modelApiKey: v.string(),
+    modelName: v.optional(v.string()),
+    url: v.string(),
+    browserbaseSessionId: v.optional(v.string()),
+    options: v.optional(
+      v.object({
+        timeout: v.optional(v.number()),
+        waitUntil: v.optional(waitUntilValidator),
+        domSettleTimeoutMs: v.optional(v.number()),
+        selfHeal: v.optional(v.boolean()),
+        systemPrompt: v.optional(v.string()),
+      }),
+    ),
+  },
+  returns: v.object({
+    sessionId: v.string(),
+    browserbaseSessionId: v.optional(v.string()),
+    cdpUrl: v.optional(v.string()),
+  }),
+  handler: async (_ctx: any, args: any) => {
+    const config: api.ApiConfig = {
+      browserbaseApiKey: args.browserbaseApiKey,
+      browserbaseProjectId: args.browserbaseProjectId,
+      modelApiKey: args.modelApiKey,
+      modelName: args.modelName,
+    };
+
+    const session = await api.startSession(config, {
+      browserbaseSessionId: args.browserbaseSessionId,
+      domSettleTimeoutMs: args.options?.domSettleTimeoutMs,
+      selfHeal: args.options?.selfHeal,
+      systemPrompt: args.options?.systemPrompt,
+    });
+
+    try {
+      await api.navigate(session.sessionId, args.url, config, {
+        waitUntil: args.options?.waitUntil,
+        timeout: args.options?.timeout,
+      });
+
+      return {
+        sessionId: session.sessionId,
+        browserbaseSessionId: session.browserbaseSessionId,
+        cdpUrl: session.cdpUrl,
+      };
+    } catch (error) {
+      await api.endSession(session.sessionId, config);
+      throw error;
+    }
+  },
+});
+
+/**
+ * End a browser session.
+ */
+export const endSession = action({
+  args: {
+    browserbaseApiKey: v.string(),
+    browserbaseProjectId: v.string(),
+    modelApiKey: v.string(),
+    sessionId: v.string(),
+  },
+  returns: v.object({ success: v.boolean() }),
+  handler: async (_ctx: any, args: any) => {
+    const config: api.ApiConfig = {
+      browserbaseApiKey: args.browserbaseApiKey,
+      browserbaseProjectId: args.browserbaseProjectId,
+      modelApiKey: args.modelApiKey,
+    };
+
+    await api.endSession(args.sessionId, config);
+    return { success: true };
+  },
+});
+
+/**
+ * Extract structured data from a web page using AI.
+ * If sessionId is provided, uses existing session (doesn't end it).
+ * Otherwise, handles full session lifecycle: start -> navigate -> extract -> end
+ */
+export const extract = action({
+  args: {
+    browserbaseApiKey: v.string(),
+    browserbaseProjectId: v.string(),
+    modelApiKey: v.string(),
+    modelName: v.optional(v.string()),
+    sessionId: v.optional(v.string()),
+    url: v.optional(v.string()),
+    instruction: v.string(),
+    schema: v.any(),
+    options: v.optional(
+      v.object({
+        timeout: v.optional(v.number()),
+        waitUntil: v.optional(waitUntilValidator),
+      }),
+    ),
+  },
+  returns: v.any(),
+  handler: async (_ctx: any, args: any) => {
+    if (!args.sessionId && !args.url) {
+      throw new Error("Either sessionId or url must be provided");
+    }
+
+    const config: api.ApiConfig = {
+      browserbaseApiKey: args.browserbaseApiKey,
+      browserbaseProjectId: args.browserbaseProjectId,
+      modelApiKey: args.modelApiKey,
+      modelName: args.modelName,
+    };
+
+    const ownSession = !args.sessionId;
+    let sessionId = args.sessionId;
+
+    if (ownSession) {
+      const session = await api.startSession(config);
+      sessionId = session.sessionId;
+    }
+
+    try {
+      if (ownSession && args.url) {
+        await api.navigate(sessionId, args.url, config, {
+          waitUntil: args.options?.waitUntil,
+          timeout: args.options?.timeout,
+        });
+      }
+
+      const result = await api.extract(
+        sessionId,
+        args.instruction,
+        args.schema,
+        config,
+      );
+
+      if (ownSession) {
+        await api.endSession(sessionId, config);
+      }
+
+      return result.result;
+    } catch (error) {
+      if (ownSession) {
+        await api.endSession(sessionId, config);
+      }
+      throw error;
+    }
+  },
+});
+
+/**
+ * Execute browser actions using natural language instructions.
+ * If sessionId is provided, uses existing session (doesn't end it).
+ * Otherwise, handles full session lifecycle: start -> navigate -> act -> end
+ */
+export const act = action({
+  args: {
+    browserbaseApiKey: v.string(),
+    browserbaseProjectId: v.string(),
+    modelApiKey: v.string(),
+    modelName: v.optional(v.string()),
+    sessionId: v.optional(v.string()),
+    url: v.optional(v.string()),
+    action: v.string(),
+    options: v.optional(
+      v.object({
+        timeout: v.optional(v.number()),
+        waitUntil: v.optional(waitUntilValidator),
+      }),
+    ),
+  },
+  returns: v.object({
+    success: v.boolean(),
+    message: v.string(),
+    actionDescription: v.string(),
+  }),
+  handler: async (_ctx: any, args: any) => {
+    if (!args.sessionId && !args.url) {
+      throw new Error("Either sessionId or url must be provided");
+    }
+
+    const config: api.ApiConfig = {
+      browserbaseApiKey: args.browserbaseApiKey,
+      browserbaseProjectId: args.browserbaseProjectId,
+      modelApiKey: args.modelApiKey,
+      modelName: args.modelName,
+    };
+
+    const ownSession = !args.sessionId;
+    let sessionId = args.sessionId;
+
+    if (ownSession) {
+      const session = await api.startSession(config);
+      sessionId = session.sessionId;
+    }
+
+    try {
+      if (ownSession && args.url) {
+        await api.navigate(sessionId, args.url, config, {
+          waitUntil: args.options?.waitUntil,
+          timeout: args.options?.timeout,
+        });
+      }
+
+      const result = await api.act(sessionId, args.action, config);
+
+      if (ownSession) {
+        await api.endSession(sessionId, config);
+      }
+
+      return {
+        success: result.result.success,
+        message: result.result.message,
+        actionDescription: result.result.actionDescription,
+      };
+    } catch (error) {
+      if (ownSession) {
+        await api.endSession(sessionId, config);
+      }
+      throw error;
+    }
+  },
+});
+
+/**
+ * Find available actions on a web page matching an instruction.
+ * If sessionId is provided, uses existing session (doesn't end it).
+ * Otherwise, handles full session lifecycle: start -> navigate -> observe -> end
+ */
+export const observe = action({
+  args: {
+    browserbaseApiKey: v.string(),
+    browserbaseProjectId: v.string(),
+    modelApiKey: v.string(),
+    modelName: v.optional(v.string()),
+    sessionId: v.optional(v.string()),
+    url: v.optional(v.string()),
+    instruction: v.string(),
+    options: v.optional(
+      v.object({
+        timeout: v.optional(v.number()),
+        waitUntil: v.optional(waitUntilValidator),
+      }),
+    ),
+  },
+  returns: v.array(observedActionValidator),
+  handler: async (_ctx: any, args: any) => {
+    if (!args.sessionId && !args.url) {
+      throw new Error("Either sessionId or url must be provided");
+    }
+
+    const config: api.ApiConfig = {
+      browserbaseApiKey: args.browserbaseApiKey,
+      browserbaseProjectId: args.browserbaseProjectId,
+      modelApiKey: args.modelApiKey,
+      modelName: args.modelName,
+    };
+
+    const ownSession = !args.sessionId;
+    let sessionId = args.sessionId;
+
+    if (ownSession) {
+      const session = await api.startSession(config);
+      sessionId = session.sessionId;
+    }
+
+    try {
+      if (ownSession && args.url) {
+        await api.navigate(sessionId, args.url, config, {
+          waitUntil: args.options?.waitUntil,
+          timeout: args.options?.timeout,
+        });
+      }
+
+      const result = await api.observe(sessionId, args.instruction, config);
+
+      if (ownSession) {
+        await api.endSession(sessionId, config);
+      }
+
+      return result.result.map((action) => ({
+        description: action.description,
+        selector: action.selector,
+        method: action.method,
+        arguments: action.arguments,
+      }));
+    } catch (error) {
+      if (ownSession) {
+        await api.endSession(sessionId, config);
+      }
+      throw error;
+    }
+  },
+});
+
+/**
+ * Execute autonomous multi-step browser automation using an AI agent.
+ * The agent interprets the instruction and decides what actions to take.
+ * If sessionId is provided, uses existing session (doesn't end it).
+ * Otherwise, handles full session lifecycle.
+ */
+export const agent = action({
+  args: {
+    browserbaseApiKey: v.string(),
+    browserbaseProjectId: v.string(),
+    modelApiKey: v.string(),
+    modelName: v.optional(v.string()),
+    sessionId: v.optional(v.string()),
+    url: v.optional(v.string()),
+    instruction: v.string(),
+    options: v.optional(
+      v.object({
+        cua: v.optional(v.boolean()),
+        maxSteps: v.optional(v.number()),
+        systemPrompt: v.optional(v.string()),
+        timeout: v.optional(v.number()),
+        waitUntil: v.optional(waitUntilValidator),
+      }),
+    ),
+  },
+  returns: v.object({
+    actions: v.array(agentActionValidator),
+    completed: v.boolean(),
+    message: v.string(),
+    success: v.boolean(),
+  }),
+  handler: async (_ctx: any, args: any) => {
+    if (!args.sessionId && !args.url) {
+      throw new Error("Either sessionId or url must be provided");
+    }
+
+    const config: api.ApiConfig = {
+      browserbaseApiKey: args.browserbaseApiKey,
+      browserbaseProjectId: args.browserbaseProjectId,
+      modelApiKey: args.modelApiKey,
+      modelName: args.modelName,
+    };
+
+    const ownSession = !args.sessionId;
+    let sessionId = args.sessionId;
+
+    if (ownSession) {
+      const session = await api.startSession(config);
+      sessionId = session.sessionId;
+    }
+
+    try {
+      if (ownSession && args.url) {
+        await api.navigate(sessionId, args.url, config, {
+          waitUntil: args.options?.waitUntil,
+          timeout: args.options?.timeout,
+        });
+      }
+
+      const result = await api.agentExecute(
+        sessionId,
+        {
+          cua: args.options?.cua,
+          systemPrompt: args.options?.systemPrompt,
+        },
+        {
+          instruction: args.instruction,
+          maxSteps: args.options?.maxSteps,
+        },
+        config,
+      );
+
+      if (ownSession) {
+        await api.endSession(sessionId, config);
+      }
+
+      return result.result;
+    } catch (error) {
+      if (ownSession) {
+        await api.endSession(sessionId, config);
+      }
+      throw error;
+    }
+  },
+});

package/src/component/schema.ts

@@ -0,0 +1,23 @@
+import { defineSchema, defineTable } from "convex/server";
+import { v } from "convex/values";
+
+export default defineSchema({
+  sessions: defineTable({
+    sessionId: v.string(),
+    startedAt: v.number(),
+    endedAt: v.optional(v.number()),
+    status: v.union(
+      v.literal("active"),
+      v.literal("completed"),
+      v.literal("error"),
+    ),
+    operation: v.union(
+      v.literal("extract"),
+      v.literal("act"),
+      v.literal("observe"),
+      v.literal("workflow"),
+    ),
+    url: v.string(),
+    error: v.optional(v.string()),
+  }).index("by_sessionId", ["sessionId"]),
+});

package/src/component/tsconfig.json

@@ -0,0 +1,25 @@
+{
+  /* This TypeScript project config describes the environment that
+   * Convex functions run in and is used to typecheck them.
+   * You can modify it, but some settings are required to use Convex.
+   */
+  "compilerOptions": {
+    /* These settings are not required by Convex and can be modified. */
+    "allowJs": true,
+    "strict": true,
+    "moduleResolution": "Bundler",
+    "jsx": "react-jsx",
+    "skipLibCheck": true,
+    "allowSyntheticDefaultImports": true,
+
+    /* These compiler options are required by Convex */
+    "target": "ESNext",
+    "lib": ["ES2021", "dom"],
+    "forceConsistentCasingInFileNames": true,
+    "module": "ESNext",
+    "isolatedModules": true,
+    "noEmit": true
+  },
+  "include": ["./**/*"],
+  "exclude": ["./_generated"]
+}