@mercuryo-ai/agentbrowse 0.2.52 → 0.2.53

package/CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+- No unreleased changes yet.
+
+## 0.2.52
+
 - switched the library extraction API to `extract(session, schema, scopeRef?)`, where
   `schema` is a plain schema object or a Zod schema
 - made `observe(...)` return a top-level flat `targets` array alongside grouped

package/README.md

@@ -111,6 +111,7 @@ try {
 
 Runnable examples live in [`examples/`](./examples/README.md):
 
+- first run `npm run build` when executing them from this repo
 - `npx tsx examples/basic.ts`
 - `npx tsx examples/attach.ts`
 - `npx tsx examples/extract.ts`
@@ -201,29 +202,12 @@ The runtime contract is intentionally small: you provide an object that can
 create an OpenAI-compatible chat-completions client.
 
 ```ts
+// Pseudocode shape only. For a runnable fetch-based adapter, see
+// `examples/extract.ts` and `docs/assistive-runtime.md`.
 import { createAgentbrowseClient } from '@mercuryo-ai/agentbrowse';
 
 const client = createAgentbrowseClient({
-  assistiveRuntime: {
-    createLlmClient: () => ({
-      async createChatCompletion(args) {
-        const { messages, response_model, image, temperature, maxOutputTokens } = args.options;
-
-        const result = await callStructuredProvider({
-          messages,
-          responseModel: response_model,
-          image,
-          temperature,
-          maxOutputTokens,
-        });
-
-        return {
-          data: result.data,
-          usage: result.usage,
-        };
-      },
-    }),
-  },
+  assistiveRuntime: createMyFetchBackedRuntime(),
 });
 ```
 

package/docs/assistive-runtime.md

@@ -47,26 +47,11 @@ your adapter returns the expected response shape.
 import { createAgentbrowseClient } from '@mercuryo-ai/agentbrowse';
 
 const client = createAgentbrowseClient({
-  assistiveRuntime: {
-    createLlmClient: () => ({
-      async createChatCompletion(args) {
-        const { messages, response_model, image, temperature, maxOutputTokens } = args.options;
-
-        const json = await callStructuredProvider({
-          messages,
-          responseModel: response_model,
-          image,
-          temperature,
-          maxOutputTokens,
-        });
-
-        return {
-          data: json.data,
-          usage: json.usage,
-        };
-      },
-    }),
-  },
+  assistiveRuntime: createOpenAiCompatibleAssistiveRuntime({
+    baseUrl: 'https://api.openai.com/v1',
+    apiKey: process.env.OPENAI_API_KEY!,
+    model: 'gpt-4.1-mini',
+  }),
 });
 ```
 
@@ -81,27 +66,101 @@ This pattern works well when:
 You can wrap the adapter once and reuse it:
 
 ```ts
+import { toJsonSchema } from '@browserbasehq/stagehand';
+import type {
+  AgentbrowseAssistiveChatCompletionOptions,
+  AgentbrowseAssistiveLlmUsage,
+} from '@mercuryo-ai/agentbrowse';
+
+type StructuredChatResponse = {
+  choices?: Array<{
+    message?: {
+      content?: string;
+    };
+  }>;
+  usage?: AgentbrowseAssistiveLlmUsage;
+};
+
+function buildMessages(options: AgentbrowseAssistiveChatCompletionOptions) {
+  const messages = [...options.messages];
+  if (!options.image) {
+    return messages;
+  }
+
+  const content: Array<
+    | { type: 'text'; text: string }
+    | { type: 'image_url'; image_url: { url: string; detail: 'auto' } }
+  > = [];
+
+  if (options.image.description?.trim()) {
+    content.push({ type: 'text', text: options.image.description.trim() });
+  }
+
+  content.push({
+    type: 'image_url',
+    image_url: {
+      url: `data:image/jpeg;base64,${options.image.buffer.toString('base64')}`,
+      detail: 'auto',
+    },
+  });
+
+  messages.push({
+    role: 'user',
+    content,
+  });
+
+  return messages;
+}
+
 function createOpenAiCompatibleAssistiveRuntime(input: {
   baseUrl: string;
   apiKey: string;
+  model: string;
 }) {
+  const baseUrl = input.baseUrl.replace(/\/$/, '');
+
   return {
     createLlmClient: () => ({
-      async createChatCompletion(args) {
-        const { messages, response_model, image, temperature, maxOutputTokens } = args.options;
-
-        const json = await callStructuredProvider({
-          baseUrl: input.baseUrl,
-          apiKey: input.apiKey,
-          messages,
-          responseModel: response_model,
-          image,
-          temperature,
-          maxOutputTokens,
+      async createChatCompletion({ options }) {
+        if (!options.response_model) {
+          throw new Error('AgentBrowse assistive extract requires response_model.');
+        }
+
+        const response = await fetch(`${baseUrl}/chat/completions`, {
+          method: 'POST',
+          headers: {
+            Authorization: `Bearer ${input.apiKey}`,
+            Accept: 'application/json',
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({
+            model: input.model,
+            messages: buildMessages(options),
+            response_format: {
+              type: 'json_schema',
+              json_schema: {
+                name: options.response_model.name,
+                strict: true,
+                schema: toJsonSchema(options.response_model.schema),
+              },
+            },
+            temperature: options.temperature,
+            max_completion_tokens: options.maxOutputTokens,
+          }),
         });
 
+        if (!response.ok) {
+          throw new Error(`assistive_provider_http_${response.status}`);
+        }
+
+        const json = (await response.json()) as StructuredChatResponse;
+        const content = json.choices?.[0]?.message?.content;
+        if (typeof content !== 'string' || content.trim().length === 0) {
+          throw new Error('assistive_provider_missing_content');
+        }
+
         return {
-          data: json.data,
+          data: JSON.parse(content),
           usage: json.usage,
         };
       },

package/examples/README.md

@@ -1,6 +1,15 @@
 # AgentBrowse Examples
 
-Run these examples from `packages/agentbrowse`:
+The published package includes these same files under `examples/`.
+
+If you run them from this repo, build once first so the self-referenced package
+entrypoint resolves to `dist/`:
+
+```bash
+npm run build
+```
+
+Then run the examples from `packages/agentbrowse`:
 
 ```bash
 npx tsx examples/basic.ts

package/examples/attach.ts

@@ -1,4 +1,4 @@
-import { attach, observe } from '../src/library.ts';
+import { attach, observe } from '@mercuryo-ai/agentbrowse';
 
 const cdpUrl = process.env.AGENTBROWSE_CDP_URL;
 

package/examples/basic.ts

@@ -1,4 +1,4 @@
-import { close, launch, observe, screenshot } from '../src/library.ts';
+import { close, launch, observe, screenshot } from '@mercuryo-ai/agentbrowse';
 
 const launchResult = await launch('https://example.com', {
   headless: false,

package/examples/extract.ts

@@ -1,6 +1,59 @@
-import { createAgentbrowseClient } from '../src/library.ts';
+import { toJsonSchema } from '@browserbasehq/stagehand';
+import {
+  createAgentbrowseClient,
+  type AgentbrowseAssistiveChatCompletionOptions,
+} from '@mercuryo-ai/agentbrowse';
+
+type StructuredChatResponse = {
+  choices?: Array<{
+    message?: {
+      content?: string;
+    };
+  }>;
+  usage?: {
+    prompt_tokens?: number;
+    completion_tokens?: number;
+    total_tokens?: number;
+  };
+};
+
+function buildMessages(options: AgentbrowseAssistiveChatCompletionOptions) {
+  const messages = [...options.messages];
+  if (!options.image) {
+    return messages;
+  }
+
+  const content: Array<
+    | { type: 'text'; text: string }
+    | { type: 'image_url'; image_url: { url: string; detail: 'auto' } }
+  > = [];
+
+  if (options.image.description?.trim()) {
+    content.push({ type: 'text', text: options.image.description.trim() });
+  }
+
+  content.push({
+    type: 'image_url',
+    image_url: {
+      url: `data:image/jpeg;base64,${options.image.buffer.toString('base64')}`,
+      detail: 'auto',
+    },
+  });
+
+  messages.push({
+    role: 'user',
+    content,
+  });
+
+  return messages;
+}
 
 const openAiApiKey = process.env.OPENAI_API_KEY;
+const openAiBaseUrl = (process.env.OPENAI_BASE_URL ?? 'https://api.openai.com/v1').replace(
+  /\/$/,
+  ''
+);
+const openAiModel = process.env.OPENAI_MODEL ?? 'gpt-4.1-mini';
 
 if (!openAiApiKey) {
   throw new Error('Set OPENAI_API_KEY before running this example.');
@@ -9,21 +62,47 @@ if (!openAiApiKey) {
 const client = createAgentbrowseClient({
   assistiveRuntime: {
     createLlmClient: () => ({
-      async createChatCompletion(args) {
-        const response = await fetch('https://api.openai.com/v1/chat/completions', {
+      async createChatCompletion({ options }) {
+        if (!options.response_model) {
+          throw new Error('AgentBrowse extract requires response_model in the assistive runtime.');
+        }
+
+        const response = await fetch(`${openAiBaseUrl}/chat/completions`, {
           method: 'POST',
           headers: {
             'content-type': 'application/json',
             authorization: `Bearer ${openAiApiKey}`,
           },
-          body: JSON.stringify(args),
+          body: JSON.stringify({
+            model: openAiModel,
+            messages: buildMessages(options),
+            response_format: {
+              type: 'json_schema',
+              json_schema: {
+                name: options.response_model.name,
+                strict: true,
+                schema: toJsonSchema(options.response_model.schema),
+              },
+            },
+            temperature: options.temperature,
+            max_completion_tokens: options.maxOutputTokens,
+          }),
         });
 
         if (!response.ok) {
           throw new Error(`openai_request_failed:${response.status}`);
         }
 
-        return (await response.json()) as any;
+        const json = (await response.json()) as StructuredChatResponse;
+        const content = json.choices?.[0]?.message?.content;
+        if (typeof content !== 'string' || content.trim().length === 0) {
+          throw new Error('openai_response_missing_content');
+        }
+
+        return {
+          data: JSON.parse(content),
+          usage: json.usage,
+        };
       },
     }),
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mercuryo-ai/agentbrowse",
-  "version": "0.2.52",
+  "version": "0.2.53",
   "type": "module",
   "description": "Browser automation primitives library for AI agents",
   "license": "MIT",