hanzi-browse 2.2.0

package/README.md

@@ -0,0 +1,182 @@
+# Hanzi Browse — MCP Server
+
+The MCP server exposes browser tools to MCP clients and forwards browser work to
+the Chrome extension over the local WebSocket relay.
+
+## Setup
+
+```bash
+cd mcp-server
+npm install
+npm run build
+```
+
+Add to your MCP config (e.g., `~/.claude/claude_desktop_config.json`):
+
+```json
+{
+  "mcpServers": {
+    "browser": {
+      "command": "node",
+      "args": ["/path/to/hanzi-browse/mcp-server/dist/index.js"]
+    }
+  }
+}
+```
+
+**Prerequisites:** The Chrome extension must be installed and running. See the [main README](../README.md) for full setup.
+
+## How It Works
+
+```text
+MCP client
+  -> mcp-server (stdio)
+  -> relay (WebSocket)
+  -> Chrome extension
+  -> browser agent
+```
+
+The extension is the browser executor. The MCP server should only manage MCP
+tool calls, local session bookkeeping, and blocking waits for completion.
+
+## Tools
+
+### `browser_start`
+
+Start a browser task. **Blocks until complete or timeout**.
+
+```
+browser_start(
+  task: "Search for flights to Tokyo on Google Flights",
+  url: "https://flights.google.com",        // optional starting URL
+  context: "Departing March 15, economy"     // optional extra info
+)
+
+→ {
+  "session_id": "abc123",
+  "status": "complete",
+  "task": "Search for flights to Tokyo...",
+  "answer": "Found 3 flights: JAL $850, ANA $920, United $780",
+  "total_steps": 8,
+  "recent_steps": ["Opened Google Flights", "Set destination to Tokyo", ...]
+}
+```
+
+### `browser_message`
+
+Send follow-up instructions to an existing session. Also blocks until the agent finishes.
+
+```
+browser_message(session_id: "abc123", message: "Book the cheapest one")
+```
+
+### `browser_status`
+
+Check known sessions and their latest status.
+
+```
+browser_status()                    // all active sessions
+browser_status(session_id: "abc123") // specific session
+```
+
+### `browser_stop`
+
+Stop a task.
+
+```
+browser_stop(session_id: "abc123")
+browser_stop(session_id: "abc123", remove: true)  // also delete session
+```
+
+### `browser_screenshot`
+
+Capture the current browser state as an image.
+
+```
+browser_screenshot(session_id: "abc123")
+```
+
+## Examples
+
+**Research:**
+```
+browser_start("Find the top 3 competitors for Acme Corp and summarize their pricing")
+```
+
+**Logged-in workflows:**
+```
+browser_start("Go to Jira, find my open tickets, and summarize what needs attention this week")
+```
+
+**Multi-turn:**
+```
+s = browser_start("Go to LinkedIn and find AI Engineer jobs in Montreal")
+→ { session_id: "x1", answer: "Found: Applied AI Engineer at Cohere" }
+
+browser_message("x1", "Click into that job and tell me the requirements")
+→ { answer: "Requirements: 3+ years Python, ML experience..." }
+
+browser_message("x1", "Apply to this job using my profile")
+→ { answer: "Application submitted successfully" }
+```
+
+**Parallel execution:**
+```
+browser_start("Check flight prices to Tokyo")
+browser_start("Check hotel prices in Shibuya")
+browser_start("Look up train pass costs")
+// All three run simultaneously
+```
+
+## Configuration
+
+| Environment Variable | Default | Description |
+|---|---|---|
+| `HANZI_IN_CHROME_MAX_SESSIONS` | `5` | Max concurrent browser tasks |
+| `WS_RELAY_PORT` | `7862` | WebSocket relay port |
+
+## Architecture
+
+```
+AI Tool (Claude Code, Cursor, etc.)
+    ↓ MCP Protocol (stdio)
+MCP Server
+    ↓ WebSocket
+Relay Server
+    ↓ WebSocket
+Chrome Extension
+    ↓ Extension agent loop
+Target Website
+```
+
+The relay server starts automatically when the MCP server connects. It routes
+messages between the MCP server and the Chrome extension and briefly queues
+messages while the extension service worker is asleep.
+
+> **Principle**: Hanzi is for real browser work in your signed-in Chrome.
+> Agents should prefer code, logs, APIs, and existing tools first. Use Hanzi when the job needs a real browser session.
+
+## Prompts
+
+The server exposes MCP prompts that clients auto-discover as slash commands:
+
+| Prompt | Description |
+|--------|-------------|
+| `linkedin-prospector` | Goal-driven LinkedIn outreach — networking, sales, partnerships, or hiring |
+| `e2e-tester` | Test your app in a real browser — reports bugs with screenshots and code references |
+| `social-poster` | Post across LinkedIn, Twitter, Reddit, HN — drafts per-platform, posts from your browser |
+
+In Claude Code, use the built-in `linkedin-prospector` prompt from the MCP prompt list.
+
+## Skills CLI
+
+```bash
+hanzi-browser skills                              # list available skills
+hanzi-browser skills install linkedin-prospector   # install SKILL.md to your project
+```
+
+Skills are portable SKILL.md files for agents that don't support MCP prompts (Cline, Codex). Each skill follows the same principle: use existing tools first, Hanzi only for real browser steps.
+
+## License
+
+[Polyform Noncommercial 1.0.0](../LICENSE)

package/dist/agent/loop.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Server-Side Managed Agent Loop
+ *
+ * Drives the browser automation agent from the server:
+ * 1. Receives a task
+ * 2. Calls Vertex AI (via callLLM) with system prompt + tools
+ * 3. For each tool_use: sends execution request to extension via WebSocket relay
+ * 4. Gets tool results back from extension
+ * 5. Feeds results back to Vertex AI
+ * 6. Repeats until end_turn or max steps
+ * 7. Returns the final answer
+ *
+ * The extension is a dumb tool executor — it only runs tools and returns results.
+ * All intelligence lives here.
+ */
+export interface AgentLoopParams {
+    /** The task description */
+    task: string;
+    /** Optional starting URL */
+    url?: string;
+    /** Optional context (form data, preferences, etc.) */
+    context?: string;
+    /** Function to execute a tool on the extension. Returns the tool result. */
+    executeTool: (toolName: string, toolInput: Record<string, any>) => Promise<ToolResult>;
+    /** Optional callback for step updates */
+    onStep?: (step: StepUpdate) => void;
+    /** Optional callback for streaming text */
+    onText?: (chunk: string) => void;
+    /** Max agent loop iterations (default: 50) */
+    maxSteps?: number;
+    /** Abort signal */
+    signal?: AbortSignal;
+}
+export interface ToolResult {
+    success: boolean;
+    output?: any;
+    error?: string;
+    /** Base64 screenshot if the tool returned one */
+    screenshot?: {
+        data: string;
+        mediaType: string;
+    };
+}
+export interface StepUpdate {
+    step: number;
+    status: "thinking" | "tool_use" | "tool_result" | "complete" | "error";
+    toolName?: string;
+    toolInput?: Record<string, any>;
+    text?: string;
+}
+export interface AgentLoopResult {
+    status: "complete" | "error" | "max_steps";
+    answer: string;
+    steps: number;
+    usage: {
+        inputTokens: number;
+        outputTokens: number;
+        apiCalls: number;
+    };
+    /** The model used for the last LLM call (for billing attribution) */
+    model?: string;
+}
+export declare function runAgentLoop(params: AgentLoopParams): Promise<AgentLoopResult>;

package/dist/agent/loop.js

@@ -0,0 +1,186 @@
+/**
+ * Server-Side Managed Agent Loop
+ *
+ * Drives the browser automation agent from the server:
+ * 1. Receives a task
+ * 2. Calls Vertex AI (via callLLM) with system prompt + tools
+ * 3. For each tool_use: sends execution request to extension via WebSocket relay
+ * 4. Gets tool results back from extension
+ * 5. Feeds results back to Vertex AI
+ * 6. Repeats until end_turn or max steps
+ * 7. Returns the final answer
+ *
+ * The extension is a dumb tool executor — it only runs tools and returns results.
+ * All intelligence lives here.
+ */
+import { callLLM } from "../llm/client.js";
+import { AGENT_TOOLS } from "./tools.js";
+import { buildSystemPrompt } from "./system-prompt.js";
+// --- Agent Loop ---
+export async function runAgentLoop(params) {
+    const { task, url, context, executeTool, onStep, onText, maxSteps = 50, signal, } = params;
+    const system = buildSystemPrompt();
+    const tools = AGENT_TOOLS;
+    const messages = [];
+    let totalUsage = { inputTokens: 0, outputTokens: 0, apiCalls: 0 };
+    let lastModel;
+    // Build initial user message
+    let userMessage = task;
+    if (url) {
+        userMessage = `Navigate to ${url} first, then: ${task}`;
+    }
+    if (context) {
+        userMessage += `\n\n<context>\n${context}\n</context>`;
+    }
+    messages.push({ role: "user", content: userMessage });
+    for (let step = 1; step <= maxSteps; step++) {
+        if (signal?.aborted) {
+            return {
+                status: "error",
+                answer: "Task was cancelled.",
+                steps: step - 1,
+                usage: totalUsage,
+                model: lastModel,
+            };
+        }
+        onStep?.({ step, status: "thinking" });
+        // Call LLM
+        let response;
+        try {
+            response = await callLLM({
+                messages,
+                system,
+                tools,
+                signal,
+                onText,
+            });
+        }
+        catch (err) {
+            console.error(`[AgentLoop] LLM call failed at step ${step}:`, err.message);
+            return {
+                status: "error",
+                answer: `LLM call failed: ${err.message}`,
+                steps: step,
+                usage: totalUsage,
+                model: lastModel,
+            };
+        }
+        totalUsage.apiCalls++;
+        totalUsage.inputTokens += response.usage?.input_tokens || 0;
+        totalUsage.outputTokens += response.usage?.output_tokens || 0;
+        if (response.model)
+            lastModel = response.model;
+        // Add assistant response to conversation
+        messages.push({ role: "assistant", content: response.content });
+        // Extract text and tool calls
+        const textBlocks = response.content.filter((b) => b.type === "text");
+        const toolUseBlocks = response.content.filter((b) => b.type === "tool_use");
+        // If no tool calls, we're done
+        if (response.stop_reason === "end_turn" || toolUseBlocks.length === 0) {
+            const answer = textBlocks.map((b) => b.text).join("\n").trim();
+            onStep?.({ step, status: "complete", text: answer });
+            return {
+                status: "complete",
+                answer: answer || "Task completed.",
+                steps: step,
+                usage: totalUsage,
+                model: lastModel,
+            };
+        }
+        // Execute each tool call
+        const allowedToolNames = new Set(tools.map((t) => t.name));
+        const toolResults = [];
+        for (const toolUse of toolUseBlocks) {
+            // Validate tool name against allowed list before forwarding to extension
+            if (!allowedToolNames.has(toolUse.name)) {
+                console.error(`[AgentLoop] LLM requested unknown tool: ${toolUse.name}`);
+                toolResults.push({
+                    type: "tool_result",
+                    tool_use_id: toolUse.id,
+                    content: [{ type: "text", text: `Error: Unknown tool "${toolUse.name}". Available tools: ${[...allowedToolNames].join(", ")}` }],
+                });
+                continue;
+            }
+            onStep?.({
+                step,
+                status: "tool_use",
+                toolName: toolUse.name,
+                toolInput: toolUse.input,
+            });
+            let result;
+            try {
+                result = await executeTool(toolUse.name, toolUse.input);
+            }
+            catch (err) {
+                // Retry once on transient errors (timeouts, relay disconnects)
+                const isTransient = err.message?.includes("timed out") ||
+                    err.message?.includes("not connected") ||
+                    err.message?.includes("Relay");
+                if (isTransient && !signal?.aborted) {
+                    console.error(`[AgentLoop] Transient error on ${toolUse.name}, retrying once: ${err.message}`);
+                    try {
+                        result = await executeTool(toolUse.name, toolUse.input);
+                    }
+                    catch (retryErr) {
+                        result = { success: false, error: `${retryErr.message} (after retry)` };
+                    }
+                }
+                else {
+                    result = { success: false, error: err.message };
+                }
+            }
+            onStep?.({ step, status: "tool_result", toolName: toolUse.name });
+            // Check abort after each tool — don't feed results back to LLM if cancelled
+            if (signal?.aborted) {
+                return {
+                    status: "error",
+                    answer: "Task was cancelled.",
+                    steps: step,
+                    usage: totalUsage,
+                    model: lastModel,
+                };
+            }
+            // Build tool result content block
+            const resultContent = [];
+            // Add text result
+            const textOutput = result.error
+                ? `Error: ${result.error}`
+                : typeof result.output === "string"
+                    ? result.output
+                    : JSON.stringify(result.output);
+            resultContent.push({ type: "text", text: textOutput });
+            // Add screenshot if present
+            if (result.screenshot) {
+                resultContent.push({
+                    type: "image",
+                    source: {
+                        type: "base64",
+                        media_type: result.screenshot.mediaType,
+                        data: result.screenshot.data,
+                    },
+                });
+            }
+            toolResults.push({
+                type: "tool_result",
+                tool_use_id: toolUse.id,
+                content: resultContent,
+            });
+        }
+        // Add tool results as user message
+        messages.push({ role: "user", content: toolResults });
+    }
+    // Exceeded max steps
+    const lastText = messages
+        .filter((m) => m.role === "assistant")
+        .flatMap((m) => Array.isArray(m.content)
+        ? m.content.filter((b) => b.type === "text").map((b) => b.text)
+        : [m.content])
+        .pop();
+    return {
+        status: "max_steps",
+        answer: lastText || "Task did not complete within the step limit.",
+        steps: maxSteps,
+        usage: totalUsage,
+        model: lastModel,
+    };
+}

package/dist/agent/system-prompt.d.ts

@@ -0,0 +1,7 @@
+/**
+ * System prompt for server-side managed agent loop.
+ */
+export declare function buildSystemPrompt(): Array<{
+    type: "text";
+    text: string;
+}>;

package/dist/agent/system-prompt.js

@@ -0,0 +1,41 @@
+/**
+ * System prompt for server-side managed agent loop.
+ */
+export function buildSystemPrompt() {
+    const now = new Date();
+    const dateStr = now.toLocaleDateString("en-US", {
+        month: "numeric",
+        day: "numeric",
+        year: "numeric",
+    });
+    const timeStr = now.toLocaleTimeString("en-US");
+    return [
+        {
+            type: "text",
+            text: `You are a web automation assistant with browser tools. Your priority is to complete the user's request efficiently and autonomously.
+
+Browser tasks often require long-running, agentic capabilities. When you encounter a user request that feels time-consuming or extensive in scope, you should be persistent and use all available context needed to accomplish the task. The user expects you to work autonomously until the task is complete. Do not ask for permission - just do it.
+
+<behavior_instructions>
+The current date is ${dateStr}, ${timeStr}.
+
+Keep responses concise and action-oriented.
+Do not use emojis unless asked.
+Do not introduce yourself. Respond to the user's request directly.
+Do not ask for permission or confirmation. Just complete the task.
+</behavior_instructions>
+
+<tool_usage_requirements>
+Use "read_page" first to get a DOM tree with numeric element IDs (backendNodeIds). This allows you to reliably target elements.
+
+Use numeric element references from read_page (e.g. "42") with the "left_click" action of the "computer" tool and the "form_input" tool. Only use coordinate-based actions when references fail.
+
+Use "get_page_text" or "read_page" to efficiently read content instead of repeatedly scrolling.
+
+ALWAYS use form_input for ANY dropdown or select element. Never use computer clicks for dropdowns.
+
+When a page shows only a loading spinner, use the computer tool with action "wait" (duration 2-3 seconds) then read_page again.
+</tool_usage_requirements>`,
+        },
+    ];
+}

package/dist/agent/tools.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Tool definitions for server-side managed agent loop.
+ *
+ * These mirror the extension's tool definitions but are used by the server
+ * when driving the agent loop via Vertex AI. The extension receives
+ * tool execution requests and returns results.
+ */
+import type { Tool } from "../llm/client.js";
+export declare const AGENT_TOOLS: Tool[];

package/dist/agent/tools.js

@@ -0,0 +1,154 @@
+/**
+ * Tool definitions for server-side managed agent loop.
+ *
+ * These mirror the extension's tool definitions but are used by the server
+ * when driving the agent loop via Vertex AI. The extension receives
+ * tool execution requests and returns results.
+ */
+export const AGENT_TOOLS = [
+    {
+        name: "read_page",
+        description: `Get a rich DOM tree of the page via Chrome DevTools Protocol. Returns interactive elements with numeric backendNodeId references (e.g., [42]<button>Submit</button>). IMPORTANT: Only use element IDs from the CURRENT output — IDs change between calls. Pierces shadow DOM and iframes automatically.`,
+        input_schema: {
+            type: "object",
+            properties: {
+                max_chars: {
+                    type: "number",
+                    description: "Maximum characters for output (default: 50000).",
+                },
+            },
+            required: [],
+        },
+    },
+    {
+        name: "find",
+        description: `Find elements on the page using natural language. Can search by purpose (e.g., "search bar", "login button") or text content. Returns up to 20 matching elements with references.`,
+        input_schema: {
+            type: "object",
+            properties: {
+                query: {
+                    type: "string",
+                    description: 'Natural language description of what to find (e.g., "search bar", "add to cart button")',
+                },
+            },
+            required: ["query"],
+        },
+    },
+    {
+        name: "form_input",
+        description: `Set values in ANY form element — text inputs, textareas, dropdowns, checkboxes, radio buttons, date pickers. For dropdowns, just pass the desired option text. ALWAYS prefer form_input over computer clicks for form fields.`,
+        input_schema: {
+            type: "object",
+            properties: {
+                ref: {
+                    type: "string",
+                    description: 'Element reference from read_page (e.g., "42") or find tool (e.g., "ref_1")',
+                },
+                value: {
+                    type: "string",
+                    description: "The value to set.",
+                },
+            },
+            required: ["ref", "value"],
+        },
+    },
+    {
+        name: "computer",
+        description: `Use a mouse and keyboard to interact with a web browser, and take screenshots.
+* Click elements using their ref from read_page or find tools.
+* Take screenshots to see the current page state.
+* Scroll to see more content.`,
+        input_schema: {
+            type: "object",
+            properties: {
+                action: {
+                    type: "string",
+                    enum: [
+                        "left_click", "right_click", "type", "screenshot", "wait",
+                        "scroll", "key", "left_click_drag", "double_click", "triple_click",
+                        "zoom", "scroll_to", "hover",
+                    ],
+                    description: "The action to perform.",
+                },
+                coordinate: {
+                    type: "array",
+                    items: { type: "number" },
+                    description: "(x, y) pixel coordinates for click/scroll actions.",
+                },
+                text: {
+                    type: "string",
+                    description: "Text to type or key(s) to press.",
+                },
+                duration: {
+                    type: "number",
+                    description: "Seconds to wait (for wait action). Max 30.",
+                },
+                scroll_direction: {
+                    type: "string",
+                    enum: ["up", "down", "left", "right"],
+                    description: "Direction to scroll.",
+                },
+                scroll_amount: {
+                    type: "number",
+                    description: "Number of scroll ticks (1-10).",
+                },
+                ref: {
+                    type: "string",
+                    description: "Element reference for click/scroll_to actions.",
+                },
+                region: {
+                    type: "array",
+                    items: { type: "number" },
+                    description: "(x0, y0, x1, y1) region for zoom action.",
+                },
+            },
+            required: ["action"],
+        },
+    },
+    {
+        name: "navigate",
+        description: `Navigate to a URL, or go forward/back in browser history.`,
+        input_schema: {
+            type: "object",
+            properties: {
+                url: {
+                    type: "string",
+                    description: 'The URL to navigate to. Use "forward"/"back" for history navigation.',
+                },
+            },
+            required: ["url"],
+        },
+    },
+    {
+        name: "get_page_text",
+        description: `Extract raw text content from the page, prioritizing article content. Ideal for reading text-heavy pages.`,
+        input_schema: {
+            type: "object",
+            properties: {
+                max_chars: {
+                    type: "number",
+                    description: "Maximum characters for output (default: 50000).",
+                },
+            },
+            required: [],
+        },
+    },
+    {
+        name: "javascript_tool",
+        description: `Execute JavaScript in the page context. Returns the result of the last expression. Do NOT use 'return' — just write the expression.`,
+        input_schema: {
+            type: "object",
+            properties: {
+                action: {
+                    type: "string",
+                    description: "Must be 'javascript_exec'.",
+                },
+                text: {
+                    type: "string",
+                    description: "JavaScript code to execute.",
+                },
+            },
+            required: ["action", "text"],
+        },
+    },
+];

package/dist/cli/detect-credentials.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Credential source detection for CLI setup.
+ *
+ * Claude Code stores OAuth tokens in one of two locations:
+ *   1. ~/.claude/.credentials.json (file-based, all platforms)
+ *   2. macOS Keychain under "Claude Code-credentials" (macOS only)
+ *
+ * The original implementation only checked (1), missing most macOS users.
+ */
+export interface CredentialSource {
+    name: string;
+    slug: 'claude' | 'codex';
+    path: string;
+}
+export interface DetectOptions {
+    platform: string;
+    homedir: string;
+    fileExists: (path: string) => boolean;
+    keychainHas: (service: string) => boolean;
+}
+export interface CredentialFlowState {
+    sourcesDetected: number;
+    anyImported: boolean;
+    manualEntryChosen: boolean;
+}
+export declare function detectCredentialSources(opts: DetectOptions): CredentialSource[];
+/**
+ * Returns an error message if setup finished with no credentials configured,
+ * or null if everything is fine.
+ */
+export declare function checkCredentialFlowResult(state: CredentialFlowState): string | null;