@53able/pi-agent-browser 0.2.0 → 0.3.0

package/README.ja-JP.md

@@ -167,6 +167,8 @@ https://example.com をブラウザ経由で読み、outputs/browser/example.md
 
 どちらのツールも対話的なセッション（TUI、またはUI対応のRPC）を必要とします。無人・headless実行では、黙って進めるのではなく拒否します。
 
+`agent_browser_open`は、各セッションが`headed: true`で起動されたかどうかも記録します。`agent_browser_handoff`が、headedで開かれていないセッションに対して呼ばれた場合、人間への警告に加えて、可視ウィンドウを開き直すための`agent_browser_open`の引数（セッション名、`headed: true`、`restore: true`、最後に開いていたURL）をそのまま提示します — どのコマンドを打ち直せばよいか推測する必要はありません。ログインや2FA、CAPTCHA、本人確認に差し掛かる可能性があるタスクでは、最初から`headed: true`で開くことを推奨します。一部の確認チャレンジは使い切り・期限付きのため、途中で開き直すとログインからやり直しになることがあります。
+
 ## Troubleshooting
 
 ```bash

package/README.md

@@ -167,6 +167,8 @@ Not every step should run unattended. Two tools hand control back to a human at
 
 Both tools require an interactive session (TUI or RPC with UI support); in headless/unattended runs they refuse rather than proceed silently.
 
+`agent_browser_open` also remembers whether each named session was launched with `headed: true`. If `agent_browser_handoff` is called for a session that was not opened headed, it automatically warns the human and includes the exact `agent_browser_open` arguments (session name, `headed: true`, `restore: true`, and the last known URL) needed to reopen a visible window — no more guessing which command to rerun. Prefer `headed: true` up front for any task that might hit a login, 2FA, CAPTCHA, or identity-verification step, since some challenges are single-use and reopening mid-flow may require restarting the login.
+
 ## Troubleshooting
 
 Run:

package/index.ts

@@ -1,6 +1,6 @@
 import type { ExtensionAPI, AgentToolUpdateCallback, ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { Type } from "typebox";
-import { access, mkdir, writeFile } from "node:fs/promises";
+import { access, mkdir, readFile, writeFile } from "node:fs/promises";
 import { constants, existsSync } from "node:fs";
 import { join, resolve, basename } from "node:path";
 import { spawn } from "node:child_process";
@@ -35,6 +35,51 @@ function scanForSensitiveContent(text: string): string[] {
   return hits;
 }
 
+// Tracks whether each agent-browser session was last opened headed, so
+// agent_browser_handoff / agent_browser_commit can tell a human definitively
+// whether a visible browser window exists for them to act in, instead of guessing.
+const SESSION_STATE_DIR = ".pi-agent-browser";
+const SESSION_STATE_FILE = "sessions.json";
+const DEFAULT_SESSION_KEY = "__default__";
+
+type SessionState = { headed: boolean; lastUrl?: string; updatedAt: string };
+
+function sessionKey(params: JsonObject): string {
+  return typeof params.session === "string" && params.session.trim() ? params.session.trim() : DEFAULT_SESSION_KEY;
+}
+
+async function readSessionStates(cwd: string): Promise<Record<string, SessionState>> {
+  try {
+    const raw = await readFile(join(cwd, SESSION_STATE_DIR, SESSION_STATE_FILE), "utf8");
+    return JSON.parse(raw) as Record<string, SessionState>;
+  } catch {
+    return {};
+  }
+}
+
+async function writeSessionState(cwd: string, session: string, patch: Partial<Omit<SessionState, "updatedAt">>): Promise<void> {
+  const dir = join(cwd, SESSION_STATE_DIR);
+  await mkdir(dir, { recursive: true });
+  const states = await readSessionStates(cwd);
+  const previous = states[session];
+  states[session] = {
+    headed: patch.headed ?? previous?.headed ?? false,
+    lastUrl: patch.lastUrl ?? previous?.lastUrl,
+    updatedAt: new Date().toISOString(),
+  };
+  await writeFile(join(dir, SESSION_STATE_FILE), JSON.stringify(states, null, 2), "utf8");
+}
+
+function reopenHeadedGuidance(session: string, lastUrl: string | undefined): string[] {
+  const sessionArg = session === DEFAULT_SESSION_KEY ? {} : { session };
+  const reopenArgs = JSON.stringify({ ...sessionArg, headed: true, restore: true, ...(lastUrl ? { url: lastUrl } : {}) });
+  return [
+    `⚠ このセッション(${session === DEFAULT_SESSION_KEY ? "既定セッション" : session})はheadless(ウィンドウ非表示)で開かれています。`,
+    `人間が直接操作できるようにするには、agent_browser_open を次の引数で呼び直してください: ${reopenArgs}`,
+    "同じセッション名なのでcookie等は復元されますが、2FA/CAPTCHAなど使い切り・期限付きのチャレンジは、再度最初からやり直しになる場合があります。",
+  ];
+}
+
 type JsonObject = Record<string, unknown>;
 
 type RunResult = {
@@ -346,6 +391,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
     promptGuidelines: [
       "Use agent_browser_snapshot after opening or navigating before clicking by @ref.",
       "Pass allowedDomains for untrusted or task-scoped browsing when possible.",
+      "Prefer headed: true whenever the task might hit a login, 2FA, CAPTCHA, or identity-verification step — a human can only intervene through agent_browser_handoff if a visible window already exists.",
     ],
     parameters: OpenSchema,
     async execute(_id, rawParams, signal, onUpdate, ctx) {
@@ -365,6 +411,12 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
       if (typeof params.url === "string" && params.url.trim()) args.push(params.url.trim());
       pushExtraArgs(args, params);
       const result = await runAgentBrowser(args, { signal, timeoutMs: params.timeoutMs as number | undefined, cwd: ctx.cwd, onUpdate });
+      if (result.ok) {
+        await writeSessionState(ctx.cwd, sessionKey(params), {
+          headed: params.headed === true,
+          lastUrl: typeof params.url === "string" && params.url.trim() ? params.url.trim() : undefined,
+        });
+      }
       return buildTextResult(result);
     },
   });
@@ -582,6 +634,9 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
       const category = String(params.category);
       const reason = String(params.reason);
       const artifacts: string[] = [];
+      const session = sessionKey(params);
+      const sessionState = (await readSessionStates(ctx.cwd))[session];
+      const headlessNotes = sessionState?.headed === true ? [] : reopenHeadedGuidance(session, sessionState?.lastUrl);
 
       if (params.screenshot !== false) {
         const path = resolveOutputPath(ctx, undefined, "handoff.png");
@@ -601,6 +656,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
               `Human handoff required (${category}) but no interactive UI is available in this run mode.`,
               `Reason: ${reason}`,
               artifacts.length ? `Evidence: ${artifacts.join(", ")}` : undefined,
+              ...headlessNotes,
               "Stop this task now and ask the operator to rerun it interactively so the handoff can be completed.",
             ].filter(Boolean).join("\n"),
           }],
@@ -609,8 +665,9 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
         };
       }
 
+      if (headlessNotes.length) ctx.ui.notify(headlessNotes.join(" "), "warning");
       ctx.ui.notify(`Human handoff needed (${category}): ${reason}`, "warning");
-      ctx.ui.setStatus("agent-browser-handoff", `⏸ Waiting for human: ${reason}`);
+      ctx.ui.setStatus("agent-browser-handoff", headlessNotes.length ? `⏸ ブラウザ非表示 — ${reason}` : `⏸ Waiting for human: ${reason}`);
       let choice: string | undefined;
       try {
         choice = await ctx.ui.select(
@@ -626,6 +683,7 @@ export default function agentBrowserExtension(pi: ExtensionAPI) {
         content: [{
           type: "text",
           text: [
+            ...headlessNotes,
             resumed
               ? "Human confirmed the handoff step is complete. Take a fresh agent_browser_snapshot before continuing."
               : "Human aborted the task during handoff. Stop and report back; do not retry automatically.",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@53able/pi-agent-browser",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Typed Pi tools for browser automation, rendered-page context extraction, screenshots, and saved session state via agent-browser.",
   "type": "module",
   "license": "MIT",

package/skills/agent-browser/SKILL.md

@@ -36,6 +36,10 @@ Two tools exist so the agent stops and hands control back to a human instead of
 
 `agent_browser_read` and `agent_browser_snapshot` also run a lightweight heuristic scan for 2FA/CAPTCHA/identity-verification keywords and prepend a warning to their output when detected — treat that warning as a signal to call `agent_browser_handoff` next, not as a blocker by itself.
 
+### If the human can't find the browser window
+
+`agent_browser_open` records whether each named session was launched with `headed: true`. When `agent_browser_handoff` runs for a session that was not opened headed, it automatically prepends a warning naming the session and the exact `agent_browser_open` arguments (including the last known URL) needed to reopen it visibly, instead of leaving the human to guess. Prefer `headed: true` from the start for any task that might hit a login, 2FA, CAPTCHA, or identity-verification step — a human can only intervene through `agent_browser_handoff` if a visible window already exists. Note that some verification challenges are single-use or session-bound, so reopening may require restarting the login flow rather than resuming mid-challenge.
+
 ## Common examples
 
 Read a public page: