@oharness/engine 0.10.0 → 0.11.0

package/dist/auth/providers/openrouter.d.ts

@@ -0,0 +1,2 @@
+import type { AuthProvider } from "../types.ts";
+export declare const openrouterAuth: AuthProvider;

package/dist/auth/providers/openrouter.js

@@ -0,0 +1,101 @@
+import { randomUUID } from "node:crypto";
+import { createPkce, listenForCallback } from "../oauth.js";
+import { parseJson } from "../../util/json.js";
+/**
+ * OpenRouter, with a sign-in that ends in an ordinary key.
+ *
+ * OpenRouter's OAuth is a public PKCE flow built for apps like this one: no
+ * client id to register, and what the code is exchanged for is not an
+ * expiring token but a permanent API key the user owns, can see and can
+ * revoke on openrouter.ai. So the sign-in is only a way of not copying a key
+ * by hand — the result is stored and sent exactly as a pasted key would be,
+ * and spends the user's own OpenRouter credits.
+ */
+const AUTHORIZE_URL = "https://openrouter.ai/auth";
+const KEY_URL = "https://openrouter.ai/api/v1/auth/keys";
+const ENV_VAR = "OPENROUTER_API_KEY";
+async function exchangeForKey(code, verifier) {
+    const response = await fetch(KEY_URL, {
+        method: "POST",
+        headers: { accept: "application/json", "content-type": "application/json" },
+        body: JSON.stringify({ code, code_verifier: verifier, code_challenge_method: "S256" }),
+    });
+    const text = await response.text();
+    const parsed = parseJson(text);
+    if (!response.ok) {
+        const error = parsed.ok ? parsed.value.error : undefined;
+        const detail = typeof error === "string" ? error : (error?.message ?? text.slice(0, 200));
+        throw new Error(`OpenRouter refused the sign-in (${response.status}): ${detail}`);
+    }
+    const key = parsed.ok ? parsed.value.key : undefined;
+    if (typeof key !== "string" || !key)
+        throw new Error("OpenRouter returned no key");
+    return key;
+}
+async function signIn() {
+    const pkce = createPkce();
+    // A path nobody can guess stands in for the state OpenRouter does not echo
+    // back; the PKCE verifier is what actually ties the code to this login.
+    const path = `/callback/${randomUUID()}`;
+    const { server, port, waitForCallback, deliver } = await listenForCallback({ path, state: null });
+    const url = new URL(AUTHORIZE_URL);
+    url.search = new URLSearchParams({
+        callback_url: `http://127.0.0.1:${port}${path}`,
+        code_challenge: pkce.challenge,
+        code_challenge_method: "S256",
+    }).toString();
+    return {
+        method: "auto",
+        url: url.toString(),
+        instructions: "Sign in to OpenRouter in your browser; this continues on its own.",
+        paste: deliver,
+        async callback() {
+            try {
+                const params = await waitForCallback;
+                const key = await exchangeForKey(params.get("code"), pkce.verifier);
+                return { type: "success", key, metadata: { source: "oauth" } };
+            }
+            catch (err) {
+                return { type: "failed", error: err instanceof Error ? err.message : String(err) };
+            }
+            finally {
+                server.close();
+                server.closeAllConnections();
+            }
+        },
+    };
+}
+function bearer(key, source) {
+    return { headers: { authorization: `Bearer ${key}` }, source };
+}
+export const openrouterAuth = {
+    id: "openrouter",
+    label: "OpenRouter",
+    methods: [
+        {
+            type: "oauth",
+            label: "Sign in with OpenRouter",
+            browser: true,
+            authorize: signIn,
+        },
+        {
+            type: "api",
+            label: "API key",
+            prompts: [{ type: "text", key: "key", message: "OpenRouter API key", placeholder: "sk-or-…", secret: true }],
+        },
+    ],
+    async loader(credential) {
+        const envKey = process.env[ENV_VAR];
+        if (envKey)
+            return bearer(envKey, `env:${ENV_VAR}`);
+        if (credential?.type === "api") {
+            return bearer(credential.key, credential.metadata?.["source"] === "oauth" ? "stored:oauth" : "stored:api");
+        }
+        return undefined;
+    },
+    async discover() {
+        const envKey = process.env[ENV_VAR];
+        return envKey ? { credential: { type: "api", key: envKey }, source: `env:${ENV_VAR}` } : undefined;
+    },
+};
+//# sourceMappingURL=openrouter.js.map

package/dist/auth/providers/openrouter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openrouter.js","sourceRoot":"","sources":["../../../src/auth/providers/openrouter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAG/C;;;;;;;;;GASG;AAEH,MAAM,aAAa,GAAG,4BAA4B,CAAC;AACnD,MAAM,OAAO,GAAG,wCAAwC,CAAC;AACzD,MAAM,OAAO,GAAG,oBAAoB,CAAC;AAErC,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,QAAgB;IAC1D,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,OAAO,EAAE;QACpC,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC3E,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,EAAE,CAAC;KACvF,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,MAAM,GAAG,SAAS,CAA2D,IAAI,CAAC,CAAC;IACzF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,KAAK,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACzD,MAAM,MAAM,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC1F,MAAM,IAAI,KAAK,CAAC,mCAAmC,QAAQ,CAAC,MAAM,MAAM,MAAM,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACrD,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IACnF,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,MAAM;IACnB,MAAM,IAAI,GAAG,UAAU,EAAE,CAAC;IAC1B,2EAA2E;IAC3E,wEAAwE;IACxE,MAAM,IAAI,GAAG,aAAa,UAAU,EAAE,EAAE,CAAC;IACzC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,EAAE,GAAG,MAAM,iBAAiB,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAElG,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,CAAC;IACnC,GAAG,CAAC,MAAM,GAAG,IAAI,eAAe,CAAC;QAC/B,YAAY,EAAE,oBAAoB,IAAI,GAAG,IAAI,EAAE;QAC/C,cAAc,EAAE,IAAI,CAAC,SAAS;QAC9B,qBAAqB,EAAE,MAAM;KAC9B,CAAC,CAAC,QAAQ,EAAE,CAAC;IAEd,OAAO;QACL,MAAM,EAAE,MAAM;QACd,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE;QACnB,YAAY,EAAE,mEAAmE;QACjF,KAAK,EAAE,OAAO;QACd,KAAK,CAAC,QAAQ;YACZ,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC;gBACrC,MAAM,GAAG,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACrE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC;YACjE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,mBAAmB,EAAE,CAAC;YAC/B,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,GAAW,EAAE,MAAc;IACzC,OAAO,EAAE,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC;AACjE,CAAC;AAED,MAAM,CAAC,MAAM,cAAc,GAAiB;IAC1C,EAAE,EAAE,YAAY;IAChB,KAAK,EAAE,YAAY;IAEnB,OAAO,EAAE;QACP;YACE,IAAI,EAAE,OAAO;YACb,KAAK,EAAE,yBAAyB;YAChC,OAAO,EAAE,IAAI;YACb,SAAS,EAAE,MAAM;SAClB;QACD;YACE,IAAI,EAAE,KAAK;YACX,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,oBAAoB,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;SAC7G;KACF;IAED,KAAK,CAAC,MAAM,CAAC,UAAU;QACrB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC,MAAM,EAAE,OAAO,OAAO,EAAE,CAAC,CAAC;QACpD,IAAI,UAAU,EAAE,IAAI,KAAK,KAAK,EAAE,CAAC;YAC/B,OAAO,MAAM,CAAC,UAAU,CAAC,GAAG,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QAC7G,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,QAAQ;QACZ,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACpC,OAAO,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACrG,CAAC;CACF,CAAC"}

package/dist/auth/types.d.ts

@@ -79,6 +79,14 @@ export type AuthorizeResult = {
      */
     method: "auto";
     callback(): Promise<CallbackResult>;
+    /**
+     * Finish the flow from the address the browser ended on, pasted back
+     * by hand — for when the browser runs on another machine (an SSH
+     * session) and its redirect to this one's loopback never arrives.
+     * Returns why the paste was not accepted, or nothing when it was, in
+     * which case `callback()` resolves as if the redirect had come in.
+     */
+    paste?(input: string): string | undefined;
 } | {
     /** The user copies a code out of the browser and pastes it back. */
     method: "code";
@@ -95,6 +103,13 @@ export type CallbackResult = ({
 export interface OAuthMethod {
     type: "oauth";
     label: string;
+    /**
+     * A vendor's own browser sign-in that asks nothing up front and finishes on
+     * a loopback redirect — Sign in with ChatGPT, OpenRouter. Marked so a UI
+     * can tell it from the OHarness account sign-ins, which ask for an email
+     * and go through the website, and offer it as the one button it is.
+     */
+    browser?: boolean;
     prompts?: Prompt[];
     authorize(inputs: Record<string, string>): Promise<AuthorizeResult>;
 }

package/dist/auth/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/auth/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAoLH,MAAM,UAAU,SAAS,CAAC,UAA2B,EAAE,MAAM,GAAG,MAAM;IACpE,OAAO,UAAU,CAAC,OAAO,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,GAAG,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;AAC7E,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/auth/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAmMH,MAAM,UAAU,SAAS,CAAC,UAA2B,EAAE,MAAM,GAAG,MAAM;IACpE,OAAO,UAAU,CAAC,OAAO,GAAG,CAAC,IAAI,UAAU,CAAC,OAAO,GAAG,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;AAC7E,CAAC"}

package/dist/browser/index.d.ts

@@ -1,5 +1,5 @@
 export { BrowserSession, originOf } from "./session.ts";
-export type { BrowserSessionOptions, ConsoleEntry, NetworkEntry } from "./session.ts";
+export type { BrowserSessionOptions, BrowserTurnShot, ConsoleEntry, NetworkEntry } from "./session.ts";
 export { createBrowserTools } from "./tools.ts";
 export { findChrome, launchChrome } from "./chrome.ts";
 export type { LaunchChromeOptions } from "./chrome.ts";

package/dist/browser/plugin.d.ts

@@ -1,6 +1,6 @@
 import type { Logger } from "../util/logger.ts";
 import type { Tool } from "../tools/types.ts";
-import type { BrowserSessionOptions } from "./session.ts";
+import type { BrowserSessionOptions, BrowserTurnShot } from "./session.ts";
 import type { BrowserProviderPlugin } from "./provider.ts";
 import type { ExternalSite, ImportedSite } from "./login-import.ts";
 import type { BrowserInputEvent, FrameOptions } from "./target.ts";
@@ -57,6 +57,11 @@ export interface BrowserUseSession {
         running: boolean;
     }>;
     close(): Promise<void>;
+    /**
+     * The turn ended: pictures of the tabs it left that the agent did not look
+     * at, or none when `capture` is false. Optional for third-party feature plugins.
+     */
+    endTurn?(capture: boolean): Promise<BrowserTurnShot[]>;
     /** Name the conversation this browser works for; see BrowserTargetContext.label. */
     setLabel?(label: () => string | undefined): void;
     /** Move to the other browser; the next call opens it. Optional for third-party feature plugins. */

package/dist/browser/plugin.js.map

@@ -1 +1 @@
-{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../../src/browser/plugin.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AA2EhD,MAAM,UAAU,sBAAsB,CAAC,OAAuC,EAAE;IAC9E,OAAO;QACL,EAAE,EAAE,aAAa;QACjB,aAAa,CAAC,OAAO;YACnB,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/C,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxD,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClD,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7C,CAAC,CAAC;YACH,IAAI,OAAO,CAAC,SAAS;gBAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvD,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,kBAAkB,CAAC,OAAyB,CAAC;KAClE,CAAC;AACJ,CAAC"}
+{"version":3,"file":"plugin.js","sourceRoot":"","sources":["../../src/browser/plugin.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAgFhD,MAAM,UAAU,sBAAsB,CAAC,OAAuC,EAAE;IAC9E,OAAO;QACL,EAAE,EAAE,aAAa;QACjB,aAAa,CAAC,OAAO;YACnB,MAAM,OAAO,GAAG,IAAI,cAAc,CAAC;gBACjC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/C,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxD,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACrD,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClD,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC7C,CAAC,CAAC;YACH,IAAI,OAAO,CAAC,SAAS;gBAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACvD,OAAO,OAAO,CAAC;QACjB,CAAC;QACD,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,kBAAkB,CAAC,OAAyB,CAAC;KAClE,CAAC;AACJ,CAAC"}

package/dist/browser/session.d.ts

@@ -4,6 +4,13 @@ import { LoginImportMemory } from "./login-import.ts";
 import type { ExternalSite, ImportedSite } from "./login-import.ts";
 import type { Snapshot, SnapshotOptions } from "./snapshot.ts";
 import type { BrowserInputEvent, BrowserPage, BrowserTargetFactory, FrameOptions } from "./target.ts";
+/** One tab as a turn left it, for the person reading the turn back. */
+export interface BrowserTurnShot {
+    mimeType: string;
+    data: string;
+    title?: string;
+    url?: string;
+}
 export interface ConsoleEntry {
     level: string;
     text: string;
@@ -80,6 +87,17 @@ export declare class BrowserSession {
     private importing;
     private probing?;
     private readonly suggestions;
+    /**
+     * Tabs the tools acted on this turn whose last state the agent has not
+     * looked at in a screenshot, oldest first. A tab whose last step was a
+     * screenshot is already pictured in the transcript.
+     */
+    private readonly unseen;
+    /**
+     * Tabs wearing the working glow this turn, with the browser they are in and
+     * the script that puts the glow back after a navigation.
+     */
+    private readonly glowing;
     constructor(opts?: BrowserSessionOptions);
     /** Name the conversation this browser works for. Read lazily, so it may change. */
     setLabel(label: () => string | undefined): void;
@@ -148,6 +166,13 @@ export declare class BrowserSession {
         x: number;
         y: number;
     }>;
+    /** The border box of a ref, in viewport CSS pixels, after scrolling it into view. */
+    boxOf(ref: string): Promise<{
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    }>;
     resolveObject(ref: string): Promise<string>;
     /** Every open tab. Starts the browser if nothing has yet. */
     tabs(): Promise<BrowserPage[]>;
@@ -241,6 +266,28 @@ export declare class BrowserSession {
     forward(event: BrowserInputEvent): Promise<void>;
     consoleEntries(): readonly ConsoleEntry[];
     networkEntries(): readonly NetworkEntry[];
+    /**
+     * Edge the tab in use with the working glow until the turn ends, so a person
+     * glancing at their browser can tell the agent is in it. Not the in-app
+     * browser: the pane around it already says so. Never seen by the model —
+     * screenshots are taken without it, and it is hidden from the
+     * accessibility tree.
+     */
+    glow(): Promise<void>;
+    /** Run `fn` with the glow out of the picture on `page`. */
+    withoutGlow<T>(page: BrowserPage, fn: () => Promise<T>): Promise<T>;
+    /** Take the glow off, from every tab or only those in one browser. */
+    private unglow;
+    /** A tool ran against the tab in use; `sawPage` when its last step was a screenshot. */
+    noteTool(sawPage: boolean): void;
+    /**
+     * The turn ended: a picture of each tab it left unseen, for the transcript.
+     *
+     * The agent works from the accessibility tree and rarely looks, so without
+     * this a browser turn is often read back with no picture of the page it
+     * left. `capture` false only forgets the turn's tabs.
+     */
+    endTurn(capture: boolean): Promise<BrowserTurnShot[]>;
     /** Let go of both browsers: the built-in one closes, the person's is detached. */
     close(): Promise<void>;
     /** Let go of one browser, leaving the other as it is. */

package/dist/browser/session.js

@@ -20,6 +20,14 @@ import { LoginImportMemory, externalBrowserName, hostInSite, listExternalSites,
  */
 const CONSOLE_LIMIT = 200;
 const NETWORK_LIMIT = 200;
+/**
+ * At most this many tabs get a picture when a turn ends: a turn that went
+ * through ten pages is read back by its last few, and each is a capture.
+ * The page shows one row of them and opens the rest from it.
+ */
+const TURN_SHOT_MAX_TABS = 6;
+/** A tab that does not answer a capture in this long is left out, not waited on. */
+const TURN_SHOT_TIMEOUT_MS = 5_000;
 function emptySide() {
     return { refs: new Map(), wired: new WeakSet() };
 }
@@ -65,6 +73,17 @@ export class BrowserSession {
     importing = false;
     probing;
     suggestions = new Map();
+    /**
+     * Tabs the tools acted on this turn whose last state the agent has not
+     * looked at in a screenshot, oldest first. A tab whose last step was a
+     * screenshot is already pictured in the transcript.
+     */
+    unseen = new Map();
+    /**
+     * Tabs wearing the working glow this turn, with the browser they are in and
+     * the script that puts the glow back after a navigation.
+     */
+    glowing = new Map();
     constructor(opts = {}) {
         this.opts = opts;
         this.policy = opts.mode === "extension" || opts.mode === "auto" ? opts.mode : "builtin";
@@ -375,6 +394,25 @@ export class BrowserSession {
             y: (quad[1] + quad[3] + quad[5] + quad[7]) / 4,
         };
     }
+    /** The border box of a ref, in viewport CSS pixels, after scrolling it into view. */
+    async boxOf(ref) {
+        const page = await this.page();
+        const backendNodeId = this.refs.get(ref);
+        if (backendNodeId === undefined) {
+            throw new BrowserError(`no such element ${ref}. Call browser_read_page to get current refs — they are replaced on every snapshot.`);
+        }
+        await page.send("DOM.scrollIntoViewIfNeeded", { backendNodeId }).catch(() => { });
+        const box = await page.send("DOM.getBoxModel", { backendNodeId }).catch(() => undefined);
+        const model = (box?.["model"] ?? {});
+        const quad = Array.isArray(model["border"]) ? model["border"] : Array.isArray(model["content"]) ? model["content"] : undefined;
+        if (!quad || quad.length < 8)
+            throw new BrowserError(`${ref} is not visible on the page`);
+        const xs = [quad[0], quad[2], quad[4], quad[6]];
+        const ys = [quad[1], quad[3], quad[5], quad[7]];
+        const x = Math.min(...xs);
+        const y = Math.min(...ys);
+        return { x, y, width: Math.max(...xs) - x, height: Math.max(...ys) - y };
+    }
     async resolveObject(ref) {
         const page = await this.page();
         const backendNodeId = this.refs.get(ref);
@@ -500,10 +538,14 @@ export class BrowserSession {
             throw new BrowserError(`no tab ${id}`);
         const wasActive = page === this.active;
         const released = Boolean(target.opened && target.release && !target.opened(id));
-        if (released)
+        if (released) {
+            await this.unglow(undefined, page);
             await target.release(page);
-        else
+        }
+        else {
+            this.glowing.delete(page);
             await page.close();
+        }
         if (wasActive)
             this.active = undefined;
         // What is left to work in: in the person's own browser only this
@@ -595,8 +637,81 @@ export class BrowserSession {
     networkEntries() {
         return this.network;
     }
+    /**
+     * Edge the tab in use with the working glow until the turn ends, so a person
+     * glancing at their browser can tell the agent is in it. Not the in-app
+     * browser: the pane around it already says so. Never seen by the model —
+     * screenshots are taken without it, and it is hidden from the
+     * accessibility tree.
+     */
+    async glow() {
+        const page = this.active;
+        if (!page || !this.target || this.target.kind === "electron" || this.glowing.has(page))
+            return;
+        const entry = { side: this.mode };
+        this.glowing.set(page, entry);
+        const added = await page.send("Page.addScriptToEvaluateOnNewDocument", { source: GLOW_ON }).catch(() => undefined);
+        if (typeof added?.["identifier"] === "string")
+            entry.script = added["identifier"];
+        await page.send("Runtime.evaluate", { expression: GLOW_ON }).catch(() => { });
+    }
+    /** Run `fn` with the glow out of the picture on `page`. */
+    async withoutGlow(page, fn) {
+        if (!this.glowing.has(page))
+            return fn();
+        await page.send("Runtime.evaluate", { expression: glowVisibility("hidden") }).catch(() => { });
+        try {
+            return await fn();
+        }
+        finally {
+            await page.send("Runtime.evaluate", { expression: glowVisibility("visible") }).catch(() => { });
+        }
+    }
+    /** Take the glow off, from every tab or only those in one browser. */
+    async unglow(side, only) {
+        const pages = [...this.glowing].filter(([page, entry]) => (!side || entry.side === side) && (!only || page === only));
+        await Promise.all(pages.map(async ([page, entry]) => {
+            this.glowing.delete(page);
+            if (entry.script)
+                await page.send("Page.removeScriptToEvaluateOnNewDocument", { identifier: entry.script }).catch(() => { });
+            await page.send("Runtime.evaluate", { expression: GLOW_OFF }).catch(() => { });
+        }));
+    }
+    /** A tool ran against the tab in use; `sawPage` when its last step was a screenshot. */
+    noteTool(sawPage) {
+        const page = this.active;
+        if (!page)
+            return;
+        this.unseen.delete(page.id);
+        if (!sawPage)
+            this.unseen.set(page.id, page);
+    }
+    /**
+     * The turn ended: a picture of each tab it left unseen, for the transcript.
+     *
+     * The agent works from the accessibility tree and rarely looks, so without
+     * this a browser turn is often read back with no picture of the page it
+     * left. `capture` false only forgets the turn's tabs.
+     */
+    async endTurn(capture) {
+        const pages = [...this.unseen.values()].slice(-TURN_SHOT_MAX_TABS);
+        this.unseen.clear();
+        // Off before the pictures: the transcript shows the page, not the glow.
+        await this.unglow();
+        if (!capture)
+            return [];
+        const shots = [];
+        for (const page of pages) {
+            const shot = await withTimeout(capturePage(page), TURN_SHOT_TIMEOUT_MS).catch(() => undefined);
+            if (shot)
+                shots.push(shot);
+        }
+        return shots;
+    }
     /** Let go of both browsers: the built-in one closes, the person's is detached. */
     async close() {
+        this.unseen.clear();
+        await this.unglow();
         this.consoleLog.length = 0;
         this.network.length = 0;
         await Promise.all([this.closeSide("builtin"), this.closeSide("extension")]);
@@ -605,6 +720,8 @@ export class BrowserSession {
     async closeSide(side) {
         const target = this.sides[side].target;
         this.sides[side] = emptySide();
+        // The person's tabs outlive the connection; the glow must not.
+        await this.unglow(side);
         await target?.close().catch(() => { });
     }
     /**
@@ -834,4 +951,62 @@ export function originOf(url) {
 function errorText(err) {
     return err instanceof Error ? err.message : String(err);
 }
+/**
+ * The working glow, put into a page from outside it.
+ *
+ * Built for pages that are not ours: a closed shadow root so the page's CSS
+ * cannot reach it, styles through the CSSOM and constructed sheets so a
+ * Content-Security-Policy or Trusted Types does not refuse them, top frame
+ * only, and nothing a click or a screen reader can land on.
+ */
+const GLOW_TAG = "oharness-glow";
+const GLOW_CSS = ".g,.g::after{position:fixed;inset:0;pointer-events:none}" +
+    ".g{box-shadow:inset 0 0 0 2px rgba(198,244,50,.9),inset 0 0 16px 2px rgba(198,244,50,.5);animation:in .5s ease}" +
+    ".g::after{content:'';box-shadow:inset 0 0 60px 12px rgba(198,244,50,.3),inset 0 0 140px 24px rgba(107,92,255,.12);animation:breathe 2.6s ease-in-out infinite}" +
+    "@keyframes in{from{opacity:0}}@keyframes breathe{0%,100%{opacity:.55}50%{opacity:1}}" +
+    "@media (prefers-reduced-motion:reduce){.g,.g::after{animation:none}}";
+const GLOW_ON = `(() => {
+  if (window.top !== window) return;
+  const put = () => {
+    const root = document.documentElement;
+    if (!root || root.querySelector(":scope > ${GLOW_TAG}")) return;
+    const host = document.createElement("${GLOW_TAG}");
+    host.setAttribute("aria-hidden", "true");
+    for (const [key, value] of [["all", "initial"], ["display", "block"], ["position", "fixed"], ["inset", "0"], ["pointer-events", "none"], ["z-index", "2147483647"]]) host.style.setProperty(key, value, "important");
+    const shadow = host.attachShadow({ mode: "closed" });
+    const sheet = new CSSStyleSheet();
+    sheet.replaceSync(${JSON.stringify(GLOW_CSS)});
+    shadow.adoptedStyleSheets = [sheet];
+    const ring = document.createElement("div");
+    ring.className = "g";
+    shadow.appendChild(ring);
+    root.appendChild(host);
+  };
+  if (document.documentElement) put(); else document.addEventListener("readystatechange", put, { once: true });
+})()`;
+const GLOW_OFF = `document.querySelector(${JSON.stringify(GLOW_TAG)})?.remove()`;
+function glowVisibility(value) {
+    return `document.querySelector(${JSON.stringify(GLOW_TAG)})?.style.setProperty("visibility", ${JSON.stringify(value)}, "important")`;
+}
+/** A tab as it is now, with what it is called. Undefined for a tab with nothing to show. */
+async function capturePage(page) {
+    const result = await page.send("Page.captureScreenshot", { format: "jpeg", quality: 70 });
+    const data = result["data"];
+    if (typeof data !== "string" || !data)
+        return undefined;
+    const about = await page
+        .send("Runtime.evaluate", { expression: "({ title: document.title, url: location.href })", returnByValue: true })
+        .then((r) => (r["result"] ?? {})["value"])
+        .catch(() => undefined);
+    const title = typeof about?.title === "string" ? about.title.trim() : "";
+    const url = typeof about?.url === "string" ? about.url : "";
+    return { mimeType: "image/jpeg", data, ...(title ? { title } : {}), ...(url ? { url } : {}) };
+}
+function withTimeout(promise, ms) {
+    let timer;
+    return Promise.race([
+        promise,
+        new Promise((_, reject) => { timer = setTimeout(() => reject(new Error("timed out")), ms); }),
+    ]).finally(() => clearTimeout(timer));
+}
 //# sourceMappingURL=session.js.map