@oharness/engine 0.7.0 → 0.8.0

package/dist/browser/screencast.js

@@ -0,0 +1,99 @@
+export class ScreencastHub {
+    page;
+    subscribers = new Set();
+    detach;
+    last;
+    constructor(page) {
+        this.page = page;
+    }
+    /** The most recent frame, for a viewer that has just arrived. */
+    async seed(quality) {
+        if (this.last)
+            return this.last;
+        try {
+            const result = await this.page.send("Page.captureScreenshot", { format: "jpeg", quality });
+            const data = result["data"];
+            if (typeof data === "string")
+                this.last = Buffer.from(data, "base64");
+        }
+        catch {
+            // A page that cannot be captured yet will produce frames when it paints.
+        }
+        return this.last;
+    }
+    async start(opts) {
+        if (this.detach)
+            return;
+        this.detach = this.page.on("Page.screencastFrame", (params) => {
+            const data = params["data"];
+            const sessionId = params["sessionId"];
+            // The ack is the backpressure: without it Chrome stops after a frame or
+            // two, which looks exactly like a page that stopped changing.
+            if (typeof sessionId === "number") {
+                void this.page.send("Page.screencastFrameAck", { sessionId }).catch(() => { });
+            }
+            if (typeof data !== "string")
+                return;
+            this.last = Buffer.from(data, "base64");
+            for (const push of this.subscribers)
+                push(this.last);
+        });
+        await this.page.send("Page.startScreencast", {
+            format: "jpeg",
+            quality: opts.quality ?? 85,
+            ...(opts.maxWidth ? { maxWidth: opts.maxWidth } : {}),
+            ...(opts.maxHeight ? { maxHeight: opts.maxHeight } : {}),
+        });
+    }
+    async stop() {
+        if (this.subscribers.size > 0 || !this.detach)
+            return;
+        this.detach();
+        this.detach = undefined;
+        await this.page.send("Page.stopScreencast").catch(() => { });
+    }
+    /** Newest frame only: a viewer behind wants the current picture, not a queue. */
+    async *frames(opts = {}) {
+        let pending = await this.seed(opts.quality ?? 85);
+        let wake;
+        let closed = false;
+        const push = (frame) => {
+            if (frame === undefined)
+                closed = true;
+            else
+                pending = frame;
+            wake?.();
+        };
+        this.subscribers.add(push);
+        await this.start(opts);
+        try {
+            for (;;) {
+                if (pending !== undefined) {
+                    const frame = pending;
+                    pending = undefined;
+                    yield frame;
+                    continue;
+                }
+                if (closed)
+                    return;
+                await new Promise((resolve) => {
+                    wake = resolve;
+                });
+                wake = undefined;
+            }
+        }
+        finally {
+            this.subscribers.delete(push);
+            await this.stop();
+        }
+    }
+    /** The page went away; every watcher ends rather than waiting on it. */
+    close() {
+        for (const push of this.subscribers)
+            push(undefined);
+        this.subscribers.clear();
+        this.detach?.();
+        this.detach = undefined;
+    }
+}
+//# sourceMappingURL=screencast.js.map

package/dist/browser/screencast.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"screencast.js","sourceRoot":"","sources":["../../src/browser/screencast.ts"],"names":[],"mappings":"AA+BA,MAAM,OAAO,aAAa;IACP,IAAI,CAAU;IACd,WAAW,GAAG,IAAI,GAAG,EAAQ,CAAC;IACvC,MAAM,CAAc;IACpB,IAAI,CAAU;IAEtB,YAAY,IAAa;QACvB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,iEAAiE;IACzD,KAAK,CAAC,IAAI,CAAC,OAAe;QAChC,IAAI,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC,IAAI,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3F,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5B,IAAI,OAAO,IAAI,KAAK,QAAQ;gBAAE,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACxE,CAAC;QAAC,MAAM,CAAC;YACP,yEAAyE;QAC3E,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,IAAkB;QACpC,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,sBAAsB,EAAE,CAAC,MAAM,EAAE,EAAE;YAC5D,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;YACtC,wEAAwE;YACxE,8DAA8D;YAC9D,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE,CAAC;gBAClC,KAAK,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,yBAAyB,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YAChF,CAAC;YACD,IAAI,OAAO,IAAI,KAAK,QAAQ;gBAAE,OAAO;YACrC,IAAI,CAAC,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YACxC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,WAAW;gBAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE;YAC3C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;YAC3B,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACzD,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,IAAI;QAChB,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO;QACtD,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;QACxB,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,iFAAiF;IACjF,KAAK,CAAC,CAAC,MAAM,CAAC,OAAqB,EAAE;QACnC,IAAI,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;QAClD,IAAI,IAA8B,CAAC;QACnC,IAAI,MAAM,GAAG,KAAK,CAAC;QAEnB,MAAM,IAAI,GAAS,CAAC,KAAK,EAAE,EAAE;YAC3B,IAAI,KAAK,KAAK,SAAS;gBAAE,MAAM,GAAG,IAAI,CAAC;;gBAClC,OAAO,GAAG,KAAK,CAAC;YACrB,IAAI,EAAE,EAAE,CAAC;QACX,CAAC,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC3B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAEvB,IAAI,CAAC;YACH,SAAS,CAAC;gBACR,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC1B,MAAM,KAAK,GAAG,OAAO,CAAC;oBACtB,OAAO,GAAG,SAAS,CAAC;oBACpB,MAAM,KAAK,CAAC;oBACZ,SAAS;gBACX,CAAC;gBACD,IAAI,MAAM;oBAAE,OAAO;gBACnB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;oBAClC,IAAI,GAAG,OAAO,CAAC;gBACjB,CAAC,CAAC,CAAC;gBACH,IAAI,GAAG,SAAS,CAAC;YACnB,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC9B,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED,wEAAwE;IACxE,KAAK;QACH,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,WAAW;YAAE,IAAI,CAAC,SAAS,CAAC,CAAC;QACrD,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAChB,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;IAC1B,CAAC;CACF"}

package/dist/browser/session.d.ts

@@ -0,0 +1,320 @@
+import type { Logger } from "../util/logger.ts";
+import type { BrowserProviderPlugin } from "./provider.ts";
+import { LoginImportMemory } from "./login-import.ts";
+import type { ExternalSite, ImportedSite } from "./login-import.ts";
+import type { Snapshot, SnapshotOptions } from "./snapshot.ts";
+import type { BrowserInputEvent, BrowserPage, BrowserTargetFactory, FrameOptions } from "./target.ts";
+export interface ConsoleEntry {
+    level: string;
+    text: string;
+    url?: string;
+    at: number;
+}
+export interface NetworkEntry {
+    requestId: string;
+    method: string;
+    url: string;
+    status?: number;
+    mimeType?: string;
+    failed?: string;
+    at: number;
+}
+export interface BrowserSessionOptions {
+    /** Overrides whatever `mode` would have chosen — the seam the desktop shell uses. */
+    target?: BrowserTargetFactory;
+    /** Host implementations. The plugin matching `mode` is selected lazily. */
+    providers?: readonly BrowserProviderPlugin[];
+    /**
+     * `builtin` starts a browser of our own; `extension` drives the person's real
+     * one through the OHarness extension; `auto` holds both and lets each call
+     * say which it needs (see `sideFor`). An explicit `target` wins over all three.
+     */
+    mode?: BrowserPolicy;
+    headless?: boolean;
+    logger?: Logger;
+    /** Where "already imported / don't ask again" is kept. Shared by default. */
+    loginMemory?: LoginImportMemory;
+    /** The owning conversation's name; see BrowserTargetContext.label. */
+    label?: () => string | undefined;
+    /** The owning conversation's project; see BrowserTargetContext.cwd. */
+    cwd?: string;
+}
+/** One of the two browsers a session can drive. */
+export type BrowserSide = "builtin" | "extension";
+/** Which of them a session may use: one of them always, or either as each call needs. */
+export type BrowserPolicy = BrowserSide | "auto";
+export declare class BrowserSession {
+    /**
+     * Both browsers' state, each started only when first used. Everything below
+     * reads the one in use through the accessors, so a call that switches
+     * browsers finds the other one's tab, refs and origin exactly as it left
+     * them — the built-in browser keeps its page while the person's is used.
+     */
+    private readonly sides;
+    private get side();
+    private get target();
+    private set target(value);
+    private get starting();
+    private set starting(value);
+    private get active();
+    private set active(value);
+    private get pageStarting();
+    private set pageStarting(value);
+    private get refs();
+    private set refs(value);
+    private get wired();
+    private set wired(value);
+    private get currentOrigin();
+    private set currentOrigin(value);
+    private readonly consoleLog;
+    private readonly network;
+    private readonly opts;
+    private sessionId?;
+    private label?;
+    /** The browser in use now. */
+    private mode;
+    /** Which browsers this session may use. */
+    private policy;
+    /** Said once, with the next tool result: a fallback the model should know about. */
+    private readonly notices;
+    private importing;
+    private probing?;
+    private readonly suggestions;
+    constructor(opts?: BrowserSessionOptions);
+    /** Name the conversation this browser works for. Read lazily, so it may change. */
+    setLabel(label: () => string | undefined): void;
+    /** Bind the lazily-created browser to its owning conversation. */
+    bind(sessionId: string): void;
+    /** Whether a browser is running, without starting one to find out. */
+    get running(): boolean;
+    /**
+     * The origin the permission engine asks about.
+     *
+     * Undefined before the first navigation, which is what makes the very first
+     * `browser_navigate` the call that gets asked: after that the origin is known
+     * and an approval for it covers the clicking and typing that follow.
+     */
+    origin(): string | undefined;
+    page(): Promise<BrowserPage>;
+    private start;
+    /**
+     * Which browser a call that can open a page will use.
+     *
+     * Pure and synchronous, because the permission check asks it before the
+     * call runs: what is approved has to be what then happens. A fixed policy
+     * always answers itself. In auto, the call's own `browser` wins — the model
+     * reads the person's intent, and says "yours" when the task needs their
+     * accounts or a page they have open. Without one, the conversation stays in
+     * the browser it is already working in, so a task never changes browsers
+     * halfway and loses the page it was on.
+     */
+    sideFor(request?: "yours" | "builtin"): BrowserSide;
+    /** The browser in use now, and which ones this session may use. */
+    current(): {
+        side: BrowserSide;
+        policy: BrowserPolicy;
+    };
+    /**
+     * Move to the browser a call asked for, before it runs. A request a fixed
+     * policy cannot honour is answered with why, rather than ignored.
+     */
+    use(request?: "yours" | "builtin"): void;
+    /** What the model has not been told yet. Each notice is handed out once. */
+    takeNotices(): string[];
+    /** Make `page` the one the tools act on, wiring its log streams once. */
+    adopt(page: BrowserPage): Promise<void>;
+    private record;
+    navigate(url: string): Promise<{
+        url: string;
+        status?: number;
+    }>;
+    private waitForUrlChange;
+    reload(): Promise<{
+        url: string;
+    }>;
+    /**
+     * Wait for the page to stop changing, with a ceiling.
+     *
+     * `Page.loadEventFired` is the signal, but a single-page app may never fire
+     * another one after the first route, so the timeout is the real contract: this
+     * returns when the page looks ready *or* soon, and never blocks a turn on a
+     * page that streams forever.
+     */
+    private settle;
+    url(): Promise<string>;
+    snapshot(opts?: SnapshotOptions): Promise<Snapshot>;
+    /** Where a ref sits on screen, in viewport coordinates. */
+    locate(ref: string): Promise<{
+        x: number;
+        y: number;
+    }>;
+    resolveObject(ref: string): Promise<string>;
+    /** Every open tab. Starts the browser if nothing has yet. */
+    tabs(): Promise<BrowserPage[]>;
+    /**
+     * Every open tab, described without attaching to any. Starts the browser if
+     * nothing has yet. In the person's own browser this is the whole window
+     * list, and a tab is only attached — and bannered — once it is selected.
+     */
+    tabList(): Promise<Array<{
+        id: string;
+        url: string;
+        title?: string;
+        active: boolean;
+        mine: boolean;
+    }>>;
+    /** Open a tab and make it the active one. */
+    newTab(url?: string): Promise<BrowserPage>;
+    /** Where a specific tab is, without switching to it. */
+    tabUrl(page: BrowserPage): Promise<string>;
+    /** A non-launching description of the tabs for the session Browser pane. */
+    paneTabs(): Promise<Array<{
+        id: string;
+        url: string;
+        title?: string;
+        active: boolean;
+        mine?: false;
+    }>>;
+    /** Switch the pane and subsequent browser tools to an already-open tab. */
+    selectPaneTab(id: string): Promise<{
+        id: string;
+        url: string;
+    }>;
+    /**
+     * Give this conversation one of the person's own tabs, because they asked —
+     * from the extension's side panel, on the tab in front of them. Their asking
+     * is what makes it this conversation's tab; it is not a permission to act on
+     * the site, which the first action there still asks for as ever.
+     */
+    adoptYourTab(id: string): Promise<{
+        id: string;
+        url: string;
+    }>;
+    /**
+     * Close one page without disturbing its siblings; the last one releases the
+     * target. A tab the person had open before the conversation was pointed at
+     * it is let go of rather than closed — it was theirs, not the agent's.
+     */
+    closePaneTab(id: string): Promise<{
+        closed: string;
+        released?: boolean;
+        active?: string;
+        running: boolean;
+    }>;
+    /**
+     * What a pane needs to draw itself without starting anything.
+     *
+     * Asked on an interval by every open page, so it must never be the call that
+     * launches a browser: somebody who opened the pane to look at it has not
+     * asked for Chrome, and a pane that started one would make the feature cost
+     * memory for people who never use it.
+     */
+    state(): {
+        running: boolean;
+        origin?: string;
+        presentation?: "stream" | "native";
+        mode?: BrowserSide;
+        policy?: BrowserPolicy;
+    };
+    /**
+     * The page's own coordinate space.
+     *
+     * A screencast frame is not the page: Chrome scales it down, so a 1280-wide
+     * viewport arrives as a 756-wide picture. A pane that treated the picture's
+     * pixels as the page's would put every human click in the wrong place — near
+     * enough to look plausible, far enough to miss the button. This is what the
+     * pane scales by.
+     */
+    viewport(): Promise<{
+        width: number;
+        height: number;
+    } | undefined>;
+    /** Frames for a viewer. Starts nothing — an idle session yields nothing. */
+    frames(opts?: FrameOptions): AsyncIterable<Buffer>;
+    /**
+     * Input from a person who has taken the pane over.
+     *
+     * The same channel the tools use, deliberately: a human click and a model
+     * click arrive at the page identically, so there is no second code path that
+     * can drift, and no state the page can be in that only one of them reaches.
+     */
+    forward(event: BrowserInputEvent): Promise<void>;
+    consoleEntries(): readonly ConsoleEntry[];
+    networkEntries(): readonly NetworkEntry[];
+    /** Let go of both browsers: the built-in one closes, the person's is detached. */
+    close(): Promise<void>;
+    /** Let go of one browser, leaving the other as it is. */
+    private closeSide;
+    /**
+     * Change which browsers this session may use.
+     *
+     * Takes effect on the next browser call. Pinning it to one browser lets go
+     * of the other — the built-in one closes, the person's is only detached,
+     * their tabs never closed. Moving to auto keeps whatever is open, since
+     * either browser is now fair game.
+     */
+    setMode(mode: BrowserPolicy): Promise<void>;
+    /**
+     * The external browser a sign-in can be imported from, and the sites it has
+     * open. Lists tabs only — nothing is attached, so the person's browser shows
+     * no debugging banner just because the dialog was opened.
+     */
+    loginImportSources(): Promise<{
+        browser: string;
+        sites: ExternalSite[];
+        current?: string;
+    }>;
+    /**
+     * Copy the chosen sites' sign-in into the built-in browser's profile.
+     *
+     * Only ever called from the confirmation dialog. In extension mode this is
+     * also the migration: the session lets go of the person's browser (detaching,
+     * never closing their tabs) and continues in the built-in one, on the site it
+     * just imported.
+     */
+    importLogin(sites: readonly string[]): Promise<{
+        browser: string;
+        imported: ImportedSite[];
+        migrated: boolean;
+        url?: string;
+    }>;
+    /**
+     * Offer an import when the built-in browser reaches a site the person has
+     * open in their own browser.
+     *
+     * Asked by the pane's state poll, so it never waits on the extension: the
+     * first call for a site starts one probe and answers nothing, and a later poll
+     * picks the result up. The probe lists tabs only and attaches to none.
+     */
+    loginImportSuggestion(): {
+        site: string;
+        browser: string;
+    } | undefined;
+    /**
+     * Whether the person has this site open in their own browser, while this
+     * session works on it in the built-in one without its sign-in. The model is
+     * told, and decides; nothing moves on the strength of it. Lists tabs only,
+     * attaches to none, and gives up quickly when the extension is not there.
+     */
+    openInYourBrowser(url: string, timeoutMs?: number): Promise<boolean>;
+    /** "Don't ask again" for one site, remembered with the shared profile. */
+    dismissLoginImport(site: string): Promise<void>;
+    private loginMemory;
+    /**
+     * The external browser, for the length of one read.
+     *
+     * In extension mode that is the session's own target. Otherwise a connection
+     * is opened for the read and closed after it, so the extension is free again
+     * and the person's tabs are detached, not closed.
+     */
+    private withExternal;
+    /**
+     * Put localStorage in place before the site's own scripts run.
+     *
+     * Written from a script injected ahead of the document rather than after the
+     * load: a signed-out page that loaded first could overwrite the keys it finds
+     * missing, and would then need a reload that sees the same state it wrote.
+     */
+    private seedLocalStorage;
+}
+export declare function originOf(url: string): string | undefined;