@proagentstore/cli 0.4.123 → 0.4.124

package/dist/browser-runner/local-apply/bridge.js

@@ -123,6 +123,7 @@ function withQuote(schema, required) {
 export class ApplyBridge {
     browser;
     host;
+    shots;
     /** Read by the runtime for the #975 diagnostic: 0 calls is the finding, not an internal detail. */
     actions = 0;
     pages = 0;
@@ -142,9 +143,47 @@ export class ApplyBridge {
     submitted = null;
     /** The last thing that stopped the work and was not resolved — the block reason if the run ends here. */
     blocked = null;
-    constructor(browser, host) {
+    constructor(browser, host, 
+    /** Replay frames (#1063), taken by the runner with every form control blurred — never a tool the CLI can call. */
+    shots) {
         this.browser = browser;
         this.host = host;
+        this.shots = shots;
+    }
+    /**
+     * The frame a successful call earns (#1063). A refused call earns none, so a model retrying a refused
+     * tool cannot fill the run's frame cap.
+     */
+    static shotPoint(name, submittedNow) {
+        if (submittedNow)
+            return "submit";
+        const points = {
+            browser_navigate: "navigated",
+            browser_navigate_back: "navigated",
+            browser_click: "clicked",
+            browser_snapshot: "snapshot",
+            browser_wait_for: "waited",
+            browser_type: "filled",
+            browser_select_option: "filled",
+            upload_artifact: "uploaded",
+            supervisor_checkpoint: "checkpoint",
+            ready_for_review: "review",
+            request_answer: "paused",
+            report_job_unavailable: "blocked",
+        };
+        return points[name] ?? null;
+    }
+    async callTool(name, args = {}) {
+        const submittedBefore = this.submitAttempted;
+        const res = await this.dispatch(name, args);
+        const point = res.isError ? null : ApplyBridge.shotPoint(name, this.submitAttempted && !submittedBefore);
+        if (point && this.shots) {
+            const seq = await this.shots.capture(this.browser, point);
+            // Metadata only: which frame, and why. The pixels — every form control blurred — stay in the runner.
+            if (seq)
+                this.host.emit({ type: "screenshot.captured", detail: { shot: seq, point } });
+        }
+        return res;
     }
     get done() {
         return this.reviewReady || this.submitAttempted || this.unavailable !== null;
@@ -189,7 +228,7 @@ export class ApplyBridge {
             },
         ];
     }
-    async callTool(name, args = {}) {
+    async dispatch(name, args) {
         if (this.done)
             return text("The application is finished for this run. Stop now.", true);
         if (name === "ready_for_review")

package/dist/browser-runner/local-apply/contract.js

@@ -29,6 +29,8 @@
 export const LOCAL_APPLY_TASK_TYPE = "local_browser.apply";
 export const LOCAL_APPLY_RUN_PATH = "/local-apply/run";
 export const LOCAL_APPLY_STATUS_PATH = "/local-apply/status";
+/** `POST {runId, afterSeq}` — a run's replay screenshots (#1063), the shape of `/local-browser/shots`. */
+export const LOCAL_APPLY_SHOTS_PATH = "/local-apply/shots";
 export const LOCAL_APPLY_RESUME_PATH = "/local-apply/resume";
 export const LOCAL_APPLY_CANCEL_PATH = "/local-apply/cancel";
 /** Read-only job-page check used before an email lead may enter material preparation. */
@@ -158,6 +160,7 @@ export const LOCAL_APPLY_EVENT_TYPES = [
     "supervisor.directive",
     "run.paused",
     "run.resumed",
+    "screenshot.captured",
     "note",
 ];
 export const LOCAL_APPLY_ENGINE_AUTHS = ["machine-login", "subscription", "api-key", "missing_login", "unknown"];
@@ -176,7 +179,7 @@ const isHttpUrl = (v) => /^https?:\/\/[^\s/]+/i.test(v);
 // from `LOCAL_APPLY_CONFIRMATION_MARKERS`, how many times the page was read, and whether it moved.
 // Each is an id, a count or a boolean — the same whitelist rule as everything else here, so no page
 // text and no typed value can ride along on a `submit.confirmed` / `submit.unconfirmed` event.
-const DETAIL_KEYS = new Set(["engine", "authMode", "engineAuth", "mode", "class", "tool", "decision", "reason", "rule", "role", "kind", "path", "sha256", "bytes", "gateId", "exitCode", "count", "status", "basis", "source", "checkpointId", "directive", "phase", "actions", "filled", "uploaded", "marker", "looks", "urlChanged", "titleChanged"]);
+const DETAIL_KEYS = new Set(["engine", "authMode", "engineAuth", "mode", "class", "tool", "decision", "reason", "rule", "role", "kind", "path", "sha256", "bytes", "gateId", "exitCode", "count", "status", "basis", "source", "checkpointId", "directive", "phase", "actions", "filled", "uploaded", "marker", "looks", "urlChanged", "titleChanged", "shot", "point"]);
 /** A validated event, or null. Detail keeps only whitelisted, primitive, bounded values. */
 export function parseLocalApplyEvent(raw) {
     if (!raw || typeof raw !== "object")

package/dist/browser-runner/local-apply/runtime.js

@@ -34,6 +34,7 @@ import { redactText, secretEnvValues } from "../local-browser/contract.js";
 import { finalText, missingLogin, signInHelp } from "../local-browser/engine.js";
 import { resolveWorkspacePath } from "../local-browser/runtime.js";
 import { ApplyBridge } from "./bridge.js";
+import { APPLY_SHOT_CAP, ShotRecorder } from "../local-browser/replay.js";
 import { confirmationSignals } from "./confirmation.js";
 import { listingUnavailableReason } from "./unavailable.js";
 import { LOCAL_APPLY_AUTH_MODES, LOCAL_APPLY_CAPS, LOCAL_APPLY_ENGINES, LOCAL_APPLY_MODES, LOCAL_APPLY_TASK_TYPE, parseLocalApplyResult, } from "./contract.js";
@@ -276,7 +277,8 @@ export class LocalApplyRuntime {
                         continue;
                     const checked = parsed.result;
                     this.runs.set(envelope.runId, {
-                        envelope, dir, token: "", state: "ended", events: raw.events, seq: raw.seq,
+                        // Replay frames are memory-only by design (#1063): a run reloaded after a restart has none.
+                        envelope, dir, token: "", state: "ended", events: raw.events, seq: raw.seq, shots: new ShotRecorder(0),
                         result: checked, allow: new Set(envelope.policy.allowDomains), grounding: [], secrets: [], activeSince: raw.endedAt,
                         activeMs: 0, waiters: [], output: [], engineAuth: checked.engineAuth, cancelled: false, timedOut: false,
                         checkpoints: new Map(), endedAt: raw.endedAt,
@@ -308,6 +310,7 @@ export class LocalApplyRuntime {
             token: randomBytes(24).toString("hex"),
             state: "running",
             events: [],
+            shots: new ShotRecorder(APPLY_SHOT_CAP, () => this.now()),
             seq: 0,
             allow: new Set(envelope.policy.allowDomains),
             grounding: [],
@@ -393,7 +396,7 @@ export class LocalApplyRuntime {
             limits: e.limits,
             mode: e.policy.mode,
             gateId: e.policy.submitGate?.gateId,
-        });
+        }, run.shots);
         if (spec.mcpConfig)
             writeFileSync(spec.mcpConfig.path, spec.mcpConfig.json, { mode: 0o600 });
         const child = this.spawn(spec.command, spec.args, { cwd: join(run.dir, "scratch"), env: spec.env, stdio: ["ignore", "pipe", "pipe"] });
@@ -638,6 +641,12 @@ export class LocalApplyRuntime {
         }
         return { runId, state: run.state };
     }
+    /** `POST /local-apply/shots {runId, afterSeq}` — the run's replay frames after `afterSeq` (#1063). */
+    shots(raw) {
+        const o = (raw && typeof raw === "object" ? raw : {});
+        const run = this.get(String(o.runId ?? ""));
+        return run.shots.page(run.envelope.runId, typeof o.afterSeq === "number" && o.afterSeq > 0 ? o.afterSeq : 0);
+    }
     status(raw) {
         const o = (raw && typeof raw === "object" ? raw : {});
         const run = this.get(String(o.runId ?? ""));

package/dist/browser-runner/local-browser/bridge.js

@@ -130,6 +130,7 @@ export function refRole(snapshot, ref) {
 export class BrowserBridge {
     browser;
     host;
+    shots;
     actions = 0;
     pages = 0;
     lastSnapshot = "";
@@ -141,9 +142,18 @@ export class BrowserBridge {
     summary = null;
     /** Tool calls that reached the bridge. Zero at exit means the CLI never used the browser (#944). */
     calls = 0;
-    constructor(browser, host) {
+    constructor(browser, host, 
+    /** Replay frames (#1063), taken by the runner at the points below — never a tool the CLI can call. */
+    shots) {
         this.browser = browser;
         this.host = host;
+        this.shots = shots;
+    }
+    /** Take a replay frame, best-effort; the trace only records that it exists (seq + capture point). */
+    async shot(point, url) {
+        const seq = await this.shots?.capture(this.browser, point, url ? { url } : {});
+        if (seq)
+            this.host.emit({ type: "screenshot.captured", ...this.currentHost(), detail: { shot: seq, point } });
     }
     /** What the CLI sees: the read-only browser tools that exist, plus the research tools. */
     async listTools() {
@@ -161,6 +171,7 @@ export class BrowserBridge {
             return this.reportFailure(args);
         if (name === "finish_research") {
             this.summary = typeof args.summary === "string" ? args.summary.slice(0, 4000) : "";
+            await this.shot("completed");
             return text("Recorded. The research is finished; stop now.");
         }
         if (WRITE_TOOLS.has(name))
@@ -179,11 +190,17 @@ export class BrowserBridge {
         if (name === "browser_snapshot") {
             this.lastSnapshot = textOf(res);
             this.host.emit({ type: "browser.snapshot", ...this.currentHost() });
+            if (!res.isError)
+                await this.shot("snapshot");
         }
+        if (name === "browser_wait_for" && !res.isError)
+            await this.shot("waited");
         if (name === "browser_navigate_back") {
             const state = await this.inspect();
-            if (state)
+            if (state) {
                 this.noteNavigation(state);
+                await this.shot("navigated", state.url);
+            }
         }
         return { content: [{ type: "text", text: textOf(res) }], ...(res.isError ? { isError: true } : {}) };
     }
@@ -244,7 +261,7 @@ export class BrowserBridge {
             await this.browser.callTool("browser_navigate_back", {}).catch(() => undefined);
             return this.refuse("browser_navigate", `${host} redirected to ${landedOn}, a site this run has not been allowed on. Open ${landedOn} with browser_navigate to ask the owner, or skip it.`);
         }
-        return this.landed(textOf(res), state);
+        return this.landed(textOf(res), state, "navigated");
     }
     async click(args) {
         const ref = typeof args.target === "string" ? args.target : typeof args.ref === "string" ? args.ref : "";
@@ -265,10 +282,10 @@ export class BrowserBridge {
             await this.browser.callTool("browser_navigate_back", {}).catch(() => undefined);
             return this.refuse("browser_click", `That link leads to ${host}, a site this run has not been allowed on. Open it with browser_navigate to ask the owner.`);
         }
-        return this.landed(textOf(res), state);
+        return this.landed(textOf(res), state, "clicked");
     }
     /** After a page change: count it, trace it, and stop for a person when the page needs one. */
-    async landed(toolText, known) {
+    async landed(toolText, known, point) {
         const state = known ?? (await this.inspect());
         if (!state)
             return text(toolText);
@@ -278,6 +295,7 @@ export class BrowserBridge {
         // Pauses, never obstacles (#947): a captcha, a sign-in, a bot check, an access-control page or a paywall is
         // for a PERSON in the browser on this machine — the run waits, and re-checks once resumed.
         const blocker = state.captcha ? "captcha" : state.login ? "login_required" : state.accessBlocked ? "access_blocked" : state.paywall ? "paywall" : null;
+        await this.shot(blocker || state.writeForm ? "blocked" : point, state.url);
         if (blocker) {
             const what = BLOCKER_WORDS[blocker];
             this.host.emit({ type: "browser.blocked", url: state.url, domain: host, detail: { reason: blocker } });

package/dist/browser-runner/local-browser/contract.js

@@ -29,6 +29,8 @@ export const LOCAL_BROWSER_RUN_PATH = "/local-browser/run";
 export const LOCAL_BROWSER_CANCEL_PATH = "/local-browser/cancel";
 export const LOCAL_BROWSER_STATUS_PATH = "/local-browser/status";
 export const LOCAL_BROWSER_RESUME_PATH = "/local-browser/resume";
+/** `POST {runId, afterSeq}` — a run's replay screenshots (#1063), pulled like its status. */
+export const LOCAL_BROWSER_SHOTS_PATH = "/local-browser/shots";
 export const LOCAL_BROWSER_ENGINES = ["claude", "codex"];
 export const LOCAL_BROWSER_AUTH_MODES = ["subscription", "machine", "api-key"];
 export const LOCAL_BROWSER_MODES = ["research_only"];
@@ -45,6 +47,7 @@ export const LOCAL_BROWSER_EVENT_TYPES = [
     "finding.parsed",
     "run.paused",
     "run.resumed",
+    "screenshot.captured",
     "note",
 ];
 export const LOCAL_BROWSER_PLATFORM_EVENT_TYPES = ["run.requested", "runner.dispatched", "run.ended", "review.decision"];

package/dist/browser-runner/local-browser/replay.js

@@ -0,0 +1,74 @@
+import { evaluateResult } from "./bridge.js";
+/** Per-run caps (#1063): enough to replay a run, not enough to make storage unbounded. */
+export const SCOUT_SHOT_CAP = 60;
+export const APPLY_SHOT_CAP = 40;
+/** One frame, as base64 characters (~768 KiB of JPEG) — the API refuses larger, so the runner never sends one. */
+export const MAX_SHOT_BASE64 = Math.ceil((768 * 1024 * 4) / 3);
+/** One pull's reply stays well under the relay's message limit. At least one frame is always returned. */
+export const SHOTS_PAGE_BASE64 = 900 * 1024;
+const MASK_ID = "__pags_replay_mask";
+/** Runs in the page through the browser's own evaluate — never offered to the CLI. */
+export const MASK_FORM_CONTROLS = `() => {
+	let s = document.getElementById(${JSON.stringify(MASK_ID)});
+	if (!s) {
+		s = document.createElement("style");
+		s.id = ${JSON.stringify(MASK_ID)};
+		s.textContent = "input, textarea, select, option, [contenteditable], [contenteditable] * { filter: blur(8px) !important; }";
+		(document.head || document.documentElement).appendChild(s);
+	}
+	return document.getElementById(${JSON.stringify(MASK_ID)}) ? "masked" : "unmasked";
+}`;
+export const UNMASK_FORM_CONTROLS = `() => { const s = document.getElementById(${JSON.stringify(MASK_ID)}); if (s) s.remove(); return "unmasked"; }`;
+const textOf = (r) => (r.content ?? []).map((c) => c.text ?? "").join("\n");
+export class ShotRecorder {
+    cap;
+    now;
+    shots = [];
+    nextSeq = 0;
+    constructor(cap, now = Date.now) {
+        this.cap = cap;
+        this.now = now;
+    }
+    /**
+     * Take one frame, best-effort. Returns its seq, or null when none was kept: the cap is reached, the
+     * form controls could not be masked (fail closed), or the browser gave no image.
+     */
+    async capture(browser, point, meta = {}) {
+        if (this.nextSeq >= this.cap)
+            return null;
+        try {
+            const masked = await browser.callTool("browser_evaluate", { function: MASK_FORM_CONTROLS });
+            if (masked.isError || evaluateResult(textOf(masked)) !== "masked")
+                return null;
+            try {
+                const res = await browser.callTool("browser_take_screenshot", { type: "jpeg" });
+                const image = res.isError ? undefined : (res.content ?? []).find((c) => c.type === "image" && typeof c.data === "string" && c.data.length > 0);
+                if (!image?.data || image.data.length > MAX_SHOT_BASE64)
+                    return null;
+                const seq = ++this.nextSeq;
+                this.shots.push({ seq, point, ...(meta.url ? { url: meta.url } : {}), ...(meta.action ? { action: meta.action } : {}), mimeType: image.mimeType ?? "image/jpeg", data: image.data, at: new Date(this.now()).toISOString() });
+                return seq;
+            }
+            finally {
+                await browser.callTool("browser_evaluate", { function: UNMASK_FORM_CONTROLS }).catch(() => undefined);
+            }
+        }
+        catch {
+            return null;
+        }
+    }
+    /** The frames after `afterSeq`, paged by size; `lastSeq` never runs past a frame not returned. */
+    page(runId, afterSeq) {
+        const out = [];
+        let size = 0;
+        for (const s of this.shots) {
+            if (s.seq <= afterSeq)
+                continue;
+            if (out.length && size + s.data.length > SHOTS_PAGE_BASE64)
+                break;
+            out.push(s);
+            size += s.data.length;
+        }
+        return { runId, shots: out, lastSeq: out.length ? out[out.length - 1].seq : afterSeq };
+    }
+}

package/dist/browser-runner/local-browser/runtime.js

@@ -25,6 +25,7 @@ import { join, resolve, sep } from "node:path";
 import { fileURLToPath } from "node:url";
 import { RunnerInputError } from "../errors.js";
 import { BrowserBridge, domainWithin } from "./bridge.js";
+import { SCOUT_SHOT_CAP, ShotRecorder } from "./replay.js";
 import { BRIDGE_ENV } from "./bridge-stdio.js";
 import { LOCAL_BROWSER_AUTH_MODES, LOCAL_BROWSER_ENGINES, LOCAL_BROWSER_TASK_TYPE, redactDetail, redactFinding, redactSourceFailure, redactText, secretEnvValues, } from "./contract.js";
 import { bridgeUnused, buildEngineSpec, finalText, missingLogin, observedEngineAuth, researchPrompt, signInHelp } from "./engine.js";
@@ -184,6 +185,7 @@ export class LocalBrowserRuntime {
             token: randomBytes(24).toString("hex"),
             state: "running",
             events: [],
+            shots: new ShotRecorder(SCOUT_SHOT_CAP, () => this.now()),
             seq: 0,
             consented: new Set(envelope.policy.consentedDomains),
             deny: new Set(envelope.policy.denyDomains),
@@ -228,7 +230,7 @@ export class LocalBrowserRuntime {
             allowListOnly: () => e.policy.allowDomains.length > 0,
             overTime: () => this.activeMs(run) >= e.limits.maxMinutes * 60_000,
             limits: e.limits,
-        });
+        }, run.shots);
         const bridgeScript = this.deps.bridgeScript ?? fileURLToPath(new URL("./bridge-stdio.js", import.meta.url));
         const spec = buildEngineSpec({
             engine: e.engine,
@@ -360,6 +362,12 @@ export class LocalBrowserRuntime {
             ...(run.result ? { result: run.result } : {}),
         };
     }
+    /** `POST /local-browser/shots {runId, afterSeq}` — the run's replay frames after `afterSeq` (#1063). */
+    shots(raw) {
+        const o = (raw && typeof raw === "object" ? raw : {});
+        const run = this.get(String(o.runId ?? ""));
+        return run.shots.page(run.envelope.runId, typeof o.afterSeq === "number" && o.afterSeq > 0 ? o.afterSeq : 0);
+    }
     /** Does this token belong to this run? The bridge forwarder's only credential. */
     authorizeBridge(runId, token) {
         const run = this.runs.get(runId);

package/dist/browser-runner/server.js

@@ -211,6 +211,9 @@ async function route(runner, req, res) {
     if (req.method === "POST" && path === "/local-browser/status") {
         return json(res, 200, runner.localBrowser.status(await readJson(req)));
     }
+    if (req.method === "POST" && path === "/local-browser/shots") {
+        return json(res, 200, runner.localBrowser.shots(await readJson(req)));
+    }
     if (req.method === "POST" && path === "/local-browser/resume") {
         return json(res, 200, runner.localBrowser.resume(await readJson(req)));
     }
@@ -237,6 +240,9 @@ async function route(runner, req, res) {
     if (req.method === "POST" && path === "/local-apply/status") {
         return json(res, 200, runner.localApply.status(await readJson(req)));
     }
+    if (req.method === "POST" && path === "/local-apply/shots") {
+        return json(res, 200, runner.localApply.shots(await readJson(req)));
+    }
     const handoff = await routeLocalApplyHandoff(runner.localApply, req.method, path, () => readJson(req));
     if (handoff)
         return json(res, 200, handoff);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@proagentstore/cli",
-  "version": "0.4.123",
+  "version": "0.4.124",
   "description": "CLI for creating, publishing, and running ProAgentStore agents",
   "license": "MIT",
   "type": "module",