@proagentstore/cli 0.4.122 → 0.4.124
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-runner/local-apply/bridge.js +48 -6
- package/dist/browser-runner/local-apply/contract.js +4 -1
- package/dist/browser-runner/local-apply/runtime.js +15 -5
- package/dist/browser-runner/local-apply/unavailable.js +40 -0
- package/dist/browser-runner/local-browser/bridge.js +23 -5
- package/dist/browser-runner/local-browser/contract.js +3 -0
- package/dist/browser-runner/local-browser/replay.js +74 -0
- package/dist/browser-runner/local-browser/runtime.js +9 -1
- package/dist/browser-runner/server.js +6 -0
- package/package.json +1 -1
|
@@ -25,6 +25,7 @@
|
|
|
25
25
|
* The trace carries classes, domains, decisions and artifact handles — never a typed value.
|
|
26
26
|
*/
|
|
27
27
|
import { classifyApplyClick } from "./contract.js";
|
|
28
|
+
import { LISTING_EXPIRED_RE, LISTING_UNAVAILABLE_RE } from "./unavailable.js";
|
|
28
29
|
import { CONFIRMATION_INTERVAL_SECONDS, CONFIRMATION_LOOKS, CONFIRMATION_TEXT_RE, confirmationMarker, observe, } from "./confirmation.js";
|
|
29
30
|
import { ELEMENT_PROBE_FN } from "../commit-guard.js";
|
|
30
31
|
import { normalize } from "../local-artifact/engine.js";
|
|
@@ -71,9 +72,13 @@ const SAFE_KEYS = /^(Tab|Shift\+Tab|Arrow(Up|Down|Left|Right)|Escape|Home|End|Pa
|
|
|
71
72
|
* The success vocabulary is INTERPOLATED from `confirmation.ts` rather than written out here
|
|
72
73
|
* (#994): a second copy of that regex is how the "sent" family — the exact word SEEK uses — was
|
|
73
74
|
* missing from the detector while being present in nobody's tests, through three live runs.
|
|
75
|
+
* The listing-gone vocabulary is interpolated from `unavailable.ts` for the same reason (#1022): its
|
|
76
|
+
* copy here lacked SEEK's "no longer advertised" through three live runs.
|
|
74
77
|
*/
|
|
75
|
-
const INSPECT_PAGE = `() => {
|
|
78
|
+
export const INSPECT_PAGE = `() => {
|
|
76
79
|
const CONFIRMATION_TEXT_SOURCE = ${JSON.stringify(CONFIRMATION_TEXT_RE.source)};
|
|
80
|
+
const UNAVAILABLE_TEXT_SOURCE = ${JSON.stringify(LISTING_UNAVAILABLE_RE.source)};
|
|
81
|
+
const EXPIRED_TEXT_SOURCE = ${JSON.stringify(LISTING_EXPIRED_RE.source)};
|
|
77
82
|
const text = (document.body ? document.body.innerText : "").slice(0, 20000).toLowerCase();
|
|
78
83
|
const has = (s) => !!document.querySelector(s);
|
|
79
84
|
return {
|
|
@@ -85,9 +90,7 @@ const INSPECT_PAGE = `() => {
|
|
|
85
90
|
duplicate: /you('ve| have) already applied|already applied (for|to) this|application (already exists|has already been (submitted|received))|duplicate application/.test(text),
|
|
86
91
|
antiBot: /unusual traffic|access (is )?denied|request (has been )?blocked|bot detected|checking (if the site connection is secure|your browser)/.test(text),
|
|
87
92
|
confirmed: new RegExp(CONFIRMATION_TEXT_SOURCE).test(text),
|
|
88
|
-
unavailable:
|
|
89
|
-
? (/expired/.test(text) ? "expired" : "unavailable")
|
|
90
|
-
: null,
|
|
93
|
+
unavailable: new RegExp(UNAVAILABLE_TEXT_SOURCE, "i").test(text) ? (new RegExp(EXPIRED_TEXT_SOURCE, "i").test(text) ? "expired" : "unavailable") : null,
|
|
91
94
|
};
|
|
92
95
|
}`;
|
|
93
96
|
const text = (t, isError = false) => ({ content: [{ type: "text", text: t }], ...(isError ? { isError: true } : {}) });
|
|
@@ -120,6 +123,7 @@ function withQuote(schema, required) {
|
|
|
120
123
|
export class ApplyBridge {
|
|
121
124
|
browser;
|
|
122
125
|
host;
|
|
126
|
+
shots;
|
|
123
127
|
/** Read by the runtime for the #975 diagnostic: 0 calls is the finding, not an internal detail. */
|
|
124
128
|
actions = 0;
|
|
125
129
|
pages = 0;
|
|
@@ -139,9 +143,47 @@ export class ApplyBridge {
|
|
|
139
143
|
submitted = null;
|
|
140
144
|
/** The last thing that stopped the work and was not resolved — the block reason if the run ends here. */
|
|
141
145
|
blocked = null;
|
|
142
|
-
constructor(browser, host
|
|
146
|
+
constructor(browser, host,
|
|
147
|
+
/** Replay frames (#1063), taken by the runner with every form control blurred — never a tool the CLI can call. */
|
|
148
|
+
shots) {
|
|
143
149
|
this.browser = browser;
|
|
144
150
|
this.host = host;
|
|
151
|
+
this.shots = shots;
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* The frame a successful call earns (#1063). A refused call earns none, so a model retrying a refused
|
|
155
|
+
* tool cannot fill the run's frame cap.
|
|
156
|
+
*/
|
|
157
|
+
static shotPoint(name, submittedNow) {
|
|
158
|
+
if (submittedNow)
|
|
159
|
+
return "submit";
|
|
160
|
+
const points = {
|
|
161
|
+
browser_navigate: "navigated",
|
|
162
|
+
browser_navigate_back: "navigated",
|
|
163
|
+
browser_click: "clicked",
|
|
164
|
+
browser_snapshot: "snapshot",
|
|
165
|
+
browser_wait_for: "waited",
|
|
166
|
+
browser_type: "filled",
|
|
167
|
+
browser_select_option: "filled",
|
|
168
|
+
upload_artifact: "uploaded",
|
|
169
|
+
supervisor_checkpoint: "checkpoint",
|
|
170
|
+
ready_for_review: "review",
|
|
171
|
+
request_answer: "paused",
|
|
172
|
+
report_job_unavailable: "blocked",
|
|
173
|
+
};
|
|
174
|
+
return points[name] ?? null;
|
|
175
|
+
}
|
|
176
|
+
async callTool(name, args = {}) {
|
|
177
|
+
const submittedBefore = this.submitAttempted;
|
|
178
|
+
const res = await this.dispatch(name, args);
|
|
179
|
+
const point = res.isError ? null : ApplyBridge.shotPoint(name, this.submitAttempted && !submittedBefore);
|
|
180
|
+
if (point && this.shots) {
|
|
181
|
+
const seq = await this.shots.capture(this.browser, point);
|
|
182
|
+
// Metadata only: which frame, and why. The pixels — every form control blurred — stay in the runner.
|
|
183
|
+
if (seq)
|
|
184
|
+
this.host.emit({ type: "screenshot.captured", detail: { shot: seq, point } });
|
|
185
|
+
}
|
|
186
|
+
return res;
|
|
145
187
|
}
|
|
146
188
|
get done() {
|
|
147
189
|
return this.reviewReady || this.submitAttempted || this.unavailable !== null;
|
|
@@ -186,7 +228,7 @@ export class ApplyBridge {
|
|
|
186
228
|
},
|
|
187
229
|
];
|
|
188
230
|
}
|
|
189
|
-
async
|
|
231
|
+
async dispatch(name, args) {
|
|
190
232
|
if (this.done)
|
|
191
233
|
return text("The application is finished for this run. Stop now.", true);
|
|
192
234
|
if (name === "ready_for_review")
|
|
@@ -29,6 +29,8 @@
|
|
|
29
29
|
export const LOCAL_APPLY_TASK_TYPE = "local_browser.apply";
|
|
30
30
|
export const LOCAL_APPLY_RUN_PATH = "/local-apply/run";
|
|
31
31
|
export const LOCAL_APPLY_STATUS_PATH = "/local-apply/status";
|
|
32
|
+
/** `POST {runId, afterSeq}` — a run's replay screenshots (#1063), the shape of `/local-browser/shots`. */
|
|
33
|
+
export const LOCAL_APPLY_SHOTS_PATH = "/local-apply/shots";
|
|
32
34
|
export const LOCAL_APPLY_RESUME_PATH = "/local-apply/resume";
|
|
33
35
|
export const LOCAL_APPLY_CANCEL_PATH = "/local-apply/cancel";
|
|
34
36
|
/** Read-only job-page check used before an email lead may enter material preparation. */
|
|
@@ -158,6 +160,7 @@ export const LOCAL_APPLY_EVENT_TYPES = [
|
|
|
158
160
|
"supervisor.directive",
|
|
159
161
|
"run.paused",
|
|
160
162
|
"run.resumed",
|
|
163
|
+
"screenshot.captured",
|
|
161
164
|
"note",
|
|
162
165
|
];
|
|
163
166
|
export const LOCAL_APPLY_ENGINE_AUTHS = ["machine-login", "subscription", "api-key", "missing_login", "unknown"];
|
|
@@ -176,7 +179,7 @@ const isHttpUrl = (v) => /^https?:\/\/[^\s/]+/i.test(v);
|
|
|
176
179
|
// from `LOCAL_APPLY_CONFIRMATION_MARKERS`, how many times the page was read, and whether it moved.
|
|
177
180
|
// Each is an id, a count or a boolean — the same whitelist rule as everything else here, so no page
|
|
178
181
|
// text and no typed value can ride along on a `submit.confirmed` / `submit.unconfirmed` event.
|
|
179
|
-
const DETAIL_KEYS = new Set(["engine", "authMode", "engineAuth", "mode", "class", "tool", "decision", "reason", "rule", "role", "kind", "path", "sha256", "bytes", "gateId", "exitCode", "count", "status", "basis", "source", "checkpointId", "directive", "phase", "actions", "filled", "uploaded", "marker", "looks", "urlChanged", "titleChanged"]);
|
|
182
|
+
const DETAIL_KEYS = new Set(["engine", "authMode", "engineAuth", "mode", "class", "tool", "decision", "reason", "rule", "role", "kind", "path", "sha256", "bytes", "gateId", "exitCode", "count", "status", "basis", "source", "checkpointId", "directive", "phase", "actions", "filled", "uploaded", "marker", "looks", "urlChanged", "titleChanged", "shot", "point"]);
|
|
180
183
|
/** A validated event, or null. Detail keeps only whitelisted, primitive, bounded values. */
|
|
181
184
|
export function parseLocalApplyEvent(raw) {
|
|
182
185
|
if (!raw || typeof raw !== "object")
|
|
@@ -34,7 +34,9 @@ import { redactText, secretEnvValues } from "../local-browser/contract.js";
|
|
|
34
34
|
import { finalText, missingLogin, signInHelp } from "../local-browser/engine.js";
|
|
35
35
|
import { resolveWorkspacePath } from "../local-browser/runtime.js";
|
|
36
36
|
import { ApplyBridge } from "./bridge.js";
|
|
37
|
+
import { APPLY_SHOT_CAP, ShotRecorder } from "../local-browser/replay.js";
|
|
37
38
|
import { confirmationSignals } from "./confirmation.js";
|
|
39
|
+
import { listingUnavailableReason } from "./unavailable.js";
|
|
38
40
|
import { LOCAL_APPLY_AUTH_MODES, LOCAL_APPLY_CAPS, LOCAL_APPLY_ENGINES, LOCAL_APPLY_MODES, LOCAL_APPLY_TASK_TYPE, parseLocalApplyResult, } from "./contract.js";
|
|
39
41
|
import { applyPrompt, buildApplyEngineSpec, observedApplyAuth } from "./engine.js";
|
|
40
42
|
import { LocalApplyHandoffRuntime } from "./runtime-handoff.js";
|
|
@@ -218,9 +220,9 @@ export class LocalApplyRuntime {
|
|
|
218
220
|
if (snapshot.isError || /captcha|verify (?:you(?:'| a)re|that you are) human|access denied|unusual traffic|checking (?:if )?your browser/i.test(page)) {
|
|
219
221
|
return { state: "unverifiable", jobUrl: parsed.toString(), reason: "access_blocked" };
|
|
220
222
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
223
|
+
const gone = listingUnavailableReason(page);
|
|
224
|
+
if (gone)
|
|
225
|
+
return { state: "unavailable", jobUrl: parsed.toString(), reason: gone };
|
|
224
226
|
// An accessible Apply control is an active path, unlike a generic careers/search URL.
|
|
225
227
|
if (/(?:button|link)\s+"[^"]{0,80}\bapply(?: now| for| here)?\b|\bapply (?:now|for (?:this )?(?:job|role|position))\b|\bstart application\b/i.test(page)) {
|
|
226
228
|
return { state: "live", jobUrl: parsed.toString(), applyUrl: parsed.toString(), evidence: "apply_control_present" };
|
|
@@ -275,7 +277,8 @@ export class LocalApplyRuntime {
|
|
|
275
277
|
continue;
|
|
276
278
|
const checked = parsed.result;
|
|
277
279
|
this.runs.set(envelope.runId, {
|
|
278
|
-
|
|
280
|
+
// Replay frames are memory-only by design (#1063): a run reloaded after a restart has none.
|
|
281
|
+
envelope, dir, token: "", state: "ended", events: raw.events, seq: raw.seq, shots: new ShotRecorder(0),
|
|
279
282
|
result: checked, allow: new Set(envelope.policy.allowDomains), grounding: [], secrets: [], activeSince: raw.endedAt,
|
|
280
283
|
activeMs: 0, waiters: [], output: [], engineAuth: checked.engineAuth, cancelled: false, timedOut: false,
|
|
281
284
|
checkpoints: new Map(), endedAt: raw.endedAt,
|
|
@@ -307,6 +310,7 @@ export class LocalApplyRuntime {
|
|
|
307
310
|
token: randomBytes(24).toString("hex"),
|
|
308
311
|
state: "running",
|
|
309
312
|
events: [],
|
|
313
|
+
shots: new ShotRecorder(APPLY_SHOT_CAP, () => this.now()),
|
|
310
314
|
seq: 0,
|
|
311
315
|
allow: new Set(envelope.policy.allowDomains),
|
|
312
316
|
grounding: [],
|
|
@@ -392,7 +396,7 @@ export class LocalApplyRuntime {
|
|
|
392
396
|
limits: e.limits,
|
|
393
397
|
mode: e.policy.mode,
|
|
394
398
|
gateId: e.policy.submitGate?.gateId,
|
|
395
|
-
});
|
|
399
|
+
}, run.shots);
|
|
396
400
|
if (spec.mcpConfig)
|
|
397
401
|
writeFileSync(spec.mcpConfig.path, spec.mcpConfig.json, { mode: 0o600 });
|
|
398
402
|
const child = this.spawn(spec.command, spec.args, { cwd: join(run.dir, "scratch"), env: spec.env, stdio: ["ignore", "pipe", "pipe"] });
|
|
@@ -637,6 +641,12 @@ export class LocalApplyRuntime {
|
|
|
637
641
|
}
|
|
638
642
|
return { runId, state: run.state };
|
|
639
643
|
}
|
|
644
|
+
/** `POST /local-apply/shots {runId, afterSeq}` — the run's replay frames after `afterSeq` (#1063). */
|
|
645
|
+
shots(raw) {
|
|
646
|
+
const o = (raw && typeof raw === "object" ? raw : {});
|
|
647
|
+
const run = this.get(String(o.runId ?? ""));
|
|
648
|
+
return run.shots.page(run.envelope.runId, typeof o.afterSeq === "number" && o.afterSeq > 0 ? o.afterSeq : 0);
|
|
649
|
+
}
|
|
640
650
|
status(raw) {
|
|
641
651
|
const o = (raw && typeof raw === "object" ? raw : {});
|
|
642
652
|
const run = this.get(String(o.runId ?? ""));
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does the page say the job listing is gone? (#1022)
|
|
3
|
+
*
|
|
4
|
+
* ── The live failure this module exists for
|
|
5
|
+
*
|
|
6
|
+
* Every Runner run for application `cc90cd13…` (runs `26d8f1c1…`, `28c8f18f…`, `9c87cd10…`, 2026-10-09
|
|
7
|
+
* and 2026-10-13) opened SEEK's listing, read "This job is no longer advertised — Jobs remain on SEEK
|
|
8
|
+
* for 30 days, unless the advertiser removes them sooner", and called `report_job_unavailable` — which
|
|
9
|
+
* the runner refused every time ("could not verify an expired or unavailable listing notice"). So each
|
|
10
|
+
* run ended `awaiting_review` with nothing to review, and a retry repeated it. The vocabulary knew
|
|
11
|
+
* "expired", "closed", "been filled", "no longer available" and "no longer accepting applications" —
|
|
12
|
+
* not "no longer advertised", SEEK's own word.
|
|
13
|
+
*
|
|
14
|
+
* It was also written TWICE — in the in-page probe (`bridge.ts` `INSPECT_PAGE`) and in the reconciliation
|
|
15
|
+
* check (`runtime.ts`) — and the probe's copy was never tested on real text, because the bridge's tests
|
|
16
|
+
* fake the probe's RESULT. That is the gap #994 closed for the confirmation wording; this closes it here:
|
|
17
|
+
* one vocabulary, interpolated into the probe, tested against the wording the boards publish.
|
|
18
|
+
*
|
|
19
|
+
* A notice, never an inference: the page must SAY the listing is gone. A missing Apply button is not
|
|
20
|
+
* this — that stays a review state.
|
|
21
|
+
*/
|
|
22
|
+
const NOUN = "(?:job|position|posting|role|opportunity|listing|advert|job ad)";
|
|
23
|
+
const GONE = "(?:has )?(?:expired|closed|been filled|been removed|is no longer (?:available|advertised|listed|accepting applications)|no longer (?:available|advertised|listed))";
|
|
24
|
+
const GONE_FIRST = "(?:expired|closed|no longer available|no longer advertised|no longer listed|no longer accepting applications)";
|
|
25
|
+
/** Specific enough to stand alone: LinkedIn's closed-listing badge names no job noun near it. */
|
|
26
|
+
const ALONE = "no longer accepting applications";
|
|
27
|
+
/**
|
|
28
|
+
* The notice, in either order ("this job is no longer advertised" / "no longer accepting applications
|
|
29
|
+
* for this role"), within 120 characters, or a phrase that needs no noun. Case-insensitive; the in-page
|
|
30
|
+
* probe compiles `.source` with "i".
|
|
31
|
+
*/
|
|
32
|
+
export const LISTING_UNAVAILABLE_RE = new RegExp(`${NOUN}.{0,120}${GONE}|${GONE_FIRST}.{0,120}${NOUN}|${ALONE}`, "i");
|
|
33
|
+
/** Which of the two reasons a notice is: "expired" when the page says so, otherwise "unavailable". */
|
|
34
|
+
export const LISTING_EXPIRED_RE = /expired/i;
|
|
35
|
+
/** The listing's own notice, read from page text — or null when the page says nothing of the kind. */
|
|
36
|
+
export function listingUnavailableReason(text) {
|
|
37
|
+
if (!LISTING_UNAVAILABLE_RE.test(text))
|
|
38
|
+
return null;
|
|
39
|
+
return LISTING_EXPIRED_RE.test(text) ? "expired" : "unavailable";
|
|
40
|
+
}
|
|
@@ -130,6 +130,7 @@ export function refRole(snapshot, ref) {
|
|
|
130
130
|
export class BrowserBridge {
|
|
131
131
|
browser;
|
|
132
132
|
host;
|
|
133
|
+
shots;
|
|
133
134
|
actions = 0;
|
|
134
135
|
pages = 0;
|
|
135
136
|
lastSnapshot = "";
|
|
@@ -141,9 +142,18 @@ export class BrowserBridge {
|
|
|
141
142
|
summary = null;
|
|
142
143
|
/** Tool calls that reached the bridge. Zero at exit means the CLI never used the browser (#944). */
|
|
143
144
|
calls = 0;
|
|
144
|
-
constructor(browser, host
|
|
145
|
+
constructor(browser, host,
|
|
146
|
+
/** Replay frames (#1063), taken by the runner at the points below — never a tool the CLI can call. */
|
|
147
|
+
shots) {
|
|
145
148
|
this.browser = browser;
|
|
146
149
|
this.host = host;
|
|
150
|
+
this.shots = shots;
|
|
151
|
+
}
|
|
152
|
+
/** Take a replay frame, best-effort; the trace only records that it exists (seq + capture point). */
|
|
153
|
+
async shot(point, url) {
|
|
154
|
+
const seq = await this.shots?.capture(this.browser, point, url ? { url } : {});
|
|
155
|
+
if (seq)
|
|
156
|
+
this.host.emit({ type: "screenshot.captured", ...this.currentHost(), detail: { shot: seq, point } });
|
|
147
157
|
}
|
|
148
158
|
/** What the CLI sees: the read-only browser tools that exist, plus the research tools. */
|
|
149
159
|
async listTools() {
|
|
@@ -161,6 +171,7 @@ export class BrowserBridge {
|
|
|
161
171
|
return this.reportFailure(args);
|
|
162
172
|
if (name === "finish_research") {
|
|
163
173
|
this.summary = typeof args.summary === "string" ? args.summary.slice(0, 4000) : "";
|
|
174
|
+
await this.shot("completed");
|
|
164
175
|
return text("Recorded. The research is finished; stop now.");
|
|
165
176
|
}
|
|
166
177
|
if (WRITE_TOOLS.has(name))
|
|
@@ -179,11 +190,17 @@ export class BrowserBridge {
|
|
|
179
190
|
if (name === "browser_snapshot") {
|
|
180
191
|
this.lastSnapshot = textOf(res);
|
|
181
192
|
this.host.emit({ type: "browser.snapshot", ...this.currentHost() });
|
|
193
|
+
if (!res.isError)
|
|
194
|
+
await this.shot("snapshot");
|
|
182
195
|
}
|
|
196
|
+
if (name === "browser_wait_for" && !res.isError)
|
|
197
|
+
await this.shot("waited");
|
|
183
198
|
if (name === "browser_navigate_back") {
|
|
184
199
|
const state = await this.inspect();
|
|
185
|
-
if (state)
|
|
200
|
+
if (state) {
|
|
186
201
|
this.noteNavigation(state);
|
|
202
|
+
await this.shot("navigated", state.url);
|
|
203
|
+
}
|
|
187
204
|
}
|
|
188
205
|
return { content: [{ type: "text", text: textOf(res) }], ...(res.isError ? { isError: true } : {}) };
|
|
189
206
|
}
|
|
@@ -244,7 +261,7 @@ export class BrowserBridge {
|
|
|
244
261
|
await this.browser.callTool("browser_navigate_back", {}).catch(() => undefined);
|
|
245
262
|
return this.refuse("browser_navigate", `${host} redirected to ${landedOn}, a site this run has not been allowed on. Open ${landedOn} with browser_navigate to ask the owner, or skip it.`);
|
|
246
263
|
}
|
|
247
|
-
return this.landed(textOf(res), state);
|
|
264
|
+
return this.landed(textOf(res), state, "navigated");
|
|
248
265
|
}
|
|
249
266
|
async click(args) {
|
|
250
267
|
const ref = typeof args.target === "string" ? args.target : typeof args.ref === "string" ? args.ref : "";
|
|
@@ -265,10 +282,10 @@ export class BrowserBridge {
|
|
|
265
282
|
await this.browser.callTool("browser_navigate_back", {}).catch(() => undefined);
|
|
266
283
|
return this.refuse("browser_click", `That link leads to ${host}, a site this run has not been allowed on. Open it with browser_navigate to ask the owner.`);
|
|
267
284
|
}
|
|
268
|
-
return this.landed(textOf(res), state);
|
|
285
|
+
return this.landed(textOf(res), state, "clicked");
|
|
269
286
|
}
|
|
270
287
|
/** After a page change: count it, trace it, and stop for a person when the page needs one. */
|
|
271
|
-
async landed(toolText, known) {
|
|
288
|
+
async landed(toolText, known, point) {
|
|
272
289
|
const state = known ?? (await this.inspect());
|
|
273
290
|
if (!state)
|
|
274
291
|
return text(toolText);
|
|
@@ -278,6 +295,7 @@ export class BrowserBridge {
|
|
|
278
295
|
// Pauses, never obstacles (#947): a captcha, a sign-in, a bot check, an access-control page or a paywall is
|
|
279
296
|
// for a PERSON in the browser on this machine — the run waits, and re-checks once resumed.
|
|
280
297
|
const blocker = state.captcha ? "captcha" : state.login ? "login_required" : state.accessBlocked ? "access_blocked" : state.paywall ? "paywall" : null;
|
|
298
|
+
await this.shot(blocker || state.writeForm ? "blocked" : point, state.url);
|
|
281
299
|
if (blocker) {
|
|
282
300
|
const what = BLOCKER_WORDS[blocker];
|
|
283
301
|
this.host.emit({ type: "browser.blocked", url: state.url, domain: host, detail: { reason: blocker } });
|
|
@@ -29,6 +29,8 @@ export const LOCAL_BROWSER_RUN_PATH = "/local-browser/run";
|
|
|
29
29
|
export const LOCAL_BROWSER_CANCEL_PATH = "/local-browser/cancel";
|
|
30
30
|
export const LOCAL_BROWSER_STATUS_PATH = "/local-browser/status";
|
|
31
31
|
export const LOCAL_BROWSER_RESUME_PATH = "/local-browser/resume";
|
|
32
|
+
/** `POST {runId, afterSeq}` — a run's replay screenshots (#1063), pulled like its status. */
|
|
33
|
+
export const LOCAL_BROWSER_SHOTS_PATH = "/local-browser/shots";
|
|
32
34
|
export const LOCAL_BROWSER_ENGINES = ["claude", "codex"];
|
|
33
35
|
export const LOCAL_BROWSER_AUTH_MODES = ["subscription", "machine", "api-key"];
|
|
34
36
|
export const LOCAL_BROWSER_MODES = ["research_only"];
|
|
@@ -45,6 +47,7 @@ export const LOCAL_BROWSER_EVENT_TYPES = [
|
|
|
45
47
|
"finding.parsed",
|
|
46
48
|
"run.paused",
|
|
47
49
|
"run.resumed",
|
|
50
|
+
"screenshot.captured",
|
|
48
51
|
"note",
|
|
49
52
|
];
|
|
50
53
|
export const LOCAL_BROWSER_PLATFORM_EVENT_TYPES = ["run.requested", "runner.dispatched", "run.ended", "review.decision"];
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { evaluateResult } from "./bridge.js";
|
|
2
|
+
/** Per-run caps (#1063): enough to replay a run, not enough to make storage unbounded. */
|
|
3
|
+
export const SCOUT_SHOT_CAP = 60;
|
|
4
|
+
export const APPLY_SHOT_CAP = 40;
|
|
5
|
+
/** One frame, as base64 characters (~768 KiB of JPEG) — the API refuses larger, so the runner never sends one. */
|
|
6
|
+
export const MAX_SHOT_BASE64 = Math.ceil((768 * 1024 * 4) / 3);
|
|
7
|
+
/** One pull's reply stays well under the relay's message limit. At least one frame is always returned. */
|
|
8
|
+
export const SHOTS_PAGE_BASE64 = 900 * 1024;
|
|
9
|
+
const MASK_ID = "__pags_replay_mask";
|
|
10
|
+
/** Runs in the page through the browser's own evaluate — never offered to the CLI. */
|
|
11
|
+
export const MASK_FORM_CONTROLS = `() => {
|
|
12
|
+
let s = document.getElementById(${JSON.stringify(MASK_ID)});
|
|
13
|
+
if (!s) {
|
|
14
|
+
s = document.createElement("style");
|
|
15
|
+
s.id = ${JSON.stringify(MASK_ID)};
|
|
16
|
+
s.textContent = "input, textarea, select, option, [contenteditable], [contenteditable] * { filter: blur(8px) !important; }";
|
|
17
|
+
(document.head || document.documentElement).appendChild(s);
|
|
18
|
+
}
|
|
19
|
+
return document.getElementById(${JSON.stringify(MASK_ID)}) ? "masked" : "unmasked";
|
|
20
|
+
}`;
|
|
21
|
+
export const UNMASK_FORM_CONTROLS = `() => { const s = document.getElementById(${JSON.stringify(MASK_ID)}); if (s) s.remove(); return "unmasked"; }`;
|
|
22
|
+
const textOf = (r) => (r.content ?? []).map((c) => c.text ?? "").join("\n");
|
|
23
|
+
export class ShotRecorder {
|
|
24
|
+
cap;
|
|
25
|
+
now;
|
|
26
|
+
shots = [];
|
|
27
|
+
nextSeq = 0;
|
|
28
|
+
constructor(cap, now = Date.now) {
|
|
29
|
+
this.cap = cap;
|
|
30
|
+
this.now = now;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Take one frame, best-effort. Returns its seq, or null when none was kept: the cap is reached, the
|
|
34
|
+
* form controls could not be masked (fail closed), or the browser gave no image.
|
|
35
|
+
*/
|
|
36
|
+
async capture(browser, point, meta = {}) {
|
|
37
|
+
if (this.nextSeq >= this.cap)
|
|
38
|
+
return null;
|
|
39
|
+
try {
|
|
40
|
+
const masked = await browser.callTool("browser_evaluate", { function: MASK_FORM_CONTROLS });
|
|
41
|
+
if (masked.isError || evaluateResult(textOf(masked)) !== "masked")
|
|
42
|
+
return null;
|
|
43
|
+
try {
|
|
44
|
+
const res = await browser.callTool("browser_take_screenshot", { type: "jpeg" });
|
|
45
|
+
const image = res.isError ? undefined : (res.content ?? []).find((c) => c.type === "image" && typeof c.data === "string" && c.data.length > 0);
|
|
46
|
+
if (!image?.data || image.data.length > MAX_SHOT_BASE64)
|
|
47
|
+
return null;
|
|
48
|
+
const seq = ++this.nextSeq;
|
|
49
|
+
this.shots.push({ seq, point, ...(meta.url ? { url: meta.url } : {}), ...(meta.action ? { action: meta.action } : {}), mimeType: image.mimeType ?? "image/jpeg", data: image.data, at: new Date(this.now()).toISOString() });
|
|
50
|
+
return seq;
|
|
51
|
+
}
|
|
52
|
+
finally {
|
|
53
|
+
await browser.callTool("browser_evaluate", { function: UNMASK_FORM_CONTROLS }).catch(() => undefined);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/** The frames after `afterSeq`, paged by size; `lastSeq` never runs past a frame not returned. */
|
|
61
|
+
page(runId, afterSeq) {
|
|
62
|
+
const out = [];
|
|
63
|
+
let size = 0;
|
|
64
|
+
for (const s of this.shots) {
|
|
65
|
+
if (s.seq <= afterSeq)
|
|
66
|
+
continue;
|
|
67
|
+
if (out.length && size + s.data.length > SHOTS_PAGE_BASE64)
|
|
68
|
+
break;
|
|
69
|
+
out.push(s);
|
|
70
|
+
size += s.data.length;
|
|
71
|
+
}
|
|
72
|
+
return { runId, shots: out, lastSeq: out.length ? out[out.length - 1].seq : afterSeq };
|
|
73
|
+
}
|
|
74
|
+
}
|
|
@@ -25,6 +25,7 @@ import { join, resolve, sep } from "node:path";
|
|
|
25
25
|
import { fileURLToPath } from "node:url";
|
|
26
26
|
import { RunnerInputError } from "../errors.js";
|
|
27
27
|
import { BrowserBridge, domainWithin } from "./bridge.js";
|
|
28
|
+
import { SCOUT_SHOT_CAP, ShotRecorder } from "./replay.js";
|
|
28
29
|
import { BRIDGE_ENV } from "./bridge-stdio.js";
|
|
29
30
|
import { LOCAL_BROWSER_AUTH_MODES, LOCAL_BROWSER_ENGINES, LOCAL_BROWSER_TASK_TYPE, redactDetail, redactFinding, redactSourceFailure, redactText, secretEnvValues, } from "./contract.js";
|
|
30
31
|
import { bridgeUnused, buildEngineSpec, finalText, missingLogin, observedEngineAuth, researchPrompt, signInHelp } from "./engine.js";
|
|
@@ -184,6 +185,7 @@ export class LocalBrowserRuntime {
|
|
|
184
185
|
token: randomBytes(24).toString("hex"),
|
|
185
186
|
state: "running",
|
|
186
187
|
events: [],
|
|
188
|
+
shots: new ShotRecorder(SCOUT_SHOT_CAP, () => this.now()),
|
|
187
189
|
seq: 0,
|
|
188
190
|
consented: new Set(envelope.policy.consentedDomains),
|
|
189
191
|
deny: new Set(envelope.policy.denyDomains),
|
|
@@ -228,7 +230,7 @@ export class LocalBrowserRuntime {
|
|
|
228
230
|
allowListOnly: () => e.policy.allowDomains.length > 0,
|
|
229
231
|
overTime: () => this.activeMs(run) >= e.limits.maxMinutes * 60_000,
|
|
230
232
|
limits: e.limits,
|
|
231
|
-
});
|
|
233
|
+
}, run.shots);
|
|
232
234
|
const bridgeScript = this.deps.bridgeScript ?? fileURLToPath(new URL("./bridge-stdio.js", import.meta.url));
|
|
233
235
|
const spec = buildEngineSpec({
|
|
234
236
|
engine: e.engine,
|
|
@@ -360,6 +362,12 @@ export class LocalBrowserRuntime {
|
|
|
360
362
|
...(run.result ? { result: run.result } : {}),
|
|
361
363
|
};
|
|
362
364
|
}
|
|
365
|
+
/** `POST /local-browser/shots {runId, afterSeq}` — the run's replay frames after `afterSeq` (#1063). */
|
|
366
|
+
shots(raw) {
|
|
367
|
+
const o = (raw && typeof raw === "object" ? raw : {});
|
|
368
|
+
const run = this.get(String(o.runId ?? ""));
|
|
369
|
+
return run.shots.page(run.envelope.runId, typeof o.afterSeq === "number" && o.afterSeq > 0 ? o.afterSeq : 0);
|
|
370
|
+
}
|
|
363
371
|
/** Does this token belong to this run? The bridge forwarder's only credential. */
|
|
364
372
|
authorizeBridge(runId, token) {
|
|
365
373
|
const run = this.runs.get(runId);
|
|
@@ -211,6 +211,9 @@ async function route(runner, req, res) {
|
|
|
211
211
|
if (req.method === "POST" && path === "/local-browser/status") {
|
|
212
212
|
return json(res, 200, runner.localBrowser.status(await readJson(req)));
|
|
213
213
|
}
|
|
214
|
+
if (req.method === "POST" && path === "/local-browser/shots") {
|
|
215
|
+
return json(res, 200, runner.localBrowser.shots(await readJson(req)));
|
|
216
|
+
}
|
|
214
217
|
if (req.method === "POST" && path === "/local-browser/resume") {
|
|
215
218
|
return json(res, 200, runner.localBrowser.resume(await readJson(req)));
|
|
216
219
|
}
|
|
@@ -237,6 +240,9 @@ async function route(runner, req, res) {
|
|
|
237
240
|
if (req.method === "POST" && path === "/local-apply/status") {
|
|
238
241
|
return json(res, 200, runner.localApply.status(await readJson(req)));
|
|
239
242
|
}
|
|
243
|
+
if (req.method === "POST" && path === "/local-apply/shots") {
|
|
244
|
+
return json(res, 200, runner.localApply.shots(await readJson(req)));
|
|
245
|
+
}
|
|
240
246
|
const handoff = await routeLocalApplyHandoff(runner.localApply, req.method, path, () => readJson(req));
|
|
241
247
|
if (handoff)
|
|
242
248
|
return json(res, 200, handoff);
|