@proagentstore/cli 0.4.34 → 0.4.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-runner/coding/engine-acts.js +199 -0
- package/dist/browser-runner/coding/headless.js +102 -1
- package/dist/browser-runner/coding/runtime.js +7 -2
- package/dist/browser-runner/index.js +43 -1
- package/dist/browser-runner/mcp-runtime.js +127 -5
- package/dist/browser-runner/reaper.js +281 -0
- package/dist/browser-runner/runner.js +16 -2
- package/dist/browser-runner/test-job-server.js +11 -0
- package/package.json +1 -1
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What the Engine actually DID — the consequential-act record (#294).
|
|
3
|
+
*
|
|
4
|
+
* A delegated run recorded its objective and its terminal outcome and nothing in between. Live,
|
|
5
|
+
* run `73ffc073` merged its own PRs to `main` unattended and the supervisor view said "done". For
|
|
6
|
+
* a read-only goal that is fine; for a goal that merges to the trunk it is trust, not review.
|
|
7
|
+
*
|
|
8
|
+
* ── WHERE THE SIGNAL COMES FROM, AND WHY IT IS NOT THE OTHER ONE ──
|
|
9
|
+
*
|
|
10
|
+
* The obvious alternative was a repo-state delta: snapshot branch/HEAD/refs before and after a run
|
|
11
|
+
* and diff them. #276 already reads `git status --short --branch` through the runner, so it is
|
|
12
|
+
* nearly free. It was rejected on a fact, not a preference: **a pull request is not a git object.**
|
|
13
|
+
* `gh pr create` changes nothing in the local repository at all, and `gh pr merge` changes a ref on
|
|
14
|
+
* GitHub that the local checkout does not learn about until somebody fetches. So the delta design
|
|
15
|
+
* cannot satisfy the one thing the issue asks for by name — "a run that opens or merges a PR leaves
|
|
16
|
+
* a record naming that act" — and would report a clean tree on `main` for the exact run that
|
|
17
|
+
* prompted the issue.
|
|
18
|
+
*
|
|
19
|
+
* What it CAN see (branch moved, tree dirty) is already reported by #276, so building it here would
|
|
20
|
+
* have duplicated the covered half and missed the uncovered one.
|
|
21
|
+
*
|
|
22
|
+
* The worry the issue actually raises about this side — "parsing terminal output is unreliable, the
|
|
23
|
+
* model's prose is not an audit trail" — does not apply to what is parsed here. This does not read
|
|
24
|
+
* the Engine's prose. Claude Code's stream-json protocol emits a `tool_use` block per tool
|
|
25
|
+
* invocation carrying the literal command string it is about to run, and a `tool_result` carrying
|
|
26
|
+
* whether it failed. `{"command":"gh pr merge 42 --squash"}` is a protocol event, the same class of
|
|
27
|
+
* fact as the `result` event #267 takes its cost figures from. It is what the Engine RAN, not what
|
|
28
|
+
* it later said about what it ran.
|
|
29
|
+
*
|
|
30
|
+
* ── THE HONEST GAP ──
|
|
31
|
+
*
|
|
32
|
+
* Only a stream-json engine (Claude Code) produces these. A raw Codex/Grok session emits nothing
|
|
33
|
+
* parseable, so it records no acts — the same gap, for the same reason, as engine usage (#267). An
|
|
34
|
+
* empty act list therefore means "nothing observed", never "nothing happened", and every consumer
|
|
35
|
+
* has to say so rather than render it as an all-clear.
|
|
36
|
+
*/
|
|
37
|
+
/** Acts whose consequences reach outside the machine, or cannot be walked back locally. */
|
|
38
|
+
const IRREVERSIBLE = new Set([
|
|
39
|
+
"pr.merge",
|
|
40
|
+
"push.trunk",
|
|
41
|
+
"push.force",
|
|
42
|
+
"branch.delete",
|
|
43
|
+
"reset.hard",
|
|
44
|
+
"clean",
|
|
45
|
+
"file.delete",
|
|
46
|
+
"release.publish",
|
|
47
|
+
"package.publish",
|
|
48
|
+
"repo.delete",
|
|
49
|
+
"deploy",
|
|
50
|
+
]);
|
|
51
|
+
/** Hard cap on the recorded command text. Long enough to read, short enough not to be a log dump. */
|
|
52
|
+
const MAX_COMMAND = 400;
|
|
53
|
+
/**
|
|
54
|
+
* Strip anything that looks like a credential before the command is stored or shown.
|
|
55
|
+
*
|
|
56
|
+
* A command line is the single most common place a token is pasted in the open
|
|
57
|
+
* (`GH_TOKEN=… git push`, `https://x-access-token:…@github.com/…`), and this record travels from
|
|
58
|
+
* the user's machine into D1 and then into a supervisor's model prompt. Redacting at the point of
|
|
59
|
+
* capture is the only place that covers all three.
|
|
60
|
+
*/
|
|
61
|
+
export function redactCommand(command) {
|
|
62
|
+
let s = String(command ?? "");
|
|
63
|
+
// Credentials embedded in a clone/push URL.
|
|
64
|
+
s = s.replace(/(https?:\/\/)[^/\s@]+@/g, "$1***@");
|
|
65
|
+
// Known token shapes, whatever they are assigned to.
|
|
66
|
+
s = s.replace(/\b(gh[pousr]_|github_pat_)[A-Za-z0-9_]{10,}/g, "$1***");
|
|
67
|
+
s = s.replace(/\bsk-[A-Za-z0-9_-]{16,}/g, "sk-***");
|
|
68
|
+
s = s.replace(/\bxox[baprs]-[A-Za-z0-9-]{10,}/g, "xox-***");
|
|
69
|
+
// Anything ASSIGNED to a secret-shaped name, whatever the value looks like.
|
|
70
|
+
s = s.replace(/\b([A-Za-z_][A-Za-z0-9_]*(?:TOKEN|SECRET|PASSWORD|PASSWD|APIKEY|API_KEY|KEY|PAT))=(\S+)/gi, "$1=***");
|
|
71
|
+
return s.trim().slice(0, MAX_COMMAND);
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Split a shell command into the pieces that each run something.
|
|
75
|
+
*
|
|
76
|
+
* Deliberately aggressive — `&&`, `||`, `;`, `|` and newlines all split, regardless of quoting.
|
|
77
|
+
* Over-splitting costs nothing (a fragment simply matches no rule); under-splitting loses an act,
|
|
78
|
+
* and the real-world shape of the case this exists for is exactly a compound line:
|
|
79
|
+
* `cd repo && git push -u origin fix && gh pr create --fill && gh pr merge --squash`.
|
|
80
|
+
*/
|
|
81
|
+
export function splitSegments(command) {
|
|
82
|
+
return String(command ?? "")
|
|
83
|
+
.split(/\|\||&&|[;|\n]/)
|
|
84
|
+
.map((s) => s.trim())
|
|
85
|
+
.filter(Boolean);
|
|
86
|
+
}
|
|
87
|
+
/** A token that is exactly the trunk, not a branch merely containing the word (`feature/main-fix`). */
|
|
88
|
+
const TRUNK_TOKEN = /(?:^|\s)(?:HEAD:)?(?:refs\/heads\/)?(?:main|master)(?:\s|$)/;
|
|
89
|
+
function prRef(segment) {
|
|
90
|
+
// `gh pr merge 42 --squash`, `gh pr merge --squash 42`, or a full URL.
|
|
91
|
+
const url = segment.match(/https?:\/\/\S*?\/pull\/(\d+)/);
|
|
92
|
+
if (url)
|
|
93
|
+
return `#${url[1]}`;
|
|
94
|
+
const num = segment.match(/\bgh\s+pr\s+(?:merge|create)\b(?:\s+-{1,2}\S+(?:=\S+)?)*\s+(\d+)\b/);
|
|
95
|
+
return num ? `#${num[1]}` : null;
|
|
96
|
+
}
|
|
97
|
+
function pushTarget(segment) {
|
|
98
|
+
// `git push [flags] <remote> <refspec>` — take the first two non-flag words after `push`.
|
|
99
|
+
const after = segment.split(/\bgit\s+push\b/)[1] ?? "";
|
|
100
|
+
const words = after.split(/\s+/).filter((w) => w && !w.startsWith("-"));
|
|
101
|
+
return words.length ? words.slice(0, 2).join(" ") : null;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Classify ONE shell segment. Returns null for ordinary work, which is nearly everything.
|
|
105
|
+
*
|
|
106
|
+
* The rules are matched in order of consequence, so `git push --force origin main` is recorded as
|
|
107
|
+
* the force-push it is rather than as an ordinary push.
|
|
108
|
+
*/
|
|
109
|
+
export function classifySegment(segment) {
|
|
110
|
+
const s = String(segment ?? "");
|
|
111
|
+
const hit = (kind, target = null) => ({
|
|
112
|
+
kind,
|
|
113
|
+
target,
|
|
114
|
+
irreversible: IRREVERSIBLE.has(kind),
|
|
115
|
+
});
|
|
116
|
+
if (/\bgh\s+pr\s+merge\b/.test(s))
|
|
117
|
+
return hit("pr.merge", prRef(s));
|
|
118
|
+
if (/\bgh\s+pr\s+create\b/.test(s))
|
|
119
|
+
return hit("pr.open", prRef(s));
|
|
120
|
+
if (/\bgh\s+release\s+create\b/.test(s))
|
|
121
|
+
return hit("release.publish");
|
|
122
|
+
if (/\bgh\s+repo\s+delete\b/.test(s))
|
|
123
|
+
return hit("repo.delete");
|
|
124
|
+
if (/\bgit\s+push\b/.test(s)) {
|
|
125
|
+
if (/(?:^|\s)(?:--force|--force-with-lease|-f)(?:=|\s|$)/.test(s))
|
|
126
|
+
return hit("push.force", pushTarget(s));
|
|
127
|
+
// `git push origin --delete x` and the older `git push origin :x` both delete a REMOTE
|
|
128
|
+
// branch — the one push that destroys rather than adds.
|
|
129
|
+
if (/(?:^|\s)--delete(?:\s|$)/.test(s) || /\s:\S+/.test(s))
|
|
130
|
+
return hit("branch.delete", pushTarget(s));
|
|
131
|
+
if (TRUNK_TOKEN.test(s.split(/\bgit\s+push\b/)[1] ?? ""))
|
|
132
|
+
return hit("push.trunk", pushTarget(s));
|
|
133
|
+
return hit("push", pushTarget(s));
|
|
134
|
+
}
|
|
135
|
+
// `-d` as well as `-D`: a supervisor asking "what did it delete" does not care that one of them
|
|
136
|
+
// refused on unmerged commits.
|
|
137
|
+
if (/\bgit\s+branch\s+(?:-\S*[dD]\b|--delete\b)/.test(s))
|
|
138
|
+
return hit("branch.delete");
|
|
139
|
+
if (/\bgit\s+reset\s+(?:\S+\s+)*--hard\b/.test(s))
|
|
140
|
+
return hit("reset.hard");
|
|
141
|
+
if (/\bgit\s+clean\b[^\n]*(?:^|\s)-\S*[fdx]/.test(s))
|
|
142
|
+
return hit("clean");
|
|
143
|
+
// Only a RECURSIVE or FORCED rm. Plain `rm scratch.txt` is housekeeping, and recording it would
|
|
144
|
+
// bury the acts that matter under noise — the record is only useful if it stays dense.
|
|
145
|
+
if (/\brm\s+(?:\S+\s+)*-\S*[rf]/.test(s))
|
|
146
|
+
return hit("file.delete");
|
|
147
|
+
if (/\b(?:npm|pnpm|yarn|bun)\s+publish\b/.test(s))
|
|
148
|
+
return hit("package.publish");
|
|
149
|
+
if (/\bwrangler\s+(?:\S+\s+)*deploy\b/.test(s))
|
|
150
|
+
return hit("deploy");
|
|
151
|
+
if (/\b(?:vercel|netlify)\s+(?:\S+\s+)*deploy\b/.test(s))
|
|
152
|
+
return hit("deploy");
|
|
153
|
+
if (/\bvercel\b[^\n]*--prod\b/.test(s))
|
|
154
|
+
return hit("deploy");
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Classify a whole command into zero or more acts.
|
|
159
|
+
*
|
|
160
|
+
* `id` is the engine's `tool_use` id; a compound command yielding two acts gets `id:0`/`id:1` so
|
|
161
|
+
* each still has its own stable dedup key. Every act carries the FULL original command as evidence,
|
|
162
|
+
* not just the segment that matched — a supervisor reading "force-pushed" needs to see what else
|
|
163
|
+
* was on that line.
|
|
164
|
+
*/
|
|
165
|
+
export function classifyCommand(id, command, at = new Date().toISOString()) {
|
|
166
|
+
const full = redactCommand(command);
|
|
167
|
+
if (!full)
|
|
168
|
+
return [];
|
|
169
|
+
const out = [];
|
|
170
|
+
for (const seg of splitSegments(command)) {
|
|
171
|
+
const c = classifySegment(seg);
|
|
172
|
+
if (!c)
|
|
173
|
+
continue;
|
|
174
|
+
out.push({
|
|
175
|
+
id: `${id}:${out.length}`,
|
|
176
|
+
kind: c.kind,
|
|
177
|
+
command: full,
|
|
178
|
+
target: c.target,
|
|
179
|
+
irreversible: c.irreversible,
|
|
180
|
+
ok: null,
|
|
181
|
+
at,
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
return out;
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Pull the command string out of a `tool_use` block's input.
|
|
188
|
+
*
|
|
189
|
+
* Keyed on the INPUT SHAPE rather than the tool name: Claude Code calls it `Bash`, but a session
|
|
190
|
+
* launched with a custom agent/tool set can present the same shell capability under another name,
|
|
191
|
+
* and matching the name would silently record nothing for it. Anything handing a tool a `command`
|
|
192
|
+
* string is running a command.
|
|
193
|
+
*/
|
|
194
|
+
export function commandFromToolInput(input) {
|
|
195
|
+
if (!input || typeof input !== "object")
|
|
196
|
+
return null;
|
|
197
|
+
const cmd = input.command;
|
|
198
|
+
return typeof cmd === "string" && cmd.trim() ? cmd : null;
|
|
199
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
|
+
import { classifyCommand, commandFromToolInput } from "./engine-acts.js";
|
|
2
3
|
import { parseEngineUsage } from "./engine-usage.js";
|
|
3
4
|
/**
|
|
4
5
|
* Merge the platform's resolved engine env over the machine's, where an EMPTY value means
|
|
@@ -33,6 +34,16 @@ import { resolveEngineAuth } from "./engine-auth.js";
|
|
|
33
34
|
* (refusing new records) would lose the turns that just happened rather than ancient ones.
|
|
34
35
|
*/
|
|
35
36
|
const MAX_PENDING_USAGE = 200;
|
|
37
|
+
/**
|
|
38
|
+
* How many un-drained consequential acts a session holds (#294).
|
|
39
|
+
*
|
|
40
|
+
* Smaller than the usage cap because acts are rare by construction — an ordinary turn produces
|
|
41
|
+
* none — so filling this means something has gone very wrong (a loop force-pushing repeatedly),
|
|
42
|
+
* and in that case the FIRST ones are the ones that explain it. So this cap drops the NEWEST rather
|
|
43
|
+
* than the oldest, the opposite of the usage queue: a spend record is fungible and the recent ones
|
|
44
|
+
* matter most, whereas the merge that started an incident is the one you cannot afford to lose.
|
|
45
|
+
*/
|
|
46
|
+
const MAX_PENDING_ACTS = 100;
|
|
36
47
|
export class HeadlessSession {
|
|
37
48
|
config;
|
|
38
49
|
/**
|
|
@@ -66,6 +77,20 @@ export class HeadlessSession {
|
|
|
66
77
|
stopped = false;
|
|
67
78
|
/** Measured engine spend not yet handed to the cloud (#267). Drained by {@link takeUsage}. */
|
|
68
79
|
pendingUsage = [];
|
|
80
|
+
/**
|
|
81
|
+
* Consequential acts observed but not yet handed to the cloud (#294). Drained by
|
|
82
|
+
* {@link takeActs}.
|
|
83
|
+
*/
|
|
84
|
+
pendingActs = [];
|
|
85
|
+
/**
|
|
86
|
+
* Acts whose `tool_result` has not come back yet, keyed by the engine's `tool_use` id.
|
|
87
|
+
*
|
|
88
|
+
* They are held here rather than published immediately so the record can say whether the
|
|
89
|
+
* command SUCCEEDED. A `gh pr merge` that failed on a branch-protection rule, published as a
|
|
90
|
+
* merge, would put an unattended merge to `main` in the audit trail that never happened — which
|
|
91
|
+
* damages the record exactly as much as missing a real one.
|
|
92
|
+
*/
|
|
93
|
+
awaitingResult = new Map();
|
|
69
94
|
/** Turn counter — only used to build a fallback id when the CLI's event has no `uuid`. */
|
|
70
95
|
usageSeq = 0;
|
|
71
96
|
/**
|
|
@@ -77,6 +102,18 @@ export class HeadlessSession {
|
|
|
77
102
|
* The queue is in memory and dies with the process, so a salt can never cause a double-count.
|
|
78
103
|
*/
|
|
79
104
|
usageRunId = Math.random().toString(36).slice(2, 10);
|
|
105
|
+
/** Fallback counter for an act whose `tool_use` block carries no id of its own. */
|
|
106
|
+
actSeq = 0;
|
|
107
|
+
/**
|
|
108
|
+
* Per-PROCESS salt for act ids, for the same reason as {@link usageRunId}.
|
|
109
|
+
*
|
|
110
|
+
* Claude Code's `tool_use` ids are unique within a conversation, not across restarts. Without a
|
|
111
|
+
* salt, a resumed session could re-emit an id the cloud has already written, and the
|
|
112
|
+
* conflict-ignoring insert would silently drop the SECOND act — a real merge, discarded because
|
|
113
|
+
* an older one happened to share an id. The queue is in memory and dies with the process, so a
|
|
114
|
+
* salt can never cause a double-write.
|
|
115
|
+
*/
|
|
116
|
+
actsRunId = Math.random().toString(36).slice(2, 10);
|
|
80
117
|
/**
|
|
81
118
|
* The engine binary could not be spawned (ENOENT, not executable).
|
|
82
119
|
*
|
|
@@ -456,13 +493,16 @@ export class HeadlessSession {
|
|
|
456
493
|
}
|
|
457
494
|
else if (block.type === "tool_use") {
|
|
458
495
|
this.push(`⚙ ${String(block.name ?? "tool")} ${shortInput(block.input)}`); // ⚙
|
|
496
|
+
this.noteAct(block);
|
|
459
497
|
}
|
|
460
498
|
}
|
|
461
499
|
break;
|
|
462
500
|
case "user": // tool results come back as a synthetic user message
|
|
463
501
|
for (const block of ev.message?.content ?? []) {
|
|
464
|
-
if (block.type === "tool_result")
|
|
502
|
+
if (block.type === "tool_result") {
|
|
465
503
|
this.push(` ↳ ${toolResult(block.content)}`); // ↳
|
|
504
|
+
this.settleAct(block);
|
|
505
|
+
}
|
|
466
506
|
}
|
|
467
507
|
break;
|
|
468
508
|
case "result": {
|
|
@@ -477,6 +517,11 @@ export class HeadlessSession {
|
|
|
477
517
|
if (this.pendingUsage.length > MAX_PENDING_USAGE)
|
|
478
518
|
this.pendingUsage.shift();
|
|
479
519
|
}
|
|
520
|
+
// The turn ended, so no further `tool_result` is coming for anything still waiting.
|
|
521
|
+
// Publish it with an UNKNOWN outcome rather than dropping it: "it ran this and we
|
|
522
|
+
// never saw whether it worked" is a materially different claim from silence, and
|
|
523
|
+
// silence is what a supervisor would read as "it did nothing".
|
|
524
|
+
this.flushAwaitingActs();
|
|
480
525
|
this.run = "idle"; // the turn is OVER — a fact, not a guess
|
|
481
526
|
break;
|
|
482
527
|
}
|
|
@@ -490,6 +535,62 @@ export class HeadlessSession {
|
|
|
490
535
|
push(line) {
|
|
491
536
|
this.transcript.push(line);
|
|
492
537
|
}
|
|
538
|
+
/**
|
|
539
|
+
* A `tool_use` block just went past — record it if the command it carries is consequential
|
|
540
|
+
* (#294). Nearly every call classifies to nothing and costs one regex sweep.
|
|
541
|
+
*/
|
|
542
|
+
noteAct(block) {
|
|
543
|
+
const command = commandFromToolInput(block.input);
|
|
544
|
+
if (!command)
|
|
545
|
+
return;
|
|
546
|
+
const toolUseId = typeof block.id === "string" && block.id ? block.id : `${this.config.id}:${this.actSeq++}`;
|
|
547
|
+
const acts = classifyCommand(`${this.actsRunId}:${toolUseId}`, command);
|
|
548
|
+
if (!acts.length)
|
|
549
|
+
return;
|
|
550
|
+
this.awaitingResult.set(toolUseId, acts);
|
|
551
|
+
}
|
|
552
|
+
/** The matching `tool_result` arrived — stamp the outcome and publish. */
|
|
553
|
+
settleAct(block) {
|
|
554
|
+
const id = typeof block.tool_use_id === "string" ? block.tool_use_id : "";
|
|
555
|
+
const acts = this.awaitingResult.get(id);
|
|
556
|
+
if (!acts)
|
|
557
|
+
return;
|
|
558
|
+
this.awaitingResult.delete(id);
|
|
559
|
+
const ok = block.is_error !== true;
|
|
560
|
+
for (const a of acts)
|
|
561
|
+
this.publishAct({ ...a, ok });
|
|
562
|
+
}
|
|
563
|
+
/** Publish everything still waiting, with an unknown outcome. */
|
|
564
|
+
flushAwaitingActs() {
|
|
565
|
+
for (const acts of this.awaitingResult.values()) {
|
|
566
|
+
for (const a of acts)
|
|
567
|
+
this.publishAct(a);
|
|
568
|
+
}
|
|
569
|
+
this.awaitingResult.clear();
|
|
570
|
+
}
|
|
571
|
+
publishAct(act) {
|
|
572
|
+
// Drops the NEWEST at the cap — see MAX_PENDING_ACTS for why this queue is the opposite
|
|
573
|
+
// way round from the usage one.
|
|
574
|
+
if (this.pendingActs.length >= MAX_PENDING_ACTS)
|
|
575
|
+
return;
|
|
576
|
+
this.pendingActs.push(act);
|
|
577
|
+
}
|
|
578
|
+
/**
|
|
579
|
+
* Hand over the consequential acts seen since the last drain, and forget them (#294).
|
|
580
|
+
*
|
|
581
|
+
* Drained rather than re-reported for the same reason as {@link takeUsage}: a 3s capture poll
|
|
582
|
+
* must not re-send the same merge forever. The cloud's insert is keyed on
|
|
583
|
+
* {@link EngineActRecord.id}, so the genuine race — the console's poll and the Pilot's own
|
|
584
|
+
* capture draining at the same moment — still cannot write the act twice.
|
|
585
|
+
*
|
|
586
|
+
* A raw engine returns an empty array, always. Nothing parses its stdout, so it has nothing to
|
|
587
|
+
* hand over; an empty list here means "not observed", never "nothing happened".
|
|
588
|
+
*/
|
|
589
|
+
takeActs() {
|
|
590
|
+
const out = this.pendingActs;
|
|
591
|
+
this.pendingActs = [];
|
|
592
|
+
return out;
|
|
593
|
+
}
|
|
493
594
|
/**
|
|
494
595
|
* Hand over the measured spend since the last drain, and forget it (#267).
|
|
495
596
|
*
|
|
@@ -110,7 +110,7 @@ export class CodingRuntime {
|
|
|
110
110
|
// exactly the question asked about a session that just stopped.
|
|
111
111
|
authResolved: session.authResolved,
|
|
112
112
|
engineRuntime: session.engineRuntime,
|
|
113
|
-
...(opts.drainUsage ? { usage: session.takeUsage() } : {}),
|
|
113
|
+
...(opts.drainUsage ? { usage: session.takeUsage(), acts: session.takeActs() } : {}),
|
|
114
114
|
};
|
|
115
115
|
}
|
|
116
116
|
/** Perform one action, then return the fresh snapshot (non-blocking, like browser act). */
|
|
@@ -143,12 +143,17 @@ export class CodingRuntime {
|
|
|
143
143
|
end(sessionId) {
|
|
144
144
|
const session = this.sessions.get(sessionId);
|
|
145
145
|
const usage = session ? session.takeUsage() : [];
|
|
146
|
+
// Acts drain here too, and for a sharper version of the same reason (#294): the LAST thing a
|
|
147
|
+
// coding run does is very often the consequential one — push, open the PR, merge it — and it
|
|
148
|
+
// happens after the final capture poll. Discarding the tail would systematically lose exactly
|
|
149
|
+
// the acts this record exists for.
|
|
150
|
+
const acts = session ? session.takeActs() : [];
|
|
146
151
|
if (session) {
|
|
147
152
|
session.stop();
|
|
148
153
|
this.sessions.delete(sessionId);
|
|
149
154
|
}
|
|
150
155
|
this.takeovers.delete(sessionId);
|
|
151
|
-
return { ok: true, usage };
|
|
156
|
+
return { ok: true, usage, acts };
|
|
152
157
|
}
|
|
153
158
|
list() {
|
|
154
159
|
return [...this.sessions.entries()].map(([sessionId, s]) => ({
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { startRunnerServer } from "./server.js";
|
|
5
|
+
import { reapOnStartup } from "./reaper.js";
|
|
5
6
|
import { randomUUID } from "node:crypto";
|
|
6
7
|
// Resilience: a stray error in any runtime must NOT take the whole runner down —
|
|
7
8
|
// that drops the tunnel and forces the user to restart `pags up` (and lose their
|
|
@@ -71,6 +72,12 @@ if (!config.token && !LOOPBACK.has(config.host)) {
|
|
|
71
72
|
`Set --token (or PAGS_RUNNER_TOKEN), or bind to 127.0.0.1.\n`);
|
|
72
73
|
process.exit(1);
|
|
73
74
|
}
|
|
75
|
+
// Recover a machine that has already leaked (#274). A browser whose launcher was
|
|
76
|
+
// SIGKILLed is reparented to init and runs forever — 41 of them took the owner's
|
|
77
|
+
// machine to load 253. Nothing in-process can clean those up, so we do it here,
|
|
78
|
+
// before we start competing for the same CPU. Narrow by construction: see the
|
|
79
|
+
// SAFETY block in reaper.ts for why the user's real Chrome can never match.
|
|
80
|
+
reapOnStartup((line) => process.stderr.write(`${line}\n`));
|
|
74
81
|
const started = await startRunnerServer(config);
|
|
75
82
|
process.stdout.write(`ProAgentStore browser runtime listening at ${started.url}\n`);
|
|
76
83
|
process.stdout.write(`Data dir: ${config.dataDir}\n`);
|
|
@@ -79,12 +86,47 @@ if (config.token)
|
|
|
79
86
|
process.stdout.write("Auth: bearer token required\n");
|
|
80
87
|
if (config.instanceId)
|
|
81
88
|
process.stdout.write(`Instance binding: ${config.instanceId}\n`);
|
|
89
|
+
/**
|
|
90
|
+
* Exit once, and ALWAYS exit (#274).
|
|
91
|
+
*
|
|
92
|
+
* Three holes lived here, and each one ended with a user reaching for `kill -9`,
|
|
93
|
+
* which is precisely the signal that orphans the browser:
|
|
94
|
+
*
|
|
95
|
+
* - `started.close()` rejects if the browser is already gone. `void shutdown()`
|
|
96
|
+
* then produced an unhandled rejection, the handler above logged it, and the
|
|
97
|
+
* process stayed up forever holding the port and the browser.
|
|
98
|
+
* - A hung `browserContext.close()` (a wedged renderer) blocked exit with no
|
|
99
|
+
* upper bound, looking identical to a freeze.
|
|
100
|
+
* - Two signals, or a signal racing the parent-death watchdog, ran the teardown
|
|
101
|
+
* twice concurrently.
|
|
102
|
+
*
|
|
103
|
+
* So: idempotent, failure-tolerant, and time-boxed. `process.exit` also runs
|
|
104
|
+
* Playwright's own synchronous exit handler, which kills any browser it launched
|
|
105
|
+
* in this process — that is what makes a clean exit leave nothing behind.
|
|
106
|
+
*/
|
|
107
|
+
let exiting = false;
|
|
82
108
|
const shutdown = async () => {
|
|
83
|
-
|
|
109
|
+
if (exiting)
|
|
110
|
+
return;
|
|
111
|
+
exiting = true;
|
|
112
|
+
const forced = setTimeout(() => {
|
|
113
|
+
process.stderr.write("[runner] shutdown timed out after 10s; exiting anyway\n");
|
|
114
|
+
process.exit(0);
|
|
115
|
+
}, 10_000);
|
|
116
|
+
forced.unref();
|
|
117
|
+
try {
|
|
118
|
+
await started.close();
|
|
119
|
+
}
|
|
120
|
+
catch (err) {
|
|
121
|
+
process.stderr.write(`[runner] shutdown error (exiting anyway): ${err instanceof Error ? err.message : String(err)}\n`);
|
|
122
|
+
}
|
|
84
123
|
process.exit(0);
|
|
85
124
|
};
|
|
86
125
|
process.on("SIGINT", () => void shutdown());
|
|
87
126
|
process.on("SIGTERM", () => void shutdown());
|
|
127
|
+
// SIGHUP was missing: closing the terminal that ran `pags up` killed the runner
|
|
128
|
+
// without ever running teardown, leaving the browser behind.
|
|
129
|
+
process.on("SIGHUP", () => void shutdown());
|
|
88
130
|
// Self-exit if our parent (the `runner connect` CLI) dies — otherwise we'd orphan
|
|
89
131
|
// and keep holding the port, making the NEXT `pags up` fail with EADDRINUSE / a 401
|
|
90
132
|
// against our stale token. When the parent dies we're reparented (ppid changes to
|
|
@@ -1,6 +1,76 @@
|
|
|
1
|
+
import { existsSync, mkdtempSync, rmSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
1
4
|
import { createConnection } from "@playwright/mcp";
|
|
2
5
|
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
3
6
|
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
|
|
7
|
+
import { sweepStalePlaywrightProfiles } from "./reaper.js";
|
|
8
|
+
/**
|
|
9
|
+
* Throwaway profile dirs created by this module and not yet removed (#274).
|
|
10
|
+
*
|
|
11
|
+
* `stop()` cannot reliably win this race on its own: the MCP server releases the
|
|
12
|
+
* browser, but Chrome's exit is asynchronous and it rewrites `Local State` on the
|
|
13
|
+
* way out — measurably AFTER `stop()` has returned, so an rmSync there leaves a
|
|
14
|
+
* re-created directory behind. The browser is only certainly gone once Playwright's
|
|
15
|
+
* own `exit` handler has run, so the last pass belongs at process exit.
|
|
16
|
+
*
|
|
17
|
+
* This also covers the dir being stranded when `stop()` is never called at all —
|
|
18
|
+
* a throw between `start()` and `stop()`, which no `try/finally` in the caller
|
|
19
|
+
* would catch either.
|
|
20
|
+
*/
|
|
21
|
+
const ownedProfileDirs = new Set();
|
|
22
|
+
let exitHookInstalled = false;
|
|
23
|
+
/**
|
|
24
|
+
* Try to remove a profile dir now. It stays registered unless it is really gone,
|
|
25
|
+
* so a dir Chrome re-creates while shutting down gets another pass at exit rather
|
|
26
|
+
* than being forgotten after one failed attempt.
|
|
27
|
+
*/
|
|
28
|
+
function removeProfileDir(dir) {
|
|
29
|
+
try {
|
|
30
|
+
rmSync(dir, { recursive: true, force: true });
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
// best effort; the dir is disk, not correctness
|
|
34
|
+
}
|
|
35
|
+
if (!existsSync(dir))
|
|
36
|
+
ownedProfileDirs.delete(dir);
|
|
37
|
+
}
|
|
38
|
+
/** Await a promise, giving up (never throwing) after `ms`. */
|
|
39
|
+
async function withTimeout(p, ms) {
|
|
40
|
+
if (!p)
|
|
41
|
+
return;
|
|
42
|
+
let timer;
|
|
43
|
+
try {
|
|
44
|
+
await Promise.race([
|
|
45
|
+
p.catch(() => undefined),
|
|
46
|
+
new Promise((resolve) => {
|
|
47
|
+
timer = setTimeout(resolve, ms);
|
|
48
|
+
}),
|
|
49
|
+
]);
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
if (timer)
|
|
53
|
+
clearTimeout(timer);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function rememberProfileDir(dir) {
|
|
57
|
+
ownedProfileDirs.add(dir);
|
|
58
|
+
if (exitHookInstalled)
|
|
59
|
+
return;
|
|
60
|
+
exitHookInstalled = true;
|
|
61
|
+
// Registered AFTER Playwright's own launch-time exit handler, so by the time
|
|
62
|
+
// this runs Playwright has already killed the browsers it started.
|
|
63
|
+
process.on("exit", () => {
|
|
64
|
+
for (const d of ownedProfileDirs) {
|
|
65
|
+
try {
|
|
66
|
+
rmSync(d, { recursive: true, force: true });
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
// exiting anyway
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
});
|
|
73
|
+
}
|
|
4
74
|
/**
|
|
5
75
|
* Hosts the INDUSTRY-STANDARD `@playwright/mcp` server in the runner and an in-process
|
|
6
76
|
* MCP client to drive it. Every browser action goes through the standard Playwright MCP
|
|
@@ -11,12 +81,51 @@ import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
|
|
|
11
81
|
export class McpRuntime {
|
|
12
82
|
server;
|
|
13
83
|
client;
|
|
84
|
+
/** A profile dir this instance created and is therefore responsible for deleting. */
|
|
85
|
+
ownedProfileDir;
|
|
14
86
|
async start(opts) {
|
|
15
87
|
if (this.client)
|
|
16
88
|
return;
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
:
|
|
89
|
+
let browser;
|
|
90
|
+
if (opts.cdpEndpoint) {
|
|
91
|
+
// Production: attach to the browser the runner already launched. Nothing
|
|
92
|
+
// is created here, so nothing can leak here.
|
|
93
|
+
browser = { cdpEndpoint: opts.cdpEndpoint };
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
// This is the ONE path in this package that produced the `#274` orphans.
|
|
97
|
+
//
|
|
98
|
+
// `isolated: true` makes @playwright/mcp call the non-persistent
|
|
99
|
+
// `chromium.launch()`, and Playwright then mkdtemps
|
|
100
|
+
// `$TMPDIR/playwright_chromiumdev_profile-XXXXXX` and deletes it only on a
|
|
101
|
+
// clean close — so every SIGKILLed parent left both a running browser AND a
|
|
102
|
+
// few hundred MB of profile behind, under a name we did not know and could
|
|
103
|
+
// not clean up afterwards.
|
|
104
|
+
//
|
|
105
|
+
// Asking for a directory WE created gets the same isolation (it is fresh
|
|
106
|
+
// every start) while making the leak addressable: we know the path, we
|
|
107
|
+
// delete it in `stop()`, and it is named so a human can see where it came
|
|
108
|
+
// from. It also keeps the throwaway-profile pattern out of our own output,
|
|
109
|
+
// so the reaper's matches are unambiguously other people's abandonment.
|
|
110
|
+
// Self-heal before adding one more. Chrome writes its final state as it
|
|
111
|
+
// dies — after `stop()`, and after our own `exit` hook — so the last
|
|
112
|
+
// skeleton of a directory can outlive the process that owned it no matter
|
|
113
|
+
// when we try to delete it. Sweeping here means a stale one never survives
|
|
114
|
+
// a second run, without needing the runner to be restarted. Only touches
|
|
115
|
+
// dirs no live process holds; see reaper.ts for the safety argument.
|
|
116
|
+
try {
|
|
117
|
+
sweepStalePlaywrightProfiles();
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
// cleanup must never stop a browser from starting
|
|
121
|
+
}
|
|
122
|
+
const userDataDir = opts.userDataDir ?? mkdtempSync(join(tmpdir(), "pags-mcp-profile-"));
|
|
123
|
+
if (!opts.userDataDir) {
|
|
124
|
+
this.ownedProfileDir = userDataDir;
|
|
125
|
+
rememberProfileDir(userDataDir);
|
|
126
|
+
}
|
|
127
|
+
browser = { userDataDir, isolated: false, launchOptions: { headless: opts.headless ?? false } };
|
|
128
|
+
}
|
|
20
129
|
// The runner is a trusted local process uploading the user's OWN résumé, which
|
|
21
130
|
// lives under the runner's data dir (outside the CWD). Lift the file-root guard
|
|
22
131
|
// (meant to stop an LLM reading arbitrary host files) so browser_file_upload can
|
|
@@ -40,9 +149,22 @@ export class McpRuntime {
|
|
|
40
149
|
return (res.content || []).map((c) => c.text ?? "").join("\n");
|
|
41
150
|
}
|
|
42
151
|
async stop() {
|
|
43
|
-
|
|
44
|
-
|
|
152
|
+
// Bounded. Closing a persistent context flushes the whole profile to disk and
|
|
153
|
+
// on a cold/slow machine that measurably exceeds 10s — and a browser wedged
|
|
154
|
+
// mid-close would otherwise block teardown with no upper limit at all, which
|
|
155
|
+
// is the state that ends in someone sending the SIGKILL that orphans it (#274).
|
|
156
|
+
// Returning early is safe: the `exit` hook and reaper.ts's sweep both still run.
|
|
157
|
+
await withTimeout(this.client?.close(), 20_000);
|
|
158
|
+
await withTimeout(this.server?.close(), 20_000);
|
|
45
159
|
this.client = undefined;
|
|
46
160
|
this.server = undefined;
|
|
161
|
+
// Remove the profile only if we made it — a caller-supplied dir belongs to
|
|
162
|
+
// the caller. A best-effort pass now, ordered after the closes so the browser
|
|
163
|
+
// has released its locks; the `exit` hook above is the pass that always runs,
|
|
164
|
+
// and reaper.ts's startup sweep covers a process that was killed outright.
|
|
165
|
+
if (this.ownedProfileDir) {
|
|
166
|
+
removeProfileDir(this.ownedProfileDir);
|
|
167
|
+
this.ownedProfileDir = undefined;
|
|
168
|
+
}
|
|
47
169
|
}
|
|
48
170
|
}
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { readdirSync, rmSync, statSync } from "node:fs";
|
|
3
|
+
import { tmpdir } from "node:os";
|
|
4
|
+
import { basename, dirname, join } from "node:path";
|
|
5
|
+
const DEFAULT_MIN_AGE_MS = 10 * 60 * 1000;
|
|
6
|
+
/**
|
|
7
|
+
* A throwaway browser-profile directory name, anchored.
|
|
8
|
+
*
|
|
9
|
+
* Two producers, both of which mkdtemp — so the trailing six characters are
|
|
10
|
+
* exactly what `fs.mkdtemp` appends, and pinning that length is what stops a
|
|
11
|
+
* user-chosen directory which merely CONTAINS this text from matching:
|
|
12
|
+
*
|
|
13
|
+
* - `playwright_<browser>dev_profile-` — Playwright's own, created per
|
|
14
|
+
* non-persistent `launch()`. These are the orphans measured in #274.
|
|
15
|
+
* - `pags-mcp-profile-` — ours, from `McpRuntime` (see mcp-runtime.ts). Chrome
|
|
16
|
+
* can rewrite this directory during its async shutdown, after `stop()` has
|
|
17
|
+
* already removed it, so it needs a sweeper too.
|
|
18
|
+
*
|
|
19
|
+
* A name matching neither is never touched, whatever it contains.
|
|
20
|
+
*/
|
|
21
|
+
const TEMP_PROFILE_NAME = /^(?:playwright_[a-z]+dev_profile|pags-mcp-profile)-[A-Za-z0-9]{6}$/;
|
|
22
|
+
/** Every path the system might hand out as the temp root, de-duplicated. */
|
|
23
|
+
export function tempRoots() {
|
|
24
|
+
const roots = new Set();
|
|
25
|
+
for (const root of [tmpdir(), process.env.TMPDIR, "/tmp"]) {
|
|
26
|
+
if (!root)
|
|
27
|
+
continue;
|
|
28
|
+
roots.add(root.replace(/\/+$/, ""));
|
|
29
|
+
}
|
|
30
|
+
return [...roots];
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Is this `--user-data-dir` a Playwright throwaway temp profile?
|
|
34
|
+
*
|
|
35
|
+
* Both halves matter. The basename pattern says "Playwright mkdtemp'd this"; the
|
|
36
|
+
* temp-root check says "and it is under the system temp dir, not somewhere a
|
|
37
|
+
* human keeps a real profile". A directory that satisfies only one is rejected.
|
|
38
|
+
*/
|
|
39
|
+
export function isPlaywrightTempProfile(userDataDir, roots = tempRoots()) {
|
|
40
|
+
if (!userDataDir)
|
|
41
|
+
return false;
|
|
42
|
+
const dir = userDataDir.replace(/\/+$/, "");
|
|
43
|
+
if (!TEMP_PROFILE_NAME.test(basename(dir)))
|
|
44
|
+
return false;
|
|
45
|
+
const parent = dirname(dir).replace(/\/+$/, "");
|
|
46
|
+
return roots.some((root) => parent === root);
|
|
47
|
+
}
|
|
48
|
+
/** Pull `--user-data-dir=<path>` out of a command line. "" when absent. */
|
|
49
|
+
export function userDataDirOf(command) {
|
|
50
|
+
// Deliberately `\S+`: every path this reaper acts on is a mkdtemp name under
|
|
51
|
+
// the temp root, which never contains a space. A path WITH a space therefore
|
|
52
|
+
// fails to parse and is skipped — the safe direction to be wrong in.
|
|
53
|
+
const m = command.match(/--user-data-dir=(\S+)/);
|
|
54
|
+
return m ? m[1] : "";
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Parse `ps -o etime` — `[[DD-]HH:]MM:SS` — into seconds. Returns 0 for anything
|
|
58
|
+
* unparseable, which reads as "brand new" and therefore never reapable.
|
|
59
|
+
*/
|
|
60
|
+
export function parseEtime(etime) {
|
|
61
|
+
const trimmed = etime.trim();
|
|
62
|
+
const m = trimmed.match(/^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)$/);
|
|
63
|
+
if (!m)
|
|
64
|
+
return 0;
|
|
65
|
+
const [, days, hours, minutes, seconds] = m;
|
|
66
|
+
return Number(days ?? 0) * 86400 + Number(hours ?? 0) * 3600 + Number(minutes) * 60 + Number(seconds);
|
|
67
|
+
}
|
|
68
|
+
/** Parse `ps -wwAo pid=,ppid=,etime=,command=` output into candidate rows. */
|
|
69
|
+
export function parsePsOutput(stdout) {
|
|
70
|
+
const out = [];
|
|
71
|
+
for (const line of stdout.split("\n")) {
|
|
72
|
+
const m = line.match(/^\s*(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/);
|
|
73
|
+
if (!m)
|
|
74
|
+
continue;
|
|
75
|
+
const userDataDir = userDataDirOf(m[4]);
|
|
76
|
+
if (!userDataDir)
|
|
77
|
+
continue;
|
|
78
|
+
out.push({ pid: Number(m[1]), ppid: Number(m[2]), ageSeconds: parseEtime(m[3]), userDataDir });
|
|
79
|
+
}
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Age of a profile directory's last write, in ms.
|
|
84
|
+
*
|
|
85
|
+
* A MISSING directory returns Infinity, and that is deliberate rather than a
|
|
86
|
+
* fallback: Playwright removes this directory only when the browser closed
|
|
87
|
+
* cleanly. A process still running on a temp profile that no longer exists is
|
|
88
|
+
* unambiguously abandoned.
|
|
89
|
+
*/
|
|
90
|
+
export function profileIdleMs(dir, now, stat = safeMtimeMs) {
|
|
91
|
+
const mtime = stat(dir);
|
|
92
|
+
return mtime === null ? Number.POSITIVE_INFINITY : now - mtime;
|
|
93
|
+
}
|
|
94
|
+
function safeMtimeMs(path) {
|
|
95
|
+
try {
|
|
96
|
+
return statSync(path).mtimeMs;
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return null;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* The whole safety decision, as one pure function — see the SAFETY block above.
|
|
104
|
+
* All four conditions must hold; any one of them failing spares the process.
|
|
105
|
+
*/
|
|
106
|
+
export function isReapable(proc, opts) {
|
|
107
|
+
if (proc.ppid !== 1)
|
|
108
|
+
return false;
|
|
109
|
+
if (!isPlaywrightTempProfile(proc.userDataDir, opts.roots))
|
|
110
|
+
return false;
|
|
111
|
+
if (proc.ageSeconds * 1000 < opts.minAgeMs)
|
|
112
|
+
return false;
|
|
113
|
+
const idle = (opts.idleMs ?? profileIdleMs)(proc.userDataDir, opts.now);
|
|
114
|
+
return idle >= opts.minAgeMs;
|
|
115
|
+
}
|
|
116
|
+
function ps() {
|
|
117
|
+
try {
|
|
118
|
+
// -ww: never truncate the command line, or the --user-data-dir we match on
|
|
119
|
+
// could be cut off and a real orphan would go unnoticed.
|
|
120
|
+
return execFileSync("ps", ["-wwAo", "pid=,ppid=,etime=,command="], { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 });
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
return "";
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
/** Every Playwright-temp-profile browser process currently alive, orphan or not. */
|
|
127
|
+
export function listPlaywrightBrowsers(psOutput = ps()) {
|
|
128
|
+
return parsePsOutput(psOutput).filter((p) => isPlaywrightTempProfile(p.userDataDir));
|
|
129
|
+
}
|
|
130
|
+
function signal(pid, sig) {
|
|
131
|
+
try {
|
|
132
|
+
process.kill(pid, sig);
|
|
133
|
+
}
|
|
134
|
+
catch {
|
|
135
|
+
// already gone, or not ours to signal — either way there is nothing to do
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
function alive(pid) {
|
|
139
|
+
try {
|
|
140
|
+
process.kill(pid, 0);
|
|
141
|
+
return true;
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
return false;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Kill abandoned Playwright browsers and delete their temp profiles.
|
|
149
|
+
*
|
|
150
|
+
* Synchronous on purpose: this runs once at runner startup, before anything else
|
|
151
|
+
* competes for the machine, and the whole point is that the CPU is already gone.
|
|
152
|
+
*/
|
|
153
|
+
export function reapOrphanedPlaywrightBrowsers(opts = {}) {
|
|
154
|
+
const log = opts.log ?? ((line) => console.warn(line));
|
|
155
|
+
const empty = { reaped: [], skippedYoung: 0 };
|
|
156
|
+
if (process.platform === "win32")
|
|
157
|
+
return empty; // no `ps`; the leak is a POSIX-orphan story
|
|
158
|
+
if (process.pid === 1) {
|
|
159
|
+
// See SAFETY note 2: as PID 1 we cannot tell an orphan from our own live child.
|
|
160
|
+
return empty;
|
|
161
|
+
}
|
|
162
|
+
const now = opts.now ?? Date.now();
|
|
163
|
+
const minAgeMs = opts.minAgeMs ?? DEFAULT_MIN_AGE_MS;
|
|
164
|
+
const all = listPlaywrightBrowsers();
|
|
165
|
+
const orphanParents = all.filter((p) => p.ppid === 1);
|
|
166
|
+
const doomed = orphanParents.filter((p) => isReapable(p, { now, minAgeMs }));
|
|
167
|
+
const skippedYoung = orphanParents.length - doomed.length;
|
|
168
|
+
if (doomed.length === 0)
|
|
169
|
+
return { reaped: [], skippedYoung };
|
|
170
|
+
const dirs = new Set(doomed.map((p) => p.userDataDir));
|
|
171
|
+
const verb = opts.dryRun ? "would reap" : "reaping";
|
|
172
|
+
log(`[runner] ${verb} ${doomed.length} abandoned Playwright browser(s) (parent gone, idle >${Math.round(minAgeMs / 60000)}m):`);
|
|
173
|
+
for (const p of doomed)
|
|
174
|
+
log(`[runner] pid ${p.pid} profile ${p.userDataDir}`);
|
|
175
|
+
if (opts.dryRun)
|
|
176
|
+
return { reaped: [...dirs], skippedYoung };
|
|
177
|
+
// Ask first. The measurement says they ignore it, but a browser that CAN exit
|
|
178
|
+
// cleanly should be given the chance to flush and remove its own profile.
|
|
179
|
+
for (const p of doomed)
|
|
180
|
+
signal(p.pid, "SIGTERM");
|
|
181
|
+
const deadline = Date.now() + 2000;
|
|
182
|
+
while (Date.now() < deadline && doomed.some((p) => alive(p.pid))) {
|
|
183
|
+
// Busy-wait: this is startup, single-purpose, and bounded at 2s. A timer
|
|
184
|
+
// would need the event loop, and callers want a settled machine on return.
|
|
185
|
+
execFileSync("sleep", ["0.1"], { stdio: "ignore" });
|
|
186
|
+
}
|
|
187
|
+
for (const p of doomed)
|
|
188
|
+
if (alive(p.pid))
|
|
189
|
+
signal(p.pid, "SIGKILL");
|
|
190
|
+
// The renderer/GPU/network helpers are children of the parent we just killed and
|
|
191
|
+
// normally follow it down. Sweep any that did not, matched by the SAME profile
|
|
192
|
+
// dirs we already cleared — never by name, never by executable.
|
|
193
|
+
for (const p of listPlaywrightBrowsers()) {
|
|
194
|
+
if (dirs.has(p.userDataDir))
|
|
195
|
+
signal(p.pid, "SIGKILL");
|
|
196
|
+
}
|
|
197
|
+
for (const dir of dirs) {
|
|
198
|
+
try {
|
|
199
|
+
rmSync(dir, { recursive: true, force: true });
|
|
200
|
+
}
|
|
201
|
+
catch {
|
|
202
|
+
// a leftover directory is untidy, not harmful
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return { reaped: [...dirs], skippedYoung };
|
|
206
|
+
}
|
|
207
|
+
/**
|
|
208
|
+
* Delete Playwright temp profile directories with no live process behind them.
|
|
209
|
+
*
|
|
210
|
+
* Separate from the process reap because the two leak independently: a browser
|
|
211
|
+
* that IS killed by `kill -9` leaves its directory behind with nobody to remove
|
|
212
|
+
* it. These are a few hundred MB each once they have been used.
|
|
213
|
+
*/
|
|
214
|
+
export function sweepStalePlaywrightProfiles(opts = {}) {
|
|
215
|
+
if (process.platform === "win32")
|
|
216
|
+
return [];
|
|
217
|
+
const now = opts.now ?? Date.now();
|
|
218
|
+
const minAgeMs = opts.minAgeMs ?? DEFAULT_MIN_AGE_MS;
|
|
219
|
+
const inUse = new Set(listPlaywrightBrowsers().map((p) => p.userDataDir));
|
|
220
|
+
const removed = [];
|
|
221
|
+
for (const root of tempRoots()) {
|
|
222
|
+
let entries;
|
|
223
|
+
try {
|
|
224
|
+
entries = readdirSync(root);
|
|
225
|
+
}
|
|
226
|
+
catch {
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
for (const name of entries) {
|
|
230
|
+
if (!TEMP_PROFILE_NAME.test(name))
|
|
231
|
+
continue;
|
|
232
|
+
const dir = join(root, name);
|
|
233
|
+
if (inUse.has(dir))
|
|
234
|
+
continue;
|
|
235
|
+
if (profileIdleMs(dir, now) < minAgeMs)
|
|
236
|
+
continue;
|
|
237
|
+
try {
|
|
238
|
+
rmSync(dir, { recursive: true, force: true });
|
|
239
|
+
removed.push(dir);
|
|
240
|
+
}
|
|
241
|
+
catch {
|
|
242
|
+
// best effort
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
if (removed.length > 0) {
|
|
247
|
+
(opts.log ?? ((l) => console.warn(l)))(`[runner] removed ${removed.length} stale Playwright temp profile dir(s)`);
|
|
248
|
+
}
|
|
249
|
+
return removed;
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* The startup entry point: recover a machine that has already leaked.
|
|
253
|
+
*
|
|
254
|
+
* On by default. Default-off would mean the users who most need it — the ones
|
|
255
|
+
* whose machine is already at load 253 — never get it, and the discriminator is
|
|
256
|
+
* narrow enough (see SAFETY) that a false positive would require a real browser
|
|
257
|
+
* to be running out of a Playwright mkdtemp directory, orphaned, and idle.
|
|
258
|
+
* PAGS_RUNNER_REAP=0 disable entirely
|
|
259
|
+
* PAGS_RUNNER_REAP_DRY_RUN=1 report what it would kill, kill nothing
|
|
260
|
+
* PAGS_RUNNER_REAP_MIN_AGE_MIN idle threshold in minutes (default 10)
|
|
261
|
+
*/
|
|
262
|
+
export function reapOnStartup(log = (l) => console.warn(l)) {
|
|
263
|
+
if (process.env.PAGS_RUNNER_REAP === "0")
|
|
264
|
+
return;
|
|
265
|
+
const minutes = Number(process.env.PAGS_RUNNER_REAP_MIN_AGE_MIN);
|
|
266
|
+
const minAgeMs = Number.isFinite(minutes) && minutes > 0 ? minutes * 60_000 : DEFAULT_MIN_AGE_MS;
|
|
267
|
+
const dryRun = process.env.PAGS_RUNNER_REAP_DRY_RUN === "1";
|
|
268
|
+
try {
|
|
269
|
+
const { reaped, skippedYoung } = reapOrphanedPlaywrightBrowsers({ minAgeMs, dryRun, log });
|
|
270
|
+
if (skippedYoung > 0)
|
|
271
|
+
log(`[runner] left ${skippedYoung} orphaned browser(s) alone — not idle long enough yet`);
|
|
272
|
+
if (!dryRun && reaped.length > 0)
|
|
273
|
+
log(`[runner] reaped ${reaped.length} orphaned browser(s); their CPU is yours again`);
|
|
274
|
+
if (!dryRun)
|
|
275
|
+
sweepStalePlaywrightProfiles({ minAgeMs, log });
|
|
276
|
+
}
|
|
277
|
+
catch (err) {
|
|
278
|
+
// Never let cleanup stop the runner from starting.
|
|
279
|
+
log(`[runner] browser reap skipped: ${err instanceof Error ? err.message : String(err)}`);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
@@ -180,12 +180,26 @@ export class LocalRunner {
|
|
|
180
180
|
void this.endTakeover(id).catch(() => undefined);
|
|
181
181
|
return task;
|
|
182
182
|
}
|
|
183
|
+
/**
|
|
184
|
+
* Tear everything down, and never let one failure strand the rest (#274).
|
|
185
|
+
*
|
|
186
|
+
* `browserContext.close()` rejects routinely — the browser crashed, or was
|
|
187
|
+
* already killed. It used to reject straight out of here, which skipped the
|
|
188
|
+
* state reset AND rejected the caller's shutdown, so the process stayed up
|
|
189
|
+
* holding a live browser until someone `kill -9`ed it. That kill is exactly
|
|
190
|
+
* what orphans the browser, so an unswallowed error here MADE the leak.
|
|
191
|
+
*/
|
|
183
192
|
async close() {
|
|
184
|
-
|
|
193
|
+
try {
|
|
194
|
+
this.coding.closeAll();
|
|
195
|
+
}
|
|
196
|
+
catch {
|
|
197
|
+
// a stuck coding session must not block the browser teardown below
|
|
198
|
+
}
|
|
185
199
|
await this.mcp?.stop().catch(() => undefined);
|
|
186
200
|
this.mcp = null;
|
|
187
201
|
this.cdpEndpoint = null;
|
|
188
|
-
await this.browserContext?.close();
|
|
202
|
+
await this.browserContext?.close().catch(() => undefined);
|
|
189
203
|
this.browserContext = null;
|
|
190
204
|
this.launchedProfileDir = null;
|
|
191
205
|
}
|
|
@@ -21,6 +21,17 @@ export async function startTestJobServer(port = 0) {
|
|
|
21
21
|
async close() {
|
|
22
22
|
await new Promise((resolve, reject) => {
|
|
23
23
|
server.close((error) => (error ? reject(error) : resolve()));
|
|
24
|
+
// `close()` stops ACCEPTING connections; it does not end the open ones, and
|
|
25
|
+
// its callback does not fire until the last socket goes away. A Playwright
|
|
26
|
+
// browser that navigated here keeps its connection alive for reuse, so the
|
|
27
|
+
// callback waited on a socket only the browser would ever close — an
|
|
28
|
+
// UNBOUNDED wait, not a slow one.
|
|
29
|
+
//
|
|
30
|
+
// That is why this could not be fixed by raising a budget, and two attempts
|
|
31
|
+
// tried: bc3f2b3 bounded `mcp.stop()`, f3e9c53 raised `hookTimeout`. The hook
|
|
32
|
+
// still burned its whole budget, red-gated CI, and blocked the API deploy on
|
|
33
|
+
// main for unrelated commits that happened to land next.
|
|
34
|
+
server.closeAllConnections();
|
|
24
35
|
});
|
|
25
36
|
},
|
|
26
37
|
};
|