@proagentstore/cli 0.4.129 → 0.4.130
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-runner/crash-reports.js +287 -0
- package/dist/browser-runner/index.js +34 -12
- package/dist/browser-runner/request-failure.js +19 -0
- package/dist/browser-runner/runner.js +4 -0
- package/dist/browser-runner/server.js +15 -2
- package/dist/browser-runner/store.js +40 -17
- package/dist/index.js +321 -21
- package/package.json +1 -1
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable crash reports for the local runner and the CLI that supervises it (#1087).
|
|
3
|
+
*
|
|
4
|
+
* Kept BYTE-IDENTICAL at
|
|
5
|
+
*
|
|
6
|
+
* packages/browser-runner/src/crash-reports.ts
|
|
7
|
+
* packages/cli/src/crash-reports.ts
|
|
8
|
+
*
|
|
9
|
+
* (`packages/cli/src/crash-reports.test.ts` fails the moment they differ) and imports only Node
|
|
10
|
+
* built-ins, so the CLI — which bundles the runner but cannot import its source — carries a copy.
|
|
11
|
+
*
|
|
12
|
+
* ── What it is for
|
|
13
|
+
*
|
|
14
|
+
* A runner crash used to be a line on stderr and nothing else: the runtime's `uncaughtException`
|
|
15
|
+
* handler printed and moved on, a request that threw answered 500 and was forgotten, and a runtime
|
|
16
|
+
* killed by a signal made `runner connect` exit with no record of the signal. This keeps a BOUNDED,
|
|
17
|
+
* SANITIZED record of each on disk — so it survives the process, an offline stretch and a restart —
|
|
18
|
+
* until the CLI, which holds the account token, has delivered it to the platform (`deliver.ts`
|
|
19
|
+
* there; `POST /v1/runner/crash-reports`, idempotent by `id`). There it is an `error_log` row that
|
|
20
|
+
* MCP `list_errors(source: "runner-crash")` reads.
|
|
21
|
+
*
|
|
22
|
+
* ── One writer per file
|
|
23
|
+
*
|
|
24
|
+
* The runtime writes `<dataDir>/crash-reports.json`; the CLI writes its own file. Nobody else writes
|
|
25
|
+
* either: the CLI records what it has DELIVERED in its own ledger rather than editing the runtime's
|
|
26
|
+
* file, so two processes never race on one file. A report delivered twice is harmless — the
|
|
27
|
+
* platform keeps one row per id.
|
|
28
|
+
*
|
|
29
|
+
* ── Atomic writes, and what they do not cover
|
|
30
|
+
*
|
|
31
|
+
* Every write goes to a temp file in the same directory, is fsync'd, then renamed over the target.
|
|
32
|
+
* A rename within one filesystem is atomic, so a reader sees the old file or the new one, never half
|
|
33
|
+
* of either; an interrupted write leaves a stray `*.tmp`, which is removed the next time the file is
|
|
34
|
+
* opened. LIMITS, stated rather than hidden:
|
|
35
|
+
* - a hard kill or power loss BEFORE the rename loses that one write (the file keeps its previous
|
|
36
|
+
* contents) — a crash that kills the process before `record` returns is not recorded;
|
|
37
|
+
* - the directory entry is not fsync'd, so a power loss just after the rename can still roll the
|
|
38
|
+
* file back to its previous version on some filesystems;
|
|
39
|
+
* - SIGKILL and an OOM kill run no handler in the dying process at all — only its PARENT can see
|
|
40
|
+
* the signal, which is why the CLI records `runner_exit` itself.
|
|
41
|
+
*
|
|
42
|
+
* ── Corruption is evidence, not an empty file
|
|
43
|
+
*
|
|
44
|
+
* A file that cannot be parsed is MOVED to `<file>.corrupt-<timestamp>` and reported, never read as
|
|
45
|
+
* empty and overwritten. Quarantined copies are bounded too ({@link MAX_QUARANTINED}).
|
|
46
|
+
*
|
|
47
|
+
* ── Bounds
|
|
48
|
+
*
|
|
49
|
+
* At most {@link MAX_REPORTS} reports per file (oldest dropped first, and COUNTED in `dropped`);
|
|
50
|
+
* a message is clipped to 1000 characters and a stack to 4000, marked as cut; credential-shaped
|
|
51
|
+
* text is masked. Metadata only (ADR 0004): no request or response bodies, no environment.
|
|
52
|
+
*/
|
|
53
|
+
import { closeSync, copyFileSync, existsSync, fsyncSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, unlinkSync, writeSync } from "node:fs";
|
|
54
|
+
import { basename, dirname, join } from "node:path";
|
|
55
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
56
|
+
export const MAX_REPORTS = 100;
|
|
57
|
+
export const MAX_QUARANTINED = 5;
|
|
58
|
+
const MAX_MESSAGE = 1000;
|
|
59
|
+
const MAX_STACK = 4000;
|
|
60
|
+
/** Mask credential-shaped text: bearer tokens, API keys, JWTs, `token=…` pairs. */
|
|
61
|
+
export function maskSecrets(text) {
|
|
62
|
+
return text
|
|
63
|
+
.replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{8,}/gi, "$1 [redacted]")
|
|
64
|
+
.replace(/\b(sk-[A-Za-z0-9_-]{8,}|ghp_[A-Za-z0-9]{8,}|gho_[A-Za-z0-9]{8,}|github_pat_[A-Za-z0-9_]{8,}|xox[abp]-[A-Za-z0-9-]{8,}|pags_runner_[A-Za-z0-9-]{8,})/g, "[redacted]")
|
|
65
|
+
.replace(/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g, "[redacted-jwt]")
|
|
66
|
+
.replace(/\b((?:api[_-]?key|token|secret|password|authorization)\s*[=:]\s*)("?)[^\s"&]{4,}\2/gi, "$1[redacted]");
|
|
67
|
+
}
|
|
68
|
+
/** A stable, non-identifying key for an account — the same as the runner lock's (`runner-lock.ts`). */
|
|
69
|
+
export function crashAccountKey(login) {
|
|
70
|
+
return createHash("sha256").update(login.trim().toLowerCase()).digest("hex").substring(0, 12);
|
|
71
|
+
}
|
|
72
|
+
/** Clip VISIBLY: a cut value says it was cut and by how much. */
|
|
73
|
+
function clip(text, max) {
|
|
74
|
+
if (text.length <= max)
|
|
75
|
+
return text;
|
|
76
|
+
return `${text.substring(0, max)}\n[cut: showing the first ${max} of ${text.length} characters]`;
|
|
77
|
+
}
|
|
78
|
+
const clean = (value, max) => clip(maskSecrets(String(value ?? "")), max);
|
|
79
|
+
const shortId = (value) => (typeof value === "string" && /^[A-Za-z0-9._:-]{1,128}$/.test(value) ? value : undefined);
|
|
80
|
+
/** The error's message, stack and cause, sanitized and bounded. */
|
|
81
|
+
export function describeError(error) {
|
|
82
|
+
if (error instanceof Error) {
|
|
83
|
+
const cause = error.cause;
|
|
84
|
+
return {
|
|
85
|
+
message: clean(`${error.name}: ${error.message}`, MAX_MESSAGE),
|
|
86
|
+
stack: error.stack ? clean(error.stack, MAX_STACK) : undefined,
|
|
87
|
+
cause: cause === undefined ? undefined : clean(cause instanceof Error ? `${cause.name}: ${cause.message}` : cause, MAX_MESSAGE),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
return { message: clean(error, MAX_MESSAGE) };
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* The runtime's own identity for its reports: the build and account `runner connect` passes it in
|
|
94
|
+
* `PAGS_RUNNER_BUILD` / `PAGS_RUNNER_ACCOUNT`. Absent for a bare `pags runner start`.
|
|
95
|
+
*/
|
|
96
|
+
export function runtimeCrashIdentity(env = process.env) {
|
|
97
|
+
return { source: "runtime", build: env.PAGS_RUNNER_BUILD || undefined, account: env.PAGS_RUNNER_ACCOUNT || undefined };
|
|
98
|
+
}
|
|
99
|
+
/** Build a report from what is known — every field bounded, ids id-shaped or dropped. */
|
|
100
|
+
export function crashReport(input) {
|
|
101
|
+
return {
|
|
102
|
+
id: shortId(input.id) ?? `crash_${randomUUID()}`,
|
|
103
|
+
at: input.at ?? new Date().toISOString(),
|
|
104
|
+
source: input.source,
|
|
105
|
+
kind: input.kind,
|
|
106
|
+
message: clean(input.message, MAX_MESSAGE),
|
|
107
|
+
...(input.stack ? { stack: clean(input.stack, MAX_STACK) } : {}),
|
|
108
|
+
...(input.cause ? { cause: clean(input.cause, MAX_MESSAGE) } : {}),
|
|
109
|
+
...(input.exitCode !== undefined ? { exitCode: input.exitCode } : {}),
|
|
110
|
+
...(input.signal !== undefined ? { signal: input.signal } : {}),
|
|
111
|
+
...(shortId(input.build) ? { build: input.build } : {}),
|
|
112
|
+
...(shortId(input.account) ? { account: input.account } : {}),
|
|
113
|
+
...(shortId(input.instanceId) ? { instanceId: input.instanceId } : {}),
|
|
114
|
+
...(shortId(input.sessionId) ? { sessionId: input.sessionId } : {}),
|
|
115
|
+
...(shortId(input.taskId) ? { taskId: input.taskId } : {}),
|
|
116
|
+
...(input.route ? { route: clean(input.route, 200) } : {}),
|
|
117
|
+
...(input.tail?.length ? { tail: input.tail.slice(-20).map((line) => clean(line, 300)) } : {}),
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Write `data` as JSON atomically: temp file in the same directory, fsync, rename. Throws on
|
|
122
|
+
* failure — the caller decides whether that may surface.
|
|
123
|
+
*/
|
|
124
|
+
export function writeJsonAtomic(path, data, mode = 0o600) {
|
|
125
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
126
|
+
const tmp = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
127
|
+
const fd = openSync(tmp, "w", mode);
|
|
128
|
+
try {
|
|
129
|
+
writeSync(fd, `${JSON.stringify(data, null, 2)}\n`);
|
|
130
|
+
fsyncSync(fd);
|
|
131
|
+
}
|
|
132
|
+
finally {
|
|
133
|
+
closeSync(fd);
|
|
134
|
+
}
|
|
135
|
+
try {
|
|
136
|
+
renameSync(tmp, path);
|
|
137
|
+
}
|
|
138
|
+
catch (error) {
|
|
139
|
+
try {
|
|
140
|
+
unlinkSync(tmp);
|
|
141
|
+
}
|
|
142
|
+
catch {
|
|
143
|
+
// The temp file is already gone; the rename's own error is the one that matters.
|
|
144
|
+
}
|
|
145
|
+
throw error;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
/** Remove temp files an interrupted {@link writeJsonAtomic} left beside `path`. */
|
|
149
|
+
export function removeStaleTemps(path) {
|
|
150
|
+
const dir = dirname(path);
|
|
151
|
+
const prefix = `${basename(path)}.`;
|
|
152
|
+
if (!existsSync(dir))
|
|
153
|
+
return;
|
|
154
|
+
for (const name of readdirSync(dir)) {
|
|
155
|
+
if (!name.startsWith(prefix) || !name.endsWith(".tmp"))
|
|
156
|
+
continue;
|
|
157
|
+
try {
|
|
158
|
+
unlinkSync(join(dir, name));
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
// Another process may own it; a temp file is never read, so leaving it is safe.
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Read a JSON file. A file that does not parse is MOVED aside to `<path>.corrupt-<timestamp>` —
|
|
167
|
+
* kept as evidence, and out of the way so the next write cannot overwrite it — and reported as
|
|
168
|
+
* `quarantined`. At most {@link MAX_QUARANTINED} quarantined copies are kept per file.
|
|
169
|
+
*/
|
|
170
|
+
export function readJsonOrQuarantine(path) {
|
|
171
|
+
removeStaleTemps(path);
|
|
172
|
+
if (!existsSync(path))
|
|
173
|
+
return { status: "missing" };
|
|
174
|
+
try {
|
|
175
|
+
return { status: "ok", value: JSON.parse(readFileSync(path, "utf-8")) };
|
|
176
|
+
}
|
|
177
|
+
catch (error) {
|
|
178
|
+
const quarantinedTo = `${path}.corrupt-${new Date().toISOString().replace(/[:.]/g, "-")}`;
|
|
179
|
+
try {
|
|
180
|
+
renameSync(path, quarantinedTo);
|
|
181
|
+
}
|
|
182
|
+
catch {
|
|
183
|
+
// Could not move it: keep a COPY instead, so the caller's next write cannot destroy the
|
|
184
|
+
// only evidence. If even that fails the report says so.
|
|
185
|
+
try {
|
|
186
|
+
copyFileSync(path, quarantinedTo);
|
|
187
|
+
}
|
|
188
|
+
catch (copyError) {
|
|
189
|
+
return { status: "quarantined", quarantinedTo: "(not preserved)", error: `${describeError(error).message}; preserving it failed: ${describeError(copyError).message}` };
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
pruneQuarantine(path);
|
|
193
|
+
return { status: "quarantined", quarantinedTo, error: describeError(error).message };
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
function pruneQuarantine(path) {
|
|
197
|
+
const dir = dirname(path);
|
|
198
|
+
const prefix = `${basename(path)}.corrupt-`;
|
|
199
|
+
const copies = readdirSync(dir).filter((name) => name.startsWith(prefix)).sort();
|
|
200
|
+
const excess = copies.length - MAX_QUARANTINED;
|
|
201
|
+
for (const name of excess > 0 ? copies.slice(0, excess) : []) {
|
|
202
|
+
try {
|
|
203
|
+
unlinkSync(join(dir, name));
|
|
204
|
+
}
|
|
205
|
+
catch {
|
|
206
|
+
// Best-effort pruning; an extra copy is a bounded cost.
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
/**
|
|
211
|
+
* One process's crash-report file. Synchronous throughout: it is called from `uncaughtException`
|
|
212
|
+
* and from an `exit` listener, where nothing asynchronous is guaranteed to run.
|
|
213
|
+
*/
|
|
214
|
+
export class CrashLog {
|
|
215
|
+
path;
|
|
216
|
+
base;
|
|
217
|
+
/** `base` stamps the reports this log writes about ITSELF (a quarantined file). */
|
|
218
|
+
constructor(path, base = { source: "runtime" }) {
|
|
219
|
+
this.path = path;
|
|
220
|
+
this.base = base;
|
|
221
|
+
}
|
|
222
|
+
/** Everything still on disk, oldest first. A corrupt file is quarantined and reported, then read as a fresh file. */
|
|
223
|
+
read() {
|
|
224
|
+
const got = readJsonOrQuarantine(this.path);
|
|
225
|
+
if (got.status === "ok") {
|
|
226
|
+
const reports = Array.isArray(got.value.reports) ? got.value.reports.filter((r) => !!r && typeof r.id === "string") : [];
|
|
227
|
+
return { reports, dropped: Number(got.value.dropped) || 0 };
|
|
228
|
+
}
|
|
229
|
+
if (got.status === "missing")
|
|
230
|
+
return { reports: [], dropped: 0 };
|
|
231
|
+
// The quarantine itself is a report in the fresh file, so the evidence is pointed at, not lost.
|
|
232
|
+
const file = {
|
|
233
|
+
reports: [crashReport({ ...this.base, kind: "store_corrupt", message: `crash-report file was unreadable and was moved to ${basename(got.quarantinedTo)}: ${got.error}` })],
|
|
234
|
+
dropped: 0,
|
|
235
|
+
};
|
|
236
|
+
this.write(file);
|
|
237
|
+
return file;
|
|
238
|
+
}
|
|
239
|
+
/** Append a report (bounded). Never throws: a crash path must not crash on its own record. */
|
|
240
|
+
record(report) {
|
|
241
|
+
try {
|
|
242
|
+
const file = this.read();
|
|
243
|
+
if (file.reports.some((r) => r.id === report.id))
|
|
244
|
+
return true;
|
|
245
|
+
file.reports.push(report);
|
|
246
|
+
const over = file.reports.length - MAX_REPORTS;
|
|
247
|
+
if (over > 0) {
|
|
248
|
+
file.reports = file.reports.slice(over);
|
|
249
|
+
file.dropped += over;
|
|
250
|
+
}
|
|
251
|
+
this.write(file);
|
|
252
|
+
return true;
|
|
253
|
+
}
|
|
254
|
+
catch (error) {
|
|
255
|
+
process.stderr.write(`[crash-reports] could not record ${report.kind}: ${describeError(error).message}\n`);
|
|
256
|
+
return false;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
/** Drop reports by id — only by this file's own writer. */
|
|
260
|
+
remove(ids) {
|
|
261
|
+
if (!ids.length)
|
|
262
|
+
return;
|
|
263
|
+
const file = this.read();
|
|
264
|
+
const gone = new Set(ids);
|
|
265
|
+
file.reports = file.reports.filter((r) => !gone.has(r.id));
|
|
266
|
+
this.write(file);
|
|
267
|
+
}
|
|
268
|
+
write(file) {
|
|
269
|
+
writeJsonAtomic(this.path, file);
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* Record process-level crashes in `log`. The process keeps running afterwards (a runner that died on
|
|
274
|
+
* every stray error would drop its relay), so `onCrash` is where a caller adds its own line.
|
|
275
|
+
*/
|
|
276
|
+
export function installCrashHandlers(log, base, onCrash) {
|
|
277
|
+
process.on("uncaughtException", (error) => {
|
|
278
|
+
const report = crashReport({ ...base, kind: "uncaught_exception", ...describeError(error) });
|
|
279
|
+
log.record(report);
|
|
280
|
+
onCrash?.(report);
|
|
281
|
+
});
|
|
282
|
+
process.on("unhandledRejection", (reason) => {
|
|
283
|
+
const report = crashReport({ ...base, kind: "unhandled_rejection", ...describeError(reason) });
|
|
284
|
+
log.record(report);
|
|
285
|
+
onCrash?.(report);
|
|
286
|
+
});
|
|
287
|
+
}
|
|
@@ -1,19 +1,10 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { homedir } from "node:os";
|
|
3
3
|
import { join } from "node:path";
|
|
4
|
+
import { CrashLog, crashReport, describeError, installCrashHandlers, runtimeCrashIdentity } from "./crash-reports.js";
|
|
4
5
|
import { startRunnerServer } from "./server.js";
|
|
5
6
|
import { reapOnStartup } from "./reaper.js";
|
|
6
7
|
import { randomUUID } from "node:crypto";
|
|
7
|
-
// Resilience: a stray error in any runtime must NOT take the whole runner down —
|
|
8
|
-
// that drops the tunnel and forces the user to restart `pags up` (and lose their
|
|
9
|
-
// session). Log it and keep serving. Per-component handlers catch most things;
|
|
10
|
-
// this is the last-resort backstop.
|
|
11
|
-
process.on("uncaughtException", (err) => {
|
|
12
|
-
process.stderr.write(`[runner] uncaught: ${err?.stack || String(err)}\n`);
|
|
13
|
-
});
|
|
14
|
-
process.on("unhandledRejection", (reason) => {
|
|
15
|
-
process.stderr.write(`[runner] unhandled rejection: ${String(reason)}\n`);
|
|
16
|
-
});
|
|
17
8
|
function arg(name, fallback) {
|
|
18
9
|
const index = process.argv.indexOf(name);
|
|
19
10
|
if (index === -1)
|
|
@@ -67,6 +58,18 @@ Endpoints:
|
|
|
67
58
|
process.exit(0);
|
|
68
59
|
}
|
|
69
60
|
const config = configFromArgs();
|
|
61
|
+
// Resilience: a stray error in any runtime must NOT take the whole runner down —
|
|
62
|
+
// that drops the tunnel and forces the user to restart `pags up` (and lose their
|
|
63
|
+
// session). Keep serving. Per-component handlers catch most things; this is the
|
|
64
|
+
// last-resort backstop — and since #1087 it leaves a DURABLE record, which the CLI
|
|
65
|
+
// delivers to the platform (MCP `list_errors`), instead of a stderr line only.
|
|
66
|
+
// Build and account come from `runner connect` (`runtimeCrashIdentity`); the CLI delivers a report
|
|
67
|
+
// only under the account that wrote it.
|
|
68
|
+
const IDENTITY = runtimeCrashIdentity();
|
|
69
|
+
const crashLog = new CrashLog(join(config.dataDir, "crash-reports.json"), IDENTITY);
|
|
70
|
+
installCrashHandlers(crashLog, IDENTITY, (report) => {
|
|
71
|
+
process.stderr.write(`[runner] ${report.kind === "uncaught_exception" ? "uncaught" : "unhandled rejection"}: ${report.stack || report.message}\n`);
|
|
72
|
+
});
|
|
70
73
|
// SECURITY: the runner's endpoints spawn CLIs and drive a browser as the user.
|
|
71
74
|
// Without a token, `authorize()` allows every request, so a tokenless runner must
|
|
72
75
|
// never be reachable off the local machine. Refuse to start if bound to a
|
|
@@ -83,8 +86,27 @@ if (!config.token && !LOOPBACK.has(config.host)) {
|
|
|
83
86
|
// machine to load 253. Nothing in-process can clean those up, so we do it here,
|
|
84
87
|
// before we start competing for the same CPU. Narrow by construction: see the
|
|
85
88
|
// SAFETY block in reaper.ts for why the user's real Chrome can never match.
|
|
86
|
-
|
|
87
|
-
|
|
89
|
+
/** A failure before the runtime serves anything: recorded, said, and the process exits non-zero. */
|
|
90
|
+
function startupFailed(stage, error) {
|
|
91
|
+
const report = crashReport({ ...IDENTITY, kind: "startup_failure", instanceId: config.instanceId, ...describeError(error), route: stage });
|
|
92
|
+
crashLog.record(report);
|
|
93
|
+
process.stderr.write(`[runner] could not start (${stage}): ${report.message}\n`);
|
|
94
|
+
process.exit(1);
|
|
95
|
+
}
|
|
96
|
+
try {
|
|
97
|
+
reapOnStartup((line) => process.stderr.write(`${line}\n`));
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
// The reaper is housekeeping: its failure is recorded, and the runtime still starts.
|
|
101
|
+
crashLog.record(crashReport({ ...IDENTITY, kind: "startup_failure", ...describeError(error), route: "reap" }));
|
|
102
|
+
}
|
|
103
|
+
let started;
|
|
104
|
+
try {
|
|
105
|
+
started = await startRunnerServer(config);
|
|
106
|
+
}
|
|
107
|
+
catch (error) {
|
|
108
|
+
startupFailed("listen", error);
|
|
109
|
+
}
|
|
88
110
|
process.stdout.write(`ProAgentStore browser runtime listening at ${started.url}\n`);
|
|
89
111
|
process.stdout.write(`Data dir: ${config.dataDir}\n`);
|
|
90
112
|
process.stdout.write("Control plane: PAGS; runtime plane: PAGS; brain placement: PAGS; runner role: tool-executor\n");
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { crashReport, describeError, runtimeCrashIdentity } from "./crash-reports.js";
|
|
2
|
+
/**
|
|
3
|
+
* A request that failed on the runtime's side (5xx), as a crash report (#1087): method, path and the
|
|
4
|
+
* correlation ids the URL and headers carry — the instance header, a `/tasks/:id`, a `/sessions/:id`.
|
|
5
|
+
* Never the body. Called from the server's one request boundary.
|
|
6
|
+
*/
|
|
7
|
+
export function recordRequestFailure(log, req, error) {
|
|
8
|
+
const path = (req.url || "").split("?")[0];
|
|
9
|
+
const instance = req.headers["x-pags-instance-id"];
|
|
10
|
+
log.record(crashReport({
|
|
11
|
+
...runtimeCrashIdentity(),
|
|
12
|
+
kind: "request_failure",
|
|
13
|
+
...describeError(error),
|
|
14
|
+
route: `${req.method ?? "?"} ${path}`,
|
|
15
|
+
instanceId: typeof instance === "string" ? instance : undefined,
|
|
16
|
+
taskId: /\/tasks\/([^/]+)/.exec(path)?.[1],
|
|
17
|
+
sessionId: /\/sessions\/([^/]+)/.exec(path)?.[1],
|
|
18
|
+
}));
|
|
19
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { spawnSync } from "node:child_process";
|
|
2
|
+
import { CrashLog, runtimeCrashIdentity } from "./crash-reports.js";
|
|
2
3
|
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
|
3
4
|
import { createRequire } from "node:module";
|
|
4
5
|
import { basename, dirname, join, resolve } from "node:path";
|
|
@@ -81,6 +82,8 @@ export class LocalRunner {
|
|
|
81
82
|
/** The profile dir Chrome was actually launched into (holds DevToolsActivePort). */
|
|
82
83
|
launchedProfileDir = null;
|
|
83
84
|
store;
|
|
85
|
+
/** This runtime's crash reports (#1087) — delivered to the platform by the CLI that runs it. */
|
|
86
|
+
crashLog;
|
|
84
87
|
/** Local tmux coding sessions (the second runtime — AgentCoder port). */
|
|
85
88
|
coding;
|
|
86
89
|
/** Local CLI browser research (#944): a signed-in Codex / Claude Code CLI researching through the policy bridge. */
|
|
@@ -99,6 +102,7 @@ export class LocalRunner {
|
|
|
99
102
|
this.config = config;
|
|
100
103
|
mkdirSync(config.dataDir, { recursive: true });
|
|
101
104
|
this.store = new RunnerStore(config.dataDir);
|
|
105
|
+
this.crashLog = new CrashLog(join(config.dataDir, "crash-reports.json"), runtimeCrashIdentity());
|
|
102
106
|
this.coding = new CodingRuntime(join(config.dataDir, "repos"));
|
|
103
107
|
const { browserFor, applyBrowserFor } = createRunnerBrowserFactories({ headless: config.headless, defaultTools: () => this.getMcp(), sharedPage: () => this.activePage });
|
|
104
108
|
this.localBrowser = new LocalBrowserRuntime({ dataDir: config.dataDir, selfUrl: () => this.selfUrl, browserFor });
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { createServer, } from "node:http";
|
|
2
2
|
import { URL } from "node:url";
|
|
3
|
+
import { recordRequestFailure } from "./request-failure.js";
|
|
3
4
|
import { LocalRunner, RunnerInputError } from "./runner.js";
|
|
4
5
|
import { probeGitSshIdentity } from "./coding/repo.js";
|
|
5
6
|
import { checkEngine } from "./coding/engine-check.js";
|
|
@@ -30,6 +31,11 @@ export function createRunnerServer(runner) {
|
|
|
30
31
|
}
|
|
31
32
|
catch (error) {
|
|
32
33
|
const status = error instanceof RunnerInputError ? error.status : 500;
|
|
34
|
+
// A request that failed on the runner's side is a crash report (#1087), not just a 500 to
|
|
35
|
+
// the caller: method, path and correlation ids from the URL and headers — never the body.
|
|
36
|
+
// A caller's bad input (`RunnerInputError`) is answered, not recorded.
|
|
37
|
+
if (status >= 500)
|
|
38
|
+
recordRequestFailure(runner.crashLog, req, error);
|
|
33
39
|
json(res, status, {
|
|
34
40
|
error: error instanceof Error ? error.message : String(error),
|
|
35
41
|
});
|
|
@@ -39,8 +45,15 @@ export function createRunnerServer(runner) {
|
|
|
39
45
|
export async function startRunnerServer(config) {
|
|
40
46
|
const runner = new LocalRunner(config);
|
|
41
47
|
const server = createRunnerServer(runner);
|
|
42
|
-
|
|
43
|
-
|
|
48
|
+
// A bind failure (EADDRINUSE, EACCES) REJECTS (#1087). It used to be an `error` event with no
|
|
49
|
+
// listener — an uncaught exception that the crash handler swallows, leaving a runtime that never
|
|
50
|
+
// started and never exited. Now the entrypoint records a `startup_failure` and exits non-zero.
|
|
51
|
+
await new Promise((resolve, reject) => {
|
|
52
|
+
server.once("error", reject);
|
|
53
|
+
server.listen(config.port, config.host, () => {
|
|
54
|
+
server.off("error", reject);
|
|
55
|
+
resolve();
|
|
56
|
+
});
|
|
44
57
|
});
|
|
45
58
|
const address = server.address();
|
|
46
59
|
const actualPort = address.port;
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { join } from "node:path";
|
|
2
|
+
import { CrashLog, crashReport, readJsonOrQuarantine, runtimeCrashIdentity, writeJsonAtomic } from "./crash-reports.js";
|
|
3
3
|
import { WORKFLOW_DRIVEN_TASKS } from "./task-types.js";
|
|
4
|
+
/** Events kept in the local store. Older ones are dropped — and counted in `droppedEvents`. */
|
|
5
|
+
export const MAX_EVENTS = 500;
|
|
4
6
|
function emptyStore() {
|
|
5
7
|
return {
|
|
6
8
|
sessions: [],
|
|
@@ -8,10 +10,26 @@ function emptyStore() {
|
|
|
8
10
|
events: [],
|
|
9
11
|
};
|
|
10
12
|
}
|
|
13
|
+
/**
|
|
14
|
+
* The runner's local sessions / tasks / events file.
|
|
15
|
+
*
|
|
16
|
+
* Crash-safe since #1087: every write is atomic (temp file + fsync + rename, `crash-reports.ts`), so
|
|
17
|
+
* a crash mid-write leaves the previous file intact rather than a truncated one; and a file that
|
|
18
|
+
* does not parse is QUARANTINED (moved to `runner-store.json.corrupt-<time>`) and reported in the
|
|
19
|
+
* crash log for delivery to the platform — it used to read as an empty store, and the next write
|
|
20
|
+
* then overwrote the evidence. Limits are those of `writeJsonAtomic`: the write in flight at a hard
|
|
21
|
+
* kill or power loss is lost, never half-applied.
|
|
22
|
+
*/
|
|
11
23
|
export class RunnerStore {
|
|
12
24
|
filePath;
|
|
25
|
+
crashLog;
|
|
13
26
|
constructor(dataDir) {
|
|
14
27
|
this.filePath = join(dataDir, "runner-store.json");
|
|
28
|
+
this.crashLog = new CrashLog(join(dataDir, "crash-reports.json"), runtimeCrashIdentity());
|
|
29
|
+
}
|
|
30
|
+
/** How many events the buffer has dropped, ever. */
|
|
31
|
+
droppedEventCount() {
|
|
32
|
+
return this.read().droppedEvents ?? 0;
|
|
15
33
|
}
|
|
16
34
|
listSessions() {
|
|
17
35
|
return this.read().sessions;
|
|
@@ -84,7 +102,10 @@ export class RunnerStore {
|
|
|
84
102
|
createdAt: new Date().toISOString(),
|
|
85
103
|
};
|
|
86
104
|
data.events.unshift(row);
|
|
87
|
-
|
|
105
|
+
if (data.events.length > MAX_EVENTS) {
|
|
106
|
+
data.droppedEvents = (data.droppedEvents ?? 0) + (data.events.length - MAX_EVENTS);
|
|
107
|
+
data.events.length = MAX_EVENTS;
|
|
108
|
+
}
|
|
88
109
|
this.write(data);
|
|
89
110
|
return row;
|
|
90
111
|
}
|
|
@@ -92,24 +113,26 @@ export class RunnerStore {
|
|
|
92
113
|
return this.read().events.slice(0, limit);
|
|
93
114
|
}
|
|
94
115
|
read() {
|
|
95
|
-
|
|
116
|
+
const got = readJsonOrQuarantine(this.filePath);
|
|
117
|
+
if (got.status === "missing")
|
|
96
118
|
return emptyStore();
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
};
|
|
104
|
-
}
|
|
105
|
-
catch {
|
|
119
|
+
if (got.status === "quarantined") {
|
|
120
|
+
// Evidence, not an empty store: the file is moved aside, the move is reported, and only
|
|
121
|
+
// then does the runner continue on a fresh store.
|
|
122
|
+
const message = `runner store was unreadable and was moved to ${got.quarantinedTo}: ${got.error}`;
|
|
123
|
+
process.stderr.write(`[store] ${message}\n`);
|
|
124
|
+
this.crashLog.record(crashReport({ ...runtimeCrashIdentity(), kind: "store_corrupt", message }));
|
|
106
125
|
return emptyStore();
|
|
107
126
|
}
|
|
127
|
+
const parsed = got.value;
|
|
128
|
+
return {
|
|
129
|
+
sessions: Array.isArray(parsed.sessions) ? parsed.sessions : [],
|
|
130
|
+
tasks: Array.isArray(parsed.tasks) ? parsed.tasks : [],
|
|
131
|
+
events: Array.isArray(parsed.events) ? parsed.events : [],
|
|
132
|
+
droppedEvents: Number(parsed.droppedEvents) || 0,
|
|
133
|
+
};
|
|
108
134
|
}
|
|
109
135
|
write(data) {
|
|
110
|
-
|
|
111
|
-
writeFileSync(this.filePath, `${JSON.stringify(data, null, 2)}\n`, {
|
|
112
|
-
mode: 0o600,
|
|
113
|
-
});
|
|
136
|
+
writeJsonAtomic(this.filePath, data);
|
|
114
137
|
}
|
|
115
138
|
}
|
package/dist/index.js
CHANGED
|
@@ -1174,7 +1174,7 @@ var publishCommand = new Command7("publish").description("Publish an agent to Pr
|
|
|
1174
1174
|
|
|
1175
1175
|
// src/commands/runner/command.ts
|
|
1176
1176
|
import { spawn as spawn4 } from "child_process";
|
|
1177
|
-
import { randomUUID as
|
|
1177
|
+
import { randomUUID as randomUUID4 } from "crypto";
|
|
1178
1178
|
import { Command as Command8 } from "commander";
|
|
1179
1179
|
|
|
1180
1180
|
// src/commands/runner/process.ts
|
|
@@ -2905,6 +2905,274 @@ function openRelaySocket(instanceId, wsBase, mintToken, localUrl, runnerToken, f
|
|
|
2905
2905
|
};
|
|
2906
2906
|
}
|
|
2907
2907
|
|
|
2908
|
+
// src/crash-reports.ts
|
|
2909
|
+
import { closeSync as closeSync2, copyFileSync, existsSync as existsSync8, fsyncSync, mkdirSync as mkdirSync7, openSync as openSync2, readdirSync, readFileSync as readFileSync8, renameSync as renameSync3, unlinkSync as unlinkSync3, writeSync as writeSync2 } from "fs";
|
|
2910
|
+
import { basename, dirname as dirname4, join as join9 } from "path";
|
|
2911
|
+
import { createHash as createHash2, randomUUID as randomUUID3 } from "crypto";
|
|
2912
|
+
var MAX_REPORTS = 100;
|
|
2913
|
+
var MAX_QUARANTINED = 5;
|
|
2914
|
+
var MAX_MESSAGE = 1e3;
|
|
2915
|
+
var MAX_STACK = 4e3;
|
|
2916
|
+
function maskSecrets(text) {
|
|
2917
|
+
return text.replace(/\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]{8,}/gi, "$1 [redacted]").replace(/\b(sk-[A-Za-z0-9_-]{8,}|ghp_[A-Za-z0-9]{8,}|gho_[A-Za-z0-9]{8,}|github_pat_[A-Za-z0-9_]{8,}|xox[abp]-[A-Za-z0-9-]{8,}|pags_runner_[A-Za-z0-9-]{8,})/g, "[redacted]").replace(/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g, "[redacted-jwt]").replace(/\b((?:api[_-]?key|token|secret|password|authorization)\s*[=:]\s*)("?)[^\s"&]{4,}\2/gi, "$1[redacted]");
|
|
2918
|
+
}
|
|
2919
|
+
function crashAccountKey(login) {
|
|
2920
|
+
return createHash2("sha256").update(login.trim().toLowerCase()).digest("hex").substring(0, 12);
|
|
2921
|
+
}
|
|
2922
|
+
function clip(text, max) {
|
|
2923
|
+
if (text.length <= max) return text;
|
|
2924
|
+
return `${text.substring(0, max)}
|
|
2925
|
+
[cut: showing the first ${max} of ${text.length} characters]`;
|
|
2926
|
+
}
|
|
2927
|
+
var clean2 = (value, max) => clip(maskSecrets(String(value ?? "")), max);
|
|
2928
|
+
var shortId = (value) => typeof value === "string" && /^[A-Za-z0-9._:-]{1,128}$/.test(value) ? value : void 0;
|
|
2929
|
+
function describeError(error) {
|
|
2930
|
+
if (error instanceof Error) {
|
|
2931
|
+
const cause = error.cause;
|
|
2932
|
+
return {
|
|
2933
|
+
message: clean2(`${error.name}: ${error.message}`, MAX_MESSAGE),
|
|
2934
|
+
stack: error.stack ? clean2(error.stack, MAX_STACK) : void 0,
|
|
2935
|
+
cause: cause === void 0 ? void 0 : clean2(cause instanceof Error ? `${cause.name}: ${cause.message}` : cause, MAX_MESSAGE)
|
|
2936
|
+
};
|
|
2937
|
+
}
|
|
2938
|
+
return { message: clean2(error, MAX_MESSAGE) };
|
|
2939
|
+
}
|
|
2940
|
+
function crashReport(input) {
|
|
2941
|
+
return {
|
|
2942
|
+
id: shortId(input.id) ?? `crash_${randomUUID3()}`,
|
|
2943
|
+
at: input.at ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
2944
|
+
source: input.source,
|
|
2945
|
+
kind: input.kind,
|
|
2946
|
+
message: clean2(input.message, MAX_MESSAGE),
|
|
2947
|
+
...input.stack ? { stack: clean2(input.stack, MAX_STACK) } : {},
|
|
2948
|
+
...input.cause ? { cause: clean2(input.cause, MAX_MESSAGE) } : {},
|
|
2949
|
+
...input.exitCode !== void 0 ? { exitCode: input.exitCode } : {},
|
|
2950
|
+
...input.signal !== void 0 ? { signal: input.signal } : {},
|
|
2951
|
+
...shortId(input.build) ? { build: input.build } : {},
|
|
2952
|
+
...shortId(input.account) ? { account: input.account } : {},
|
|
2953
|
+
...shortId(input.instanceId) ? { instanceId: input.instanceId } : {},
|
|
2954
|
+
...shortId(input.sessionId) ? { sessionId: input.sessionId } : {},
|
|
2955
|
+
...shortId(input.taskId) ? { taskId: input.taskId } : {},
|
|
2956
|
+
...input.route ? { route: clean2(input.route, 200) } : {},
|
|
2957
|
+
...input.tail?.length ? { tail: input.tail.slice(-20).map((line) => clean2(line, 300)) } : {}
|
|
2958
|
+
};
|
|
2959
|
+
}
|
|
2960
|
+
function writeJsonAtomic(path, data, mode = 384) {
|
|
2961
|
+
mkdirSync7(dirname4(path), { recursive: true });
|
|
2962
|
+
const tmp = `${path}.${process.pid}.${randomUUID3()}.tmp`;
|
|
2963
|
+
const fd = openSync2(tmp, "w", mode);
|
|
2964
|
+
try {
|
|
2965
|
+
writeSync2(fd, `${JSON.stringify(data, null, 2)}
|
|
2966
|
+
`);
|
|
2967
|
+
fsyncSync(fd);
|
|
2968
|
+
} finally {
|
|
2969
|
+
closeSync2(fd);
|
|
2970
|
+
}
|
|
2971
|
+
try {
|
|
2972
|
+
renameSync3(tmp, path);
|
|
2973
|
+
} catch (error) {
|
|
2974
|
+
try {
|
|
2975
|
+
unlinkSync3(tmp);
|
|
2976
|
+
} catch {
|
|
2977
|
+
}
|
|
2978
|
+
throw error;
|
|
2979
|
+
}
|
|
2980
|
+
}
|
|
2981
|
+
function removeStaleTemps(path) {
|
|
2982
|
+
const dir = dirname4(path);
|
|
2983
|
+
const prefix = `${basename(path)}.`;
|
|
2984
|
+
if (!existsSync8(dir)) return;
|
|
2985
|
+
for (const name of readdirSync(dir)) {
|
|
2986
|
+
if (!name.startsWith(prefix) || !name.endsWith(".tmp")) continue;
|
|
2987
|
+
try {
|
|
2988
|
+
unlinkSync3(join9(dir, name));
|
|
2989
|
+
} catch {
|
|
2990
|
+
}
|
|
2991
|
+
}
|
|
2992
|
+
}
|
|
2993
|
+
function readJsonOrQuarantine(path) {
|
|
2994
|
+
removeStaleTemps(path);
|
|
2995
|
+
if (!existsSync8(path)) return { status: "missing" };
|
|
2996
|
+
try {
|
|
2997
|
+
return { status: "ok", value: JSON.parse(readFileSync8(path, "utf-8")) };
|
|
2998
|
+
} catch (error) {
|
|
2999
|
+
const quarantinedTo = `${path}.corrupt-${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}`;
|
|
3000
|
+
try {
|
|
3001
|
+
renameSync3(path, quarantinedTo);
|
|
3002
|
+
} catch {
|
|
3003
|
+
try {
|
|
3004
|
+
copyFileSync(path, quarantinedTo);
|
|
3005
|
+
} catch (copyError) {
|
|
3006
|
+
return { status: "quarantined", quarantinedTo: "(not preserved)", error: `${describeError(error).message}; preserving it failed: ${describeError(copyError).message}` };
|
|
3007
|
+
}
|
|
3008
|
+
}
|
|
3009
|
+
pruneQuarantine(path);
|
|
3010
|
+
return { status: "quarantined", quarantinedTo, error: describeError(error).message };
|
|
3011
|
+
}
|
|
3012
|
+
}
|
|
3013
|
+
function pruneQuarantine(path) {
|
|
3014
|
+
const dir = dirname4(path);
|
|
3015
|
+
const prefix = `${basename(path)}.corrupt-`;
|
|
3016
|
+
const copies = readdirSync(dir).filter((name) => name.startsWith(prefix)).sort();
|
|
3017
|
+
const excess = copies.length - MAX_QUARANTINED;
|
|
3018
|
+
for (const name of excess > 0 ? copies.slice(0, excess) : []) {
|
|
3019
|
+
try {
|
|
3020
|
+
unlinkSync3(join9(dir, name));
|
|
3021
|
+
} catch {
|
|
3022
|
+
}
|
|
3023
|
+
}
|
|
3024
|
+
}
|
|
3025
|
+
var CrashLog = class {
|
|
3026
|
+
/** `base` stamps the reports this log writes about ITSELF (a quarantined file). */
|
|
3027
|
+
constructor(path, base = { source: "runtime" }) {
|
|
3028
|
+
this.path = path;
|
|
3029
|
+
this.base = base;
|
|
3030
|
+
}
|
|
3031
|
+
path;
|
|
3032
|
+
base;
|
|
3033
|
+
/** Everything still on disk, oldest first. A corrupt file is quarantined and reported, then read as a fresh file. */
|
|
3034
|
+
read() {
|
|
3035
|
+
const got = readJsonOrQuarantine(this.path);
|
|
3036
|
+
if (got.status === "ok") {
|
|
3037
|
+
const reports = Array.isArray(got.value.reports) ? got.value.reports.filter((r) => !!r && typeof r.id === "string") : [];
|
|
3038
|
+
return { reports, dropped: Number(got.value.dropped) || 0 };
|
|
3039
|
+
}
|
|
3040
|
+
if (got.status === "missing") return { reports: [], dropped: 0 };
|
|
3041
|
+
const file = {
|
|
3042
|
+
reports: [crashReport({ ...this.base, kind: "store_corrupt", message: `crash-report file was unreadable and was moved to ${basename(got.quarantinedTo)}: ${got.error}` })],
|
|
3043
|
+
dropped: 0
|
|
3044
|
+
};
|
|
3045
|
+
this.write(file);
|
|
3046
|
+
return file;
|
|
3047
|
+
}
|
|
3048
|
+
/** Append a report (bounded). Never throws: a crash path must not crash on its own record. */
|
|
3049
|
+
record(report) {
|
|
3050
|
+
try {
|
|
3051
|
+
const file = this.read();
|
|
3052
|
+
if (file.reports.some((r) => r.id === report.id)) return true;
|
|
3053
|
+
file.reports.push(report);
|
|
3054
|
+
const over = file.reports.length - MAX_REPORTS;
|
|
3055
|
+
if (over > 0) {
|
|
3056
|
+
file.reports = file.reports.slice(over);
|
|
3057
|
+
file.dropped += over;
|
|
3058
|
+
}
|
|
3059
|
+
this.write(file);
|
|
3060
|
+
return true;
|
|
3061
|
+
} catch (error) {
|
|
3062
|
+
process.stderr.write(`[crash-reports] could not record ${report.kind}: ${describeError(error).message}
|
|
3063
|
+
`);
|
|
3064
|
+
return false;
|
|
3065
|
+
}
|
|
3066
|
+
}
|
|
3067
|
+
/** Drop reports by id — only by this file's own writer. */
|
|
3068
|
+
remove(ids) {
|
|
3069
|
+
if (!ids.length) return;
|
|
3070
|
+
const file = this.read();
|
|
3071
|
+
const gone = new Set(ids);
|
|
3072
|
+
file.reports = file.reports.filter((r) => !gone.has(r.id));
|
|
3073
|
+
this.write(file);
|
|
3074
|
+
}
|
|
3075
|
+
write(file) {
|
|
3076
|
+
writeJsonAtomic(this.path, file);
|
|
3077
|
+
}
|
|
3078
|
+
};
|
|
3079
|
+
|
|
3080
|
+
// src/crash-delivery.ts
|
|
3081
|
+
import { existsSync as existsSync9, readFileSync as readFileSync9 } from "fs";
|
|
3082
|
+
import { homedir as homedir4 } from "os";
|
|
3083
|
+
import { join as join10 } from "path";
|
|
3084
|
+
var BATCH = 50;
|
|
3085
|
+
var LEDGER_MAX = 1e3;
|
|
3086
|
+
var TIMEOUT_MS = 1e4;
|
|
3087
|
+
function runnerDataDir(dataDir, env = process.env) {
|
|
3088
|
+
return dataDir || env.PAGS_RUNNER_DATA_DIR || join10(homedir4(), ".config", "proagentstore", "browser-runner");
|
|
3089
|
+
}
|
|
3090
|
+
function crashPaths(configDir2, account, dataDir) {
|
|
3091
|
+
const key = crashAccountKey(account);
|
|
3092
|
+
return {
|
|
3093
|
+
runtimeFile: join10(runnerDataDir(dataDir), "crash-reports.json"),
|
|
3094
|
+
cliFile: join10(configDir2, `runner-crashes-${key}.json`),
|
|
3095
|
+
ledgerFile: join10(configDir2, `crash-delivered-${key}.json`)
|
|
3096
|
+
};
|
|
3097
|
+
}
|
|
3098
|
+
function readRuntimeReports(path) {
|
|
3099
|
+
if (!existsSync9(path)) return [];
|
|
3100
|
+
try {
|
|
3101
|
+
const parsed = JSON.parse(readFileSync9(path, "utf-8"));
|
|
3102
|
+
return Array.isArray(parsed.reports) ? parsed.reports.filter((r) => r && typeof r.id === "string") : [];
|
|
3103
|
+
} catch {
|
|
3104
|
+
return [];
|
|
3105
|
+
}
|
|
3106
|
+
}
|
|
3107
|
+
function readLedger(path) {
|
|
3108
|
+
const got = readJsonOrQuarantine(path);
|
|
3109
|
+
return got.status === "ok" && Array.isArray(got.value.ids) ? got.value.ids.filter((id) => typeof id === "string") : [];
|
|
3110
|
+
}
|
|
3111
|
+
async function deliverCrashReports(opts) {
|
|
3112
|
+
const key = crashAccountKey(opts.account);
|
|
3113
|
+
const cliLog = new CrashLog(opts.paths.cliFile, { source: "cli", account: key });
|
|
3114
|
+
const acked = new Set(readLedger(opts.paths.ledgerFile));
|
|
3115
|
+
const mine = (r) => r.account === key && !acked.has(r.id);
|
|
3116
|
+
const pending = [...readRuntimeReports(opts.paths.runtimeFile).filter(mine), ...cliLog.read().reports.filter(mine)];
|
|
3117
|
+
if (!pending.length) return { delivered: 0, pending: 0 };
|
|
3118
|
+
const doFetch = opts.fetchImpl ?? fetch;
|
|
3119
|
+
const accepted = [];
|
|
3120
|
+
let error;
|
|
3121
|
+
for (let i = 0; i < pending.length; i += BATCH) {
|
|
3122
|
+
const batch = pending.slice(i, i + BATCH);
|
|
3123
|
+
try {
|
|
3124
|
+
const res = await doFetch(`${opts.apiBase.replace(/\/$/, "")}/v1/errors/runner-crashes`, {
|
|
3125
|
+
method: "POST",
|
|
3126
|
+
headers: { Authorization: `Bearer ${opts.token}`, "Content-Type": "application/json" },
|
|
3127
|
+
body: JSON.stringify({ reports: batch }),
|
|
3128
|
+
signal: AbortSignal.timeout(TIMEOUT_MS)
|
|
3129
|
+
});
|
|
3130
|
+
if (!res.ok) {
|
|
3131
|
+
error = `the platform answered ${res.status}`;
|
|
3132
|
+
break;
|
|
3133
|
+
}
|
|
3134
|
+
const body = await res.json().catch(() => ({}));
|
|
3135
|
+
if (Array.isArray(body.accepted)) accepted.push(...body.accepted.filter((id) => typeof id === "string"));
|
|
3136
|
+
} catch (e) {
|
|
3137
|
+
error = e instanceof Error ? e.message : String(e);
|
|
3138
|
+
break;
|
|
3139
|
+
}
|
|
3140
|
+
}
|
|
3141
|
+
if (accepted.length) {
|
|
3142
|
+
const ledger = [...acked, ...accepted];
|
|
3143
|
+
writeJsonAtomic(opts.paths.ledgerFile, { ids: ledger.slice(Math.max(0, ledger.length - LEDGER_MAX)) });
|
|
3144
|
+
cliLog.remove(accepted);
|
|
3145
|
+
}
|
|
3146
|
+
return { delivered: accepted.length, pending: pending.length - accepted.length, ...error ? { error } : {} };
|
|
3147
|
+
}
|
|
3148
|
+
var OutputTail = class {
|
|
3149
|
+
constructor(max) {
|
|
3150
|
+
this.max = max;
|
|
3151
|
+
}
|
|
3152
|
+
max;
|
|
3153
|
+
buffer = [];
|
|
3154
|
+
partial = "";
|
|
3155
|
+
push(chunk) {
|
|
3156
|
+
const parts = (this.partial + chunk.toString()).split("\n");
|
|
3157
|
+
this.partial = parts.pop() ?? "";
|
|
3158
|
+
for (const line of parts) {
|
|
3159
|
+
if (!line.trim()) continue;
|
|
3160
|
+
this.buffer.push(line);
|
|
3161
|
+
if (this.buffer.length > this.max) this.buffer.shift();
|
|
3162
|
+
}
|
|
3163
|
+
}
|
|
3164
|
+
lines() {
|
|
3165
|
+
return this.partial.trim() ? [...this.buffer, this.partial].slice(-this.max) : [...this.buffer];
|
|
3166
|
+
}
|
|
3167
|
+
};
|
|
3168
|
+
function recordChildExit(o) {
|
|
3169
|
+
const account = crashAccountKey(o.account);
|
|
3170
|
+
const how = o.signal ? `was killed by ${o.signal}` : `exited unexpectedly${o.code !== null ? ` (code ${o.code})` : ""}`;
|
|
3171
|
+
const report = crashReport({ source: "cli", account, build: o.build, kind: o.kind, message: `${o.what} ${how}`, exitCode: o.code, signal: o.signal, tail: o.tail });
|
|
3172
|
+
new CrashLog(o.paths.cliFile, { source: "cli", account }).record(report);
|
|
3173
|
+
return report;
|
|
3174
|
+
}
|
|
3175
|
+
|
|
2908
3176
|
// src/commands/runner/command.ts
|
|
2909
3177
|
import { hostname as hostname5 } from "os";
|
|
2910
3178
|
function collectCapability(value, previous = []) {
|
|
@@ -2918,7 +3186,7 @@ function createRunnerCommand() {
|
|
|
2918
3186
|
await startRunnerForeground(opts);
|
|
2919
3187
|
});
|
|
2920
3188
|
command.command("connect <instanceIds...>").description("Start ONE local runtime, connect via WebSocket relay, and register it for every given PAGS instance").option("--host <host>", "Host to bind", "127.0.0.1").option("--port <port>", "Port to bind", "49171").option("--data-dir <path>", "Runner data directory").option("--token <token>", "Runner bearer token. Defaults to PAGS_RUNNER_TOKEN or a generated token").option("--headless", "Run Playwright headless").option("--api-base <url>", "PAGS API base URL").option("--pags-token <token>", "PAGS session token. Defaults to PAGS_TOKEN").option("--runner-version <version>", "Runner version").option("--force", "Take over from another connected machine").option("--watch-instances", "Attach newly eligible agents while running, without a restart").action(async (instanceIds, opts) => {
|
|
2921
|
-
const runnerToken = clean(opts.token) || clean(process.env.PAGS_RUNNER_TOKEN) || `pags_runner_${
|
|
3189
|
+
const runnerToken = clean(opts.token) || clean(process.env.PAGS_RUNNER_TOKEN) || `pags_runner_${randomUUID4()}`;
|
|
2922
3190
|
const host = clean(opts.host) || "127.0.0.1";
|
|
2923
3191
|
const port = clean(opts.port) || String(await findFreePort2(49171));
|
|
2924
3192
|
const lockAccount = clean(process.env.PAGS_LOCK_ACCOUNT) || "";
|
|
@@ -2939,6 +3207,8 @@ function createRunnerCommand() {
|
|
|
2939
3207
|
const primary = instanceIds[0];
|
|
2940
3208
|
const runnerOpts = { ...opts, host, port, token: runnerToken };
|
|
2941
3209
|
const spec = runnerSpawnSpec(runnerOpts);
|
|
3210
|
+
const crashAccount = lockAccount || loadSession()?.user.login || "";
|
|
3211
|
+
const crashes = crashAccount ? crashPaths(configDir(), crashAccount, opts.dataDir) : null;
|
|
2942
3212
|
const runner = spawn4(spec.command, spec.args, {
|
|
2943
3213
|
cwd: spec.cwd,
|
|
2944
3214
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -2948,16 +3218,36 @@ function createRunnerCommand() {
|
|
|
2948
3218
|
// replace is an authorisation rather than a signal aimed at a name pattern.
|
|
2949
3219
|
env: {
|
|
2950
3220
|
...process.env,
|
|
3221
|
+
PAGS_RUNNER_BUILD: CLI_VERSION,
|
|
3222
|
+
...crashAccount ? { PAGS_RUNNER_ACCOUNT: crashAccountKey(crashAccount) } : {},
|
|
2951
3223
|
...lockHeld ? { PAGS_RUNNER_RSID: lockHeld.rsid, PAGS_RUNNER_CONTROL_NONCE: lockHeld.nonce, PAGS_RUNNER_LAUNCH: lockHeld.launch } : {}
|
|
2952
3224
|
}
|
|
2953
3225
|
});
|
|
3226
|
+
const tail = new OutputTail(20);
|
|
2954
3227
|
runner.stdout?.on("data", (data) => process.stdout.write(data));
|
|
2955
|
-
runner.stderr?.on("data", (data) =>
|
|
3228
|
+
runner.stderr?.on("data", (data) => {
|
|
3229
|
+
process.stderr.write(data);
|
|
3230
|
+
tail.push(data);
|
|
3231
|
+
});
|
|
3232
|
+
const deliver = async () => {
|
|
3233
|
+
if (!crashes) return;
|
|
3234
|
+
const token = clean(opts.pagsToken) || clean(process.env.PAGS_TOKEN) || clean(loadSession()?.token);
|
|
3235
|
+
if (!token) return;
|
|
3236
|
+
const result = await deliverCrashReports({ apiBase: pagsApiBase(opts.apiBase), token, account: crashAccount, paths: crashes });
|
|
3237
|
+
if (result.delivered) writeLine(`Delivered ${result.delivered} crash report(s) to ProAgentStore.`);
|
|
3238
|
+
};
|
|
2956
3239
|
let shuttingDown = false;
|
|
2957
|
-
runner.on("exit", (code) => {
|
|
3240
|
+
runner.on("exit", (code, signal) => {
|
|
2958
3241
|
if (shuttingDown) return;
|
|
2959
|
-
|
|
2960
|
-
|
|
3242
|
+
const how = signal ? `was killed by ${signal}` : `exited unexpectedly${code !== null ? ` (code ${code})` : ""}`;
|
|
3243
|
+
writeError(`Local browser runtime ${how}. Run \`pags up\` again to reconnect.`);
|
|
3244
|
+
if (crashes) recordChildExit({ paths: crashes, account: crashAccount, build: CLI_VERSION, kind: "runner_exit", what: "the local runtime", code, signal, tail: tail.lines() });
|
|
3245
|
+
const finish = () => process.exit(code ?? 1);
|
|
3246
|
+
const deadline = setTimeout(finish, 5e3);
|
|
3247
|
+
void deliver().catch(() => void 0).finally(() => {
|
|
3248
|
+
clearTimeout(deadline);
|
|
3249
|
+
finish();
|
|
3250
|
+
});
|
|
2961
3251
|
});
|
|
2962
3252
|
const shutdown = () => {
|
|
2963
3253
|
shuttingDown = true;
|
|
@@ -2977,6 +3267,8 @@ function createRunnerCommand() {
|
|
|
2977
3267
|
await waitForLocalRunner({ url: localUrl, token: runnerToken, instanceId: primary });
|
|
2978
3268
|
writeLine(`Local browser runtime healthy at ${localUrl}`);
|
|
2979
3269
|
await connectViaRelay(instanceIds, localUrl, runnerToken, opts, Boolean(opts.force), Boolean(opts.watchInstances));
|
|
3270
|
+
void deliver().catch(() => void 0);
|
|
3271
|
+
setInterval(() => void deliver().catch(() => void 0), 5 * 6e4).unref();
|
|
2980
3272
|
await new Promise((resolvePromise) => {
|
|
2981
3273
|
runner.on("exit", () => resolvePromise());
|
|
2982
3274
|
});
|
|
@@ -3098,27 +3390,27 @@ function createRunnerCommand() {
|
|
|
3098
3390
|
var runnerCommand = createRunnerCommand();
|
|
3099
3391
|
|
|
3100
3392
|
// src/commands/status.ts
|
|
3101
|
-
import { readFileSync as
|
|
3393
|
+
import { readFileSync as readFileSync11, statSync as statSync2, watchFile } from "fs";
|
|
3102
3394
|
import { Command as Command9 } from "commander";
|
|
3103
3395
|
|
|
3104
3396
|
// src/background.ts
|
|
3105
3397
|
import { spawn as nodeSpawn } from "child_process";
|
|
3106
|
-
import { closeSync as
|
|
3107
|
-
import { dirname as
|
|
3398
|
+
import { closeSync as closeSync3, existsSync as existsSync10, mkdirSync as mkdirSync8, openSync as openSync3, readFileSync as readFileSync10, renameSync as renameSync4, statSync, writeFileSync as writeFileSync6 } from "fs";
|
|
3399
|
+
import { dirname as dirname5, join as join11 } from "path";
|
|
3108
3400
|
var BACKGROUND_ENV = "PAGS_BACKGROUND";
|
|
3109
3401
|
var BACKGROUND_LOG_ENV = "PAGS_BACKGROUND_LOG";
|
|
3110
3402
|
var BACKGROUND_RESTART_ENV = "PAGS_BACKGROUND_RESTART";
|
|
3111
3403
|
var LOG_ROTATE_BYTES = 5 * 1024 * 1024;
|
|
3112
|
-
var backgroundLogPath = (account, env = process.env) =>
|
|
3113
|
-
var backgroundStatePath = (account, env = process.env) =>
|
|
3404
|
+
var backgroundLogPath = (account, env = process.env) => join11(configDir(env), "logs", `runner-${accountKey(account)}.log`);
|
|
3405
|
+
var backgroundStatePath = (account, env = process.env) => join11(configDir(env), `background-${accountKey(account)}.json`);
|
|
3114
3406
|
function writeBackgroundState(path, state) {
|
|
3115
|
-
|
|
3407
|
+
mkdirSync8(dirname5(path), { recursive: true });
|
|
3116
3408
|
writeFileSync6(path, `${JSON.stringify(state)}
|
|
3117
3409
|
`, { mode: 384 });
|
|
3118
3410
|
}
|
|
3119
3411
|
function readBackgroundState(path) {
|
|
3120
3412
|
try {
|
|
3121
|
-
const parsed = JSON.parse(
|
|
3413
|
+
const parsed = JSON.parse(readFileSync10(path, "utf-8"));
|
|
3122
3414
|
return parsed?.v === 1 && Number.isInteger(parsed.pid) ? parsed : null;
|
|
3123
3415
|
} catch {
|
|
3124
3416
|
return null;
|
|
@@ -3130,7 +3422,7 @@ function rotateLog(path, maxBytes = LOG_ROTATE_BYTES) {
|
|
|
3130
3422
|
} catch {
|
|
3131
3423
|
return false;
|
|
3132
3424
|
}
|
|
3133
|
-
|
|
3425
|
+
renameSync4(path, `${path}.1`);
|
|
3134
3426
|
return true;
|
|
3135
3427
|
}
|
|
3136
3428
|
function shellQuote(arg, platform = process.platform) {
|
|
@@ -3142,16 +3434,16 @@ function backgroundRestartCommand(execPath, script, upArgs, platform = process.p
|
|
|
3142
3434
|
return platform === "win32" ? `set ${BACKGROUND_RESTART_ENV}=1&& ${argv}` : `${BACKGROUND_RESTART_ENV}=1 ${argv}`;
|
|
3143
3435
|
}
|
|
3144
3436
|
function spawnDetached(command, args, env, logPath, spawn5 = nodeSpawn) {
|
|
3145
|
-
|
|
3437
|
+
mkdirSync8(dirname5(logPath), { recursive: true });
|
|
3146
3438
|
rotateLog(logPath);
|
|
3147
|
-
const fd =
|
|
3439
|
+
const fd = openSync3(logPath, "a", 384);
|
|
3148
3440
|
try {
|
|
3149
3441
|
const child = spawn5(command, args, { detached: true, stdio: ["ignore", fd, fd], env, windowsHide: true });
|
|
3150
3442
|
child.unref();
|
|
3151
3443
|
if (!child.pid) throw new Error("the runner process did not start");
|
|
3152
3444
|
return child.pid;
|
|
3153
3445
|
} finally {
|
|
3154
|
-
|
|
3446
|
+
closeSync3(fd);
|
|
3155
3447
|
}
|
|
3156
3448
|
}
|
|
3157
3449
|
var sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
@@ -3175,8 +3467,8 @@ async function waitUntilTakeable(read, opts = {}) {
|
|
|
3175
3467
|
}
|
|
3176
3468
|
}
|
|
3177
3469
|
function tailLines(path, n) {
|
|
3178
|
-
if (!
|
|
3179
|
-
const lines =
|
|
3470
|
+
if (!existsSync10(path)) return null;
|
|
3471
|
+
const lines = readFileSync10(path, "utf-8").split("\n");
|
|
3180
3472
|
if (lines.at(-1) === "") lines.pop();
|
|
3181
3473
|
return lines.slice(Math.max(0, lines.length - n));
|
|
3182
3474
|
}
|
|
@@ -3231,7 +3523,7 @@ var logsCommand = new Command9("logs").description("Show the background runner's
|
|
|
3231
3523
|
watchFile(path, { interval: 1e3 }, (cur) => {
|
|
3232
3524
|
if (cur.size < offset) offset = 0;
|
|
3233
3525
|
if (cur.size === offset) return;
|
|
3234
|
-
process.stdout.write(
|
|
3526
|
+
process.stdout.write(readFileSync11(path).subarray(offset, cur.size));
|
|
3235
3527
|
offset = cur.size;
|
|
3236
3528
|
});
|
|
3237
3529
|
});
|
|
@@ -3735,14 +4027,21 @@ var upCommand = new Command10("up").description("Start the browser runner for al
|
|
|
3735
4027
|
}
|
|
3736
4028
|
};
|
|
3737
4029
|
let childDead = false;
|
|
4030
|
+
let stopping = false;
|
|
3738
4031
|
child.stdout?.on("data", handleOutput);
|
|
3739
4032
|
child.stderr?.on("data", handleOutput);
|
|
3740
|
-
|
|
4033
|
+
const crashes = crashPaths(configDir(), session.user.login);
|
|
4034
|
+
const recordConnectExit = (code, signal, tail) => {
|
|
4035
|
+
recordChildExit({ paths: crashes, account: session.user.login, build: CLI_VERSION2, kind: "connect_exit", what: "runner connect", code, signal, tail });
|
|
4036
|
+
void deliverCrashReports({ apiBase: API_BASE3, token: session.token, account: session.user.login, paths: crashes }).catch(() => void 0);
|
|
4037
|
+
};
|
|
4038
|
+
child.on("exit", (code, signal) => {
|
|
3741
4039
|
if (code === RUNNER_RESTART_EXIT_CODE) {
|
|
3742
4040
|
restartUp("Runner updated remotely \u2014 restarting pags up on the new version\u2026", { PAGS_NO_PROMPT: "1" });
|
|
3743
4041
|
return;
|
|
3744
4042
|
}
|
|
3745
4043
|
childDead = true;
|
|
4044
|
+
if (code !== 0 && !stopping) recordConnectExit(code, signal ?? null, logs.slice(-20));
|
|
3746
4045
|
if (code === null) {
|
|
3747
4046
|
state.runner = "error";
|
|
3748
4047
|
state.lastEvent = "Runner stopped by another process (signal) \u2014 it is not connected any more";
|
|
@@ -3765,6 +4064,7 @@ var upCommand = new Command10("up").description("Start the browser runner for al
|
|
|
3765
4064
|
process.exit(1);
|
|
3766
4065
|
}
|
|
3767
4066
|
const shutdown = () => {
|
|
4067
|
+
stopping = true;
|
|
3768
4068
|
child.kill();
|
|
3769
4069
|
clearScreen();
|
|
3770
4070
|
writeLine(" Runner stopped.");
|