@saadnvd1/agent-os 0.3.678 → 0.3.684
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/lib/agents/brief.ts +7 -0
- package/lib/code-health/reports.ts +38 -1
- package/lib/code-health/source.ts +1 -1
- package/lib/demo-reseed.ts +4 -0
- package/lib/dev-instance/spawnServer.ts +10 -6
- package/lib/doctor/checks.ts +1 -1
- package/lib/env-setup.ts +2 -0
- package/lib/files.ts +1 -0
- package/lib/health/facts.ts +13 -3
- package/lib/health/spin-tick.ts +293 -0
- package/lib/health/spin.ts +158 -0
- package/lib/health/tick.ts +10 -2
- package/lib/health/types.ts +2 -0
- package/lib/orchestrator/brief.ts +1 -1
- package/lib/share/pdf.ts +32 -1
- package/lib/tasks/setup.ts +9 -2
- package/lib/worktree-deps/clone.ts +10 -0
- package/lib/worktree-deps/fastcopy.ts +8 -1
- package/lib/worktree-deps/index.ts +14 -19
- package/lib/worktree-deps/install.ts +17 -24
- package/package.json +8 -15
- package/scripts/check-native.mjs +86 -0
- package/scripts/code-health/prepass.ts +3 -3
- package/scripts/install-deps +24 -0
- package/scripts/lib/commands.sh +2 -2
- package/scripts/lib/deps-swap.sh +68 -0
- package/scripts/lib/deps-swap.test.ts +115 -0
- package/scripts/lib/pnpm.sh +36 -0
- package/scripts/lib/prerequisites.sh +9 -9
- package/scripts/publish +3 -2
- package/scripts/redeploy +25 -8
- package/scripts/setup.sh +4 -4
- package/tsconfig.json +1 -0
- package/scripts/ensure-native.mjs +0 -23
package/README.md
CHANGED
|
@@ -43,7 +43,7 @@ and check on all of it from your phone.
|
|
|
43
43
|
|
|
44
44
|
## Install
|
|
45
45
|
|
|
46
|
-
You need Node.js
|
|
46
|
+
You need Node.js 22+, tmux, and at least one agent CLI (the installer offers to set up what's missing).
|
|
47
47
|
|
|
48
48
|
```bash
|
|
49
49
|
npm install -g @saadnvd1/agent-os
|
|
@@ -74,7 +74,7 @@ npm run check # typecheck, lint, format and tests: what CI runs
|
|
|
74
74
|
npm run screenshots # regenerate screenshots/ from a demo instance with fake data
|
|
75
75
|
```
|
|
76
76
|
|
|
77
|
-
A pre-commit hook checks formatting, secrets and banned terms on what is staged. Route handlers may not call `*Sync` functions, and git runs only through `lib/git-exec.ts`. More in [docs/reference/development.md](docs/reference/development.md).
|
|
77
|
+
Dependencies install with pnpm, frozen to the lockfile, with a 7-day cooldown on new versions and an allowlist for install scripts: see [docs/reference/dependencies.md](docs/reference/dependencies.md) (it also covers adding a package that needs an install script and the override for an urgent security fix). A pre-commit hook checks formatting, secrets and banned terms on what is staged. Route handlers may not call `*Sync` functions, and git runs only through `lib/git-exec.ts`. More in [docs/reference/development.md](docs/reference/development.md).
|
|
78
78
|
|
|
79
79
|
## License
|
|
80
80
|
|
package/lib/agents/brief.ts
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
// What every agent AgentOS starts is told about waiting; the note sent after
|
|
2
|
+
// AgentOS stops a spinning background job repeats it (lib/health/spin.ts).
|
|
3
|
+
export const WAIT_ADVICE =
|
|
4
|
+
"To wait for a long command, run it with run_in_background and end your turn; you're woken when it exits. Don't start a second job to poll for the first. Any wait loop must sleep on each pass and test a condition that can actually become true (pgrep for the exact process, a log line), never a fixed sleep.";
|
|
5
|
+
|
|
1
6
|
// What every agent AgentOS starts is told about the network it's on.
|
|
2
7
|
export const BUS_BRIEF = `You are one of several coding agents running in AgentOS. Other sessions may be working on related projects at the same time, and you can talk to them with the \`aos\` command:
|
|
3
8
|
|
|
@@ -18,4 +23,6 @@ Messages arrive in your terminal as lines starting with "[AgentOS message from .
|
|
|
18
23
|
|
|
19
24
|
AgentOS itself is running your session, on this machine. Never stop processes with pkill/killall by pattern. Stop your own dev server with \`npm run dev:stop\` (or kill the PID you started). Never stop, kill or restart the AgentOS server, or whatever listens on its port: that ends every session, yours included. If it needs a restart, make that the very last thing you do and say so first, or ask the user to do it.
|
|
20
25
|
|
|
26
|
+
${WAIT_ADVICE}
|
|
27
|
+
|
|
21
28
|
Python: run a repo's Python through its own tooling: \`uv run …\` where it has a pyproject.toml or uv.lock (\`uv\` is at ~/.local/bin/uv), the repo's documented test script (for example \`npm run pytest\` in dashboards), or its venv. Never a bare \`python3\` for repo code, and never \`pip install\` into Homebrew or system Python. Make temp files with \`mktemp\`, not \`$TMPDIR/…\`.`;
|
|
@@ -24,13 +24,50 @@ interface NpmVuln {
|
|
|
24
24
|
| { name?: string; version?: string; isSemVerMajor?: boolean };
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
interface PnpmAdvisory {
|
|
28
|
+
module_name?: string;
|
|
29
|
+
title?: string;
|
|
30
|
+
severity?: string;
|
|
31
|
+
patched_versions?: string | null;
|
|
32
|
+
findings?: { paths?: string[] }[];
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// pnpm audit lists advisories, not packages: one npm-style entry per module.
|
|
36
|
+
function fromAdvisories(
|
|
37
|
+
advisories: Record<string, PnpmAdvisory>
|
|
38
|
+
): Record<string, NpmVuln> {
|
|
39
|
+
const out: Record<string, NpmVuln> = {};
|
|
40
|
+
for (const a of Object.values(advisories)) {
|
|
41
|
+
const name = a.module_name ?? "";
|
|
42
|
+
if (!name) continue;
|
|
43
|
+
const paths = (a.findings ?? []).flatMap((f) => f.paths ?? []);
|
|
44
|
+
const prev = out[name];
|
|
45
|
+
const rank = (v?: string) => (v === "critical" ? 2 : v === "high" ? 1 : 0);
|
|
46
|
+
out[name] = {
|
|
47
|
+
severity:
|
|
48
|
+
rank(a.severity) >= rank(prev?.severity) ? a.severity : prev?.severity,
|
|
49
|
+
isDirect:
|
|
50
|
+
Boolean(prev?.isDirect) || paths.some((p) => /^\.>[^>]+$/.test(p)),
|
|
51
|
+
via: [...(prev?.via ?? []), a.title ?? ""],
|
|
52
|
+
fixAvailable: a.patched_versions
|
|
53
|
+
? { name, version: a.patched_versions }
|
|
54
|
+
: false,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
return out;
|
|
58
|
+
}
|
|
59
|
+
|
|
27
60
|
export function summarizeNpmAudit(json: unknown): NpmAuditSummary {
|
|
28
61
|
const data = (json ?? {}) as {
|
|
29
62
|
metadata?: { vulnerabilities?: Record<string, number> };
|
|
30
63
|
vulnerabilities?: Record<string, NpmVuln>;
|
|
64
|
+
advisories?: Record<string, PnpmAdvisory>;
|
|
31
65
|
};
|
|
32
66
|
const counts = { ...(data.metadata?.vulnerabilities ?? {}) };
|
|
33
|
-
const
|
|
67
|
+
const vulnerabilities = data.advisories
|
|
68
|
+
? fromAdvisories(data.advisories)
|
|
69
|
+
: (data.vulnerabilities ?? {});
|
|
70
|
+
const serious = Object.entries(vulnerabilities)
|
|
34
71
|
.filter(([, v]) => v.severity === "high" || v.severity === "critical")
|
|
35
72
|
.map(([name, v]) => {
|
|
36
73
|
const fix = v.fixAvailable;
|
|
@@ -16,7 +16,7 @@ const CODE = /\.(ts|tsx|mts|cts|js|mjs|cjs|jsx)$/;
|
|
|
16
16
|
const TEST = /(\.test\.|\.spec\.|(^|\/)__tests__\/|(^|\/)vitest\.setup\.)/;
|
|
17
17
|
// Generated, vendored or built: measured by other means or not ours.
|
|
18
18
|
const NOT_OURS =
|
|
19
|
-
/(^|\/)(node_modules|\.next[^/]*|dist|out|coverage|public
|
|
19
|
+
/(^|\/)(node_modules|\.deps|\.next[^/]*|dist|out|coverage|public)\/|\.min\.js$|(^|\/)next-env\.d\.ts$/;
|
|
20
20
|
|
|
21
21
|
export const isCode = (p: string) => CODE.test(p) && !NOT_OURS.test(p);
|
|
22
22
|
export const isTest = (p: string) => TEST.test(p);
|
package/lib/demo-reseed.ts
CHANGED
|
@@ -57,6 +57,10 @@ export function restoreDemo(db: Database.Database, file: string): void {
|
|
|
57
57
|
db.transaction(() => {
|
|
58
58
|
for (const { name } of tables) {
|
|
59
59
|
if (KEEP_TABLES.has(name)) continue;
|
|
60
|
+
// The search index (an FTS table, its shadow tables, its keys) is
|
|
61
|
+
// kept up to date by triggers on chat_items; copying it would
|
|
62
|
+
// fight them, and shadow tables can't be written at all.
|
|
63
|
+
if (name.startsWith("chat_search")) continue;
|
|
60
64
|
// Stored columns only: a generated one can't be written.
|
|
61
65
|
const cols = (
|
|
62
66
|
db.prepare(`SELECT * FROM pragma_table_xinfo(?)`).all(name) as {
|
|
@@ -20,12 +20,16 @@ export function spawnServer(opts: {
|
|
|
20
20
|
}): ChildProcess {
|
|
21
21
|
const { repo, port, env, stdio, detached, native } = opts;
|
|
22
22
|
assertNotLivePort(port, process.env, readLiveRecord());
|
|
23
|
-
const run = (script: string) =>
|
|
24
|
-
execFileSync(
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
23
|
+
const run = (script: string, ...args: string[]) =>
|
|
24
|
+
execFileSync(
|
|
25
|
+
process.execPath,
|
|
26
|
+
[path.join(repo, "scripts", script), ...args],
|
|
27
|
+
{
|
|
28
|
+
cwd: repo,
|
|
29
|
+
stdio: "ignore",
|
|
30
|
+
}
|
|
31
|
+
);
|
|
32
|
+
if (native) run("check-native.mjs", "--rebuild");
|
|
29
33
|
run("gen-migrations-index.mjs");
|
|
30
34
|
return spawn(process.execPath, ["--import", "tsx", "server.ts"], {
|
|
31
35
|
cwd: repo,
|
package/lib/doctor/checks.ts
CHANGED
|
@@ -11,7 +11,7 @@ import { getAllProviderDefinitions } from "../providers/registry";
|
|
|
11
11
|
|
|
12
12
|
// The oldest Node major AgentOS runs on; scripts/lib/prerequisites.sh must agree
|
|
13
13
|
// (lib/doctor/drift.test.ts fails when they don't).
|
|
14
|
-
export const MIN_NODE_MAJOR =
|
|
14
|
+
export const MIN_NODE_MAJOR = 22;
|
|
15
15
|
|
|
16
16
|
const TOOL_MS = 5_000;
|
|
17
17
|
const CACHE_MS = 60 * 1000;
|
package/lib/env-setup.ts
CHANGED
|
@@ -22,6 +22,8 @@ export interface SetupStep {
|
|
|
22
22
|
success: boolean;
|
|
23
23
|
output?: string;
|
|
24
24
|
error?: string;
|
|
25
|
+
// The step worked, but the agent should know something about it.
|
|
26
|
+
warning?: string;
|
|
25
27
|
}
|
|
26
28
|
|
|
27
29
|
// The stages a worktree's setup goes through, in order, for progress.
|
package/lib/files.ts
CHANGED
package/lib/health/facts.ts
CHANGED
|
@@ -76,7 +76,12 @@ function queue(): QueueFact[] {
|
|
|
76
76
|
|
|
77
77
|
// Background jobs still running in a chat whose agent is alive, that have
|
|
78
78
|
// been going long enough to be worth a look at their output.
|
|
79
|
-
|
|
79
|
+
export interface BgTask extends BgFact {
|
|
80
|
+
outputFile?: string;
|
|
81
|
+
taskType?: string;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function runningBgTasks(now: number, minAgeMs: number): BgTask[] {
|
|
80
85
|
const live = new Set(runningWorkers());
|
|
81
86
|
if (!live.size) return [];
|
|
82
87
|
const sessions = db
|
|
@@ -86,12 +91,12 @@ function backgroundJobs(now: number, c: HealthConfig): BgFact[] {
|
|
|
86
91
|
WHERE s.archived_at IS NULL AND s.view = 'chat'`
|
|
87
92
|
)
|
|
88
93
|
.all() as { id: string; name: string; workspace_id: string | null }[];
|
|
89
|
-
const out:
|
|
94
|
+
const out: BgTask[] = [];
|
|
90
95
|
for (const s of sessions) {
|
|
91
96
|
if (!live.has(s.id)) continue;
|
|
92
97
|
for (const t of itemsOfKind(s.id, "task")) {
|
|
93
98
|
if (t.status !== "running" || t.ambient) continue;
|
|
94
|
-
if (now - t.createdAt <
|
|
99
|
+
if (now - t.createdAt < minAgeMs) continue;
|
|
95
100
|
const tail = taskOutputTail(t.taskId, t.outputFile);
|
|
96
101
|
out.push({
|
|
97
102
|
sessionId: s.id,
|
|
@@ -101,12 +106,17 @@ function backgroundJobs(now: number, c: HealthConfig): BgFact[] {
|
|
|
101
106
|
description: t.description,
|
|
102
107
|
startedAt: t.createdAt,
|
|
103
108
|
outputAt: tail?.updatedAt ?? null,
|
|
109
|
+
outputFile: t.outputFile,
|
|
110
|
+
taskType: t.taskType,
|
|
104
111
|
});
|
|
105
112
|
}
|
|
106
113
|
}
|
|
107
114
|
return out;
|
|
108
115
|
}
|
|
109
116
|
|
|
117
|
+
const backgroundJobs = (now: number, c: HealthConfig): BgFact[] =>
|
|
118
|
+
runningBgTasks(now, c.bgQuietMinutes * MIN);
|
|
119
|
+
|
|
110
120
|
// Every running background job, for the heartbeat a restart reads back.
|
|
111
121
|
export function liveBackgroundJobs(): string[] {
|
|
112
122
|
const live = new Set(runningWorkers());
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The I/O half of bg-spin (spin.ts decides): once a minute, for each old
|
|
3
|
+
* enough background job of a live chat, find its process tree, sample the CPU
|
|
4
|
+
* of its shells, and stop a job that spins. A stop is by PID, the job's own
|
|
5
|
+
* tree only: SIGTERM, then SIGKILL for what is still the same process after a
|
|
6
|
+
* grace. Never a pattern, never a group, never AgentOS, tmux or the agent.
|
|
7
|
+
* With no old background job there is no process I/O at all.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { execFile } from "child_process";
|
|
11
|
+
import fs from "fs";
|
|
12
|
+
import { sendMessage } from "../bus";
|
|
13
|
+
import { findOutputFile } from "../chat/task-output";
|
|
14
|
+
import { queueEventOnce } from "../orchestrator/events";
|
|
15
|
+
import { untrusted } from "../orchestrator/untrusted";
|
|
16
|
+
import { processTable, type ProcRow } from "../process-table";
|
|
17
|
+
import { runningBgTasks, type BgTask } from "./facts";
|
|
18
|
+
import { close, recordTrips } from "./store";
|
|
19
|
+
import {
|
|
20
|
+
KILL_GRACE_MS,
|
|
21
|
+
SPIN_KEEP_MS,
|
|
22
|
+
SPIN_MIN_AGE_MS,
|
|
23
|
+
baseOf,
|
|
24
|
+
classifySpin,
|
|
25
|
+
jobTree,
|
|
26
|
+
killPlan,
|
|
27
|
+
parseCpuTime,
|
|
28
|
+
pickRoot,
|
|
29
|
+
safeToKill,
|
|
30
|
+
shellPids,
|
|
31
|
+
spinEventLine,
|
|
32
|
+
spinNote,
|
|
33
|
+
type SpinSample,
|
|
34
|
+
} from "./spin";
|
|
35
|
+
import type { Trip } from "./types";
|
|
36
|
+
|
|
37
|
+
export interface SpinDeps {
|
|
38
|
+
tasks: (now: number) => BgTask[];
|
|
39
|
+
// Output file -> the PIDs holding it open.
|
|
40
|
+
holders: (files: string[]) => Promise<Map<string, number[]>>;
|
|
41
|
+
table: () => Promise<ProcRow[] | null>;
|
|
42
|
+
cpuSeconds: (pids: number[]) => Promise<Map<number, number>>;
|
|
43
|
+
kill: (pid: number, signal: "SIGTERM" | "SIGKILL") => void;
|
|
44
|
+
sleep: (ms: number) => Promise<void>;
|
|
45
|
+
tell: (sessionId: string, body: string) => Promise<void>;
|
|
46
|
+
event: (
|
|
47
|
+
workspaceId: string,
|
|
48
|
+
key: string,
|
|
49
|
+
subject: string,
|
|
50
|
+
line: string,
|
|
51
|
+
now: number
|
|
52
|
+
) => void;
|
|
53
|
+
record: (trip: Trip, now: number) => void;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const run = (cmd: string, args: string[]): Promise<string> =>
|
|
57
|
+
new Promise((resolve) =>
|
|
58
|
+
execFile(cmd, args, { timeout: 5000, maxBuffer: 4 << 20 }, (_err, out) =>
|
|
59
|
+
resolve(out ?? "")
|
|
60
|
+
)
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
// `lsof -F pn`: a `p<pid>` line, then `n<name>` for each file it holds.
|
|
64
|
+
export function parseHolders(out: string): Map<string, number[]> {
|
|
65
|
+
const map = new Map<string, number[]>();
|
|
66
|
+
let pid = 0;
|
|
67
|
+
for (const line of out.split("\n")) {
|
|
68
|
+
if (line.startsWith("p")) pid = Number(line.slice(1)) || 0;
|
|
69
|
+
else if (line.startsWith("n") && pid) {
|
|
70
|
+
const list = map.get(line.slice(1)) ?? [];
|
|
71
|
+
if (!list.includes(pid)) list.push(pid);
|
|
72
|
+
map.set(line.slice(1), list);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return map;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Linux counts reaped children in /proc; macOS ps's TIME already includes them.
|
|
79
|
+
function procCpu(pid: number): number | null {
|
|
80
|
+
try {
|
|
81
|
+
const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
82
|
+
const f = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
|
|
83
|
+
return (
|
|
84
|
+
(Number(f[11]) + Number(f[12]) + Number(f[13]) + Number(f[14])) / 100
|
|
85
|
+
);
|
|
86
|
+
} catch {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async function cpuSeconds(pids: number[]): Promise<Map<number, number>> {
|
|
92
|
+
const out = new Map<number, number>();
|
|
93
|
+
if (!pids.length) return out;
|
|
94
|
+
if (process.platform === "linux") {
|
|
95
|
+
for (const pid of pids) {
|
|
96
|
+
const sec = procCpu(pid);
|
|
97
|
+
if (sec !== null) out.set(pid, sec);
|
|
98
|
+
}
|
|
99
|
+
return out;
|
|
100
|
+
}
|
|
101
|
+
const text = await run("ps", ["-o", "pid=,time=", "-p", pids.join(",")]);
|
|
102
|
+
for (const line of text.split("\n")) {
|
|
103
|
+
const m = /^\s*(\d+)\s+(\S+)\s*$/.exec(line);
|
|
104
|
+
const sec = m ? parseCpuTime(m[2]) : null;
|
|
105
|
+
if (m && sec !== null) out.set(Number(m[1]), sec);
|
|
106
|
+
}
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
export const realSpin: SpinDeps = {
|
|
111
|
+
tasks: (now) =>
|
|
112
|
+
runningBgTasks(now, SPIN_MIN_AGE_MS).map((t) => ({
|
|
113
|
+
...t,
|
|
114
|
+
outputFile: t.outputFile ?? findOutputFile(t.taskId) ?? undefined,
|
|
115
|
+
})),
|
|
116
|
+
holders: async (files) =>
|
|
117
|
+
parseHolders(await run("lsof", ["-F", "pn", "--", ...files])),
|
|
118
|
+
table: () => processTable(0),
|
|
119
|
+
cpuSeconds,
|
|
120
|
+
kill: (pid, signal) => process.kill(pid, signal),
|
|
121
|
+
sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
|
|
122
|
+
tell: async (sessionId, body) => {
|
|
123
|
+
await sendMessage({
|
|
124
|
+
fromId: null,
|
|
125
|
+
to: sessionId,
|
|
126
|
+
body,
|
|
127
|
+
fromLabel: "AgentOS health",
|
|
128
|
+
origin: { kind: "system", label: "Health", body },
|
|
129
|
+
});
|
|
130
|
+
},
|
|
131
|
+
event: (ws, key, subject, line, now) => {
|
|
132
|
+
queueEventOnce(ws, key, subject, line, now);
|
|
133
|
+
},
|
|
134
|
+
record: (trip, now) => {
|
|
135
|
+
const { opened } = recordTrips([trip], [], now);
|
|
136
|
+
// History only: nothing to wake a watcher for.
|
|
137
|
+
for (const row of opened) close(row.id, "stopped", now);
|
|
138
|
+
},
|
|
139
|
+
};
|
|
140
|
+
|
|
141
|
+
interface Track {
|
|
142
|
+
root: number;
|
|
143
|
+
comm: string;
|
|
144
|
+
samples: SpinSample[];
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const tracks = new Map<string, Track>();
|
|
148
|
+
let running = false;
|
|
149
|
+
|
|
150
|
+
export function resetSpin(): void {
|
|
151
|
+
tracks.clear();
|
|
152
|
+
running = false;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const keyOf = (t: BgTask) => `${t.sessionId}:${t.taskId}`;
|
|
156
|
+
|
|
157
|
+
// SIGTERM the whole job, root first; SIGKILL what is still the same process.
|
|
158
|
+
async function stopTree(
|
|
159
|
+
root: number,
|
|
160
|
+
table: ProcRow[],
|
|
161
|
+
deps: SpinDeps
|
|
162
|
+
): Promise<number[]> {
|
|
163
|
+
const plan = killPlan(root, table);
|
|
164
|
+
const comm = new Map(table.map((r) => [r.pid, r.comm]));
|
|
165
|
+
const signal = (pid: number, sig: "SIGTERM" | "SIGKILL") => {
|
|
166
|
+
try {
|
|
167
|
+
deps.kill(pid, sig);
|
|
168
|
+
} catch {
|
|
169
|
+
// Already gone.
|
|
170
|
+
}
|
|
171
|
+
};
|
|
172
|
+
for (const pid of plan) signal(pid, "SIGTERM");
|
|
173
|
+
await deps.sleep(KILL_GRACE_MS);
|
|
174
|
+
const after = new Map((await deps.table())?.map((r) => [r.pid, r.comm]));
|
|
175
|
+
for (const pid of plan)
|
|
176
|
+
if (after.has(pid) && after.get(pid) === comm.get(pid))
|
|
177
|
+
signal(pid, "SIGKILL");
|
|
178
|
+
return plan;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
async function sampleAll(
|
|
182
|
+
jobs: BgTask[],
|
|
183
|
+
now: number,
|
|
184
|
+
deps: SpinDeps
|
|
185
|
+
): Promise<{ job: BgTask; track: Track; table: ProcRow[] }[]> {
|
|
186
|
+
const table = await deps.table();
|
|
187
|
+
if (!table) return [];
|
|
188
|
+
const need = jobs.filter((j) => !tracks.get(keyOf(j)) && j.outputFile);
|
|
189
|
+
if (need.length) {
|
|
190
|
+
const held = await deps.holders(need.map((j) => j.outputFile as string));
|
|
191
|
+
for (const j of need) {
|
|
192
|
+
const root = pickRoot(held.get(j.outputFile as string) ?? [], table);
|
|
193
|
+
const row = table.find((r) => r.pid === root);
|
|
194
|
+
if (row)
|
|
195
|
+
tracks.set(keyOf(j), { root: row.pid, comm: row.comm, samples: [] });
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
const live: { job: BgTask; track: Track; table: ProcRow[] }[] = [];
|
|
199
|
+
for (const j of jobs) {
|
|
200
|
+
const track = tracks.get(keyOf(j));
|
|
201
|
+
if (!track) continue;
|
|
202
|
+
const row = table.find((r) => r.pid === track.root);
|
|
203
|
+
// Gone, or the PID is someone else's now.
|
|
204
|
+
if (!row || row.comm !== track.comm) {
|
|
205
|
+
tracks.delete(keyOf(j));
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
live.push({ job: j, track, table });
|
|
209
|
+
}
|
|
210
|
+
const pids = live.flatMap((l) => shellPids(jobTree(l.track.root, table)));
|
|
211
|
+
const cpu = await deps.cpuSeconds(pids);
|
|
212
|
+
for (const { job, track } of live) {
|
|
213
|
+
const sec = shellPids(jobTree(track.root, table)).reduce(
|
|
214
|
+
(sum, pid) => sum + (cpu.get(pid) ?? 0),
|
|
215
|
+
0
|
|
216
|
+
);
|
|
217
|
+
track.samples.push({ at: now, cpuSec: sec, outputAt: job.outputAt });
|
|
218
|
+
track.samples = track.samples.filter((s) => now - s.at <= SPIN_KEEP_MS);
|
|
219
|
+
}
|
|
220
|
+
return live;
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
export async function spinTick(
|
|
224
|
+
now = Date.now(),
|
|
225
|
+
deps: SpinDeps = realSpin
|
|
226
|
+
): Promise<void> {
|
|
227
|
+
if (running) return;
|
|
228
|
+
running = true;
|
|
229
|
+
try {
|
|
230
|
+
// Shell jobs only: a subagent task has no shell to spin.
|
|
231
|
+
const jobs = deps
|
|
232
|
+
.tasks(now)
|
|
233
|
+
.filter((t) => !t.taskType || t.taskType === "local_bash");
|
|
234
|
+
const names = new Set(jobs.map(keyOf));
|
|
235
|
+
for (const k of tracks.keys()) if (!names.has(k)) tracks.delete(k);
|
|
236
|
+
if (!jobs.length) return;
|
|
237
|
+
for (const { job, track, table } of await sampleAll(jobs, now, deps)) {
|
|
238
|
+
if (classifySpin(track.samples) !== "spinning") continue;
|
|
239
|
+
if (!safeToKill(track.root, table)) {
|
|
240
|
+
console.warn(
|
|
241
|
+
`[health] bg-spin: not stopping pid ${track.root} (${baseOf(track.comm)}) in ${job.sessionName}: it is not a plain job of the agent`
|
|
242
|
+
);
|
|
243
|
+
continue;
|
|
244
|
+
}
|
|
245
|
+
const pids = await stopTree(track.root, table, deps);
|
|
246
|
+
tracks.delete(keyOf(job));
|
|
247
|
+
console.warn(
|
|
248
|
+
`[health] bg-spin: stopped "${job.description}" in ${job.sessionName} (session ${job.sessionId}): pid ${track.root}, tree ${pids.join(",")}`
|
|
249
|
+
);
|
|
250
|
+
await announce(job, track.root, now, deps);
|
|
251
|
+
}
|
|
252
|
+
} finally {
|
|
253
|
+
running = false;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
async function announce(
|
|
258
|
+
job: BgTask,
|
|
259
|
+
pid: number,
|
|
260
|
+
now: number,
|
|
261
|
+
deps: SpinDeps
|
|
262
|
+
): Promise<void> {
|
|
263
|
+
const line = spinEventLine(job.sessionName, job.description, pid);
|
|
264
|
+
try {
|
|
265
|
+
deps.record(
|
|
266
|
+
{
|
|
267
|
+
check: "bg-spin",
|
|
268
|
+
subject: keyOf(job),
|
|
269
|
+
severity: "high",
|
|
270
|
+
line,
|
|
271
|
+
detail: { sessionId: job.sessionId, taskId: job.taskId, pid },
|
|
272
|
+
workspaceId: job.workspaceId,
|
|
273
|
+
},
|
|
274
|
+
now
|
|
275
|
+
);
|
|
276
|
+
} catch (error) {
|
|
277
|
+
console.error("[health] bg-spin record:", error);
|
|
278
|
+
}
|
|
279
|
+
if (job.workspaceId)
|
|
280
|
+
deps.event(
|
|
281
|
+
job.workspaceId,
|
|
282
|
+
`bg-spin:${keyOf(job)}`,
|
|
283
|
+
job.sessionId,
|
|
284
|
+
// The description is the agent's own text.
|
|
285
|
+
`Health (tasks): ${untrusted("health detector", line)}`,
|
|
286
|
+
now
|
|
287
|
+
);
|
|
288
|
+
try {
|
|
289
|
+
await deps.tell(job.sessionId, spinNote(job.description, pid));
|
|
290
|
+
} catch (error) {
|
|
291
|
+
console.error("[health] bg-spin note:", error);
|
|
292
|
+
}
|
|
293
|
+
}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* bg-spin: a background job whose shell burns a core with nothing printed is a
|
|
3
|
+
* wait loop that can never exit (`until grep -q "" /dev/null; do :; done`).
|
|
4
|
+
* The pure half: deciding it, picking what to stop, and the words about it.
|
|
5
|
+
* No I/O here; spin-tick.ts reads the processes and stops them.
|
|
6
|
+
*
|
|
7
|
+
* Only the CPU of shells in the job's tree is counted. A loop's work happens
|
|
8
|
+
* in the shell itself (builtins, or children reaped into its total), while a
|
|
9
|
+
* build's lives in node, cc or cargo, so a silent heavy build never reads as
|
|
10
|
+
* spinning.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import path from "path";
|
|
14
|
+
import { WAIT_ADVICE } from "../agents/brief";
|
|
15
|
+
import type { ProcRow } from "../process-table";
|
|
16
|
+
import { MIN } from "./types";
|
|
17
|
+
|
|
18
|
+
// Share of one core, every interval of the window.
|
|
19
|
+
export const SPIN_CORES = 0.8;
|
|
20
|
+
// How long it has to hold, with nothing new in the output.
|
|
21
|
+
export const SPIN_WINDOW_MS = 4 * MIN;
|
|
22
|
+
// Younger jobs aren't sampled.
|
|
23
|
+
export const SPIN_MIN_AGE_MS = SPIN_WINDOW_MS;
|
|
24
|
+
// A longer gap between samples (missed ticks, a sleeping laptop) starts over.
|
|
25
|
+
export const SPIN_MAX_GAP_MS = 3 * MIN;
|
|
26
|
+
// SIGTERM, then SIGKILL for whatever is still there.
|
|
27
|
+
export const KILL_GRACE_MS = 3000;
|
|
28
|
+
// Samples older than this are no use to the decision.
|
|
29
|
+
export const SPIN_KEEP_MS = SPIN_WINDOW_MS + 2 * MIN;
|
|
30
|
+
|
|
31
|
+
const SHELLS = new Set(["sh", "bash", "zsh", "dash", "fish", "ksh"]);
|
|
32
|
+
const AGENTS = new Set(["claude"]);
|
|
33
|
+
// Never part of a job's tree to stop.
|
|
34
|
+
const NEVER = new Set(["tmux", "claude"]);
|
|
35
|
+
|
|
36
|
+
export const baseOf = (comm: string): string =>
|
|
37
|
+
path.basename(comm).replace(/^-/, "");
|
|
38
|
+
|
|
39
|
+
export interface SpinSample {
|
|
40
|
+
at: number;
|
|
41
|
+
// Cumulative CPU seconds of the tree's shells.
|
|
42
|
+
cpuSec: number;
|
|
43
|
+
// The output file's last write, if it has one.
|
|
44
|
+
outputAt: number | null;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export type SpinClass =
|
|
48
|
+
| "warming"
|
|
49
|
+
| "sleeping"
|
|
50
|
+
| "busy-with-output"
|
|
51
|
+
| "spinning";
|
|
52
|
+
|
|
53
|
+
export function classifySpin(samples: SpinSample[]): SpinClass {
|
|
54
|
+
if (samples.length < 2) return "warming";
|
|
55
|
+
// The unbroken run of samples ending now.
|
|
56
|
+
let from = samples.length - 1;
|
|
57
|
+
while (from > 0 && samples[from].at - samples[from - 1].at <= SPIN_MAX_GAP_MS)
|
|
58
|
+
from--;
|
|
59
|
+
const run = samples.slice(from);
|
|
60
|
+
const last = run[run.length - 1];
|
|
61
|
+
// Start at the newest sample at least a window old.
|
|
62
|
+
let start = -1;
|
|
63
|
+
for (let i = 0; i < run.length; i++)
|
|
64
|
+
if (last.at - run[i].at >= SPIN_WINDOW_MS) start = i;
|
|
65
|
+
if (start < 0) return "warming";
|
|
66
|
+
const win = run.slice(start);
|
|
67
|
+
for (let i = 1; i < win.length; i++) {
|
|
68
|
+
const wall = (win[i].at - win[i - 1].at) / 1000;
|
|
69
|
+
const cpu = Math.max(0, win[i].cpuSec - win[i - 1].cpuSec);
|
|
70
|
+
if (wall <= 0 || cpu / wall < SPIN_CORES) return "sleeping";
|
|
71
|
+
}
|
|
72
|
+
const wrote = win.some((s) => s.outputAt !== null && s.outputAt > win[0].at);
|
|
73
|
+
return wrote ? "busy-with-output" : "spinning";
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// Of the processes holding a task's output file open, the job's root: the
|
|
77
|
+
// one whose parent doesn't hold it too.
|
|
78
|
+
export function pickRoot(holders: number[], table: ProcRow[]): number | null {
|
|
79
|
+
const held = new Set(holders);
|
|
80
|
+
const roots = table.filter((r) => held.has(r.pid) && !held.has(r.ppid));
|
|
81
|
+
return roots.length === 1 ? roots[0].pid : null;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
export function jobTree(root: number, table: ProcRow[]): ProcRow[] {
|
|
85
|
+
const kids = new Map<number, ProcRow[]>();
|
|
86
|
+
for (const r of table) {
|
|
87
|
+
const list = kids.get(r.ppid);
|
|
88
|
+
if (list) list.push(r);
|
|
89
|
+
else kids.set(r.ppid, [r]);
|
|
90
|
+
}
|
|
91
|
+
const byPid = new Map(table.map((r) => [r.pid, r]));
|
|
92
|
+
const out: ProcRow[] = [];
|
|
93
|
+
const seen = new Set<number>();
|
|
94
|
+
const stack = [byPid.get(root)].filter((r): r is ProcRow => !!r);
|
|
95
|
+
while (stack.length) {
|
|
96
|
+
const at = stack.shift() as ProcRow;
|
|
97
|
+
if (seen.has(at.pid)) continue;
|
|
98
|
+
seen.add(at.pid);
|
|
99
|
+
out.push(at);
|
|
100
|
+
stack.push(...(kids.get(at.pid) ?? []));
|
|
101
|
+
}
|
|
102
|
+
return out;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export const shellPids = (tree: ProcRow[]): number[] =>
|
|
106
|
+
tree.filter((r) => SHELLS.has(baseOf(r.comm))).map((r) => r.pid);
|
|
107
|
+
|
|
108
|
+
// Whether this tree is a background job of an agent and nothing else: a
|
|
109
|
+
// shell directly under the agent, with nothing of AgentOS, tmux or the agent
|
|
110
|
+
// inside it.
|
|
111
|
+
export function safeToKill(
|
|
112
|
+
root: number,
|
|
113
|
+
table: ProcRow[],
|
|
114
|
+
protectedPids: number[] = [process.pid, process.ppid]
|
|
115
|
+
): boolean {
|
|
116
|
+
const head = table.find((r) => r.pid === root);
|
|
117
|
+
if (!head || root <= 1) return false;
|
|
118
|
+
if (!SHELLS.has(baseOf(head.comm))) return false;
|
|
119
|
+
const parent = table.find((r) => r.pid === head.ppid);
|
|
120
|
+
if (!parent || !AGENTS.has(baseOf(parent.comm))) return false;
|
|
121
|
+
return !jobTree(root, table).some(
|
|
122
|
+
(r) =>
|
|
123
|
+
protectedPids.includes(r.pid) ||
|
|
124
|
+
r.pid === parent.pid ||
|
|
125
|
+
NEVER.has(baseOf(r.comm))
|
|
126
|
+
);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// The PIDs to signal: the root first, so a loop can't respawn children, then
|
|
130
|
+
// everything under it.
|
|
131
|
+
export const killPlan = (root: number, table: ProcRow[]): number[] =>
|
|
132
|
+
jobTree(root, table).map((r) => r.pid);
|
|
133
|
+
|
|
134
|
+
// ps's TIME column: [[D-]H:]M:SS[.cc] to seconds.
|
|
135
|
+
export function parseCpuTime(text: string): number | null {
|
|
136
|
+
const m = /^\s*(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)\s*$/.exec(text);
|
|
137
|
+
if (!m) return null;
|
|
138
|
+
return (
|
|
139
|
+
Number(m[1] ?? 0) * 86400 +
|
|
140
|
+
Number(m[2] ?? 0) * 3600 +
|
|
141
|
+
Number(m[3]) * 60 +
|
|
142
|
+
Number(m[4])
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export function spinNote(description: string, pid: number): string {
|
|
147
|
+
const mins = Math.round(SPIN_WINDOW_MS / MIN);
|
|
148
|
+
return `AgentOS stopped your background job "${description}" (PID ${pid}): its shell used about a core for ${mins}+ minutes with no output, which is a wait loop that can't exit. ${WAIT_ADVICE}`;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export function spinEventLine(
|
|
152
|
+
sessionName: string,
|
|
153
|
+
description: string,
|
|
154
|
+
pid: number
|
|
155
|
+
): string {
|
|
156
|
+
const mins = Math.round(SPIN_WINDOW_MS / MIN);
|
|
157
|
+
return `Stopped background job "${description}" (PID ${pid}) in ${sessionName}: a shell wait loop holding a core for ${mins}+ minutes with no output. The task was told.`;
|
|
158
|
+
}
|