atris 3.70.0 → 3.71.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -0
- package/atris/skills/README.md +1 -0
- package/atris/skills/campaigns/SKILL.md +81 -0
- package/atris/skills/meta/SKILL.md +52 -8
- package/ax +226 -42
- package/bin/atris.js +38 -8
- package/commands/autoland.js +176 -7
- package/commands/autopilot.js +71 -11
- package/commands/campaign.js +779 -0
- package/commands/compile.js +19 -8
- package/commands/doctor.js +58 -0
- package/commands/dream.js +18 -3
- package/commands/engine.js +147 -5
- package/commands/improve.js +119 -9
- package/commands/init.js +18 -1
- package/commands/integrations.js +17 -5
- package/commands/interview.js +15 -3
- package/commands/land.js +22 -6
- package/commands/link.js +1 -0
- package/commands/member.js +359 -94
- package/commands/mission.js +249 -36
- package/commands/report.js +42 -8
- package/commands/run.js +9 -2
- package/commands/scout.js +9 -1
- package/commands/task.js +71 -17
- package/commands/team.js +212 -4
- package/commands/unknowns.js +15 -1
- package/commands/verify.js +46 -0
- package/commands/worktree.js +108 -15
- package/commands/write.js +12 -1
- package/commands/youtube.js +17 -2
- package/lib/accept-verify-gate.js +1 -0
- package/lib/account-bound.js +13 -0
- package/lib/auto-accept-certified.js +1 -0
- package/lib/autoland.js +13 -4
- package/lib/campaign-cadence.js +131 -0
- package/lib/campaign-core.js +330 -0
- package/lib/campaign-holdout.js +65 -0
- package/lib/campaign-loop.js +246 -0
- package/lib/campaign-missions.js +237 -0
- package/lib/campaign-new.js +251 -0
- package/lib/campaign-protect.js +75 -0
- package/lib/campaign-reading.js +147 -0
- package/lib/campaign-run-lock.js +59 -0
- package/lib/campaign-schema.js +196 -0
- package/lib/campaign-store.js +380 -0
- package/lib/campaign-sync.js +302 -0
- package/lib/campaign-trace.js +300 -0
- package/lib/conductor-artifacts.js +15 -1
- package/lib/daily-log.js +11 -1
- package/lib/dispatch-scout.js +18 -0
- package/lib/engine-ask.js +33 -3
- package/lib/engine-registry.js +2 -0
- package/lib/engine-resource.js +518 -0
- package/lib/fleet.js +125 -16
- package/lib/init-commit.js +80 -0
- package/lib/intents.js +8 -0
- package/lib/known-commands.js +1 -1
- package/lib/land-green.js +58 -7
- package/lib/landing-proof.js +352 -0
- package/lib/member-autoland.js +395 -18
- package/lib/member-board.js +7 -1
- package/lib/member-map-fold.js +126 -0
- package/lib/member-map-inbox.js +80 -0
- package/lib/member-progress.js +48 -1
- package/lib/member-report.js +127 -8
- package/lib/member-self-next-job.js +97 -0
- package/lib/member-task-pick.js +41 -14
- package/lib/member-up.js +249 -16
- package/lib/roster-prep.js +25 -2
- package/lib/runner-command.js +5 -0
- package/lib/task-db.js +47 -11
- package/lib/task-explanation.js +7 -0
- package/lib/task-list-keeper.js +8 -1
- package/lib/task-receipt.js +6 -1
- package/mcp/atris-mcp/index.mjs +212 -10
- package/package.json +2 -1
- package/scripts/det/engine-reading.js +37 -0
- package/scripts/det/ytnotes +29 -5
package/lib/dispatch-scout.js
CHANGED
|
@@ -331,10 +331,28 @@ function verifyScoutPack(rawPack, { task, worktreePath, allowedFiles, expectedCo
|
|
|
331
331
|
return Buffer.byteLength(JSON.stringify(pack)) <= SCOUT_MAX_PACK_BYTES ? pack : null;
|
|
332
332
|
}
|
|
333
333
|
|
|
334
|
+
// A task whose title already names the file it changes, by a path that is in
|
|
335
|
+
// the repo or sits in one of its folders, gains nothing from a scout: the
|
|
336
|
+
// engine knows where to start. Measured 2026-10-15 on five one-file tasks: the
|
|
337
|
+
// scout ran 21 to 118 seconds before each engine, longer than the engine's own
|
|
338
|
+
// 37 to 53 second build, and pushed the flight from about 95 to 196 seconds.
|
|
339
|
+
const NAMED_PATH_PATTERN = /(?:^|[\s"'`(])((?:[\w.-]+\/)+[\w.-]+|[\w-]+\.[A-Za-z0-9]{1,8})(?=$|[\s"'`),:;]|\.(?:\s|$))/g;
|
|
340
|
+
|
|
341
|
+
function titleNamesItsFiles(task, worktreePath) {
|
|
342
|
+
const title = String(task && task.title || '');
|
|
343
|
+
const named = [...title.matchAll(NAMED_PATH_PATTERN)].map((match) => match[1]);
|
|
344
|
+
if (!named.length) return false;
|
|
345
|
+
const tracked = trackedFiles(worktreePath);
|
|
346
|
+
const files = new Set(tracked);
|
|
347
|
+
const folders = new Set(tracked.map((file) => path.posix.dirname(file)));
|
|
348
|
+
return named.every((ref) => files.has(ref) || (ref.includes('/') && folders.has(path.posix.dirname(ref))));
|
|
349
|
+
}
|
|
350
|
+
|
|
334
351
|
async function buildVerifiedScoutPack({ task, worktreePath, ask = defaultScoutAsk }) {
|
|
335
352
|
try {
|
|
336
353
|
const commit = checkoutCommit(worktreePath);
|
|
337
354
|
if (!commit) return null;
|
|
355
|
+
if (titleNamesItsFiles(task, worktreePath)) return null;
|
|
338
356
|
const seed = seedScoutContext({ task, worktreePath });
|
|
339
357
|
if (seed.allowedFiles.length < 2) return null;
|
|
340
358
|
const job = {
|
package/lib/engine-ask.js
CHANGED
|
@@ -21,6 +21,7 @@ const {
|
|
|
21
21
|
createEngineLiveLog,
|
|
22
22
|
engineTerminalStatus,
|
|
23
23
|
} = require('./engine-job-lifecycle');
|
|
24
|
+
const { recordEngineResource } = require('./engine-resource');
|
|
24
25
|
|
|
25
26
|
const DEFAULT_ASK_CONCURRENCY = 3;
|
|
26
27
|
const DEFAULT_ASK_TIMEOUT_MS = 120000;
|
|
@@ -278,8 +279,9 @@ function buildReadOnlyEngineInvocation(engineName, prompt, modelName = '', {
|
|
|
278
279
|
const request = guardedPrompt(prompt);
|
|
279
280
|
const effortArgs = askEffortArgs(engine, effort);
|
|
280
281
|
|
|
281
|
-
if (engine === 'atris-fast' || engine === 'composer') {
|
|
282
|
-
|
|
282
|
+
if (engine === 'atris-fast' || engine === 'composer' || engine === 'mercury') {
|
|
283
|
+
const lane = engine === 'mercury' ? '--mercury' : '--fast';
|
|
284
|
+
return { engine, bin: profile.bin, args: [lane, '--cloud', '--print', request] };
|
|
283
285
|
}
|
|
284
286
|
if (engine === 'claude' || engine === 'fable' || engine === 'haiku') {
|
|
285
287
|
return {
|
|
@@ -545,10 +547,37 @@ async function runEngineAskJobs(jobs, {
|
|
|
545
547
|
} = {}) {
|
|
546
548
|
const answers = new Array(jobs.length);
|
|
547
549
|
let nextIndex = 0;
|
|
548
|
-
const
|
|
550
|
+
const launch = executeAskJob || (async (job) => {
|
|
549
551
|
const invocation = buildReadOnlyEngineInvocation(job.engine, job.prompt, job.model, { timeoutMs, effort: job.effort || '' });
|
|
550
552
|
return runAskProcess(invocation, { cwd: invocation.cwd || root, timeoutMs, signal, onOutputChunk });
|
|
551
553
|
});
|
|
554
|
+
// Machine memory and load at both ends, like a dispatch, so a killed ask
|
|
555
|
+
// still says whether the machine was short on memory. This wraps a
|
|
556
|
+
// caller's own launcher too (link, the dispatch scout); one that throws
|
|
557
|
+
// still gets its end line before the error passes on.
|
|
558
|
+
const execute = async (job, context) => {
|
|
559
|
+
const resource = { engine: canonicalEngineName(job.engine) || job.engine, task: `ask ${job.label || job.engine}`, run_id: crypto.randomUUID() };
|
|
560
|
+
recordEngineResource(root, { phase: 'start', ...resource });
|
|
561
|
+
const startedAt = Date.now();
|
|
562
|
+
const writeEnd = (ran, error = '') => recordEngineResource(root, {
|
|
563
|
+
phase: 'end',
|
|
564
|
+
...resource,
|
|
565
|
+
signal: ran.signal || null,
|
|
566
|
+
timed_out: ran.timed_out,
|
|
567
|
+
error,
|
|
568
|
+
duration_ms: Number.isFinite(ran.duration_ms) ? ran.duration_ms : Date.now() - startedAt,
|
|
569
|
+
});
|
|
570
|
+
let result;
|
|
571
|
+
try {
|
|
572
|
+
result = await launch(job, context);
|
|
573
|
+
} catch (error) {
|
|
574
|
+
writeEnd({}, String(error && error.message || error).slice(-2000));
|
|
575
|
+
throw error;
|
|
576
|
+
}
|
|
577
|
+
// A SIGKILL's cause goes on the answer so the failure line can name it.
|
|
578
|
+
const end = writeEnd(result || {});
|
|
579
|
+
return end && end.cause && !(result && result.timed_out) ? { ...result, kill_cause: end.cause } : result;
|
|
580
|
+
};
|
|
552
581
|
const workerCount = Math.min(concurrency, jobs.length);
|
|
553
582
|
|
|
554
583
|
async function work() {
|
|
@@ -700,6 +729,7 @@ function createEngineAskReceipt(root, receipt) {
|
|
|
700
729
|
function compactFailure(answer) {
|
|
701
730
|
if (answer.reason === 'cancelled' || answer.cancelled) return 'cancelled';
|
|
702
731
|
if (answer.reason === 'timeout') return `timed out after ${Math.round(answer.duration_ms / 1000)}s`;
|
|
732
|
+
if (answer.kill_cause) return `stopped by ${answer.signal || 'SIGKILL'}; ${answer.kill_cause}`;
|
|
703
733
|
const detail = String(answer.stderr || '').trim().split(/\r?\n/).find(Boolean);
|
|
704
734
|
return detail || String(answer.reason || 'failed').replace(/_/g, ' ');
|
|
705
735
|
}
|
package/lib/engine-registry.js
CHANGED
|
@@ -160,6 +160,8 @@ function engineFailureHealthStatus(result) {
|
|
|
160
160
|
// fallback_order. Haiku searches first wherever the claude CLI is installed.
|
|
161
161
|
const ENGINE_SEED_META = Object.freeze({
|
|
162
162
|
'atris-fast': Object.freeze({ tier: 'fast', roles: Object.freeze(['navigator']), models: Object.freeze(['atris fast']), duty: 'learning', fallback_order: 10, hosted: true }),
|
|
163
|
+
// Test model on a capped free tier: a roster pick may use it, the router never hands it work.
|
|
164
|
+
mercury: Object.freeze({ tier: 'fast', roles: Object.freeze([]), roster_only_roles: Object.freeze(['navigator']), models: Object.freeze(['mercury 3 (test model)']), fallback_order: 100, hosted: true }),
|
|
163
165
|
// roster_only_roles: jobs a roster pick may give this engine, but the router
|
|
164
166
|
// never hands it on its own. Claude can own search when picked; with no
|
|
165
167
|
// pick, search goes to haiku, then other installed tools, then atris-fast.
|
|
@@ -0,0 +1,518 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// How the machine looked when an engine run started and ended: free memory,
|
|
4
|
+
// swap, and load, one JSON line per event in .atris/state/engine-resource.jsonl.
|
|
5
|
+
// A run killed with SIGKILL leaves no report of its own, so the end line is
|
|
6
|
+
// the only evidence for why. Memory fields are read on macOS only; elsewhere
|
|
7
|
+
// they stay null rather than a guess.
|
|
8
|
+
|
|
9
|
+
const fs = require('fs');
|
|
10
|
+
const os = require('os');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const { spawnSync } = require('child_process');
|
|
13
|
+
|
|
14
|
+
const RESOURCE_LOG = path.join('.atris', 'state', 'engine-resource.jsonl');
|
|
15
|
+
|
|
16
|
+
// Below this free percent, or above this share of swap in use, a SIGKILL is
|
|
17
|
+
// most likely the kernel reclaiming memory.
|
|
18
|
+
const LOW_FREE_PCT = 20;
|
|
19
|
+
const HIGH_SWAP_SHARE = 0.8;
|
|
20
|
+
|
|
21
|
+
// Tests that leave out a root default to the working folder, which is this
|
|
22
|
+
// checkout. Under the test runner, never write there, or test runs read back
|
|
23
|
+
// later as real engine kills.
|
|
24
|
+
const OWN_CHECKOUT = path.resolve(__dirname, '..');
|
|
25
|
+
|
|
26
|
+
// Keep start, end, and sweep evidence in the main project after a linked
|
|
27
|
+
// build folder is removed, using the same lookup as roster run history.
|
|
28
|
+
function resourceLogRoot(root) {
|
|
29
|
+
const { mainCheckoutRoot } = require('./engine-registry');
|
|
30
|
+
const main = mainCheckoutRoot(root);
|
|
31
|
+
return main && fs.existsSync(main) ? main : root;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function resourceLogFile(root) {
|
|
35
|
+
const project = resourceLogRoot(root);
|
|
36
|
+
if (process.env.NODE_TEST_CONTEXT && (path.resolve(root) === OWN_CHECKOUT
|
|
37
|
+
|| path.resolve(project) === path.resolve(resourceLogRoot(OWN_CHECKOUT)))) return null;
|
|
38
|
+
return path.join(project, RESOURCE_LOG);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// The log only grows, so once it passes MAX_LOG_BYTES it is cut back to its
|
|
42
|
+
// newest KEEP_LOG_BYTES, which covers the tail history reads. An older start
|
|
43
|
+
// line stays when its end was not cut with it, so a run still going, or one
|
|
44
|
+
// whose end survived, keeps its pair. Starts with no end ever, from runs that
|
|
45
|
+
// died with the parent, are capped so they cannot fill the log instead.
|
|
46
|
+
const MAX_LOG_BYTES = 4 * 1024 * 1024;
|
|
47
|
+
const KEEP_LOG_BYTES = 2 * 1024 * 1024;
|
|
48
|
+
const MAX_KEPT_STARTS = 200;
|
|
49
|
+
|
|
50
|
+
function compactResourceLog(file) {
|
|
51
|
+
if (fs.statSync(file).size <= MAX_LOG_BYTES) return;
|
|
52
|
+
const lines = fs.readFileSync(file, 'utf8').split('\n').filter(Boolean);
|
|
53
|
+
let cut = lines.length;
|
|
54
|
+
let kept = 0;
|
|
55
|
+
while (cut > 0 && kept + Buffer.byteLength(lines[cut - 1]) + 1 <= KEEP_LOG_BYTES) {
|
|
56
|
+
cut -= 1;
|
|
57
|
+
kept += Buffer.byteLength(lines[cut]) + 1;
|
|
58
|
+
}
|
|
59
|
+
const dropped = lines.slice(0, cut).map((line) => {
|
|
60
|
+
try { return { line, entry: JSON.parse(line) || {} }; } catch { return { line, entry: {} }; }
|
|
61
|
+
});
|
|
62
|
+
const endedHere = new Set(dropped.filter(({ entry }) => entry.phase === 'end' && entry.run_id).map(({ entry }) => entry.run_id));
|
|
63
|
+
const starts = dropped
|
|
64
|
+
.filter(({ entry }) => entry.phase === 'start' && entry.run_id && !endedHere.has(entry.run_id))
|
|
65
|
+
.slice(-MAX_KEPT_STARTS)
|
|
66
|
+
.map(({ line }) => line);
|
|
67
|
+
const temp = `${file}.${process.pid}.tmp`;
|
|
68
|
+
fs.writeFileSync(temp, [...starts, ...lines.slice(cut)].map((line) => `${line}\n`).join(''));
|
|
69
|
+
fs.renameSync(temp, file);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// A write failure is swallowed: recording the machine's state must never
|
|
73
|
+
// fail the run or sweep it describes.
|
|
74
|
+
function appendResourceLine(root, entry) {
|
|
75
|
+
const file = resourceLogFile(root);
|
|
76
|
+
if (!file) return;
|
|
77
|
+
try {
|
|
78
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
79
|
+
try { compactResourceLog(file); } catch {}
|
|
80
|
+
fs.appendFileSync(file, `${JSON.stringify(entry)}\n`);
|
|
81
|
+
} catch {}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Full paths, so a run launched from cron or launchd with a narrow PATH still
|
|
85
|
+
// reads swap and agent sockets. A tool missing from its usual folder falls
|
|
86
|
+
// back to the PATH lookup.
|
|
87
|
+
const SYSTEM_TOOLS = {
|
|
88
|
+
memory_pressure: '/usr/bin/memory_pressure',
|
|
89
|
+
sysctl: '/usr/sbin/sysctl',
|
|
90
|
+
ps: '/bin/ps',
|
|
91
|
+
lsof: '/usr/sbin/lsof',
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
function defaultRun(cmd, args) {
|
|
95
|
+
const full = SYSTEM_TOOLS[cmd];
|
|
96
|
+
return spawnSync(full && fs.existsSync(full) ? full : cmd, args, { encoding: 'utf8', timeout: 3000 });
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function parseFreePct(text) {
|
|
100
|
+
const match = /memory free percentage:\s*(\d+(?:\.\d+)?)%/i.exec(String(text || ''));
|
|
101
|
+
return match ? Number(match[1]) : null;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function parseSwap(text) {
|
|
105
|
+
const total = /total\s*=\s*(\d+(?:\.\d+)?)M/i.exec(String(text || ''));
|
|
106
|
+
const used = /used\s*=\s*(\d+(?:\.\d+)?)M/i.exec(String(text || ''));
|
|
107
|
+
return {
|
|
108
|
+
swap_used_mb: used ? Number(used[1]) : null,
|
|
109
|
+
swap_total_mb: total ? Number(total[1]) : null,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function readOutput(run, cmd, args) {
|
|
114
|
+
try {
|
|
115
|
+
const result = run(cmd, args);
|
|
116
|
+
return result && result.status === 0 ? String(result.stdout || '') : '';
|
|
117
|
+
} catch {
|
|
118
|
+
return '';
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function readMachineResources(deps = {}) {
|
|
123
|
+
const platform = deps.platform || process.platform;
|
|
124
|
+
const loadavg = deps.loadavg || os.loadavg;
|
|
125
|
+
const load = loadavg().map((n) => Math.round(n * 100) / 100);
|
|
126
|
+
if (platform !== 'darwin') return { free_pct: null, swap_used_mb: null, swap_total_mb: null, load };
|
|
127
|
+
const run = deps.run || defaultRun;
|
|
128
|
+
return {
|
|
129
|
+
free_pct: parseFreePct(readOutput(run, 'memory_pressure', ['-Q'])),
|
|
130
|
+
...parseSwap(readOutput(run, 'sysctl', ['vm.swapusage'])),
|
|
131
|
+
load,
|
|
132
|
+
};
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function memoryWasLow(snapshot) {
|
|
136
|
+
if (!snapshot) return false;
|
|
137
|
+
if (Number.isFinite(snapshot.free_pct) && snapshot.free_pct < LOW_FREE_PCT) return true;
|
|
138
|
+
const { swap_used_mb: used, swap_total_mb: total } = snapshot;
|
|
139
|
+
return Number.isFinite(used) && Number.isFinite(total) && total > 0 && used / total >= HIGH_SWAP_SHARE;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const LOW_MEMORY_CAUSE = 'killed (machine memory low)';
|
|
143
|
+
const UNKNOWN_CAUSE = 'killed (cause unknown)';
|
|
144
|
+
// The dispatch sent this kill itself when the run passed its time limit, so
|
|
145
|
+
// the cause is known and memory readings do not change it.
|
|
146
|
+
const TIME_LIMIT_CAUSE = 'killed (time limit reached)';
|
|
147
|
+
|
|
148
|
+
function killedCauseText(snapshot) {
|
|
149
|
+
return memoryWasLow(snapshot) ? LOW_MEMORY_CAUSE : UNKNOWN_CAUSE;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// Appends one line and returns it.
|
|
153
|
+
function recordEngineResource(root, { phase, engine, task, run_id, signal, duration_ms, timed_out, error } = {}, deps = {}) {
|
|
154
|
+
const now = deps.now || (() => new Date().toISOString());
|
|
155
|
+
const entry = {
|
|
156
|
+
at: now(),
|
|
157
|
+
phase: phase || 'start',
|
|
158
|
+
engine: engine || '',
|
|
159
|
+
task: task || '',
|
|
160
|
+
...readMachineResources(deps),
|
|
161
|
+
};
|
|
162
|
+
if (run_id) entry.run_id = run_id;
|
|
163
|
+
if (entry.phase === 'end' && Number.isFinite(duration_ms) && duration_ms >= 0) entry.duration_ms = duration_ms;
|
|
164
|
+
if (entry.phase === 'start') {
|
|
165
|
+
// The process that will write the end line; if it is gone and no end
|
|
166
|
+
// came, the run was lost with it. A short helper writing for a longer
|
|
167
|
+
// caller, like a shell script, passes the caller's pid.
|
|
168
|
+
entry.pid = Number.isInteger(deps.pid) && deps.pid > 0 ? deps.pid : process.pid;
|
|
169
|
+
entry.leaked_ssh_agents = null;
|
|
170
|
+
try {
|
|
171
|
+
const uid = deps.uid ?? (process.getuid ? process.getuid() : null);
|
|
172
|
+
if (uid !== null) {
|
|
173
|
+
const read = deps.readSshAgents || readSshAgents;
|
|
174
|
+
const { agents, referenced, unread = [] } = read(deps);
|
|
175
|
+
// A partial scan cannot establish a count, even when no agents were found.
|
|
176
|
+
if (!unread.length) entry.leaked_ssh_agents = findLeakedSshAgents(agents, { referenced, uid }).length;
|
|
177
|
+
}
|
|
178
|
+
} catch {}
|
|
179
|
+
}
|
|
180
|
+
if (signal) entry.signal = signal;
|
|
181
|
+
if (entry.phase === 'end' && timed_out === true) entry.timed_out = true;
|
|
182
|
+
if (entry.phase === 'end' && typeof error === 'string' && error) entry.error = error;
|
|
183
|
+
if (signal === 'SIGKILL') entry.cause = entry.timed_out ? TIME_LIMIT_CAUSE : killedCauseText(entry);
|
|
184
|
+
appendResourceLine(root, entry);
|
|
185
|
+
return entry;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// Start and end readings around one engine launch that reports failure by
|
|
189
|
+
// throwing, like the autopilot and run phases. The returned end takes the
|
|
190
|
+
// thrown error, or nothing on success, writes once, and returns its line.
|
|
191
|
+
function startEngineRun(root, { engine, task } = {}, deps = {}) {
|
|
192
|
+
const resource = { engine: engine || 'runner', task: task || '', run_id: require('crypto').randomUUID() };
|
|
193
|
+
recordEngineResource(root, { phase: 'start', ...resource }, deps);
|
|
194
|
+
const startedAt = process.hrtime.bigint();
|
|
195
|
+
let ended = null;
|
|
196
|
+
return (err = null) => {
|
|
197
|
+
if (ended) return ended;
|
|
198
|
+
const timedOut = Boolean(err && err.code === 'ETIMEDOUT');
|
|
199
|
+
const signal = err && err.signal ? err.signal : null;
|
|
200
|
+
const stderr = err ? String(err.stderr || '') : '';
|
|
201
|
+
const error = err && !signal && !timedOut ? (stderr.trim() ? stderr : String(err.message || '')).slice(-2000) : '';
|
|
202
|
+
ended = recordEngineResource(root, { phase: 'end', ...resource, signal, timed_out: timedOut, error, duration_ms: Math.round(Number(process.hrtime.bigint() - startedAt) / 1e6) }, deps);
|
|
203
|
+
return ended;
|
|
204
|
+
};
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// Shell startup files that run `eval $(ssh-agent -s)` start a fresh agent for
|
|
208
|
+
// every engine shell, and the agent outlives the shell. An agent counts as
|
|
209
|
+
// leaked only when it was started in that plain shape, its parent is gone, it
|
|
210
|
+
// is older than ten minutes, it belongs to this user, and no live process
|
|
211
|
+
// still holds its pid in SSH_AGENT_PID. The macOS system agent runs with -l
|
|
212
|
+
// and is never listed.
|
|
213
|
+
const LEAK_MIN_AGE_SEC = 600;
|
|
214
|
+
const SHELL_AGENT_FLAGS = new Set(['-s', '-c']);
|
|
215
|
+
|
|
216
|
+
// ps elapsed time: [[dd-]hh:]mm:ss.
|
|
217
|
+
function parseElapsed(text) {
|
|
218
|
+
const match = /^(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+)$/.exec(String(text || '').trim());
|
|
219
|
+
if (!match) return null;
|
|
220
|
+
const [, days, hours, minutes, seconds] = match.map((part) => Number(part || 0));
|
|
221
|
+
return ((days * 24 + hours) * 60 + minutes) * 60 + seconds;
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function isShellAgent(args) {
|
|
225
|
+
const [program, ...flags] = String(args || '').trim().split(/\s+/);
|
|
226
|
+
return path.basename(program || '') === 'ssh-agent' && flags.every((flag) => SHELL_AGENT_FLAGS.has(flag));
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function readSshAgents(deps = {}) {
|
|
230
|
+
const run = deps.run || defaultRun;
|
|
231
|
+
const agents = [];
|
|
232
|
+
let processes;
|
|
233
|
+
try { processes = run('ps', ['-A', '-o', 'pid=,ppid=,uid=,etime=,args=']); } catch {}
|
|
234
|
+
if (!processes || processes.status !== 0) return { agents, referenced: new Set(), unread: ['process list'] };
|
|
235
|
+
for (const line of String(processes.stdout || '').split('\n')) {
|
|
236
|
+
const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(.+?)\s*$/.exec(line);
|
|
237
|
+
if (!match) continue;
|
|
238
|
+
const program = match[5].split(/\s+/)[0];
|
|
239
|
+
if (path.basename(program) !== 'ssh-agent') continue;
|
|
240
|
+
agents.push({
|
|
241
|
+
pid: Number(match[1]),
|
|
242
|
+
ppid: Number(match[2]),
|
|
243
|
+
uid: Number(match[3]),
|
|
244
|
+
age_sec: parseElapsed(match[4]),
|
|
245
|
+
args: match[5],
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
const referenced = new Set();
|
|
249
|
+
const authSockets = new Set();
|
|
250
|
+
const environments = readOutput(run, 'ps', ['-A', '-E', '-o', 'pid=,command=']);
|
|
251
|
+
// Without the environment list nothing can be proven unused. unread names
|
|
252
|
+
// each reader that failed, so the sweep can say an empty list is not clean.
|
|
253
|
+
if (!environments.trim()) return { agents, referenced: new Set(agents.map((agent) => agent.pid)), unread: ['process environments'] };
|
|
254
|
+
for (const line of environments.split('\n')) {
|
|
255
|
+
const owner = Number((/^\s*(\d+)/.exec(line) || [])[1]);
|
|
256
|
+
for (const ref of line.matchAll(/\bSSH_AGENT_PID=(\d+)/g)) {
|
|
257
|
+
if (Number(ref[1]) !== owner) referenced.add(Number(ref[1]));
|
|
258
|
+
}
|
|
259
|
+
for (const ref of line.matchAll(/\bSSH_AUTH_SOCK=(\S+)/g)) authSockets.add(ref[1]);
|
|
260
|
+
}
|
|
261
|
+
const own = Number(process.env.SSH_AGENT_PID);
|
|
262
|
+
if (Number.isInteger(own) && own > 0) referenced.add(own);
|
|
263
|
+
if (process.env.SSH_AUTH_SOCK) authSockets.add(process.env.SSH_AUTH_SOCK);
|
|
264
|
+
// A shell can reach an agent by its socket alone, so an agent whose socket
|
|
265
|
+
// is named in a live SSH_AUTH_SOCK is in use. An agent whose socket could
|
|
266
|
+
// not be read is treated as in use too.
|
|
267
|
+
const sockets = new Map();
|
|
268
|
+
let socketPid = null;
|
|
269
|
+
for (const line of readOutput(run, 'lsof', ['-a', '-U', '-c', 'ssh-agent', '-F', 'pn']).split('\n')) {
|
|
270
|
+
if (line.startsWith('p')) socketPid = Number(line.slice(1));
|
|
271
|
+
else if (line.startsWith('n/') && socketPid) sockets.set(socketPid, [...(sockets.get(socketPid) || []), line.slice(1)]);
|
|
272
|
+
}
|
|
273
|
+
for (const agent of agents) {
|
|
274
|
+
const own = sockets.get(agent.pid) || [];
|
|
275
|
+
if (!own.length || own.some((socket) => authSockets.has(socket))) referenced.add(agent.pid);
|
|
276
|
+
}
|
|
277
|
+
// With no agents running, lsof finding no sockets is the right answer.
|
|
278
|
+
return { agents, referenced, unread: agents.some((agent) => !(sockets.get(agent.pid) || []).length) ? ['agent sockets'] : [] };
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// Shell startup files that run an ssh-agent start a new one in every shell
|
|
282
|
+
// that reads them, which is how agents pile up under engine shells. Names
|
|
283
|
+
// each uncommented line that runs ssh-agent through eval or command
|
|
284
|
+
// substitution, unless the same line checks for a running agent; a guard on
|
|
285
|
+
// an earlier line is not seen, so this is a lead. A line that is only an eval
|
|
286
|
+
// gets a fix that starts an agent only when no agent socket is reachable.
|
|
287
|
+
const SHELL_STARTUP_FILES = ['.zshenv', '.zprofile', '.zshrc', '.zlogin', '.bash_profile', '.bashrc', '.profile'];
|
|
288
|
+
|
|
289
|
+
function findAgentStartLines({ home = os.homedir() } = {}) {
|
|
290
|
+
const found = [];
|
|
291
|
+
for (const name of SHELL_STARTUP_FILES) {
|
|
292
|
+
let text;
|
|
293
|
+
try { text = fs.readFileSync(path.join(home, name), 'utf8'); } catch { continue; }
|
|
294
|
+
text.split('\n').forEach((line, i) => {
|
|
295
|
+
const code = line.replace(/(^|\s)#.*$/, '').trim();
|
|
296
|
+
if (!/\bssh-agent\b/.test(code) || !/\beval\b|\$\(|`/.test(code)) return;
|
|
297
|
+
if (/\bSSH_(AUTH_SOCK|AGENT_PID)\b/.test(code)) return;
|
|
298
|
+
const lone = /^eval\s/.test(code) && !/[;&|]|\b(if|then|fi)\b/.test(code);
|
|
299
|
+
found.push({ file: `~/${name}`, line: i + 1, ...(lone ? { fix: `[ -S "$SSH_AUTH_SOCK" ] || ${code}` } : {}) });
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
return found;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
// One plain line per startup line found, shared by the sweep and the
|
|
306
|
+
// landing status so both name the source and its fix the same way.
|
|
307
|
+
function agentStartLineText({ file, line, fix }) {
|
|
308
|
+
return `likely source: ${file} line ${line} starts a new ssh-agent in every shell that reads it${fix ? `; to start one only when none is reachable, change it to: ${fix}` : ''}`;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
function findLeakedSshAgents(agents, { referenced = new Set(), uid, minAgeSec = LEAK_MIN_AGE_SEC } = {}) {
|
|
312
|
+
return (agents || []).filter((agent) => agent.ppid === 1
|
|
313
|
+
&& agent.uid === uid
|
|
314
|
+
&& Number.isFinite(agent.age_sec) && agent.age_sec >= minAgeSec
|
|
315
|
+
&& isShellAgent(agent.args)
|
|
316
|
+
&& !referenced.has(agent.pid));
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// Reads the machine again right before stopping anything, so an agent that a
|
|
320
|
+
// shell picked up since the list was made, or a pid reused by a younger agent,
|
|
321
|
+
// is skipped. Stops with SIGTERM and writes one sweep line to the resource log.
|
|
322
|
+
function stopLeakedSshAgents(root, leaked, { read = readSshAgents, uid, kill = process.kill.bind(process), now = () => new Date().toISOString() } = {}) {
|
|
323
|
+
const recheck = read();
|
|
324
|
+
const unread = recheck.unread || [];
|
|
325
|
+
const current = new Map(recheck.agents.map((agent) => [agent.pid, agent]));
|
|
326
|
+
const still = new Map(findLeakedSshAgents(recheck.agents, { referenced: recheck.referenced, uid }).map((agent) => [agent.pid, agent]));
|
|
327
|
+
const stopped = [];
|
|
328
|
+
const failed = [];
|
|
329
|
+
const skipped = [];
|
|
330
|
+
const replaced = [];
|
|
331
|
+
for (const { pid, age_sec } of leaked) {
|
|
332
|
+
if (!still.has(pid) || still.get(pid).age_sec < age_sec) {
|
|
333
|
+
skipped.push(pid);
|
|
334
|
+
if (current.has(pid) && current.get(pid).age_sec < age_sec) replaced.push(pid);
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
try {
|
|
338
|
+
kill(pid, 'SIGTERM');
|
|
339
|
+
stopped.push(pid);
|
|
340
|
+
} catch (err) {
|
|
341
|
+
failed.push({ pid, code: (err && err.code) || 'error' });
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
// How many still looked leaked once this sweep was done, so the growth
|
|
345
|
+
// note can count from here instead of from the pile it just cleared.
|
|
346
|
+
const left = unread.length ? null : Math.max(0, still.size - stopped.length);
|
|
347
|
+
appendResourceLine(root, { at: now(), phase: 'sweep', stopped, failed: failed.map((f) => f.pid), skipped, ...(replaced.length ? { replaced } : {}), leaked_ssh_agents: left, ...(unread.length ? { unread } : {}) });
|
|
348
|
+
return { stopped, failed, skipped, ...(replaced.length ? { replaced } : {}), ...(unread.length ? { unread } : {}) };
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
// The newest sweep and SIGKILL from the resource log, how many SIGKILLs were
|
|
352
|
+
// blamed on low memory, the runs whose start has no matching end, and the
|
|
353
|
+
// runs closed after their parent died, and the newest run that failed with
|
|
354
|
+
// a message without being killed. Kills and closes are also counted for
|
|
355
|
+
// the last day, with time limit kills apart and the newest other kill kept so
|
|
356
|
+
// a run of timeouts cannot hide a crash. Reads
|
|
357
|
+
// up to the size cap, which covers the whole log once it has been cut back,
|
|
358
|
+
// so an unfinished start kept at the top is still seen; only a log that grew
|
|
359
|
+
// past the cap before the cap existed is read from its tail until the next
|
|
360
|
+
// append cuts it. A line that does not parse is skipped, and starts from
|
|
361
|
+
// before run ids existed are not counted.
|
|
362
|
+
const HISTORY_TAIL_BYTES = MAX_LOG_BYTES;
|
|
363
|
+
const DAY_MS = 24 * 3600 * 1000;
|
|
364
|
+
|
|
365
|
+
// A pid owned by another user still answers with EPERM. A reused pid reads
|
|
366
|
+
// as alive, so a lost run can be undercounted but never invented.
|
|
367
|
+
function processAlive(pid) {
|
|
368
|
+
try {
|
|
369
|
+
process.kill(pid, 0);
|
|
370
|
+
return true;
|
|
371
|
+
} catch (err) {
|
|
372
|
+
return Boolean(err && err.code === 'EPERM');
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
function scanResourceLog(root, { now = Date.now() } = {}) {
|
|
377
|
+
const dayAgo = now - DAY_MS;
|
|
378
|
+
const history = { last_sweep: null, last_kill: null, kills: 0, low_memory_kills: 0, time_limit_kills: 0, last_other_kill: null, kills_day: 0, low_memory_kills_day: 0, time_limit_kills_day: 0, last_day_other_kill: null, open_runs: 0, parent_gone: 0, still_running: 0, last_open: null, lost_closed: 0, lost_closed_day: 0, last_lost_closed: null, first_day_leak_reading: null, last_error: null, errors_day: 0, last_day_error: null, read_error: null };
|
|
379
|
+
const open = new Map();
|
|
380
|
+
let text = '';
|
|
381
|
+
try {
|
|
382
|
+
const file = path.join(resourceLogRoot(root), RESOURCE_LOG);
|
|
383
|
+
const size = fs.statSync(file).size;
|
|
384
|
+
const start = Math.max(0, size - HISTORY_TAIL_BYTES);
|
|
385
|
+
const fd = fs.openSync(file, 'r');
|
|
386
|
+
try {
|
|
387
|
+
const buffer = Buffer.alloc(size - start);
|
|
388
|
+
fs.readSync(fd, buffer, 0, buffer.length, start);
|
|
389
|
+
text = buffer.toString('utf8');
|
|
390
|
+
} finally {
|
|
391
|
+
fs.closeSync(fd);
|
|
392
|
+
}
|
|
393
|
+
if (start > 0) text = text.slice(text.indexOf('\n') + 1);
|
|
394
|
+
} catch (err) {
|
|
395
|
+
// No log yet means no runs; any other failure means the runs are unknown.
|
|
396
|
+
const readError = err && err.code !== 'ENOENT' ? String(err.code || err.message || err) : null;
|
|
397
|
+
return { history, open, readError };
|
|
398
|
+
}
|
|
399
|
+
for (const line of text.split('\n')) {
|
|
400
|
+
let entry;
|
|
401
|
+
try { entry = JSON.parse(line); } catch { continue; }
|
|
402
|
+
if (!entry || typeof entry !== 'object') continue;
|
|
403
|
+
if (entry.phase === 'sweep') history.last_sweep = entry;
|
|
404
|
+
if (entry.phase === 'start' && entry.run_id) open.set(entry.run_id, entry);
|
|
405
|
+
// The oldest leaked ssh-agent count a run start recorded in the last day,
|
|
406
|
+
// so the status can say how fast the pile is growing. A sweep that
|
|
407
|
+
// stopped agents starts the count over from what it left.
|
|
408
|
+
if (Number.isFinite(entry.leaked_ssh_agents) && Date.parse(entry.at) >= dayAgo) {
|
|
409
|
+
const cleared = entry.phase === 'sweep' && Array.isArray(entry.stopped) && entry.stopped.length > 0;
|
|
410
|
+
if (cleared || (entry.phase === 'start' && !history.first_day_leak_reading)) history.first_day_leak_reading = entry;
|
|
411
|
+
}
|
|
412
|
+
if (entry.phase === 'end' && entry.run_id) open.delete(entry.run_id);
|
|
413
|
+
if (entry.phase === 'end' && entry.cause === PARENT_GONE_CAUSE) {
|
|
414
|
+
history.last_lost_closed = entry;
|
|
415
|
+
history.lost_closed += 1;
|
|
416
|
+
if (Date.parse(entry.at) >= dayAgo) history.lost_closed_day += 1;
|
|
417
|
+
}
|
|
418
|
+
// The newest run that failed with a message but was not killed, such as
|
|
419
|
+
// an engine that could not start; kills are named on their own.
|
|
420
|
+
if (entry.phase === 'end' && !entry.signal && typeof entry.error === 'string' && entry.error.trim()) {
|
|
421
|
+
history.last_error = entry;
|
|
422
|
+
if (Date.parse(entry.at) >= dayAgo) {
|
|
423
|
+
history.errors_day += 1;
|
|
424
|
+
history.last_day_error = entry;
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
if (entry.phase === 'end' && entry.signal === 'SIGKILL') {
|
|
428
|
+
history.last_kill = entry;
|
|
429
|
+
history.kills += 1;
|
|
430
|
+
if (entry.cause === LOW_MEMORY_CAUSE) history.low_memory_kills += 1;
|
|
431
|
+
if (entry.cause === TIME_LIMIT_CAUSE) history.time_limit_kills += 1;
|
|
432
|
+
else history.last_other_kill = entry;
|
|
433
|
+
if (Date.parse(entry.at) >= dayAgo) {
|
|
434
|
+
history.kills_day += 1;
|
|
435
|
+
if (entry.cause === LOW_MEMORY_CAUSE) history.low_memory_kills_day += 1;
|
|
436
|
+
if (entry.cause === TIME_LIMIT_CAUSE) history.time_limit_kills_day += 1;
|
|
437
|
+
else history.last_day_other_kill = entry;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
return { history, open };
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
const PARENT_GONE_CAUSE = 'parent gone';
|
|
445
|
+
|
|
446
|
+
function parentGone(entry) {
|
|
447
|
+
return Number.isInteger(entry.pid) && entry.pid > 0 && !processAlive(entry.pid);
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
// The first line of a run's error, short enough for one status line.
|
|
451
|
+
function errorReason(entry) {
|
|
452
|
+
const text = entry && typeof entry.error === 'string' ? entry.error : '';
|
|
453
|
+
return (text.split('\n').map((line) => line.trim()).find(Boolean) || '').slice(0, 200);
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
function readResourceHistory(root, { now } = {}) {
|
|
457
|
+
const { history, open, readError } = scanResourceLog(root, now === undefined ? {} : { now });
|
|
458
|
+
// An empty history from a log that could not be read is not a quiet one.
|
|
459
|
+
if (readError) history.read_error = readError;
|
|
460
|
+
history.open_runs = open.size;
|
|
461
|
+
for (const entry of open.values()) {
|
|
462
|
+
if (!Number.isInteger(entry.pid) || entry.pid <= 0) continue;
|
|
463
|
+
if (parentGone(entry)) history.parent_gone += 1;
|
|
464
|
+
else history.still_running += 1;
|
|
465
|
+
}
|
|
466
|
+
history.last_open = [...open.values()].pop() || null;
|
|
467
|
+
return history;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
// Growth since the oldest count a run start recorded in the last day, or
|
|
471
|
+
// since the newest sweep that stopped agents; quiet when the pile has not
|
|
472
|
+
// grown, so only a filling machine reads as urgent.
|
|
473
|
+
// The landing status and the sweep both print it.
|
|
474
|
+
function leakGrowthNote(reading, now, at = Date.now()) {
|
|
475
|
+
if (!reading || !(now > reading.leaked_ssh_agents)) return '';
|
|
476
|
+
const when = `${String(reading.at || '').slice(0, 16).replace('T', ' ')} UTC`;
|
|
477
|
+
const hours = (at - Date.parse(reading.at)) / 3600000;
|
|
478
|
+
const rate = hours >= 0.25 ? `, about ${Math.round((now - reading.leaked_ssh_agents) / hours)} more an hour` : '';
|
|
479
|
+
const from = reading.phase === 'sweep' ? `${reading.leaked_ssh_agents} left by the sweep` : reading.leaked_ssh_agents;
|
|
480
|
+
return `, up from ${from} at ${when}${rate}`;
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
// Writes an end line for each unfinished run whose parent process is gone,
|
|
484
|
+
// so the log stops carrying it as open. The line has no machine readings:
|
|
485
|
+
// they would describe the sweep, not the moment the run was lost.
|
|
486
|
+
function closeLostRuns(root, { now = () => new Date().toISOString() } = {}) {
|
|
487
|
+
const closed = [];
|
|
488
|
+
const { open, readError } = scanResourceLog(root);
|
|
489
|
+
if (readError) throw new Error(`could not read the engine run log: ${readError}`);
|
|
490
|
+
for (const entry of open.values()) {
|
|
491
|
+
if (!parentGone(entry)) continue;
|
|
492
|
+
const end = { at: now(), phase: 'end', engine: entry.engine || '', task: entry.task || '', run_id: entry.run_id, cause: PARENT_GONE_CAUSE, closed_by: 'sweep' };
|
|
493
|
+
appendResourceLine(root, end);
|
|
494
|
+
closed.push(end);
|
|
495
|
+
}
|
|
496
|
+
return closed;
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
module.exports = {
|
|
500
|
+
RESOURCE_LOG,
|
|
501
|
+
MAX_LOG_BYTES,
|
|
502
|
+
readResourceHistory,
|
|
503
|
+
errorReason,
|
|
504
|
+
leakGrowthNote,
|
|
505
|
+
closeLostRuns,
|
|
506
|
+
stopLeakedSshAgents,
|
|
507
|
+
LEAK_MIN_AGE_SEC,
|
|
508
|
+
parseElapsed,
|
|
509
|
+
readSshAgents,
|
|
510
|
+
findLeakedSshAgents,
|
|
511
|
+
findAgentStartLines,
|
|
512
|
+
agentStartLineText,
|
|
513
|
+
readMachineResources,
|
|
514
|
+
memoryWasLow,
|
|
515
|
+
killedCauseText,
|
|
516
|
+
recordEngineResource,
|
|
517
|
+
startEngineRun,
|
|
518
|
+
};
|