@hivemagnus/magnus-agent 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +3 -0
- package/README.md +47 -2
- package/deploy/magnus-agent-unprivileged.service +5 -0
- package/deploy/magnus-agent.service +5 -0
- package/dist/agent.d.ts +2 -0
- package/dist/agent.js +18 -3
- package/dist/env-catalog.js +2 -1
- package/dist/executors/historical-mutation.d.ts +5 -6
- package/dist/executors/historical-mutation.js +15 -2
- package/dist/executors/reviewed-historical-records.d.ts +22 -0
- package/dist/executors/reviewed-historical-records.js +117 -0
- package/dist/executors/scheduler-runner.js +12 -1
- package/dist/upgrade/install-service.js +15 -5
- package/dist/upgrade/routes.js +6 -1
- package/dist/upgrade/self-upgrade.d.ts +85 -10
- package/dist/upgrade/self-upgrade.js +459 -57
- package/launcher/launcher.mjs +383 -50
- package/package.json +5 -2
package/launcher/launcher.mjs
CHANGED
|
@@ -6,61 +6,169 @@
|
|
|
6
6
|
* staged the agent installed versions/<to>, pointed ./current at it and exited
|
|
7
7
|
* trial the new version runs; it must report one accepted heartbeat
|
|
8
8
|
* (state "confirmed") before the deadline
|
|
9
|
-
* confirmed done
|
|
9
|
+
* confirmed done; ./last-good points at the confirmed version
|
|
10
10
|
* reverted the trial failed (crash loop or no heartbeat in time); ./current
|
|
11
|
-
* points back at versions/<from>
|
|
11
|
+
* points back at versions/<from> (or ./last-good)
|
|
12
|
+
* failed the upgrade could not be started or reverted; ./current was left as it is
|
|
12
13
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
14
|
+
* A launcher write that fails (full or read-only disk) never stops the agent
|
|
15
|
+
* from starting: the launcher logs it and runs the safest version it can. The
|
|
16
|
+
* launcher has no dependencies. `magnus-agent install-service` installs it, and
|
|
17
|
+
* a confirmed agent replaces it with its own copy only when that copy has a
|
|
18
|
+
* higher LAUNCHER_PROTOCOL. It never escalates privileges.
|
|
15
19
|
*/
|
|
16
20
|
import { spawn } from 'node:child_process';
|
|
17
|
-
import { closeSync, existsSync, fsyncSync, lchownSync, openSync, readFileSync, renameSync, rmSync, statSync,
|
|
21
|
+
import { closeSync, existsSync, fsyncSync, lchownSync, openSync, readdirSync, readFileSync, readlinkSync, renameSync, rmSync, statSync,
|
|
22
|
+
symlinkSync, writeSync } from 'node:fs';
|
|
18
23
|
import { dirname, join, resolve } from 'node:path';
|
|
24
|
+
import { performance } from 'node:perf_hooks';
|
|
19
25
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
20
26
|
|
|
21
|
-
|
|
27
|
+
/** 2: last-good, trial liveness marker, durable writes, state lock, MAGNUS_AGENT_LAUNCHER_CONFIRM_SECONDS. */
|
|
28
|
+
export const LAUNCHER_PROTOCOL = 2;
|
|
22
29
|
export const STATE_FILE = 'upgrade-state.json';
|
|
30
|
+
export const LOCK_FILE = 'upgrade-state.lock';
|
|
31
|
+
/** Written by an agent on trial whose heartbeats cannot reach the control API. */
|
|
32
|
+
export const ALIVE_FILE = 'trial-alive.json';
|
|
33
|
+
export const CRASH_FILE = 'launcher-crashes.json';
|
|
34
|
+
/** The trial window grows while the new version is alive but the control API is unreachable, up to this many windows. */
|
|
35
|
+
export const EXTENSION_FACTOR = 3;
|
|
36
|
+
export const ALIVE_FRESH_MS = 60_000;
|
|
37
|
+
/** Launches with an unreadable state file, within RAPID_WINDOW_MS, before falling back to ./last-good. */
|
|
38
|
+
export const RAPID_CRASHES = 3;
|
|
39
|
+
const RAPID_WINDOW_MS = 5 * 60_000;
|
|
40
|
+
const HEALTHY_RUN_MS = 2 * 60_000;
|
|
23
41
|
const POLL_MS = 2_000;
|
|
42
|
+
const ENTRY = ['node_modules', '@hivemagnus', 'magnus-agent', 'dist', 'index.js'];
|
|
43
|
+
|
|
44
|
+
const errorCode = error => error?.code ?? error?.message ?? String(error);
|
|
45
|
+
|
|
46
|
+
export function entryOf(dir, version) {
|
|
47
|
+
return join(dir, 'versions', version, ...ENTRY);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function complete(dir, version) {
|
|
51
|
+
return typeof version === 'string' && version !== '' && existsSync(entryOf(dir, version));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** The X of a `<name> -> versions/X` link, or null. */
|
|
55
|
+
export function linkVersion(dir, name) {
|
|
56
|
+
try {
|
|
57
|
+
const target = readlinkSync(join(dir, name));
|
|
58
|
+
const match = /^versions[\\/]([^\\/]+)$/.exec(target);
|
|
59
|
+
return match ? match[1] : null;
|
|
60
|
+
} catch {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function revertTarget(state, facts) {
|
|
66
|
+
if (!facts.complete) return state.from ?? null;
|
|
67
|
+
if (facts.complete(state.from)) return state.from;
|
|
68
|
+
if (facts.lastGood && facts.lastGood !== state.to && facts.complete(facts.lastGood)) return facts.lastGood;
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function revertDecision(state, reason, facts, stamp) {
|
|
73
|
+
const target = revertTarget(state, facts);
|
|
74
|
+
const fullReason = target === null ? `${reason}; versions/${state.from} is missing and no other complete version exists, so ./current was left as it is`
|
|
75
|
+
: target !== state.from ? `${reason}; versions/${state.from} is missing, so ./current points at last-good ${target}` : reason;
|
|
76
|
+
return { action: 'revert', trial: false, target, reason: fullReason,
|
|
77
|
+
next: { ...state, state: target === null ? 'failed' : 'reverted', reason: fullReason, updatedAt: stamp } };
|
|
78
|
+
}
|
|
24
79
|
|
|
25
80
|
/**
|
|
26
81
|
* Pure decision for one launch.
|
|
27
82
|
* @param {null | Record<string, any>} state
|
|
28
83
|
* @param {number} now epoch ms
|
|
29
84
|
* @param {number} confirmMs trial window
|
|
30
|
-
* @
|
|
85
|
+
* @param {{ current?: string | null, lastGood?: string | null, complete?: (version: string) => boolean }} [facts]
|
|
86
|
+
* current is the ./current link target (`versions/X`); complete tells whether versions/X can run
|
|
87
|
+
* @returns {{ action: 'run' | 'revert', next: Record<string, any> | null, trial: boolean, reason?: string, target?: string | null }}
|
|
31
88
|
*/
|
|
32
|
-
export function decideLaunch(state, now, confirmMs) {
|
|
89
|
+
export function decideLaunch(state, now, confirmMs, facts = {}) {
|
|
33
90
|
if (!state || typeof state !== 'object') return { action: 'run', next: null, trial: false };
|
|
34
91
|
const maxAttempts = Number.isInteger(state.maxAttempts) && state.maxAttempts > 0 ? state.maxAttempts : 2;
|
|
35
92
|
const stamp = new Date(now).toISOString();
|
|
36
93
|
if (state.state === 'staged') {
|
|
94
|
+
const expected = join('versions', String(state.to));
|
|
95
|
+
if (facts.current !== undefined && facts.current !== expected) {
|
|
96
|
+
const reason = `./current points at ${facts.current ?? 'nothing'}, not ${expected}; the upgrade to ${state.to} was not switched in`;
|
|
97
|
+
return { action: 'run', trial: false, reason, next: { ...state, state: 'failed', reason, updatedAt: stamp } };
|
|
98
|
+
}
|
|
37
99
|
return { action: 'run', trial: true,
|
|
38
100
|
next: { ...state, state: 'trial', attempt: 1, maxAttempts, trialDeadline: new Date(now + confirmMs).toISOString(), updatedAt: stamp } };
|
|
39
101
|
}
|
|
40
102
|
if (state.state === 'trial') {
|
|
41
|
-
|
|
103
|
+
// A deadline further away than one window means the wall clock moved back: clamp it.
|
|
104
|
+
const parsed = Date.parse(state.trialDeadline ?? '');
|
|
105
|
+
const deadline = Number.isFinite(parsed) ? Math.min(parsed, now + confirmMs) : Number.NaN;
|
|
42
106
|
const attempt = Number.isInteger(state.attempt) ? state.attempt : maxAttempts;
|
|
43
107
|
if (!Number.isFinite(deadline) || now >= deadline) {
|
|
44
|
-
return
|
|
45
|
-
next: { ...state, state: 'reverted', updatedAt: stamp } };
|
|
46
|
-
}
|
|
47
|
-
if (attempt >= maxAttempts) {
|
|
48
|
-
return { action: 'revert', trial: false, reason: `${state.to} exited ${attempt} time(s) before reporting a heartbeat`,
|
|
49
|
-
next: { ...state, state: 'reverted', updatedAt: stamp } };
|
|
108
|
+
return revertDecision(state, `no confirmed heartbeat from ${state.to} within ${Math.round(confirmMs / 1000)} s`, facts, stamp);
|
|
50
109
|
}
|
|
51
|
-
|
|
110
|
+
if (attempt >= maxAttempts) return revertDecision(state, `${state.to} exited ${attempt} time(s) before reporting a heartbeat`, facts, stamp);
|
|
111
|
+
return { action: 'run', trial: true, next: { ...state, attempt: attempt + 1, trialDeadline: new Date(deadline).toISOString(), updatedAt: stamp } };
|
|
52
112
|
}
|
|
53
113
|
return { action: 'run', next: null, trial: false };
|
|
54
114
|
}
|
|
55
115
|
|
|
56
|
-
|
|
116
|
+
/**
|
|
117
|
+
* Pure decision for one poll while a trial runs. Times are monotonic
|
|
118
|
+
* milliseconds since this launch; `now` (wall clock) only stamps the state and
|
|
119
|
+
* judges the agent's liveness marker.
|
|
120
|
+
* @returns {{ action: 'stop' | 'wait' | 'extend' | 'revert', windowMs?: number, next?: Record<string, any>, reason?: string, target?: string | null }}
|
|
121
|
+
*/
|
|
122
|
+
export function decideWatch(state, { to, elapsedMs, windowMs, confirmMs, now, alive = null, facts = {} }) {
|
|
123
|
+
if (!state || typeof state !== 'object' || state.state !== 'trial' || state.to !== to) return { action: 'stop' };
|
|
124
|
+
if (elapsedMs < windowMs) return { action: 'wait' };
|
|
125
|
+
const stamp = new Date(now).toISOString();
|
|
126
|
+
const unreachable = alive && alive.version === to && Number.isFinite(alive.at) && Math.abs(now - alive.at) < ALIVE_FRESH_MS;
|
|
127
|
+
const capMs = EXTENSION_FACTOR * confirmMs;
|
|
128
|
+
if (unreachable && windowMs < capMs) {
|
|
129
|
+
const next = Math.min(windowMs + confirmMs, capMs);
|
|
130
|
+
return { action: 'extend', windowMs: next, reason: `control API unreachable from ${to}; trial extended to ${Math.round(next / 1000)} s`,
|
|
131
|
+
next: { ...state, trialDeadline: new Date(now + next - elapsedMs).toISOString(), extended: (state.extended ?? 0) + 1, updatedAt: stamp } };
|
|
132
|
+
}
|
|
133
|
+
const seconds = Math.round(windowMs / 1000);
|
|
134
|
+
const reason = unreachable
|
|
135
|
+
? `control API unreachable from ${to}${alive.error ? ` (${String(alive.error).slice(0, 200)})` : ''}: no confirmed heartbeat within ${seconds} s`
|
|
136
|
+
: `no confirmed heartbeat from ${to} within ${seconds} s`;
|
|
137
|
+
return revertDecision(state, reason, facts, stamp);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** Pure: whether repeated launches with an unreadable state file should fall back to ./last-good. */
|
|
141
|
+
export function decideCorrupt(starts, now, { current, lastGood, complete: isComplete = () => true }) {
|
|
142
|
+
const recent = (Array.isArray(starts) ? starts : []).filter(at => Number.isFinite(at) && now - at >= 0 && now - at < RAPID_WINDOW_MS);
|
|
143
|
+
recent.push(now);
|
|
144
|
+
if (recent.length >= RAPID_CRASHES && lastGood && lastGood !== current && isComplete(lastGood)) return { fallback: lastGood, starts: [] };
|
|
145
|
+
return { fallback: null, starts: recent };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** @returns {{ state: Record<string, any> | null, corrupt: boolean }} */
|
|
149
|
+
export function readStateFile(dir) {
|
|
150
|
+
let text;
|
|
151
|
+
try {
|
|
152
|
+
text = readFileSync(join(dir, STATE_FILE), 'utf8');
|
|
153
|
+
} catch (error) {
|
|
154
|
+
return { state: null, corrupt: error?.code !== 'ENOENT' };
|
|
155
|
+
}
|
|
57
156
|
try {
|
|
58
|
-
|
|
157
|
+
const value = JSON.parse(text);
|
|
158
|
+
return value && typeof value === 'object' && !Array.isArray(value) ? { state: value, corrupt: false } : { state: null, corrupt: true };
|
|
59
159
|
} catch {
|
|
60
|
-
return null;
|
|
160
|
+
return { state: null, corrupt: true };
|
|
61
161
|
}
|
|
62
162
|
}
|
|
63
163
|
|
|
164
|
+
export function readState(dir) {
|
|
165
|
+
return readStateFile(dir).state;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function readJson(path) {
|
|
169
|
+
try { return JSON.parse(readFileSync(path, 'utf8')); } catch { return null; }
|
|
170
|
+
}
|
|
171
|
+
|
|
64
172
|
/** Running as root in a directory another account owns: hand what we write to that owner. */
|
|
65
173
|
function adoptOwner(dir, path) {
|
|
66
174
|
if (process.getuid?.() !== 0) return;
|
|
@@ -68,30 +176,117 @@ function adoptOwner(dir, path) {
|
|
|
68
176
|
if (owner.uid !== 0) lchownSync(path, owner.uid, owner.gid);
|
|
69
177
|
}
|
|
70
178
|
|
|
71
|
-
|
|
72
|
-
|
|
179
|
+
function fsyncDir(dir) {
|
|
180
|
+
try {
|
|
181
|
+
const fd = openSync(dir, 'r');
|
|
182
|
+
try { fsyncSync(fd); } finally { closeSync(fd); }
|
|
183
|
+
} catch {
|
|
184
|
+
// some filesystems cannot fsync a directory
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/** temp file, fsync, rename, fsync the directory. */
|
|
189
|
+
export function writeDurable(dir, name, text) {
|
|
190
|
+
const path = join(dir, name);
|
|
73
191
|
const temporary = `${path}.${process.pid}.tmp`;
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
192
|
+
try {
|
|
193
|
+
const fd = openSync(temporary, 'w', 0o600);
|
|
194
|
+
try { writeSync(fd, text); fsyncSync(fd); } finally { closeSync(fd); }
|
|
195
|
+
adoptOwner(dir, temporary);
|
|
196
|
+
renameSync(temporary, path);
|
|
197
|
+
} catch (error) {
|
|
198
|
+
try { rmSync(temporary, { force: true }); } catch { /* best effort */ }
|
|
199
|
+
throw error;
|
|
200
|
+
}
|
|
201
|
+
fsyncDir(dir);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
export function writeState(dir, state) {
|
|
205
|
+
writeDurable(dir, STATE_FILE, `${JSON.stringify(state, null, 2)}\n`);
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Atomically point `<dir>/<name>` at versions/<version>. */
|
|
209
|
+
export function pointLink(dir, name, version) {
|
|
210
|
+
const temporary = join(dir, `${name}.${process.pid}.tmp`);
|
|
211
|
+
try {
|
|
212
|
+
rmSync(temporary, { force: true });
|
|
213
|
+
symlinkSync(join('versions', version), temporary, 'dir');
|
|
214
|
+
adoptOwner(dir, temporary);
|
|
215
|
+
renameSync(temporary, join(dir, name));
|
|
216
|
+
} catch (error) {
|
|
217
|
+
try { rmSync(temporary, { force: true }); } catch { /* best effort */ }
|
|
218
|
+
throw error;
|
|
219
|
+
}
|
|
220
|
+
fsyncDir(dir);
|
|
79
221
|
}
|
|
80
222
|
|
|
81
223
|
export function pointCurrentAt(dir, version) {
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
224
|
+
pointLink(dir, 'current', version);
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const sleeper = new Int32Array(new SharedArrayBuffer(4));
|
|
228
|
+
|
|
229
|
+
function lockIsStale(path, staleMs) {
|
|
230
|
+
try {
|
|
231
|
+
const pid = Number.parseInt(readFileSync(path, 'utf8'), 10);
|
|
232
|
+
if (Number.isInteger(pid) && pid > 0) {
|
|
233
|
+
try { process.kill(pid, 0); } catch (error) { if (error?.code === 'ESRCH') return true; }
|
|
234
|
+
}
|
|
235
|
+
return Date.now() - statSync(path).mtimeMs > staleMs;
|
|
236
|
+
} catch {
|
|
237
|
+
return false; // released meanwhile: retry
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Exclusive lock (O_EXCL) around a read-modify-write of the state file, shared
|
|
243
|
+
* with the agent. Failing to take it throws an error with `lockFailed: true`;
|
|
244
|
+
* errors from `fn` pass through unchanged.
|
|
245
|
+
*/
|
|
246
|
+
export function withLock(dir, fn, { timeoutMs = 3_000, staleMs = 15_000 } = {}) {
|
|
247
|
+
const path = join(dir, LOCK_FILE);
|
|
248
|
+
const until = Date.now() + timeoutMs;
|
|
249
|
+
let fd;
|
|
250
|
+
for (;;) {
|
|
251
|
+
try {
|
|
252
|
+
fd = openSync(path, 'wx', 0o600);
|
|
253
|
+
break;
|
|
254
|
+
} catch (error) {
|
|
255
|
+
if (error?.code !== 'EEXIST') throw Object.assign(error, { lockFailed: true });
|
|
256
|
+
if (lockIsStale(path, staleMs)) { try { rmSync(path, { force: true }); } catch { /* retry */ } continue; }
|
|
257
|
+
if (Date.now() >= until) throw Object.assign(new Error(`${path} is held by another process`), { code: 'ELOCKED', lockFailed: true });
|
|
258
|
+
Atomics.wait(sleeper, 0, 0, 25);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
try { writeSync(fd, String(process.pid)); } finally { closeSync(fd); }
|
|
262
|
+
try {
|
|
263
|
+
return fn();
|
|
264
|
+
} finally {
|
|
265
|
+
try { rmSync(path, { force: true }); } catch { /* stale lock is detected by pid */ }
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
/** Run `fn` under the lock, or without it when the lock cannot be taken (read-only disk): starting the agent matters more. */
|
|
270
|
+
function lockedOrNot(dir, fn, log) {
|
|
271
|
+
try {
|
|
272
|
+
return withLock(dir, fn);
|
|
273
|
+
} catch (error) {
|
|
274
|
+
if (!error?.lockFailed) throw error;
|
|
275
|
+
log(`cannot lock ${STATE_FILE} (${errorCode(error)}); continuing without the lock`);
|
|
276
|
+
return fn();
|
|
277
|
+
}
|
|
87
278
|
}
|
|
88
279
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
280
|
+
/** Stale `<name>.<pid>.tmp` files from an interrupted write. Nothing else writes while the launcher starts. */
|
|
281
|
+
export function sweepTemporaries(dir, log = () => {}) {
|
|
282
|
+
for (const folder of [dir, join(dir, 'launcher')]) {
|
|
283
|
+
let names = [];
|
|
284
|
+
try { names = readdirSync(folder); } catch { continue; }
|
|
285
|
+
for (const name of names) {
|
|
286
|
+
if (!/\.\d+\.tmp$/.test(name)) continue;
|
|
287
|
+
try { rmSync(join(folder, name), { recursive: true, force: true }); } catch (error) { log(`cannot remove ${join(folder, name)} (${errorCode(error)})`); }
|
|
288
|
+
}
|
|
289
|
+
}
|
|
95
290
|
}
|
|
96
291
|
|
|
97
292
|
export function confirmSeconds(env) {
|
|
@@ -99,41 +294,178 @@ export function confirmSeconds(env) {
|
|
|
99
294
|
return Number.isInteger(value) && value >= 10 ? Math.min(value, 3600) : 180;
|
|
100
295
|
}
|
|
101
296
|
|
|
297
|
+
const realOps = {
|
|
298
|
+
readStateFile, writeState, pointCurrentAt, linkVersion, complete,
|
|
299
|
+
readCrashes: dir => readJson(join(dir, CRASH_FILE))?.starts ?? [],
|
|
300
|
+
writeCrashes: (dir, starts) => writeDurable(dir, CRASH_FILE, `${JSON.stringify({ starts })}\n`),
|
|
301
|
+
clearCrashes: dir => rmSync(join(dir, CRASH_FILE), { force: true }),
|
|
302
|
+
lock: (dir, fn, log) => lockedOrNot(dir, fn, log),
|
|
303
|
+
};
|
|
304
|
+
|
|
305
|
+
/** Switch and record a revert decision. Returns whether ./current now points at the revert target. */
|
|
306
|
+
function applyRevert(dir, decision, ops, log) {
|
|
307
|
+
const next = { ...decision.next };
|
|
308
|
+
let switched = false;
|
|
309
|
+
if (decision.target) {
|
|
310
|
+
try {
|
|
311
|
+
ops.pointCurrentAt(dir, decision.target);
|
|
312
|
+
switched = true;
|
|
313
|
+
} catch (error) {
|
|
314
|
+
next.state = 'failed';
|
|
315
|
+
next.reason = `${decision.reason}; switching ./current to ${decision.target} failed (${errorCode(error)})`;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
try {
|
|
319
|
+
ops.writeState(dir, next);
|
|
320
|
+
} catch (error) {
|
|
321
|
+
log(`cannot record the ${next.state} state (${errorCode(error)}); running ./current anyway`);
|
|
322
|
+
}
|
|
323
|
+
log(`${switched ? 'reverted' : 'could not revert'} ${next.to} -> ${decision.target ?? '(none)'}: ${next.reason}`);
|
|
324
|
+
return switched;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/**
|
|
328
|
+
* Apply the launch decision for `dir`. Never throws: a state or link write
|
|
329
|
+
* that fails falls back to running the previous (or current) version without
|
|
330
|
+
* a trial, so a full or read-only disk cannot turn into a crash loop.
|
|
331
|
+
* @returns {{ trial: boolean, to: string | null, remainingMs: number }}
|
|
332
|
+
*/
|
|
333
|
+
export function prepareLaunch(dir, now, confirmMs, log = () => {}, overrides = {}) {
|
|
334
|
+
const ops = { ...realOps, ...overrides };
|
|
335
|
+
let plan = { trial: false, to: null, remainingMs: 0 };
|
|
336
|
+
const body = () => {
|
|
337
|
+
const { state, corrupt } = ops.readStateFile(dir);
|
|
338
|
+
const current = ops.linkVersion(dir, 'current');
|
|
339
|
+
const lastGood = ops.linkVersion(dir, 'last-good');
|
|
340
|
+
if (corrupt) {
|
|
341
|
+
const verdict = decideCorrupt(ops.readCrashes(dir), now, { current, lastGood, complete: version => ops.complete(dir, version) });
|
|
342
|
+
if (!verdict.fallback) {
|
|
343
|
+
log(`${STATE_FILE} is unreadable; running ./current (${current ?? 'unknown'})`);
|
|
344
|
+
try { ops.writeCrashes(dir, verdict.starts); } catch { /* counted again next time */ }
|
|
345
|
+
return;
|
|
346
|
+
}
|
|
347
|
+
try {
|
|
348
|
+
ops.pointCurrentAt(dir, verdict.fallback);
|
|
349
|
+
log(`${STATE_FILE} is unreadable and ${current} keeps exiting; switched ./current to last-good ${verdict.fallback}`);
|
|
350
|
+
ops.writeState(dir, { state: 'reverted', from: current, to: current, requestId: null, updatedAt: new Date(now).toISOString(),
|
|
351
|
+
reason: `${STATE_FILE} was unreadable and ${current} exited ${RAPID_CRASHES} times; switched to last-good ${verdict.fallback}` });
|
|
352
|
+
ops.clearCrashes(dir);
|
|
353
|
+
} catch (error) {
|
|
354
|
+
log(`last-good fallback to ${verdict.fallback} failed (${errorCode(error)}); running ./current`);
|
|
355
|
+
}
|
|
356
|
+
return;
|
|
357
|
+
}
|
|
358
|
+
try { ops.clearCrashes(dir); } catch { /* harmless */ }
|
|
359
|
+
const facts = { current: current === null ? null : join('versions', current), lastGood, complete: version => ops.complete(dir, version) };
|
|
360
|
+
const decision = decideLaunch(state, now, confirmMs, facts);
|
|
361
|
+
if (decision.action === 'revert') { applyRevert(dir, decision, ops, log); return; }
|
|
362
|
+
if (!decision.next) return;
|
|
363
|
+
try {
|
|
364
|
+
ops.writeState(dir, decision.next);
|
|
365
|
+
} catch (error) {
|
|
366
|
+
log(`cannot record the ${decision.next.state} state for ${state.to} (${errorCode(error)})`);
|
|
367
|
+
if (!decision.trial) return;
|
|
368
|
+
// A trial that cannot be recorded could never be reverted: run the previous version instead.
|
|
369
|
+
if (state.from && ops.complete(dir, state.from)) {
|
|
370
|
+
try {
|
|
371
|
+
ops.pointCurrentAt(dir, state.from);
|
|
372
|
+
log(`switched ./current back to ${state.from}; ${state.to} was not started`);
|
|
373
|
+
} catch (linkError) {
|
|
374
|
+
log(`cannot switch ./current back to ${state.from} (${errorCode(linkError)}); running ./current without a trial`);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
379
|
+
if (decision.next.state === 'failed') log(decision.reason);
|
|
380
|
+
if (decision.trial) plan = { trial: true, to: state.to, remainingMs: Math.max(0, Date.parse(decision.next.trialDeadline) - now) };
|
|
381
|
+
};
|
|
382
|
+
try {
|
|
383
|
+
ops.lock(dir, body, log);
|
|
384
|
+
} catch (error) {
|
|
385
|
+
log(`upgrade bookkeeping failed (${errorCode(error)}); running ./current`);
|
|
386
|
+
plan = { trial: false, to: null, remainingMs: 0 };
|
|
387
|
+
}
|
|
388
|
+
return plan;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** One watcher poll: extend, revert or keep waiting. Returns 'reverted' when the child must stop. */
|
|
392
|
+
export function watchTrial(dir, plan, elapsedMs, windowMs, confirmMs, log = () => {}, overrides = {}) {
|
|
393
|
+
const ops = { ...realOps, ...overrides };
|
|
394
|
+
let outcome = { action: 'wait', windowMs };
|
|
395
|
+
const body = () => {
|
|
396
|
+
const { state } = ops.readStateFile(dir);
|
|
397
|
+
const now = Date.now();
|
|
398
|
+
const facts = { lastGood: ops.linkVersion(dir, 'last-good'), complete: version => ops.complete(dir, version) };
|
|
399
|
+
const verdict = decideWatch(state, { to: plan.to, elapsedMs, windowMs, confirmMs, now, alive: readJson(join(dir, ALIVE_FILE)), facts });
|
|
400
|
+
if (verdict.action === 'stop' || verdict.action === 'wait') { outcome = { action: verdict.action, windowMs }; return; }
|
|
401
|
+
if (verdict.action === 'extend') {
|
|
402
|
+
try { ops.writeState(dir, verdict.next); } catch (error) { log(`cannot record the extended trial (${errorCode(error)})`); }
|
|
403
|
+
log(verdict.reason);
|
|
404
|
+
outcome = { action: 'extend', windowMs: verdict.windowMs };
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
outcome = { action: applyRevert(dir, verdict, ops, log) ? 'reverted' : 'stop', windowMs };
|
|
408
|
+
};
|
|
409
|
+
try {
|
|
410
|
+
ops.lock(dir, body, log);
|
|
411
|
+
} catch (error) {
|
|
412
|
+
log(`trial check failed (${errorCode(error)})`);
|
|
413
|
+
}
|
|
414
|
+
return outcome;
|
|
415
|
+
}
|
|
416
|
+
|
|
102
417
|
function main() {
|
|
418
|
+
if (process.argv.includes('--protocol')) { console.log(String(LAUNCHER_PROTOCOL)); return; }
|
|
103
419
|
const dir = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
420
|
+
const log = message => console.error(`[launcher] ${message}`);
|
|
421
|
+
const configured = process.env.AGENT_INSTALL_DIR;
|
|
422
|
+
if (configured && resolve(configured) !== dir) log(`AGENT_INSTALL_DIR=${configured} differs from the launcher's directory; using ${dir}`);
|
|
104
423
|
const confirmMs = confirmSeconds(process.env) * 1000;
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
else if (decision.next) writeState(dir, decision.next);
|
|
424
|
+
sweepTemporaries(dir, log);
|
|
425
|
+
const plan = prepareLaunch(dir, Date.now(), confirmMs, log);
|
|
108
426
|
|
|
109
|
-
|
|
427
|
+
let entry = join(dir, 'current', ...ENTRY);
|
|
110
428
|
if (!existsSync(entry)) {
|
|
111
|
-
|
|
112
|
-
|
|
429
|
+
const lastGood = linkVersion(dir, 'last-good');
|
|
430
|
+
if (!plan.trial && lastGood && lastGood !== linkVersion(dir, 'current') && complete(dir, lastGood)) {
|
|
431
|
+
try {
|
|
432
|
+
pointCurrentAt(dir, lastGood);
|
|
433
|
+
log(`${entry} is missing; switched ./current to last-good ${lastGood}`);
|
|
434
|
+
entry = join(dir, 'current', ...ENTRY);
|
|
435
|
+
} catch (error) {
|
|
436
|
+
log(`${entry} is missing and switching to last-good failed (${errorCode(error)})`);
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
if (!existsSync(entry)) {
|
|
440
|
+
log(`${entry} is missing; reinstall with magnus-agent install-service`);
|
|
441
|
+
process.exit(1);
|
|
442
|
+
}
|
|
113
443
|
}
|
|
444
|
+
const startedAt = performance.now();
|
|
114
445
|
const child = spawn(process.execPath, [entry, 'start'], {
|
|
115
446
|
cwd: dir,
|
|
116
447
|
stdio: 'inherit',
|
|
117
448
|
env: {
|
|
118
449
|
...process.env,
|
|
119
|
-
AGENT_INSTALL_DIR:
|
|
450
|
+
AGENT_INSTALL_DIR: dir,
|
|
120
451
|
MAGNUS_AGENT_LAUNCHER: '1',
|
|
121
452
|
MAGNUS_AGENT_LAUNCHER_PROTOCOL: String(LAUNCHER_PROTOCOL),
|
|
453
|
+
MAGNUS_AGENT_LAUNCHER_CONFIRM_SECONDS: String(confirmMs / 1000),
|
|
122
454
|
},
|
|
123
455
|
});
|
|
124
456
|
for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) process.on(signal, () => child.kill(signal));
|
|
125
457
|
|
|
126
458
|
let failedTrial = false;
|
|
127
459
|
let watcher;
|
|
128
|
-
if (
|
|
460
|
+
if (plan.trial) {
|
|
461
|
+
let windowMs = plan.remainingMs;
|
|
129
462
|
watcher = setInterval(() => {
|
|
130
|
-
const
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
if (
|
|
463
|
+
const outcome = watchTrial(dir, plan, performance.now() - startedAt, windowMs, confirmMs, log);
|
|
464
|
+
windowMs = outcome.windowMs;
|
|
465
|
+
if (outcome.action === 'stop') clearInterval(watcher);
|
|
466
|
+
if (outcome.action === 'reverted') {
|
|
134
467
|
clearInterval(watcher);
|
|
135
468
|
failedTrial = true;
|
|
136
|
-
revert(dir, verdict);
|
|
137
469
|
child.kill('SIGTERM');
|
|
138
470
|
setTimeout(() => child.kill('SIGKILL'), 10_000).unref();
|
|
139
471
|
}
|
|
@@ -141,6 +473,7 @@ function main() {
|
|
|
141
473
|
}
|
|
142
474
|
child.on('exit', (code, signal) => {
|
|
143
475
|
if (watcher) clearInterval(watcher);
|
|
476
|
+
if (performance.now() - startedAt >= HEALTHY_RUN_MS) { try { realOps.clearCrashes(dir); } catch { /* harmless */ } }
|
|
144
477
|
// Any exit restarts through systemd (Restart=always); a failed trial exits non-zero.
|
|
145
478
|
process.exit(failedTrial ? 1 : code ?? (signal ? 1 : 0));
|
|
146
479
|
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hivemagnus/magnus-agent",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.3",
|
|
4
4
|
"description": "Magnus node agent: a small daemon that enrolls a Linux host with a magnus-server and runs authenticated health, command, cron, file and skill management, with optional Hermes Agent, Codex and Ardent management.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"magnus",
|
|
@@ -42,6 +42,9 @@
|
|
|
42
42
|
"engines": {
|
|
43
43
|
"node": "^24.22.1 || ^26.13.0"
|
|
44
44
|
},
|
|
45
|
+
"magnusAgent": {
|
|
46
|
+
"minLauncher": 1
|
|
47
|
+
},
|
|
45
48
|
"publishConfig": {
|
|
46
49
|
"access": "public"
|
|
47
50
|
},
|
|
@@ -55,7 +58,7 @@
|
|
|
55
58
|
"clean": "node -e \"const fs=require('node:fs');fs.rmSync('dist',{recursive:true,force:true});fs.rmSync('tsconfig.tsbuildinfo',{force:true})\""
|
|
56
59
|
},
|
|
57
60
|
"dependencies": {
|
|
58
|
-
"@hivemagnus/magnus-contracts": "1.1.
|
|
61
|
+
"@hivemagnus/magnus-contracts": "1.1.3",
|
|
59
62
|
"@hono/node-server": "^2.1.5",
|
|
60
63
|
"dotenv": "^18.0.9",
|
|
61
64
|
"hono": "^4.13.16",
|