@hivemagnus/magnus-server 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +30 -0
- package/README.md +22 -3
- package/agent-version.json +3 -1
- package/dist/cli/agents.js +111 -0
- package/dist/cli/args.js +19 -4
- package/dist/cli/auto-migrate.js +85 -0
- package/dist/cli/env-catalog.js +17 -0
- package/dist/cli/migrations.js +117 -0
- package/dist/cli/preflight.js +5 -14
- package/dist/cli/upgrade.js +55 -0
- package/dist/cli.js +121 -7
- package/dist/config/env.js +26 -0
- package/dist/config/features.js +22 -0
- package/dist/config/route-aliases.js +73 -0
- package/dist/index.js +13 -5
- package/dist/openapi/spec.js +72 -31
- package/dist/routes/agent-upgrades.js +42 -0
- package/dist/routes/heartbeat.js +8 -1
- package/dist/routes/self-update.js +19 -0
- package/dist/routes/system-updates.js +27 -0
- package/dist/sdk/node-api-client.js +4 -3
- package/dist/security/approval-assertion.js +23 -1
- package/dist/security/native-completion-registry.js +2 -0
- package/dist/server.js +5 -0
- package/dist/services/agent-upgrade-service.js +293 -0
- package/dist/services/agent-version-warnings.js +31 -0
- package/dist/services/self-update-service.js +11 -8
- package/dist/services/update-check-service.js +178 -0
- package/dist/services/webhook-service.js +13 -10
- package/dist/services/webhook-types.js +29 -0
- package/dist/utils/runtime-file.js +23 -0
- package/package.json +3 -3
|
@@ -390,9 +390,10 @@ export class NodeApiClient {
|
|
|
390
390
|
async removeSkill(skillName) {
|
|
391
391
|
return this.mutate('DELETE', `skills/${encodeURIComponent(skillName)}`);
|
|
392
392
|
}
|
|
393
|
-
// ──
|
|
394
|
-
|
|
395
|
-
|
|
393
|
+
// ── Agent self-upgrade ──
|
|
394
|
+
/** Stage an exact agent version; the agent restarts into it through its launcher. */
|
|
395
|
+
async upgradeAgent(request) {
|
|
396
|
+
return this.mutate('POST', 'agent-upgrade', request, 15 * 60_000);
|
|
396
397
|
}
|
|
397
398
|
// ── Ping ──
|
|
398
399
|
async ping() {
|
|
@@ -2,6 +2,26 @@ import { createHash, createPublicKey, timingSafeEqual, verify } from 'node:crypt
|
|
|
2
2
|
import { readFileSync } from 'node:fs';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { HttpError } from '../middleware/error-handler.js';
|
|
5
|
+
/** Signature domain prefix for approval assertions; the signed message is `<prefix>.v<version>\n<canonical payload>`. */
|
|
6
|
+
export const DEFAULT_APPROVAL_ASSERTION_DOMAIN = 'magnus.approval-assertion';
|
|
7
|
+
export const approvalAssertionDomainPattern = /^[a-z0-9](?:[a-z0-9.-]{0,98}[a-z0-9])?$/;
|
|
8
|
+
/**
|
|
9
|
+
* Accepted signature domain prefixes, in preference order. The first one is the
|
|
10
|
+
* domain this server would sign with. `MAGNUS_APPROVAL_ASSERTION_DOMAINS`
|
|
11
|
+
* (comma-separated) lets older signers that use another prefix keep working.
|
|
12
|
+
*/
|
|
13
|
+
export function approvalAssertionDomains(value = process.env['MAGNUS_APPROVAL_ASSERTION_DOMAINS']) {
|
|
14
|
+
if (value === undefined || value.trim() === '')
|
|
15
|
+
return [DEFAULT_APPROVAL_ASSERTION_DOMAIN];
|
|
16
|
+
const domains = value.split(',').map(item => item.trim()).filter(Boolean);
|
|
17
|
+
if (!domains.length || domains.some(domain => !approvalAssertionDomainPattern.test(domain)))
|
|
18
|
+
throw new Error('MAGNUS_APPROVAL_ASSERTION_DOMAINS must list domain prefixes made of a-z, 0-9, "." and "-"');
|
|
19
|
+
return [...new Set(domains)];
|
|
20
|
+
}
|
|
21
|
+
/** The exact bytes signed for an approval payload under a domain prefix. */
|
|
22
|
+
export function approvalSigningMessage(payload, domain = approvalAssertionDomains()[0]) {
|
|
23
|
+
return Buffer.from(`${domain}.v${payload.version}\n` + approvalPayloadCanonical(payload));
|
|
24
|
+
}
|
|
5
25
|
const identifier = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9:._-]{0,159}$/);
|
|
6
26
|
const digest = z.string().regex(/^[a-f0-9]{64}$/);
|
|
7
27
|
const payloadFields = {
|
|
@@ -49,7 +69,9 @@ export function verifyApprovalAssertion(raw) {
|
|
|
49
69
|
const key = createPublicKey(inline ?? (encoded
|
|
50
70
|
? { key: Buffer.from(encoded, 'base64'), format: 'der', type: 'spki' }
|
|
51
71
|
: readFileSync(path, 'utf8')));
|
|
52
|
-
|
|
72
|
+
const signatureBytes = Buffer.from(signature, 'base64url');
|
|
73
|
+
valid = key.asymmetricKeyType === 'ed25519' && approvalAssertionDomains()
|
|
74
|
+
.some(domain => verify(null, approvalSigningMessage(payload, domain), key, signatureBytes));
|
|
53
75
|
}
|
|
54
76
|
catch {
|
|
55
77
|
valid = false;
|
|
@@ -20,6 +20,8 @@ export const NATIVE_COMPLETION_REGISTRY = {
|
|
|
20
20
|
completionMode: 'buffered-synchronous-http', timeoutCeilingSeconds: 3600, resourceClassifier: 'node-scheduler' },
|
|
21
21
|
{ id: 'ardent-action', method: 'POST', pathMatcher: { kind: 'exact', path: '/ardent/action' },
|
|
22
22
|
completionMode: 'buffered-synchronous-http', timeoutCeilingSeconds: 900, resourceClassifier: 'node-ardent' },
|
|
23
|
+
{ id: 'agent-self-upgrade', method: 'POST', pathMatcher: { kind: 'exact', path: '/agent-upgrade' },
|
|
24
|
+
completionMode: 'buffered-synchronous-http', timeoutCeilingSeconds: 900, resourceClassifier: 'node-filesystem' },
|
|
23
25
|
],
|
|
24
26
|
};
|
|
25
27
|
export const NATIVE_COMPLETION_REGISTRY_SHA256 = createHash('sha256')
|
package/dist/server.js
CHANGED
|
@@ -30,6 +30,8 @@ import { libraryRouter } from './routes/library.js';
|
|
|
30
30
|
import { rolesRouter } from './routes/roles.js';
|
|
31
31
|
import { notificationsRouter } from './routes/notifications.js';
|
|
32
32
|
import { selfUpdateRouter } from './routes/self-update.js';
|
|
33
|
+
import { systemUpdatesRouter } from './routes/system-updates.js';
|
|
34
|
+
import { agentUpgradesRouter } from './routes/agent-upgrades.js';
|
|
33
35
|
import { scanRouter } from './routes/scan.js';
|
|
34
36
|
import { terminalTicketRouter } from './routes/terminal-ticket.js';
|
|
35
37
|
import { usersRouter, authRouter } from './routes/users.js';
|
|
@@ -93,6 +95,7 @@ const v1 = new Hono();
|
|
|
93
95
|
// ── Rate limiting on sensitive endpoints (permissive — Beta) ──
|
|
94
96
|
v1.use('/auth/*', rateLimitMiddleware({ windowMs: 60_000, max: 20, keyPrefix: 'rl:auth', failClosed: true }));
|
|
95
97
|
v1.use('/deploy/*', rateLimitMiddleware({ windowMs: 60_000, max: 10, keyPrefix: 'rl:deploy', failClosed: true }));
|
|
98
|
+
v1.use('/agents/upgrade', rateLimitMiddleware({ windowMs: 60_000, max: 10, keyPrefix: 'rl:agent-upgrade', failClosed: true }));
|
|
96
99
|
v1.use('/self-update/*', rateLimitMiddleware({ windowMs: 60_000, max: 10, keyPrefix: 'rl:selfupdate', failClosed: true }));
|
|
97
100
|
v1.use('/commands/*', rateLimitMiddleware({ windowMs: 60_000, max: 60, keyPrefix: 'rl:commands' }));
|
|
98
101
|
v1.use('/quick-command/*', rateLimitMiddleware({ windowMs: 60_000, max: 30, keyPrefix: 'rl:quickcmd' }));
|
|
@@ -125,6 +128,8 @@ v1.route('/library', libraryRouter);
|
|
|
125
128
|
v1.route('/roles', rolesRouter);
|
|
126
129
|
v1.route('/notifications', notificationsRouter);
|
|
127
130
|
v1.route('/self-update', selfUpdateRouter);
|
|
131
|
+
v1.route('/system/updates', systemUpdatesRouter);
|
|
132
|
+
v1.route('/agents/upgrade', agentUpgradesRouter);
|
|
128
133
|
v1.route('/scan', scanRouter);
|
|
129
134
|
v1.route('/terminal', terminalTicketRouter);
|
|
130
135
|
v1.route('/users', usersRouter);
|
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Server-driven agent upgrades (MAGNUS_AGENT_UPGRADES). The server pins one
|
|
3
|
+
* exact version and its npm integrity, asks one canary agent to stage it,
|
|
4
|
+
* waits until that agent reports the version on trial as confirmed, then
|
|
5
|
+
* upgrades the rest a few at a time. The first revert or failure stops the
|
|
6
|
+
* run. Agents decide for themselves whether they can upgrade
|
|
7
|
+
* (MAGNUS_AGENT_SELF_UPGRADE, launcher layout) and report it with every
|
|
8
|
+
* heartbeat, so the console can grey out the ones that cannot.
|
|
9
|
+
*/
|
|
10
|
+
import { randomUUID } from 'node:crypto';
|
|
11
|
+
import { AGENT_SELF_UPGRADE_PROTOCOL, agentSelfUpgradeStatusSchema, checkUpgradeAllowed, compareVersions, isVersion, } from '@hivemagnus/magnus-contracts';
|
|
12
|
+
import { upgradeSettings } from '../config/features.js';
|
|
13
|
+
import { AGENT_VERSION } from '../config/version.js';
|
|
14
|
+
import { getDb } from '../db/client.js';
|
|
15
|
+
import { HttpError } from '../middleware/error-handler.js';
|
|
16
|
+
import { NodeApiClient } from '../sdk/node-api-client.js';
|
|
17
|
+
import { getNodeManagerSecret } from '../utils/env.js';
|
|
18
|
+
import { getNodeBaseUrl } from '../utils/node-url.js';
|
|
19
|
+
import { SENTINEL_NODE_IDS } from '../utils/sentinel-nodes.js';
|
|
20
|
+
import { logAuditEvent } from './audit-service.js';
|
|
21
|
+
export const AGENT_PACKAGE = '@hivemagnus/magnus-agent';
|
|
22
|
+
/** Heartbeats are 30 s apart; older than this and the node counts as offline for upgrades. */
|
|
23
|
+
const FRESH_HEARTBEAT_MS = 2 * 60_000;
|
|
24
|
+
/** Install time plus the agent's confirmation window (default 180 s), with margin. */
|
|
25
|
+
const CONFIRM_TIMEOUT_MS = 20 * 60_000;
|
|
26
|
+
const POLL_MS = 5_000;
|
|
27
|
+
const RUN_TTL_SECONDS = 7 * 24 * 60 * 60;
|
|
28
|
+
function parseMetadata(raw) {
|
|
29
|
+
try {
|
|
30
|
+
const parsed = JSON.parse(raw ?? '{}');
|
|
31
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed) ? parsed : {};
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return {};
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
export function storedSelfUpgrade(metadataRaw) {
|
|
38
|
+
const value = parseMetadata(metadataRaw)['agentSelfUpgrade'];
|
|
39
|
+
const parsed = agentSelfUpgradeStatusSchema.safeParse(value);
|
|
40
|
+
if (!parsed.success)
|
|
41
|
+
return null;
|
|
42
|
+
const extra = value;
|
|
43
|
+
return { ...parsed.data, nodeVersion: typeof extra.nodeVersion === 'string' ? extra.nodeVersion : null,
|
|
44
|
+
observedAt: typeof extra.observedAt === 'string' ? extra.observedAt : new Date(0).toISOString() };
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Store the self-upgrade report from a heartbeat, keep Agent.status in step
|
|
48
|
+
* (UPDATING while a version is staged or on trial) and audit each revert once.
|
|
49
|
+
*/
|
|
50
|
+
export async function recordSelfUpgradeReport(node, body) {
|
|
51
|
+
const parsed = agentSelfUpgradeStatusSchema.safeParse(body.selfUpgrade);
|
|
52
|
+
if (!parsed.success)
|
|
53
|
+
return;
|
|
54
|
+
const db = getDb();
|
|
55
|
+
const row = await db.node.findUnique({ where: { id: node.id }, select: { metadata: true } });
|
|
56
|
+
const metadata = parseMetadata(row?.metadata);
|
|
57
|
+
const previous = storedSelfUpgrade(row?.metadata);
|
|
58
|
+
const nodeVersion = typeof body.nodeVersion === 'string' ? body.nodeVersion.slice(0, 32) : null;
|
|
59
|
+
const next = { ...parsed.data, nodeVersion, observedAt: new Date().toISOString() };
|
|
60
|
+
const unchanged = previous && JSON.stringify({ ...previous, observedAt: '' }) === JSON.stringify({ ...next, observedAt: '' })
|
|
61
|
+
&& Date.now() - Date.parse(previous.observedAt) < 10 * 60_000;
|
|
62
|
+
if (!unchanged) {
|
|
63
|
+
await db.node.update({ where: { id: node.id }, data: { metadata: JSON.stringify({ ...metadata, agentSelfUpgrade: next, nodeVersion }) } });
|
|
64
|
+
}
|
|
65
|
+
const state = parsed.data.upgrade?.state;
|
|
66
|
+
const updating = state === 'staged' || state === 'trial';
|
|
67
|
+
await db.agent.updateMany({
|
|
68
|
+
where: { nodeId: node.id, status: updating ? { not: 'UPDATING' } : 'UPDATING' },
|
|
69
|
+
data: { status: updating ? 'UPDATING' : 'RUNNING' },
|
|
70
|
+
});
|
|
71
|
+
const upgrade = parsed.data.upgrade;
|
|
72
|
+
if (upgrade && (upgrade.state === 'reverted' || upgrade.state === 'confirmed')
|
|
73
|
+
&& upgrade.requestId && previous?.upgrade?.requestId === upgrade.requestId && previous.upgrade.state !== upgrade.state) {
|
|
74
|
+
await logAuditEvent({
|
|
75
|
+
action: upgrade.state === 'reverted' ? 'agent.upgrade_reverted' : 'agent.upgrade',
|
|
76
|
+
actorId: 'system:agent-upgrade', actorType: 'system', targetType: 'node', targetId: node.id,
|
|
77
|
+
details: { node: node.name, from: upgrade.from, to: upgrade.to, requestId: upgrade.requestId, result: upgrade.state, reason: upgrade.reason ?? null },
|
|
78
|
+
}).catch(() => undefined);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/** The exact version (default: the one this server ships) and its npm integrity. */
|
|
82
|
+
export async function resolveAgentRelease(options = {}) {
|
|
83
|
+
const version = options.version ?? AGENT_VERSION;
|
|
84
|
+
if (!isVersion(version))
|
|
85
|
+
throw new HttpError(400, 'INVALID_VERSION', `${version} is not an X.Y.Z version`);
|
|
86
|
+
if (options.integrity) {
|
|
87
|
+
if (!/^sha512-[A-Za-z0-9+/]+={0,2}$/.test(options.integrity))
|
|
88
|
+
throw new HttpError(400, 'INVALID_INTEGRITY', 'integrity must be sha512-<base64>');
|
|
89
|
+
return { version, integrity: options.integrity };
|
|
90
|
+
}
|
|
91
|
+
const registry = upgradeSettings().updateRegistry;
|
|
92
|
+
const url = `${registry}/${AGENT_PACKAGE.replace('/', '%2f')}/${version}`;
|
|
93
|
+
let response;
|
|
94
|
+
try {
|
|
95
|
+
response = await (options.fetchImpl ?? fetch)(url, { headers: { accept: 'application/json' }, signal: AbortSignal.timeout(15_000) });
|
|
96
|
+
}
|
|
97
|
+
catch (error) {
|
|
98
|
+
throw new HttpError(502, 'REGISTRY_UNREACHABLE', `Cannot read ${AGENT_PACKAGE}@${version} from ${registry}: ${error instanceof Error ? error.message : String(error)}. Pass the integrity explicitly for offline installs.`);
|
|
99
|
+
}
|
|
100
|
+
if (response.status === 404)
|
|
101
|
+
throw new HttpError(404, 'AGENT_VERSION_NOT_PUBLISHED', `${AGENT_PACKAGE}@${version} is not published on ${registry}`);
|
|
102
|
+
if (!response.ok)
|
|
103
|
+
throw new HttpError(502, 'REGISTRY_ERROR', `${registry} answered ${response.status} for ${AGENT_PACKAGE}@${version}`);
|
|
104
|
+
const document = await response.json();
|
|
105
|
+
const integrity = document.dist?.integrity;
|
|
106
|
+
if (!integrity?.startsWith('sha512-'))
|
|
107
|
+
throw new HttpError(502, 'REGISTRY_ERROR', `${AGENT_PACKAGE}@${version} has no sha512 integrity`);
|
|
108
|
+
return { version, integrity };
|
|
109
|
+
}
|
|
110
|
+
/** Pure, for tests: why one node can or cannot take `target` now. */
|
|
111
|
+
export function classifyCandidate(row, target, now = Date.now()) {
|
|
112
|
+
const version = row.agent?.version ?? 'unknown';
|
|
113
|
+
const report = storedSelfUpgrade(row.metadata);
|
|
114
|
+
const base = { nodeId: row.id, name: row.name, version, upgrade: report?.upgrade ?? null, layout: report?.layout ?? null };
|
|
115
|
+
const no = (reason) => ({ ...base, eligible: false, reason });
|
|
116
|
+
if (row.status !== 'ONLINE' || !row.lastHeartbeatAt || now - row.lastHeartbeatAt.getTime() > FRESH_HEARTBEAT_MS)
|
|
117
|
+
return no('offline or no recent heartbeat');
|
|
118
|
+
if (!isVersion(version))
|
|
119
|
+
return no('agent version unknown');
|
|
120
|
+
if (compareVersions(version, target) === 0)
|
|
121
|
+
return no('already on this version');
|
|
122
|
+
const decision = checkUpgradeAllowed(version, target);
|
|
123
|
+
if (!decision.allowed)
|
|
124
|
+
return no(decision.kind === 'downgrade' ? `runs ${version}, newer than ${target}` : `${version} -> ${target} crosses a major version; upgrade this node by hand`);
|
|
125
|
+
if (!report)
|
|
126
|
+
return no(`magnus-agent ${version} cannot upgrade itself (needs 1.1.0 or later and \`sudo magnus-agent install-service\`)`);
|
|
127
|
+
if (report.protocol < AGENT_SELF_UPGRADE_PROTOCOL)
|
|
128
|
+
return no('agent self-upgrade protocol too old');
|
|
129
|
+
if (!report.enabled)
|
|
130
|
+
return no(report.reason ?? 'self-upgrade disabled on this node');
|
|
131
|
+
if (report.upgrade && (report.upgrade.state === 'staged' || report.upgrade.state === 'trial'))
|
|
132
|
+
return no(`an upgrade to ${report.upgrade.to} is in progress`);
|
|
133
|
+
return { ...base, eligible: true, reason: null };
|
|
134
|
+
}
|
|
135
|
+
export async function listUpgradeCandidates(target, nodeIds) {
|
|
136
|
+
const rows = await getDb().node.findMany({
|
|
137
|
+
where: { id: { notIn: SENTINEL_NODE_IDS, ...(nodeIds?.length ? { in: [...nodeIds] } : {}) }, status: { not: 'DECOMMISSIONED' } },
|
|
138
|
+
select: { id: true, name: true, status: true, lastHeartbeatAt: true, metadata: true, agent: { select: { version: true } } },
|
|
139
|
+
orderBy: { name: 'asc' },
|
|
140
|
+
});
|
|
141
|
+
return rows.map(row => classifyCandidate(row, target));
|
|
142
|
+
}
|
|
143
|
+
const runs = new Map();
|
|
144
|
+
async function saveRun(run) {
|
|
145
|
+
runs.set(run.id, run);
|
|
146
|
+
try {
|
|
147
|
+
const { getRedis } = await import('../queue/redis.js');
|
|
148
|
+
await getRedis().set(`agent-upgrade-run:${run.id}`, JSON.stringify(run), 'EX', RUN_TTL_SECONDS);
|
|
149
|
+
await getRedis().set('agent-upgrade-run:latest', run.id, 'EX', RUN_TTL_SECONDS);
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
// in-memory only
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
export async function getUpgradeRun(id) {
|
|
156
|
+
const local = runs.get(id);
|
|
157
|
+
if (local)
|
|
158
|
+
return local;
|
|
159
|
+
try {
|
|
160
|
+
const { getRedis } = await import('../queue/redis.js');
|
|
161
|
+
const raw = await getRedis().get(`agent-upgrade-run:${id}`);
|
|
162
|
+
return raw ? JSON.parse(raw) : null;
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
return null;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
export async function latestUpgradeRun() {
|
|
169
|
+
try {
|
|
170
|
+
const { getRedis } = await import('../queue/redis.js');
|
|
171
|
+
const id = await getRedis().get('agent-upgrade-run:latest');
|
|
172
|
+
return id ? getUpgradeRun(id) : null;
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
return [...runs.values()].at(-1) ?? null;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
function clientFor(node) {
|
|
179
|
+
return new NodeApiClient({ baseUrl: getNodeBaseUrl(node.ipAddress, node.port), resourceNodeId: node.id, nodeToken: getNodeManagerSecret(),
|
|
180
|
+
timeout: 15 * 60_000 });
|
|
181
|
+
}
|
|
182
|
+
export function defaultDeps() {
|
|
183
|
+
return {
|
|
184
|
+
request: async (nodeId, body) => {
|
|
185
|
+
const node = await getDb().node.findUniqueOrThrow({ where: { id: nodeId }, select: { id: true, ipAddress: true, port: true } });
|
|
186
|
+
await clientFor(node).upgradeAgent(body);
|
|
187
|
+
},
|
|
188
|
+
observe: async (nodeId) => {
|
|
189
|
+
const row = await getDb().node.findUnique({ where: { id: nodeId }, select: { metadata: true, agent: { select: { version: true } } } });
|
|
190
|
+
return { version: row?.agent?.version ?? null, report: storedSelfUpgrade(row?.metadata) };
|
|
191
|
+
},
|
|
192
|
+
sleep: ms => new Promise(resolve => setTimeout(resolve, ms)),
|
|
193
|
+
now: () => Date.now(),
|
|
194
|
+
save: saveRun,
|
|
195
|
+
confirmTimeoutMs: CONFIRM_TIMEOUT_MS,
|
|
196
|
+
pollMs: POLL_MS,
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
/** Ask one node to upgrade and wait for its confirmed (or reverted) report. */
|
|
200
|
+
async function upgradeOne(run, entry, deps) {
|
|
201
|
+
entry.state = 'requested';
|
|
202
|
+
await deps.save(run);
|
|
203
|
+
try {
|
|
204
|
+
await deps.request(entry.nodeId, { version: run.version, integrity: run.integrity, requestId: entry.requestId, packageName: AGENT_PACKAGE });
|
|
205
|
+
}
|
|
206
|
+
catch (error) {
|
|
207
|
+
entry.state = 'failed';
|
|
208
|
+
entry.detail = (error instanceof Error ? error.message : String(error)).slice(0, 500);
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
const deadline = deps.now() + deps.confirmTimeoutMs;
|
|
212
|
+
while (deps.now() < deadline) {
|
|
213
|
+
await deps.sleep(deps.pollMs);
|
|
214
|
+
const { version, report } = await deps.observe(entry.nodeId);
|
|
215
|
+
const upgrade = report?.upgrade;
|
|
216
|
+
if (upgrade?.requestId === entry.requestId && upgrade.state === 'reverted') {
|
|
217
|
+
entry.state = 'reverted';
|
|
218
|
+
entry.detail = upgrade.reason ?? 'the agent switched back to its previous version';
|
|
219
|
+
return;
|
|
220
|
+
}
|
|
221
|
+
if (version === run.version && (upgrade?.requestId !== entry.requestId || upgrade.state === 'confirmed')) {
|
|
222
|
+
entry.state = 'confirmed';
|
|
223
|
+
entry.detail = null;
|
|
224
|
+
return;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
entry.state = 'failed';
|
|
228
|
+
entry.detail = `no confirmation within ${Math.round(deps.confirmTimeoutMs / 60_000)} min`;
|
|
229
|
+
}
|
|
230
|
+
/** Canary first, then batches of `concurrency`. Stops at the first node that does not confirm. */
|
|
231
|
+
export async function executeUpgradeRun(run, deps) {
|
|
232
|
+
const queue = run.nodes.filter(node => node.state === 'pending');
|
|
233
|
+
const [canary, ...rest] = queue;
|
|
234
|
+
const finish = async (status, message) => {
|
|
235
|
+
for (const node of run.nodes)
|
|
236
|
+
if (node.state === 'pending')
|
|
237
|
+
node.state = 'skipped';
|
|
238
|
+
run.status = status;
|
|
239
|
+
run.message = message;
|
|
240
|
+
run.finishedAt = new Date(deps.now()).toISOString();
|
|
241
|
+
await deps.save(run);
|
|
242
|
+
return run;
|
|
243
|
+
};
|
|
244
|
+
if (!canary)
|
|
245
|
+
return finish('succeeded', 'no eligible nodes');
|
|
246
|
+
await upgradeOne(run, canary, deps);
|
|
247
|
+
await deps.save(run);
|
|
248
|
+
if (canary.state !== 'confirmed')
|
|
249
|
+
return finish('failed', `canary ${canary.name} did not confirm (${canary.state}: ${canary.detail ?? ''}); the other nodes were not touched`);
|
|
250
|
+
for (let index = 0; index < rest.length; index += run.concurrency) {
|
|
251
|
+
const batch = rest.slice(index, index + run.concurrency);
|
|
252
|
+
await Promise.all(batch.map(node => upgradeOne(run, node, deps)));
|
|
253
|
+
await deps.save(run);
|
|
254
|
+
const bad = batch.find(node => node.state !== 'confirmed');
|
|
255
|
+
if (bad)
|
|
256
|
+
return finish('failed', `${bad.name} did not confirm (${bad.state}: ${bad.detail ?? ''}); remaining nodes were skipped`);
|
|
257
|
+
}
|
|
258
|
+
return finish('succeeded', `${run.nodes.filter(node => node.state === 'confirmed').length} agent(s) upgraded to ${run.version}`);
|
|
259
|
+
}
|
|
260
|
+
/** Validate, create the run and start it in the background. */
|
|
261
|
+
export async function startUpgradeRun(options) {
|
|
262
|
+
const settings = upgradeSettings();
|
|
263
|
+
if (!settings.agentUpgrades)
|
|
264
|
+
throw new HttpError(403, 'AGENT_UPGRADES_DISABLED', 'Server-driven agent upgrades are off (MAGNUS_AGENT_UPGRADES=false)');
|
|
265
|
+
const release = await resolveAgentRelease({ version: options.version, integrity: options.integrity });
|
|
266
|
+
const candidates = await listUpgradeCandidates(release.version, options.nodeIds);
|
|
267
|
+
const eligible = candidates.filter(candidate => candidate.eligible);
|
|
268
|
+
if (eligible.length === 0) {
|
|
269
|
+
const reasons = candidates.slice(0, 5).map(candidate => `${candidate.name}: ${candidate.reason}`).join('; ');
|
|
270
|
+
throw new HttpError(409, 'NO_ELIGIBLE_AGENTS', `No selected agent can be upgraded now${reasons ? ` (${reasons})` : ''}`);
|
|
271
|
+
}
|
|
272
|
+
const running = await latestUpgradeRun();
|
|
273
|
+
if (running?.status === 'running')
|
|
274
|
+
throw new HttpError(409, 'AGENT_UPGRADE_RUNNING', `Upgrade run ${running.id} is still in progress`);
|
|
275
|
+
const run = {
|
|
276
|
+
id: randomUUID(), version: release.version, integrity: release.integrity, actorId: options.actorId, status: 'running',
|
|
277
|
+
startedAt: new Date().toISOString(), finishedAt: null, concurrency: settings.agentUpgradeConcurrency, message: null,
|
|
278
|
+
nodes: eligible.map(candidate => ({ nodeId: candidate.nodeId, name: candidate.name, from: candidate.version, state: 'pending', requestId: randomUUID(), detail: null })),
|
|
279
|
+
};
|
|
280
|
+
const deps = options.deps ?? defaultDeps();
|
|
281
|
+
await deps.save(run);
|
|
282
|
+
await logAuditEvent({ action: 'agent.upgrade', actorId: options.actorId, targetType: 'fleet', targetId: run.id,
|
|
283
|
+
details: { version: run.version, nodes: run.nodes.map(node => node.name) } }).catch(() => undefined);
|
|
284
|
+
const done = executeUpgradeRun(run, deps).catch(async (error) => {
|
|
285
|
+
run.status = 'failed';
|
|
286
|
+
run.message = error instanceof Error ? error.message : String(error);
|
|
287
|
+
run.finishedAt = new Date().toISOString();
|
|
288
|
+
await deps.save(run);
|
|
289
|
+
return run;
|
|
290
|
+
});
|
|
291
|
+
return { run, candidates, done };
|
|
292
|
+
}
|
|
293
|
+
//# sourceMappingURL=agent-upgrade-service.js.map
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { compareVersions, isVersion } from '@hivemagnus/magnus-contracts';
|
|
2
|
+
import { agentFixCommand } from './update-check-service.js';
|
|
3
|
+
export function agentVersionStatus(agentVersion, expected) {
|
|
4
|
+
if (!agentVersion || !isVersion(agentVersion) || !isVersion(expected))
|
|
5
|
+
return { updateAvailable: false, relation: 'unknown' };
|
|
6
|
+
const order = compareVersions(agentVersion, expected);
|
|
7
|
+
return { updateAvailable: order < 0, relation: order === 0 ? 'current' : order < 0 ? 'behind' : 'ahead' };
|
|
8
|
+
}
|
|
9
|
+
const warned = new Map();
|
|
10
|
+
/**
|
|
11
|
+
* One loud log line per node and version pair, with the exact fix. Mixed
|
|
12
|
+
* versions work within a major, but features and fixes need matching agents.
|
|
13
|
+
*/
|
|
14
|
+
export function warnOnAgentVersionMismatch(node, agentVersion, expected, log = line => console.warn(line)) {
|
|
15
|
+
const status = agentVersionStatus(agentVersion, expected);
|
|
16
|
+
if (status.relation !== 'behind' && status.relation !== 'ahead') {
|
|
17
|
+
warned.delete(node.id);
|
|
18
|
+
return status;
|
|
19
|
+
}
|
|
20
|
+
const key = `${agentVersion}->${expected}`;
|
|
21
|
+
if (warned.get(node.id) === key)
|
|
22
|
+
return status;
|
|
23
|
+
warned.set(node.id, key);
|
|
24
|
+
const major = agentVersion.split('.')[0] !== expected.split('.')[0];
|
|
25
|
+
log(status.relation === 'behind'
|
|
26
|
+
? `[agents] VERSION MISMATCH: node ${node.name} runs magnus-agent ${agentVersion}, this server ships ${expected}`
|
|
27
|
+
+ `${major ? ' (different major version: some features will not work)' : ''}. Fix: ${agentFixCommand([node.name])}`
|
|
28
|
+
: `[agents] VERSION MISMATCH: node ${node.name} runs magnus-agent ${agentVersion}, newer than this server (${expected}). Upgrade the server: magnus upgrade`);
|
|
29
|
+
return status;
|
|
30
|
+
}
|
|
31
|
+
//# sourceMappingURL=agent-version-warnings.js.map
|
|
@@ -24,16 +24,17 @@ export async function isGitRepo() {
|
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
26
|
let lastChecked = null;
|
|
27
|
-
/** Read the
|
|
27
|
+
/** Read the versions from the package files on disk (they change after a pull and rebuild). */
|
|
28
28
|
async function readVersionFromDisk() {
|
|
29
|
-
const
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
29
|
+
const read = async (file) => {
|
|
30
|
+
try {
|
|
31
|
+
return JSON.parse(await readFile(path.join(REPO_ROOT, file), 'utf-8')).version ?? 'unknown';
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return 'unknown';
|
|
35
|
+
}
|
|
36
36
|
};
|
|
37
|
+
return { agent: await read('agent-version.json'), controlApi: await read('package.json') };
|
|
37
38
|
}
|
|
38
39
|
/** Run a git command in the repo root. */
|
|
39
40
|
async function git(...args) {
|
|
@@ -195,6 +196,8 @@ export async function getUpdateStatus() {
|
|
|
195
196
|
gitCommit: commit,
|
|
196
197
|
lastChecked,
|
|
197
198
|
isGitRepo: gitRepo,
|
|
199
|
+
legacyGitSelfUpdate: (await import('../config/features.js')).upgradeSettings().legacyGitSelfUpdate,
|
|
200
|
+
upgradeCommand: 'magnus upgrade',
|
|
198
201
|
};
|
|
199
202
|
}
|
|
200
203
|
//# sourceMappingURL=self-update-service.js.map
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Opt-out update notices (MAGNUS_UPDATE_CHECK). Once a day the server reads
|
|
3
|
+
* the npm registry document of @hivemagnus/magnus (one anonymous GET; nothing
|
|
4
|
+
* about this install is sent) and summarizes the releases between the running
|
|
5
|
+
* version and the newest one of the same major: release notes, new required
|
|
6
|
+
* settings and migrations that are slow or one-way. A newer major is reported
|
|
7
|
+
* separately and never offered as a one-click upgrade.
|
|
8
|
+
*/
|
|
9
|
+
import { compareVersions, isVersion, latestVersion, manifestsFromPackument, summarizeUpgradePath, } from '@hivemagnus/magnus-contracts';
|
|
10
|
+
import { upgradeSettings } from '../config/features.js';
|
|
11
|
+
import { AGENT_VERSION, CONTROL_API_VERSION } from '../config/version.js';
|
|
12
|
+
export const META_PACKAGE = '@hivemagnus/magnus';
|
|
13
|
+
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
14
|
+
const CACHE_KEY = 'update-check:v1';
|
|
15
|
+
export function emptyResult(enabled, error = null) {
|
|
16
|
+
return {
|
|
17
|
+
enabled, checkedAt: null, error, current: CONTROL_API_VERSION, latest: null, latestMajor: null, updateAvailable: false,
|
|
18
|
+
breaking: false, releases: [], newRequiredEnv: [], attention: [], command: 'magnus upgrade',
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export function summarizePackument(packument, current, env, checkedAt) {
|
|
22
|
+
const versions = Object.keys(packument?.versions ?? {}).filter(isVersion);
|
|
23
|
+
const manifests = manifestsFromPackument(packument);
|
|
24
|
+
const sameMajor = latestVersion(versions, { sameMajorAs: current });
|
|
25
|
+
const newest = latestVersion(versions);
|
|
26
|
+
const latest = sameMajor && compareVersions(sameMajor, current) > 0 ? sameMajor : null;
|
|
27
|
+
const latestMajor = newest && compareVersions(newest, current) > 0 && newest.split('.')[0] !== current.split('.')[0] ? newest : null;
|
|
28
|
+
const path = latest ? summarizeUpgradePath(manifests, current, latest) : null;
|
|
29
|
+
return {
|
|
30
|
+
enabled: true,
|
|
31
|
+
checkedAt,
|
|
32
|
+
error: null,
|
|
33
|
+
current,
|
|
34
|
+
latest,
|
|
35
|
+
latestMajor,
|
|
36
|
+
updateAvailable: latest !== null,
|
|
37
|
+
breaking: path?.breaking ?? false,
|
|
38
|
+
releases: (path?.releases ?? []).map(release => ({
|
|
39
|
+
version: release.version, summary: release.summary, notesUrl: release.notesUrl ?? null, breaking: release.breaking,
|
|
40
|
+
})),
|
|
41
|
+
newRequiredEnv: (path?.newRequiredEnv ?? []).map(variable => ({
|
|
42
|
+
name: variable.name, component: variable.component, secret: variable.secret, description: variable.description,
|
|
43
|
+
missingOnServer: variable.component === 'server' && !env[variable.name],
|
|
44
|
+
})),
|
|
45
|
+
attention: (path?.attention ?? []).map(migration => ({ name: migration.name, impact: migration.impact, note: migration.note, version: migration.version })),
|
|
46
|
+
command: latest ? `magnus upgrade --to ${latest}` : 'magnus upgrade',
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
/** One registry read. Errors are reported in the result, never thrown. */
|
|
50
|
+
export async function checkForUpdate(options = {}) {
|
|
51
|
+
const env = options.env ?? process.env;
|
|
52
|
+
const settings = upgradeSettings(env);
|
|
53
|
+
const current = options.current ?? CONTROL_API_VERSION;
|
|
54
|
+
if (!settings.updateCheck)
|
|
55
|
+
return { ...emptyResult(false), current };
|
|
56
|
+
const url = `${settings.updateRegistry}/${META_PACKAGE.replace('/', '%2f')}`;
|
|
57
|
+
const checkedAt = (options.now ?? (() => new Date()))().toISOString();
|
|
58
|
+
try {
|
|
59
|
+
const response = await (options.fetchImpl ?? fetch)(url, { headers: { accept: 'application/json' }, signal: AbortSignal.timeout(15_000) });
|
|
60
|
+
if (response.status === 404)
|
|
61
|
+
return { ...emptyResult(true), current, checkedAt };
|
|
62
|
+
if (!response.ok)
|
|
63
|
+
return { ...emptyResult(true, `registry answered ${response.status}`), current, checkedAt };
|
|
64
|
+
return summarizePackument(await response.json(), current, env, checkedAt);
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
return { ...emptyResult(true, `registry unreachable: ${error instanceof Error ? error.message : String(error)}`), current, checkedAt };
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
/** The `updates` line of `check`: never a failure, only a notice. */
|
|
71
|
+
export function updateResult(update) {
|
|
72
|
+
if (!update.enabled)
|
|
73
|
+
return { name: 'updates', status: 'pass', message: 'registry check off (MAGNUS_UPDATE_CHECK=false)' };
|
|
74
|
+
if (update.error)
|
|
75
|
+
return { name: 'updates', status: 'warn', message: `could not check: ${update.error}` };
|
|
76
|
+
const major = update.latestMajor ? `; ${update.latestMajor} is a new major version (see UPGRADING.md)` : '';
|
|
77
|
+
if (!update.updateAvailable)
|
|
78
|
+
return { name: 'updates', status: 'pass', message: `${update.current} is the newest ${update.current.split('.')[0]}.x release${major}` };
|
|
79
|
+
const env = update.newRequiredEnv.map(variable => `${variable.name}${variable.missingOnServer ? ' (not set here)' : ''}`);
|
|
80
|
+
return {
|
|
81
|
+
name: 'updates',
|
|
82
|
+
status: 'warn',
|
|
83
|
+
message: `${update.latest} is available: ${update.command}`
|
|
84
|
+
+ (env.length ? `; new required settings: ${env.join(', ')}` : '')
|
|
85
|
+
+ (update.attention.length ? `; slow or one-way migrations: ${update.attention.map(m => `${m.name} (${m.impact})`).join(', ')}` : '')
|
|
86
|
+
+ major,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
let cached = null;
|
|
90
|
+
let timer;
|
|
91
|
+
async function redis() {
|
|
92
|
+
const { getRedis } = await import('../queue/redis.js');
|
|
93
|
+
return getRedis();
|
|
94
|
+
}
|
|
95
|
+
async function store(result) {
|
|
96
|
+
cached = result;
|
|
97
|
+
try {
|
|
98
|
+
await (await redis()).set(CACHE_KEY, JSON.stringify(result), 'PX', DAY_MS + 60 * 60 * 1000);
|
|
99
|
+
}
|
|
100
|
+
catch {
|
|
101
|
+
// The in-memory copy is enough; Redis only shares the result across restarts.
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
/** Latest known result; refreshes when there is none yet or `refresh` is set. */
|
|
105
|
+
export async function getUpdateCheck(options = {}) {
|
|
106
|
+
if (!upgradeSettings().updateCheck)
|
|
107
|
+
return emptyResult(false);
|
|
108
|
+
if (!options.refresh) {
|
|
109
|
+
if (cached && cached.current === CONTROL_API_VERSION)
|
|
110
|
+
return cached;
|
|
111
|
+
try {
|
|
112
|
+
const raw = await (await redis()).get(CACHE_KEY);
|
|
113
|
+
const parsed = raw ? JSON.parse(raw) : null;
|
|
114
|
+
if (parsed && parsed.current === CONTROL_API_VERSION) {
|
|
115
|
+
cached = parsed;
|
|
116
|
+
return parsed;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
// fall through to a live check
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
const result = await checkForUpdate();
|
|
124
|
+
await store(result);
|
|
125
|
+
return result;
|
|
126
|
+
}
|
|
127
|
+
export function startUpdateCheck() {
|
|
128
|
+
if (!upgradeSettings().updateCheck || timer)
|
|
129
|
+
return;
|
|
130
|
+
const run = () => {
|
|
131
|
+
void getUpdateCheck({ refresh: true }).then(result => {
|
|
132
|
+
if (result.updateAvailable)
|
|
133
|
+
console.log(`[updates] Magnus ${result.latest} is available (running ${result.current}): ${result.command}`);
|
|
134
|
+
if (result.latestMajor)
|
|
135
|
+
console.log(`[updates] Magnus ${result.latestMajor} (new major) is available; see UPGRADING.md`);
|
|
136
|
+
});
|
|
137
|
+
};
|
|
138
|
+
// First check a minute after startup (with jitter), then daily.
|
|
139
|
+
timer = setTimeout(function tick() {
|
|
140
|
+
run();
|
|
141
|
+
timer = setTimeout(tick, DAY_MS);
|
|
142
|
+
timer.unref();
|
|
143
|
+
}, 60_000 + Math.floor(Math.random() * 60_000));
|
|
144
|
+
timer.unref();
|
|
145
|
+
}
|
|
146
|
+
export function stopUpdateCheck() {
|
|
147
|
+
if (timer)
|
|
148
|
+
clearTimeout(timer);
|
|
149
|
+
timer = undefined;
|
|
150
|
+
}
|
|
151
|
+
/** Classify agent versions against the version this server ships with. */
|
|
152
|
+
export function classifyAgentVersions(agents, expected = AGENT_VERSION) {
|
|
153
|
+
const skew = { expected, current: 0, behind: [], ahead: [], unknown: [] };
|
|
154
|
+
for (const agent of agents) {
|
|
155
|
+
if (!isVersion(agent.version) || !isVersion(expected))
|
|
156
|
+
skew.unknown.push(agent);
|
|
157
|
+
else {
|
|
158
|
+
const order = compareVersions(agent.version, expected);
|
|
159
|
+
if (order === 0)
|
|
160
|
+
skew.current++;
|
|
161
|
+
else
|
|
162
|
+
(order < 0 ? skew.behind : skew.ahead).push(agent);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
return skew;
|
|
166
|
+
}
|
|
167
|
+
export async function agentVersionSkew() {
|
|
168
|
+
const { getDb } = await import('../db/client.js');
|
|
169
|
+
const agents = await getDb().agent.findMany({ select: { version: true, node: { select: { id: true, name: true } } } });
|
|
170
|
+
return classifyAgentVersions(agents.map(agent => ({ nodeId: agent.node.id, name: agent.node.name, version: agent.version })));
|
|
171
|
+
}
|
|
172
|
+
/** The fix for an agent that is behind, spelled out for logs, `check` and the console. */
|
|
173
|
+
export function agentFixCommand(nodeNames) {
|
|
174
|
+
return nodeNames.length > 0 && nodeNames.length <= 5
|
|
175
|
+
? `magnus-server agents upgrade --nodes ${nodeNames.join(',')}`
|
|
176
|
+
: 'magnus-server agents upgrade';
|
|
177
|
+
}
|
|
178
|
+
//# sourceMappingURL=update-check-service.js.map
|