@agentpulselabs/cli-plugin-pulse 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +2 -2
- package/src/commands/pulse/cutover.js +34 -10
- package/src/daemon/main.js +1 -0
- package/src/lib/cutover.js +51 -16
- package/src/lib/cutoverRequest.js +485 -0
- package/src/lib/cutoverWiring.js +6 -3
- package/src/lib/daemonProcess.js +43 -6
- package/src/lib/serviceManager.js +3 -1
- package/src/runner/runnerService.js +11 -2
package/README.md
CHANGED
|
@@ -89,6 +89,15 @@ These need your team's Pulse cloud-first flag on, set by a team owner or admin,
|
|
|
89
89
|
|
|
90
90
|
Everything sent goes through the same secret and path filter as the mirror, and the cloud filters it again.
|
|
91
91
|
|
|
92
|
+
## Cutover through the daemon (1.1.3)
|
|
93
|
+
|
|
94
|
+
`agentpulse pulse cutover` and `--rollback` ask the running Pulse daemon to do the work, so the command itself never needs `launchctl` or a permission prompt. Only `--dry-run`, which only reads, runs in the command.
|
|
95
|
+
|
|
96
|
+
- The command writes an owner-only `cutover.request` (`id`, `mode`, `requestedAt`, `requestedBy`), one at a time. It then follows `cutover.json`, printing each step as an in-process run would, and exits with the run's result: 0 when it is done, or 1 with the run's error code when the run fails, the daemon refuses the request, or nobody takes it within 2 minutes (the request is then withdrawn). It exits 2 when it cannot tell how the run ended: the daemon stopped, or 2 hours passed.
|
|
97
|
+
- The daemon takes one request at a time, and runs it beside its heartbeat. It refuses a request that is more than 10 minutes old, a duplicate, a malformed request, or a rollback with nothing to roll back; the reason is listed under `refusals` in `cutover.json`. It unloads only the old hub's LaunchAgents (`com.agentpulse.agents-hub`, `com.agentpulse.keepawake`), never its own, and then serves 127.0.0.1:8890 itself. `daemon-health.json` shows `cutover: { state, step, requestId }`. A restart asked for while a cutover runs waits until the cutover ends.
|
|
98
|
+
- If the daemon stops in the middle of a request, it finishes the request when it starts again, and records what it did under `recovery`. A cutover that had begun is rolled back, including one stopped after its last check passed. A rollback is run again unless it had begun loading the old hub. In that case the daemon only checks that the old hub holds the port, and otherwise leaves it to you, because restoring `hub.db` again could lose what the old hub has written since.
|
|
99
|
+
- With a daemon older than 1.1.3, or with `--in-process`, the command runs the cutover itself as 1.1.2 did, which needs `launchctl`.
|
|
100
|
+
|
|
92
101
|
## Bus identity and delivery states (1.1.1)
|
|
93
102
|
|
|
94
103
|
After cutover the runner hosts the bus, and `hub.mjs bus` works as before with these additions:
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agentpulselabs/cli-plugin-pulse",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.3",
|
|
4
4
|
"description": "AgentPulse CLI plugin for installing and operating the Pulse local connector (replaces the deprecated build-discipline Pulse plugin).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
"scripts": {
|
|
22
22
|
"test": "node --import ./test/support/register-loader.mjs --test \"test/*.test.js\" \"test/hub/*.test.js\"",
|
|
23
23
|
"contract:check": "node scripts/sync-contract.mjs --local",
|
|
24
|
-
"check": "node --check src/commands/pulse/install.js && node --check src/commands/pulse/pair.js && node --check src/commands/pulse/login.js && node --check src/commands/pulse/start.js && node --check src/commands/pulse/status.js && node --check src/commands/pulse/doctor.js && node --check src/commands/pulse/stop.js && node --check src/commands/pulse/restart.js && node --check src/lib/selfRestart.js && node --check src/lib/subprocess.js && node --check src/lib/daemonProcess.js && node --check src/commands/pulse/mirror.js && node --check src/commands/pulse/hub-write.js && node --check src/lib/localHubWriter.js && node --check src/lib/hubActions.js && node --check src/lib/localHub.js && node --check src/lib/pathRedaction.js && node --check src/lib/rulePacks.js && node --check src/lib/mirrorContract.js && node --check src/lib/graphWindows.js && node --check src/lib/mirrorSource.js && node --check src/lib/mirrorSync.js && node --check src/lib/mirrorService.js && node --check src/lib/mirrorDaemon.js && node --check src/lib/daemonComposition.js && node --check src/lib/daemonRuntime.js && node --check src/lib/daemonLoop.js && node --check src/daemon/main.js && node --check src/lib/sqlite.js && node --check src/lib/nodeFloor.js && node --check src/lib/runnerStore.js && node --check src/lib/hubDbReader.js && node --check src/lib/channelScanner.js && node --check src/lib/uploadRedactor.js && node --check src/lib/hubScan.js && node --check src/lib/outboxFlusher.js && node --check src/lib/hubSnapshot.js && node --check src/lib/reconcileView.js && node --check src/lib/reconcile.js && node --check src/lib/shadowService.js && node --check src/lib/cloudFirstCommands.js && node --check src/commands/pulse/import.js && node --check src/commands/pulse/shadow.js && node --check src/commands/pulse/parity.js && node --check src/runner/main.js && node --check src/runner/runnerProcess.js && node --check src/runner/facade.js && node --check src/runner/bridge.js && node --check src/runner/runnerChild.js && node --check src/runner/applier.js && node --check src/runner/capture.js && node --check src/runner/hubCapture.js && node --check src/clients/hub.mjs && node --check src/clients/hub-bridge.mjs && node --check src/runner/busIdentity.js && node --check src/runner/deliveryStates.js && node --check src/lib/doctorWarnings.js && node --check src/hub/lib/closure.mjs && node --check src/runner/channelDiscipline.js",
|
|
24
|
+
"check": "node --check src/commands/pulse/install.js && node --check src/commands/pulse/pair.js && node --check src/commands/pulse/login.js && node --check src/commands/pulse/start.js && node --check src/commands/pulse/status.js && node --check src/commands/pulse/doctor.js && node --check src/commands/pulse/stop.js && node --check src/commands/pulse/restart.js && node --check src/lib/selfRestart.js && node --check src/lib/subprocess.js && node --check src/lib/daemonProcess.js && node --check src/commands/pulse/mirror.js && node --check src/commands/pulse/hub-write.js && node --check src/lib/localHubWriter.js && node --check src/lib/hubActions.js && node --check src/lib/localHub.js && node --check src/lib/pathRedaction.js && node --check src/lib/rulePacks.js && node --check src/lib/mirrorContract.js && node --check src/lib/graphWindows.js && node --check src/lib/mirrorSource.js && node --check src/lib/mirrorSync.js && node --check src/lib/mirrorService.js && node --check src/lib/mirrorDaemon.js && node --check src/lib/daemonComposition.js && node --check src/lib/daemonRuntime.js && node --check src/lib/daemonLoop.js && node --check src/daemon/main.js && node --check src/lib/sqlite.js && node --check src/lib/nodeFloor.js && node --check src/lib/runnerStore.js && node --check src/lib/hubDbReader.js && node --check src/lib/channelScanner.js && node --check src/lib/uploadRedactor.js && node --check src/lib/hubScan.js && node --check src/lib/outboxFlusher.js && node --check src/lib/hubSnapshot.js && node --check src/lib/reconcileView.js && node --check src/lib/reconcile.js && node --check src/lib/shadowService.js && node --check src/lib/cutover.js && node --check src/lib/cutoverWiring.js && node --check src/lib/cutoverRequest.js && node --check src/commands/pulse/cutover.js && node --check src/lib/cloudFirstCommands.js && node --check src/commands/pulse/import.js && node --check src/commands/pulse/shadow.js && node --check src/commands/pulse/parity.js && node --check src/runner/main.js && node --check src/runner/runnerProcess.js && node --check src/runner/facade.js && node --check src/runner/bridge.js && node --check src/runner/runnerChild.js && node --check src/runner/applier.js && node --check src/runner/capture.js && node --check src/runner/hubCapture.js && node --check src/clients/hub.mjs && node --check src/clients/hub-bridge.mjs && node --check src/runner/busIdentity.js && node --check src/runner/deliveryStates.js && node --check src/lib/doctorWarnings.js && node --check src/hub/lib/closure.mjs && node --check src/runner/channelDiscipline.js",
|
|
25
25
|
"test:t2": "node --import ./test/support/register-loader.mjs --test --test-timeout=120000 \"test/t2/*.test.js\"",
|
|
26
26
|
"test:t3": "node --import ./test/support/register-loader.mjs --test --test-concurrency=1 --test-timeout=240000 \"test/t3/*.test.js\""
|
|
27
27
|
},
|
|
@@ -1,10 +1,13 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
1
2
|
import { Flags } from '@oclif/core';
|
|
2
3
|
import BaseCommand from '../../lib/BaseCommand.js';
|
|
3
4
|
import { defaultConfigDirectory } from '../../lib/application.js';
|
|
5
|
+
import { FOLLOW_UNKNOWN_EXIT, followCutoverRequest, takesCutoverRequests, writeCutoverRequest } from '../../lib/cutoverRequest.js';
|
|
4
6
|
import { createCutover } from '../../lib/cutoverWiring.js';
|
|
7
|
+
import { readOwnerOnlyJson } from '../../lib/storage.js';
|
|
5
8
|
|
|
6
9
|
export default class PulseCutover extends BaseCommand {
|
|
7
|
-
static description = 'Move this machine\'s local Agent Hub into the Pulse daemon (a short freeze, a backup kept), check first with --dry-run, or go back with --rollback (restores the backup; loses only what was written after the switch).';
|
|
10
|
+
static description = 'Move this machine\'s local Agent Hub into the Pulse daemon (a short freeze, a backup kept), check first with --dry-run, or go back with --rollback (restores the backup; loses only what was written after the switch). The running daemon does the work, so no launchctl or permission prompt is needed here; --in-process runs it in this process instead.';
|
|
8
11
|
static examples = [
|
|
9
12
|
'<%= config.bin %> pulse cutover --dry-run',
|
|
10
13
|
'<%= config.bin %> pulse cutover --smoke-channel agentpulse-agent-hub/cf-phase3',
|
|
@@ -14,26 +17,47 @@ export default class PulseCutover extends BaseCommand {
|
|
|
14
17
|
'dry-run': Flags.boolean({ description: 'Check everything and say what would change; change nothing', default: false }),
|
|
15
18
|
rollback: Flags.boolean({ description: 'Back to the local hub from the backup', default: false, exclusive: ['dry-run'] }),
|
|
16
19
|
'smoke-channel': Flags.string({ description: 'A task channel (<repo>/<task>) for the smoke test\'s one channel post' }),
|
|
20
|
+
'in-process': Flags.boolean({ description: 'Run the cutover in this process, as Pulse 1.1.2 did, instead of asking the daemon (needs launchctl)', default: false }),
|
|
17
21
|
};
|
|
18
22
|
|
|
19
23
|
async run() {
|
|
20
24
|
const { flags } = await this.parse(PulseCutover);
|
|
21
|
-
const
|
|
25
|
+
const configDir = this.configDir || defaultConfigDirectory();
|
|
26
|
+
const log = (event, details) => this.log(`${event} ${JSON.stringify(details)}`);
|
|
27
|
+
const inProcess = async () => this.cutover || createCutover({ configDir, log });
|
|
22
28
|
if (flags['dry-run']) {
|
|
23
|
-
const report = await
|
|
29
|
+
const report = await (await inProcess()).dryRun();
|
|
24
30
|
for (const check of report.checks) this.log(`${check.ok ? 'ok ' : 'FAIL'} ${check.name}${check.detail ? `: ${check.detail}` : ''}${!check.ok && check.fix ? ` (fix: ${check.fix})` : ''}`);
|
|
25
31
|
this.log(`Outage while it runs: ${report.outage.what}; at most ${Math.round(report.outage.boundMs / 60_000)} min.`);
|
|
26
32
|
this.log(report.message);
|
|
27
33
|
if (!report.ok) process.exitCode = 1;
|
|
28
34
|
return report;
|
|
29
35
|
}
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
36
|
+
const health = await readOwnerOnlyJson(path.join(configDir, 'daemon-health.json')).catch(() => null);
|
|
37
|
+
const viaDaemon = !flags['in-process'] && takesCutoverRequests(health);
|
|
38
|
+
if (!viaDaemon && !flags['in-process']) {
|
|
39
|
+
this.log(`The Pulse daemon ${health?.connectorVersion ? `(connector ${health.connectorVersion}) ` : ''}does not take cutover requests or is not running; running it in this process.`);
|
|
40
|
+
}
|
|
41
|
+
const mode = flags.rollback ? 'rollback' : 'cutover';
|
|
42
|
+
let result;
|
|
43
|
+
if (viaDaemon) {
|
|
44
|
+
const request = await writeCutoverRequest({ configDir, mode, smokeChannel: mode === 'cutover' ? flags['smoke-channel'] ?? null : null, health });
|
|
45
|
+
this.log(`Asked the Pulse daemon (pid ${health.pid}) to ${mode === 'rollback' ? 'roll back' : 'cut over'}: request ${request.id}.`);
|
|
46
|
+
try {
|
|
47
|
+
result = await followCutoverRequest({ configDir, request, log, ...this.follow });
|
|
48
|
+
} catch (error) {
|
|
49
|
+
if (error.exit === FOLLOW_UNKNOWN_EXIT) this.fail(error.code, error.message, error.exit);
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
} else {
|
|
53
|
+
const cutover = await inProcess();
|
|
54
|
+
result = mode === 'rollback' ? await cutover.rollback() : await cutover.run({ smokeChannel: flags['smoke-channel'] ?? null });
|
|
55
|
+
}
|
|
56
|
+
if (mode === 'rollback') {
|
|
57
|
+
this.log(result.message);
|
|
58
|
+
return result;
|
|
34
59
|
}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
return journal;
|
|
60
|
+
this.log(`Cut over: the Pulse daemon serves 127.0.0.1:${result.port}. The backup is ${result.backup.file}. Go back any time with \`agentpulse pulse cutover --rollback\`.`);
|
|
61
|
+
return result;
|
|
38
62
|
}
|
|
39
63
|
}
|
package/src/daemon/main.js
CHANGED
|
@@ -10,6 +10,7 @@ const MIRROR_FAILURES = new Set(['mirror_failed', 'mirror_blocked', 'mirror_stre
|
|
|
10
10
|
const ERROR_EVENTS = new Set([
|
|
11
11
|
'job_failed', 'job_refused', 'job_settle_failed', 'cycle_stalled', 'drain_timed_out', 'daemon_start_failed',
|
|
12
12
|
'daemon_shutdown_failed', 'source_watch_refused', 'source_watch_failed',
|
|
13
|
+
'cutover_failed', 'cutover_request_failed', 'cutover_request_refused', 'cutover_interrupted', 'cutover_service_failed',
|
|
13
14
|
]);
|
|
14
15
|
|
|
15
16
|
/** One redacted line per job, local hub or restart event; failures go to daemon-error.log, the rest to daemon.log. */
|
package/src/lib/cutover.js
CHANGED
|
@@ -5,6 +5,7 @@ import { setTimeout as sleep } from 'node:timers/promises';
|
|
|
5
5
|
import { clientState, installClients, restoreClients } from './clientInstaller.js';
|
|
6
6
|
import { HubError } from './errors.js';
|
|
7
7
|
import { RunnerStore, SEED_META } from './runnerStore.js';
|
|
8
|
+
import { SERVICE_LABEL } from './serviceManager.js';
|
|
8
9
|
import { readOwnerOnlyJson, writeOwnerOnlyJson } from './storage.js';
|
|
9
10
|
|
|
10
11
|
export const CUTOVER_FILE = 'cutover.json';
|
|
@@ -20,6 +21,9 @@ export const STEPS = Object.freeze(['sign_in_moved', 'shims_offline', 'hub_unloa
|
|
|
20
21
|
|
|
21
22
|
const sha256File = async (file) => createHash('sha256').update(await readFile(file)).digest('hex');
|
|
22
23
|
|
|
24
|
+
/** What a fresh journal keeps from the last one: the daemon's request history (CUT-03). */
|
|
25
|
+
const carried = (previous) => Object.fromEntries(['requests', 'refusals'].filter((key) => previous?.[key]).map((key) => [key, previous[key]]));
|
|
26
|
+
|
|
23
27
|
/** `version >= floor` on the numeric x.y.z core; anything unparsable is older. */
|
|
24
28
|
function atLeast(version, floor) {
|
|
25
29
|
const parse = (v) => String(v ?? '').match(/^(\d+)\.(\d+)\.(\d+)/)?.slice(1).map(Number) ?? null;
|
|
@@ -34,20 +38,23 @@ function atLeast(version, floor) {
|
|
|
34
38
|
* a backup kept; `--rollback` restores that backup and the old hub. Every effect goes through an injected seam:
|
|
35
39
|
* `services` (launchd, by label), `probe` (who holds hub.db), `cloudFirst` (Phase 2's final import and reconcile),
|
|
36
40
|
* `daemon` (the facade's health and the old hub's), `smoke`, `backupDb`, `applySpool` and `signIn` (the TokenStore). Each step is journaled in
|
|
37
|
-
* cutover.json before the next starts; a failure at any step after the freeze began rolls back by itself.
|
|
41
|
+
* cutover.json before the next starts; a failure at any step after the freeze began rolls back by itself. `serve` is
|
|
42
|
+
* called after each change of `cloudFirst.mode`: run inside the daemon (CUT-03) it starts or stops the daemon's own
|
|
43
|
+
* runner at once instead of on its next heartbeat.
|
|
38
44
|
*/
|
|
39
45
|
export class Cutover {
|
|
40
46
|
constructor({
|
|
41
|
-
configDir, hubHome, port = 8890, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn,
|
|
47
|
+
configDir, hubHome, port = 8890, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn, serve = async () => {},
|
|
42
48
|
labels = OLD_HUB_LABELS, holderWaitMs = 120_000, facadeWaitMs = 90_000, pollMs = 1_000, now = Date.now, wait = sleep, log = () => {},
|
|
43
49
|
}) {
|
|
50
|
+
if (labels.includes(SERVICE_LABEL)) throw new HubError('HUB_CUTOVER_LABELS', `The cutover never unloads the Pulse daemon itself (${SERVICE_LABEL})`);
|
|
44
51
|
this.configDir = configDir;
|
|
45
52
|
this.configFile = path.join(configDir, 'config.json');
|
|
46
53
|
this.journalFile = path.join(configDir, CUTOVER_FILE);
|
|
47
54
|
this.hubHome = hubHome;
|
|
48
55
|
this.hubDb = path.join(hubHome, 'state', 'hub.db');
|
|
49
56
|
this.backupFile = `${this.hubDb}${BACKUP_SUFFIX}`;
|
|
50
|
-
Object.assign(this, { port, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn, labels, holderWaitMs, facadeWaitMs, pollMs, now, wait, log });
|
|
57
|
+
Object.assign(this, { port, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn, serve, labels, holderWaitMs, facadeWaitMs, pollMs, now, wait, log });
|
|
51
58
|
}
|
|
52
59
|
|
|
53
60
|
async journal() {
|
|
@@ -188,11 +195,17 @@ export class Cutover {
|
|
|
188
195
|
};
|
|
189
196
|
}
|
|
190
197
|
|
|
191
|
-
/**
|
|
192
|
-
|
|
198
|
+
/**
|
|
199
|
+
* The cutover. Resolves with the journal; a failure after the freeze began rolls back and rethrows. `request` is the
|
|
200
|
+
* daemon's accepted cutover.request (CUT-03), kept on the journal so its CLI can follow it.
|
|
201
|
+
*/
|
|
202
|
+
async run({ smokeChannel = null, request = null } = {}) {
|
|
193
203
|
const preflight = await this.dryRun();
|
|
194
204
|
if (!preflight.ok) throw new HubError('HUB_CUTOVER_NOT_READY', preflight.message, { details: { checks: preflight.checks.filter((c) => !c.ok) } });
|
|
195
|
-
const journal = {
|
|
205
|
+
const journal = {
|
|
206
|
+
state: 'running', startedAt: new Date(this.now()).toISOString(), hubHome: this.hubHome, port: this.port, steps: {}, loadedBefore: [],
|
|
207
|
+
runBy: request ? 'daemon' : 'cli', pid: process.pid, ...carried(await this.journal()), ...(request ? { request } : {}),
|
|
208
|
+
};
|
|
196
209
|
await writeOwnerOnlyJson(this.journalFile, journal);
|
|
197
210
|
try {
|
|
198
211
|
// Before the freeze: the sign-in moves to this config's own Keychain item (the 12:52 DECISION); no outage.
|
|
@@ -240,6 +253,7 @@ export class Cutover {
|
|
|
240
253
|
cloudFirst: { ...(config.cloudFirst || {}), mode: 'runner', hubHome: this.hubHome, port: this.port, cutoverSeq, seedFrom: this.backupFile, seedSha256: journal.backup.sha256, shadow: false },
|
|
241
254
|
});
|
|
242
255
|
journal.cutoverSeq = cutoverSeq;
|
|
256
|
+
await this.serve();
|
|
243
257
|
const serving = this.now() + this.facadeWaitMs;
|
|
244
258
|
while (!(await this.daemon.facadeUp(this.port))) {
|
|
245
259
|
if (this.now() >= serving) throw new HubError('HUB_CUTOVER_FACADE', `the Pulse daemon did not serve 127.0.0.1:${this.port} within ${this.facadeWaitMs} ms`);
|
|
@@ -271,27 +285,33 @@ export class Cutover {
|
|
|
271
285
|
*/
|
|
272
286
|
async rollback({ reason = 'asked' } = {}) {
|
|
273
287
|
const journal = await this.journal();
|
|
274
|
-
const report = { command: 'cutover', rollback: true, reason, steps: [] };
|
|
275
|
-
|
|
288
|
+
const report = { command: 'cutover', rollback: true, reason, startedAt: new Date(this.now()).toISOString(), steps: [] };
|
|
289
|
+
// Each step is journaled as it ends, so a CLI following a daemon's rollback (CUT-03) prints it.
|
|
290
|
+
const step = async (name, detail = {}) => {
|
|
291
|
+
report.steps.push({ name, ...detail });
|
|
292
|
+
await writeOwnerOnlyJson(this.journalFile, { ...journal, rollback: report });
|
|
293
|
+
this.log('rollback_step', { name, ...detail });
|
|
294
|
+
};
|
|
276
295
|
const config = await this.#config();
|
|
277
296
|
if (config.cloudFirst?.mode === 'runner') {
|
|
278
297
|
await writeOwnerOnlyJson(this.configFile, { ...config, cloudFirst: { ...config.cloudFirst, mode: 'rolled_back' } });
|
|
298
|
+
await this.serve();
|
|
279
299
|
const deadline = this.now() + this.facadeWaitMs;
|
|
280
300
|
while (await this.daemon.facadeUp(this.port)) {
|
|
281
301
|
if (this.now() >= deadline) throw new HubError('HUB_ROLLBACK_FACADE', `the Pulse daemon still serves 127.0.0.1:${this.port}`);
|
|
282
302
|
await this.wait(this.pollMs);
|
|
283
303
|
}
|
|
284
|
-
step('daemon_left_port');
|
|
304
|
+
await step('daemon_left_port');
|
|
285
305
|
}
|
|
286
306
|
if (journal.cutoverAt) {
|
|
287
307
|
const lost = await this.#exportLost(journal.cutoverAt);
|
|
288
308
|
report.lost = lost;
|
|
289
|
-
step('lost_exported', lost);
|
|
309
|
+
await step('lost_exported', lost);
|
|
290
310
|
}
|
|
291
311
|
const marked = await this.#markSeedRolledBack();
|
|
292
|
-
if (marked) step('runner_db_marked', marked);
|
|
312
|
+
if (marked) await step('runner_db_marked', marked);
|
|
293
313
|
await restoreClients({ hubHome: this.hubHome });
|
|
294
|
-
step('entry_points_restored');
|
|
314
|
+
await step('entry_points_restored');
|
|
295
315
|
if (journal.steps?.backed_up && (await stat(this.backupFile).catch(() => null))) {
|
|
296
316
|
for (const suffix of ['-wal', '-shm']) await rm(`${this.hubDb}${suffix}`, { force: true });
|
|
297
317
|
const restored = `${this.hubDb}.pulse-restore`;
|
|
@@ -300,13 +320,15 @@ export class Cutover {
|
|
|
300
320
|
await rm(this.hubDb, { force: true });
|
|
301
321
|
await copyFile(restored, this.hubDb);
|
|
302
322
|
await rm(restored, { force: true });
|
|
303
|
-
step('hub_db_restored', { from: this.backupFile, sha256: await sha256File(this.backupFile) });
|
|
323
|
+
await step('hub_db_restored', { from: this.backupFile, sha256: await sha256File(this.backupFile) });
|
|
304
324
|
}
|
|
305
325
|
const spool = await this.applySpool(this.hubDb);
|
|
306
|
-
step('spool_applied', spool);
|
|
326
|
+
await step('spool_applied', spool);
|
|
307
327
|
const loadedBefore = journal.loadedBefore?.length ? journal.loadedBefore : this.labels;
|
|
328
|
+
// Journaled before the first load: from here the old hub may have hub.db open, so it is never restored again.
|
|
329
|
+
await step('hub_loading', { labels: loadedBefore });
|
|
308
330
|
for (const label of loadedBefore) await this.services.restoreLabel(label);
|
|
309
|
-
step('hub_loaded', { labels: loadedBefore });
|
|
331
|
+
await step('hub_loaded', { labels: loadedBefore });
|
|
310
332
|
const deadline = this.now() + this.facadeWaitMs;
|
|
311
333
|
while (!(await this.daemon.oldHubUp(this.port, { timeoutMs: ROLLBACK_PROBE_MS }))) {
|
|
312
334
|
if (this.now() >= deadline) {
|
|
@@ -317,7 +339,7 @@ export class Cutover {
|
|
|
317
339
|
}
|
|
318
340
|
await this.wait(this.pollMs);
|
|
319
341
|
}
|
|
320
|
-
step('hub_serving');
|
|
342
|
+
await step('hub_serving');
|
|
321
343
|
report.message = report.lost?.count
|
|
322
344
|
? `Rolled back to the local hub. ${report.lost.count} write(s) made after the switch (${Object.entries(report.lost.kinds).map(([kind, n]) => `${kind} ${n}`).join(', ')}) are not in hub.db; they are listed in ${report.lost.file}.`
|
|
323
345
|
: 'Rolled back to the local hub. Nothing written after the switch was lost.';
|
|
@@ -325,6 +347,19 @@ export class Cutover {
|
|
|
325
347
|
return report;
|
|
326
348
|
}
|
|
327
349
|
|
|
350
|
+
/**
|
|
351
|
+
* CUT-03 boot recovery of a rollback that had begun loading the old hub: whether the old hub holds the port again
|
|
352
|
+
* (it answers, or it is hung but owned by its job), waiting up to facadeWaitMs. Reads only.
|
|
353
|
+
*/
|
|
354
|
+
async verifyRolledBack() {
|
|
355
|
+
const deadline = this.now() + this.facadeWaitMs;
|
|
356
|
+
for (;;) {
|
|
357
|
+
const onPort = await this.#oldHubOnPort();
|
|
358
|
+
if (onPort.ok || this.now() >= deadline) return onPort;
|
|
359
|
+
await this.wait(this.pollMs);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
328
363
|
/**
|
|
329
364
|
* After the runner left the port: the seed is marked rolled back, so the next cutover replaces it with its own
|
|
330
365
|
* backup instead of serving this attempt's state (B1). Null when runner.db was never seeded.
|
|
@@ -0,0 +1,485 @@
|
|
|
1
|
+
import { randomBytes, randomUUID } from 'node:crypto';
|
|
2
|
+
import { chmod, link, open, rename, rm, unlink } from 'node:fs/promises';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { setTimeout as sleep } from 'node:timers/promises';
|
|
5
|
+
import { CUTOVER_FILE, STEPS } from './cutover.js';
|
|
6
|
+
import { HubError } from './errors.js';
|
|
7
|
+
import { readOwnerOnlyJson, writeOwnerOnlyJson } from './storage.js';
|
|
8
|
+
|
|
9
|
+
export const CUTOVER_REQUEST_FILE = 'cutover.request';
|
|
10
|
+
/** A request the daemon renamed off the disk to read it; one a crash leaves behind is refused on start, never run. */
|
|
11
|
+
export const CUTOVER_CLAIMED_FILE = 'cutover.request.claimed';
|
|
12
|
+
/** The request protocol a daemon serves, in every heartbeat; a daemon without it gets the cutover run in-process. */
|
|
13
|
+
export const CUTOVER_PROTOCOL = 1;
|
|
14
|
+
export const REQUEST_STALE_MS = 10 * 60_000;
|
|
15
|
+
/** The exit code of a follow that cannot tell how the request ended (timed out, or the daemon stopped). */
|
|
16
|
+
export const FOLLOW_UNKNOWN_EXIT = 2;
|
|
17
|
+
const FUTURE_SLACK_MS = 60_000;
|
|
18
|
+
const HEALTH_FRESH_MS = 60_000;
|
|
19
|
+
const MODES = new Set(['cutover', 'rollback']);
|
|
20
|
+
const SMOKE_CHANNEL = /^[A-Za-z0-9._-]{1,100}\/[A-Za-z0-9._-]{1,100}$/;
|
|
21
|
+
const HISTORY = 50;
|
|
22
|
+
const TICK_MS = 2_000;
|
|
23
|
+
const ACCEPT_MS = 2 * 60_000;
|
|
24
|
+
// The worst case of one request: holders (2 min), the final import (its 60 min wait), the reconcile (10 min), the
|
|
25
|
+
// facade (90 s) and smoke, then the rollback that follows a failure (facade 90 s, spool 2 min, hub 90 s): about 80 min.
|
|
26
|
+
const FOLLOW_MS = 2 * 3_600_000;
|
|
27
|
+
const POLL_MS = 1_000;
|
|
28
|
+
|
|
29
|
+
const iso = (ms) => new Date(ms).toISOString();
|
|
30
|
+
const atOrAfter = (when, since) => Date.parse(when) >= Date.parse(since);
|
|
31
|
+
|
|
32
|
+
function pidAlive(pid) {
|
|
33
|
+
try {
|
|
34
|
+
process.kill(pid, 0);
|
|
35
|
+
return true;
|
|
36
|
+
} catch (error) {
|
|
37
|
+
return error.code === 'EPERM';
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Rollback steps from which the old hub may have hub.db open: a rollback that reached one is never run again. */
|
|
42
|
+
const RELOADING = new Set(['hub_loading', 'hub_loaded', 'hub_serving']);
|
|
43
|
+
|
|
44
|
+
/** The steps of the rollback journaled since `since` (this request's), by name. */
|
|
45
|
+
function rollbackProgress(journal, since) {
|
|
46
|
+
const rollback = journal.rollback;
|
|
47
|
+
return rollback?.startedAt && atOrAfter(rollback.startedAt, since) ? rollback.steps.map((entry) => entry.name) : [];
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** The health state a finished request left. */
|
|
51
|
+
function endState(request) {
|
|
52
|
+
const { outcome } = request;
|
|
53
|
+
if (outcome.state === 'done') return request.mode === 'cutover' ? 'serving' : 'rolled_back';
|
|
54
|
+
return outcome.rolledBack ? 'rolled_back' : 'failed';
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The daemon's side of `pulse cutover` (CUT-03). It serves cutover.request on its own timer, beside the heartbeat, so
|
|
59
|
+
* the runner a cutover switches on can start while the cutover waits for it. One request runs at a time, in this
|
|
60
|
+
* process, with the real Cutover from `createCutover({ log })`, and only while `canTake()` (no restart is pending).
|
|
61
|
+
* A request is claimed by renaming it off the disk, checked, then recorded in cutover.json (`request`, and its id in
|
|
62
|
+
* `requests`) before the claimed file is deleted, so a crash never runs it twice; a stale, duplicate, malformed or
|
|
63
|
+
* pointless one is refused into `refusals`. On start, a request a crash cut off is finished as 1.1.2 would after a
|
|
64
|
+
* failure (the 20:05:57 DECISION): a cutover that had begun rolls back, a rollback runs again, unless either had begun
|
|
65
|
+
* loading the old hub (`hub_loading`): restoring hub.db under it would lose its writes, so boot only checks that the old
|
|
66
|
+
* hub holds the port and records rolled_back, or HUB_ROLLBACK_INTERRUPTED for a human. Each boot action is recorded in
|
|
67
|
+
* the journal's `recovery` and in health.
|
|
68
|
+
*/
|
|
69
|
+
export class CutoverService {
|
|
70
|
+
constructor({ directory, createCutover, canTake = () => true, now = Date.now, tickMs = TICK_MS, log = () => {} }) {
|
|
71
|
+
this.directory = directory;
|
|
72
|
+
this.createCutover = createCutover;
|
|
73
|
+
this.canTake = canTake;
|
|
74
|
+
this.now = now;
|
|
75
|
+
this.tickMs = tickMs;
|
|
76
|
+
this.log = log;
|
|
77
|
+
this.journalFile = path.join(directory, CUTOVER_FILE);
|
|
78
|
+
this.requestFile = path.join(directory, CUTOVER_REQUEST_FILE);
|
|
79
|
+
this.claimedFile = path.join(directory, CUTOVER_CLAIMED_FILE);
|
|
80
|
+
this.timer = null;
|
|
81
|
+
this.running = null;
|
|
82
|
+
this.recovered = false;
|
|
83
|
+
this.stopped = false;
|
|
84
|
+
this.active = false;
|
|
85
|
+
this.state = null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
start() {
|
|
89
|
+
this.stopped = false;
|
|
90
|
+
this.#schedule(0);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** Stops taking requests. A cutover running now is not waited for: one cut off is finished by the next start. */
|
|
94
|
+
async stop() {
|
|
95
|
+
this.stopped = true;
|
|
96
|
+
clearTimeout(this.timer);
|
|
97
|
+
this.timer = null;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** Whether a request is running: the daemon holds any restart until it ends. */
|
|
101
|
+
busy() {
|
|
102
|
+
return this.active;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** `{ state, step, requestId }` for daemon-health.json, or null before any request. */
|
|
106
|
+
health() {
|
|
107
|
+
return this.state;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
#schedule(waitMs) {
|
|
111
|
+
clearTimeout(this.timer);
|
|
112
|
+
if (this.stopped) return;
|
|
113
|
+
this.timer = setTimeout(() => {
|
|
114
|
+
this.timer = null;
|
|
115
|
+
void this.tick().finally(() => this.#schedule(this.tickMs));
|
|
116
|
+
}, waitMs);
|
|
117
|
+
this.timer.unref?.();
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** One pass: finish what a crash cut off (first pass only), then take one request if there is one. Never rejects. */
|
|
121
|
+
tick() {
|
|
122
|
+
this.running ??= this.#pass().finally(() => {
|
|
123
|
+
this.running = null;
|
|
124
|
+
});
|
|
125
|
+
return this.running;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
async #pass() {
|
|
129
|
+
try {
|
|
130
|
+
if (!this.recovered) {
|
|
131
|
+
this.recovered = true;
|
|
132
|
+
await this.#recover();
|
|
133
|
+
}
|
|
134
|
+
if (this.stopped || !this.canTake()) return;
|
|
135
|
+
try {
|
|
136
|
+
await rename(this.requestFile, this.claimedFile);
|
|
137
|
+
} catch (error) {
|
|
138
|
+
if (error.code === 'ENOENT') return;
|
|
139
|
+
throw error;
|
|
140
|
+
}
|
|
141
|
+
await this.#take();
|
|
142
|
+
} catch (error) {
|
|
143
|
+
this.log('cutover_service_failed', { code: error.code || 'HUB_CUTOVER_SERVICE_FAILED', message: String(error.message).slice(0, 200) });
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
async #journal() {
|
|
148
|
+
return (await readOwnerOnlyJson(this.journalFile)) || { state: 'none', steps: {} };
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async #claimed() {
|
|
152
|
+
try {
|
|
153
|
+
return { request: await readOwnerOnlyJson(this.claimedFile) };
|
|
154
|
+
} catch (error) {
|
|
155
|
+
return { request: null, unreadable: error.code === 'HUB_INVALID_CONFIG' ? 'not JSON' : `${error.code || 'unreadable'}: ${String(error.message).slice(0, 120)}` };
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** Why this request is refused, or null. */
|
|
160
|
+
#refusal(request, journal) {
|
|
161
|
+
if (!request || typeof request !== 'object') return 'invalid: not a JSON object';
|
|
162
|
+
if (typeof request.id !== 'string' || !/^[A-Za-z0-9._:-]{1,100}$/.test(request.id)) return 'invalid: no usable id';
|
|
163
|
+
if (!MODES.has(request.mode)) return `invalid: mode ${JSON.stringify(request.mode ?? null)} is neither cutover nor rollback`;
|
|
164
|
+
if (request.smokeChannel != null && !(typeof request.smokeChannel === 'string' && SMOKE_CHANNEL.test(request.smokeChannel))) {
|
|
165
|
+
return 'invalid: smokeChannel is not <repo>/<task>';
|
|
166
|
+
}
|
|
167
|
+
const at = Date.parse(request.requestedAt);
|
|
168
|
+
if (!Number.isFinite(at)) return 'invalid: no requestedAt';
|
|
169
|
+
if (at - this.now() > FUTURE_SLACK_MS) return `invalid: requestedAt ${request.requestedAt} is in the future`;
|
|
170
|
+
const ageMs = this.now() - at;
|
|
171
|
+
if (ageMs > REQUEST_STALE_MS) return `stale: requested ${Math.round(ageMs / 60_000)} min ago, more than ${REQUEST_STALE_MS / 60_000} min`;
|
|
172
|
+
if (journal.request?.id === request.id || (journal.requests || []).includes(request.id)) return `duplicate: request ${request.id} was already taken`;
|
|
173
|
+
const ended = journal.request?.outcome?.at;
|
|
174
|
+
if (ended && at < Date.parse(ended)) return `arrived while another request ran: requested ${request.requestedAt}, before request ${journal.request.id} ended at ${ended}`;
|
|
175
|
+
if (journal.state === 'running' && journal.runBy === 'cli' && Number.isInteger(journal.pid) && pidAlive(journal.pid)) {
|
|
176
|
+
return `busy: pid ${journal.pid} is running a cutover in-process (started ${journal.startedAt})`;
|
|
177
|
+
}
|
|
178
|
+
if (request.mode === 'rollback') {
|
|
179
|
+
if (journal.state === 'none' || journal.state === 'rolled_back') return `nothing to roll back: cutover.json is ${journal.state}`;
|
|
180
|
+
if ((journal.rollback?.steps || []).some((entry) => RELOADING.has(entry.name))) {
|
|
181
|
+
return 'a rollback already loaded the old hub again: another would restore hub.db under it; check it by hand';
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
return null;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
async #take() {
|
|
188
|
+
const { request, unreadable } = await this.#claimed();
|
|
189
|
+
const journal = await this.#journal();
|
|
190
|
+
const reason = unreadable ? `invalid: ${unreadable}` : this.#refusal(request, journal);
|
|
191
|
+
if (reason) {
|
|
192
|
+
await this.#refuse(request, reason, journal);
|
|
193
|
+
await rm(this.claimedFile, { force: true });
|
|
194
|
+
return;
|
|
195
|
+
}
|
|
196
|
+
// A restart requested while this was read: the request goes back for the next process, or is refused if it can't.
|
|
197
|
+
if (this.stopped || !this.canTake()) {
|
|
198
|
+
const back = await link(this.claimedFile, this.requestFile).then(() => true, () => false);
|
|
199
|
+
if (!back) await this.#refuse(request, 'the Pulse daemon is restarting; ask again once it is back', journal);
|
|
200
|
+
await rm(this.claimedFile, { force: true });
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
this.active = true;
|
|
204
|
+
try {
|
|
205
|
+
const accepted = {
|
|
206
|
+
id: request.id, mode: request.mode, requestedAt: request.requestedAt, requestedBy: request.requestedBy ?? null,
|
|
207
|
+
...(request.mode === 'cutover' && request.smokeChannel ? { smokeChannel: request.smokeChannel } : {}),
|
|
208
|
+
acceptedAt: iso(this.now()),
|
|
209
|
+
};
|
|
210
|
+
await writeOwnerOnlyJson(this.journalFile, { ...journal, request: accepted, requests: [...(journal.requests || []), accepted.id].slice(-HISTORY) });
|
|
211
|
+
await rm(this.claimedFile, { force: true });
|
|
212
|
+
this.log('cutover_request_accepted', { id: accepted.id, mode: accepted.mode });
|
|
213
|
+
await this.#perform(accepted, accepted.mode);
|
|
214
|
+
} finally {
|
|
215
|
+
this.active = false;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
async #refuse(request, reason, journal) {
|
|
220
|
+
const refusal = {
|
|
221
|
+
id: typeof request?.id === 'string' ? request.id : null, mode: request?.mode ?? null, requestedAt: request?.requestedAt ?? null,
|
|
222
|
+
reason, at: iso(this.now()),
|
|
223
|
+
};
|
|
224
|
+
await writeOwnerOnlyJson(this.journalFile, { ...journal, refusals: [...(journal.refusals || []), refusal].slice(-HISTORY) });
|
|
225
|
+
this.state = { state: 'refused', requestId: refusal.id, reason };
|
|
226
|
+
this.log('cutover_request_refused', { id: refusal.id, reason });
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Runs `action` (cutover or rollback) for an accepted request and records its outcome on the journal. `asOutcome`
|
|
231
|
+
* is what a rollback that finishes a cut-off cutover reports, rolled back, instead of a plain done; `recovered` names
|
|
232
|
+
* the boot action that started it, for health.
|
|
233
|
+
*/
|
|
234
|
+
async #perform(request, action, { asOutcome = null, recovered = null } = {}) {
|
|
235
|
+
const tag = recovered ? { recovered } : {};
|
|
236
|
+
this.state = { state: 'running', mode: action, step: null, requestId: request.id, since: iso(this.now()), ...tag };
|
|
237
|
+
const log = (event, details = {}) => {
|
|
238
|
+
if (event === 'cutover_step') this.state = { ...this.state, step: details.step };
|
|
239
|
+
if (event === 'rollback_step') this.state = { ...this.state, phase: 'rollback', step: details.name };
|
|
240
|
+
this.log(event, details);
|
|
241
|
+
};
|
|
242
|
+
// Health says how it ended before the journal does, so whoever sees the outcome sees the same in health.
|
|
243
|
+
try {
|
|
244
|
+
const cutover = await this.createCutover({ log });
|
|
245
|
+
if (action === 'cutover') {
|
|
246
|
+
await cutover.run({ smokeChannel: request.smokeChannel ?? null, request });
|
|
247
|
+
this.state = { state: 'serving', step: this.state.step, requestId: request.id, ...tag };
|
|
248
|
+
await this.#finish(request.id, { state: 'done' });
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
const report = await cutover.rollback({ reason: asOutcome?.code ?? 'asked' });
|
|
252
|
+
this.state = { state: 'rolled_back', step: this.state.step, requestId: request.id, ...tag };
|
|
253
|
+
await this.#finish(request.id, asOutcome ? { ...asOutcome, rolledBack: true } : { state: 'done', message: report.message });
|
|
254
|
+
} catch (error) {
|
|
255
|
+
const code = error.code || 'HUB_CUTOVER_FAILED';
|
|
256
|
+
this.state = { state: error.rollback ? 'rolled_back' : 'failed', step: this.state.step, requestId: request.id, code, ...tag };
|
|
257
|
+
this.log('cutover_request_failed', { id: request.id, code });
|
|
258
|
+
await this.#finish(request.id, {
|
|
259
|
+
state: 'failed', code, message: String(error.message).slice(0, 500), rolledBack: Boolean(error.rollback),
|
|
260
|
+
...(error.details?.checks ? { checks: error.details.checks } : {}),
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
/** Records the request's outcome on the journal, with `extra` journal fields (a boot recovery's). */
|
|
266
|
+
async #finish(id, outcome, extra = {}) {
|
|
267
|
+
const journal = await this.#journal();
|
|
268
|
+
const request = journal.request?.id === id ? journal.request : { id };
|
|
269
|
+
await writeOwnerOnlyJson(this.journalFile, { ...journal, ...extra, request: { ...request, outcome: { ...outcome, at: iso(this.now()) } } });
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/** A boot recovery that ends the request without running anything: the outcome and what boot did, in both files. */
|
|
273
|
+
async #recovered(request, action, outcome, state, extra = {}) {
|
|
274
|
+
const recovery = { at: iso(this.now()), requestId: request.id, action };
|
|
275
|
+
this.state = { state, requestId: request.id, recovered: action, ...(outcome.code ? { code: outcome.code } : {}) };
|
|
276
|
+
this.log('cutover_recovered', { id: request.id, action });
|
|
277
|
+
await this.#finish(request.id, outcome, { ...extra, recovery });
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/** On start: a claim a crash left is refused; a request that was running is finished (see the class). */
|
|
281
|
+
async #recover() {
|
|
282
|
+
const claimed = await this.#claimed();
|
|
283
|
+
if (claimed.request || claimed.unreadable) {
|
|
284
|
+
const journal = await this.#journal();
|
|
285
|
+
if (!(claimed.request && (journal.requests || []).includes(claimed.request.id))) {
|
|
286
|
+
await this.#refuse(claimed.request, 'interrupted: the daemon stopped while it was reading this request', journal);
|
|
287
|
+
}
|
|
288
|
+
await rm(this.claimedFile, { force: true });
|
|
289
|
+
}
|
|
290
|
+
const journal = await this.#journal();
|
|
291
|
+
const request = journal.request;
|
|
292
|
+
if (!request?.id) return;
|
|
293
|
+
if (request.outcome) {
|
|
294
|
+
this.state = { state: endState(request), requestId: request.id, ...(request.outcome.code ? { code: request.outcome.code } : {}) };
|
|
295
|
+
return;
|
|
296
|
+
}
|
|
297
|
+
const since = request.acceptedAt;
|
|
298
|
+
const began = Boolean(journal.startedAt) && atOrAfter(journal.startedAt, since);
|
|
299
|
+
const rolledBack = journal.state === 'rolled_back' && atOrAfter(journal.rolledBackAt, since);
|
|
300
|
+
const reloading = rollbackProgress(journal, since).some((name) => RELOADING.has(name));
|
|
301
|
+
const failure = journal.failure ?? { code: 'HUB_CUTOVER_INTERRUPTED', message: 'The Pulse daemon stopped while it ran this cutover.' };
|
|
302
|
+
// What a cutover that ended rolled back reports; a rollback request that ended so is simply done.
|
|
303
|
+
const rolledBackOutcome = (message) => (request.mode === 'rollback'
|
|
304
|
+
? { state: 'done', message }
|
|
305
|
+
: { state: 'failed', code: failure.code, message: failure.message, rolledBack: true });
|
|
306
|
+
this.active = true;
|
|
307
|
+
try {
|
|
308
|
+
if (request.mode === 'cutover' && journal.state === 'done' && began) return await this.#recovered(request, 'finished_before_stop', { state: 'done' }, 'serving');
|
|
309
|
+
if (rolledBack) return await this.#recovered(request, 'rolled_back_before_stop', rolledBackOutcome(journal.rollback?.message ?? null), 'rolled_back');
|
|
310
|
+
if (request.mode === 'cutover' && !(journal.state === 'running' && began)) {
|
|
311
|
+
return await this.#recovered(request, 'never_began', {
|
|
312
|
+
state: 'failed', code: 'HUB_CUTOVER_INTERRUPTED', message: 'The Pulse daemon stopped before this cutover began; nothing was changed.', rolledBack: false,
|
|
313
|
+
}, 'failed');
|
|
314
|
+
}
|
|
315
|
+
if (reloading) return await this.#verifyReloaded(request, journal, rolledBackOutcome);
|
|
316
|
+
if (request.mode === 'cutover') {
|
|
317
|
+
const cutFailure = journal.failure ?? {
|
|
318
|
+
step: STEPS.find((name) => !journal.steps?.[name]) ?? null, code: 'HUB_CUTOVER_INTERRUPTED',
|
|
319
|
+
message: 'The Pulse daemon stopped while it ran this cutover; it rolled back on start.',
|
|
320
|
+
};
|
|
321
|
+
const recovery = { at: iso(this.now()), requestId: request.id, action: 'rolled_back_on_start' };
|
|
322
|
+
await writeOwnerOnlyJson(this.journalFile, { ...journal, failure: cutFailure, recovery });
|
|
323
|
+
this.log('cutover_interrupted', { id: request.id, mode: 'cutover', step: cutFailure.step });
|
|
324
|
+
return await this.#perform(request, 'rollback', { asOutcome: { state: 'failed', code: 'HUB_CUTOVER_INTERRUPTED', message: cutFailure.message }, recovered: 'rolled_back_on_start' });
|
|
325
|
+
}
|
|
326
|
+
await writeOwnerOnlyJson(this.journalFile, { ...journal, recovery: { at: iso(this.now()), requestId: request.id, action: 'rollback_run_again' } });
|
|
327
|
+
this.log('cutover_interrupted', { id: request.id, mode: 'rollback' });
|
|
328
|
+
return await this.#perform(request, 'rollback', { recovered: 'rollback_run_again' });
|
|
329
|
+
} finally {
|
|
330
|
+
this.active = false;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/**
|
|
335
|
+
* The 20:05:57 DECISION: a rollback that had begun loading the old hub is never run again (it would restore hub.db
|
|
336
|
+
* under the running hub). Boot only checks that the old hub holds the port, and records rolled_back if it does.
|
|
337
|
+
*/
|
|
338
|
+
async #verifyReloaded(request, journal, rolledBackOutcome) {
|
|
339
|
+
this.state = { state: 'running', mode: 'verify', step: 'hub_loading', requestId: request.id, since: iso(this.now()) };
|
|
340
|
+
let onPort;
|
|
341
|
+
try {
|
|
342
|
+
onPort = await (await this.createCutover({ log: this.log })).verifyRolledBack();
|
|
343
|
+
} catch (error) {
|
|
344
|
+
onPort = { ok: false, detail: `the check failed: ${error.code || 'error'}: ${String(error.message).slice(0, 200)}` };
|
|
345
|
+
}
|
|
346
|
+
if (onPort.ok) {
|
|
347
|
+
const at = iso(this.now());
|
|
348
|
+
const message = `Rolled back to the local hub: the Pulse daemon restarted after the old hub was loaded again, and found it on the port (${onPort.detail}).`;
|
|
349
|
+
return this.#recovered(request, 'verified_old_hub', rolledBackOutcome(message), 'rolled_back', {
|
|
350
|
+
state: 'rolled_back', rolledBackAt: at, rollback: { ...journal.rollback, recoveredOnBoot: true, verified: onPort.detail, message },
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
return this.#recovered(request, 'old_hub_not_serving', {
|
|
354
|
+
state: 'failed', code: 'HUB_ROLLBACK_INTERRUPTED', rolledBack: false,
|
|
355
|
+
message: `The Pulse daemon stopped after its rollback began loading the old hub, and the old hub does not hold the port now (${onPort.detail}). It is not rolled back twice, which would restore hub.db under a hub that may have it open: check the hub by hand.`,
|
|
356
|
+
}, 'failed');
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
/** Whether the daemon behind this health file is alive and serves cutover.request. */
|
|
361
|
+
export function takesCutoverRequests(health, now = Date.now()) {
|
|
362
|
+
const ageMs = now - Date.parse(health?.heartbeatAt);
|
|
363
|
+
return ageMs <= HEALTH_FRESH_MS && Number(health?.cutoverProtocol) >= CUTOVER_PROTOCOL;
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/** Creates `file` owner-only with `value`, whole or not at all, and only if it does not exist yet. */
|
|
367
|
+
async function createExclusive(file, value) {
|
|
368
|
+
const temporary = path.join(path.dirname(file), `.${path.basename(file)}.${process.pid}.${randomBytes(6).toString('hex')}.tmp`);
|
|
369
|
+
const handle = await open(temporary, 'wx', 0o600);
|
|
370
|
+
try {
|
|
371
|
+
await handle.writeFile(`${JSON.stringify(value, null, 2)}\n`, 'utf8');
|
|
372
|
+
await handle.sync();
|
|
373
|
+
} finally {
|
|
374
|
+
await handle.close();
|
|
375
|
+
}
|
|
376
|
+
try {
|
|
377
|
+
await chmod(temporary, 0o600);
|
|
378
|
+
await link(temporary, file);
|
|
379
|
+
return true;
|
|
380
|
+
} catch (error) {
|
|
381
|
+
if (error.code === 'EEXIST') return false;
|
|
382
|
+
throw error;
|
|
383
|
+
} finally {
|
|
384
|
+
await rm(temporary, { force: true });
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/** Writes an owner-only cutover.request for the daemon, exclusively: one request at a time. */
|
|
389
|
+
export async function writeCutoverRequest({ configDir, mode, smokeChannel = null, health = null, now = Date.now, id = randomUUID(), requestedBy = null }) {
|
|
390
|
+
const pending = (detail) => new HubError('HUB_CUTOVER_PENDING', `Another cutover request is ${detail}; wait for it to finish (see ${path.join(configDir, CUTOVER_FILE)}).`);
|
|
391
|
+
if (health?.cutover?.state === 'running') throw pending(`running in the Pulse daemon (${health.cutover.requestId}${health.cutover.step ? `, at ${health.cutover.step}` : ''})`);
|
|
392
|
+
const request = {
|
|
393
|
+
id, mode, requestedAt: iso(now()),
|
|
394
|
+
requestedBy: requestedBy ?? { pid: process.pid, session: process.env.CURSOR_CONVERSATION_ID || null },
|
|
395
|
+
...(smokeChannel ? { smokeChannel } : {}),
|
|
396
|
+
};
|
|
397
|
+
if (!(await createExclusive(path.join(configDir, CUTOVER_REQUEST_FILE), request))) throw pending('waiting for the Pulse daemon');
|
|
398
|
+
return request;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/**
|
|
402
|
+
* Follows a request through cutover.json, printing each step as `log(event, details)` exactly as an in-process run
|
|
403
|
+
* logs it. It resolves with the journal (a cutover) or the rollback report, or throws: the journal's code for a run
|
|
404
|
+
* that failed (or rolled back), HUB_CUTOVER_REFUSED, or HUB_CUTOVER_NOT_ACCEPTED when nobody took it within
|
|
405
|
+
* `acceptMs` (it is withdrawn). It ends without an outcome, with `exit` FOLLOW_UNKNOWN_EXIT, when the daemon's heartbeat
|
|
406
|
+
* goes stale (it finishes the request when it starts again) or after `timeoutMs`, the worst case of a run.
|
|
407
|
+
*/
|
|
408
|
+
export async function followCutoverRequest({
|
|
409
|
+
configDir, request, log = () => {}, now = Date.now, wait = sleep, pollMs = POLL_MS, acceptMs = ACCEPT_MS, timeoutMs = FOLLOW_MS,
|
|
410
|
+
healthFreshMs = HEALTH_FRESH_MS,
|
|
411
|
+
}) {
|
|
412
|
+
const journalFile = path.join(configDir, CUTOVER_FILE);
|
|
413
|
+
const requestFile = path.join(configDir, CUTOVER_REQUEST_FILE);
|
|
414
|
+
const healthFile = path.join(configDir, 'daemon-health.json');
|
|
415
|
+
const started = now();
|
|
416
|
+
const printed = { steps: new Set(), failure: false, rollback: 0 };
|
|
417
|
+
let withDaemon = false;
|
|
418
|
+
const print = (journal) => {
|
|
419
|
+
const since = journal.request.acceptedAt;
|
|
420
|
+
if (request.mode === 'cutover' && journal.startedAt && atOrAfter(journal.startedAt, since)) {
|
|
421
|
+
for (const name of STEPS) {
|
|
422
|
+
if (!journal.steps?.[name] || printed.steps.has(name)) continue;
|
|
423
|
+
printed.steps.add(name);
|
|
424
|
+
const { at: _at, ...detail } = journal.steps[name];
|
|
425
|
+
log('cutover_step', { step: name, ...detail });
|
|
426
|
+
}
|
|
427
|
+
if (journal.failure && !printed.failure) {
|
|
428
|
+
printed.failure = true;
|
|
429
|
+
log('cutover_failed', journal.failure);
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
if (rollbackProgress(journal, since).length) {
|
|
433
|
+
for (const entry of journal.rollback.steps.slice(printed.rollback)) log('rollback_step', entry);
|
|
434
|
+
printed.rollback = journal.rollback.steps.length;
|
|
435
|
+
}
|
|
436
|
+
};
|
|
437
|
+
const unknown = (code, message) => new HubError(code, message, { exit: FOLLOW_UNKNOWN_EXIT });
|
|
438
|
+
for (;;) {
|
|
439
|
+
const journal = await readOwnerOnlyJson(journalFile).catch(() => null);
|
|
440
|
+
const refused = journal?.refusals?.find((entry) => entry.id === request.id);
|
|
441
|
+
if (refused) throw new HubError('HUB_CUTOVER_REFUSED', `The Pulse daemon refused request ${request.id}: ${refused.reason}`, { details: { refused } });
|
|
442
|
+
const health = await readOwnerOnlyJson(healthFile).catch(() => null);
|
|
443
|
+
// A long synchronous stretch (a rescan) can delay a heartbeat; a daemon whose pid is alive gets three windows.
|
|
444
|
+
const ageMs = now() - Date.parse(health?.heartbeatAt);
|
|
445
|
+
const stale = !(ageMs <= healthFreshMs) && (!(ageMs <= 3 * healthFreshMs) || !(Number.isInteger(health?.pid) && pidAlive(health.pid)));
|
|
446
|
+
if (journal?.request?.id === request.id) {
|
|
447
|
+
print(journal);
|
|
448
|
+
const outcome = journal.request.outcome;
|
|
449
|
+
if (outcome?.state === 'done') return request.mode === 'cutover' ? journal : journal.rollback;
|
|
450
|
+
if (outcome) {
|
|
451
|
+
throw Object.assign(
|
|
452
|
+
new HubError(outcome.code || 'HUB_CUTOVER_FAILED', outcome.message || 'the cutover failed', { details: { ...(outcome.checks ? { checks: outcome.checks } : {}), rolledBack: Boolean(outcome.rolledBack) } }),
|
|
453
|
+
outcome.rolledBack ? { rollback: journal.rollback ?? null } : {},
|
|
454
|
+
);
|
|
455
|
+
}
|
|
456
|
+
if (stale) {
|
|
457
|
+
throw unknown('HUB_CUTOVER_DAEMON_STOPPED', `The Pulse daemon stopped (last heartbeat ${health?.heartbeatAt ?? 'never'}) while it ran request ${request.id}. When it starts again it finishes the request (a cutover that had begun rolls back); follow it in ${journalFile}.`);
|
|
458
|
+
}
|
|
459
|
+
} else if (!withDaemon && (stale || now() - started >= acceptMs)) {
|
|
460
|
+
// Withdrawn only while it is still ours on the disk: gone means the daemon claimed it, so it is followed.
|
|
461
|
+
const pending = await readOwnerOnlyJson(requestFile).catch(() => null);
|
|
462
|
+
let withdrawn = false;
|
|
463
|
+
if (pending?.id === request.id) {
|
|
464
|
+
try {
|
|
465
|
+
await unlink(requestFile);
|
|
466
|
+
withdrawn = true;
|
|
467
|
+
} catch (error) {
|
|
468
|
+
if (error.code !== 'ENOENT') throw error;
|
|
469
|
+
}
|
|
470
|
+
}
|
|
471
|
+
if (withdrawn) {
|
|
472
|
+
throw new HubError('HUB_CUTOVER_NOT_ACCEPTED', stale
|
|
473
|
+
? `The Pulse daemon is not running (last heartbeat ${health?.heartbeatAt ?? 'never'}), so request ${request.id} was withdrawn. Start it with \`agentpulse pulse start\`.`
|
|
474
|
+
: `The Pulse daemon did not take request ${request.id} within ${Math.round(acceptMs / 1000)} s, so it was withdrawn. Check it with \`agentpulse pulse status\`.`);
|
|
475
|
+
}
|
|
476
|
+
withDaemon = true;
|
|
477
|
+
} else if (withDaemon && stale) {
|
|
478
|
+
throw unknown('HUB_CUTOVER_DAEMON_STOPPED', `The Pulse daemon stopped (last heartbeat ${health?.heartbeatAt ?? 'never'}) after it claimed request ${request.id}; when it starts again it refuses or finishes it. Follow it in ${journalFile}.`);
|
|
479
|
+
}
|
|
480
|
+
if (now() - started >= timeoutMs) {
|
|
481
|
+
throw unknown('HUB_CUTOVER_FOLLOW_TIMEOUT', `Request ${request.id} is still running in the Pulse daemon after ${Math.round(timeoutMs / 60_000)} min; follow it in ${journalFile} or with \`agentpulse pulse status\`.`);
|
|
482
|
+
}
|
|
483
|
+
await wait(pollMs);
|
|
484
|
+
}
|
|
485
|
+
}
|
package/src/lib/cutoverWiring.js
CHANGED
|
@@ -109,14 +109,17 @@ export function portProbes(tokenFor) {
|
|
|
109
109
|
};
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
-
/**
|
|
113
|
-
|
|
112
|
+
/**
|
|
113
|
+
* The production seams of `pulse cutover`: launchd, lsof, the Phase 2 import and reconcile, and the local port.
|
|
114
|
+
* `serve` is the daemon's own runner switch when the daemon runs it (CUT-03); in-process it is left to the heartbeat.
|
|
115
|
+
*/
|
|
116
|
+
export async function createCutover({ configDir, log = () => {}, port = 8890, serve = undefined }) {
|
|
114
117
|
const config = await readOwnerOnlyJson(path.join(configDir, 'config.json'));
|
|
115
118
|
if (!config) throw new HubError('HUB_NOT_INSTALLED', 'Pulse is not installed here; run `agentpulse pulse install`');
|
|
116
119
|
const hubHome = config.cloudFirst?.hubHome || config.mirror?.hub?.dir || (await discoverLocalHub({ config })).dir;
|
|
117
120
|
const tokenFor = () => hubToken(hubHome);
|
|
118
121
|
return new Cutover({
|
|
119
|
-
configDir, hubHome, port, log,
|
|
122
|
+
configDir, hubHome, port, log, serve,
|
|
120
123
|
services: new ServiceManager({}),
|
|
121
124
|
probe: { holders: (files) => lsofHolders(files), portOwners: (p) => lsofPortOwners(p) },
|
|
122
125
|
cloudFirst: new CloudFirstCommands({ configDir }),
|
package/src/lib/daemonProcess.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
|
+
import { CUTOVER_PROTOCOL, CutoverService } from './cutoverRequest.js';
|
|
3
|
+
import { createCutover } from './cutoverWiring.js';
|
|
2
4
|
import { createProductionDependencies } from './daemonComposition.js';
|
|
3
5
|
import { DaemonLock } from './daemonLock.js';
|
|
4
6
|
import { createHeartbeat, MIRROR_UPDATE_STEP, TERMINAL_STATES } from './daemonLoop.js';
|
|
@@ -60,7 +62,8 @@ const inside = (child, parent) => {
|
|
|
60
62
|
* - start_failed: it could not start while it was holding restarts back.
|
|
61
63
|
* A code or request restart drains (no new claim; running jobs finish and settle) for up to drainMs; a stall aborts at
|
|
62
64
|
* once. Every trigger goes through one check that runs each tickMs. While the daemon has restarted too often it holds
|
|
63
|
-
* the restart (health says restart_loop) until the window clears
|
|
65
|
+
* the restart (health says restart_loop) until the window clears, and while it runs a cutover request (CUT-03) it holds
|
|
66
|
+
* it until the cutover ends, whose own deadlines bound it. SIGTERM always wins and exits 0.
|
|
64
67
|
*/
|
|
65
68
|
export async function runDaemon({
|
|
66
69
|
directory,
|
|
@@ -74,6 +77,8 @@ export async function runDaemon({
|
|
|
74
77
|
createMirror = (options) => new MirrorController(options),
|
|
75
78
|
createShadow = (options) => new ShadowService(options),
|
|
76
79
|
createRunner = (options) => new RunnerService(options),
|
|
80
|
+
createCutoverService = (options) => new CutoverService(options),
|
|
81
|
+
cutoverFactory = createCutover,
|
|
77
82
|
dependencyOptions = {},
|
|
78
83
|
runtimeOptions = {},
|
|
79
84
|
lockOptions = {},
|
|
@@ -85,7 +90,7 @@ export async function runDaemon({
|
|
|
85
90
|
const lock = new DaemonLock({ directory, ...lockOptions });
|
|
86
91
|
await lock.acquire();
|
|
87
92
|
// The first cycle (token exchange, reconnect, claim) can outlast `pulse start`'s wait; say we are up before it.
|
|
88
|
-
await lock.heartbeat({ state: 'starting' });
|
|
93
|
+
await lock.heartbeat({ state: 'starting', cutoverProtocol: CUTOVER_PROTOCOL });
|
|
89
94
|
let startingHealthAt = Date.now();
|
|
90
95
|
const guard = new RestartGuard({ directory, now });
|
|
91
96
|
await guard.recordStart();
|
|
@@ -99,6 +104,11 @@ export async function runDaemon({
|
|
|
99
104
|
let runtime = null;
|
|
100
105
|
let shadow = null;
|
|
101
106
|
let runnerService = null;
|
|
107
|
+
let cutoverService = null;
|
|
108
|
+
let cutoverDeferralLogged = false;
|
|
109
|
+
// Every read of config.json is numbered, so the runner never applies one older than it already has (CUT-03).
|
|
110
|
+
let configGeneration = 0;
|
|
111
|
+
let heartbeatGeneration = 0;
|
|
102
112
|
let stopServices = async () => {};
|
|
103
113
|
let timer = null;
|
|
104
114
|
let watcher = null;
|
|
@@ -207,6 +217,14 @@ export async function runDaemon({
|
|
|
207
217
|
const requestedAt = await consumeRestartRequest(directory);
|
|
208
218
|
if (requestedAt !== null && requestedAt < bootedAt) log('restart_request_satisfied', { requestedAt: new Date(requestedAt).toISOString() });
|
|
209
219
|
else if (requestedAt !== null) pending ??= { reason: 'request' };
|
|
220
|
+
if (pending && cutoverService?.busy()) {
|
|
221
|
+
if (!cutoverDeferralLogged) {
|
|
222
|
+
cutoverDeferralLogged = true;
|
|
223
|
+
log('restart_deferred', { reason: pending.reason, cutover: cutoverService.health()?.requestId ?? null });
|
|
224
|
+
}
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
cutoverDeferralLogged = false;
|
|
210
228
|
if (pending) restart(pending.reason);
|
|
211
229
|
} catch (error) {
|
|
212
230
|
log('restart_check_failed', { code: error.code || 'HUB_RESTART_CHECK_FAILED' });
|
|
@@ -217,11 +235,15 @@ export async function runDaemon({
|
|
|
217
235
|
|
|
218
236
|
const extraHealth = () => {
|
|
219
237
|
const loop = degraded();
|
|
238
|
+
const cutover = cutoverService?.health();
|
|
220
239
|
return {
|
|
221
240
|
...(loop ? { degraded: loop } : {}),
|
|
222
241
|
...(restarting ? { restarting: { ...restarting, draining: runtime?.activeJobIds() ?? [] } } : {}),
|
|
242
|
+
...(pending && cutoverService?.busy() ? { restartDeferred: { reason: pending.reason, untilCutoverEnds: cutover?.requestId ?? null } } : {}),
|
|
223
243
|
...(shadow ? { shadow: shadow.health() } : {}),
|
|
224
244
|
...(runnerService ? { runner: runnerService.health() } : {}),
|
|
245
|
+
cutoverProtocol: CUTOVER_PROTOCOL,
|
|
246
|
+
...(cutover ? { cutover } : {}),
|
|
225
247
|
};
|
|
226
248
|
};
|
|
227
249
|
|
|
@@ -325,19 +347,34 @@ export async function runDaemon({
|
|
|
325
347
|
shadow = createShadow({ directory, transportFactory: production.transportFactory, tokenStore: production.tokenStore, log: mirrorLog });
|
|
326
348
|
runnerService = createRunner({ directory, transportFactory: production.transportFactory, log });
|
|
327
349
|
heartbeat = createHeartbeat({
|
|
328
|
-
readConfig: () =>
|
|
350
|
+
readConfig: () => {
|
|
351
|
+
heartbeatGeneration = ++configGeneration;
|
|
352
|
+
return readOwnerOnlyJson(configFile);
|
|
353
|
+
},
|
|
329
354
|
runtime,
|
|
330
355
|
mirror,
|
|
331
356
|
shadow,
|
|
332
|
-
runner: runnerService,
|
|
357
|
+
runner: { update: (config) => runnerService.update(config, { generation: heartbeatGeneration }) },
|
|
333
358
|
lock,
|
|
334
359
|
ledger,
|
|
335
360
|
ownerStore,
|
|
336
361
|
progress: cycleProgress,
|
|
337
362
|
extraHealth,
|
|
338
363
|
});
|
|
364
|
+
// A cutover request runs here, beside the heartbeat; its facade step starts this daemon's runner at once.
|
|
365
|
+
const serve = async () => {
|
|
366
|
+
const generation = ++configGeneration;
|
|
367
|
+
runnerService.update(await readOwnerOnlyJson(configFile), { generation });
|
|
368
|
+
};
|
|
369
|
+
cutoverService = createCutoverService({
|
|
370
|
+
directory,
|
|
371
|
+
log,
|
|
372
|
+
canTake: () => !exiting && !restarting && !pending,
|
|
373
|
+
createCutover: ({ log: stepLog }) => cutoverFactory({ configDir: directory, log: stepLog, serve }),
|
|
374
|
+
});
|
|
375
|
+
cutoverService.start();
|
|
339
376
|
stopServices = async ({ abortCode }) => {
|
|
340
|
-
const results = await Promise.allSettled([mirror.stop(), shadow.stop(), runnerService.stop(), runtime.shutdown({ abortCode })]);
|
|
377
|
+
const results = await Promise.allSettled([cutoverService.stop(), mirror.stop(), shadow.stop(), runnerService.stop(), runtime.shutdown({ abortCode })]);
|
|
341
378
|
const failed = results.find((result) => result.status === 'rejected');
|
|
342
379
|
if (failed) throw failed.reason;
|
|
343
380
|
};
|
|
@@ -367,7 +404,7 @@ export async function runDaemon({
|
|
|
367
404
|
// A cycle that ends after shutdown released the lock has no health left to report.
|
|
368
405
|
if (exiting) return;
|
|
369
406
|
await Promise.resolve()
|
|
370
|
-
.then(() => lock.heartbeat({ state: 'error', code: error.code || 'HUB_DAEMON_LOOP_FAILED' }))
|
|
407
|
+
.then(() => lock.heartbeat({ state: 'error', code: error.code || 'HUB_DAEMON_LOOP_FAILED', ...extraHealth() }))
|
|
371
408
|
.catch(() => {});
|
|
372
409
|
wait = t.errorRetryMs;
|
|
373
410
|
} finally {
|
|
@@ -10,7 +10,9 @@ import {
|
|
|
10
10
|
} from './storage.js';
|
|
11
11
|
import { execFileBounded as execFile } from './subprocess.js';
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
/** The Pulse daemon's own service label; the cutover that runs inside it must never retire it. */
|
|
14
|
+
export const SERVICE_LABEL = 'com.agentpulse.hub';
|
|
15
|
+
const LABEL = SERVICE_LABEL;
|
|
14
16
|
const SYSTEMD_UNIT = 'agentpulse-hub.service';
|
|
15
17
|
const START_ATTEMPTS = 6;
|
|
16
18
|
const START_RETRY_MS = 250;
|
|
@@ -38,13 +38,22 @@ export class RunnerService {
|
|
|
38
38
|
this.log = log;
|
|
39
39
|
this.now = now;
|
|
40
40
|
this.key = null;
|
|
41
|
+
this.generation = -Infinity;
|
|
41
42
|
this.current = null;
|
|
42
43
|
this.restarts = 0;
|
|
43
44
|
this.state = { state: 'off' };
|
|
44
45
|
}
|
|
45
46
|
|
|
46
|
-
/**
|
|
47
|
-
|
|
47
|
+
/**
|
|
48
|
+
* Start, keep or stop the runner for this config. Never waits on the runner. `generation` orders the reads of
|
|
49
|
+
* config.json: a config read before the last one applied is ignored, so a heartbeat that read it before a cutover's
|
|
50
|
+
* `serve` cannot switch the runner back (CUT-03).
|
|
51
|
+
*/
|
|
52
|
+
update(config, { generation = null } = {}) {
|
|
53
|
+
if (generation !== null) {
|
|
54
|
+
if (generation < this.generation) return this.state;
|
|
55
|
+
this.generation = generation;
|
|
56
|
+
}
|
|
48
57
|
const on = runnerMode(config) && pairedStatus(config).ok;
|
|
49
58
|
const key = on ? JSON.stringify({ c: config.cloudFirst, m: config.paired.machineId, v: config.paired.policyVersion }) : null;
|
|
50
59
|
if (key === this.key) return this.state;
|