@agentpulselabs/cli-plugin-pulse 1.1.2 → 1.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -0
- package/package.json +3 -2
- package/src/commands/pulse/cutover.js +34 -10
- package/src/daemon/main.js +1 -0
- package/src/hub/core/db.mjs +21 -5
- package/src/hub/workers/manager.mjs +44 -6
- package/src/lib/contract.sha256 +1 -1
- package/src/lib/cutover.js +51 -16
- package/src/lib/cutoverRequest.js +485 -0
- package/src/lib/cutoverWiring.js +6 -3
- package/src/lib/daemonProcess.js +43 -6
- package/src/lib/daemonRuntime.js +25 -7
- package/src/lib/hubDbReader.js +6 -0
- package/src/lib/hubScan.js +1 -0
- package/src/lib/ledger.js +21 -12
- package/src/lib/lifecycle.js +3 -1
- package/src/lib/runnerStore.js +61 -49
- package/src/lib/serviceManager.js +3 -1
- package/src/runner/bridge.js +8 -3
- package/src/runner/capture.js +83 -20
- package/src/runner/channelDiscipline.js +48 -14
- package/src/runner/channelWalk.js +42 -0
- package/src/runner/deliveryStates.js +22 -9
- package/src/runner/directoryCache.js +34 -0
- package/src/runner/facade.js +71 -14
- package/src/runner/hubCapture.js +105 -56
- package/src/runner/loopMonitor.js +89 -0
- package/src/runner/runnerHub.js +3 -1
- package/src/runner/runnerProcess.js +58 -16
- package/src/runner/runnerService.js +29 -4
- package/src/runner/serial.js +5 -0
- package/src/runner/spool.js +46 -11
package/README.md
CHANGED
|
@@ -89,6 +89,15 @@ These need your team's Pulse cloud-first flag on, set by a team owner or admin,
|
|
|
89
89
|
|
|
90
90
|
Everything sent goes through the same secret and path filter as the mirror, and the cloud filters it again.
|
|
91
91
|
|
|
92
|
+
## Cutover through the daemon (1.1.3)
|
|
93
|
+
|
|
94
|
+
`agentpulse pulse cutover` and `--rollback` ask the running Pulse daemon to do the work, so the command itself never needs `launchctl` or a permission prompt. Only `--dry-run`, which only reads, runs in the command.
|
|
95
|
+
|
|
96
|
+
- The command writes an owner-only `cutover.request` (`id`, `mode`, `requestedAt`, `requestedBy`), one at a time. It then follows `cutover.json`, printing each step as an in-process run would, and exits with the run's result: 0 when it is done, or 1 with the run's error code when the run fails, the daemon refuses the request, or nobody takes it within 2 minutes (the request is then withdrawn). It exits 2 when it cannot tell how the run ended: the daemon stopped, or 2 hours passed.
|
|
97
|
+
- The daemon takes one request at a time, and runs it beside its heartbeat. It refuses a request that is more than 10 minutes old, a duplicate, a malformed request, or a rollback with nothing to roll back; the reason is listed under `refusals` in `cutover.json`. It unloads only the old hub's LaunchAgents (`com.agentpulse.agents-hub`, `com.agentpulse.keepawake`), never its own, and then serves 127.0.0.1:8890 itself. `daemon-health.json` shows `cutover: { state, step, requestId }`. A restart asked for while a cutover runs waits until the cutover ends.
|
|
98
|
+
- If the daemon stops in the middle of a request, it finishes the request when it starts again, and records what it did under `recovery`. A cutover that had begun is rolled back, including one stopped after its last check passed. A rollback is run again unless it had begun loading the old hub. In that case the daemon only checks that the old hub holds the port, and otherwise leaves it to you, because restoring `hub.db` again could lose what the old hub has written since.
|
|
99
|
+
- With a daemon older than 1.1.3, or with `--in-process`, the command runs the cutover itself as 1.1.2 did, which needs `launchctl`.
|
|
100
|
+
|
|
92
101
|
## Bus identity and delivery states (1.1.1)
|
|
93
102
|
|
|
94
103
|
After cutover the runner hosts the bus, and `hub.mjs bus` works as before with these additions:
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agentpulselabs/cli-plugin-pulse",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.5",
|
|
4
4
|
"description": "AgentPulse CLI plugin for installing and operating the Pulse local connector (replaces the deprecated build-discipline Pulse plugin).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -21,8 +21,9 @@
|
|
|
21
21
|
"scripts": {
|
|
22
22
|
"test": "node --import ./test/support/register-loader.mjs --test \"test/*.test.js\" \"test/hub/*.test.js\"",
|
|
23
23
|
"contract:check": "node scripts/sync-contract.mjs --local",
|
|
24
|
-
"check": "node --check src/commands/pulse/install.js && node --check src/commands/pulse/pair.js && node --check src/commands/pulse/login.js && node --check src/commands/pulse/start.js && node --check src/commands/pulse/status.js && node --check src/commands/pulse/doctor.js && node --check src/commands/pulse/stop.js && node --check src/commands/pulse/restart.js && node --check src/lib/selfRestart.js && node --check src/lib/subprocess.js && node --check src/lib/daemonProcess.js && node --check src/commands/pulse/mirror.js && node --check src/commands/pulse/hub-write.js && node --check src/lib/localHubWriter.js && node --check src/lib/hubActions.js && node --check src/lib/localHub.js && node --check src/lib/pathRedaction.js && node --check src/lib/rulePacks.js && node --check src/lib/mirrorContract.js && node --check src/lib/graphWindows.js && node --check src/lib/mirrorSource.js && node --check src/lib/mirrorSync.js && node --check src/lib/mirrorService.js && node --check src/lib/mirrorDaemon.js && node --check src/lib/daemonComposition.js && node --check src/lib/daemonRuntime.js && node --check src/lib/daemonLoop.js && node --check src/daemon/main.js && node --check src/lib/sqlite.js && node --check src/lib/nodeFloor.js && node --check src/lib/runnerStore.js && node --check src/lib/hubDbReader.js && node --check src/lib/channelScanner.js && node --check src/lib/uploadRedactor.js && node --check src/lib/hubScan.js && node --check src/lib/outboxFlusher.js && node --check src/lib/hubSnapshot.js && node --check src/lib/reconcileView.js && node --check src/lib/reconcile.js && node --check src/lib/shadowService.js && node --check src/lib/cloudFirstCommands.js && node --check src/commands/pulse/import.js && node --check src/commands/pulse/shadow.js && node --check src/commands/pulse/parity.js && node --check src/runner/main.js && node --check src/runner/runnerProcess.js && node --check src/runner/facade.js && node --check src/runner/bridge.js && node --check src/runner/runnerChild.js && node --check src/runner/applier.js && node --check src/runner/capture.js && node --check src/runner/hubCapture.js && node --check src/clients/hub.mjs && node --check src/clients/hub-bridge.mjs && node --check src/runner/busIdentity.js && node --check src/runner/deliveryStates.js && node --check src/lib/doctorWarnings.js && node --check src/hub/lib/closure.mjs && node --check src/runner/channelDiscipline.js",
|
|
24
|
+
"check": "node --check src/commands/pulse/install.js && node --check src/commands/pulse/pair.js && node --check src/commands/pulse/login.js && node --check src/commands/pulse/start.js && node --check src/commands/pulse/status.js && node --check src/commands/pulse/doctor.js && node --check src/commands/pulse/stop.js && node --check src/commands/pulse/restart.js && node --check src/lib/selfRestart.js && node --check src/lib/subprocess.js && node --check src/lib/daemonProcess.js && node --check src/commands/pulse/mirror.js && node --check src/commands/pulse/hub-write.js && node --check src/lib/localHubWriter.js && node --check src/lib/hubActions.js && node --check src/lib/localHub.js && node --check src/lib/pathRedaction.js && node --check src/lib/rulePacks.js && node --check src/lib/mirrorContract.js && node --check src/lib/graphWindows.js && node --check src/lib/mirrorSource.js && node --check src/lib/mirrorSync.js && node --check src/lib/mirrorService.js && node --check src/lib/mirrorDaemon.js && node --check src/lib/daemonComposition.js && node --check src/lib/daemonRuntime.js && node --check src/lib/daemonLoop.js && node --check src/daemon/main.js && node --check src/lib/sqlite.js && node --check src/lib/nodeFloor.js && node --check src/lib/runnerStore.js && node --check src/lib/hubDbReader.js && node --check src/lib/channelScanner.js && node --check src/lib/uploadRedactor.js && node --check src/lib/hubScan.js && node --check src/lib/outboxFlusher.js && node --check src/lib/hubSnapshot.js && node --check src/lib/reconcileView.js && node --check src/lib/reconcile.js && node --check src/lib/shadowService.js && node --check src/lib/cutover.js && node --check src/lib/cutoverWiring.js && node --check src/lib/cutoverRequest.js && node --check src/commands/pulse/cutover.js && node --check src/lib/cloudFirstCommands.js && node --check src/commands/pulse/import.js && node --check src/commands/pulse/shadow.js && node --check src/commands/pulse/parity.js && node --check src/runner/main.js && node --check src/runner/runnerProcess.js && node --check src/runner/facade.js && node --check src/runner/bridge.js && node --check src/runner/runnerChild.js && node --check src/runner/applier.js && node --check src/runner/capture.js && node --check src/runner/loopMonitor.js && node --check src/runner/channelWalk.js && node --check src/runner/serial.js && node --check src/runner/directoryCache.js && node --check src/runner/hubCapture.js && node --check src/clients/hub.mjs && node --check src/clients/hub-bridge.mjs && node --check src/runner/busIdentity.js && node --check src/runner/deliveryStates.js && node --check src/lib/doctorWarnings.js && node --check src/hub/lib/closure.mjs && node --check src/runner/channelDiscipline.js",
|
|
25
25
|
"test:t2": "node --import ./test/support/register-loader.mjs --test --test-timeout=120000 \"test/t2/*.test.js\"",
|
|
26
|
+
"test:perf": "node --import ./test/support/register-loader.mjs --test --test-concurrency=1 --test-timeout=900000 \"test/perf/*.test.js\"",
|
|
26
27
|
"test:t3": "node --import ./test/support/register-loader.mjs --test --test-concurrency=1 --test-timeout=240000 \"test/t3/*.test.js\""
|
|
27
28
|
},
|
|
28
29
|
"engines": {
|
|
@@ -1,10 +1,13 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
1
2
|
import { Flags } from '@oclif/core';
|
|
2
3
|
import BaseCommand from '../../lib/BaseCommand.js';
|
|
3
4
|
import { defaultConfigDirectory } from '../../lib/application.js';
|
|
5
|
+
import { FOLLOW_UNKNOWN_EXIT, followCutoverRequest, takesCutoverRequests, writeCutoverRequest } from '../../lib/cutoverRequest.js';
|
|
4
6
|
import { createCutover } from '../../lib/cutoverWiring.js';
|
|
7
|
+
import { readOwnerOnlyJson } from '../../lib/storage.js';
|
|
5
8
|
|
|
6
9
|
export default class PulseCutover extends BaseCommand {
|
|
7
|
-
static description = 'Move this machine\'s local Agent Hub into the Pulse daemon (a short freeze, a backup kept), check first with --dry-run, or go back with --rollback (restores the backup; loses only what was written after the switch).';
|
|
10
|
+
static description = 'Move this machine\'s local Agent Hub into the Pulse daemon (a short freeze, a backup kept), check first with --dry-run, or go back with --rollback (restores the backup; loses only what was written after the switch). The running daemon does the work, so no launchctl or permission prompt is needed here; --in-process runs it in this process instead.';
|
|
8
11
|
static examples = [
|
|
9
12
|
'<%= config.bin %> pulse cutover --dry-run',
|
|
10
13
|
'<%= config.bin %> pulse cutover --smoke-channel agentpulse-agent-hub/cf-phase3',
|
|
@@ -14,26 +17,47 @@ export default class PulseCutover extends BaseCommand {
|
|
|
14
17
|
'dry-run': Flags.boolean({ description: 'Check everything and say what would change; change nothing', default: false }),
|
|
15
18
|
rollback: Flags.boolean({ description: 'Back to the local hub from the backup', default: false, exclusive: ['dry-run'] }),
|
|
16
19
|
'smoke-channel': Flags.string({ description: 'A task channel (<repo>/<task>) for the smoke test\'s one channel post' }),
|
|
20
|
+
'in-process': Flags.boolean({ description: 'Run the cutover in this process, as Pulse 1.1.2 did, instead of asking the daemon (needs launchctl)', default: false }),
|
|
17
21
|
};
|
|
18
22
|
|
|
19
23
|
async run() {
|
|
20
24
|
const { flags } = await this.parse(PulseCutover);
|
|
21
|
-
const
|
|
25
|
+
const configDir = this.configDir || defaultConfigDirectory();
|
|
26
|
+
const log = (event, details) => this.log(`${event} ${JSON.stringify(details)}`);
|
|
27
|
+
const inProcess = async () => this.cutover || createCutover({ configDir, log });
|
|
22
28
|
if (flags['dry-run']) {
|
|
23
|
-
const report = await
|
|
29
|
+
const report = await (await inProcess()).dryRun();
|
|
24
30
|
for (const check of report.checks) this.log(`${check.ok ? 'ok ' : 'FAIL'} ${check.name}${check.detail ? `: ${check.detail}` : ''}${!check.ok && check.fix ? ` (fix: ${check.fix})` : ''}`);
|
|
25
31
|
this.log(`Outage while it runs: ${report.outage.what}; at most ${Math.round(report.outage.boundMs / 60_000)} min.`);
|
|
26
32
|
this.log(report.message);
|
|
27
33
|
if (!report.ok) process.exitCode = 1;
|
|
28
34
|
return report;
|
|
29
35
|
}
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
36
|
+
const health = await readOwnerOnlyJson(path.join(configDir, 'daemon-health.json')).catch(() => null);
|
|
37
|
+
const viaDaemon = !flags['in-process'] && takesCutoverRequests(health);
|
|
38
|
+
if (!viaDaemon && !flags['in-process']) {
|
|
39
|
+
this.log(`The Pulse daemon ${health?.connectorVersion ? `(connector ${health.connectorVersion}) ` : ''}does not take cutover requests or is not running; running it in this process.`);
|
|
40
|
+
}
|
|
41
|
+
const mode = flags.rollback ? 'rollback' : 'cutover';
|
|
42
|
+
let result;
|
|
43
|
+
if (viaDaemon) {
|
|
44
|
+
const request = await writeCutoverRequest({ configDir, mode, smokeChannel: mode === 'cutover' ? flags['smoke-channel'] ?? null : null, health });
|
|
45
|
+
this.log(`Asked the Pulse daemon (pid ${health.pid}) to ${mode === 'rollback' ? 'roll back' : 'cut over'}: request ${request.id}.`);
|
|
46
|
+
try {
|
|
47
|
+
result = await followCutoverRequest({ configDir, request, log, ...this.follow });
|
|
48
|
+
} catch (error) {
|
|
49
|
+
if (error.exit === FOLLOW_UNKNOWN_EXIT) this.fail(error.code, error.message, error.exit);
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
} else {
|
|
53
|
+
const cutover = await inProcess();
|
|
54
|
+
result = mode === 'rollback' ? await cutover.rollback() : await cutover.run({ smokeChannel: flags['smoke-channel'] ?? null });
|
|
55
|
+
}
|
|
56
|
+
if (mode === 'rollback') {
|
|
57
|
+
this.log(result.message);
|
|
58
|
+
return result;
|
|
34
59
|
}
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
return journal;
|
|
60
|
+
this.log(`Cut over: the Pulse daemon serves 127.0.0.1:${result.port}. The backup is ${result.backup.file}. Go back any time with \`agentpulse pulse cutover --rollback\`.`);
|
|
61
|
+
return result;
|
|
38
62
|
}
|
|
39
63
|
}
|
package/src/daemon/main.js
CHANGED
|
@@ -10,6 +10,7 @@ const MIRROR_FAILURES = new Set(['mirror_failed', 'mirror_blocked', 'mirror_stre
|
|
|
10
10
|
const ERROR_EVENTS = new Set([
|
|
11
11
|
'job_failed', 'job_refused', 'job_settle_failed', 'cycle_stalled', 'drain_timed_out', 'daemon_start_failed',
|
|
12
12
|
'daemon_shutdown_failed', 'source_watch_refused', 'source_watch_failed',
|
|
13
|
+
'cutover_failed', 'cutover_request_failed', 'cutover_request_refused', 'cutover_interrupted', 'cutover_service_failed',
|
|
13
14
|
]);
|
|
14
15
|
|
|
15
16
|
/** One redacted line per job, local hub or restart event; failures go to daemon-error.log, the rest to daemon.log. */
|
package/src/hub/core/db.mjs
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import fs from 'node:fs';
|
|
4
4
|
import path from 'node:path';
|
|
5
5
|
import { DatabaseSync } from 'node:sqlite';
|
|
6
|
+
import { isBusy } from './ticks.mjs';
|
|
6
7
|
|
|
7
8
|
export const SCHEMA_VERSION = 1;
|
|
8
9
|
|
|
@@ -231,22 +232,37 @@ export function withTx(db, fn, label = null) {
|
|
|
231
232
|
}
|
|
232
233
|
|
|
233
234
|
const pause = (ms) => { if (ms > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); };
|
|
235
|
+
/** How long a busy chunk of writeInChunks waits before each new try; the wait blocks its thread, the collect worker's. */
|
|
236
|
+
const BUSY_PAUSES_MS = [250, 500, 1000, 2000];
|
|
234
237
|
|
|
235
238
|
/**
|
|
236
239
|
* Runs `steps` (functions) in as many short write transactions as it takes for none to hold the lock past `budgetMs`,
|
|
237
240
|
* pausing `pauseMs` between them so writers on other connections get in. Inside an open transaction it simply joins it.
|
|
238
|
-
*
|
|
241
|
+
* A chunk that finds the database busy past busy_timeout is rolled back and run again from its first step, after
|
|
242
|
+
* `busyPausesMs`, so another writer's long transaction delays a pass instead of failing it. Returns how many
|
|
243
|
+
* transactions it used.
|
|
239
244
|
*/
|
|
240
|
-
export function writeInChunks(db, steps, { budgetMs = 50, pauseMs = 15, label = 'chunked write', onPause = null } = {}) {
|
|
245
|
+
export function writeInChunks(db, steps, { budgetMs = 50, pauseMs = 15, label = 'chunked write', onPause = null, busyPausesMs = BUSY_PAUSES_MS } = {}) {
|
|
241
246
|
if (TX.get(db)) { for (const step of steps) step(); return 0; }
|
|
242
247
|
let i = 0;
|
|
243
248
|
let chunks = 0;
|
|
249
|
+
let busy = 0;
|
|
244
250
|
while (i < steps.length) {
|
|
245
251
|
if (chunks) { pause(pauseMs); if (onPause) onPause(); }
|
|
246
252
|
const started = performance.now();
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
253
|
+
const first = i;
|
|
254
|
+
try {
|
|
255
|
+
withTx(db, () => {
|
|
256
|
+
do steps[i++](); while (i < steps.length && performance.now() - started < budgetMs);
|
|
257
|
+
}, `${label} (chunk ${chunks + 1})`);
|
|
258
|
+
} catch (e) {
|
|
259
|
+
if (!isBusy(e) || busy >= busyPausesMs.length) throw e;
|
|
260
|
+
txTiming.log(`${label}: database busy, chunk ${chunks + 1} again in ${busyPausesMs[busy]} ms`);
|
|
261
|
+
pause(busyPausesMs[busy++]);
|
|
262
|
+
i = first;
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
265
|
+
busy = 0;
|
|
250
266
|
chunks++;
|
|
251
267
|
}
|
|
252
268
|
return chunks;
|
|
@@ -20,6 +20,10 @@ const MAX_OUTPUT = 2 * 1024 * 1024;
|
|
|
20
20
|
/** Answers took 0.7-9.8 min in one chat resumed for two days; a fresh, small chat answers well inside this. */
|
|
21
21
|
export const RUN_TIMEOUT_MS = 5 * 60e3;
|
|
22
22
|
const MAX_ATTEMPTS = 3;
|
|
23
|
+
/** Claims also count a run's own lapsed claim taken back after a stall; past this many, a message no run finishes is given up. */
|
|
24
|
+
const MAX_CLAIMS = 10;
|
|
25
|
+
/** A run that ends while the lease is away (a stalled runner lapses it for a moment) waits this long for it to come back. */
|
|
26
|
+
export const HELD_WAIT_MS = 60e3;
|
|
23
27
|
/** Runs cut off by restarts do not use up attempts; past this many restarts a message is given up, not looped on. */
|
|
24
28
|
export const MAX_RESTARTS = 3;
|
|
25
29
|
/** Runs at once, one chat each. One, as before: a second question waits one run at most, FYIs none. */
|
|
@@ -129,10 +133,19 @@ ${voice || ''}`.trim();
|
|
|
129
133
|
const topicOf = (sender) => (sender === 'user' ? 'user' : `agent/${sender}`);
|
|
130
134
|
const errorText = (error) => String(error && error.message ? error.message : error);
|
|
131
135
|
|
|
136
|
+
/** Why a cursor-agent run failed (exit, signal, stderr), never its command line: that holds the whole prompt. */
|
|
137
|
+
function runError(error) {
|
|
138
|
+
if (!error || !String(error.message || '').startsWith('Command failed:')) return errorText(error);
|
|
139
|
+
const how = error.killed ? `timed out (${error.signal})` : error.code != null ? `exit ${error.code}` : `signal ${error.signal}`;
|
|
140
|
+
const said = String(error.stderr || error.stdout || '').trim().split('\n').slice(-3).join(' ').slice(0, 300);
|
|
141
|
+
return `cursor-agent ${how}${said ? `: ${said}` : ''}`;
|
|
142
|
+
}
|
|
143
|
+
|
|
132
144
|
export function createManager({
|
|
133
145
|
hub, state, run = runFile, onChange = () => {}, parallel = PARALLEL, now = Date.now, log = console.error, ackRetryMs = RETRY_MS,
|
|
134
146
|
held = () => true,
|
|
135
147
|
fyi = null,
|
|
148
|
+
heldWaitMs = HELD_WAIT_MS,
|
|
136
149
|
}) {
|
|
137
150
|
const configFile = path.join(state, 'manager.json');
|
|
138
151
|
const statusFile = path.join(state, 'manager-status.json');
|
|
@@ -143,6 +156,8 @@ export function createManager({
|
|
|
143
156
|
const { messaging, registry, leases } = hub;
|
|
144
157
|
/** message id → { message, leaseToken, slot, startedAt, abort } */
|
|
145
158
|
const inflight = new Map();
|
|
159
|
+
/** message id → { count, error } of its runs that failed; claims alone are no attempts. */
|
|
160
|
+
const failures = new Map();
|
|
146
161
|
const runs = new Set();
|
|
147
162
|
let lastHeartbeat = 0;
|
|
148
163
|
let timer = null;
|
|
@@ -222,6 +237,15 @@ export function createManager({
|
|
|
222
237
|
}, 'manager answer'), { delaysMs: ackRetryMs, label: `manager answer to ${m.id}`, log });
|
|
223
238
|
}
|
|
224
239
|
|
|
240
|
+
/** Only the lease holder settles; a stall that lapsed the lease here gets it back moments later. */
|
|
241
|
+
async function heldAgain(signal) {
|
|
242
|
+
for (let waited = 0; !held(); waited += 250) {
|
|
243
|
+
if (signal.aborted || waited >= heldWaitMs) return false;
|
|
244
|
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
|
245
|
+
}
|
|
246
|
+
return !signal.aborted;
|
|
247
|
+
}
|
|
248
|
+
|
|
225
249
|
async function answer(f) {
|
|
226
250
|
const { message: m, slot, abort } = f;
|
|
227
251
|
const chat = await chatFor(slot, abort.signal);
|
|
@@ -236,6 +260,10 @@ export function createManager({
|
|
|
236
260
|
if (envelope && envelope.is_error) throw new Error(`cursor-agent failed: ${String(textOf(envelope) || 'error').slice(0, 300)}`);
|
|
237
261
|
const actions = parseManagerActions(stdout);
|
|
238
262
|
if (!actions.reply) throw new Error('manager returned no reply');
|
|
263
|
+
if (!(await heldAgain(abort.signal))) {
|
|
264
|
+
if (!abort.signal.aborted) log(`manager answer to ${m.id} held back: the manager lease is not held here`);
|
|
265
|
+
return null;
|
|
266
|
+
}
|
|
239
267
|
await settle(m, actions);
|
|
240
268
|
recordTurn(slot, chat.chatId, m, prompt, actions.reply);
|
|
241
269
|
if (registry.fact('manager-restart', m.id)) registry.dropFact('manager-restart', m.id);
|
|
@@ -252,11 +280,16 @@ export function createManager({
|
|
|
252
280
|
const done = answer(f).then((replied) => ({ replied }), (error) => ({ error })).then(({ replied, error }) => {
|
|
253
281
|
inflight.delete(m.id);
|
|
254
282
|
runs.delete(done);
|
|
255
|
-
if (f.abort.signal.aborted) return;
|
|
256
|
-
|
|
283
|
+
if (f.abort.signal.aborted) { log(`manager run for ${m.id} was stopped before it finished; it is not counted as a try`); return; }
|
|
284
|
+
const why = error ? runError(error).slice(0, 500) : null;
|
|
285
|
+
if (error) {
|
|
286
|
+
const failed = { count: ((failures.get(m.id) || {}).count || 0) + 1, error: why };
|
|
287
|
+
failures.set(m.id, failed);
|
|
288
|
+
log(`manager failed for ${m.id} (try ${failed.count} of ${MAX_ATTEMPTS}): ${why}`);
|
|
289
|
+
} else if (replied) failures.delete(m.id);
|
|
257
290
|
status({
|
|
258
291
|
...(replied || {}), state: error ? 'error' : inflight.size ? 'working' : 'online', messageId: error ? m.id : null,
|
|
259
|
-
error: error ?
|
|
292
|
+
error: why, inflight: inflightView(), ...(error ? { lastFailure: { id: m.id, from: m.from, error: why, at: now() } } : {}),
|
|
260
293
|
});
|
|
261
294
|
}).catch((e) => log(`manager status after ${m.id} failed: ${errorText(e)}`));
|
|
262
295
|
runs.add(done);
|
|
@@ -315,9 +348,14 @@ export function createManager({
|
|
|
315
348
|
const own = inflight.get(message.id);
|
|
316
349
|
if (own) { own.leaseToken = message.leaseToken; continue; }
|
|
317
350
|
if (isFyi(message)) { if (!(fyi && fyi(message))) messaging.ack('manager', message.id, {}, registry.directory()); continue; }
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
351
|
+
const failed = failures.get(message.id);
|
|
352
|
+
if ((failed && failed.count >= MAX_ATTEMPTS) || message.attempts > MAX_CLAIMS) {
|
|
353
|
+
const tries = failed ? failed.count : message.attempts - 1;
|
|
354
|
+
const last = failed ? failed.error : `none of its ${tries} runs finished (each was cut off by a restart or a lost lease)`;
|
|
355
|
+
messaging.ack('manager', message.id, { reply: `I could not answer this after ${tries} tries, so I stopped retrying. Last error: ${last}`, fromName: 'Manager' }, registry.directory());
|
|
356
|
+
failures.delete(message.id);
|
|
357
|
+
log(`manager gave up on ${message.id} after ${tries} tries: ${last}`);
|
|
358
|
+
status({ gaveUp: { id: message.id, from: message.from, tries, error: last, at: t } });
|
|
321
359
|
continue;
|
|
322
360
|
}
|
|
323
361
|
dispatch(message);
|
package/src/lib/contract.sha256
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
|
|
1
|
+
c9db92b38d0c978d4fa18d8a731fadbf3013a19659b524d615583ad099a8fafc connector-contract.json
|
|
2
2
|
7b7a59c08797c100b46436123dfcb09b517730357fa90cd0890da9b80308faf3 mirror-contract.json
|
package/src/lib/cutover.js
CHANGED
|
@@ -5,6 +5,7 @@ import { setTimeout as sleep } from 'node:timers/promises';
|
|
|
5
5
|
import { clientState, installClients, restoreClients } from './clientInstaller.js';
|
|
6
6
|
import { HubError } from './errors.js';
|
|
7
7
|
import { RunnerStore, SEED_META } from './runnerStore.js';
|
|
8
|
+
import { SERVICE_LABEL } from './serviceManager.js';
|
|
8
9
|
import { readOwnerOnlyJson, writeOwnerOnlyJson } from './storage.js';
|
|
9
10
|
|
|
10
11
|
export const CUTOVER_FILE = 'cutover.json';
|
|
@@ -20,6 +21,9 @@ export const STEPS = Object.freeze(['sign_in_moved', 'shims_offline', 'hub_unloa
|
|
|
20
21
|
|
|
21
22
|
const sha256File = async (file) => createHash('sha256').update(await readFile(file)).digest('hex');
|
|
22
23
|
|
|
24
|
+
/** What a fresh journal keeps from the last one: the daemon's request history (CUT-03). */
|
|
25
|
+
const carried = (previous) => Object.fromEntries(['requests', 'refusals'].filter((key) => previous?.[key]).map((key) => [key, previous[key]]));
|
|
26
|
+
|
|
23
27
|
/** `version >= floor` on the numeric x.y.z core; anything unparsable is older. */
|
|
24
28
|
function atLeast(version, floor) {
|
|
25
29
|
const parse = (v) => String(v ?? '').match(/^(\d+)\.(\d+)\.(\d+)/)?.slice(1).map(Number) ?? null;
|
|
@@ -34,20 +38,23 @@ function atLeast(version, floor) {
|
|
|
34
38
|
* a backup kept; `--rollback` restores that backup and the old hub. Every effect goes through an injected seam:
|
|
35
39
|
* `services` (launchd, by label), `probe` (who holds hub.db), `cloudFirst` (Phase 2's final import and reconcile),
|
|
36
40
|
* `daemon` (the facade's health and the old hub's), `smoke`, `backupDb`, `applySpool` and `signIn` (the TokenStore). Each step is journaled in
|
|
37
|
-
* cutover.json before the next starts; a failure at any step after the freeze began rolls back by itself.
|
|
41
|
+
* cutover.json before the next starts; a failure at any step after the freeze began rolls back by itself. `serve` is
|
|
42
|
+
* called after each change of `cloudFirst.mode`: run inside the daemon (CUT-03) it starts or stops the daemon's own
|
|
43
|
+
* runner at once instead of on its next heartbeat.
|
|
38
44
|
*/
|
|
39
45
|
export class Cutover {
|
|
40
46
|
constructor({
|
|
41
|
-
configDir, hubHome, port = 8890, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn,
|
|
47
|
+
configDir, hubHome, port = 8890, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn, serve = async () => {},
|
|
42
48
|
labels = OLD_HUB_LABELS, holderWaitMs = 120_000, facadeWaitMs = 90_000, pollMs = 1_000, now = Date.now, wait = sleep, log = () => {},
|
|
43
49
|
}) {
|
|
50
|
+
if (labels.includes(SERVICE_LABEL)) throw new HubError('HUB_CUTOVER_LABELS', `The cutover never unloads the Pulse daemon itself (${SERVICE_LABEL})`);
|
|
44
51
|
this.configDir = configDir;
|
|
45
52
|
this.configFile = path.join(configDir, 'config.json');
|
|
46
53
|
this.journalFile = path.join(configDir, CUTOVER_FILE);
|
|
47
54
|
this.hubHome = hubHome;
|
|
48
55
|
this.hubDb = path.join(hubHome, 'state', 'hub.db');
|
|
49
56
|
this.backupFile = `${this.hubDb}${BACKUP_SUFFIX}`;
|
|
50
|
-
Object.assign(this, { port, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn, labels, holderWaitMs, facadeWaitMs, pollMs, now, wait, log });
|
|
57
|
+
Object.assign(this, { port, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn, serve, labels, holderWaitMs, facadeWaitMs, pollMs, now, wait, log });
|
|
51
58
|
}
|
|
52
59
|
|
|
53
60
|
async journal() {
|
|
@@ -188,11 +195,17 @@ export class Cutover {
|
|
|
188
195
|
};
|
|
189
196
|
}
|
|
190
197
|
|
|
191
|
-
/**
|
|
192
|
-
|
|
198
|
+
/**
|
|
199
|
+
* The cutover. Resolves with the journal; a failure after the freeze began rolls back and rethrows. `request` is the
|
|
200
|
+
* daemon's accepted cutover.request (CUT-03), kept on the journal so its CLI can follow it.
|
|
201
|
+
*/
|
|
202
|
+
async run({ smokeChannel = null, request = null } = {}) {
|
|
193
203
|
const preflight = await this.dryRun();
|
|
194
204
|
if (!preflight.ok) throw new HubError('HUB_CUTOVER_NOT_READY', preflight.message, { details: { checks: preflight.checks.filter((c) => !c.ok) } });
|
|
195
|
-
const journal = {
|
|
205
|
+
const journal = {
|
|
206
|
+
state: 'running', startedAt: new Date(this.now()).toISOString(), hubHome: this.hubHome, port: this.port, steps: {}, loadedBefore: [],
|
|
207
|
+
runBy: request ? 'daemon' : 'cli', pid: process.pid, ...carried(await this.journal()), ...(request ? { request } : {}),
|
|
208
|
+
};
|
|
196
209
|
await writeOwnerOnlyJson(this.journalFile, journal);
|
|
197
210
|
try {
|
|
198
211
|
// Before the freeze: the sign-in moves to this config's own Keychain item (the 12:52 DECISION); no outage.
|
|
@@ -240,6 +253,7 @@ export class Cutover {
|
|
|
240
253
|
cloudFirst: { ...(config.cloudFirst || {}), mode: 'runner', hubHome: this.hubHome, port: this.port, cutoverSeq, seedFrom: this.backupFile, seedSha256: journal.backup.sha256, shadow: false },
|
|
241
254
|
});
|
|
242
255
|
journal.cutoverSeq = cutoverSeq;
|
|
256
|
+
await this.serve();
|
|
243
257
|
const serving = this.now() + this.facadeWaitMs;
|
|
244
258
|
while (!(await this.daemon.facadeUp(this.port))) {
|
|
245
259
|
if (this.now() >= serving) throw new HubError('HUB_CUTOVER_FACADE', `the Pulse daemon did not serve 127.0.0.1:${this.port} within ${this.facadeWaitMs} ms`);
|
|
@@ -271,27 +285,33 @@ export class Cutover {
|
|
|
271
285
|
*/
|
|
272
286
|
async rollback({ reason = 'asked' } = {}) {
|
|
273
287
|
const journal = await this.journal();
|
|
274
|
-
const report = { command: 'cutover', rollback: true, reason, steps: [] };
|
|
275
|
-
|
|
288
|
+
const report = { command: 'cutover', rollback: true, reason, startedAt: new Date(this.now()).toISOString(), steps: [] };
|
|
289
|
+
// Each step is journaled as it ends, so a CLI following a daemon's rollback (CUT-03) prints it.
|
|
290
|
+
const step = async (name, detail = {}) => {
|
|
291
|
+
report.steps.push({ name, ...detail });
|
|
292
|
+
await writeOwnerOnlyJson(this.journalFile, { ...journal, rollback: report });
|
|
293
|
+
this.log('rollback_step', { name, ...detail });
|
|
294
|
+
};
|
|
276
295
|
const config = await this.#config();
|
|
277
296
|
if (config.cloudFirst?.mode === 'runner') {
|
|
278
297
|
await writeOwnerOnlyJson(this.configFile, { ...config, cloudFirst: { ...config.cloudFirst, mode: 'rolled_back' } });
|
|
298
|
+
await this.serve();
|
|
279
299
|
const deadline = this.now() + this.facadeWaitMs;
|
|
280
300
|
while (await this.daemon.facadeUp(this.port)) {
|
|
281
301
|
if (this.now() >= deadline) throw new HubError('HUB_ROLLBACK_FACADE', `the Pulse daemon still serves 127.0.0.1:${this.port}`);
|
|
282
302
|
await this.wait(this.pollMs);
|
|
283
303
|
}
|
|
284
|
-
step('daemon_left_port');
|
|
304
|
+
await step('daemon_left_port');
|
|
285
305
|
}
|
|
286
306
|
if (journal.cutoverAt) {
|
|
287
307
|
const lost = await this.#exportLost(journal.cutoverAt);
|
|
288
308
|
report.lost = lost;
|
|
289
|
-
step('lost_exported', lost);
|
|
309
|
+
await step('lost_exported', lost);
|
|
290
310
|
}
|
|
291
311
|
const marked = await this.#markSeedRolledBack();
|
|
292
|
-
if (marked) step('runner_db_marked', marked);
|
|
312
|
+
if (marked) await step('runner_db_marked', marked);
|
|
293
313
|
await restoreClients({ hubHome: this.hubHome });
|
|
294
|
-
step('entry_points_restored');
|
|
314
|
+
await step('entry_points_restored');
|
|
295
315
|
if (journal.steps?.backed_up && (await stat(this.backupFile).catch(() => null))) {
|
|
296
316
|
for (const suffix of ['-wal', '-shm']) await rm(`${this.hubDb}${suffix}`, { force: true });
|
|
297
317
|
const restored = `${this.hubDb}.pulse-restore`;
|
|
@@ -300,13 +320,15 @@ export class Cutover {
|
|
|
300
320
|
await rm(this.hubDb, { force: true });
|
|
301
321
|
await copyFile(restored, this.hubDb);
|
|
302
322
|
await rm(restored, { force: true });
|
|
303
|
-
step('hub_db_restored', { from: this.backupFile, sha256: await sha256File(this.backupFile) });
|
|
323
|
+
await step('hub_db_restored', { from: this.backupFile, sha256: await sha256File(this.backupFile) });
|
|
304
324
|
}
|
|
305
325
|
const spool = await this.applySpool(this.hubDb);
|
|
306
|
-
step('spool_applied', spool);
|
|
326
|
+
await step('spool_applied', spool);
|
|
307
327
|
const loadedBefore = journal.loadedBefore?.length ? journal.loadedBefore : this.labels;
|
|
328
|
+
// Journaled before the first load: from here the old hub may have hub.db open, so it is never restored again.
|
|
329
|
+
await step('hub_loading', { labels: loadedBefore });
|
|
308
330
|
for (const label of loadedBefore) await this.services.restoreLabel(label);
|
|
309
|
-
step('hub_loaded', { labels: loadedBefore });
|
|
331
|
+
await step('hub_loaded', { labels: loadedBefore });
|
|
310
332
|
const deadline = this.now() + this.facadeWaitMs;
|
|
311
333
|
while (!(await this.daemon.oldHubUp(this.port, { timeoutMs: ROLLBACK_PROBE_MS }))) {
|
|
312
334
|
if (this.now() >= deadline) {
|
|
@@ -317,7 +339,7 @@ export class Cutover {
|
|
|
317
339
|
}
|
|
318
340
|
await this.wait(this.pollMs);
|
|
319
341
|
}
|
|
320
|
-
step('hub_serving');
|
|
342
|
+
await step('hub_serving');
|
|
321
343
|
report.message = report.lost?.count
|
|
322
344
|
? `Rolled back to the local hub. ${report.lost.count} write(s) made after the switch (${Object.entries(report.lost.kinds).map(([kind, n]) => `${kind} ${n}`).join(', ')}) are not in hub.db; they are listed in ${report.lost.file}.`
|
|
323
345
|
: 'Rolled back to the local hub. Nothing written after the switch was lost.';
|
|
@@ -325,6 +347,19 @@ export class Cutover {
|
|
|
325
347
|
return report;
|
|
326
348
|
}
|
|
327
349
|
|
|
350
|
+
/**
|
|
351
|
+
* CUT-03 boot recovery of a rollback that had begun loading the old hub: whether the old hub holds the port again
|
|
352
|
+
* (it answers, or it is hung but owned by its job), waiting up to facadeWaitMs. Reads only.
|
|
353
|
+
*/
|
|
354
|
+
async verifyRolledBack() {
|
|
355
|
+
const deadline = this.now() + this.facadeWaitMs;
|
|
356
|
+
for (;;) {
|
|
357
|
+
const onPort = await this.#oldHubOnPort();
|
|
358
|
+
if (onPort.ok || this.now() >= deadline) return onPort;
|
|
359
|
+
await this.wait(this.pollMs);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
328
363
|
/**
|
|
329
364
|
* After the runner left the port: the seed is marked rolled back, so the next cutover replaces it with its own
|
|
330
365
|
* backup instead of serving this attempt's state (B1). Null when runner.db was never seeded.
|