@agentpulselabs/cli-plugin-pulse 1.1.2 → 1.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -89,6 +89,15 @@ These need your team's Pulse cloud-first flag on, set by a team owner or admin,
89
89
 
90
90
  Everything sent goes through the same secret and path filter as the mirror, and the cloud filters it again.
91
91
 
92
+ ## Cutover through the daemon (1.1.3)
93
+
94
+ `agentpulse pulse cutover` and `--rollback` ask the running Pulse daemon to do the work, so the command itself never needs `launchctl` or a permission prompt. Only `--dry-run`, which only reads, runs in the command.
95
+
96
+ - The command writes an owner-only `cutover.request` (`id`, `mode`, `requestedAt`, `requestedBy`), one at a time. It then follows `cutover.json`, printing each step as an in-process run would, and exits with the run's result: 0 when it is done, or 1 with the run's error code when the run fails, the daemon refuses the request, or nobody takes it within 2 minutes (the request is then withdrawn). It exits 2 when it cannot tell how the run ended: the daemon stopped, or 2 hours passed.
97
+ - The daemon takes one request at a time, and runs it beside its heartbeat. It refuses a request that is more than 10 minutes old, a duplicate, a malformed request, or a rollback with nothing to roll back; the reason is listed under `refusals` in `cutover.json`. It unloads only the old hub's LaunchAgents (`com.agentpulse.agents-hub`, `com.agentpulse.keepawake`), never its own, and then serves 127.0.0.1:8890 itself. `daemon-health.json` shows `cutover: { state, step, requestId }`. A restart asked for while a cutover runs waits until the cutover ends.
98
+ - If the daemon stops in the middle of a request, it finishes the request when it starts again, and records what it did under `recovery`. A cutover that had begun is rolled back, including one stopped after its last check passed. A rollback is run again unless it had begun loading the old hub. In that case the daemon only checks that the old hub holds the port, and otherwise leaves it to you, because restoring `hub.db` again could lose what the old hub has written since.
99
+ - With a daemon older than 1.1.3, or with `--in-process`, the command runs the cutover itself as 1.1.2 did, which needs `launchctl`.
100
+
92
101
  ## Bus identity and delivery states (1.1.1)
93
102
 
94
103
  After cutover the runner hosts the bus, and `hub.mjs bus` works as before with these additions:
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@agentpulselabs/cli-plugin-pulse",
3
- "version": "1.1.2",
3
+ "version": "1.1.5",
4
4
  "description": "AgentPulse CLI plugin for installing and operating the Pulse local connector (replaces the deprecated build-discipline Pulse plugin).",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -21,8 +21,9 @@
21
21
  "scripts": {
22
22
  "test": "node --import ./test/support/register-loader.mjs --test \"test/*.test.js\" \"test/hub/*.test.js\"",
23
23
  "contract:check": "node scripts/sync-contract.mjs --local",
24
- "check": "node --check src/commands/pulse/install.js && node --check src/commands/pulse/pair.js && node --check src/commands/pulse/login.js && node --check src/commands/pulse/start.js && node --check src/commands/pulse/status.js && node --check src/commands/pulse/doctor.js && node --check src/commands/pulse/stop.js && node --check src/commands/pulse/restart.js && node --check src/lib/selfRestart.js && node --check src/lib/subprocess.js && node --check src/lib/daemonProcess.js && node --check src/commands/pulse/mirror.js && node --check src/commands/pulse/hub-write.js && node --check src/lib/localHubWriter.js && node --check src/lib/hubActions.js && node --check src/lib/localHub.js && node --check src/lib/pathRedaction.js && node --check src/lib/rulePacks.js && node --check src/lib/mirrorContract.js && node --check src/lib/graphWindows.js && node --check src/lib/mirrorSource.js && node --check src/lib/mirrorSync.js && node --check src/lib/mirrorService.js && node --check src/lib/mirrorDaemon.js && node --check src/lib/daemonComposition.js && node --check src/lib/daemonRuntime.js && node --check src/lib/daemonLoop.js && node --check src/daemon/main.js && node --check src/lib/sqlite.js && node --check src/lib/nodeFloor.js && node --check src/lib/runnerStore.js && node --check src/lib/hubDbReader.js && node --check src/lib/channelScanner.js && node --check src/lib/uploadRedactor.js && node --check src/lib/hubScan.js && node --check src/lib/outboxFlusher.js && node --check src/lib/hubSnapshot.js && node --check src/lib/reconcileView.js && node --check src/lib/reconcile.js && node --check src/lib/shadowService.js && node --check src/lib/cloudFirstCommands.js && node --check src/commands/pulse/import.js && node --check src/commands/pulse/shadow.js && node --check src/commands/pulse/parity.js && node --check src/runner/main.js && node --check src/runner/runnerProcess.js && node --check src/runner/facade.js && node --check src/runner/bridge.js && node --check src/runner/runnerChild.js && node --check src/runner/applier.js && node --check src/runner/capture.js && node --check src/runner/hubCapture.js && node --check src/clients/hub.mjs && node --check src/clients/hub-bridge.mjs && node --check src/runner/busIdentity.js && node --check src/runner/deliveryStates.js && node --check src/lib/doctorWarnings.js && node --check src/hub/lib/closure.mjs && node --check src/runner/channelDiscipline.js",
24
+ "check": "node --check src/commands/pulse/install.js && node --check src/commands/pulse/pair.js && node --check src/commands/pulse/login.js && node --check src/commands/pulse/start.js && node --check src/commands/pulse/status.js && node --check src/commands/pulse/doctor.js && node --check src/commands/pulse/stop.js && node --check src/commands/pulse/restart.js && node --check src/lib/selfRestart.js && node --check src/lib/subprocess.js && node --check src/lib/daemonProcess.js && node --check src/commands/pulse/mirror.js && node --check src/commands/pulse/hub-write.js && node --check src/lib/localHubWriter.js && node --check src/lib/hubActions.js && node --check src/lib/localHub.js && node --check src/lib/pathRedaction.js && node --check src/lib/rulePacks.js && node --check src/lib/mirrorContract.js && node --check src/lib/graphWindows.js && node --check src/lib/mirrorSource.js && node --check src/lib/mirrorSync.js && node --check src/lib/mirrorService.js && node --check src/lib/mirrorDaemon.js && node --check src/lib/daemonComposition.js && node --check src/lib/daemonRuntime.js && node --check src/lib/daemonLoop.js && node --check src/daemon/main.js && node --check src/lib/sqlite.js && node --check src/lib/nodeFloor.js && node --check src/lib/runnerStore.js && node --check src/lib/hubDbReader.js && node --check src/lib/channelScanner.js && node --check src/lib/uploadRedactor.js && node --check src/lib/hubScan.js && node --check src/lib/outboxFlusher.js && node --check src/lib/hubSnapshot.js && node --check src/lib/reconcileView.js && node --check src/lib/reconcile.js && node --check src/lib/shadowService.js && node --check src/lib/cutover.js && node --check src/lib/cutoverWiring.js && node --check src/lib/cutoverRequest.js && node --check src/commands/pulse/cutover.js && node --check src/lib/cloudFirstCommands.js && node --check src/commands/pulse/import.js && node --check src/commands/pulse/shadow.js && node --check src/commands/pulse/parity.js && node --check src/runner/main.js && node --check src/runner/runnerProcess.js && node --check src/runner/facade.js && node --check src/runner/bridge.js && node --check src/runner/runnerChild.js && node --check src/runner/applier.js && node --check src/runner/capture.js && node --check src/runner/loopMonitor.js && node --check src/runner/channelWalk.js && node --check src/runner/serial.js && node --check src/runner/directoryCache.js && node --check src/runner/hubCapture.js && node --check src/clients/hub.mjs && node --check src/clients/hub-bridge.mjs && node --check src/runner/busIdentity.js && node --check src/runner/deliveryStates.js && node --check src/lib/doctorWarnings.js && node --check src/hub/lib/closure.mjs && node --check src/runner/channelDiscipline.js",
25
25
  "test:t2": "node --import ./test/support/register-loader.mjs --test --test-timeout=120000 \"test/t2/*.test.js\"",
26
+ "test:perf": "node --import ./test/support/register-loader.mjs --test --test-concurrency=1 --test-timeout=900000 \"test/perf/*.test.js\"",
26
27
  "test:t3": "node --import ./test/support/register-loader.mjs --test --test-concurrency=1 --test-timeout=240000 \"test/t3/*.test.js\""
27
28
  },
28
29
  "engines": {
@@ -1,10 +1,13 @@
1
+ import path from 'node:path';
1
2
  import { Flags } from '@oclif/core';
2
3
  import BaseCommand from '../../lib/BaseCommand.js';
3
4
  import { defaultConfigDirectory } from '../../lib/application.js';
5
+ import { FOLLOW_UNKNOWN_EXIT, followCutoverRequest, takesCutoverRequests, writeCutoverRequest } from '../../lib/cutoverRequest.js';
4
6
  import { createCutover } from '../../lib/cutoverWiring.js';
7
+ import { readOwnerOnlyJson } from '../../lib/storage.js';
5
8
 
6
9
  export default class PulseCutover extends BaseCommand {
7
- static description = 'Move this machine\'s local Agent Hub into the Pulse daemon (a short freeze, a backup kept), check first with --dry-run, or go back with --rollback (restores the backup; loses only what was written after the switch).';
10
+ static description = 'Move this machine\'s local Agent Hub into the Pulse daemon (a short freeze, a backup kept), check first with --dry-run, or go back with --rollback (restores the backup; loses only what was written after the switch). The running daemon does the work, so no launchctl or permission prompt is needed here; --in-process runs it in this process instead.';
8
11
  static examples = [
9
12
  '<%= config.bin %> pulse cutover --dry-run',
10
13
  '<%= config.bin %> pulse cutover --smoke-channel agentpulse-agent-hub/cf-phase3',
@@ -14,26 +17,47 @@ export default class PulseCutover extends BaseCommand {
14
17
  'dry-run': Flags.boolean({ description: 'Check everything and say what would change; change nothing', default: false }),
15
18
  rollback: Flags.boolean({ description: 'Back to the local hub from the backup', default: false, exclusive: ['dry-run'] }),
16
19
  'smoke-channel': Flags.string({ description: 'A task channel (<repo>/<task>) for the smoke test\'s one channel post' }),
20
+ 'in-process': Flags.boolean({ description: 'Run the cutover in this process, as Pulse 1.1.2 did, instead of asking the daemon (needs launchctl)', default: false }),
17
21
  };
18
22
 
19
23
  async run() {
20
24
  const { flags } = await this.parse(PulseCutover);
21
- const cutover = this.cutover || await createCutover({ configDir: defaultConfigDirectory(), log: (event, details) => this.log(`${event} ${JSON.stringify(details)}`) });
25
+ const configDir = this.configDir || defaultConfigDirectory();
26
+ const log = (event, details) => this.log(`${event} ${JSON.stringify(details)}`);
27
+ const inProcess = async () => this.cutover || createCutover({ configDir, log });
22
28
  if (flags['dry-run']) {
23
- const report = await cutover.dryRun();
29
+ const report = await (await inProcess()).dryRun();
24
30
  for (const check of report.checks) this.log(`${check.ok ? 'ok ' : 'FAIL'} ${check.name}${check.detail ? `: ${check.detail}` : ''}${!check.ok && check.fix ? ` (fix: ${check.fix})` : ''}`);
25
31
  this.log(`Outage while it runs: ${report.outage.what}; at most ${Math.round(report.outage.boundMs / 60_000)} min.`);
26
32
  this.log(report.message);
27
33
  if (!report.ok) process.exitCode = 1;
28
34
  return report;
29
35
  }
30
- if (flags.rollback) {
31
- const report = await cutover.rollback();
32
- this.log(report.message);
33
- return report;
36
+ const health = await readOwnerOnlyJson(path.join(configDir, 'daemon-health.json')).catch(() => null);
37
+ const viaDaemon = !flags['in-process'] && takesCutoverRequests(health);
38
+ if (!viaDaemon && !flags['in-process']) {
39
+ this.log(`The Pulse daemon ${health?.connectorVersion ? `(connector ${health.connectorVersion}) ` : ''}does not take cutover requests or is not running; running it in this process.`);
40
+ }
41
+ const mode = flags.rollback ? 'rollback' : 'cutover';
42
+ let result;
43
+ if (viaDaemon) {
44
+ const request = await writeCutoverRequest({ configDir, mode, smokeChannel: mode === 'cutover' ? flags['smoke-channel'] ?? null : null, health });
45
+ this.log(`Asked the Pulse daemon (pid ${health.pid}) to ${mode === 'rollback' ? 'roll back' : 'cut over'}: request ${request.id}.`);
46
+ try {
47
+ result = await followCutoverRequest({ configDir, request, log, ...this.follow });
48
+ } catch (error) {
49
+ if (error.exit === FOLLOW_UNKNOWN_EXIT) this.fail(error.code, error.message, error.exit);
50
+ throw error;
51
+ }
52
+ } else {
53
+ const cutover = await inProcess();
54
+ result = mode === 'rollback' ? await cutover.rollback() : await cutover.run({ smokeChannel: flags['smoke-channel'] ?? null });
55
+ }
56
+ if (mode === 'rollback') {
57
+ this.log(result.message);
58
+ return result;
34
59
  }
35
- const journal = await cutover.run({ smokeChannel: flags['smoke-channel'] ?? null });
36
- this.log(`Cut over: the Pulse daemon serves 127.0.0.1:${journal.port}. The backup is ${journal.backup.file}. Go back any time with \`agentpulse pulse cutover --rollback\`.`);
37
- return journal;
60
+ this.log(`Cut over: the Pulse daemon serves 127.0.0.1:${result.port}. The backup is ${result.backup.file}. Go back any time with \`agentpulse pulse cutover --rollback\`.`);
61
+ return result;
38
62
  }
39
63
  }
@@ -10,6 +10,7 @@ const MIRROR_FAILURES = new Set(['mirror_failed', 'mirror_blocked', 'mirror_stre
10
10
  const ERROR_EVENTS = new Set([
11
11
  'job_failed', 'job_refused', 'job_settle_failed', 'cycle_stalled', 'drain_timed_out', 'daemon_start_failed',
12
12
  'daemon_shutdown_failed', 'source_watch_refused', 'source_watch_failed',
13
+ 'cutover_failed', 'cutover_request_failed', 'cutover_request_refused', 'cutover_interrupted', 'cutover_service_failed',
13
14
  ]);
14
15
 
15
16
  /** One redacted line per job, local hub or restart event; failures go to daemon-error.log, the rest to daemon.log. */
@@ -3,6 +3,7 @@
3
3
  import fs from 'node:fs';
4
4
  import path from 'node:path';
5
5
  import { DatabaseSync } from 'node:sqlite';
6
+ import { isBusy } from './ticks.mjs';
6
7
 
7
8
  export const SCHEMA_VERSION = 1;
8
9
 
@@ -231,22 +232,37 @@ export function withTx(db, fn, label = null) {
231
232
  }
232
233
 
233
234
  const pause = (ms) => { if (ms > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); };
235
+ /** How long a busy chunk of writeInChunks waits before each new try; the wait blocks its thread, the collect worker's. */
236
+ const BUSY_PAUSES_MS = [250, 500, 1000, 2000];
234
237
 
235
238
  /**
236
239
  * Runs `steps` (functions) in as many short write transactions as it takes for none to hold the lock past `budgetMs`,
237
240
  * pausing `pauseMs` between them so writers on other connections get in. Inside an open transaction it simply joins it.
238
- * Returns how many transactions it used.
241
+ * A chunk that finds the database busy past busy_timeout is rolled back and run again from its first step, after
242
+ * `busyPausesMs`, so another writer's long transaction delays a pass instead of failing it. Returns how many
243
+ * transactions it used.
239
244
  */
240
- export function writeInChunks(db, steps, { budgetMs = 50, pauseMs = 15, label = 'chunked write', onPause = null } = {}) {
245
+ export function writeInChunks(db, steps, { budgetMs = 50, pauseMs = 15, label = 'chunked write', onPause = null, busyPausesMs = BUSY_PAUSES_MS } = {}) {
241
246
  if (TX.get(db)) { for (const step of steps) step(); return 0; }
242
247
  let i = 0;
243
248
  let chunks = 0;
249
+ let busy = 0;
244
250
  while (i < steps.length) {
245
251
  if (chunks) { pause(pauseMs); if (onPause) onPause(); }
246
252
  const started = performance.now();
247
- withTx(db, () => {
248
- do steps[i++](); while (i < steps.length && performance.now() - started < budgetMs);
249
- }, `${label} (chunk ${chunks + 1})`);
253
+ const first = i;
254
+ try {
255
+ withTx(db, () => {
256
+ do steps[i++](); while (i < steps.length && performance.now() - started < budgetMs);
257
+ }, `${label} (chunk ${chunks + 1})`);
258
+ } catch (e) {
259
+ if (!isBusy(e) || busy >= busyPausesMs.length) throw e;
260
+ txTiming.log(`${label}: database busy, chunk ${chunks + 1} again in ${busyPausesMs[busy]} ms`);
261
+ pause(busyPausesMs[busy++]);
262
+ i = first;
263
+ continue;
264
+ }
265
+ busy = 0;
250
266
  chunks++;
251
267
  }
252
268
  return chunks;
@@ -20,6 +20,10 @@ const MAX_OUTPUT = 2 * 1024 * 1024;
20
20
  /** Answers took 0.7-9.8 min in one chat resumed for two days; a fresh, small chat answers well inside this. */
21
21
  export const RUN_TIMEOUT_MS = 5 * 60e3;
22
22
  const MAX_ATTEMPTS = 3;
23
+ /** Claims also count a run's own lapsed claim taken back after a stall; past this many, a message no run finishes is given up. */
24
+ const MAX_CLAIMS = 10;
25
+ /** A run that ends while the lease is away (a stalled runner lapses it for a moment) waits this long for it to come back. */
26
+ export const HELD_WAIT_MS = 60e3;
23
27
  /** Runs cut off by restarts do not use up attempts; past this many restarts a message is given up, not looped on. */
24
28
  export const MAX_RESTARTS = 3;
25
29
  /** Runs at once, one chat each. One, as before: a second question waits one run at most, FYIs none. */
@@ -129,10 +133,19 @@ ${voice || ''}`.trim();
129
133
  const topicOf = (sender) => (sender === 'user' ? 'user' : `agent/${sender}`);
130
134
  const errorText = (error) => String(error && error.message ? error.message : error);
131
135
 
136
+ /** Why a cursor-agent run failed (exit, signal, stderr), never its command line: that holds the whole prompt. */
137
+ function runError(error) {
138
+ if (!error || !String(error.message || '').startsWith('Command failed:')) return errorText(error);
139
+ const how = error.killed ? `timed out (${error.signal})` : error.code != null ? `exit ${error.code}` : `signal ${error.signal}`;
140
+ const said = String(error.stderr || error.stdout || '').trim().split('\n').slice(-3).join(' ').slice(0, 300);
141
+ return `cursor-agent ${how}${said ? `: ${said}` : ''}`;
142
+ }
143
+
132
144
  export function createManager({
133
145
  hub, state, run = runFile, onChange = () => {}, parallel = PARALLEL, now = Date.now, log = console.error, ackRetryMs = RETRY_MS,
134
146
  held = () => true,
135
147
  fyi = null,
148
+ heldWaitMs = HELD_WAIT_MS,
136
149
  }) {
137
150
  const configFile = path.join(state, 'manager.json');
138
151
  const statusFile = path.join(state, 'manager-status.json');
@@ -143,6 +156,8 @@ export function createManager({
143
156
  const { messaging, registry, leases } = hub;
144
157
  /** message id → { message, leaseToken, slot, startedAt, abort } */
145
158
  const inflight = new Map();
159
+ /** message id → { count, error } of its runs that failed; claims alone are no attempts. */
160
+ const failures = new Map();
146
161
  const runs = new Set();
147
162
  let lastHeartbeat = 0;
148
163
  let timer = null;
@@ -222,6 +237,15 @@ export function createManager({
222
237
  }, 'manager answer'), { delaysMs: ackRetryMs, label: `manager answer to ${m.id}`, log });
223
238
  }
224
239
 
240
+ /** Only the lease holder settles; a stall that lapsed the lease here gets it back moments later. */
241
+ async function heldAgain(signal) {
242
+ for (let waited = 0; !held(); waited += 250) {
243
+ if (signal.aborted || waited >= heldWaitMs) return false;
244
+ await new Promise((resolve) => setTimeout(resolve, 250));
245
+ }
246
+ return !signal.aborted;
247
+ }
248
+
225
249
  async function answer(f) {
226
250
  const { message: m, slot, abort } = f;
227
251
  const chat = await chatFor(slot, abort.signal);
@@ -236,6 +260,10 @@ export function createManager({
236
260
  if (envelope && envelope.is_error) throw new Error(`cursor-agent failed: ${String(textOf(envelope) || 'error').slice(0, 300)}`);
237
261
  const actions = parseManagerActions(stdout);
238
262
  if (!actions.reply) throw new Error('manager returned no reply');
263
+ if (!(await heldAgain(abort.signal))) {
264
+ if (!abort.signal.aborted) log(`manager answer to ${m.id} held back: the manager lease is not held here`);
265
+ return null;
266
+ }
239
267
  await settle(m, actions);
240
268
  recordTurn(slot, chat.chatId, m, prompt, actions.reply);
241
269
  if (registry.fact('manager-restart', m.id)) registry.dropFact('manager-restart', m.id);
@@ -252,11 +280,16 @@ export function createManager({
252
280
  const done = answer(f).then((replied) => ({ replied }), (error) => ({ error })).then(({ replied, error }) => {
253
281
  inflight.delete(m.id);
254
282
  runs.delete(done);
255
- if (f.abort.signal.aborted) return;
256
- if (error) log(`manager failed for ${m.id}: ${errorText(error)}`);
283
+ if (f.abort.signal.aborted) { log(`manager run for ${m.id} was stopped before it finished; it is not counted as a try`); return; }
284
+ const why = error ? runError(error).slice(0, 500) : null;
285
+ if (error) {
286
+ const failed = { count: ((failures.get(m.id) || {}).count || 0) + 1, error: why };
287
+ failures.set(m.id, failed);
288
+ log(`manager failed for ${m.id} (try ${failed.count} of ${MAX_ATTEMPTS}): ${why}`);
289
+ } else if (replied) failures.delete(m.id);
257
290
  status({
258
291
  ...(replied || {}), state: error ? 'error' : inflight.size ? 'working' : 'online', messageId: error ? m.id : null,
259
- error: error ? errorText(error).slice(0, 500) : null, inflight: inflightView(),
292
+ error: why, inflight: inflightView(), ...(error ? { lastFailure: { id: m.id, from: m.from, error: why, at: now() } } : {}),
260
293
  });
261
294
  }).catch((e) => log(`manager status after ${m.id} failed: ${errorText(e)}`));
262
295
  runs.add(done);
@@ -315,9 +348,14 @@ export function createManager({
315
348
  const own = inflight.get(message.id);
316
349
  if (own) { own.leaseToken = message.leaseToken; continue; }
317
350
  if (isFyi(message)) { if (!(fyi && fyi(message))) messaging.ack('manager', message.id, {}, registry.directory()); continue; }
318
- if (message.attempts > MAX_ATTEMPTS) {
319
- const last = readJson(statusFile, {}).error || 'unknown error';
320
- messaging.ack('manager', message.id, { reply: `I could not answer this after ${MAX_ATTEMPTS} tries, so I stopped retrying. Last error: ${last}`, fromName: 'Manager' }, registry.directory());
351
+ const failed = failures.get(message.id);
352
+ if ((failed && failed.count >= MAX_ATTEMPTS) || message.attempts > MAX_CLAIMS) {
353
+ const tries = failed ? failed.count : message.attempts - 1;
354
+ const last = failed ? failed.error : `none of its ${tries} runs finished (each was cut off by a restart or a lost lease)`;
355
+ messaging.ack('manager', message.id, { reply: `I could not answer this after ${tries} tries, so I stopped retrying. Last error: ${last}`, fromName: 'Manager' }, registry.directory());
356
+ failures.delete(message.id);
357
+ log(`manager gave up on ${message.id} after ${tries} tries: ${last}`);
358
+ status({ gaveUp: { id: message.id, from: message.from, tries, error: last, at: t } });
321
359
  continue;
322
360
  }
323
361
  dispatch(message);
@@ -1,2 +1,2 @@
1
- ce1142e1e133c8aff45a999d0d4b67faa6177f420ddcd1b449af81f8e806d8d0 connector-contract.json
1
+ c9db92b38d0c978d4fa18d8a731fadbf3013a19659b524d615583ad099a8fafc connector-contract.json
2
2
  7b7a59c08797c100b46436123dfcb09b517730357fa90cd0890da9b80308faf3 mirror-contract.json
@@ -5,6 +5,7 @@ import { setTimeout as sleep } from 'node:timers/promises';
5
5
  import { clientState, installClients, restoreClients } from './clientInstaller.js';
6
6
  import { HubError } from './errors.js';
7
7
  import { RunnerStore, SEED_META } from './runnerStore.js';
8
+ import { SERVICE_LABEL } from './serviceManager.js';
8
9
  import { readOwnerOnlyJson, writeOwnerOnlyJson } from './storage.js';
9
10
 
10
11
  export const CUTOVER_FILE = 'cutover.json';
@@ -20,6 +21,9 @@ export const STEPS = Object.freeze(['sign_in_moved', 'shims_offline', 'hub_unloa
20
21
 
21
22
  const sha256File = async (file) => createHash('sha256').update(await readFile(file)).digest('hex');
22
23
 
24
+ /** What a fresh journal keeps from the last one: the daemon's request history (CUT-03). */
25
+ const carried = (previous) => Object.fromEntries(['requests', 'refusals'].filter((key) => previous?.[key]).map((key) => [key, previous[key]]));
26
+
23
27
  /** `version >= floor` on the numeric x.y.z core; anything unparsable is older. */
24
28
  function atLeast(version, floor) {
25
29
  const parse = (v) => String(v ?? '').match(/^(\d+)\.(\d+)\.(\d+)/)?.slice(1).map(Number) ?? null;
@@ -34,20 +38,23 @@ function atLeast(version, floor) {
34
38
  * a backup kept; `--rollback` restores that backup and the old hub. Every effect goes through an injected seam:
35
39
  * `services` (launchd, by label), `probe` (who holds hub.db), `cloudFirst` (Phase 2's final import and reconcile),
36
40
  * `daemon` (the facade's health and the old hub's), `smoke`, `backupDb`, `applySpool` and `signIn` (the TokenStore). Each step is journaled in
37
- * cutover.json before the next starts; a failure at any step after the freeze began rolls back by itself.
41
+ * cutover.json before the next starts; a failure at any step after the freeze began rolls back by itself. `serve` is
42
+ * called after each change of `cloudFirst.mode`: run inside the daemon (CUT-03) it starts or stops the daemon's own
43
+ * runner at once instead of on its next heartbeat.
38
44
  */
39
45
  export class Cutover {
40
46
  constructor({
41
- configDir, hubHome, port = 8890, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn,
47
+ configDir, hubHome, port = 8890, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn, serve = async () => {},
42
48
  labels = OLD_HUB_LABELS, holderWaitMs = 120_000, facadeWaitMs = 90_000, pollMs = 1_000, now = Date.now, wait = sleep, log = () => {},
43
49
  }) {
50
+ if (labels.includes(SERVICE_LABEL)) throw new HubError('HUB_CUTOVER_LABELS', `The cutover never unloads the Pulse daemon itself (${SERVICE_LABEL})`);
44
51
  this.configDir = configDir;
45
52
  this.configFile = path.join(configDir, 'config.json');
46
53
  this.journalFile = path.join(configDir, CUTOVER_FILE);
47
54
  this.hubHome = hubHome;
48
55
  this.hubDb = path.join(hubHome, 'state', 'hub.db');
49
56
  this.backupFile = `${this.hubDb}${BACKUP_SUFFIX}`;
50
- Object.assign(this, { port, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn, labels, holderWaitMs, facadeWaitMs, pollMs, now, wait, log });
57
+ Object.assign(this, { port, services, probe, cloudFirst, daemon, smoke, backupDb, applySpool, signIn, serve, labels, holderWaitMs, facadeWaitMs, pollMs, now, wait, log });
51
58
  }
52
59
 
53
60
  async journal() {
@@ -188,11 +195,17 @@ export class Cutover {
188
195
  };
189
196
  }
190
197
 
191
- /** The cutover. Resolves with the journal; a failure after the freeze began rolls back and rethrows. */
192
- async run({ smokeChannel = null } = {}) {
198
+ /**
199
+ * The cutover. Resolves with the journal; a failure after the freeze began rolls back and rethrows. `request` is the
200
+ * daemon's accepted cutover.request (CUT-03), kept on the journal so its CLI can follow it.
201
+ */
202
+ async run({ smokeChannel = null, request = null } = {}) {
193
203
  const preflight = await this.dryRun();
194
204
  if (!preflight.ok) throw new HubError('HUB_CUTOVER_NOT_READY', preflight.message, { details: { checks: preflight.checks.filter((c) => !c.ok) } });
195
- const journal = { state: 'running', startedAt: new Date(this.now()).toISOString(), hubHome: this.hubHome, port: this.port, steps: {}, loadedBefore: [] };
205
+ const journal = {
206
+ state: 'running', startedAt: new Date(this.now()).toISOString(), hubHome: this.hubHome, port: this.port, steps: {}, loadedBefore: [],
207
+ runBy: request ? 'daemon' : 'cli', pid: process.pid, ...carried(await this.journal()), ...(request ? { request } : {}),
208
+ };
196
209
  await writeOwnerOnlyJson(this.journalFile, journal);
197
210
  try {
198
211
  // Before the freeze: the sign-in moves to this config's own Keychain item (the 12:52 DECISION); no outage.
@@ -240,6 +253,7 @@ export class Cutover {
240
253
  cloudFirst: { ...(config.cloudFirst || {}), mode: 'runner', hubHome: this.hubHome, port: this.port, cutoverSeq, seedFrom: this.backupFile, seedSha256: journal.backup.sha256, shadow: false },
241
254
  });
242
255
  journal.cutoverSeq = cutoverSeq;
256
+ await this.serve();
243
257
  const serving = this.now() + this.facadeWaitMs;
244
258
  while (!(await this.daemon.facadeUp(this.port))) {
245
259
  if (this.now() >= serving) throw new HubError('HUB_CUTOVER_FACADE', `the Pulse daemon did not serve 127.0.0.1:${this.port} within ${this.facadeWaitMs} ms`);
@@ -271,27 +285,33 @@ export class Cutover {
271
285
  */
272
286
  async rollback({ reason = 'asked' } = {}) {
273
287
  const journal = await this.journal();
274
- const report = { command: 'cutover', rollback: true, reason, steps: [] };
275
- const step = (name, detail = {}) => { report.steps.push({ name, ...detail }); this.log('rollback_step', { name, ...detail }); };
288
+ const report = { command: 'cutover', rollback: true, reason, startedAt: new Date(this.now()).toISOString(), steps: [] };
289
+ // Each step is journaled as it ends, so a CLI following a daemon's rollback (CUT-03) prints it.
290
+ const step = async (name, detail = {}) => {
291
+ report.steps.push({ name, ...detail });
292
+ await writeOwnerOnlyJson(this.journalFile, { ...journal, rollback: report });
293
+ this.log('rollback_step', { name, ...detail });
294
+ };
276
295
  const config = await this.#config();
277
296
  if (config.cloudFirst?.mode === 'runner') {
278
297
  await writeOwnerOnlyJson(this.configFile, { ...config, cloudFirst: { ...config.cloudFirst, mode: 'rolled_back' } });
298
+ await this.serve();
279
299
  const deadline = this.now() + this.facadeWaitMs;
280
300
  while (await this.daemon.facadeUp(this.port)) {
281
301
  if (this.now() >= deadline) throw new HubError('HUB_ROLLBACK_FACADE', `the Pulse daemon still serves 127.0.0.1:${this.port}`);
282
302
  await this.wait(this.pollMs);
283
303
  }
284
- step('daemon_left_port');
304
+ await step('daemon_left_port');
285
305
  }
286
306
  if (journal.cutoverAt) {
287
307
  const lost = await this.#exportLost(journal.cutoverAt);
288
308
  report.lost = lost;
289
- step('lost_exported', lost);
309
+ await step('lost_exported', lost);
290
310
  }
291
311
  const marked = await this.#markSeedRolledBack();
292
- if (marked) step('runner_db_marked', marked);
312
+ if (marked) await step('runner_db_marked', marked);
293
313
  await restoreClients({ hubHome: this.hubHome });
294
- step('entry_points_restored');
314
+ await step('entry_points_restored');
295
315
  if (journal.steps?.backed_up && (await stat(this.backupFile).catch(() => null))) {
296
316
  for (const suffix of ['-wal', '-shm']) await rm(`${this.hubDb}${suffix}`, { force: true });
297
317
  const restored = `${this.hubDb}.pulse-restore`;
@@ -300,13 +320,15 @@ export class Cutover {
300
320
  await rm(this.hubDb, { force: true });
301
321
  await copyFile(restored, this.hubDb);
302
322
  await rm(restored, { force: true });
303
- step('hub_db_restored', { from: this.backupFile, sha256: await sha256File(this.backupFile) });
323
+ await step('hub_db_restored', { from: this.backupFile, sha256: await sha256File(this.backupFile) });
304
324
  }
305
325
  const spool = await this.applySpool(this.hubDb);
306
- step('spool_applied', spool);
326
+ await step('spool_applied', spool);
307
327
  const loadedBefore = journal.loadedBefore?.length ? journal.loadedBefore : this.labels;
328
+ // Journaled before the first load: from here the old hub may have hub.db open, so it is never restored again.
329
+ await step('hub_loading', { labels: loadedBefore });
308
330
  for (const label of loadedBefore) await this.services.restoreLabel(label);
309
- step('hub_loaded', { labels: loadedBefore });
331
+ await step('hub_loaded', { labels: loadedBefore });
310
332
  const deadline = this.now() + this.facadeWaitMs;
311
333
  while (!(await this.daemon.oldHubUp(this.port, { timeoutMs: ROLLBACK_PROBE_MS }))) {
312
334
  if (this.now() >= deadline) {
@@ -317,7 +339,7 @@ export class Cutover {
317
339
  }
318
340
  await this.wait(this.pollMs);
319
341
  }
320
- step('hub_serving');
342
+ await step('hub_serving');
321
343
  report.message = report.lost?.count
322
344
  ? `Rolled back to the local hub. ${report.lost.count} write(s) made after the switch (${Object.entries(report.lost.kinds).map(([kind, n]) => `${kind} ${n}`).join(', ')}) are not in hub.db; they are listed in ${report.lost.file}.`
323
345
  : 'Rolled back to the local hub. Nothing written after the switch was lost.';
@@ -325,6 +347,19 @@ export class Cutover {
325
347
  return report;
326
348
  }
327
349
 
350
+ /**
351
+ * CUT-03 boot recovery of a rollback that had begun loading the old hub: whether the old hub holds the port again
352
+ * (it answers, or it is hung but owned by its job), waiting up to facadeWaitMs. Reads only.
353
+ */
354
+ async verifyRolledBack() {
355
+ const deadline = this.now() + this.facadeWaitMs;
356
+ for (;;) {
357
+ const onPort = await this.#oldHubOnPort();
358
+ if (onPort.ok || this.now() >= deadline) return onPort;
359
+ await this.wait(this.pollMs);
360
+ }
361
+ }
362
+
328
363
  /**
329
364
  * After the runner left the port: the seed is marked rolled back, so the next cutover replaces it with its own
330
365
  * backup instead of serving this attempt's state (B1). Null when runner.db was never seeded.