@agentpulselabs/cli-plugin-pulse 0.2.0 → 1.0.0

package/src/lib/daemonLoop.js

@@ -0,0 +1,69 @@
+export const TERMINAL_STATES = new Set(['revoked', 'upgrade_required', 'auth_required', 'stopped']);
+const LOOP_RETRY_MS = 5_000;
+/** The heartbeat step that waits on the mirror; a pass that keeps reading or pushing keeps it alive. */
+export const MIRROR_UPDATE_STEP = 'mirror_update';
+
+/**
+ * One daemon heartbeat: re-read config.json, run a runtime cycle, then always hand the same fresh config to the mirror
+ * and publish health. A failed cycle must not leave the mirror on an old policy version, so the cycle's error becomes
+ * the heartbeat's state instead of skipping the rest. progress(step) names each step for the watchdog; extraHealth()
+ * adds fields such as the restart state to every health write.
+ */
+export function createHeartbeat({
+  readConfig, runtime, mirror, lock, ledger, ownerStore, progress = () => {}, extraHealth = () => ({}),
+}) {
+  let last = null;
+  const health = (config, status) => ({
+    state: status.state,
+    ...(status.code ? { code: status.code } : {}),
+    ...(status.message ? { message: status.message } : {}),
+    ...(status.accepts ? { accepts: status.accepts } : {}),
+    paired: Boolean(config?.paired),
+    protocolVersion: config?.protocolVersion ?? 1,
+    recovery: ledger.recover(),
+    ownedProcesses: Object.values(ownerStore.data.owners)
+      .filter((owner) => owner.state === 'running')
+      .map((owner) => ({
+        processRef: owner.processRef,
+        jobId: owner.jobId,
+        attemptNo: owner.attemptNo,
+        fenceToken: owner.fenceToken,
+      })),
+    recovering: runtime.recovering(),
+    settleFailed: ledger.settleFailed().slice(0, 5),
+    ...(runtime.lastJobError ? { lastJobError: runtime.lastJobError } : {}),
+    ...extraHealth(),
+  });
+  // Writes are chained and each builds its health when its turn comes, so a republish never overwrites a newer cycle.
+  let writing = Promise.resolve();
+  const write = (override = null) => {
+    writing = writing.catch(() => {}).then(async () => {
+      await lock.heartbeat(health(last?.config ?? null, { ...(last?.status ?? {}), ...override }));
+      heartbeat.writtenAt = Date.now();
+    });
+    return writing;
+  };
+  const heartbeat = async () => {
+    progress('config');
+    const config = await readConfig();
+    runtime.updateConfig(config);
+    let status;
+    try {
+      progress('cycle');
+      status = await runtime.runOnce();
+    } catch (error) {
+      status = { state: 'error', code: error.code || 'HUB_DAEMON_LOOP_FAILED', retryAfterMs: LOOP_RETRY_MS };
+    }
+    progress(MIRROR_UPDATE_STEP);
+    // A cloud that refused the contract would refuse every batch too, so the mirror waits for the next session.
+    await mirror.update(TERMINAL_STATES.has(status.state) || status.code === 'contract_unsupported' ? null : config);
+    progress('health');
+    last = { config, status };
+    await write();
+    return status;
+  };
+  /** Writes health again from the last cycle, with status fields overridden, while a cycle is still running. */
+  heartbeat.republish = write;
+  heartbeat.writtenAt = 0;
+  return heartbeat;
+}

package/src/lib/daemonProcess.js

@@ -0,0 +1,366 @@
+import path from 'node:path';
+import { createProductionDependencies } from './daemonComposition.js';
+import { DaemonLock } from './daemonLock.js';
+import { createHeartbeat, MIRROR_UPDATE_STEP, TERMINAL_STATES } from './daemonLoop.js';
+import { DaemonRuntime } from './daemonRuntime.js';
+import { assertNever, HubError } from './errors.js';
+import { DurableLedger } from './ledger.js';
+import { LocalHubReader } from './localHub.js';
+import { MIRROR_PATH } from './mirrorContract.js';
+import { MirrorController } from './mirrorDaemon.js';
+import { ProcessOwnerStore } from './processOwnership.js';
+import {
+  consumeRestartRequest,
+  RESTART_EXIT_CODE,
+  RestartGuard,
+  restartRequested,
+  Watchdog,
+  watchSource,
+} from './selfRestart.js';
+import { readOwnerOnlyJson } from './storage.js';
+
+// The longest a healthy cycle goes between two steps: a transport retry backoff (capped at 60 s) or a held claim (35 s).
+// A running job counts its lease renewals, every 10 s, as steps.
+const CYCLE_INTERVAL_MS = 60_000;
+// The mirror reconciles every 60 s, and a pass reads or pushes at least once per request timeout (30 s).
+const MIRROR_INTERVAL_MS = 60_000;
+const STALL_FACTOR = 3;
+const MIRROR_LOOPS = ['mirror', 'mirror windows'];
+/** The failure code of a job a restart had to abort: a drain that ran out of time, or a stalled loop. */
+export const RESTART_ABORT_CODE = 'HUB_DAEMON_RESTARTING';
+const DEFAULT_TIMING = Object.freeze({
+  cycleStallMs: STALL_FACTOR * CYCLE_INTERVAL_MS,
+  mirrorStallMs: STALL_FACTOR * MIRROR_INTERVAL_MS,
+  // An agent turn can run for many minutes; a code or request restart waits this long for it before aborting it.
+  drainMs: 15 * 60_000,
+  tickMs: 1_000,
+  healthEveryMs: 10_000,
+  watchDebounceMs: 2_000,
+  watchPollMs: 5_000,
+  watchRearmMs: 30_000,
+  shutdownMs: 20_000,
+  errorRetryMs: 5_000,
+});
+
+const inside = (child, parent) => {
+  const relative = path.relative(path.resolve(parent), path.resolve(child));
+  return relative === '' || (!relative.startsWith('..') && !path.isAbsolute(relative));
+};
+
+/**
+ * The Pulse daemon process. Besides the cycle and the mirror it restarts itself, by exiting RESTART_EXIT_CODE for
+ * launchd or systemd to start it again:
+ * - code: its plugin sources changed;
+ * - request: restart.request appeared in the config directory after this process booted;
+ * - stall: the cycle or a mirror pass made no progress for its limit;
+ * - start_failed: it could not start while it was holding restarts back.
+ * A code or request restart drains (no new claim; running jobs finish and settle) for up to drainMs; a stall aborts at
+ * once. Every trigger goes through one check that runs each tickMs. While the daemon has restarted too often it holds
+ * the restart (health says restart_loop) until the window clears. SIGTERM always wins and exits 0.
+ */
+export async function runDaemon({
+  directory,
+  pluginRoot,
+  logDir = null,
+  log = () => {},
+  mirrorLog = () => {},
+  exit = (code) => process.exit(code),
+  signals = process,
+  createDependencies = createProductionDependencies,
+  createMirror = (options) => new MirrorController(options),
+  dependencyOptions = {},
+  runtimeOptions = {},
+  lockOptions = {},
+  now = Date.now,
+  timing = {},
+}) {
+  const t = { ...DEFAULT_TIMING, ...timing };
+  const bootedAt = Date.now();
+  const lock = new DaemonLock({ directory, ...lockOptions });
+  await lock.acquire();
+  // The first cycle (token exchange, reconnect, claim) can outlast `pulse start`'s wait; say we are up before it.
+  await lock.heartbeat({ state: 'starting' });
+  let startingHealthAt = Date.now();
+  const guard = new RestartGuard({ directory, now });
+  await guard.recordStart();
+
+  let exiting = false;
+  let pending = null;
+  let restarting = null;
+  let deferralLogged = false;
+  let failure = null;
+  let heartbeat = null;
+  let runtime = null;
+  let stopServices = async () => {};
+  let timer = null;
+  let watcher = null;
+  let checking = false;
+  let finishing = null;
+  let signalled = false;
+
+  const degraded = () => {
+    const loop = guard.status();
+    if (!loop.looping) return null;
+    return {
+      code: 'restart_loop',
+      restarts: loop.restarts,
+      windowMs: loop.windowMs,
+      clearsAt: new Date(loop.clearsAt).toISOString(),
+      ...(pending ? { pending: pending.reason } : {}),
+    };
+  };
+
+  const finish = (code, { failCode = code, abortCode = null } = {}) => {
+    if (finishing) return finishing;
+    exiting = true;
+    clearTimeout(timer);
+    clearInterval(ticker);
+    watcher?.close();
+    let deadline;
+    const timedOut = new Promise((_, reject) => {
+      deadline = setTimeout(() => reject(new HubError('HUB_SHUTDOWN_TIMEOUT', 'Daemon shutdown did not finish')), t.shutdownMs);
+    });
+    finishing = Promise.race([stopServices({ abortCode }), timedOut])
+      .then(
+        () => code,
+        (error) => {
+          log('daemon_shutdown_failed', { code: error.code || 'HUB_SHUTDOWN_FAILED' });
+          return failCode;
+        }
+      )
+      .then(async (exitCode) => {
+        clearTimeout(deadline);
+        // A service stuck past shutdownMs dies with this process; the next one must not find the lock held.
+        await lock.release().catch((error) => log('daemon_lock_release_failed', { code: error.code || 'HUB_LOCK_RELEASE_FAILED' }));
+        exit(signalled ? 0 : exitCode);
+      });
+    return finishing;
+  };
+
+  const drainThenExit = async (reason) => {
+    restarting = { reason, since: new Date().toISOString() };
+    clearTimeout(timer);
+    let deadline;
+    const drained = runtime ? runtime.drain().then(() => true) : Promise.resolve(true);
+    const outOfTime = new Promise((resolve) => {
+      deadline = setTimeout(() => resolve(false), t.drainMs);
+    });
+    const finished = await Promise.race([drained, outOfTime]);
+    clearTimeout(deadline);
+    if (exiting) return;
+    if (!finished) log('drain_timed_out', { reason, draining: runtime?.activeJobIds() ?? [] });
+    await finish(RESTART_EXIT_CODE, finished ? {} : { abortCode: RESTART_ABORT_CODE });
+  };
+
+  const restart = (reason) => {
+    log('reload_requested', pending);
+    switch (reason) {
+      case 'code':
+      case 'request':
+        void drainThenExit(reason);
+        break;
+      case 'stall':
+      case 'start_failed':
+        void finish(RESTART_EXIT_CODE, { abortCode: RESTART_ABORT_CODE });
+        break;
+      default:
+        assertNever(reason, 'restart reason');
+    }
+  };
+
+  // A cycle writes health when it ends; a long job, a drain or a stall would leave it stale, so it is republished.
+  const publishHealth = async () => {
+    if (Date.now() - Math.max(heartbeat?.writtenAt ?? 0, startingHealthAt) < t.healthEveryMs) return;
+    if (heartbeat?.writtenAt) {
+      await heartbeat.republish(failure);
+      return;
+    }
+    startingHealthAt = Date.now();
+    await lock.heartbeat({ state: 'starting', ...failure, ...extraHealth() });
+  };
+
+  const check = async () => {
+    if (exiting || checking) return;
+    checking = true;
+    try {
+      watchdog.check();
+      await publishHealth();
+      if (restarting) return;
+      const loop = degraded();
+      if (loop) {
+        if (!pending && (await restartRequested(directory))) pending = { reason: 'request' };
+        if (pending && !deferralLogged) {
+          deferralLogged = true;
+          log('restart_deferred', { reason: pending.reason, restarts: loop.restarts, clearsAt: loop.clearsAt });
+        }
+        return;
+      }
+      deferralLogged = false;
+      const requestedAt = await consumeRestartRequest(directory);
+      if (requestedAt !== null && requestedAt < bootedAt) log('restart_request_satisfied', { requestedAt: new Date(requestedAt).toISOString() });
+      else if (requestedAt !== null) pending ??= { reason: 'request' };
+      if (pending) restart(pending.reason);
+    } catch (error) {
+      log('restart_check_failed', { code: error.code || 'HUB_RESTART_CHECK_FAILED' });
+    } finally {
+      checking = false;
+    }
+  };
+
+  const extraHealth = () => {
+    const loop = degraded();
+    return {
+      ...(loop ? { degraded: loop } : {}),
+      ...(restarting ? { restarting: { ...restarting, draining: runtime?.activeJobIds() ?? [] } } : {}),
+    };
+  };
+
+  const watchdog = new Watchdog({
+    now,
+    onStall: (stall) => {
+      log('cycle_stalled', stall);
+      failure = { state: 'stalled', code: 'cycle_stalled' };
+      if (restarting) {
+        void finish(RESTART_EXIT_CODE, { abortCode: RESTART_ABORT_CODE });
+        return;
+      }
+      pending ??= { reason: 'stall', loop: stall.loop, awaiting: stall.awaiting };
+    },
+  });
+  const ticker = setInterval(() => void check(), t.tickMs);
+  const onSignal = () => {
+    signalled = true;
+    return finish(0, { failCode: 1 });
+  };
+  signals.once('SIGTERM', onSignal);
+  signals.once('SIGINT', onSignal);
+  const handle = { stop: onSignal };
+
+  const refused = [['config', directory], ['logs', logDir]].find(([, dir]) => dir && inside(dir, pluginRoot));
+  if (refused) {
+    log('source_watch_refused', { reason: `${refused[0]}_inside_plugin` });
+  } else {
+    try {
+      watcher = await watchSource({
+        root: pluginRoot,
+        debounceMs: t.watchDebounceMs,
+        pollMs: t.watchPollMs,
+        rearmMs: t.watchRearmMs,
+        onError: (error) => log('source_watch_polling', { code: error.code || 'HUB_WATCH_FAILED' }),
+        onChange: (files) => {
+          pending ??= { reason: 'code', changed: files.length, files: files.slice(0, 5) };
+          void check();
+        },
+      });
+    } catch (error) {
+      log('source_watch_failed', { code: error.code || 'HUB_WATCH_FAILED' });
+    }
+  }
+
+  const cycleProgress = (step) => watchdog.progress('cycle', { step });
+  // A heartbeat waiting on the mirror is as alive as the mirror: a long pass that keeps reading or pushing is no stall.
+  // A windows pass runs beside the change passes (see MirrorService) and is watched as a loop of its own; reads and
+  // pushes are not told apart by lane, so they count as progress for both.
+  const mirrorProgress = (call) => {
+    for (const loop of MIRROR_LOOPS) watchdog.progress(loop, { call });
+    if (watchdog.loops.get('cycle')?.step === MIRROR_UPDATE_STEP) watchdog.progress('cycle', { call: `mirror ${call}` });
+  };
+  try {
+    const ledger = await DurableLedger.open(directory);
+    const ownerStore = await ProcessOwnerStore.open(directory);
+    const configFile = path.join(directory, 'config.json');
+    const initialConfig = await readOwnerOnlyJson(configFile);
+    const production = await createDependencies({
+      config: initialConfig,
+      configDir: directory,
+      log,
+      onTransportAttempt: ({ path: requestPath, phase }) => {
+        const call = `${phase} ${requestPath}`;
+        if (requestPath === MIRROR_PATH) mirrorProgress(call);
+        else watchdog.progress('cycle', { call });
+      },
+      ...dependencyOptions,
+    });
+    runtime = new DaemonRuntime({
+      config: initialConfig,
+      ledger,
+      ownerStore,
+      transportFactory: production.transportFactory,
+      handlersFactory: production.handlersFactory,
+      sessionResolver: production.sessionResolver,
+      settlementObserver: production.settlementObserver,
+      log,
+      progress: cycleProgress,
+      ...runtimeOptions,
+    });
+    const mirror = createMirror({
+      directory,
+      transportFactory: production.transportFactory,
+      tokenStore: production.tokenStore,
+      log: mirrorLog,
+      createReader: (options) => new LocalHubReader({ ...options, onRead: (pathname) => mirrorProgress(`read ${pathname}`) }),
+      serviceOptions: {
+        onPassStart: ({ lane } = {}) => {
+          const loop = lane === 'windows' ? 'mirror windows' : 'mirror';
+          const entry = watchdog.begin(loop, 'pass', t.mirrorStallMs);
+          mirrorProgress('pass start');
+          return { loop, entry };
+        },
+        onPassEnd: ({ loop, entry }) => {
+          mirrorProgress('pass end');
+          watchdog.end(loop, entry);
+        },
+      },
+    });
+    heartbeat = createHeartbeat({
+      readConfig: () => readOwnerOnlyJson(configFile),
+      runtime,
+      mirror,
+      lock,
+      ledger,
+      ownerStore,
+      progress: cycleProgress,
+      extraHealth,
+    });
+    stopServices = async ({ abortCode }) => {
+      const results = await Promise.allSettled([mirror.stop(), runtime.shutdown({ abortCode })]);
+      const failed = results.find((result) => result.status === 'rejected');
+      if (failed) throw failed.reason;
+    };
+  } catch (error) {
+    runtime = null;
+    const code = error.code || 'HUB_DAEMON_FAILED';
+    log('daemon_start_failed', { code, message: error.message });
+    if (!degraded()) {
+      await finish(1);
+      return handle;
+    }
+    failure = { state: 'error', code };
+    pending ??= { reason: 'start_failed', code };
+    void check();
+    return handle;
+  }
+
+  const schedule = async () => {
+    if (exiting || restarting) return;
+    const entry = watchdog.begin('cycle', 'start', t.cycleStallMs);
+    let wait;
+    try {
+      const status = await heartbeat();
+      if (TERMINAL_STATES.has(status.state)) return;
+      wait = status.retryAfterMs || 15_000;
+    } catch (error) {
+      // A cycle that ends after shutdown released the lock has no health left to report.
+      if (exiting) return;
+      await Promise.resolve()
+        .then(() => lock.heartbeat({ state: 'error', code: error.code || 'HUB_DAEMON_LOOP_FAILED' }))
+        .catch(() => {});
+      wait = t.errorRetryMs;
+    } finally {
+      watchdog.end('cycle', entry);
+    }
+    if (!exiting && !restarting) timer = setTimeout(() => void schedule(), wait);
+  };
+  void schedule();
+  return handle;
+}