@agentpulselabs/cli-plugin-pulse 0.1.0 → 1.0.0

package/src/lib/daemonRuntime.js

@@ -0,0 +1,820 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { fullJitterDelay } from './backoff.js';
+import { createClaudeDriver, createCursorDriver, DriverRegistry } from './drivers.js';
+import { assertNever, HubError } from './errors.js';
+import { HUB_JOB_TYPES, IntentRegistry } from './intents.js';
+import { AttemptCoordinator } from './lifecycle.js';
+import { CONTRACT_VERSION } from './mirrorContract.js';
+import { assertPairedConfig } from './pairedConfig.js';
+import { ProcessSupervisor } from './processSupervisor.js';
+import { redactSecretsDeep } from './redaction.js';
+
+const WRITE_JOBS = new Set(['worktree.create', 'agent.start', 'agent.turn', 'agent.stop', ...HUB_JOB_TYPES]);
+// `pulse hub-write` bumps the cloud grant version before this daemon re-reads config.json on its next cycle; until the
+// grace runs out a stale policy is a reconnect, not a stop.
+const POLICY_STALE_GRACE_MS = 60_000;
+const TRANSIENT_AUTH_CODES = new Set([
+  'HUB_REFRESH_UNAVAILABLE',
+  'HUB_REFRESH_LOCKED',
+  'HUB_TARGET_UNAVAILABLE',
+  'HUB_TRANSPORT_UNAVAILABLE',
+]);
+const AGENT_JOBS = new Set(['agent.start', 'agent.turn']);
+// Acks are idempotent, so a cloud 5xx, 429 or dropped connection is retried in place; the delays stay well inside the
+// 30 s lease, which the renew timer keeps alive meanwhile.
+const ACK_RETRY_DELAYS_MS = [1_000, 2_000, 4_000];
+
+function transientAckError(error) {
+  const status = error?.details?.status;
+  return error?.code === 'HUB_TRANSPORT_UNAVAILABLE' || status === 429 || status >= 500;
+}
+
+// A settlement or event upload the cloud refused for this machine as a whole, not for one job: it still stops the cycle.
+// HUB_TRANSPORT_UNAVAILABLE is left to the job: a cloud that keeps failing one job's calls must not stop the others, and
+// a real outage already fails the cycle's claim.
+function machineWideError(error) {
+  return (TRANSIENT_AUTH_CODES.has(error?.code) && error.code !== 'HUB_TRANSPORT_UNAVAILABLE')
+    || ['policy_stale', 'upgrade_required', 'HUB_LOGIN_REQUIRED', 'HUB_SCOPE_DENIED'].includes(error?.code)
+    || String(error?.code || '').startsWith('machine_');
+}
+
+// The cloud refused one job's settlement for good; the attempt is reconciled instead of retried.
+const RECONCILE_CODES = new Set(['stale_fence', 'idempotency_conflict', 'HUB_AGENT_SESSION_BINDING_MISMATCH', 'HUB_SETTLEMENT_ACK_MISMATCH']);
+// One job's settlement is retried with backoff for up to this many failures or this long (the cloud lease), then given
+// up locally so it never holds up claims; the cloud lapses the job into connector_no_result.
+const SETTLE_GIVE_UP_FAILURES = 5;
+const SETTLE_GIVE_UP_MS = 30_000;
+// The cloud allows 12 heartbeats and 6 claims a minute per machine (CONNECTOR_LIMITS); past them it answers 429 with a
+// 20 s Retry-After. An open session is reused, so these two set the steady call rate.
+const HEARTBEAT_INTERVAL_MS = 15_000;
+const CLAIM_SPACING_MS = 10_000;
+// A cloud that refuses this connector's contract version is asked again this late, or at the next daemon start.
+export const CONTRACT_RETRY_MS = 60 * 60 * 1_000;
+
+/** An event as uploaded: its payload through the contract's secret table, so a key in agent output never leaves. */
+function cloudEvent(event) {
+  return {
+    sourceEventId: event.sourceEventId,
+    sequence: event.sequence,
+    type: event.type,
+    payload: redactSecretsDeep(event.payload).value,
+  };
+}
+
+function attemptTuple(job) {
+  return {
+    jobId: job.id,
+    attemptNo: job.attemptNo ?? job.attempt_no,
+    fenceToken: job.fenceToken ?? job.fence_token,
+    mode: WRITE_JOBS.has(job.type) ? 'write' : 'read',
+    jobType: job.type,
+    idempotencyKey: job.idempotencyKey ?? job.idempotency_key,
+  };
+}
+
+/** Whether the cloud can take an acknowledgement under this attempt and fence. */
+function ackableTuple(tuple) {
+  return [tuple.attemptNo, tuple.fenceToken].every((value) => Number.isSafeInteger(value) && value >= 1);
+}
+
+export class DaemonRuntime {
+  constructor({
+    config,
+    ledger,
+    ownerStore,
+    transportFactory,
+    handlersFactory,
+    processSupervisor = null,
+    bootId = randomUUID(),
+    renewIntervalMs = 10_000,
+    sessionResolver = null,
+    settlementObserver = null,
+    random = Math.random,
+    now = Date.now,
+    log = () => {},
+    ackRetryDelaysMs = ACK_RETRY_DELAYS_MS,
+    delay = sleep,
+    progress = () => {},
+    heartbeatIntervalMs = HEARTBEAT_INTERVAL_MS,
+    claimSpacingMs = CLAIM_SPACING_MS,
+  }) {
+    this.progress = progress;
+    this.heartbeatIntervalMs = heartbeatIntervalMs;
+    this.claimSpacingMs = claimSpacingMs;
+    this.session = null;
+    this.lastHeartbeat = null;
+    this.log = log;
+    this.ackRetryDelaysMs = ackRetryDelaysMs;
+    this.delay = delay;
+    this.lastJobError = null;
+    this.settleFailures = new Map();
+    this.config = config;
+    this.random = random;
+    this.now = now;
+    this.policyStaleSince = null;
+    this.consecutiveFailures = 0;
+    this.ledger = ledger;
+    this.ownerStore = ownerStore;
+    this.transportFactory = transportFactory;
+    this.handlersFactory = handlersFactory;
+    this.processSupervisor =
+      processSupervisor || new ProcessSupervisor({ ownerStore });
+    this.bootId = bootId;
+    this.renewIntervalMs = renewIntervalMs;
+    this.sessionResolver = sessionResolver;
+    this.settlementObserver = settlementObserver;
+    this.state = 'created';
+    this.stopping = false;
+    this.draining = false;
+    this.abortCode = null;
+    this.running = null;
+    this.activeControllers = new Map();
+  }
+
+  async #flush(transport) {
+    const pending = await this.ledger.pendingEvents(100);
+    const groups = new Map();
+    for (const event of pending) {
+      const attempt = this.ledger.attempt(event.jobId);
+      if (!attempt) continue;
+      const key = `${attempt.jobId}\0${attempt.attemptNo}\0${attempt.fenceToken}`;
+      if (!groups.has(key)) groups.set(key, { attempt, events: [] });
+      groups.get(key).events.push(cloudEvent(event));
+    }
+    for (const { attempt, events } of groups.values()) {
+      let response;
+      try {
+        response = await transport.uploadEvents({
+          jobId: attempt.jobId,
+          attemptNo: attempt.attemptNo,
+          fenceToken: attempt.fenceToken,
+          events,
+        });
+      } catch (error) {
+        if (machineWideError(error)) throw error;
+        const trace = { jobId: attempt.jobId, type: attempt.jobType, attemptNo: attempt.attemptNo, code: error.code || 'HUB_EVENT_UPLOAD_FAILED' };
+        if (transientAckError(error)) {
+          this.log('events_upload_failed', trace);
+          continue;
+        }
+        const { discarded } = await this.ledger.discardEvents(attempt.jobId, trace.code);
+        this.log('events_discarded', { ...trace, discarded });
+        continue;
+      }
+      if (Array.isArray(response.events)) {
+        await this.ledger.acknowledgeEvents(
+          response.events.map((event) => ({
+            sourceEventId: event.sourceEventId,
+            cloudCursor: event.cloudCursor,
+          }))
+        );
+      }
+    }
+  }
+
+  #compose(grant) {
+    if (
+      !grant ||
+      !Number.isInteger(Number(grant.version)) ||
+      Number(grant.version) !== Number(this.config.paired?.policyVersion) ||
+      !Array.isArray(grant.capabilities) ||
+      !Array.isArray(grant.roots) ||
+      !grant.providerPolicy ||
+      !Number.isInteger(grant.providerPolicy.maxConcurrency) ||
+      grant.providerPolicy.maxConcurrency < 1 ||
+      grant.providerPolicy.maxConcurrency > 16 ||
+      !Number.isInteger(grant.providerPolicy.maxTokensPerRun) ||
+      grant.providerPolicy.maxTokensPerRun < 1 ||
+      grant.providerPolicy.maxTokensPerRun > 10_000_000 ||
+      !Number.isFinite(grant.providerPolicy.maxCostUsd) ||
+      grant.providerPolicy.maxCostUsd < 0 ||
+      grant.providerPolicy.maxCostUsd > 10_000
+    ) {
+      throw new HubError('HUB_POLICY_STALE', 'Signed cloud grant is missing or does not match local policy');
+    }
+    const localRoots = new Map(
+      (this.config.localRoots || []).map((root) => [root.rootId, root])
+    );
+    for (const root of grant.roots) {
+      const local = localRoots.get(root.rootId);
+      if (
+        !local ||
+        local.repoWrite !== root.repoWrite ||
+        local.grantCanonicalPathHash !== root.canonicalPathHash
+      ) {
+        throw new HubError(
+          'HUB_POLICY_STALE',
+          `Signed root ${root.rootId} does not match the local approval`
+        );
+      }
+    }
+    const providerPolicy = grant.providerPolicy || {};
+    const policy = {
+      version: Number(grant.version),
+      capabilities: grant.capabilities || [],
+      rootIds: (grant.roots || []).map((root) => root.rootId),
+      providers: providerPolicy.providers || {},
+      roleModes: {
+        architect: ['read'],
+        coder: ['write'],
+        reviewer: ['read'],
+        auditor: ['read'],
+      },
+      limits: {
+        maxConcurrency: providerPolicy.maxConcurrency,
+        maxTokensPerRun: providerPolicy.maxTokensPerRun,
+        maxCostUsd: providerPolicy.maxCostUsd,
+        maxWallMs: providerPolicy.maxWallMs || 20 * 60 * 1_000,
+        maxOutputBytes: providerPolicy.maxOutputBytes || 4 * 1024 * 1024,
+      },
+    };
+    const drivers = new DriverRegistry({
+      drivers: [
+        createCursorDriver({ supervisor: this.processSupervisor }),
+        createClaudeDriver({ supervisor: this.processSupervisor }),
+      ],
+      policy,
+    });
+    const coordinator = new AttemptCoordinator({
+      ledger: this.ledger,
+      supervisor: this.processSupervisor,
+    });
+    const handlers = this.handlersFactory?.({
+      grant,
+      policy,
+      drivers,
+      coordinator,
+      processSupervisor: this.processSupervisor,
+    });
+    if (!handlers) {
+      throw new HubError('HUB_HANDLER_REGISTRY_UNAVAILABLE', 'Local handler registry is unavailable');
+    }
+    return {
+      coordinator,
+      intents: new IntentRegistry({ handlers, policy }),
+    };
+  }
+
+  /** Log a job that could not run and keep it for the health file, by id, type and code only. */
+  #jobError(event, trace, code) {
+    this.log(event, { ...trace, code });
+    this.lastJobError = { jobId: trace.jobId, type: trace.type, code, at: new Date().toISOString() };
+  }
+
+  #terminalTransportState(error) {
+    if (['machine_revoked', 'machine_unknown', 'machine_forbidden'].includes(error.code)) return 'revoked';
+    if (['policy_stale', 'upgrade_required'].includes(error.code)) return 'upgrade_required';
+    if (['HUB_LOGIN_REQUIRED', 'HUB_SCOPE_DENIED', 'HUB_TARGET_CHANGED', 'bearer_required', 'caller_unknown']
+      .includes(error.code)) {
+      return 'auth_required';
+    }
+    return null;
+  }
+
+  /** Settle a claim the ledger refused as failed under its own attempt, so the cloud records why instead of expiring it. */
+  async #refuseClaim(tuple, code, transport) {
+    const errorSummary = 'Pulse refused this attempt before running it.';
+    const response = await this.#acknowledge(transport, tuple.jobId, {
+      attemptNo: tuple.attemptNo,
+      fenceToken: tuple.fenceToken,
+      state: 'failed',
+      resultDigest: createHash('sha256').update(JSON.stringify({ code, summary: errorSummary })).digest('hex'),
+      idempotencyKey: createHash('sha256').update(`${tuple.idempotencyKey}\0refused\0${tuple.attemptNo}`).digest('hex'),
+      errorCode: code,
+      errorSummary,
+    });
+    this.log('job_acked', { jobId: tuple.jobId, type: tuple.jobType, attemptNo: tuple.attemptNo, state: response?.job?.state ?? 'failed' });
+  }
+
+  /** One acknowledgement, retried on transient failures; anything else, or the last failure, is thrown. */
+  async #acknowledge(transport, jobId, payload) {
+    for (let retry = 0; ; retry += 1) {
+      try {
+        return await transport.acknowledge(jobId, payload);
+      } catch (error) {
+        if (!transientAckError(error) || retry >= this.ackRetryDelaysMs.length) throw error;
+        this.log('ack_retry', { jobId, state: payload.state, retry: retry + 1, code: error.code || 'HUB_ACK_FAILED' });
+        await this.delay(this.ackRetryDelaysMs[retry]);
+      }
+    }
+  }
+
+  #runningIdempotency(value) {
+    return createHash('sha256').update(`${value}\0running`).digest('hex');
+  }
+
+  async #settleAttempt(attempt, transport) {
+    if (attempt.cloudSettled || attempt.state === 'fenced') return;
+    if (!attempt.settlement?.idempotencyKey) {
+      throw new HubError('HUB_SETTLEMENT_INVALID', 'Durable terminal settlement envelope is missing');
+    }
+    const response = await this.#acknowledge(
+      transport,
+      attempt.jobId,
+      structuredClone(attempt.settlement)
+    );
+    const acknowledged = response?.job;
+    if (
+      acknowledged?.id !== attempt.jobId ||
+      acknowledged?.state !== attempt.state ||
+      (acknowledged.attempt_no !== undefined &&
+        Number(acknowledged.attempt_no) !== attempt.attemptNo) ||
+      (acknowledged.fence_token !== undefined &&
+        Number(acknowledged.fence_token) !== attempt.fenceToken) ||
+      (acknowledged.result_digest !== undefined &&
+        acknowledged.result_digest !== attempt.resultDigest)
+    ) {
+      throw new HubError('HUB_SETTLEMENT_ACK_MISMATCH', 'Cloud settlement acknowledgement does not match');
+    }
+    this.log('job_acked', { jobId: attempt.jobId, type: attempt.jobType, attemptNo: attempt.attemptNo, state: attempt.state });
+    await this.settlementObserver?.(attempt, response);
+    await this.ledger.markSettled(attempt, attempt.resultDigest);
+  }
+
+  /**
+   * Settle one terminal attempt without letting it stop the cycle. A refusal for good reconciles it; any other job
+   * failure is retried on later cycles with backoff, then given up as settle_failed. Machine-wide errors still throw.
+   */
+  async #settleIsolated(attempt, transport, trace = { jobId: attempt.jobId, type: attempt.jobType, attemptNo: attempt.attemptNo }) {
+    if (attempt.settleFailed || attempt.cloudSettled) return;
+    const pending = this.settleFailures.get(attempt.jobId);
+    if (pending && pending.retryAt > this.now()) return;
+    try {
+      await this.#settleAttempt(attempt, transport);
+      this.settleFailures.delete(attempt.jobId);
+    } catch (error) {
+      if (machineWideError(error)) throw error;
+      if (RECONCILE_CODES.has(error.code)) {
+        this.settleFailures.delete(attempt.jobId);
+        await this.settlementObserver?.(attempt, null, error);
+        await this.ledger.markReconciled(attempt, error.code);
+        return;
+      }
+      await this.#noteSettleFailure(attempt, error.code || 'HUB_SETTLEMENT_FAILED', trace);
+    }
+  }
+
+  async #noteSettleFailure(attempt, code, trace) {
+    const entry = this.settleFailures.get(attempt.jobId) || { failures: 0, since: this.now() };
+    entry.failures += 1;
+    entry.code = code;
+    entry.retryAt = this.now() + Math.min(SETTLE_GIVE_UP_MS, 1_000 * 2 ** (entry.failures - 1));
+    this.settleFailures.set(attempt.jobId, entry);
+    this.#jobError('job_settle_failed', { ...trace, state: attempt.state, failures: entry.failures }, code);
+    if (entry.failures < SETTLE_GIVE_UP_FAILURES && this.now() - entry.since < SETTLE_GIVE_UP_MS) return;
+    this.settleFailures.delete(attempt.jobId);
+    await this.ledger.markSettleFailed(attempt, code);
+    this.log('job_settle_abandoned', { ...trace, state: attempt.state, failures: entry.failures, code });
+  }
+
+  /** Jobs whose settlement is being retried, for the health file. */
+  recovering() {
+    return [...this.settleFailures].map(([jobId, entry]) => ({ jobId, code: entry.code, failures: entry.failures }));
+  }
+
+  async #applyReconnect(opened, composition, transport) {
+    for (const remote of opened.attempts || []) {
+      const local = this.ledger.attempt(remote.jobId);
+      if (!local) continue;
+      const tuple = {
+        jobId: local.jobId,
+        attemptNo: local.attemptNo,
+        fenceToken: local.fenceToken,
+      };
+      switch (remote.disposition) {
+        case 'continue':
+          break;
+        case 'cancel':
+          await composition.coordinator.cancel(tuple);
+          await this.#settleIsolated(this.ledger.attempt(local.jobId), transport);
+          break;
+        case 'fenced':
+          if (
+            ['cancelled', 'succeeded', 'failed'].includes(local.state) &&
+            !local.cloudSettled
+          ) {
+            await this.#settleIsolated(local, transport);
+          } else if (!['cancelled', 'succeeded', 'failed', 'fenced'].includes(local.state)) {
+            await composition.coordinator.fence(tuple);
+          }
+          break;
+        case 'upload_terminal':
+          await this.#settleIsolated(local, transport);
+          break;
+        default:
+          assertNever(remote.disposition, 'reconnect disposition');
+      }
+    }
+    for (const cancellation of opened.desiredCancellations || []) {
+      const jobId = typeof cancellation === 'string' ? cancellation : cancellation.jobId;
+      const local = this.ledger.attempt(jobId);
+      if (!local || ['cancelled', 'succeeded', 'failed', 'fenced'].includes(local.state)) continue;
+      await composition.coordinator.cancel({
+        jobId: local.jobId,
+        attemptNo: local.attemptNo,
+        fenceToken: local.fenceToken,
+      });
+      await this.#settleIsolated(this.ledger.attempt(jobId), transport);
+    }
+  }
+
+  async #runJob(job, composition, transport) {
+    const tuple = attemptTuple(job);
+    const trace = { jobId: tuple.jobId, type: job.type, attemptNo: tuple.attemptNo };
+    let claimed;
+    try {
+      claimed = await composition.coordinator.claim(tuple);
+    } catch (error) {
+      const code = error.code || 'HUB_JOB_FAILED';
+      this.#jobError('job_refused', trace, code);
+      if (!ackableTuple(tuple)) throw error;
+      await this.#refuseClaim(tuple, code, transport);
+      return;
+    }
+    this.log('job_claimed', { ...trace, ...(claimed?.prior ? { priorState: claimed.prior.state } : {}) });
+    const controller = new AbortController();
+    this.activeControllers.set(job.id, controller);
+    let started = false;
+    let renewTimer;
+    let renewInFlight = false;
+    let renewError = null;
+    let cancellationRequested = false;
+    let deadlineExpired = false;
+    let deadlineTimer;
+    const renew = async () => {
+      if (renewInFlight || renewError) return;
+      renewInFlight = true;
+      try {
+        await transport.renew(job.id, {
+          attemptNo: tuple.attemptNo,
+          fenceToken: tuple.fenceToken,
+        });
+        const heartbeat = await transport.heartbeat({
+          cloudEventCursor: this.ledger.snapshot().cloudCursor,
+          localEventCursor: this.ledger.snapshot().acceptedLocalCursor,
+          activeAttempts: [{
+            jobId: tuple.jobId,
+            attemptNo: tuple.attemptNo,
+            fenceToken: tuple.fenceToken,
+            state: 'running',
+          }],
+        });
+        this.progress(`job ${job.type} renewed`);
+        if (
+          (heartbeat.cancellations || []).some((entry) => {
+            const jobId = typeof entry === 'string' ? entry : entry.jobId ?? entry.id;
+            const attemptNo = entry.attemptNo ?? entry.attempt_no;
+            const fenceToken = entry.fenceToken ?? entry.fence_token;
+            return (
+              jobId === tuple.jobId &&
+              (attemptNo === undefined || Number(attemptNo) === tuple.attemptNo) &&
+              (fenceToken === undefined || Number(fenceToken) === tuple.fenceToken)
+            );
+          })
+        ) {
+          cancellationRequested = true;
+          controller.abort();
+        }
+      } catch (error) {
+        renewError = error;
+        controller.abort();
+      } finally {
+        renewInFlight = false;
+      }
+    };
+    const markStarted = async (processIdentity) => {
+      if (started) throw new HubError('HUB_PROCESS_IDENTITY_IMMUTABLE', 'Attempt already started');
+      started = true;
+      await composition.coordinator.started({ ...tuple, processIdentity });
+      this.log('job_started', trace);
+      await this.#acknowledge(transport, job.id, {
+        attemptNo: tuple.attemptNo,
+        fenceToken: tuple.fenceToken,
+        state: 'running',
+        idempotencyKey: this.#runningIdempotency(tuple.idempotencyKey),
+      });
+      renewTimer = setInterval(() => void renew(), this.renewIntervalMs);
+      renewTimer.unref?.();
+    };
+    try {
+      const sessionBinding = this.sessionResolver?.(job) || null;
+      const context = {
+        sessionProvider: sessionBinding?.provider ?? null,
+        sessionBinding,
+        onProcessStart: markStarted,
+        attempt: tuple,
+        priorAttempt: claimed?.prior ? structuredClone(claimed.prior) : null,
+        cloudJob: structuredClone(job),
+        signal: controller.signal,
+      };
+      const deadline = job.deadlineAt ?? job.deadline_at;
+      if (deadline && Date.parse(deadline) <= Date.now()) {
+        throw new HubError('HUB_JOB_DEADLINE_EXPIRED', 'Job deadline has expired');
+      }
+      if (deadline) {
+        const remaining = Date.parse(deadline) - Date.now();
+        deadlineTimer = setTimeout(() => {
+          deadlineExpired = true;
+          controller.abort();
+        }, remaining);
+        deadlineTimer.unref?.();
+      }
+      if (!AGENT_JOBS.has(job.type)) {
+        await markStarted(undefined);
+      }
+      const result = await composition.intents.execute(
+        {
+          type: job.type,
+          capability: job.capability,
+          rootId: job.rootId ?? job.root_id,
+          arguments: job.arguments,
+          policyVersion: job.policyVersion ?? job.policy_version,
+          idempotencyKey: job.idempotencyKey ?? job.idempotency_key,
+        },
+        context
+      );
+      if (AGENT_JOBS.has(job.type) && !started) {
+        throw new HubError(
+          'HUB_PROCESS_OWNERSHIP_REQUIRED',
+          'Agent handler returned without recording durable process ownership'
+        );
+      }
+      if (renewError) throw renewError;
+      const terminal = await composition.coordinator.complete({ ...tuple, result });
+      this.log('job_succeeded', { ...trace, ...(typeof result?.outcome === 'string' ? { outcome: result.outcome } : {}) });
+      await this.#settleIsolated(terminal, transport, trace);
+    } catch (error) {
+      const restartAbort = controller.signal.aborted && !cancellationRequested && !deadlineExpired && !renewError ? this.abortCode : null;
+      this.#jobError('job_failed', trace, deadlineExpired ? 'HUB_JOB_DEADLINE_EXPIRED' : renewError?.code || restartAbort || error.code || 'HUB_JOB_FAILED');
+      const current = this.ledger.attempt(job.id);
+      if (current && !['cancelled', 'succeeded', 'failed', 'fenced'].includes(current.state)) {
+        if (cancellationRequested) {
+          const terminal = await composition.coordinator.cancel(tuple);
+          await this.#settleIsolated(terminal, transport, trace);
+          return;
+        }
+        if (
+          renewError &&
+          ['stale_fence', 'policy_stale', 'machine_revoked', 'machine_unknown', 'machine_forbidden'].includes(
+            renewError.code
+          )
+        ) {
+          await composition.coordinator.fence(tuple);
+          return;
+        }
+        const terminal = await composition.coordinator.fail({
+          ...tuple,
+          code: deadlineExpired
+            ? 'HUB_JOB_DEADLINE_EXPIRED'
+            : renewError?.code || restartAbort || error.code || 'HUB_JOB_FAILED',
+          summary: deadlineExpired
+            ? 'Job deadline expired during execution.'
+            : renewError?.message || (restartAbort ? 'The Pulse daemon restarted before the job finished.' : error.message),
+        });
+        await this.#settleIsolated(terminal, transport, trace);
+      }
+    } finally {
+      clearInterval(renewTimer);
+      clearTimeout(deadlineTimer);
+      this.activeControllers.delete(job.id);
+    }
+  }
+
+  /**
+   * The cloud session lasts until another session opens for this machine or it is revoked; it does not time out. So it
+   * is opened once and reused, and reopened when the transport or policy version changes or after any failed cycle
+   * (a 409 session_invalid or policy_stale, an auth failure, an outage), which drops it in #runCycle.
+   */
+  async #exchange(transport) {
+    const policyVersion = this.config.paired.policyVersion;
+    const reuse = this.session?.transport === transport && this.session.policyVersion === policyVersion;
+    let grant;
+    if (reuse) {
+      ({ grant } = this.session);
+      await this.#heartbeat(transport);
+      this.progress('recovery');
+      await this.#retrySettlements(transport);
+    } else {
+      this.session = null;
+      const opened = await this.#openSession(transport);
+      await this.#heartbeat(transport, { force: true });
+      ({ grant } = opened);
+      const composition = this.#compose(grant);
+      this.progress('recovery');
+      await this.#applyReconnect(opened, composition, transport);
+      this.session = { transport, policyVersion, grant };
+    }
+    const composition = this.#compose(grant);
+    this.progress('events');
+    await this.#flush(transport);
+    if (this.stopping) return { state: 'stopping', processed: 0 };
+    if (this.draining) return { state: 'draining', processed: 0 };
+    this.progress('claim');
+    const claimedAt = this.now();
+    const claimed = await transport.poll({
+      waitSeconds: 25,
+      maxJobs: 1,
+      cloudCursor: this.ledger.snapshot().cloudCursor,
+      localCursor: this.ledger.snapshot().acceptedLocalCursor,
+      activeAttempts: [],
+    });
+    const jobs = claimed.jobs || [];
+    for (const job of jobs) {
+      if (this.stopping) break;
+      this.progress(`job ${job.type}`);
+      await this.#runJob(job, composition, transport);
+    }
+    this.progress('events');
+    await this.#flush(transport);
+    this.state = 'running';
+    const cloudWait = Number(claimed.retryAfterMs) || 1_000;
+    const budgetWait = jobs.length ? 0 : this.claimSpacingMs - (this.now() - claimedAt);
+    return {
+      state: this.state,
+      processed: jobs.length,
+      retryAfterMs: Math.max(250, Math.min(60_000, Math.max(cloudWait, budgetWait))),
+    };
+  }
+
+  #openSession(transport) {
+    this.progress('session');
+    return transport.reconnect({
+      bootId: this.bootId,
+      policyVersion: this.config.paired.policyVersion,
+      capabilities: this.config.capabilities || [],
+      cloudEventCursor: this.ledger.snapshot().cloudCursor,
+      localEventCursor: this.ledger.snapshot().acceptedLocalCursor,
+      activeAttempts: Object.values(this.ledger.snapshot().attempts)
+        .filter(
+          (attempt) =>
+            attempt.state !== 'fenced' &&
+            !attempt.settleFailed &&
+            (!['cancelled', 'succeeded', 'failed'].includes(attempt.state) ||
+              !attempt.cloudSettled ||
+              !attempt.terminalUploaded)
+        )
+        .map((attempt) => ({
+          jobId: attempt.jobId,
+          attemptNo: attempt.attemptNo,
+          fenceToken: attempt.fenceToken,
+          state: attempt.state === 'cancel_requested' ? 'running' : attempt.state,
+        })),
+    });
+  }
+
+  /** Sent when the session opens, when the running attempts change, and otherwise once per heartbeatIntervalMs. */
+  async #heartbeat(transport, { force = false } = {}) {
+    const snapshot = this.ledger.snapshot();
+    const activeAttempts = Object.values(snapshot.attempts)
+      .filter((attempt) => ['leased', 'running', 'cancel_requested'].includes(attempt.state))
+      .map((attempt) => ({
+        jobId: attempt.jobId,
+        attemptNo: attempt.attemptNo,
+        fenceToken: attempt.fenceToken,
+        state: attempt.state === 'cancel_requested' ? 'cancel_acknowledged' : attempt.state,
+      }));
+    const key = JSON.stringify(activeAttempts);
+    const due = !this.lastHeartbeat || this.now() - this.lastHeartbeat.at >= this.heartbeatIntervalMs;
+    if (!force && !due && this.lastHeartbeat.key === key) return;
+    this.progress('heartbeat');
+    await transport.heartbeat({
+      cloudEventCursor: snapshot.cloudCursor,
+      localEventCursor: snapshot.acceptedLocalCursor,
+      activeAttempts,
+    });
+    this.lastHeartbeat = { at: this.now(), key };
+  }
+
+  /** On a reused session no reconnect hands back upload_terminal, so settlements that failed are retried from here. */
+  async #retrySettlements(transport) {
+    for (const jobId of [...this.settleFailures.keys()]) {
+      const attempt = this.ledger.attempt(jobId);
+      if (attempt) await this.#settleIsolated(attempt, transport);
+    }
+  }
+
+  async #runCycle() {
+    if (!this.config.paired) {
+      this.state = 'idle_unpaired';
+      return { state: this.state, processed: 0 };
+    }
+    assertPairedConfig(this.config);
+    if (!this.transportFactory || !this.handlersFactory) {
+      throw new HubError('HUB_DAEMON_COMPOSITION_INCOMPLETE', 'Daemon dependencies are incomplete');
+    }
+    try {
+      const status = await this.#exchange(await this.transportFactory(this.config));
+      this.consecutiveFailures = 0;
+      this.policyStaleSince = null;
+      return status;
+    } catch (error) {
+      this.session = null;
+      if (error.code === 'contract_unsupported') {
+        this.state = 'error';
+        const accepts = error.details?.accepts ?? null;
+        return {
+          state: this.state,
+          processed: 0,
+          code: error.code,
+          accepts,
+          message: `Pulse accepts ${accepts ?? 'another mirror contract'} and this connector speaks mirror/${CONTRACT_VERSION}. `
+            + 'Update the connector, or once Pulse is updated run `agentpulse pulse restart`; it asks again within an hour.',
+          retryAfterMs: CONTRACT_RETRY_MS,
+        };
+      }
+      if (error.code === 'policy_stale') {
+        this.policyStaleSince ??= this.now();
+        if (this.now() - this.policyStaleSince < POLICY_STALE_GRACE_MS) {
+          this.state = 'reconnecting';
+          return { state: this.state, processed: 0, code: error.code, retryAfterMs: 2_000 };
+        }
+      }
+      if (TRANSIENT_AUTH_CODES.has(error.code)) {
+        this.consecutiveFailures += 1;
+        this.state = 'reconnecting';
+        return {
+          state: this.state,
+          processed: 0,
+          code: error.code,
+          retryAfterMs: 1_000 + fullJitterDelay(this.consecutiveFailures - 1, { random: this.random }),
+        };
+      }
+      const terminalState = this.#terminalTransportState(error);
+      if (!terminalState) throw error;
+      this.state = terminalState;
+      this.stopping = true;
+      return { state: this.state, processed: 0, code: error.code };
+    }
+  }
+
+  runOnce() {
+    if (this.running) return this.running;
+    this.running = this.#runCycle().finally(() => {
+      this.running = null;
+    });
+    return this.running;
+  }
+
+  updateConfig(config) {
+    if (!config || typeof config !== 'object') {
+      throw new HubError('HUB_INVALID_CONFIG', 'Daemon configuration is unavailable');
+    }
+    this.config = config;
+  }
+
+  stop() {
+    this.stopping = true;
+    this.state = 'stopping';
+  }
+
+  /**
+   * Finish in-flight work without taking more: no new claim starts, but the job running now and any job an in-flight
+   * claim returns still run and settle. Resolves when the current cycle is over.
+   */
+  async drain() {
+    this.draining = true;
+    await this.running?.catch(() => {});
+  }
+
+  activeJobIds() {
+    return [...this.activeControllers.keys()];
+  }
+
+  /** abortCode is the failure code an aborted job settles with, so a restart is told apart from a job's own failure. */
+  async shutdown({ abortCode = null } = {}) {
+    this.abortCode = abortCode;
+    this.stop();
+    for (const controller of this.activeControllers.values()) controller.abort();
+    if (this.running) {
+      let timer;
+      try {
+        await Promise.race([
+          this.running,
+          new Promise((_, reject) => {
+            timer = setTimeout(
+              () => reject(new HubError('HUB_SHUTDOWN_TIMEOUT', 'Active runtime cycle did not stop')),
+              15_000
+            );
+          }),
+        ]);
+      } finally {
+        clearTimeout(timer);
+      }
+    }
+    const running = Object.values(this.ownerStore.data.owners).filter(
+      (owner) => owner.state === 'running'
+    );
+    for (const identity of running) {
+      const outcome = await this.processSupervisor.cancel(identity);
+      if (!outcome.verifiedStopped) {
+        throw new HubError(
+          'HUB_CANCELLATION_UNPROVEN',
+          `Could not prove process ${identity.processRef} stopped during shutdown`
+        );
+      }
+    }
+    this.state = 'stopped';
+    return { state: this.state, stoppedProcesses: running.length };
+  }
+}