@serve.zone/interfaces 32.68.0 → 32.70.0

package/ts/data/nodeaction.ts

@@ -0,0 +1,755 @@
+import { canonicalizeStrictJson, compareWireTokens, createCanonicalJsonSha256Hex } from '../private/canonicaljson.js';
+import { readDataElements, readDataProperties, readExactDataRecord } from '../private/dataproperties.js';
+import { isClusterNodeId } from '../private/nodeidentifier.js';
+import { isCanonicalTokenList } from '../private/tokenlist.js';
+import { protocolRefusalHttpStatus, readProtocolOffer, type IProtocolOffer } from '../protocol.js';
+import type { TNodeActionName, TNodeActionStatus } from './clusternode.js';
+import { isReleaseVersion } from './releaseversion.js';
+
+/**
+ * Every node action, in canonical token order, so a list of them (a Spark's `actionExecutors`)
+ * is compared byte for byte.
+ */
+export const nodeActionNames = Object.freeze([
+  'reboot',
+  'serveZoneServiceUpdate',
+  'systemUpgrade',
+] as const satisfies readonly TNodeActionName[]);
+
+/** Fails to compile when `nodeActionNames` leaves out a `TNodeActionName`. */
+type TNodeActionNamesComplete = Exclude<TNodeActionName, typeof nodeActionNames[number]> extends never ? true : never;
+const nodeActionNamesComplete: TNodeActionNamesComplete = true;
+void nodeActionNamesComplete;
+
+export const isNodeActionName = (valueArg: unknown): valueArg is TNodeActionName => (
+  nodeActionNames.some((nameArg) => nameArg === valueArg)
+);
+
+/**
+ * Whether each action state is still open or ended.
+ *
+ * The four values the Swarm-era carrier shipped keep their meaning: `pending` is the requested
+ * state of every action, `running` is a Swarm-era action its daemon started, and `succeeded` and
+ * `failed` end one. The node runtime's lifecycle (`INodeActionLifecycle`) adds the rest:
+ *
+ * - `claimed`: the node's Spark took the action (`claim`), stating the boot it took it in.
+ * - `executing`, `draining`, `rebooting`: the phase Spark reports, only ever forward
+ *   (`sparkNodeActionPhaseOrder`).
+ * - `verifying`: Cloudly saw the evidence the action changed the node (`nodeActionVerification`)
+ *   and waits for the node's runtime to come back.
+ * - `expired`: never claimed in its claim window, or a Swarm-era record no carrier can deliver
+ *   (`TNodeActionExpiryReason`).
+ * - `cancelled`: an operator withdrew it while it was still `pending`.
+ * - `unverified`: its deadline passed before Cloudly could verify it. An operator judges the node;
+ *   Cloudly never retries it.
+ */
+export const nodeActionStatusKinds = Object.freeze({
+  pending: 'open',
+  running: 'open',
+  claimed: 'open',
+  executing: 'open',
+  draining: 'open',
+  rebooting: 'open',
+  verifying: 'open',
+  succeeded: 'terminal',
+  failed: 'terminal',
+  expired: 'terminal',
+  cancelled: 'terminal',
+  unverified: 'terminal',
+} as const satisfies Record<TNodeActionStatus, 'open' | 'terminal'>);
+
+/** Every open state, the ones the one-open-action-per-node rule counts. */
+export const nodeActionOpenStatuses = Object.freeze((Object.keys(nodeActionStatusKinds) as TNodeActionStatus[])
+  .filter((statusArg) => nodeActionStatusKinds[statusArg] === 'open'));
+
+export const isNodeActionOpen = (statusArg: TNodeActionStatus): boolean => (
+  nodeActionStatusKinds[statusArg] === 'open'
+);
+
+/**
+ * The phases Spark reports while it executes a claimed action, in the only order it may report
+ * them. A phase may be skipped, never repeated or reported after a later one.
+ */
+export const sparkNodeActionPhaseOrder = Object.freeze(['claimed', 'executing', 'draining', 'rebooting'] as const);
+
+/** A phase Spark reports with a `phase` transition. */
+export type TSparkNodeActionPhase = Exclude<typeof sparkNodeActionPhaseOrder[number], 'claimed'>;
+
+/**
+ * Time budgets of the node runtime's action lifecycle, in milliseconds.
+ *
+ * A requested action waits `claimWindowMs` for its node's Spark to claim it, then expires. A claimed
+ * action must be verified within its completion budget (`nodeActionCompletionBudgetMs`), measured
+ * from the claim, or it ends `unverified`. Each budget is the sum of the steps the action takes. One
+ * budget is read from Spark's code; every other one is chosen here, and each member says which.
+ */
+export const nodeActionContract = Object.freeze({
+  /** Chosen: how long a requested action waits for its node's Spark to claim it (10 min). */
+  claimWindowMs: 600_000,
+  /**
+   * Spark code: `sparkNodeStopTimeoutSeconds`, the node unit's `TimeoutStopSec=` (522 s), in which
+   * Pallet stops every workload it runs.
+   */
+  drainMs: 522_000,
+  /** Chosen: a host's reboot until Spark reports from the new boot (10 min). */
+  bootMs: 600_000,
+  /**
+   * Chosen: how long Pallet takes to start and register again after the node runtime starts
+   * (712 s, from the 2026-11-05 survey; Spark states no start budget of its own).
+   */
+  startMs: 712_000,
+  /** Chosen: downloading and checking the target's node bundle before anything stops (10 min). */
+  downloadMs: 600_000,
+  /** Chosen: the host's package upgrade (30 min). */
+  hostUpgradeMs: 1_800_000,
+  /** Entries an action's timeline keeps; each state is entered at most once. */
+  maximumTimelineEntries: 16,
+  /** Characters of operator text one timeline entry or failure carries. */
+  maximumDetailLength: 512,
+  /** Services one impact names; a node holding more is answered `impact-unreadable`. */
+  maximumImpactServices: 1024,
+  /** Characters of one service name an impact carries. */
+  maximumServiceNameLength: 256,
+} as const);
+
+/**
+ * How long a claimed action has until Cloudly must have verified it, from the claim.
+ *
+ * A reboot drains, boots and starts the node. An update downloads the target, then restarts the
+ * node runtime on it: a drain and a start, no boot. A host upgrade runs the package upgrade and,
+ * with `reboot`, the reboot after it.
+ */
+export const nodeActionCompletionBudgetMs = (actionNameArg: TNodeActionName, rebootArg: boolean): number => {
+  const c = nodeActionContract;
+  switch (actionNameArg) {
+    case 'reboot':
+      return c.drainMs + c.bootMs + c.startMs;
+    case 'serveZoneServiceUpdate':
+      return c.downloadMs + c.drainMs + c.startMs;
+    case 'systemUpgrade':
+      return c.hostUpgradeMs + (rebootArg ? c.drainMs + c.bootMs + c.startMs : 0);
+  }
+};
+
+/** Whether an action reboots the host, which is how Cloudly verifies it (`nodeActionVerification`). */
+export const nodeActionReboots = (actionNameArg: TNodeActionName, rebootArg: boolean): boolean => (
+  actionNameArg === 'reboot' || (actionNameArg === 'systemUpgrade' && rebootArg)
+);
+
+/**
+ * What Cloudly accepts as evidence that a claimed action changed the node, by what the action does.
+ *
+ * - `new-boot`: an action that reboots is verified by a heartbeat, accepted after the claim, whose
+ *   `boot.id` differs from the boot the claim stated.
+ * - `target-version`: an update is verified by a heartbeat, accepted after the claim, whose bundle
+ *   version is the action's target version.
+ * - `upgrade-complete`: a host upgrade without a reboot is verified by Spark's `complete` and a
+ *   heartbeat accepted after it; a heartbeat that states `boot.rebootRequired` fails the action
+ *   `reboot-required` instead, because applying the upgrade needs a reboot the operator did not
+ *   request. Cloudly never escalates to one.
+ *
+ * An action that restarts the node runtime (every action but a host upgrade without a reboot) is
+ * then only `succeeded` once the node's Pallet registered a runtime session after the claim and
+ * Cloudly accepted an observation of every replica slot on the node after that registration.
+ */
+export type TNodeActionVerification = 'new-boot' | 'target-version' | 'upgrade-complete';
+
+export const nodeActionVerification = (
+  actionNameArg: TNodeActionName,
+  rebootArg: boolean,
+): TNodeActionVerification => nodeActionReboots(actionNameArg, rebootArg) ? 'new-boot'
+  : actionNameArg === 'serveZoneServiceUpdate' ? 'target-version' : 'upgrade-complete';
+
+/** Whether the action restarts the node runtime and is verified by its return (see `nodeActionVerification`). */
+export const nodeActionRestartsRuntime = (actionNameArg: TNodeActionName, rebootArg: boolean): boolean => (
+  nodeActionVerification(actionNameArg, rebootArg) !== 'upgrade-complete'
+);
+
+/**
+ * Why an action ended `failed`, each with the remedy an operator is told.
+ *
+ * The first group is stated by Spark (`TSparkNodeActionFailureCode`), the second by Cloudly.
+ */
+export const nodeActionFailureRemedies = Object.freeze({
+  'download-failed': 'Spark could not download the target release. Check that the node reaches '
+    + 'code.foss.global, then request the update again.',
+  'installer-mismatch': 'The downloaded install.sh does not have the SHA-256 the target states. '
+    + 'Nothing was installed; set the fleet Spark target again, then request the update again.',
+  'install-failed': 'The node bundle installer failed and left the installed release in place. '
+    + 'Read the node\'s Spark journal, then request the update again.',
+  'drain-failed': 'Pallet did not stop its workloads within the stop budget. Read the node\'s '
+    + 'Pallet journal before requesting the action again.',
+  'reboot-failed': 'The host did not accept the reboot. Read the node\'s Spark journal.',
+  'host-upgrade-failed': 'The host package upgrade failed. Read the node\'s Spark journal, repair '
+    + 'the package state on the host, then request the upgrade again.',
+  'executor-failed': 'Spark could not run the action. Read the node\'s Spark journal.',
+  'reboot-required': 'The host upgrade finished and applying it needs a reboot. Request a system '
+    + 'upgrade with reboot, or a reboot, when the node may go down.',
+  'target-below-node': 'The node runs a newer Spark than the action\'s target, so the update '
+    + 'would downgrade it. Set the fleet Spark target to the release the node should run.',
+} as const);
+
+export type TNodeActionFailureCode = keyof typeof nodeActionFailureRemedies;
+
+/** The failure codes Spark may state with a `fail` transition. */
+export const sparkNodeActionFailureCodes = Object.freeze([
+  'download-failed',
+  'drain-failed',
+  'executor-failed',
+  'host-upgrade-failed',
+  'install-failed',
+  'installer-mismatch',
+  'reboot-failed',
+] as const satisfies readonly TNodeActionFailureCode[]);
+
+export type TSparkNodeActionFailureCode = typeof sparkNodeActionFailureCodes[number];
+
+/**
+ * Why an action ended `expired`.
+ *
+ * - `claim-window-passed`: no Spark claimed it within `nodeActionContract.claimWindowMs`.
+ * - `carrier-retired`: a Swarm-era record on a node that holds no Swarm-era session, which no
+ *   carrier delivers: it states none of the frozen parameters and target the node runtime's
+ *   lifecycle needs. Cloudly keeps the record.
+ */
+export type TNodeActionExpiryReason = 'claim-window-passed' | 'carrier-retired';
+
+export interface INodeActionExpiry {
+  reason: TNodeActionExpiryReason;
+  at: number;
+}
+
+/**
+ * The Spark release an update moves a node to: the fleet Spark target as it stood when the
+ * operator requested the action, frozen into the action.
+ */
+export interface INodeActionTarget {
+  /** `MAJOR.MINOR.PATCH`, without `v`. */
+  version: string;
+  /** The SHA-256 of the release's `install.sh`, as `SHA256SUMS.txt` states it (lowercase hex). */
+  installerSha256: string;
+  /** The fleet target's revision this copy was taken from. */
+  targetRevision: number;
+}
+
+/**
+ * What an operator confirmed when requesting a node runtime action, frozen into the action.
+ *
+ * The consent is the typed hostname and the digest of the impact preview the operator read:
+ * Cloudly refuses the request when either no longer matches what it reads now.
+ */
+export interface INodeActionConfirmation {
+  /** The hostname the operator typed; must equal the node's reported hostname. */
+  confirmedHostname: string;
+  /** `impactDigest` of the preview the operator confirmed (`computeNodeActionImpactDigest`). */
+  impactDigest: string;
+  /**
+   * Consent to restarting the Corestore the preview names on this node. Required `true` when the
+   * impact names one (`corestore`), and never assumed: a Corestore restart is never a default.
+   */
+  corestoreRestart: boolean;
+  /** `systemUpgrade` only: reboot after the upgrade, as part of this one action. */
+  reboot: boolean;
+  /**
+   * `serveZoneServiceUpdate` only, and required there: the revision of the fleet Spark target the
+   * preview showed (`INodeActionPreview.target.targetRevision`). Cloudly refuses the update
+   * (`target-changed`) when the target it would freeze into the action has another revision, so the
+   * consent names the exact release the node is moved to. Absent for every other action.
+   */
+  targetRevision?: number;
+}
+
+/** One service the action interrupts, as the impact preview names it. */
+export interface INodeActionImpactService {
+  serviceId: string;
+  name: string;
+  /** Replica slots of this service on the node. */
+  slotsOnNode: number;
+  /** Replica slots of this service on other nodes; `0` means the service is down while the node is. */
+  slotsElsewhere: number;
+  /** The service is the cluster's designated ingress. */
+  clusterIngress: boolean;
+}
+
+/**
+ * What a node action interrupts, read from Cloudly's database when it is previewed and again when
+ * it is requested.
+ *
+ * - `clusterNodeCount` 1: the node is its cluster, so the action is a full cluster outage.
+ * - `relayHost`: the node hosts its cluster's relay, so every inter-node connection of the cluster
+ *   is interrupted while it is down. The relay never moves for an action.
+ * - `corestore`: the node runs a Corestore (`IClusterNode.data.corestore`), which restarts with it.
+ * - `services`: every service with a replica slot on the node, sorted by `serviceId`.
+ */
+export interface INodeActionImpact {
+  nodeId: string;
+  clusterId: string;
+  /** The hostname the node reports, which a confirmation types; `null` while it reports none. */
+  hostname: string | null;
+  clusterNodeCount: number;
+  relayHost: boolean;
+  corestore: boolean;
+  services: INodeActionImpactService[];
+}
+
+/** The SHA-256 (lowercase hex) of one impact's canonical JSON, which a confirmation names. */
+export const computeNodeActionImpactDigest = async (impactArg: INodeActionImpact): Promise<string> => {
+  const fail = (reasonArg: string): never => { throw new Error(`Node action impact rejected: ${reasonArg}`); };
+  return createCanonicalJsonSha256Hex(canonicalizeStrictJson(impactArg, fail, 'node action impact'), fail);
+};
+
+/** When Spark claimed an action, and in which boot of the host. */
+export interface INodeActionClaim {
+  claimedAt: number;
+  /** The host's `boot.id` the claim stated. */
+  bootId: string;
+  /** The Spark bundle version the node reported when it claimed. */
+  bundleVersion: string | null;
+}
+
+/** Why an action failed, and the remedy the operator was told (`nodeActionFailureRemedies`). */
+export interface INodeActionFailure {
+  code: TNodeActionFailureCode;
+  remedy: string;
+  /** Spark's or Cloudly's own text, bounded by `nodeActionContract.maximumDetailLength`. */
+  detail: string | null;
+}
+
+/** Who moved an action into a state. */
+export type TNodeActionActor = 'operator' | 'spark' | 'cloudly';
+
+/** One state an action entered, in order. */
+export interface INodeActionTransition {
+  status: TNodeActionStatus;
+  at: number;
+  actor: TNodeActionActor;
+  detail: string | null;
+}
+
+/**
+ * The node runtime's lifecycle of one action: what was confirmed and frozen at the request, and
+ * every state since. Absent on a Swarm-era record, which no carrier but the Swarm-era daemon's
+ * delivers.
+ *
+ * `revision` advances with every state change. Each change is one transaction that matches the
+ * action on its expected state and revision, so two writers never both move one action.
+ */
+export interface INodeActionLifecycle {
+  clusterId: string;
+  revision: number;
+  confirmation: INodeActionConfirmation;
+  impact: INodeActionImpact;
+  /** `serveZoneServiceUpdate` only. */
+  target: INodeActionTarget | null;
+  /** The claim window's end while `pending`; the completion deadline once claimed. */
+  deadlineAt: number;
+  claim: INodeActionClaim | null;
+  failure: INodeActionFailure | null;
+  timeline: INodeActionTransition[];
+}
+
+/**
+ * Every reason Cloudly answers instead of admitting, previewing or cancelling a node action. Each
+ * travels by name as a `TypedResponseError`, and a refused request changes nothing.
+ */
+export const nodeActionRefusals = Object.freeze([
+  /** No node of that id exists. */
+  'node-missing',
+  /** The node reported through no Spark that carries actions: it never reported at all. */
+  'node-action-undeliverable',
+  /** The node's Spark does not state that it executes this action (`ISparkNodeReport.actionExecutors`). */
+  'node-action-unsupported',
+  /**
+   * System upgrade on a node runtime node: its Spark does not state the `systemUpgrade` executor,
+   * which Spark states only once its host-upgrade executor passed the lab qualification.
+   */
+  'system-upgrade-unqualified',
+  /** The node already has an open action. */
+  'node-action-open',
+  /** Another node of the cluster has an open action; a cluster runs one disruptive action at a time. */
+  'cluster-action-open',
+  /** An update without a fleet Spark target. */
+  'spark-target-unset',
+  /** An update whose target is below the Spark bundle version the node reports. */
+  'spark-target-below-node',
+  /** An update of a node that is held at its release (`ISparkNodeTargetHold`). */
+  'spark-target-held',
+  /** The request states no confirmation, or one that is not well-formed for this action. */
+  'confirmation-invalid',
+  /** The typed hostname is not the hostname the node reports. */
+  'confirmation-hostname-mismatch',
+  /** What the action interrupts changed since the preview the confirmation names. */
+  'impact-changed',
+  /** An update whose confirmation names another fleet Spark target revision than the one Cloudly would freeze. */
+  'target-changed',
+  /** The impact names a Corestore on the node and the confirmation does not consent to restarting it. */
+  'corestore-consent-required',
+  /** Cloudly could not read what the action interrupts. */
+  'impact-unreadable',
+  /** No action of that id on that node. */
+  'node-action-missing',
+  /** Only a `pending` action is cancelled. */
+  'node-action-not-cancellable',
+  /** The action moved since the revision the request names. */
+  'node-action-revision-stale',
+] as const);
+
+export type TNodeActionRefusalCode = typeof nodeActionRefusals[number];
+
+/** What requesting one action on one node would do, and whether Cloudly would admit it now. */
+export interface INodeActionPreview {
+  nodeId: string;
+  actionName: TNodeActionName;
+  impact: INodeActionImpact;
+  /** What a confirmation names (`INodeActionConfirmation.impactDigest`). */
+  impactDigest: string;
+  /** The Spark bundle version the node reports; `null` while it reports none. */
+  nodeBundleVersion: string | null;
+  /** Whether the node's Spark states that the host needs a reboot; `null` while it states nothing. */
+  rebootRequired: boolean | null;
+  /** `serveZoneServiceUpdate` only: the fleet target the update moves the node to. */
+  target: INodeActionTarget | null;
+  /** Why a request would be refused now, in `nodeActionRefusals` order; empty when it would be admitted. */
+  refusals: TNodeActionRefusalCode[];
+}
+
+const maximumHostnameLength = 253;
+const hostnamePattern = /^[a-z0-9](?:[a-z0-9.-]{0,251}[a-z0-9])?$/;
+const sha256HexPattern = /^[a-f0-9]{64}$/;
+
+/**
+ * Validates one confirmation for one action and names every reason it is refused.
+ *
+ * Shape only: an update must state `targetRevision`, every other action must not. Whether it
+ * matches the node, the impact and the fleet target Cloudly reads now is Cloudly's decision
+ * (`confirmation-hostname-mismatch`, `impact-changed`, `target-changed`, `corestore-consent-required`).
+ */
+export const validateNodeActionConfirmation = (
+  valueArg: unknown,
+  actionNameArg: TNodeActionName,
+  pathArg = 'node action confirmation',
+): string[] => {
+  // An update names the target revision it confirmed; no other action does.
+  const update = actionNameArg === 'serveZoneServiceUpdate';
+  const value = readExactDataRecord(valueArg, ['confirmedHostname', 'impactDigest', 'corestoreRestart', 'reboot',
+    ...(update ? ['targetRevision'] : [])]);
+  if (value === null) {
+    return [update
+      ? `${pathArg} must use its exact schema, with the targetRevision the preview showed`
+      : `${pathArg} must use its exact schema, without a targetRevision`];
+  }
+  const errors: string[] = [];
+  if (update && (!Number.isSafeInteger(value.targetRevision) || (value.targetRevision as number) < 1)) {
+    errors.push(`${pathArg} targetRevision must be a positive integer`);
+  }
+  if (typeof value.confirmedHostname !== 'string' || value.confirmedHostname.length > maximumHostnameLength
+    || !hostnamePattern.test(value.confirmedHostname)) {
+    errors.push(`${pathArg} confirmedHostname must be a lowercase hostname`);
+  }
+  if (typeof value.impactDigest !== 'string' || !sha256HexPattern.test(value.impactDigest)) {
+    errors.push(`${pathArg} impactDigest must be a lowercase SHA-256 hex digest`);
+  }
+  if (typeof value.corestoreRestart !== 'boolean') errors.push(`${pathArg} corestoreRestart must be a boolean`);
+  if (typeof value.reboot !== 'boolean') {
+    errors.push(`${pathArg} reboot must be a boolean`);
+  } else if (value.reboot && actionNameArg !== 'systemUpgrade') {
+    errors.push(`${pathArg} reboot is stated for systemUpgrade only`);
+  }
+  return errors;
+};
+
+/** HTTPS POST body asking for the action Cloudly holds for this node. Never log, persist, or return nodeToken. */
+export interface ISparkNodeActionOfferRequest {
+  nodeId: string;
+  nodeToken: string;
+  /** Read and negotiated before the node is authenticated; see `sparkNodeActionContracts`. */
+  protocol: IProtocolOffer;
+}
+
+/**
+ * One action as Spark executes it: the node runtime's view of an open action.
+ *
+ * `status` is the action's state when Cloudly answered; `revision` is what Spark's next transition
+ * names. `claim` states the boot Spark claimed it in, so a Spark that restarted mid-action resumes
+ * by replaying the transition it had sent rather than claiming again.
+ */
+export interface ISparkNodeActionAssignment {
+  id: string;
+  actionName: TNodeActionName;
+  status: Extract<TNodeActionStatus, 'pending' | 'claimed' | 'executing' | 'draining' | 'rebooting' | 'verifying'>;
+  revision: number;
+  deadlineAt: number;
+  /** `systemUpgrade` only: reboot after the upgrade. */
+  reboot: boolean;
+  /** `serveZoneServiceUpdate` only: install exactly this release, after checking its installer's SHA-256. */
+  target: INodeActionTarget | null;
+  claim: { bootId: string } | null;
+}
+
+/** What Cloudly answers the offer route: the node's one open action, or `null`. */
+export interface ISparkNodeActionOffer {
+  nodeId: string;
+  action: ISparkNodeActionAssignment | null;
+  /** The controller's offer, so the node judges the answer it acts on. */
+  protocol: IProtocolOffer;
+}
+
+/**
+ * One step Spark states about the action it executes.
+ *
+ * - `claim`: take a `pending` action, in the boot `bootId` names.
+ * - `phase`: the action entered a later phase (`sparkNodeActionPhaseOrder`). A reboot drains and
+ *   then reboots; an update executes (downloads and installs) and then drains (restarts the node
+ *   runtime on the new release); a host upgrade executes and, with `reboot`, drains and reboots.
+ * - `complete`: a host upgrade without a reboot finished; Cloudly verifies it from the next heartbeat.
+ * - `fail`: the action failed with one of `sparkNodeActionFailureCodes`.
+ *
+ * Every other ending is Cloudly's: it verifies an action from what the node reports
+ * (`nodeActionVerification`), and a deadline that passes ends one `unverified`.
+ */
+export type TSparkNodeActionTransition =
+  | { kind: 'claim'; bootId: string }
+  | { kind: 'phase'; phase: TSparkNodeActionPhase }
+  | { kind: 'complete' }
+  | { kind: 'fail'; code: TSparkNodeActionFailureCode; detail: string };
+
+/** HTTPS POST body stating one transition. Never log, persist, or return nodeToken. */
+export interface ISparkNodeActionTransitionRequest {
+  nodeId: string;
+  nodeToken: string;
+  /** Read and negotiated before the node is authenticated; see `sparkNodeActionContracts`. */
+  protocol: IProtocolOffer;
+  actionId: string;
+  /** The revision of the assignment the transition was decided from. */
+  expectedRevision: number;
+  transition: TSparkNodeActionTransition;
+}
+
+/**
+ * Why Cloudly did not apply a transition.
+ *
+ * - `action-unknown`: no action of that id on this node.
+ * - `action-closed`: the action ended; `action` states it no more, Spark stops executing it.
+ * - `revision-stale`: the action moved since `expectedRevision`; `action` states where it stands.
+ * - `transition-invalid`: the action's state or kind does not admit this transition.
+ */
+export type TSparkNodeActionTransitionRefusal = 'action-unknown' | 'action-closed' | 'revision-stale'
+  | 'transition-invalid';
+
+/**
+ * What Cloudly answers a transition with.
+ *
+ * `replayed` answers a transition Cloudly had already applied at exactly that revision, so a
+ * Spark that lost the answer and sends it again is told it took effect. `action` is the action as
+ * it stands after the answer, `null` once it is no longer open.
+ */
+export type TSparkNodeActionTransitionAnswer =
+  | { outcome: 'applied' | 'replayed'; action: ISparkNodeActionAssignment | null; protocol: IProtocolOffer }
+  | { outcome: 'refused'; reason: TSparkNodeActionTransitionRefusal; action: ISparkNodeActionAssignment | null;
+    protocol: IProtocolOffer };
+
+/**
+ * The node runtime's action routes.
+ *
+ * Spark asks the offer route after each accepted heartbeat and states every step on the transition
+ * route. Both are authenticated with the node's Spark credential exactly like
+ * `sparkNodeHeartbeatContract` and judged in the same server order: the offer first, within
+ * `maxRequestBytes`, answered with `protocolRefusalStatus` and an `IProtocolRefusal` body on
+ * refusal, before the node is authenticated. The server's offer on these routes names, as its
+ * minimum, the release that introduced them, so a Spark below it is refused by name and is never
+ * handed an action.
+ */
+export const sparkNodeActionContracts = Object.freeze({
+  offer: Object.freeze({
+    endpoint: '/spark/nodes/action',
+    maxRequestBytes: 1024,
+    maxResponseBytes: 4096,
+  } as const),
+  transition: Object.freeze({
+    endpoint: '/spark/nodes/action/transition',
+    maxRequestBytes: 2048,
+    maxResponseBytes: 4096,
+  } as const),
+  protocolRefusalStatus: protocolRefusalHttpStatus,
+} as const);
+
+const fail = (): never => { throw new Error('Spark node action contract rejected.'); };
+function requireValue(valueArg: unknown): asserts valueArg {
+  if (!valueArg) fail();
+}
+const record = (valueArg: unknown, keysArg: readonly string[]): Record<string, unknown> => {
+  const result = readExactDataRecord(valueArg, keysArg);
+  requireValue(result !== null);
+  return result;
+};
+const nodeId = (valueArg: unknown): string => (isClusterNodeId(valueArg) ? valueArg : fail());
+const nodeToken = (valueArg: unknown): string => {
+  requireValue(typeof valueArg === 'string' && /^secureToken[a-f0-9]{64}$/.test(valueArg));
+  return valueArg;
+};
+const integer = (valueArg: unknown, minimumArg: number): number => {
+  requireValue(Number.isSafeInteger(valueArg) && !Object.is(valueArg, -0) && (valueArg as number) >= minimumArg);
+  return valueArg as number;
+};
+/** An action id as Cloudly mints them: a short URL-safe token. */
+const actionId = (valueArg: unknown): string => {
+  requireValue(typeof valueArg === 'string' && /^[A-Za-z0-9_-]{1,64}$/.test(valueArg));
+  return valueArg;
+};
+/** The kernel's `boot_id`: one lowercase UUID. */
+export const isHostBootId = (valueArg: unknown): valueArg is string => (
+  typeof valueArg === 'string'
+  && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/.test(valueArg)
+);
+const bootId = (valueArg: unknown): string => (isHostBootId(valueArg) ? valueArg : fail());
+const operatorText = (valueArg: unknown): string => {
+  requireValue(typeof valueArg === 'string' && valueArg.length <= nodeActionContract.maximumDetailLength
+    && /^[\x20-\x7e]*$/.test(valueArg));
+  return valueArg;
+};
+
+const snapshotTarget = (valueArg: unknown): INodeActionTarget => {
+  const value = record(valueArg, ['version', 'installerSha256', 'targetRevision']);
+  requireValue(isReleaseVersion(value.version));
+  requireValue(typeof value.installerSha256 === 'string' && sha256HexPattern.test(value.installerSha256));
+  return Object.freeze({
+    version: value.version, installerSha256: value.installerSha256,
+    targetRevision: integer(value.targetRevision, 1),
+  });
+};
+
+const assignmentStatuses = ['pending', 'claimed', 'executing', 'draining', 'rebooting', 'verifying'] as const;
+
+/** One assignment, exactly its keys; detached and frozen. */
+export const snapshotSparkNodeActionAssignment = (valueArg: unknown): ISparkNodeActionAssignment => {
+  const value = record(valueArg, ['id', 'actionName', 'status', 'revision', 'deadlineAt', 'reboot', 'target', 'claim']);
+  requireValue(isNodeActionName(value.actionName));
+  const status = assignmentStatuses.find((statusArg) => statusArg === value.status) ?? fail();
+  requireValue(typeof value.reboot === 'boolean' && (!value.reboot || value.actionName === 'systemUpgrade'));
+  const target = value.target === null ? null : snapshotTarget(value.target);
+  requireValue((target !== null) === (value.actionName === 'serveZoneServiceUpdate'));
+  const claim = value.claim === null ? null : Object.freeze({ bootId: bootId(record(value.claim, ['bootId']).bootId) });
+  requireValue((claim === null) === (status === 'pending'));
+  return Object.freeze({
+    id: actionId(value.id), actionName: value.actionName, status,
+    revision: integer(value.revision, 1), deadlineAt: integer(value.deadlineAt, 1),
+    reboot: value.reboot, target, claim,
+  });
+};
+
+export const snapshotSparkNodeActionOfferRequest = (valueArg: unknown): ISparkNodeActionOfferRequest => {
+  const value = record(valueArg, ['nodeId', 'nodeToken', 'protocol']);
+  return Object.freeze({
+    nodeId: nodeId(value.nodeId), nodeToken: nodeToken(value.nodeToken), protocol: readProtocolOffer(value),
+  });
+};
+
+export const snapshotSparkNodeActionOffer = (valueArg: unknown): ISparkNodeActionOffer => {
+  const value = record(valueArg, ['nodeId', 'action', 'protocol']);
+  return Object.freeze({
+    nodeId: nodeId(value.nodeId),
+    action: value.action === null ? null : snapshotSparkNodeActionAssignment(value.action),
+    protocol: readProtocolOffer(value),
+  });
+};
+
+const snapshotTransition = (valueArg: unknown): TSparkNodeActionTransition => {
+  const properties = readDataProperties(valueArg);
+  requireValue(properties !== null);
+  switch (properties.kind) {
+    case 'claim':
+      return Object.freeze({ kind: 'claim', bootId: bootId(record(valueArg, ['kind', 'bootId']).bootId) });
+    case 'phase': {
+      const phase = record(valueArg, ['kind', 'phase']).phase;
+      const known = sparkNodeActionPhaseOrder.find((phaseArg) => phaseArg !== 'claimed' && phaseArg === phase);
+      return Object.freeze({ kind: 'phase', phase: (known ?? fail()) as TSparkNodeActionPhase });
+    }
+    case 'complete':
+      record(valueArg, ['kind']);
+      return Object.freeze({ kind: 'complete' });
+    case 'fail': {
+      const value = record(valueArg, ['kind', 'code', 'detail']);
+      const code = sparkNodeActionFailureCodes.find((codeArg) => codeArg === value.code) ?? fail();
+      return Object.freeze({ kind: 'fail', code, detail: operatorText(value.detail) });
+    }
+    default:
+      return fail();
+  }
+};
+
+export const snapshotSparkNodeActionTransitionRequest = (valueArg: unknown): ISparkNodeActionTransitionRequest => {
+  const value = record(valueArg, ['nodeId', 'nodeToken', 'protocol', 'actionId', 'expectedRevision', 'transition']);
+  return Object.freeze({
+    nodeId: nodeId(value.nodeId), nodeToken: nodeToken(value.nodeToken), protocol: readProtocolOffer(value),
+    actionId: actionId(value.actionId), expectedRevision: integer(value.expectedRevision, 1),
+    transition: snapshotTransition(value.transition),
+  });
+};
+
+const transitionRefusals: readonly TSparkNodeActionTransitionRefusal[] = [
+  'action-unknown', 'action-closed', 'revision-stale', 'transition-invalid',
+];
+
+export const snapshotSparkNodeActionTransitionAnswer = (valueArg: unknown): TSparkNodeActionTransitionAnswer => {
+  const properties = readDataProperties(valueArg);
+  requireValue(properties !== null);
+  if (properties.outcome === 'refused') {
+    const value = record(valueArg, ['outcome', 'reason', 'action', 'protocol']);
+    const reason = transitionRefusals.find((reasonArg) => reasonArg === value.reason) ?? fail();
+    return Object.freeze({
+      outcome: 'refused', reason,
+      action: value.action === null ? null : snapshotSparkNodeActionAssignment(value.action),
+      protocol: readProtocolOffer(value),
+    });
+  }
+  const value = record(valueArg, ['outcome', 'action', 'protocol']);
+  requireValue(value.outcome === 'applied' || value.outcome === 'replayed');
+  return Object.freeze({
+    outcome: value.outcome as 'applied' | 'replayed',
+    action: value.action === null ? null : snapshotSparkNodeActionAssignment(value.action),
+    protocol: readProtocolOffer(value),
+  });
+};
+
+/**
+ * Whether an open action admits one transition from Spark, judged from its state alone.
+ *
+ * `claim` takes a `pending` action. A `phase` moves strictly forward in `sparkNodeActionPhaseOrder`
+ * from `claimed` or an earlier phase, and only into a phase the action has: `rebooting` only for
+ * an action that reboots, `draining` for every action but a host upgrade without a reboot.
+ * `complete` ends the execution of a host upgrade without a reboot. `fail` ends any claimed action
+ * that Cloudly is not yet verifying.
+ */
+export const admitsSparkNodeActionTransition = (
+  actionArg: Pick<ISparkNodeActionAssignment, 'actionName' | 'status' | 'reboot'>,
+  transitionArg: TSparkNodeActionTransition,
+): boolean => {
+  const executing = (sparkNodeActionPhaseOrder as readonly string[]).indexOf(actionArg.status);
+  switch (transitionArg.kind) {
+    case 'claim':
+      return actionArg.status === 'pending';
+    case 'phase': {
+      if (executing < 0) return false;
+      const next = sparkNodeActionPhaseOrder.indexOf(transitionArg.phase);
+      if (next <= executing) return false;
+      if (transitionArg.phase === 'rebooting') return nodeActionReboots(actionArg.actionName, actionArg.reboot);
+      if (transitionArg.phase === 'draining') return nodeActionRestartsRuntime(actionArg.actionName, actionArg.reboot);
+      return actionArg.actionName !== 'reboot';
+    }
+    case 'complete':
+      return executing >= 1 && nodeActionVerification(actionArg.actionName, actionArg.reboot) === 'upgrade-complete';
+    case 'fail':
+      return executing >= 0;
+  }
+};
+
+/** True when a value is a canonical list of action names: known names, strictly increasing, at least one. */
+export const isNodeActionNameList = (valueArg: unknown): valueArg is TNodeActionName[] => {
+  const elements = readDataElements(valueArg, nodeActionNames.length);
+  return elements !== null && elements.length > 0 && isCanonicalTokenList(elements, nodeActionNames);
+};
+
+/** Sorts action names into canonical order, so a list built from a set compares byte for byte. */
+export const sortNodeActionNames = (namesArg: Iterable<TNodeActionName>): TNodeActionName[] => (
+  [...new Set(namesArg)].sort(compareWireTokens)
+);