@starci/hfs 4.2.17 → 4.4.0

package/stack/jobs.mjs

@@ -0,0 +1,51 @@
+// jobs.mjs - `starci app stack jobs <action>`: runs the app's declared job CLI (deploy.jobs.command + the action) inside the running task of
+// deploy.jobs.service on the manager, through the ssh runner, and relays its stdout. Job rows hold names and error texts, never payloads: the app
+// CLI decides what is printed, this module only relays it. stats, failed, open and watchdog are read-only; retry and retry-failed are effects
+// that need --execute (without it a dry-run line says what would run).
+import { firstLine, lines as splitLines } from './context.mjs';
+import { failure, refusal } from './errors.mjs';
+import { remoteDocker, withRunner } from './remote.mjs';
+
+const READ_ONLY = new Set(['stats', 'failed', 'open', 'watchdog']);
+const EFFECTS = new Set(['retry', 'retry-failed']);
+const OUTBOX_ID = /^[A-Za-z0-9-]{1,64}$/u;
+const WHOLE_NUMBER = /^[0-9]{1,4}$/u;
+
+function validCount(value, label) {
+  if (!WHOLE_NUMBER.test(value) || Number(value) < 1 || Number(value) > 720) throw refusal('STACK_JOBS_ARGUMENT', `${label} must be a whole number from 1 to 720`);
+}
+
+/** Validates the action and its arguments; every argument reaches a remote command only after this. */
+function validate(action, args) {
+  if (!READ_ONLY.has(action) && !EFFECTS.has(action)) throw refusal('STACK_JOBS_ARGUMENT', `jobs takes ${[...READ_ONLY, ...EFFECTS].join(', ')}`);
+  const maximum = action === 'stats' || action === 'watchdog' ? 0 : 1;
+  if (args.length > maximum) throw refusal('STACK_JOBS_ARGUMENT', `jobs ${action} takes at most ${maximum} argument(s)`);
+  if (action === 'retry') {
+    if (args.length !== 1 || !OUTBOX_ID.test(args[0])) throw refusal('STACK_JOBS_ARGUMENT', 'jobs retry takes one outbox id (letters, digits, dashes, up to 64)');
+  } else if (args.length === 1) validCount(args[0], action === 'retry-failed' ? 'hours' : 'the row count');
+}
+
+function runningTask(run, config) {
+  const service = `${config.stack}_${config.jobs.service}`;
+  const result = remoteDocker(run, ['ps', '-q', '--filter', `label=com.docker.swarm.service.name=${service}`, '--filter', 'status=running']);
+  const task = result.status === 0 ? splitLines(result.stdout)[0] : undefined;
+  if (task) return task;
+  const reason = result.status === 0 ? 'none is running' : firstLine(result.stderr);
+  throw failure('STACK_JOBS_NO_TASK', `no running task of service ${service} on the manager (${reason})`);
+}
+
+/** jobs: relays one action of the app's job CLI. */
+export async function runJobs(ctx, config, { action = 'stats', args = [], execute = false, sshCmd = null } = {}, deps = {}) {
+  if (!config.jobs) throw refusal('STACK_JOBS_UNDECLARED', 'the environment declares no deploy.jobs block');
+  validate(action, args);
+  if (EFFECTS.has(action) && !execute) {
+    return { ok: true, lines: [`dry run: would run jobs ${[action, ...args].join(' ')} in the running ${config.jobs.service} task (add --execute)`], data: { action, status: null } };
+  }
+  return withRunner(ctx, config, { sshCmd }, deps, (run, notes) => {
+    const task = runningTask(run, config);
+    const result = remoteDocker(run, ['exec', task, ...config.jobs.command, action, ...args]);
+    const lines = [...notes, ...splitLines(result.stdout)];
+    if (result.status !== 0) lines.push(`FAIL  jobs ${action} exited ${result.status}: ${firstLine(result.stderr)}`);
+    return { ok: result.status === 0, lines, data: { action, status: result.status } };
+  });
+}

package/stack/live-restore.mjs

@@ -0,0 +1,177 @@
+// live-restore.mjs - docker `live-restore` is incompatible with swarm mode. `prepare` detects it; with --fix-live-restore, --owner-approved and
+// --execute it backs up /etc/docker/daemon.json, sets live-restore false and restarts dockerd through passwordless sudo.
+// A dockerd restart stops every container (a restart policy does not reliably bring them back: on a real host 25 of 25 stayed down), so the fix
+//   1. snapshots every running container first (name, compose project and service, depends_on, image) and the answer of every public host,
+//   2. after the restart waits for docker, gives the restart policies a short grace, then `docker start`s every snapshotted container that is
+//      not running: databases and infrastructure first, applications next, edge proxies last, compose depends_on respected inside a group,
+//   3. re-checks the hosts against the baseline and FAILS loudly, listing every container and host still down.
+// Everything runs ON the manager through the ssh runner.
+import { approvalMissing } from './errors.mjs';
+import { remoteDocker, shellQuote } from './remote.mjs';
+
+const DAEMON_JSON = '/etc/docker/daemon.json';
+const WAIT_SECONDS = 180;
+const GRACE_SECONDS = 45;
+const HOST_WAIT_SECONDS = 120;
+const POLL_MS = 3000;
+const INFRA = /(?:postgres|postgis|pgbouncer|mysql|mariadb|mongo|redis|valkey|memcached|kafka|zookeeper|nats|rabbit|minio|elastic|opensearch|qdrant|clickhouse|cassandra|-db\b|_db\b|^db$|database)/iu;
+const EDGE = /(?:traefik|nginx|caddy|haproxy|envoy|edge|proxy)/iu;
+const FORMAT = '{{.Name}}|{{index .Config.Labels "com.docker.compose.project"}}|{{index .Config.Labels "com.docker.compose.service"}}|{{index .Config.Labels "com.docker.compose.depends_on"}}|{{.HostConfig.RestartPolicy.Name}}|{{.Config.Image}}';
+
+/** True when dockerd runs with live-restore (docker info reports it). */
+function liveRestoreEnabled(run) {
+  return remoteDocker(run, ['info', '--format', '{{.LiveRestoreEnabled}}']).stdout.trim() === 'true';
+}
+
+/** Read-only check used by prepare and doctor. */
+export function checkLiveRestore(run, report) {
+  if (liveRestoreEnabled(run)) report.bad('docker live-restore is enabled and is incompatible with swarm: run "starci app stack prepare --fix-live-restore --owner-approved --execute"');
+  else report.ok('docker live-restore is off');
+}
+
+/** depends_on label: "db:service_started:false,cache:service_healthy:true" -> ['db', 'cache']. */
+const dependenciesOf = (label) => label.split(',').map((item) => item.split(':')[0].trim()).filter(Boolean);
+
+function parseSnapshot(line) {
+  const [name, project = '', service = '', depends = '', restart = '', image = ''] = line.split('|');
+  return { name: name.replace(/^\//u, ''), project, service, depends: dependenciesOf(depends), restart, image };
+}
+
+/** Every running container with the facts needed to start it again in order. */
+export function snapshotContainers(run) {
+  const names = remoteDocker(run, ['ps', '--format', '{{.Names}}']).stdout.split(/\r?\n/u).filter(Boolean);
+  return names.map((name) => parseSnapshot(remoteDocker(run, ['inspect', '--format', FORMAT, name]).stdout.trim()));
+}
+
+const rankOf = (entry) => {
+  if (INFRA.test(`${entry.service} ${entry.name} ${entry.image}`)) return 0;
+  return EDGE.test(`${entry.service} ${entry.name} ${entry.image}`) ? 2 : 1;
+};
+
+/** Start order: infrastructure, applications, edge; inside a rank, compose dependencies first (stable, cycles keep their input order). */
+export function startOrder(snapshot) {
+  const ordered = [];
+  const placed = new Set();
+  const place = (entry, trail) => {
+    if (placed.has(entry.name) || trail.has(entry.name)) return;
+    const next = new Set(trail).add(entry.name);
+    const same = (other) => other.project === entry.project && entry.depends.includes(other.service) && rankOf(other) === rankOf(entry);
+    for (const dependency of snapshot.filter(same)) place(dependency, next);
+    placed.add(entry.name);
+    ordered.push(entry);
+  };
+  for (const rank of [0, 1, 2]) for (const entry of snapshot.filter((item) => rankOf(item) === rank)) place(entry, new Set());
+  return ordered;
+}
+
+function rewrittenDaemonJson(run) {
+  const read = run(`sudo -n cat ${shellQuote(DAEMON_JSON)}`);
+  if (read.status !== 0) return { error: `cannot read ${DAEMON_JSON}: ${read.stderr.trim().split(/\r?\n/u)[0]}` };
+  let parsed;
+  try { parsed = JSON.parse(read.stdout.trim() || '{}'); } catch { return { error: `${DAEMON_JSON} is not valid JSON; fix it by hand` }; }
+  return { text: `${JSON.stringify({ ...parsed, 'live-restore': false }, null, 2)}\n` };
+}
+
+const running = (run, name) => remoteDocker(run, ['inspect', '--format', '{{.State.Running}}', name]).stdout.trim() === 'true';
+const dockerUp = (run) => remoteDocker(run, ['info', '--format', '{{.ServerVersion}}']).status === 0;
+const notRunning = (run, names) => names.filter((name) => !running(run, name));
+
+async function waitUntil(ctx, seconds, done) {
+  const deadline = ctx.now() + seconds * 1000;
+  while (!done() && ctx.now() < deadline) await ctx.sleep(POLL_MS);
+  return done();
+}
+
+/** The hosts to prove, from the declared ingress: the verify list, else every upstream host (path /). */
+function hostsToProve(config) {
+  const ingress = config.ingress;
+  if (!ingress) return [];
+  const verify = ingress.verify.map((item) => ({ host: item.host, path: item.path }));
+  const seen = new Set(verify.map((item) => item.host));
+  const plain = ingress.upstreams.flatMap((entry) => entry.hosts).filter((host) => !seen.has(host) && !ingress.drop.includes(host));
+  return [...verify, ...[...new Set(plain)].map((host) => ({ host, path: '/' }))];
+}
+
+const probeClass = (run, entry) => {
+  const target = `https://${entry.host}${entry.path}`;
+  const result = run(['curl', '-sS', '-o', '/dev/null', '--max-time', '10', '-w', '%{http_code}', '--resolve', shellQuote(`${entry.host}:443:127.0.0.1`), shellQuote(target)].join(' '));
+  const code = Number(result.stdout.trim());
+  return code >= 200 && code < 500 ? Math.floor(code / 100) : 0;
+};
+
+const baselineOf = (run, hosts) => new Map(hosts.map((entry) => [entry.host, probeClass(run, entry)]));
+const hostsDown = (run, hosts, baseline) => hosts.filter((entry) => baseline.get(entry.host) !== 0 && probeClass(run, entry) !== baseline.get(entry.host));
+
+function startMissing(run, report, order) {
+  const failed = [];
+  for (const entry of order.filter((item) => !running(run, item.name))) {
+    const started = remoteDocker(run, ['start', entry.name]);
+    if (started.status === 0) report.note(`started ${entry.name}`);
+    else {
+      failed.push(entry.name);
+      report.bad(`cannot start ${entry.name}: ${started.stderr.trim().split(/\r?\n/u)[0]}`);
+    }
+  }
+  return failed;
+}
+
+async function recover(ctx, run, report, snapshot, hosts, baseline) {
+  const order = startOrder(snapshot);
+  const names = order.map((entry) => entry.name);
+  await waitUntil(ctx, WAIT_SECONDS, () => dockerUp(run));
+  await waitUntil(ctx, GRACE_SECONDS, () => notRunning(run, names).length === 0);
+  startMissing(run, report, order);
+  await waitUntil(ctx, WAIT_SECONDS, () => notRunning(run, names).length === 0);
+  const still = notRunning(run, names);
+  const back = names.filter((name) => !still.includes(name));
+  report.note(`containers back (${back.length} of ${names.length}): ${back.join(', ') || 'none'}`);
+  if (still.length > 0) report.bad(`CONTAINERS STILL DOWN after the restart: ${still.join(', ')}`);
+  await waitUntil(ctx, HOST_WAIT_SECONDS, () => hostsDown(run, hosts, baseline).length === 0);
+  const down = hostsDown(run, hosts, baseline).map((entry) => entry.host);
+  if (down.length > 0) report.bad(`PUBLIC HOSTS STILL DOWN against the pre-restart baseline: ${down.join(', ')}`);
+  else if (hosts.length > 0) report.ok(`all ${hosts.length} public host(s) answer as before the restart`);
+  return { back, missing: still, hostsDown: down };
+}
+
+/**
+ * The fix. Needs execute AND ownerApproved (else a dry-run line, or approvalMissing when execute is given without approval).
+ * Returns { changed, back, missing, hostsDown, snapshot }.
+ */
+export async function fixLiveRestore(ctx, run, report, { execute, ownerApproved, config }) {
+  const idle = { changed: false, back: [], missing: [], hostsDown: [], snapshot: [] };
+  if (!liveRestoreEnabled(run)) {
+    report.note('docker live-restore is already off: nothing to change');
+    return idle;
+  }
+  if (!execute) {
+    report.note('dry run: would snapshot the running containers and the public hosts, back up daemon.json, set live-restore false, restart docker, start every container that does not come back (databases, applications, edge) and re-check the hosts (needs --execute and --owner-approved)');
+    return idle;
+  }
+  if (!ownerApproved) throw approvalMissing('STACK_LIVE_RESTORE_UNAPPROVED', 'restarting dockerd restarts every container; pass --owner-approved after the owner said yes');
+  const rewritten = rewrittenDaemonJson(run);
+  if (rewritten.error) {
+    report.bad(rewritten.error);
+    return idle;
+  }
+  const snapshot = snapshotContainers(run);
+  const hosts = hostsToProve(config);
+  const baseline = baselineOf(run, hosts);
+  report.note(`snapshot: ${snapshot.length} running container(s), ${hosts.length} public host(s) in the baseline`);
+  const stamp = new Date(ctx.now()).toISOString().replaceAll(/[-:.]/gu, '').slice(0, 15);
+  const backup = `${DAEMON_JSON}.bak-${stamp}`;
+  const steps = [
+    [`sudo -n cp -p ${shellQuote(DAEMON_JSON)} ${shellQuote(backup)}`, undefined],
+    [`sudo -n tee ${shellQuote(DAEMON_JSON)} >/dev/null`, rewritten.text],
+    ['sudo -n systemctl restart docker', undefined],
+  ];
+  for (const [command, input] of steps) {
+    const done = run(command, input === undefined ? {} : { input });
+    if (done.status !== 0) {
+      report.bad(`live-restore fix stopped at "${command.split(' ').slice(0, 4).join(' ')}": ${done.stderr.trim().split(/\r?\n/u)[0]} (daemon.json backup: ${backup})`);
+      return { ...idle, snapshot };
+    }
+  }
+  report.note(`daemon.json backed up to ${backup}; live-restore set to false; docker restarted`);
+  const outcome = await recover(ctx, run, report, snapshot, hosts, baseline);
+  return { changed: true, snapshot, ...outcome };
+}

package/stack/prepare.mjs

@@ -0,0 +1,266 @@
+// prepare.mjs - `starci app stack prepare` and `doctor`: read-only checks of the Swarm manager, plus (prepare, --execute) the missing external
+// volumes and the ONE-TIME, owner-approved `docker swarm init`. Every docker call runs ON the manager through the ssh runner (remote.mjs), so the
+// same code serves CI and the owner's machine. Lines are `ok    text` / `FAIL  text`; a failing check makes the result not ok but all checks run.
+import fs from 'node:fs';
+import { firstLine, lines as splitLines } from './context.mjs';
+import { RELEASE_ID } from './digest.mjs';
+import { approvalMissing, refusal } from './errors.mjs';
+import { installAge } from './age-install.mjs';
+import { describeClash, edgeCollisions } from './edge-alias.mjs';
+import { checkLiveRestore, fixLiveRestore } from './live-restore.mjs';
+import { remoteDocker, shellQuote, withRunner } from './remote.mjs';
+
+const MIN_FREE_GB = 10;
+const ADDRESS = /^[0-9a-zA-Z.:-]+$/u;
+const SWARM_FORMAT = '{{.Swarm.LocalNodeState}} {{.Swarm.ControlAvailable}}';
+const OK = 'ok    ';
+const BAD = 'FAIL  ';
+
+function newReport() {
+  const entries = [];
+  const add = (mark) => (text) => { entries.push({ mark, text }); };
+  return {
+    ok: add(OK),
+    bad: add(BAD),
+    note: add(''),
+    failed: () => entries.some((entry) => entry.mark === BAD),
+    lines: () => entries.map((entry) => `${entry.mark}${entry.text}`),
+    checks: () => entries.filter((entry) => entry.mark !== '').map((entry) => ({ ok: entry.mark === OK, text: entry.text })),
+  };
+}
+
+function swarmInfo(run) {
+  const result = remoteDocker(run, ['info', '--format', SWARM_FORMAT]);
+  const text = result.stdout.trim();
+  return { reachable: result.status === 0, active: text === 'active true', state: text.split(' ')[0], manager: text.split(' ')[1] };
+}
+
+const dockerText = (run, args) => remoteDocker(run, args).stdout.trim();
+
+function checkDaemon(run, report) {
+  if (!swarmInfo(run).reachable) {
+    report.bad('docker daemon not reachable');
+    return false;
+  }
+  report.ok(`docker daemon reachable (${dockerText(run, ['version', '--format', '{{.Server.Version}}'])})`);
+  return true;
+}
+
+function checkPlatform(run, report) {
+  const platform = dockerText(run, ['info', '--format', '{{.OSType}}/{{.Architecture}}']);
+  if (platform === 'linux/x86_64') report.ok('linux/x86_64');
+  else report.bad(`unsupported platform ${platform}`);
+}
+
+function checkSwarm(run, report) {
+  const info = swarmInfo(run);
+  if (info.active) report.ok('swarm active, this node is a manager');
+  else report.bad(`swarm is '${info.state}' (manager: ${info.manager}): the owner approves "starci app stack prepare --init-swarm --owner-approved --execute" once`);
+}
+
+function checkNodes(run, report) {
+  const count = splitLines(remoteDocker(run, ['node', 'ls', '-q']).stdout).length;
+  if (count <= 1) report.ok('single-manager swarm');
+  else report.ok(`swarm has ${count} nodes: review quorum and placement before relying on it`);
+}
+
+const volumeExists = (run, volume) => remoteDocker(run, ['volume', 'inspect', volume]).status === 0;
+
+function checkVolumes(run, report, config) {
+  for (const volume of config.volumes) {
+    if (volumeExists(run, volume)) report.ok(`volume ${volume}`);
+    else report.bad(`volume ${volume} missing (prepare creates it)`);
+  }
+}
+
+function checkDisk(run, report) {
+  const result = run('target=/var/lib/docker; [ -d "$target" ] || target=/; echo "$target"; df -Pk "$target"');
+  const [target, , row = ''] = splitLines(result.stdout);
+  const availableKb = Number(row.trim().split(/\s+/u)[3]);
+  if (result.status !== 0 || !Number.isFinite(availableKb)) {
+    report.bad('disk: cannot read the free space');
+    return;
+  }
+  const gigabytes = Math.floor(availableKb / 1_048_576);
+  if (gigabytes >= MIN_FREE_GB) report.ok(`disk: ${gigabytes} GB free under ${target}`);
+  else report.bad(`disk: only ${gigabytes} GB free under ${target}`);
+}
+
+const releasesBase = (config) => config.remote.releasesBase ?? `/srv/${config.stack}`;
+
+function directoryState(run, config) {
+  const quoted = shellQuote(releasesBase(config));
+  return run(`if [ ! -d ${quoted} ]; then echo missing; elif [ ! -w ${quoted} ]; then echo readonly; else echo writable; fi`).stdout.trim();
+}
+
+function checkReleasesDirectory(run, report, config) {
+  const base = releasesBase(config);
+  const state = directoryState(run, config);
+  if (state === 'writable') report.ok(`releases directory ${base} is writable`);
+  else if (state === 'readonly') report.bad(`releases directory ${base} is not writable by the deploy user`);
+  else report.bad(`releases directory ${base} is missing (create it once, owned by the deploy user)`);
+}
+
+/** The checks both verbs share; false when the daemon cannot be reached (nothing else can be judged). */
+function baseChecks(run, report, config) {
+  if (!checkDaemon(run, report)) return false;
+  checkLiveRestore(run, report);
+  checkPlatform(run, report);
+  checkSwarm(run, report);
+  checkNodes(run, report);
+  checkVolumes(run, report, config);
+  checkDisk(run, report);
+  checkReleasesDirectory(run, report, config);
+  return true;
+}
+
+function advertiseArguments(ctx) {
+  const address = ctx.env.STARCI_ADVERTISE_ADDR;
+  if (!address) return [];
+  if (!ADDRESS.test(address)) throw refusal('STACK_SWARM_ADVERTISE_ADDR', `STARCI_ADVERTISE_ADDR must match ${ADDRESS}`);
+  const listen = ctx.env.STARCI_SWARM_LISTEN_ADDR ?? '127.0.0.1:2377';
+  if (!ADDRESS.test(listen)) throw refusal('STACK_SWARM_LISTEN_ADDR', `STARCI_SWARM_LISTEN_ADDR must match ${ADDRESS}`);
+  // the control port stays on loopback unless the owner widens it: a single manager needs no inbound swarm port
+  return ['--advertise-addr', address, '--listen-addr', listen];
+}
+
+function initSwarm(run, report, { execute, advertise }) {
+  if (swarmInfo(run).active) {
+    report.note('swarm already active: nothing to initialize (never re-initialized)');
+    return false;
+  }
+  if (!execute) {
+    report.note('dry run: would run docker swarm init (needs --execute and --owner-approved)');
+    return false;
+  }
+  const result = remoteDocker(run, ['swarm', 'init', ...advertise]);
+  if (result.status !== 0) {
+    report.bad(`docker swarm init failed: ${firstLine(result.stderr)}`);
+    return false;
+  }
+  report.note('SWARM INITIALIZED. Record the node id; back up /var/lib/docker/swarm (docker swarm unlock-key if autolock is enabled later).');
+  return true;
+}
+
+function createVolumes(run, report, config, execute) {
+  const created = [];
+  for (const volume of config.volumes.filter((name) => !volumeExists(run, name))) {
+    if (!execute) {
+      report.note(`dry run: would create volume ${volume}`);
+      continue;
+    }
+    const result = remoteDocker(run, ['volume', 'create', volume]);
+    if (result.status === 0) {
+      report.note(`created volume ${volume}`);
+      created.push(volume);
+    } else report.bad(`cannot create volume ${volume}: ${firstLine(result.stderr)}`);
+  }
+  return created;
+}
+
+/** The backup directories the deploy block declares (volume backups and the ingress store backups), without duplicates. */
+const backupDirectories = (config) => [...new Set([config.backup?.dir, config.ingress?.backup?.dir].filter((dir) => typeof dir === 'string'))];
+
+/** Creates one declared backup directory (mode 750, owned by the deploy user): as the user first, with passwordless sudo when the parent is not theirs. */
+function ensureDirectory(run, report, dir) {
+  const quoted = shellQuote(dir);
+  if (run(`test -d ${quoted}`).status === 0) return;
+  const made = run(`install -d -m 750 ${quoted} || sudo -n install -d -o "$(id -un)" -m 750 ${quoted}`);
+  if (made.status === 0) report.note(`created ${dir} (mode 750, owned by the deploy user)`);
+  else report.bad(`cannot create ${dir}: ${firstLine(made.stderr)} (declare a backup dir under the releases root, or create it once with sudo)`);
+}
+
+/** Creates the releases directory and installs age (the backup encryptor) once, with passwordless sudo when the host allows it. */
+function ensureTools(run, report, config) {
+  const base = releasesBase(config);
+  const quoted = shellQuote(base);
+  if (directoryState(run, config) === 'missing') {
+    const made = run(`sudo -n mkdir -p ${quoted} && sudo -n chown "$(id -un)" ${quoted}`);
+    if (made.status === 0) report.note(`created ${base} owned by the deploy user`);
+    else report.bad(`cannot create ${base}: ${firstLine(made.stderr)} (needs passwordless sudo once, or create it by hand)`);
+  }
+  for (const dir of [`${base}/releases`, ...backupDirectories(config)]) ensureDirectory(run, report, dir);
+  installAge(run, report);
+}
+
+/** prepare: read-only checks; with execute also the missing volumes and (initSwarm, approved) the one-time swarm init. */
+export async function prepareHost(ctx, config, { execute = false, initSwarm: wantInit = false, fixLiveRestore: wantFix = false, ownerApproved = false, sshCmd = null } = {}, deps = {}) {
+  if (wantInit && execute && !ownerApproved && ctx.env.STARCI_SWARM_INIT_APPROVED !== '1') {
+    throw approvalMissing('STACK_SWARM_INIT_UNAPPROVED', 'docker swarm init is an owner-approved effect; pass --owner-approved (or set STARCI_SWARM_INIT_APPROVED=1) after the owner said yes');
+  }
+  const advertise = wantInit ? advertiseArguments(ctx) : [];
+  return withRunner(ctx, config, { sshCmd }, deps, async (run, notes) => {
+    const report = newReport();
+    notes.forEach(report.note);
+    const data = { created: [], swarmInitialized: false };
+    if (swarmInfo(run).reachable) {
+      if (wantFix) data.liveRestore = await fixLiveRestore(ctx, run, report, { execute, ownerApproved, config });
+      if (wantInit) data.swarmInitialized = initSwarm(run, report, { execute, advertise });
+      data.created = createVolumes(run, report, config, execute);
+      if (execute) ensureTools(run, report, config);
+    }
+    baseChecks(run, report, config);
+    return { ok: !report.failed(), lines: report.lines(), data: { ...data, checks: report.checks() } };
+  });
+}
+
+function checkServices(run, report, config) {
+  const result = remoteDocker(run, ['service', 'ls', '--filter', `label=com.docker.stack.namespace=${config.stack}`, '--format', '{{.Name}} {{.Replicas}} {{.Image}}']);
+  if (result.status !== 0) {
+    report.bad(`cannot list the services of stack ${config.stack}`);
+    return 0;
+  }
+  const services = splitLines(result.stdout);
+  report.ok(`services of stack ${config.stack}: ${services.length}`);
+  for (const service of services) report.note(`      ${service}`);
+  return services.length;
+}
+
+function managedCount(run, kind, stack) {
+  return splitLines(remoteDocker(run, [kind, 'ls', '--filter', `label=${stack}.managed=true`, '-q']).stdout).length;
+}
+
+function checkPointers(run, report, config) {
+  const base = shellQuote(releasesBase(config));
+  const pointers = {};
+  for (const pointer of ['current', 'previous']) {
+    const value = run(`cd ${base} 2>/dev/null && [ -L ${pointer} ] && basename "$(readlink ${pointer})" || true`).stdout.trim();
+    pointers[pointer] = value === '' ? null : value;
+    if (value === '') report.ok(`release ${pointer}: none`);
+    else if (RELEASE_ID.test(value)) report.ok(`release ${pointer}: ${value}`);
+    else report.bad(`release ${pointer} names an unexpected value`);
+  }
+  return pointers;
+}
+
+/** The short service names of the stack source against the names the external (edge) networks already resolve (STACK_DEPLOY_ALIAS_SHARED at deploy). */
+async function checkEdgeNames(run, report, config) {
+  if (!config.stackFile) return;
+  let found;
+  try {
+    found = await edgeCollisions(run, config, fs.readFileSync(config.stackFile, 'utf8'));
+  } catch (error) {
+    report.bad(`edge networks: ${error.message}`);
+    return;
+  }
+  for (const clash of found.clashes) report.bad(`STACK_DEPLOY_ALIAS_SHARED: ${describeClash(clash)}`);
+  if (found.clashes.length === 0) report.ok(`edge networks (${found.networks.length}): no short service name is resolved by another tenant`);
+}
+
+/** doctor: the prepare checks plus services, managed secrets and configs and the release pointers. Changes nothing. */
+export async function doctorHost(ctx, config, { sshCmd = null } = {}, deps = {}) {
+  return withRunner(ctx, config, { sshCmd }, deps, async (run, notes) => {
+    const report = newReport();
+    notes.forEach(report.note);
+    const data = { services: 0, secrets: 0, configs: 0, releases: { current: null, previous: null } };
+    if (baseChecks(run, report, config)) {
+      data.services = checkServices(run, report, config);
+      data.secrets = managedCount(run, 'secret', config.stack);
+      data.configs = managedCount(run, 'config', config.stack);
+      report.ok(`managed secrets: ${data.secrets}, configs: ${data.configs}`);
+      data.releases = checkPointers(run, report, config);
+      await checkEdgeNames(run, report, config);
+    }
+    return { ok: !report.failed(), lines: report.lines(), data: { ...data, checks: report.checks() } };
+  });
+}

package/stack/private-file.mjs

@@ -0,0 +1,22 @@
+// private-file.mjs - writes a file only its owner may use. On POSIX the mode is enough. Windows OpenSSH refuses a key, a known_hosts or a config
+// file whose ACL still carries inherited entries ("Bad permissions ... UNKNOWN\UNKNOWN"), so on win32 the inheritance is removed with icacls and the
+// current user alone is granted access. `ctx.platform` and `ctx.username` are optional seams for the specs; the defaults are the real ones.
+import fs from 'node:fs';
+import os from 'node:os';
+import { refusal } from './errors.mjs';
+import { firstLine, tool } from './context.mjs';
+
+/** On win32: `icacls <file> /inheritance:r /grant:r <user>:<rights>` (R = read, F = full, to let a cleanup overwrite the file). Elsewhere: nothing. */
+export function restrictToUser(ctx, file, rights = 'R') {
+  if ((ctx.platform ?? process.platform) !== 'win32') return;
+  const user = ctx.username ?? os.userInfo().username;
+  const run = tool(ctx, 'icacls', [file, '/inheritance:r', '/grant:r', `${user}:${rights}`]);
+  if (run.status !== 0) throw refusal('STACK_REMOTE_ACL', `cannot restrict ${file} to the current user with icacls: ${firstLine(run.stderr) || firstLine(run.stdout) || 'icacls failed'}`);
+}
+
+/** Writes `text` to `file` with owner-only access (mode on POSIX, icacls on win32). */
+export function writePrivate(ctx, file, text) {
+  fs.writeFileSync(file, text, { mode: 0o600 });
+  fs.chmodSync(file, 0o600);
+  restrictToUser(ctx, file);
+}

package/stack/provision.mjs

@@ -0,0 +1,90 @@
+// provision.mjs - creates the immutable, versioned Swarm secrets and config objects a release names, on the daemon DOCKER_HOST points at.
+// Each secret value is opened in memory and piped to `docker secret create <runtimeName> -` on stdin: it never reaches argv, a file, the
+// output or an error message. A name that exists is left alone (same value, same name); a name is never reused for another value, and
+// an earlier version is never removed here (prune runs only after the release is accepted).
+import fs from 'node:fs';
+import { firstLine, lines as splitLines, tool } from './context.mjs';
+import { failure, refusal } from './errors.mjs';
+import { isReleaseId } from './digest.mjs';
+import { configBytes, configObjects, loadSealed } from './sealed.mjs';
+
+const STALE = 'render the release again';
+
+function readRelease(options) {
+  if (options.release) return options.release;
+  if (!options.releaseFile) throw refusal('STACK_RELEASE_MISSING', 'provision needs a release record (release or releaseFile)');
+  try {
+    return JSON.parse(fs.readFileSync(options.releaseFile, 'utf8'));
+  } catch {
+    throw refusal('STACK_RELEASE_INVALID', `${options.releaseFile} is not a readable release record`);
+  }
+}
+
+function releaseId(release) {
+  const id = String(release.releaseDigest).slice(0, 16);
+  if (!isReleaseId(id) || !Array.isArray(release.secrets) || !Array.isArray(release.configs)) throw refusal('STACK_RELEASE_INVALID', 'the release record has no valid releaseDigest, secrets and configs');
+  return id;
+}
+
+function existing(ctx, kind) {
+  const run = tool(ctx, 'docker', [kind, 'ls', '--format', '{{.Name}}']);
+  if (run.error || run.status !== 0) throw failure('STACK_PROVISION_FAILED', `docker ${kind} ls failed: ${run.error?.message ?? firstLine(run.stderr)}`);
+  return new Set(splitLines(run.stdout));
+}
+
+function createFailure(run, what, value) {
+  const text = run.error?.message ?? firstLine(run.stderr);
+  return failure('STACK_PROVISION_FAILED', `docker ${what} failed: ${value === undefined ? text : text.replaceAll(value, '[value]')}`);
+}
+
+function create(ctx, kind, labels, name, input) {
+  const label = labels.flatMap((item) => ['--label', item]);
+  return tool(ctx, 'docker', [kind, 'create', ...label, name, '-'], { input });
+}
+
+function plan(ctx, config, release, sealed) {
+  const secrets = release.secrets.map((entry) => {
+    const secret = sealed.secrets[entry.key];
+    if (secret?.runtimeName !== entry.runtimeName) throw refusal('STACK_RELEASE_STALE', `the sealed custody no longer matches the release record for ${entry.key}; ${STALE}`);
+    return { key: entry.key, name: entry.runtimeName, value: secret.value };
+  });
+  const files = new Map(configObjects(config).map((object) => [object.name, object.file]));
+  const configs = release.configs.map((entry) => {
+    if (!files.has(entry.name)) throw refusal('STACK_RELEASE_STALE', `the config ${entry.name} does not match the committed files; ${STALE}`);
+    return { logical: entry.logical, name: entry.name, file: files.get(entry.name) };
+  });
+  const haveSecrets = existing(ctx, 'secret');
+  const haveConfigs = existing(ctx, 'config');
+  return {
+    secrets: secrets.filter((item) => !haveSecrets.has(item.name)),
+    kept: secrets.filter((item) => haveSecrets.has(item.name)).length,
+    configs: configs.filter((item) => !haveConfigs.has(item.name)),
+  };
+}
+
+function createAll(ctx, config, id, todo) {
+  const managed = `${config.stack}.managed=true`;
+  const tagged = `${config.stack}.release=${id}`;
+  for (const item of todo.secrets) {
+    const run = create(ctx, 'secret', [managed, `${config.stack}.key=${item.key}`, tagged], item.name, item.value);
+    if (run.error || run.status !== 0) throw createFailure(run, `secret create for ${item.key}`, item.value);
+  }
+  for (const item of todo.configs) {
+    const run = create(ctx, 'config', [managed, tagged], item.name, configBytes(item.file));
+    if (run.error || run.status !== 0) throw createFailure(run, `config create for ${item.logical}`);
+  }
+}
+
+/** Creates the missing secrets and configs of a release; without `execute` it only reports what it would create. */
+export async function provisionRelease(ctx, config, options = {}) {
+  const release = readRelease(options);
+  const id = releaseId(release);
+  const todo = plan(ctx, config, release, loadSealed(ctx, config));
+  const data = { created: todo.secrets.length, kept: todo.kept, configsCreated: todo.configs.length };
+  if (options.execute !== true) {
+    const keys = todo.secrets.map((item) => item.key).join(', ');
+    return { ok: true, lines: [`dry run: would create ${data.created} secrets (${keys}), ${data.configsCreated} configs; ${data.kept} kept`], data };
+  }
+  createAll(ctx, config, id, todo);
+  return { ok: true, lines: [`provision: secrets created ${data.created}, kept ${data.kept}; configs created ${data.configsCreated}`], data };
+}

package/stack/prune.mjs

@@ -0,0 +1,46 @@
+// prune.mjs - removes the managed Swarm secrets and configs that none of the kept releases names. A name is never reused and an earlier
+// version is never deleted before convergence, so this runs only after a release was accepted. Docker refuses to remove an object a
+// service still uses; such a refusal is reported and left alone (never forced). Without `execute` it only lists what it would remove.
+import fs from 'node:fs';
+import { firstLine, lines as splitLines, tool } from './context.mjs';
+import { failure, refusal } from './errors.mjs';
+
+const KINDS = ['secret', 'config'];
+
+/** The runtime names a kept release.json holds: secret runtimeNames and config names (names only, never values). */
+function keptNames(files) {
+  const keep = new Set();
+  for (const file of files) {
+    let release;
+    try { release = JSON.parse(fs.readFileSync(file, 'utf8')); } catch { throw refusal('STACK_PRUNE_KEEP_INVALID', `${file} is not a readable release.json`); }
+    for (const secret of release.secrets ?? []) keep.add(secret.runtimeName);
+    for (const config of release.configs ?? []) keep.add(config.name);
+  }
+  return keep;
+}
+
+function managed(ctx, config, kind) {
+  const list = tool(ctx, 'docker', [kind, 'ls', '--filter', `label=${config.stack}.managed=true`, '--format', '{{.Name}}']);
+  if (list.status !== 0) throw failure('STACK_PRUNE_LIST', `docker ${kind} ls failed: ${firstLine(list.stderr)}`);
+  return splitLines(list.stdout);
+}
+
+/**
+ * Prunes superseded managed secrets and configs. `keepFiles` are the release.json files to keep (at least one).
+ * Returns { ok, lines, data: { kept, candidates, removed, inUse } }.
+ */
+export async function pruneStack(ctx, config, { keepFiles = [], execute = false } = {}) {
+  if (keepFiles.length === 0) throw refusal('STACK_PRUNE_NO_KEEP', 'prune needs --keep <release.json> (at least the accepted release)');
+  const keep = keptNames(keepFiles);
+  const out = { lines: [], removed: 0, inUse: 0, candidates: 0 };
+  for (const kind of KINDS) {
+    for (const name of managed(ctx, config, kind).filter((entry) => !keep.has(entry))) {
+      out.candidates += 1;
+      if (!execute) out.lines.push(`dry run: would remove ${kind} ${name}`);
+      else if (tool(ctx, 'docker', [kind, 'rm', name]).status === 0) out.removed += 1;
+      else { out.inUse += 1; out.lines.push(`${kind} ${name} is still in use; left alone`); }
+    }
+  }
+  out.lines.push(execute ? `prune: removed ${out.removed} superseded objects, ${out.inUse} still in use` : `dry run: ${out.candidates} superseded objects would be removed`);
+  return { ok: true, lines: out.lines, data: { kept: keep.size, candidates: out.candidates, removed: out.removed, inUse: out.inUse } };
+}