@starci/hfs 4.11.22 → 4.12.0

package/stack/jobs.d.mts

@@ -0,0 +1,12 @@
+import type { DeployConfig } from './config.mjs';
+import type { StackContext } from './context.mjs';
+import type { ActionResult, RunnerDeps } from './remote.mjs';
+/** What `jobs` takes: the app CLI action and its arguments. */
+export interface JobsOptions {
+    action?: string | undefined;
+    args?: string[];
+    execute?: boolean;
+    sshCmd?: string | null;
+}
+/** jobs: relays one action of the app's job CLI. */
+export declare function runJobs(ctx: StackContext, config: DeployConfig, { action, args, execute, sshCmd }?: JobsOptions, deps?: RunnerDeps): Promise<ActionResult>;

package/stack/jobs.mjs

@@ -1,51 +1,52 @@
-// jobs.mjs - `starci app stack jobs <action>`: runs the app's declared job CLI (deploy.jobs.command + the action) inside the running task of
-// deploy.jobs.service on the manager, through the ssh runner, and relays its stdout. Job rows hold names and error texts, never payloads: the app
-// CLI decides what is printed, this module only relays it. stats, failed, open and watchdog are read-only; retry and retry-failed are effects
-// that need --execute (without it a dry-run line says what would run).
 import { firstLine, lines as splitLines } from './context.mjs';
 import { failure, refusal } from './errors.mjs';
 import { remoteDocker, withRunner } from './remote.mjs';
-
+const declaresJobs = (config) => Boolean(config.jobs);
 const READ_ONLY = new Set(['stats', 'failed', 'open', 'watchdog']);
 const EFFECTS = new Set(['retry', 'retry-failed']);
 const OUTBOX_ID = /^[A-Za-z0-9-]{1,64}$/u;
 const WHOLE_NUMBER = /^[0-9]{1,4}$/u;
-
 function validCount(value, label) {
-  if (!WHOLE_NUMBER.test(value) || Number(value) < 1 || Number(value) > 720) throw refusal('STACK_JOBS_ARGUMENT', `${label} must be a whole number from 1 to 720`);
+    if (!WHOLE_NUMBER.test(value) || Number(value) < 1 || Number(value) > 720)
+        throw refusal('STACK_JOBS_ARGUMENT', `${label} must be a whole number from 1 to 720`);
 }
-
 /** Validates the action and its arguments; every argument reaches a remote command only after this. */
 function validate(action, args) {
-  if (!READ_ONLY.has(action) && !EFFECTS.has(action)) throw refusal('STACK_JOBS_ARGUMENT', `jobs takes ${[...READ_ONLY, ...EFFECTS].join(', ')}`);
-  const maximum = action === 'stats' || action === 'watchdog' ? 0 : 1;
-  if (args.length > maximum) throw refusal('STACK_JOBS_ARGUMENT', `jobs ${action} takes at most ${maximum} argument(s)`);
-  if (action === 'retry') {
-    if (args.length !== 1 || !OUTBOX_ID.test(args[0])) throw refusal('STACK_JOBS_ARGUMENT', 'jobs retry takes one outbox id (letters, digits, dashes, up to 64)');
-  } else if (args.length === 1) validCount(args[0], action === 'retry-failed' ? 'hours' : 'the row count');
+    if (!READ_ONLY.has(action) && !EFFECTS.has(action))
+        throw refusal('STACK_JOBS_ARGUMENT', `jobs takes ${[...READ_ONLY, ...EFFECTS].join(', ')}`);
+    const maximum = action === 'stats' || action === 'watchdog' ? 0 : 1;
+    if (args.length > maximum)
+        throw refusal('STACK_JOBS_ARGUMENT', `jobs ${action} takes at most ${maximum} argument(s)`);
+    if (action === 'retry') {
+        if (args.length !== 1 || !OUTBOX_ID.test(args[0]))
+            throw refusal('STACK_JOBS_ARGUMENT', 'jobs retry takes one outbox id (letters, digits, dashes, up to 64)');
+    }
+    else if (args.length === 1)
+        validCount(args[0], action === 'retry-failed' ? 'hours' : 'the row count');
 }
-
 function runningTask(run, config) {
-  const service = `${config.stack}_${config.jobs.service}`;
-  const result = remoteDocker(run, ['ps', '-q', '--filter', `label=com.docker.swarm.service.name=${service}`, '--filter', 'status=running']);
-  const task = result.status === 0 ? splitLines(result.stdout)[0] : undefined;
-  if (task) return task;
-  const reason = result.status === 0 ? 'none is running' : firstLine(result.stderr);
-  throw failure('STACK_JOBS_NO_TASK', `no running task of service ${service} on the manager (${reason})`);
+    const service = `${config.stack}_${config.jobs.service}`;
+    const result = remoteDocker(run, ['ps', '-q', '--filter', `label=com.docker.swarm.service.name=${service}`, '--filter', 'status=running']);
+    const task = result.status === 0 ? splitLines(result.stdout)[0] : undefined;
+    if (task)
+        return task;
+    const reason = result.status === 0 ? 'none is running' : firstLine(result.stderr);
+    throw failure('STACK_JOBS_NO_TASK', `no running task of service ${service} on the manager (${reason})`);
 }
-
 /** jobs: relays one action of the app's job CLI. */
 export async function runJobs(ctx, config, { action = 'stats', args = [], execute = false, sshCmd = null } = {}, deps = {}) {
-  if (!config.jobs) throw refusal('STACK_JOBS_UNDECLARED', 'the environment declares no deploy.jobs block');
-  validate(action, args);
-  if (EFFECTS.has(action) && !execute) {
-    return { ok: true, lines: [`dry run: would run jobs ${[action, ...args].join(' ')} in the running ${config.jobs.service} task (add --execute)`], data: { action, status: null } };
-  }
-  return withRunner(ctx, config, { sshCmd }, deps, (run, notes) => {
-    const task = runningTask(run, config);
-    const result = remoteDocker(run, ['exec', task, ...config.jobs.command, action, ...args]);
-    const lines = [...notes, ...splitLines(result.stdout)];
-    if (result.status !== 0) lines.push(`FAIL  jobs ${action} exited ${result.status}: ${firstLine(result.stderr)}`);
-    return { ok: result.status === 0, lines, data: { action, status: result.status } };
-  });
+    if (!declaresJobs(config))
+        throw refusal('STACK_JOBS_UNDECLARED', 'the environment declares no deploy.jobs block');
+    validate(action, args);
+    if (EFFECTS.has(action) && !execute) {
+        return { ok: true, lines: [`dry run: would run jobs ${[action, ...args].join(' ')} in the running ${config.jobs.service} task (add --execute)`], data: { action, status: null } };
+    }
+    return withRunner(ctx, config, { sshCmd }, deps, (run, notes) => {
+        const task = runningTask(run, config);
+        const result = remoteDocker(run, ['exec', task, ...config.jobs.command, action, ...args]);
+        const lines = [...notes, ...splitLines(result.stdout)];
+        if (result.status !== 0)
+            lines.push(`FAIL  jobs ${action} exited ${result.status}: ${firstLine(result.stderr)}`);
+        return { ok: result.status === 0, lines, data: { action, status: result.status } };
+    });
 }

package/stack/live-restore.d.mts

@@ -0,0 +1,35 @@
+import type { DeployConfig } from './config.mjs';
+import type { Report, StackContext } from './context.mjs';
+import type { Runner } from './remote.mjs';
+/** A running container with the facts needed to start it again in order. */
+export interface ContainerSnapshot {
+    name: string;
+    project: string;
+    service: string;
+    depends: string[];
+    restart: string;
+    image: string;
+}
+/** What the fix did. */
+export interface LiveRestoreOutcome {
+    changed: boolean;
+    back: string[];
+    missing: string[];
+    hostsDown: string[];
+    snapshot: ContainerSnapshot[];
+}
+/** Read-only check used by prepare (its dry run and --execute). */
+export declare function checkLiveRestore(run: Runner, report: Report): void;
+/** Every running container with the facts needed to start it again in order. */
+export declare function snapshotContainers(run: Runner): ContainerSnapshot[];
+/** Start order: infrastructure, applications, edge; inside a rank, compose dependencies first (stable, cycles keep their input order). */
+export declare function startOrder(snapshot: ContainerSnapshot[]): ContainerSnapshot[];
+/**
+ * The fix. Needs execute AND ownerApproved (else a dry-run line, or approvalMissing when execute is given without approval).
+ * Returns { changed, back, missing, hostsDown, snapshot }.
+ */
+export declare function fixLiveRestore(ctx: StackContext, run: Runner, report: Report, { execute, ownerApproved, config }: {
+    execute: boolean;
+    ownerApproved: boolean;
+    config: DeployConfig;
+}): Promise<LiveRestoreOutcome>;

package/stack/live-restore.mjs

@@ -1,14 +1,5 @@
-// live-restore.mjs - docker `live-restore` is incompatible with swarm mode. `prepare` detects it; with --fix-live-restore, --owner-approved and
-// --execute it backs up /etc/docker/daemon.json, sets live-restore false and restarts dockerd through passwordless sudo.
-// A dockerd restart stops every container (a restart policy does not reliably bring them back: on a real host 25 of 25 stayed down), so the fix
-//   1. snapshots every running container first (name, compose project and service, depends_on, image) and the answer of every public host,
-//   2. after the restart waits for docker, gives the restart policies a short grace, then `docker start`s every snapshotted container that is
-//      not running: databases and infrastructure first, applications next, edge proxies last, compose depends_on respected inside a group,
-//   3. re-checks the hosts against the baseline and FAILS loudly, listing every container and host still down.
-// Everything runs ON the manager through the ssh runner.
 import { approvalMissing } from './errors.mjs';
 import { remoteDocker, shellQuote } from './remote.mjs';
-
 const DAEMON_JSON = '/etc/docker/daemon.json';
 const WAIT_SECONDS = 180;
 const GRACE_SECONDS = 45;
@@ -17,161 +8,165 @@ const POLL_MS = 3000;
 const INFRA = /(?:postgres|postgis|pgbouncer|mysql|mariadb|mongo|redis|valkey|memcached|kafka|zookeeper|nats|rabbit|minio|elastic|opensearch|qdrant|clickhouse|cassandra|-db\b|_db\b|^db$|database)/iu;
 const EDGE = /(?:traefik|nginx|caddy|haproxy|envoy|edge|proxy)/iu;
 const FORMAT = '{{.Name}}|{{index .Config.Labels "com.docker.compose.project"}}|{{index .Config.Labels "com.docker.compose.service"}}|{{index .Config.Labels "com.docker.compose.depends_on"}}|{{.HostConfig.RestartPolicy.Name}}|{{.Config.Image}}';
-
 /** True when dockerd runs with live-restore (docker info reports it). */
 function liveRestoreEnabled(run) {
-  return remoteDocker(run, ['info', '--format', '{{.LiveRestoreEnabled}}']).stdout.trim() === 'true';
+    return remoteDocker(run, ['info', '--format', '{{.LiveRestoreEnabled}}']).stdout.trim() === 'true';
 }
-
-/** Read-only check used by prepare and doctor. */
+/** Read-only check used by prepare (its dry run and --execute). */
 export function checkLiveRestore(run, report) {
-  if (liveRestoreEnabled(run)) report.bad('docker live-restore is enabled and is incompatible with swarm: run "starci app stack prepare --fix-live-restore --owner-approved --execute"');
-  else report.ok('docker live-restore is off');
+    if (liveRestoreEnabled(run))
+        report.bad('docker live-restore is enabled and is incompatible with swarm: run "starci app stack prepare --env <env> --fix-live-restore --owner-approved --execute"');
+    else
+        report.ok('docker live-restore is off');
 }
-
 /** depends_on label: "db:service_started:false,cache:service_healthy:true" -> ['db', 'cache']. */
-const dependenciesOf = (label) => label.split(',').map((item) => item.split(':')[0].trim()).filter(Boolean);
-
+const dependenciesOf = (label) => label.split(',').map((item) => (item.split(':')[0] ?? '').trim()).filter(Boolean);
 function parseSnapshot(line) {
-  const [name, project = '', service = '', depends = '', restart = '', image = ''] = line.split('|');
-  return { name: name.replace(/^\//u, ''), project, service, depends: dependenciesOf(depends), restart, image };
+    const [name = '', project = '', service = '', depends = '', restart = '', image = ''] = line.split('|');
+    return { name: name.replace(/^\//u, ''), project, service, depends: dependenciesOf(depends), restart, image };
 }
-
 /** Every running container with the facts needed to start it again in order. */
 export function snapshotContainers(run) {
-  const names = remoteDocker(run, ['ps', '--format', '{{.Names}}']).stdout.split(/\r?\n/u).filter(Boolean);
-  return names.map((name) => parseSnapshot(remoteDocker(run, ['inspect', '--format', FORMAT, name]).stdout.trim()));
+    const names = remoteDocker(run, ['ps', '--format', '{{.Names}}']).stdout.split(/\r?\n/u).filter(Boolean);
+    return names.map((name) => parseSnapshot(remoteDocker(run, ['inspect', '--format', FORMAT, name]).stdout.trim()));
 }
-
 const rankOf = (entry) => {
-  if (INFRA.test(`${entry.service} ${entry.name} ${entry.image}`)) return 0;
-  return EDGE.test(`${entry.service} ${entry.name} ${entry.image}`) ? 2 : 1;
+    if (INFRA.test(`${entry.service} ${entry.name} ${entry.image}`))
+        return 0;
+    return EDGE.test(`${entry.service} ${entry.name} ${entry.image}`) ? 2 : 1;
 };
-
 /** Start order: infrastructure, applications, edge; inside a rank, compose dependencies first (stable, cycles keep their input order). */
 export function startOrder(snapshot) {
-  const ordered = [];
-  const placed = new Set();
-  const place = (entry, trail) => {
-    if (placed.has(entry.name) || trail.has(entry.name)) return;
-    const next = new Set(trail).add(entry.name);
-    const same = (other) => other.project === entry.project && entry.depends.includes(other.service) && rankOf(other) === rankOf(entry);
-    for (const dependency of snapshot.filter(same)) place(dependency, next);
-    placed.add(entry.name);
-    ordered.push(entry);
-  };
-  for (const rank of [0, 1, 2]) for (const entry of snapshot.filter((item) => rankOf(item) === rank)) place(entry, new Set());
-  return ordered;
+    const ordered = [];
+    const placed = new Set();
+    const place = (entry, trail) => {
+        if (placed.has(entry.name) || trail.has(entry.name))
+            return;
+        const next = new Set(trail).add(entry.name);
+        const same = (other) => other.project === entry.project && entry.depends.includes(other.service) && rankOf(other) === rankOf(entry);
+        for (const dependency of snapshot.filter(same))
+            place(dependency, next);
+        placed.add(entry.name);
+        ordered.push(entry);
+    };
+    for (const rank of [0, 1, 2])
+        for (const entry of snapshot.filter((item) => rankOf(item) === rank))
+            place(entry, new Set());
+    return ordered;
 }
-
 function rewrittenDaemonJson(run) {
-  const read = run(`sudo -n cat ${shellQuote(DAEMON_JSON)}`);
-  if (read.status !== 0) return { error: `cannot read ${DAEMON_JSON}: ${read.stderr.trim().split(/\r?\n/u)[0]}` };
-  let parsed;
-  try { parsed = JSON.parse(read.stdout.trim() || '{}'); } catch { return { error: `${DAEMON_JSON} is not valid JSON; fix it by hand` }; }
-  return { text: `${JSON.stringify({ ...parsed, 'live-restore': false }, null, 2)}\n` };
+    const read = run(`sudo -n cat ${shellQuote(DAEMON_JSON)}`);
+    if (read.status !== 0)
+        return { error: `cannot read ${DAEMON_JSON}: ${read.stderr.trim().split(/\r?\n/u)[0]}` };
+    let parsed;
+    try {
+        parsed = JSON.parse(read.stdout.trim() || '{}');
+    }
+    catch {
+        return { error: `${DAEMON_JSON} is not valid JSON; fix it by hand` };
+    }
+    return { text: `${JSON.stringify({ ...parsed, 'live-restore': false }, null, 2)}\n` };
 }
-
 const running = (run, name) => remoteDocker(run, ['inspect', '--format', '{{.State.Running}}', name]).stdout.trim() === 'true';
 const dockerUp = (run) => remoteDocker(run, ['info', '--format', '{{.ServerVersion}}']).status === 0;
 const notRunning = (run, names) => names.filter((name) => !running(run, name));
-
 async function waitUntil(ctx, seconds, done) {
-  const deadline = ctx.now() + seconds * 1000;
-  while (!done() && ctx.now() < deadline) await ctx.sleep(POLL_MS);
-  return done();
+    const deadline = ctx.now() + seconds * 1000;
+    while (!done() && ctx.now() < deadline)
+        await ctx.sleep(POLL_MS);
+    return done();
 }
-
 /** The hosts to prove, from the declared ingress: the verify list, else every upstream host (path /). */
 function hostsToProve(config) {
-  const ingress = config.ingress;
-  if (!ingress) return [];
-  const verify = ingress.verify.map((item) => ({ host: item.host, path: item.path }));
-  const seen = new Set(verify.map((item) => item.host));
-  const plain = ingress.upstreams.flatMap((entry) => entry.hosts).filter((host) => !seen.has(host) && !ingress.drop.includes(host));
-  return [...verify, ...[...new Set(plain)].map((host) => ({ host, path: '/' }))];
+    const ingress = config.ingress;
+    if (!ingress)
+        return [];
+    const verify = ingress.verify.map((item) => ({ host: item.host, path: item.path }));
+    const seen = new Set(verify.map((item) => item.host));
+    const plain = ingress.upstreams.flatMap((entry) => entry.hosts).filter((host) => !seen.has(host) && !ingress.drop.includes(host));
+    return [...verify, ...[...new Set(plain)].map((host) => ({ host, path: '/' }))];
 }
-
 const probeClass = (run, entry) => {
-  const target = `https://${entry.host}${entry.path}`;
-  const result = run(['curl', '-sS', '-o', '/dev/null', '--max-time', '10', '-w', '%{http_code}', '--resolve', shellQuote(`${entry.host}:443:127.0.0.1`), shellQuote(target)].join(' '));
-  const code = Number(result.stdout.trim());
-  return code >= 200 && code < 500 ? Math.floor(code / 100) : 0;
+    const target = `https://${entry.host}${entry.path}`;
+    const result = run(['curl', '-sS', '-o', '/dev/null', '--max-time', '10', '-w', '%{http_code}', '--resolve', shellQuote(`${entry.host}:443:127.0.0.1`), shellQuote(target)].join(' '));
+    const code = Number(result.stdout.trim());
+    return code >= 200 && code < 500 ? Math.floor(code / 100) : 0;
 };
-
 const baselineOf = (run, hosts) => new Map(hosts.map((entry) => [entry.host, probeClass(run, entry)]));
 const hostsDown = (run, hosts, baseline) => hosts.filter((entry) => baseline.get(entry.host) !== 0 && probeClass(run, entry) !== baseline.get(entry.host));
-
 function startMissing(run, report, order) {
-  const failed = [];
-  for (const entry of order.filter((item) => !running(run, item.name))) {
-    const started = remoteDocker(run, ['start', entry.name]);
-    if (started.status === 0) report.note(`started ${entry.name}`);
-    else {
-      failed.push(entry.name);
-      report.bad(`cannot start ${entry.name}: ${started.stderr.trim().split(/\r?\n/u)[0]}`);
+    const failed = [];
+    for (const entry of order.filter((item) => !running(run, item.name))) {
+        const started = remoteDocker(run, ['start', entry.name]);
+        if (started.status === 0)
+            report.note(`started ${entry.name}`);
+        else {
+            failed.push(entry.name);
+            report.bad(`cannot start ${entry.name}: ${started.stderr.trim().split(/\r?\n/u)[0]}`);
+        }
     }
-  }
-  return failed;
+    return failed;
 }
-
 async function recover(ctx, run, report, snapshot, hosts, baseline) {
-  const order = startOrder(snapshot);
-  const names = order.map((entry) => entry.name);
-  await waitUntil(ctx, WAIT_SECONDS, () => dockerUp(run));
-  await waitUntil(ctx, GRACE_SECONDS, () => notRunning(run, names).length === 0);
-  startMissing(run, report, order);
-  await waitUntil(ctx, WAIT_SECONDS, () => notRunning(run, names).length === 0);
-  const still = notRunning(run, names);
-  const back = names.filter((name) => !still.includes(name));
-  report.note(`containers back (${back.length} of ${names.length}): ${back.join(', ') || 'none'}`);
-  if (still.length > 0) report.bad(`CONTAINERS STILL DOWN after the restart: ${still.join(', ')}`);
-  await waitUntil(ctx, HOST_WAIT_SECONDS, () => hostsDown(run, hosts, baseline).length === 0);
-  const down = hostsDown(run, hosts, baseline).map((entry) => entry.host);
-  if (down.length > 0) report.bad(`PUBLIC HOSTS STILL DOWN against the pre-restart baseline: ${down.join(', ')}`);
-  else if (hosts.length > 0) report.ok(`all ${hosts.length} public host(s) answer as before the restart`);
-  return { back, missing: still, hostsDown: down };
+    const order = startOrder(snapshot);
+    const names = order.map((entry) => entry.name);
+    await waitUntil(ctx, WAIT_SECONDS, () => dockerUp(run));
+    await waitUntil(ctx, GRACE_SECONDS, () => notRunning(run, names).length === 0);
+    startMissing(run, report, order);
+    await waitUntil(ctx, WAIT_SECONDS, () => notRunning(run, names).length === 0);
+    const still = notRunning(run, names);
+    const back = names.filter((name) => !still.includes(name));
+    report.note(`containers back (${back.length} of ${names.length}): ${back.join(', ') || 'none'}`);
+    if (still.length > 0)
+        report.bad(`CONTAINERS STILL DOWN after the restart: ${still.join(', ')}`);
+    await waitUntil(ctx, HOST_WAIT_SECONDS, () => hostsDown(run, hosts, baseline).length === 0);
+    const down = hostsDown(run, hosts, baseline).map((entry) => entry.host);
+    if (down.length > 0)
+        report.bad(`PUBLIC HOSTS STILL DOWN against the pre-restart baseline: ${down.join(', ')}`);
+    else if (hosts.length > 0)
+        report.ok(`all ${hosts.length} public host(s) answer as before the restart`);
+    return { back, missing: still, hostsDown: down };
 }
-
 /**
  * The fix. Needs execute AND ownerApproved (else a dry-run line, or approvalMissing when execute is given without approval).
  * Returns { changed, back, missing, hostsDown, snapshot }.
  */
 export async function fixLiveRestore(ctx, run, report, { execute, ownerApproved, config }) {
-  const idle = { changed: false, back: [], missing: [], hostsDown: [], snapshot: [] };
-  if (!liveRestoreEnabled(run)) {
-    report.note('docker live-restore is already off: nothing to change');
-    return idle;
-  }
-  if (!execute) {
-    report.note('dry run: would snapshot the running containers and the public hosts, back up daemon.json, set live-restore false, restart docker, start every container that does not come back (databases, applications, edge) and re-check the hosts (needs --execute and --owner-approved)');
-    return idle;
-  }
-  if (!ownerApproved) throw approvalMissing('STACK_LIVE_RESTORE_UNAPPROVED', 'restarting dockerd restarts every container; pass --owner-approved after the owner said yes');
-  const rewritten = rewrittenDaemonJson(run);
-  if (rewritten.error) {
-    report.bad(rewritten.error);
-    return idle;
-  }
-  const snapshot = snapshotContainers(run);
-  const hosts = hostsToProve(config);
-  const baseline = baselineOf(run, hosts);
-  report.note(`snapshot: ${snapshot.length} running container(s), ${hosts.length} public host(s) in the baseline`);
-  const stamp = new Date(ctx.now()).toISOString().replaceAll(/[-:.]/gu, '').slice(0, 15);
-  const backup = `${DAEMON_JSON}.bak-${stamp}`;
-  const steps = [
-    [`sudo -n cp -p ${shellQuote(DAEMON_JSON)} ${shellQuote(backup)}`, undefined],
-    [`sudo -n tee ${shellQuote(DAEMON_JSON)} >/dev/null`, rewritten.text],
-    ['sudo -n systemctl restart docker', undefined],
-  ];
-  for (const [command, input] of steps) {
-    const done = run(command, input === undefined ? {} : { input });
-    if (done.status !== 0) {
-      report.bad(`live-restore fix stopped at "${command.split(' ').slice(0, 4).join(' ')}": ${done.stderr.trim().split(/\r?\n/u)[0]} (daemon.json backup: ${backup})`);
-      return { ...idle, snapshot };
+    const idle = { changed: false, back: [], missing: [], hostsDown: [], snapshot: [] };
+    if (!liveRestoreEnabled(run)) {
+        report.note('docker live-restore is already off: nothing to change');
+        return idle;
+    }
+    if (!execute) {
+        report.note('dry run: would snapshot the running containers and the public hosts, back up daemon.json, set live-restore false, restart docker, start every container that does not come back (databases, applications, edge) and re-check the hosts (needs --execute and --owner-approved)');
+        return idle;
+    }
+    if (!ownerApproved)
+        throw approvalMissing('STACK_LIVE_RESTORE_UNAPPROVED', 'restarting dockerd restarts every container; pass --owner-approved after the owner said yes');
+    const rewritten = rewrittenDaemonJson(run);
+    if (rewritten.text === undefined) {
+        report.bad(rewritten.error);
+        return idle;
+    }
+    const snapshot = snapshotContainers(run);
+    const hosts = hostsToProve(config);
+    const baseline = baselineOf(run, hosts);
+    report.note(`snapshot: ${snapshot.length} running container(s), ${hosts.length} public host(s) in the baseline`);
+    const stamp = new Date(ctx.now()).toISOString().replaceAll(/[-:.]/gu, '').slice(0, 15);
+    const backup = `${DAEMON_JSON}.bak-${stamp}`;
+    const steps = [
+        [`sudo -n cp -p ${shellQuote(DAEMON_JSON)} ${shellQuote(backup)}`, undefined],
+        [`sudo -n tee ${shellQuote(DAEMON_JSON)} >/dev/null`, rewritten.text],
+        ['sudo -n systemctl restart docker', undefined],
+    ];
+    for (const [command, input] of steps) {
+        const done = run(command, input === undefined ? {} : { input });
+        if (done.status !== 0) {
+            report.bad(`live-restore fix stopped at "${command.split(' ').slice(0, 4).join(' ')}": ${done.stderr.trim().split(/\r?\n/u)[0]} (daemon.json backup: ${backup})`);
+            return { ...idle, snapshot };
+        }
     }
-  }
-  report.note(`daemon.json backed up to ${backup}; live-restore set to false; docker restarted`);
-  const outcome = await recover(ctx, run, report, snapshot, hosts, baseline);
-  return { changed: true, snapshot, ...outcome };
+    report.note(`daemon.json backed up to ${backup}; live-restore set to false; docker restarted`);
+    const outcome = await recover(ctx, run, report, snapshot, hosts, baseline);
+    return { changed: true, snapshot, ...outcome };
 }

package/stack/post-deploy.d.mts

@@ -0,0 +1,8 @@
+import type { DeployConfig } from './config.mjs';
+import type { StackContext } from './context.mjs';
+import type { ActionResult } from './remote.mjs';
+/**
+ * Runs the declared post-deploy tasks in order and stops at the first failure.
+ * Returns { ok, lines, data: { tasks: [{ name, ok }] } }; the lines read `PASS|FAIL  post-deploy <name>  detail (N output lines withheld)`.
+ */
+export declare function postDeployStack(ctx: StackContext, config: DeployConfig): Promise<ActionResult>;

package/stack/post-deploy.mjs

@@ -6,43 +6,43 @@
 // CI log, so stdout and stderr are never relayed (`starci app stack task <name>` is the operator verb that shows them).
 import { findInOrder } from '../runtime/scripts/lib/in-order.mjs';
 import { lines as splitLines, tool } from './context.mjs';
-
 /** The running container of a service on this daemon, or undefined. */
 function runningContainer(ctx, config, service) {
-  const found = tool(ctx, 'docker', ['ps', '-q', '--filter', `label=com.docker.swarm.service.name=${config.stack}_${service}`, '--filter', 'status=running']);
-  return splitLines(found.stdout)[0];
+    const found = tool(ctx, 'docker', ['ps', '-q', '--filter', `label=com.docker.swarm.service.name=${config.stack}_${service}`, '--filter', 'status=running']);
+    return splitLines(found.stdout)[0];
 }
-
 /** The verdict of one finished `docker exec`: a timeout, a launch failure or a non-zero exit is a failure, whatever it printed. */
 function verdictOf(run, timeoutSeconds) {
-  if (run.error?.code === 'ETIMEDOUT') return { ok: false, detail: `timed out after ${timeoutSeconds}s` };
-  if (run.error) return { ok: false, detail: 'could not run' };
-  if (run.status === null || run.status === undefined) return { ok: false, detail: 'ended without an exit status' };
-  return run.status === 0 ? { ok: true, detail: 'ok' } : { ok: false, detail: `exit ${run.status}` };
+    if (run.error?.code === 'ETIMEDOUT')
+        return { ok: false, detail: `timed out after ${timeoutSeconds}s` };
+    if (run.error)
+        return { ok: false, detail: 'could not run' };
+    if (run.status === null || run.status === undefined)
+        return { ok: false, detail: 'ended without an exit status' };
+    return run.status === 0 ? { ok: true, detail: 'ok' } : { ok: false, detail: `exit ${run.status}` };
 }
-
 /** Runs one declared task; returns { name, ok, detail, outputLines }. */
 function runTask(ctx, config, name) {
-  const declared = config.tasks[name];
-  const container = runningContainer(ctx, config, declared.service);
-  if (!container) return { name, ok: false, detail: `no running container of ${config.stack}_${declared.service} on the daemon`, outputLines: 0 };
-  const run = tool(ctx, 'docker', ['exec', container, ...declared.command], { timeout: declared.timeoutSeconds * 1000, killSignal: 'SIGKILL' });
-  return { name, ...verdictOf(run, declared.timeoutSeconds), outputLines: splitLines(run.stdout).length + splitLines(run.stderr).length };
+    const declared = config.tasks[name]; // every postDeploy name is a declared task (stack-deploy-config.mjs)
+    const container = runningContainer(ctx, config, declared.service);
+    if (!container)
+        return { name, ok: false, detail: `no running container of ${config.stack}_${declared.service} on the daemon`, outputLines: 0 };
+    const run = tool(ctx, 'docker', ['exec', container, ...declared.command], { timeout: declared.timeoutSeconds * 1000, killSignal: 'SIGKILL' });
+    return { name, ...verdictOf(run, declared.timeoutSeconds), outputLines: splitLines(run.stdout).length + splitLines(run.stderr).length };
 }
-
 /**
  * Runs the declared post-deploy tasks in order and stops at the first failure.
  * Returns { ok, lines, data: { tasks: [{ name, ok }] } }; the lines read `PASS|FAIL  post-deploy <name>  detail (N output lines withheld)`.
  */
 export async function postDeployStack(ctx, config) {
-  const results = [];
-  await findInOrder(config.postDeploy, async (name) => {
-    results.push(runTask(ctx, config, name));
-    return !results.at(-1).ok;
-  });
-  return {
-    ok: results.every((entry) => entry.ok),
-    lines: results.map((entry) => `${entry.ok ? 'PASS' : 'FAIL'}  post-deploy ${entry.name}  ${entry.detail} (${entry.outputLines} output line${entry.outputLines === 1 ? '' : 's'} withheld)`),
-    data: { tasks: results.map(({ name, ok }) => ({ name, ok })) },
-  };
+    const results = [];
+    await findInOrder(config.postDeploy, async (name) => {
+        results.push(runTask(ctx, config, name));
+        return !results.at(-1)?.ok;
+    });
+    return {
+        ok: results.every((entry) => entry.ok),
+        lines: results.map((entry) => `${entry.ok ? 'PASS' : 'FAIL'}  post-deploy ${entry.name}  ${entry.detail} (${entry.outputLines} output line${entry.outputLines === 1 ? '' : 's'} withheld)`),
+        data: { tasks: results.map(({ name, ok }) => ({ name, ok })) },
+    };
 }

package/stack/prepare.d.mts

@@ -0,0 +1,24 @@
+import type { DeployConfig } from './config.mjs';
+import type { Report, StackContext } from './context.mjs';
+import type { ActionResult, Runner, RunnerDeps } from './remote.mjs';
+/** What `prepare` takes. */
+export interface PrepareOptions {
+    execute?: boolean;
+    initSwarm?: boolean;
+    fixLiveRestore?: boolean;
+    ownerApproved?: boolean;
+    sshCmd?: string | null;
+}
+/** The `current` and `previous` release pointers (null for an absent link). */
+export interface ReleasePointers {
+    current: string | null;
+    previous: string | null;
+}
+/** prepare: read-only checks; with execute also the missing volumes and (initSwarm, approved) the one-time swarm init. */
+export declare function prepareHost(ctx: StackContext, config: DeployConfig, { execute, initSwarm: wantInit, fixLiveRestore: wantFix, ownerApproved, sshCmd }?: PrepareOptions, deps?: RunnerDeps): Promise<ActionResult>;
+/** The release pointers (`current`, `previous`) of the stack's releases directory: {current, previous} (null for an absent link); an unreadable pointer is reported bad, never read as absent. */
+export declare function checkPointers(run: Runner, report: Report, config: DeployConfig): ReleasePointers;
+/** The dry run of prepare: the prepare checks plus services, managed secrets and configs and the release pointers. Changes nothing. */
+export declare function doctorHost(ctx: StackContext, config: DeployConfig, { sshCmd }?: {
+    sshCmd?: string | null;
+}, deps?: RunnerDeps): Promise<ActionResult>;