@starci/hfs 4.2.17 → 4.4.0

package/stack/image-scan.mjs

@@ -0,0 +1,152 @@
+// image-scan.mjs - the guard before a PUBLIC image is pushed: layers of a public package are public, so the built image is scanned for secret material
+// and the push is refused on any hit. Three sources, all read from the local docker daemon (no registry, no credentials):
+//   history      the build instructions (docker history): secret patterns, and build arguments that are not on the public build-args list
+//   environment  the baked ENV: secret patterns, and non-empty values of variables named like a secret
+//   filesystem   the exported container filesystem (a tar, read by header): files named like secret material outside the library directories, and the
+//                content of small text files for secret patterns
+// A hit names the rule and the place (a path, a history line, a variable NAME), never the matched text.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { firstLine, lines as splitLines, tool } from './context.mjs';
+
+const SECRET_PATTERNS = [
+  ['a private key block', /-----BEGIN (?:[A-Z]+ )*PRIVATE KEY-----/u],
+  ['an age secret key', /AGE-SECRET-KEY-1[0-9A-Z]{20,}/u],
+  ['a sealed secret envelope', /ENC\[AES256_GCM,/u],
+  ['sops metadata', /"sops"\s*:\s*\{|^sops:\s*$/mu],
+  ['a GitHub token', /\bgh[pousr]_[A-Za-z0-9]{30,}\b/u],
+  ['an AWS access key id', /\bAKIA[0-9A-Z]{16}\b/u],
+  ['a Slack token', /\bxox[abprs]-[A-Za-z0-9-]{10,}/u],
+  ['a live API key', /\b(?:sk|rk)_live_[A-Za-z0-9]{16,}/u],
+];
+const SECRET_NAME = /(?:SECRET|TOKEN|PASSWORD|PASSWD|PRIVATE|CREDENTIAL|API_?KEY|ACCESS_?KEY)/iu;
+const SECRET_FILE = /(?:^|\/)(?:\.env(?!\.(?:example|sample|template)$)[^/]*|secrets?\.env|id_(?:rsa|dsa|ecdsa|ed25519)|[^/]+\.(?:pem|key|p12|pfx|jks|enc)|\.npmrc|\.netrc|\.git-credentials|\.sops\.yaml|keys\.txt)$/u;
+/** Directories whose files are libraries or the base system, not the app: test certificates and sample keys live there by design. */
+const LIBRARY_DIR = /^(?:usr\/(?:share|lib|local\/lib|include)|etc\/(?:ssl|ca-certificates|pki)|lib|var\/lib|opt\/yarn|root\/\.(?:npm|cache)|home\/[^/]+\/\.(?:npm|cache)|(?:[^/]+\/)*node_modules)(?:\/|$)/u;
+const TEXT_FILE = /\.(?:[cm]?js|ts|json|ya?ml|env|txt|conf|cfg|ini|sh|toml|properties|md)$/u;
+const MAX_TEXT_BYTES = 128 * 1024;
+const BLOCK = 512;
+const BUILD_ARG_PREFIX = /^\|\d+ ((?:[A-Za-z_][A-Za-z0-9_]*=\S*\s?)+)/u;
+const ARG_INSTRUCTION = /\bARG\s+([A-Za-z_][A-Za-z0-9_]*)/gu;
+
+const cleanName = (name) => name.replace(/^\.?\//u, '').replace(/\/$/u, '');
+const secretPatternIn = (text) => SECRET_PATTERNS.find(([, pattern]) => pattern.test(text))?.[0] ?? null;
+
+/** Reads one NUL-terminated field of a tar header. */
+const field = (header, from, length) => header.toString('utf8', from, from + length).replace(/\0.*$/su, '');
+
+function sizeOf(header) {
+  if (header[124] & 0x80) {
+    let value = 0;
+    for (let index = 125; index < 136; index += 1) value = (value * 256) + header[index];
+    return value;
+  }
+  return Number.parseInt(field(header, 124, 12).trim() || '0', 8);
+}
+
+const paxPath = (body) => body.toString('utf8').split('\n').map((record) => /^\d+ path=(.*)$/u.exec(record)).find(Boolean)?.[1] ?? null;
+
+/**
+ * Walks a tar file by header: calls visit({ name, type, size, read }) for every entry (`read()` returns the content, called only for files worth
+ * reading). Handles ustar prefixes, GNU long names (L) and PAX paths (x). Content is never loaded unless the visitor asks for it.
+ */
+export function walkTar(file, visit) {
+  const fd = fs.openSync(file, 'r');
+  try {
+    const header = Buffer.alloc(BLOCK);
+    let position = 0;
+    let pending = null;
+    for (;;) {
+      if (fs.readSync(fd, header, 0, BLOCK, position) < BLOCK || header.every((byte) => byte === 0)) return;
+      const size = sizeOf(header);
+      const type = String.fromCharCode(header[156] || 48);
+      const prefix = field(header, 345, 155);
+      const own = field(header, 0, 100);
+      const contentAt = position + BLOCK;
+      const read = () => { const body = Buffer.alloc(size); fs.readSync(fd, body, 0, size, contentAt); return body; };
+      if (type === 'L' || type === 'x') {
+        const body = read();
+        pending = type === 'L' ? body.toString('utf8').replace(/\0.*$/su, '') : paxPath(body) ?? pending;
+      } else {
+        visit({ name: cleanName(pending ?? (prefix ? `${prefix}/${own}` : own)), type, size, read });
+        pending = null;
+      }
+      position = contentAt + (Math.ceil(size / BLOCK) * BLOCK);
+    }
+  } finally {
+    fs.closeSync(fd);
+  }
+}
+
+function historyHits(historyText, allowedArgs) {
+  const hits = [];
+  splitLines(historyText).forEach((line, index) => {
+    const found = secretPatternIn(line);
+    if (found) hits.push({ kind: 'history', where: `history line ${index + 1}`, why: found });
+    const names = [...(BUILD_ARG_PREFIX.exec(line)?.[1].trim().split(/\s+/u).map((pair) => pair.split('=')[0]) ?? []), ...[...line.matchAll(ARG_INSTRUCTION)].map((match) => match[1])];
+    for (const name of new Set(names)) if (!allowedArgs.has(name)) hits.push({ kind: 'build-arg', where: `history line ${index + 1}`, why: `build argument ${name} is not on the public build-args list` });
+  });
+  return hits;
+}
+
+function environmentHits(envJson) {
+  let values = [];
+  try { values = JSON.parse(envJson); } catch { values = []; }
+  return (Array.isArray(values) ? values : []).flatMap((entry) => {
+    const at = String(entry).indexOf('=');
+    const name = String(entry).slice(0, at);
+    const value = String(entry).slice(at + 1);
+    const found = secretPatternIn(value);
+    if (found) return [{ kind: 'env', where: `environment variable ${name}`, why: found }];
+    return SECRET_NAME.test(name) && value !== '' ? [{ kind: 'env', where: `environment variable ${name}`, why: 'a variable named like a secret holds a value' }] : [];
+  });
+}
+
+function fileHits(tarFile) {
+  const hits = [];
+  walkTar(tarFile, ({ name, type, size, read }) => {
+    if (!['0', '7'].includes(type) || LIBRARY_DIR.test(name)) return;
+    if (SECRET_FILE.test(name)) hits.push({ kind: 'file', where: name, why: 'a file named like secret material' });
+    else if (TEXT_FILE.test(name) && size > 0 && size <= MAX_TEXT_BYTES) {
+      const found = secretPatternIn(read().toString('utf8'));
+      if (found) hits.push({ kind: 'content', where: name, why: found });
+    }
+  });
+  return hits;
+}
+
+/** The filesystem of the image as a tar file (create, export to a file, remove), or an error line. */
+function exportFilesystem(ctx, ref, directory) {
+  const created = tool(ctx, 'docker', ['create', ref]);
+  const id = firstLine(created.stdout);
+  if (created.status !== 0 || id === '') return { error: `docker create failed: ${firstLine(created.stderr)}` };
+  const tarFile = path.join(directory, 'filesystem.tar');
+  const exported = tool(ctx, 'docker', ['export', '-o', tarFile, id]);
+  tool(ctx, 'docker', ['rm', '-f', id]);
+  return exported.status === 0 ? { tarFile } : { error: `docker export failed: ${firstLine(exported.stderr)}` };
+}
+
+/**
+ * Scans a built image. `allowedArgs` = the names of the public build arguments that were passed. Returns { ok, hits, errors }:
+ * errors are scan steps that could not run (the push is refused too: an unscanned public image is not pushed).
+ */
+export function scanImage(ctx, ref, { allowedArgs }) {
+  const hits = [];
+  const errors = [];
+  const history = tool(ctx, 'docker', ['history', '--no-trunc', '--format', '{{.CreatedBy}}', ref]);
+  if (history.status === 0) hits.push(...historyHits(history.stdout, new Set(allowedArgs)));
+  else errors.push(`docker history failed: ${firstLine(history.stderr)}`);
+  const environment = tool(ctx, 'docker', ['inspect', '--format', '{{json .Config.Env}}', ref]);
+  if (environment.status === 0) hits.push(...environmentHits(environment.stdout));
+  else errors.push(`docker inspect failed: ${firstLine(environment.stderr)}`);
+  const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'stack-scan-'));
+  try {
+    const exported = exportFilesystem(ctx, ref, directory);
+    if (exported.error) errors.push(exported.error);
+    else hits.push(...fileHits(exported.tarFile));
+  } finally {
+    fs.rmSync(directory, { recursive: true, force: true });
+  }
+  return { ok: hits.length === 0 && errors.length === 0, hits, errors };
+}

package/stack/ingress-create.mjs

@@ -0,0 +1,114 @@
+// ingress-create.mjs - `starci app stack ingress create`: deploys the SHARED ingress as its own swarm stack with ZERO replicas, so the service exists
+// (rehearse and takeover need it) while nothing binds a port and the live edge is untouched. The stack is rendered from the deploy block:
+// the pinned image, the whole legacy dynamic directory bind-mounted read-only, the persisted certificates volume, the attachable overlay (external),
+// host-mode 80/443 (they bind only when a task starts) and no ACME email. The static arguments are the declared `ingress.args` or, byte for byte, the
+// ones the live edge container runs with (read-only inspect), so the new proxy behaves exactly like the old one. It refuses to start a task.
+import { refusal } from './errors.mjs';
+import { remoteDocker, shellQuote } from './remote.mjs';
+
+const ACME_EMAIL = /acme\.email=(.+)$/iu;
+const SERVICE_NETWORK = 'edge';
+
+const parse = (text) => { try { return JSON.parse(text); } catch { return null; } };
+const asArray = (value) => (Array.isArray(value) ? value : []);
+
+/** The static arguments and the two mount targets of the live edge container (inspect formats read Args and Mounts only, never the environment). */
+function readLegacy(run, ingress) {
+  const container = ingress.legacy.container;
+  const args = parse(remoteDocker(run, ['inspect', '--format', '{{json .Args}}', container]).stdout);
+  const mounts = asArray(parse(remoteDocker(run, ['inspect', '--format', '{{json .Mounts}}', container]).stdout));
+  const target = (match) => mounts.find(match)?.Destination;
+  return {
+    args: ingress.args ?? asArray(args).filter((arg) => typeof arg === 'string'),
+    dynamicTarget: target((mount) => mount.Source === ingress.legacy.dynamicDir),
+    acmeTarget: target((mount) => mount.Name === ingress.acmeVolume),
+  };
+}
+
+/** The stack model (compose spec as JSON, which is valid YAML). */
+export function stackModel(ingress, { args, dynamicTarget, acmeTarget }) {
+  const ports = [80, 443].map((port) => ({ target: port, published: port, protocol: 'tcp', mode: 'host' }));
+  return {
+    services: {
+      [ingress.service]: {
+        image: ingress.image,
+        command: args,
+        ports,
+        volumes: [
+          { type: 'bind', source: ingress.legacy.dynamicDir, target: dynamicTarget, read_only: true },
+          { type: 'volume', source: 'acme', target: acmeTarget },
+        ],
+        networks: [SERVICE_NETWORK],
+        logging: { driver: 'json-file', options: { 'max-size': '10m', 'max-file': '5' } },
+        deploy: {
+          replicas: 0,
+          placement: { constraints: ['node.role == manager'] },
+          update_config: { order: 'stop-first', failure_action: 'rollback' },
+          restart_policy: { condition: 'any' },
+        },
+      },
+    },
+    networks: { [SERVICE_NETWORK]: { external: true, name: ingress.network } },
+    volumes: { acme: { external: true, name: ingress.acmeVolume } },
+  };
+}
+
+/** The guard that no task can start from this model. */
+export function refuseRunning(model) {
+  for (const [name, service] of Object.entries(model.services)) {
+    if (service.deploy?.replicas !== 0) throw refusal('STACK_INGRESS_WOULD_START', `the ingress service ${name} would start a task; create deploys it with replicas 0 only`);
+  }
+}
+
+function emailProblem(args) {
+  const configured = args.map((arg) => ACME_EMAIL.exec(arg)).find((match) => match !== null);
+  return configured ? 'the arguments set an ACME email; a new email registers a new ACME account: leave it unset at takeover' : null;
+}
+
+const swarmActive = (run) => remoteDocker(run, ['info', '--format', '{{.Swarm.LocalNodeState}} {{.Swarm.ControlAvailable}}']).stdout.trim() === 'active true';
+const overlayAttachable = (run, network) => remoteDocker(run, ['network', 'inspect', '--format', '{{.Driver}} {{.Attachable}}', network]).stdout.trim() === 'overlay true';
+const volumePresent = (run, volume) => remoteDocker(run, ['volume', 'inspect', volume]).status === 0;
+
+/** Replica state of the service: { exists, desired }. */
+function serviceState(run, name) {
+  const listed = remoteDocker(run, ['service', 'ls', '--filter', `name=${name}`, '--format', '{{.Name}} {{.Replicas}}']);
+  const row = String(listed.stdout).split('\n').map((line) => line.trim().split(' ')).find(([found]) => found === name);
+  if (listed.status !== 0 || !row) return { exists: false, desired: 0 };
+  return { exists: true, desired: Number(String(row[1]).split('/')[1]) };
+}
+
+function checks(run, ingress, legacy, service) {
+  const directory = run(`test -d ${shellQuote(ingress.legacy.dynamicDir)}`).status === 0;
+  return [
+    ['an image is declared (deploy.ingress.image, pinned by digest)', typeof ingress.image === 'string', 'declare deploy.ingress.image'],
+    ['static arguments known (declared, or read from the live edge)', legacy.args.length > 0, `cannot read the arguments of ${ingress.legacy.container}`],
+    ['no ACME email configured', emailProblem(legacy.args) === null, emailProblem(legacy.args)],
+    ['dynamic directory and certificates mounts found on the live edge', typeof legacy.dynamicTarget === 'string' && typeof legacy.acmeTarget === 'string', 'the live edge container does not mount the dynamic directory and the certificates volume'],
+    ['swarm active (manager)', swarmActive(run), 'this host is not an active swarm manager'],
+    [`network ${ingress.network} is an attachable overlay`, overlayAttachable(run, ingress.network), 'run "starci app stack ingress attach --execute" first'],
+    [`certificates volume ${ingress.acmeVolume} present`, volumePresent(run, ingress.acmeVolume), 'the persisted certificates volume is missing'],
+    [`dynamic directory ${ingress.legacy.dynamicDir} present`, directory, 'the dynamic directory is missing'],
+    [`service ${ingress.name}_${ingress.service} is not running a task`, service.desired === 0, `it already runs ${service.desired} replica(s): use takeover or scale it by hand`],
+  ].map(([name, ok, detail]) => ({ name, ok, detail }));
+}
+
+/** `ingress create`: dry run unless execute. Returns { ok, lines, data }. */
+export function createIngress(run, ingress, execute) {
+  const legacy = readLegacy(run, ingress);
+  const stack = `${ingress.name}_${ingress.service}`;
+  const results = checks(run, ingress, legacy, serviceState(run, stack));
+  const lines = results.map((item) => (item.ok ? `precondition ok: ${item.name}` : `precondition FAILED: ${item.name} (${item.detail})`));
+  if (results.some((item) => !item.ok)) return { ok: false, lines: [...lines, 'create refused: nothing was changed'], data: { steps: [] } };
+  const model = stackModel(ingress, legacy);
+  refuseRunning(model);
+  const summary = `stack ${ingress.name}: service ${ingress.service} image ${ingress.image}, ${legacy.args.length} static argument(s), host-mode 80/443, replicas 0`;
+  if (!execute) return { ok: true, lines: [...lines, `dry run: would deploy ${summary}`], data: { steps: [] } };
+  const deployed = run(['docker', ...['stack', 'deploy', '--detach=true', '-c', '-', ingress.name].map(shellQuote)].join(' '), { input: JSON.stringify(model) });
+  if (deployed.status !== 0) return { ok: false, lines: [...lines, `docker stack deploy failed: ${String(deployed.stderr).trim().split('\n')[0]}`], data: { steps: [{ step: 'deploy', ok: false }] } };
+  const tasks = remoteDocker(run, ['service', 'ps', stack, '--filter', 'desired-state=running', '-q']).stdout.trim();
+  if (tasks !== '') {
+    remoteDocker(run, ['service', 'scale', `${stack}=0`]);
+    return { ok: false, lines: [...lines, 'REFUSED: a task of the ingress service started; it was scaled back to 0'], data: { steps: [{ step: 'deploy', ok: true }, { step: 'no-task', ok: false }] } };
+  }
+  return { ok: true, lines: [...lines, `deployed ${summary}; no task runs and no port is bound`], data: { steps: [{ step: 'deploy', ok: true }, { step: 'no-task', ok: true }] } };
+}

package/stack/ingress-publish.mjs

@@ -0,0 +1,273 @@
+// ingress-publish.mjs - `starci app stack ingress publish`: puts the app's own routes file (deploy.ingress.routes) into the dynamic directory the
+// shared ingress serves, under `publish.file`, and retires the old generic file(s) listed in `publish.supersedes`. The directory is shared with
+// tenants that write their own files: those files are NEVER touched; the only remote writes and removals are `publish.file`, its `.tmp` upload
+// name and the superseded names.
+//   Preconditions (all or nothing): the routes file parses, no router or service name clashes with a tenant file, the ingress service runs, the old
+//   edge does not, every service of the app stack is healthy, the age recipient and the backup directory exist.
+//   --execute: baseline of every host -> age backup of each superseded file AND of the current publish.file when it exists -> upload as <file>.tmp (ignored by the file provider because of its
+//   extension) -> ONE remote command swaps it in and removes the superseded files -> every host is verified (declared status exact, otherwise the
+//   baseline status class; a host that did not answer before is new: it must answer 2xx-4xx over a certificate valid for its name, whatever the
+//   default certificate was; certificates of hosts that answered before unchanged) -> any failure puts the superseded files AND the previous
+//   publish.file back from memory (publish.file is removed only when there was none) and re-verifies the baseline (RESTORED ok / RESTORE FAILED (manual: ...)).
+// Route files may carry header values or tokens: they are held in memory only and NEVER printed; lines carry names, hosts and statuses.
+import fs from 'node:fs';
+import path from 'node:path';
+import { isPlainObject as plain } from '../runtime/engine/plain-object.mjs';
+import { parseYaml } from '../runtime/engine/yaml.mjs';
+import { firstLine } from './context.mjs';
+import { refusal } from './errors.mjs';
+import { remoteDocker, shellQuote } from './remote.mjs';
+
+const FILE_MARK = /^#@@FILE (.+)$/mu;
+const HEALTHY = /^([1-9]\d*)\/\1$/u;
+const VERIFY_ATTEMPTS = 6;
+const VERIFY_MS = 2000;
+
+const asMap = (value) => (plain(value) ? value : {});
+const classOf = (code) => Math.floor(code / 100);
+const answers = (code) => code >= 200 && code < 500;
+const label = (target) => `${target.host}${target.path}`;
+const result = (ok, lines, steps, extra = {}) => ({ ok, lines, data: { ok, steps, ...extra } });
+const quote = (value) => shellQuote(value);
+const shScript = (script) => `sh -c ${quote(script)}`;
+
+/** The app's own routes file: { text, routers, services } or { error }. */
+function readOwn(config, ingress) {
+  let text = '';
+  let doc = null;
+  try {
+    text = fs.readFileSync(path.join(config.envDir, ingress.routes), 'utf8');
+    doc = parseYaml(text);
+  } catch {
+    return { error: `${ingress.routes} cannot be read as YAML` };
+  }
+  const http = doc?.http;
+  if (!plain(http) || !plain(http.routers) || !plain(http.services)) return { error: `${ingress.routes} needs http.routers and http.services mappings` };
+  return { text, routers: Object.keys(http.routers), services: Object.keys(http.services) };
+}
+
+/** The route files of the live directory: Map name -> text (listing only; the contents are never printed). */
+function readFiles(run, kit, dir) {
+  const listed = run(kit.liveCommand(dir));
+  if (listed.status !== 0) return { error: `cannot read ${dir} on the manager: ${firstLine(listed.stderr)}` };
+  const parts = String(listed.stdout).split(FILE_MARK);
+  const files = new Map();
+  for (let index = 1; index < parts.length; index += 2) files.set(parts[index], parts[index + 1]);
+  return { files };
+}
+
+/** Router and service names of the other files that the own file would collide with (superseded files and the target are ignored). */
+function clashes(files, own, ignore) {
+  const found = [];
+  const unreadable = [];
+  for (const [name, body] of files) {
+    if (ignore.has(name)) continue;
+    let doc = null;
+    try { doc = parseYaml(body); } catch { unreadable.push(name); continue; }
+    const http = asMap(asMap(doc).http);
+    for (const router of Object.keys(asMap(http.routers))) if (own.routers.includes(router)) found.push(`router ${router} (${name})`);
+    for (const service of Object.keys(asMap(http.services))) if (own.services.includes(service)) found.push(`service ${service} (${name})`);
+  }
+  return { found, unreadable };
+}
+
+/** Every service of the app stack is N/N with N >= 1. */
+function stackHealth(run, stack) {
+  const listed = remoteDocker(run, ['stack', 'services', stack, '--format', '{{.Name}} {{.Replicas}}']);
+  const rows = String(listed.stdout).split('\n').map((line) => line.trim()).filter(Boolean).map((line) => line.split(' '));
+  const unhealthy = rows.filter(([, replicas]) => !HEALTHY.test(replicas ?? '')).map(([name]) => name);
+  return { ok: listed.status === 0 && rows.length > 0 && unhealthy.length === 0, unhealthy, count: rows.length };
+}
+
+function recipientOf(kit, config) {
+  try {
+    return { recipient: kit.readRecipient(config.repoRoot) };
+  } catch (error) {
+    return { error: error.message };
+  }
+}
+
+const legacyRunning = (run, ingress) => String(remoteDocker(run, ['inspect', '--format', '{{.State.Running}}', ingress.legacy.container]).stdout).trim() === 'true';
+
+function preconditions(run, config, ingress, kit, context) {
+  const { own, live, recipient } = context;
+  const ignore = new Set([ingress.publish.file, ...ingress.publish.supersedes]);
+  const swarm = remoteDocker(run, ['info', '--format', '{{.Swarm.LocalNodeState}} {{.Swarm.ControlAvailable}}']);
+  const service = kit.serviceState(run, `${ingress.name}_${ingress.service}`);
+  const health = stackHealth(run, config.stack);
+  const clash = (own.error || live.error) ? { found: [], unreadable: [] } : clashes(live.files, own, ignore);
+  const checks = [
+    [`${ingress.routes} parses with routers and services`, !own.error, own.error],
+    [`${ingress.legacy.dynamicDir} is readable`, !live.error, live.error],
+    ['no router or service name clashes with a file of another tenant', clash.found.length === 0, clash.found.join(', ')],
+    ['every other file of the directory can be checked', clash.unreadable.length === 0, `not valid YAML: ${clash.unreadable.join(', ')}`],
+    ['swarm active (manager)', swarm.status === 0 && String(swarm.stdout).trim() === 'active true', 'this host is not an active swarm manager'],
+    [`ingress service ${ingress.name}_${ingress.service} is running`, service.desired >= 1 && service.running >= 1, 'run the takeover first'],
+    [`old edge ${ingress.legacy.container} is not running`, !legacyRunning(run, ingress), 'an old edge watching the same directory would load the new file too'],
+    [`every service of the stack ${config.stack} is healthy`, health.ok, health.count === 0 ? 'no service listed' : `not healthy: ${health.unhealthy.join(', ')}`],
+    ['age recipient and backup directory declared', recipient.error === undefined && ingress.backup !== null, recipient.error ?? 'declare deploy.ingress.backup'],
+  ];
+  return checks.map(([name, ok, detail]) => ({ name, ok, detail }));
+}
+
+// ---- remote writes: only publish.file, its .tmp and the superseded names ------------------------------------------------------------------
+
+const remoteFile = (ingress, name) => `${ingress.legacy.dynamicDir}/${name}`;
+
+function readRemote(run, ingress, name) {
+  const read = run(`cat -- ${quote(remoteFile(ingress, name))}`);
+  return read.status === 0 ? String(read.stdout) : null;
+}
+
+/** Uploads `content` as <name>.tmp (never a route file name: the file provider ignores it). Returns an error text or null. */
+function upload(run, ingress, name, content) {
+  const tmp = remoteFile(ingress, `${name}.tmp`);
+  const done = run(shScript(`umask 022 && cat > ${quote(tmp)}`), { input: content });
+  return done.status === 0 ? null : `uploading ${name}.tmp FAILED: ${firstLine(done.stderr)}`;
+}
+
+/** Moves each <name>.tmp over <name> and removes the given files in ONE remote command. */
+function swapIn(run, ingress, moves, removals) {
+  const steps = moves.map((name) => {
+    const tmp = remoteFile(ingress, `${name}.tmp`);
+    return `mv -f ${quote(tmp)} ${quote(remoteFile(ingress, name))}`;
+  });
+  if (removals.length > 0) steps.push(`rm -f ${removals.map((name) => quote(remoteFile(ingress, name))).join(' ')}`);
+  const done = run(shScript(steps.join(' && ')));
+  return done.status === 0 ? null : firstLine(done.stderr);
+}
+
+/** Age-encrypted copy of one file into the backup directory, fed through stdin (the content never touches argv). */
+function backupOne(ctx, run, ingress, kit, { recipient, name, content }) {
+  const dir = ingress.backup.dir;
+  const file = `${dir}/${name}-${kit.stampOf(ctx.now())}.age`;
+  const script = ['umask 077', `mkdir -p ${quote(dir)}`, `age -r ${quote(recipient)} -o ${quote(file)}`, `test -s ${quote(file)}`].join(' && ');
+  const done = run(`bash -c ${quote(script)}`, { input: content });
+  return done.status === 0 ? { file } : { error: `backup of ${name} FAILED: ${firstLine(done.stderr)}` };
+}
+
+// ---- verification -----------------------------------------------------------------------------------------------------------------------
+
+/** A host that did not answer before the swap (status 0: unrouted, with no certificate or the default one) is new: it gets a fresh certificate. */
+const isNew = (entry) => !answers(entry.code);
+
+/**
+ * After the swap: a declared status is exact; otherwise the baseline class. A new host must answer 2xx-4xx (curl verifies the certificate against the
+ * name, so an answer proves a valid certificate for it) and is never compared with its old default certificate; a host that answered before keeps
+ * its certificate.
+ */
+function forwardOk(entry, seen) {
+  let statusOk = answers(seen.code);
+  if (entry.target.status !== null) statusOk = seen.code === entry.target.status;
+  else if (!isNew(entry)) statusOk = classOf(seen.code) === classOf(entry.code);
+  return statusOk && (isNew(entry) || entry.certificate === null || seen.certificate === entry.certificate);
+}
+
+/** After a restore: every host that answered before answers in the same class with the same certificate. */
+const restoreOk = (entry, seen) => classOf(seen.code) === classOf(entry.code) && seen.certificate === entry.certificate;
+
+function certificateState(entry, seen) {
+  if (isNew(entry) || entry.certificate === null) return 'new';
+  return seen.certificate === entry.certificate ? 'equal' : 'different';
+}
+
+function verify(ctx, run, kit, entries, accept) {
+  return kit.settle(ctx, VERIFY_ATTEMPTS, VERIFY_MS, () => {
+    const rows = entries.map((entry) => {
+      const seen = kit.observe(run, entry.target);
+      return { entry, code: seen.code, same: certificateState(entry, seen), ok: accept(entry, seen) };
+    });
+    return { ok: rows.every((row) => row.ok), rows };
+  });
+}
+
+function rowLine(row) {
+  return `${row.ok ? 'ok' : 'MISMATCH'} ${label(row.entry.target)}: status ${row.code} (baseline ${row.entry.code}), certificate serial ${row.same}`;
+}
+
+const hostsData = (rows) => rows.map((row) => ({ host: row.entry.target.host, path: row.entry.target.path, status: row.code, certificate: row.same }));
+
+/** Writes the superseded files and the previous publish.file back from memory (one atomic swap; publish.file is deleted only when there was none) and re-verifies the baseline. */
+async function restore(ctx, run, ingress, kit, context) {
+  const { contents, previous, base, steps } = context;
+  const back = new Map(contents);
+  if (previous !== null) back.set(ingress.publish.file, previous);
+  const names = [...back.keys()];
+  const failedUpload = names.map((name) => upload(run, ingress, name, back.get(name))).find((error) => error !== null);
+  const failedSwap = failedUpload ?? swapIn(run, ingress, names, previous === null ? [ingress.publish.file] : []);
+  steps.push('restore');
+  const manual = `RESTORE FAILED (manual: encrypted backups in ${ingress.backup.dir})`;
+  if (failedSwap) return { ok: false, line: manual };
+  const proof = await verify(ctx, run, kit, base.filter((entry) => answers(entry.code)), restoreOk);
+  steps.push('restore-verify');
+  return proof.ok ? { ok: true, line: 'RESTORED ok: the superseded files are back and every host answers as before' } : { ok: false, line: manual };
+}
+
+async function apply(ctx, run, ingress, kit, context) {
+  const { own, base, lines, steps, contents, backups, previous } = context;
+  const uploadError = upload(run, ingress, ingress.publish.file, own.text);
+  steps.push('upload');
+  if (uploadError) return result(false, [...lines, uploadError, 'publish refused: nothing was changed'], steps);
+  lines.push(`uploaded ${ingress.publish.file}.tmp`);
+  const swapError = swapIn(run, ingress, [ingress.publish.file], [...contents.keys()]);
+  steps.push('swap');
+  const proof = swapError ? { ok: false, rows: [] } : await verify(ctx, run, kit, base, forwardOk);
+  steps.push('verify');
+  lines.push(...proof.rows.map(rowLine));
+  if (!swapError && proof.ok) {
+    lines.push(`published ${ingress.publish.file}; retired ${[...contents.keys()].join(', ') || 'nothing'}; ${proof.rows.length} host(s) verified`);
+    return result(true, lines, steps, { backups, hosts: hostsData(proof.rows), restored: null });
+  }
+  lines.push(swapError ? `swap FAILED: ${swapError}` : 'verification FAILED');
+  const back = await restore(ctx, run, ingress, kit, { contents, previous, base, steps });
+  lines.push(back.line);
+  return result(false, lines, steps, { backups, hosts: hostsData(proof.rows), restored: back.ok ? 'ok' : 'failed' });
+}
+
+/** Reads and backs up every superseded file that exists, then the current publish.file (about to be overwritten); returns { contents, previous, backups } or { error }. */
+function retire(ctx, run, ingress, kit, { recipient, names }) {
+  const contents = new Map();
+  const backups = [];
+  for (const name of names) {
+    const content = readRemote(run, ingress, name);
+    if (content === null) continue;
+    const saved = backupOne(ctx, run, ingress, kit, { recipient, name, content });
+    if (saved.error) return { error: saved.error };
+    contents.set(name, content);
+    backups.push(saved.file);
+  }
+  const previous = readRemote(run, ingress, ingress.publish.file);
+  if (previous !== null) {
+    const saved = backupOne(ctx, run, ingress, kit, { recipient, name: ingress.publish.file, content: previous });
+    if (saved.error) return { error: saved.error };
+    backups.push(saved.file);
+  }
+  return { contents, previous, backups };
+}
+
+/** `ingress publish`: see the top of the file. `kit` carries the shared probe helpers of ingress.mjs. */
+export async function publishIngress(ctx, run, config, ingress, execute, kit) {
+  if (ingress.publish === null) throw refusal('STACK_INGRESS_PUBLISH_UNDECLARED', 'deploy.ingress.publish ({ file, supersedes }) is not declared: nothing to publish');
+  if (ingress.routes === null) throw refusal('STACK_INGRESS_PUBLISH_NO_ROUTES', 'deploy.ingress.routes is not declared: there is no routes file to publish');
+  const steps = ['preconditions'];
+  const own = readOwn(config, ingress);
+  const live = readFiles(run, kit, ingress.legacy.dynamicDir);
+  const { recipient, error } = recipientOf(kit, config);
+  const checks = preconditions(run, config, ingress, kit, { own, live, recipient: { error } });
+  const lines = checks.map((check) => (check.ok ? `precondition ok: ${check.name}` : `precondition FAILED: ${check.name} (${check.detail})`));
+  if (checks.some((check) => !check.ok)) return result(false, [...lines, 'publish refused: nothing was changed'], steps);
+  const present = ingress.publish.supersedes.filter((name) => live.files.has(name));
+  if (readRemote(run, ingress, ingress.publish.file) === own.text && present.length === 0) return result(true, [...lines, `nothing to do: ${ingress.publish.file} is current and no superseded file exists`], steps, { changed: false });
+  const base = kit.baselines(run, kit.targets(config, ingress));
+  steps.push('baseline');
+  for (const entry of base) lines.push(`baseline ${label(entry.target)}: status ${entry.code}`);
+  if (!execute) {
+    lines.push(`dry run: would back up ${present.join(', ') || 'no superseded file'} (age) to ${ingress.backup.dir}, upload ${ingress.publish.file}, swap it in, retire ${present.join(', ') || 'nothing'} and verify ${base.length} host(s), restoring on any failure`);
+    return result(true, lines, steps, { changed: false });
+  }
+  const retired = retire(ctx, run, ingress, kit, { recipient, names: present });
+  if (retired.error) return result(false, [...lines, retired.error, 'publish refused: nothing was changed'], steps);
+  steps.push('backup');
+  lines.push(`backups written: ${retired.backups.join(', ') || 'none needed'}`);
+  return apply(ctx, run, ingress, kit, { own, base, lines, steps, ...retired });
+}