@starci/hfs 4.3.0 → 4.4.0

package/stack/build.mjs

@@ -0,0 +1,173 @@
+// build.mjs - `starci app stack build --env <env> [--push] [--execute]`: builds, from the CURRENT checkout, every image the deploy block declares
+// (docker buildx, linux/amd64, no provenance so the digest is one manifest), passing the public build values of build-args.mjs, and prints
+// `--image <service>=<registry/name@sha256:digest>` lines ready for `stack deploy`. It refuses a dirty tree and a HEAD that carries no release tag
+// (the images must come from the released commit). It never reads registry credentials: pushing uses the operator's own `docker login`.
+// The image name is `<registry host>/<owner>/<project>-<app>:<full sha>`, the owner being the GitHub owner of `origin` (ghcr.io only).
+// With deploy.registry.visibility `public` (the default) every image is scanned for secret material BEFORE it is pushed (image-scan.mjs: the layers of
+// a public package are public), and after the push the real visibility of the package is checked with an ANONYMOUS manifest fetch of the pushed
+// digest; a package that is still private is reported with the one owner step that changes it (GitHub has no API for that) and the run fails.
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { buildArgs } from './build-args.mjs';
+import { firstLine, lines as splitLines, tool } from './context.mjs';
+import { failure, refusal } from './errors.mjs';
+import { requireAgeIdentity } from './sealed.mjs';
+import { scanImage } from './image-scan.mjs';
+
+const RELEASE_TAG = /^v\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/u;
+const GITHUB_REMOTE = /github\.com[:/]([A-Za-z0-9._-]+)\/[A-Za-z0-9._-]+?(?:\.git)?$/u;
+const BUILD_ARG = /^([A-Z][A-Z0-9_]*)=(.*)$/u;
+const SIDES = ['be', 'fe'];
+const MAX_HITS_SHOWN = 20;
+const MANIFEST_TYPES = ['application/vnd.oci.image.index.v1+json', 'application/vnd.oci.image.manifest.v1+json', 'application/vnd.docker.distribution.manifest.list.v2+json', 'application/vnd.docker.distribution.manifest.v2+json'];
+
+const git = (ctx, config, args) => tool(ctx, 'git', args, { cwd: config.repoRoot });
+
+/** The release facts of the checkout: { sha, tag } or a refusal (dirty tree, no release tag). */
+function checkout(ctx, config) {
+  const status = git(ctx, config, ['status', '--porcelain']);
+  if (status.status !== 0) throw failure('STACK_BUILD_GIT', `git status failed: ${firstLine(status.stderr)}`);
+  const dirty = splitLines(status.stdout);
+  if (dirty.length > 0) throw refusal('STACK_BUILD_DIRTY', `the working tree is not clean (${dirty.length} path(s), first: ${dirty[0]}): images must come from the released commit`);
+  const sha = splitLines(git(ctx, config, ['rev-parse', 'HEAD']).stdout)[0] ?? '';
+  if (!/^[0-9a-f]{40}$/u.test(sha)) throw failure('STACK_BUILD_GIT', 'cannot read HEAD');
+  const tag = splitLines(git(ctx, config, ['tag', '--points-at', 'HEAD']).stdout).find((name) => RELEASE_TAG.test(name));
+  if (!tag) throw refusal('STACK_BUILD_UNTAGGED', 'HEAD carries no release tag (vMAJOR.MINOR.PATCH): build only the released commit');
+  return { sha, tag };
+}
+
+function ownerOf(ctx, config) {
+  const remote = git(ctx, config, ['remote', 'get-url', 'origin']);
+  const match = GITHUB_REMOTE.exec(firstLine(remote.stdout));
+  if (remote.status !== 0 || !match) throw refusal('STACK_BUILD_OWNER', 'origin is not a github.com remote: the image owner cannot be derived');
+  return match[1].toLowerCase();
+}
+
+/** The Dockerfile of an app: be/apps/<app>/Dockerfile or fe/apps/<app>/Dockerfile; ambiguity and absence are refusals. */
+function dockerfileOf(config, app) {
+  const found = SIDES.map((side) => `${side}/apps/${app}/Dockerfile`).filter((file) => fs.existsSync(path.join(config.repoRoot, file)));
+  if (found.length === 0) throw refusal('STACK_BUILD_NO_DOCKERFILE', `no Dockerfile for app ${app} (be/apps/${app}/Dockerfile or fe/apps/${app}/Dockerfile)`);
+  if (found.length > 1) throw refusal('STACK_BUILD_AMBIGUOUS', `app ${app} has a Dockerfile on both sides: ${found.join(', ')}`);
+  return found[0];
+}
+
+async function argsOf(ctx, config, app, hasBuildArgs) {
+  if (!hasBuildArgs) return [];
+  const result = await buildArgs(ctx, config, { app });
+  return result.lines.map((line) => BUILD_ARG.exec(line)).filter(Boolean).map((match) => [match[1], match[2]]);
+}
+
+function plan(config, { sha, owner }) {
+  return Object.entries(config.images).map(([service, entry]) => ({
+    service,
+    app: entry.app,
+    dockerfile: dockerfileOf(config, entry.app),
+    ref: `${config.registry.host}/${owner}/${config.project}-${entry.app}:${sha}`,
+    hasBuildArgs: Object.keys(entry.buildArgs.values).length > 0 || entry.buildArgs.sealed !== null,
+  }));
+}
+
+function buildOne(ctx, config, item, args, push) {
+  const metadata = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'stack-build-')), 'metadata.json');
+  const argv = ['buildx', 'build', '--platform', 'linux/amd64', '--provenance=false', '--file', item.dockerfile, '--tag', item.ref, '--metadata-file', metadata,
+    ...args.flatMap(([key, value]) => ['--build-arg', `${key}=${value}`]), push ? '--push' : '--load', '.'];
+  const run = tool(ctx, 'docker', argv, { cwd: config.repoRoot });
+  let digest = null;
+  try { digest = JSON.parse(fs.readFileSync(metadata, 'utf8'))['containerimage.digest'] ?? null; } catch { digest = null; }
+  fs.rmSync(path.dirname(metadata), { recursive: true, force: true });
+  if (run.status !== 0) return { ok: false, error: firstLine(run.stderr) || 'docker buildx failed' };
+  return { ok: true, digest };
+}
+
+/** `docker push` of a scanned local image; the digest is the one docker prints. */
+function pushOne(ctx, config, item) {
+  const run = tool(ctx, 'docker', ['push', item.ref], { cwd: config.repoRoot });
+  if (run.status !== 0) return { ok: false, error: firstLine(run.stderr) || 'docker push failed' };
+  return { ok: true, digest: /digest:\s*(sha256:[0-9a-f]{64})/u.exec(String(run.stdout))?.[1] ?? null };
+}
+
+function scanRefusal(item, scan) {
+  const shown = scan.hits.slice(0, MAX_HITS_SHOWN).map((hit) => `  ${hit.kind}: ${hit.where} (${hit.why})`);
+  const more = scan.hits.length > MAX_HITS_SHOWN ? [`  ... and ${scan.hits.length - MAX_HITS_SHOWN} more`] : [];
+  const head = `REFUSED to push ${item.service}: the image would be public and the scan found ${scan.hits.length} hit(s) and ${scan.errors.length} failed step(s) (places only, never the text)`;
+  return [head, ...shown, ...more, ...scan.errors.map((error) => `  scan error: ${error}`)];
+}
+
+/** One image: built (and pushed in the same step when private); built, scanned and only then pushed when public. Returns { ok, digest, lines }. */
+function processImage(ctx, config, item, args, { push }) {
+  const isPublic = config.registry.visibility === 'public';
+  const built = buildOne(ctx, config, item, args, push && !isPublic);
+  if (!built.ok) return { ok: false, lines: [`build of ${item.service} FAILED: ${built.error}`] };
+  if (!isPublic) return { ok: true, digest: built.digest, lines: [] };
+  const scan = scanImage(ctx, item.ref, { allowedArgs: args.map(([key]) => key) });
+  if (!scan.ok) return { ok: false, lines: scanRefusal(item, scan) };
+  const clean = `scanned ${item.service}: no secret material found`;
+  if (!push) return { ok: true, digest: null, lines: [clean] };
+  const pushed = pushOne(ctx, config, item);
+  if (!pushed.ok) return { ok: false, lines: [clean, `push of ${item.service} FAILED: ${pushed.error}`] };
+  return { ok: true, digest: pushed.digest, lines: [clean] };
+}
+
+const ownerStep = (name) => `owner step: GitHub -> Packages -> ${name} -> Package settings -> Change visibility -> Public (GitHub has no API for this)`;
+
+/** 'public' when an ANONYMOUS client can fetch the manifest of the pushed digest, 'private' when the registry refuses it, 'unknown' on a network error. */
+async function visibilityOf(ctx, host, repository, digest) {
+  try {
+    const token = await ctx.fetch(`https://${host}/token?scope=repository:${repository}:pull&service=${host}`, { signal: AbortSignal.timeout(15000) });
+    const bearer = token.ok ? (await token.json()).token : null;
+    const headers = { Accept: MANIFEST_TYPES.join(', '), ...(bearer ? { Authorization: `Bearer ${bearer}` } : {}) };
+    const manifest = await ctx.fetch(`https://${host}/v2/${repository}/manifests/${digest}`, { headers, signal: AbortSignal.timeout(15000) });
+    return manifest.status === 200 ? 'public' : 'private';
+  } catch {
+    return 'unknown';
+  }
+}
+
+/** After the push: the real visibility of every pushed package against the declared one. Returns { ok, lines, actual }. */
+async function checkVisibility(ctx, config, images) {
+  const declared = config.registry.visibility;
+  const lines = [];
+  const actual = {};
+  let ok = true;
+  for (const [service, ref] of Object.entries(images)) {
+    const [name, digest] = ref.split('@');
+    const repository = name.slice(config.registry.host.length + 1);
+    const packageName = repository.slice(repository.indexOf('/') + 1);
+    actual[service] = await visibilityOf(ctx, config.registry.host, repository, digest);
+    if (actual[service] === 'unknown') lines.push(`visibility of ${packageName} could not be checked (network)`);
+    else if (actual[service] === declared) lines.push(`visibility of ${packageName}: ${actual[service]}, as declared`);
+    else if (declared === 'public') {
+      ok = false;
+      lines.push(`visibility of ${packageName} is PRIVATE but deploy.registry.visibility is public`, ownerStep(packageName));
+    } else lines.push(`note: ${packageName} is public although deploy.registry.visibility is private; change it in Package settings or declare public`);
+  }
+  return { ok, lines, actual };
+}
+
+/** `build`: dry run unless execute; pushing needs --push as well. */
+export async function buildImages(ctx, config, { push = false, execute = false } = {}) {
+  if (Object.keys(config.images).length === 0) throw refusal('STACK_BUILD_NO_IMAGES', 'the deploy block declares no images');
+  if (config.registry.host !== 'ghcr.io') throw refusal('STACK_BUILD_REGISTRY', `build supports ghcr.io only (deploy.registry.host is ${config.registry.host})`);
+  const release = checkout(ctx, config);
+  const items = plan(config, { sha: release.sha, owner: ownerOf(ctx, config) });
+  const lines = [`release ${release.tag} at ${release.sha.slice(0, 12)}: ${items.length} image(s), registry ${config.registry.host} (${config.registry.visibility})`, ...items.map((item) => `  ${item.service}: ${item.dockerfile} -> ${item.ref}`)];
+  if (!execute) {
+    const scan = config.registry.visibility === 'public' ? ', scan for secret material' : '';
+    return { ok: true, lines: [...lines, `dry run: would build${scan}${push ? ' and push' : ' (local only, no digest without --push)'}; nothing was built`], data: { tag: release.tag, images: {} } };
+  }
+  if (items.some((item) => config.images[item.service].buildArgs.sealed !== null)) requireAgeIdentity(ctx, config, 'build');
+  const images = {};
+  for (const item of items) {
+    const outcome = processImage(ctx, config, item, await argsOf(ctx, config, item.app, item.hasBuildArgs), { push });
+    lines.push(...outcome.lines);
+    if (!outcome.ok) return { ok: false, lines, data: { tag: release.tag, images } };
+    if (push && !outcome.digest) return { ok: false, lines: [...lines, `${item.service} was pushed but docker reported no digest`], data: { tag: release.tag, images } };
+    if (push) images[item.service] = `${item.ref.slice(0, item.ref.lastIndexOf(':'))}@${outcome.digest}`;
+    lines.push(push ? `pushed ${item.service}: ${images[item.service]}` : `built ${item.service} (local image ${item.ref}; push it with --push to get a digest)`);
+  }
+  const visibility = push ? await checkVisibility(ctx, config, images) : { ok: true, lines: [], actual: {} };
+  lines.push(...visibility.lines);
+  if (push) lines.push('', 'for the deploy:', `${Object.entries(images).map(([service, ref]) => `--image ${service}=${ref}`).join(' ')} --source-sha ${release.sha}`);
+  return { ok: visibility.ok, lines, data: { tag: release.tag, sha: release.sha, images, visibility: visibility.actual } };
+}

package/stack/cli.mjs

@@ -0,0 +1,130 @@
+// cli.mjs - `starci app stack <action>` for the Swarm deploy tooling: flags, dry run versus --execute, dispatch, text and --json output.
+//   starci app stack prepare|doctor|connect|disconnect|build-args|render|deploy|converge|verify|rollback|prune|backup|jobs|task --env <env> [flags]
+// (up, down and status are the test-world verbs and are dispatched by packages/hfs/src/main.mjs.)
+// Every action module exports one async function (ctx, config, options) -> { ok, lines, data }; a refusal throws StackError.
+import path from 'node:path';
+import { stackContext } from './context.mjs';
+import { loadDeployConfig } from './config.mjs';
+import { withDockerTransport } from './docker-transport.mjs';
+import { StackError, EXIT, refusal } from './errors.mjs';
+
+/** The actions that drive the docker CLI of the operator machine (pointed at the manager by docker-transport.mjs when the key is sealed). */
+const DOCKER_ACTIONS = new Set(['deploy', 'accept', 'converge', 'verify', 'rollback', 'prune']);
+
+export const ACTIONS = Object.freeze(['prepare', 'doctor', 'connect', 'disconnect', 'build-args', 'render', 'deploy', 'converge', 'verify', 'rollback', 'prune', 'backup', 'jobs', 'task', 'ingress', 'retire', 'build', 'accept']);
+
+const BOOLEAN = new Set(['--execute', '--json', '--swarm-state', '--init-swarm', '--owner-approved', '--fix-live-restore', '--skip-public', '--skip-internal', '--allow-unpinned', '--push']);
+const VALUE = new Set(['--env', '--image', '--source-sha', '--previous', '--out', '--release', '--since', '--timeout', '--retry', '--keep', '--app', '--ssh-cmd', '--extra-compose', '--project']);
+const REPEAT = new Set(['--image', '--keep', '--extra-compose', '--project']);
+
+/** Splits argv into { flags, positional }; an unknown flag or a flag without its value is a refusal. */
+export function parseFlags(argv) {
+  const flags = {};
+  const positional = [];
+  for (let index = 0; index < argv.length; index += 1) {
+    const arg = argv[index];
+    if (BOOLEAN.has(arg)) flags[arg.slice(2)] = true;
+    else if (VALUE.has(arg)) {
+      const value = argv[index + 1];
+      if (value === undefined || value.startsWith('--')) throw refusal('STACK_USAGE', `${arg} needs a value`);
+      index += 1;
+      const key = arg.slice(2);
+      flags[key] = REPEAT.has(arg) ? [...(flags[key] ?? []), value] : value;
+    } else if (arg.startsWith('--')) throw refusal('STACK_USAGE', `unknown flag ${arg}`);
+    else positional.push(arg);
+  }
+  return { flags, positional };
+}
+
+/** `--image api=ghcr.io/o/x@sha256:...` (repeatable or comma separated) -> { api: 'ghcr.io/o/x@sha256:...' }. */
+export function parseImages(values = []) {
+  const out = {};
+  for (const item of values.flatMap((value) => value.split(','))) {
+    const at = item.indexOf('=');
+    const service = item.slice(0, at);
+    if (at < 1 || at === item.length - 1 || Object.hasOwn(out, service)) throw refusal('STACK_USAGE', `--image takes <service>=<reference>, once per service (got ${service || item})`);
+    out[service] = item.slice(at + 1);
+  }
+  return out;
+}
+
+const number = (flags, name) => {
+  if (flags[name] === undefined) return undefined;
+  const value = Number(flags[name]);
+  if (!Number.isFinite(value) || value < 0) throw refusal('STACK_USAGE', `--${name} must be a number of seconds`);
+  return value;
+};
+
+/** The options object each action module receives, from the parsed flags. */
+const OPTIONS = {
+  prepare: ({ flags }) => ({ execute: flags.execute === true, initSwarm: flags['init-swarm'] === true, ownerApproved: flags['owner-approved'] === true, fixLiveRestore: flags['fix-live-restore'] === true }),
+  doctor: () => ({}),
+  connect: ({ flags }) => ({ execute: flags.execute === true }),
+  disconnect: ({ flags }) => ({ execute: flags.execute === true }),
+  'build-args': ({ flags }) => ({ app: flags.app }),
+  render: ({ flags, cwd }) => ({
+    images: parseImages(flags.image), sourceSha: flags['source-sha'], previous: flags.previous ?? null,
+    out: flags.out ? path.resolve(cwd, flags.out) : null, allowUnpinned: flags['allow-unpinned'] === true,
+    extraCompose: (flags['extra-compose'] ?? []).map((file) => path.resolve(cwd, file)),
+  }),
+  deploy: ({ flags, cwd }) => ({
+    images: parseImages(flags.image), sourceSha: flags['source-sha'], execute: flags.execute === true,
+    out: flags.out ? path.resolve(cwd, flags.out) : null, timeoutSeconds: number(flags, 'timeout'), retrySeconds: number(flags, 'retry'), skipPublic: flags['skip-public'] === true,
+  }),
+  converge: ({ flags }) => ({ since: flags.since, timeoutSeconds: number(flags, 'timeout') }),
+  verify: ({ flags }) => ({ retrySeconds: number(flags, 'retry'), skipPublic: flags['skip-public'] === true, skipInternal: flags['skip-internal'] === true }),
+  rollback: ({ flags }) => ({ release: flags.release ?? null, execute: flags.execute === true, timeoutSeconds: number(flags, 'timeout') }),
+  prune: ({ flags, cwd }) => ({ keepFiles: (flags.keep ?? []).map((file) => path.resolve(cwd, file)), execute: flags.execute === true }),
+  backup: ({ flags }) => ({ swarmState: flags['swarm-state'] === true, execute: flags.execute === true }),
+  ingress: ({ flags, positional }) => ({ action: positional[0], execute: flags.execute === true, args: positional.slice(1) }),
+  build: ({ flags }) => ({ push: flags.push === true, execute: flags.execute === true }),
+  accept: ({ flags }) => ({ release: flags.release ?? null, execute: flags.execute === true, retrySeconds: number(flags, 'retry') }),
+  retire: ({ flags }) => ({ projects: flags.project ?? [], execute: flags.execute === true }),
+  jobs: ({ flags, positional }) => ({ action: positional[0], args: positional.slice(1), execute: flags.execute === true }),
+  task: ({ flags, positional }) => ({ name: positional[0], args: positional.slice(1), execute: flags.execute === true }),
+};
+
+const MODULES = {
+  prepare: ['./prepare.mjs', 'prepareHost'], doctor: ['./prepare.mjs', 'doctorHost'],
+  connect: ['./remote.mjs', 'connectCi'], disconnect: ['./remote.mjs', 'disconnectCi'],
+  'build-args': ['./build-args.mjs', 'buildArgs'], render: ['./render.mjs', 'renderRelease'],
+  deploy: ['./deploy.mjs', 'deployRelease'], converge: ['./converge.mjs', 'convergeStack'],
+  verify: ['./verify.mjs', 'verifyStack'], rollback: ['./deploy.mjs', 'rollbackRelease'],
+  prune: ['./prune.mjs', 'pruneStack'], backup: ['./backup.mjs', 'backupVolumes'], jobs: ['./jobs.mjs', 'runJobs'], task: ['./task.mjs', 'runTask'],
+  ingress: ['./ingress.mjs', 'runIngress'],
+  retire: ['./retire.mjs', 'retireCompose'],
+  build: ['./build.mjs', 'buildImages'],
+  accept: ['./deploy.mjs', 'acceptRelease'],
+};
+
+async function handlerOf(action, importer) {
+  const [file, name] = MODULES[action];
+  return (await importer(file))[name];
+}
+
+function print(ctx, action, flags, result) {
+  if (flags.json) ctx.stdout(`${JSON.stringify({ verb: action, ok: result.ok, ...result.data }, null, 2)}\n`);
+  else for (const line of result.lines ?? []) ctx.stdout(`${line}\n`);
+}
+
+/** Runs one action; returns the exit code. `io.importer` lets a spec load fake action modules. */
+export async function stackMain(argv, io = {}) {
+  const ctx = stackContext(io);
+  try {
+    const { flags, positional } = parseFlags(argv.slice(1));
+    const action = argv[0];
+    if (!ACTIONS.includes(action)) throw refusal('STACK_USAGE', `starci app stack takes ${[...ACTIONS, 'up', 'down', 'status'].join(', ')}`);
+    const config = loadDeployConfig({ repoRoot: ctx.cwd, env: flags.env ?? 'production' });
+    const options = OPTIONS[action]({ flags, positional, cwd: ctx.cwd });
+    const handler = await handlerOf(action, io.importer ?? ((file) => import(file)));
+    const sshCmd = flags['ssh-cmd'] ?? null;
+    const call = (context) => handler(context, config, { ...options, sshCmd });
+    const result = DOCKER_ACTIONS.has(action) ? await withDockerTransport(ctx, config, { sshCmd }, call) : await call(ctx);
+    print(ctx, action, flags, result);
+    return result.ok ? EXIT.ok : EXIT.failed;
+  } catch (error) {
+    if (!(error instanceof StackError)) throw error;
+    ctx.stderr(`starci app stack: ${error.message}\n`);
+    return error.exit;
+  }
+}

package/stack/config.mjs

@@ -0,0 +1,57 @@
+// config.mjs - loads `environments.<env>.deploy` of .starcistacks/application-stacks.yaml for the stack verbs.
+// The block is read and judged by the same module `starci app check` uses (runtime/scripts/hfs/stack-deploy-config.mjs).
+import fs from 'node:fs';
+import path from 'node:path';
+import { parseYaml } from '../runtime/engine/yaml.mjs';
+import { DEPLOY_CONFIG_CODE, NAME, normalizeDeploy } from '../runtime/scripts/hfs/stack-deploy-config.mjs';
+import { refusal } from './errors.mjs';
+
+export const STACKS_DIR = '.starcistacks';
+export const STACKS_FILE = 'application-stacks.yaml';
+
+const readYaml = (file, what) => {
+  let source;
+  try { source = fs.readFileSync(file, 'utf8'); } catch { throw refusal('STACKS_DEPLOY_FILE_MISSING', `${what} (${file}) is not readable`); }
+  try { return parseYaml(source); } catch { throw refusal('STACKS_DEPLOY_FILE_INVALID', `${what} is not valid YAML`); }
+};
+
+const projectOf = (repoRoot) => {
+  const doc = JSON.parse(fs.readFileSync(path.join(repoRoot, 'hfs.json'), 'utf8'));
+  return String(doc.project);
+};
+
+const sealedMembers = (secretsDir) => {
+  try { return new Set(fs.readdirSync(secretsDir).filter((name) => name.endsWith('.enc')).map((name) => name.slice(0, -4))); } catch { return new Set(); }
+};
+
+const stackServices = (stackFile) => {
+  const model = readYaml(stackFile, 'the stack source');
+  return new Set(Object.keys(model?.services ?? {}));
+};
+
+/**
+ * The normalised deploy configuration of one environment, with absolute paths:
+ *   { env, project, stack, repoRoot, envDir, stackFile, secretsDir, members, services, ...deploy block }.
+ * Throws StackError (exit 2) naming every problem when the block is missing or invalid.
+ */
+export function loadDeployConfig({ repoRoot, env }) {
+  if (!NAME.test(String(env))) throw refusal(DEPLOY_CONFIG_CODE, `--env ${env} must match ${NAME}`);
+  const declaration = readYaml(path.join(repoRoot, STACKS_DIR, STACKS_FILE), STACKS_FILE);
+  const environment = declaration?.environments?.[env];
+  if (!environment) throw refusal('STACKS_DEPLOY_ENV_UNKNOWN', `${STACKS_FILE} declares no environment ${env}`);
+  if (environment.runtime !== 'docker-swarm') throw refusal('STACKS_DEPLOY_NOT_SWARM', `environment ${env} runs ${environment.runtime}; stack deploy verbs need docker-swarm`);
+  if (environment.deploy === undefined) throw refusal('STACKS_DEPLOY_UNDECLARED', `environment ${env} has no deploy block in ${STACKS_FILE}`);
+  const envDir = path.join(repoRoot, STACKS_DIR, env);
+  const stackFile = path.join(envDir, String(environment.composeFiles?.[0] ?? ''));
+  const secretsDir = path.join(envDir, 'secrets');
+  const services = stackServices(stackFile);
+  const members = sealedMembers(secretsDir);
+  const probe = { exists: (rel) => fs.existsSync(path.join(envDir, rel)), services, members };
+  const project = projectOf(repoRoot);
+  const { config, problems } = normalizeDeploy(environment.deploy, { project, probe });
+  if (!config) {
+    const first = problems.slice(0, 6).map((item) => `${item.at}: ${item.message}`).join('; ');
+    throw refusal(DEPLOY_CONFIG_CODE, `${problems.length} problem${problems.length === 1 ? '' : 's'} in environments.${env}.deploy: ${first}`);
+  }
+  return { env, project, repoRoot, envDir, stackFile, secretsDir, members, services, ...config };
+}

package/stack/context.mjs

@@ -0,0 +1,57 @@
+// context.mjs - the seams every stack module runs through: spawning tools, sleeping, the clock and the output.
+// Production uses the real implementations; the specs inject fakes (tests/helpers/stack-fakes.mjs).
+import { spawnSync } from 'node:child_process';
+import { defaultSops } from '../runtime/scripts/hfs/secret-sops.mjs';
+
+const MAX_BUFFER = 1 << 24;
+
+const realSpawn = (command, args, options = {}) => {
+  const run = spawnSync(command, args, { encoding: 'utf8', maxBuffer: MAX_BUFFER, ...options });
+  return { status: run.status, stdout: run.stdout ?? '', stderr: run.stderr ?? '', error: run.error };
+};
+
+const realSleep = (ms) => new Promise((done) => { setTimeout(done, ms); });
+
+/**
+ * The context of one stack invocation.
+ *   cwd, env       the app root and the process environment
+ *   spawn          (command, args, { input, env, cwd }) -> { status, stdout, stderr, error }; never a shell
+ *   sleep, now     await sleep(ms); now() -> epoch ms
+ *   stdout, stderr text writers
+ *   fetch          the HTTP seam of verify (url, init) -> Response
+ *   sops           the secret seam (decrypt(file, format) -> map; decryptBinary(file) -> text); `io.sops` or, on first use, the app's own sops/age tools
+ */
+export function stackContext(io = {}) {
+  const cwd = io.cwd ?? process.cwd();
+  const env = io.env ?? process.env;
+  let sops = io.sops;
+  return Object.freeze({
+    cwd,
+    env,
+    spawn: io.spawn ?? realSpawn,
+    sleep: io.sleep ?? realSleep,
+    now: io.now ?? Date.now,
+    stdout: io.stdout ?? ((text) => process.stdout.write(text)),
+    stderr: io.stderr ?? ((text) => process.stderr.write(text)),
+    fetch: io.fetch ?? globalThis.fetch,
+    /** The secret seam: the app's own sops/age tools, created on first use so verbs that open no custody never need them. */
+    get sops() {
+      sops ??= defaultSops(env, cwd);
+      return sops;
+    },
+  });
+}
+
+/** Runs one tool; `env` replaces the child environment when given, `input` is written to its stdin. */
+export const tool = (ctx, command, args, options = {}) => ctx.spawn(command, args, options);
+
+/** Runs a tool and returns stdout, or throws the first stderr line. Used where a failure has no recovery. */
+export function toolOutput(ctx, command, args, options = {}) {
+  const run = tool(ctx, command, args, options);
+  if (run.error) throw new Error(`cannot run ${command} (${run.error.message})`);
+  if (run.status !== 0) throw new Error(`${command} ${args.slice(0, 2).join(' ')}: ${firstLine(run.stderr)}`);
+  return run.stdout;
+}
+
+export const firstLine = (text) => String(text ?? '').trim().split(/\r?\n/u)[0];
+export const lines = (text) => String(text ?? '').split(/\r?\n/u).filter(Boolean);

package/stack/converge.mjs

@@ -0,0 +1,107 @@
+// converge.mjs - observes whether a stack deploy has converged: desired versus running replicas, the update state of every service and
+// failed or rejected tasks of the NEW service spec created since the deploy began (`docker stack deploy --detach` returns before any of this is
+// true). The old tasks still restarting while a rolling update replaces them carry the previous spec: their failures are not the deploy's, so a
+// task counts only when its spec equals the service's current task template. The update must also have reached `completed`.
+// Two consecutive agreeing observations are required, so a task that starts and immediately exits is caught by the second look.
+// Read-only: every call is a docker query. Time comes from ctx.now(), waits from ctx.sleep(), tools through ctx.spawn.
+import { repeatInOrder } from '../runtime/scripts/lib/in-order.mjs';
+import { lines as splitLines, toolOutput } from './context.mjs';
+import { refusal } from './errors.mjs';
+
+const POLL_MS = 5000;
+const TAB = String.fromCodePoint(9);
+// The spec is JSON (it never holds a raw tab), so it is split off at the last tab; the error text before it may hold anything.
+const TASK_FORMAT = `{{.DesiredState}}|{{.Status.State}}|{{.Status.Timestamp}}|{{.Status.Err}}${TAB}{{json .Spec}}`;
+
+const docker = (ctx, args) => toolOutput(ctx, 'docker', args);
+
+/** The JSON text with object keys sorted at every depth, so two equal specs compare equal whatever the key order. */
+function canonical(value) {
+  if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
+  if (value !== null && typeof value === 'object') return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`;
+  return JSON.stringify(value);
+}
+
+function sameSpec(taskSpec, template) {
+  try {
+    return template === undefined || canonical(JSON.parse(taskSpec)) === template;
+  } catch {
+    return true;
+  }
+}
+
+/** Failed or rejected tasks of the current spec created since the deploy began, and how many tasks run as desired. */
+function readTasks(ctx, name, sinceMs, template) {
+  const ids = splitLines(docker(ctx, ['service', 'ps', name, '--no-trunc', '-q']));
+  const rows = ids.length === 0 ? [] : splitLines(docker(ctx, ['inspect', '--type', 'task', '--format', TASK_FORMAT, ...ids]));
+  const out = { running: 0, failed: [] };
+  for (const row of rows) {
+    const cut = row.lastIndexOf(TAB);
+    const [wanted, state, timestamp, ...rest] = (cut < 0 ? row : row.slice(0, cut)).split('|');
+    if (wanted === 'running' && state === 'running') out.running += 1;
+    const current = cut < 0 || sameSpec(row.slice(cut + 1), template);
+    if ((state === 'failed' || state === 'rejected') && current && Date.parse(timestamp) >= sinceMs) out.failed.push(`${name}: task ${state}: ${rest.join('|')}`);
+  }
+  return out;
+}
+
+/** One service: { pending, failed } lines. A rolled back or paused update since the deploy began fails it at once. */
+function observeService(ctx, name, sinceMs) {
+  const inspect = JSON.parse(docker(ctx, ['service', 'inspect', name, '--format', '{{json .}}']));
+  const desired = inspect.Spec?.Mode?.Replicated?.Replicas ?? 1;
+  const update = inspect.UpdateStatus;
+  const recentUpdate = Boolean(update) && (update.StartedAt ? Date.parse(update.StartedAt) : 0) >= sinceMs;
+  if (recentUpdate && /rollback|paused/u.test(update.State)) return { pending: [], failed: [`${name}: update ${update.State}: ${update.Message ?? ''}`] };
+  const pending = recentUpdate && update.State !== 'completed' ? [`${name}: update in progress (${update.State})`] : [];
+  const taskTemplate = inspect.Spec?.TaskTemplate;
+  const tasks = readTasks(ctx, name, sinceMs, taskTemplate === undefined ? undefined : canonical(taskTemplate));
+  if (tasks.running !== desired) pending.push(`${name}: ${tasks.running}/${desired} running`);
+  return { pending, failed: tasks.failed };
+}
+
+/** One observation over every service of the stack: { ok, pending, failed, services }. */
+function observe(ctx, config, sinceMs) {
+  const services = splitLines(docker(ctx, ['service', 'ls', '--filter', `label=com.docker.stack.namespace=${config.stack}`, '--format', '{{.Name}}']));
+  const result = { pending: services.length === 0 ? [`no service of stack ${config.stack} yet`] : [], failed: [], services };
+  for (const name of services) {
+    const found = observeService(ctx, name, sinceMs);
+    result.pending.push(...found.pending);
+    result.failed.push(...found.failed);
+  }
+  return { ...result, ok: result.pending.length === 0 && result.failed.length === 0 };
+}
+
+/** The observation, or a pending line when docker failed (a thrown docker error is never a crash). */
+function tryObserve(ctx, config, sinceMs, previous) {
+  try {
+    return observe(ctx, config, sinceMs);
+  } catch (error) {
+    return { ok: false, pending: [String(error.message)], failed: [], services: previous.services };
+  }
+}
+
+const result = (ok, lines, last) => ({ ok, lines, data: { services: last.services, failed: last.failed, pending: last.pending } });
+
+/**
+ * Waits until the stack converged. `since` is the ISO instant the deploy began (required). Returns { ok, lines, data: { services, failed, pending } };
+ * on failure the lines name the service and the Swarm task error text.
+ */
+export async function convergeStack(ctx, config, { since, timeoutSeconds = 600 } = {}) {
+  const sinceMs = Date.parse(since);
+  if (Number.isNaN(sinceMs)) throw refusal('STACK_CONVERGE_SINCE', '--since must be an ISO instant such as 2026-01-31T10:00:00Z');
+  const deadline = ctx.now() + timeoutSeconds * 1000;
+  let last = { pending: ['not observed'], failed: [], services: [] };
+  let stable = 0;
+  const outcome = await repeatInOrder(async () => {
+    if (ctx.now() >= deadline) return 'timeout';
+    last = tryObserve(ctx, config, sinceMs, last);
+    if (last.failed.length > 0) return 'failed';
+    stable = last.ok ? stable + 1 : 0;
+    if (stable >= 2) return 'converged';
+    await ctx.sleep(POLL_MS);
+    return undefined;
+  });
+  if (outcome === 'converged') return result(true, [`converge: ${last.services.join(', ')}: desired replicas running, no failed task`], last);
+  const reasons = [...last.failed, ...last.pending].map((line) => `  ${line}`);
+  return result(false, [`converge: ${last.failed.length > 0 ? 'FAILED' : 'TIMEOUT'}`, ...reasons], last);
+}