@ultimat3/cli 23.0.0 → 25.0.0

package/src/cmd-deploy-start-first.ts

@@ -0,0 +1,178 @@
+// One compose role rolled start-first: the new containers come up beside the old ones, are waited
+// on until healthy, and only then are the old ones stopped and removed. Converges on the declared
+// replica count, so a rerun after a half-finished deploy finishes it rather than adding replicas.
+
+import type { ExecOptions, Runner } from './exec';
+
+/**
+ * Bounds `--wait`: a new container that never reports healthy fails the step rather than holding
+ * the deploy open. Five minutes is past any `start_period` the scaffold declares (30s) plus a cold
+ * boot, and `--wait` already returns at once when a container turns `unhealthy` or exits.
+ */
+export const COMPOSE_WAIT_TIMEOUT_SECONDS = 300;
+
+/** The label compose stamps on each container: the hash of the service definition it was made from. */
+const CONFIG_HASH = 'com.docker.compose.config-hash';
+const INSPECT_FORMAT = `{{.Id}} {{.Created}} {{index .Config.Labels "${CONFIG_HASH}"}}`;
+
+/** The `up` that starts the new replicas beside the running ones; `--no-recreate` keeps the old. */
+export function startFirstUp(
+  base: readonly string[],
+  role: string,
+  scale: number | undefined,
+): readonly string[] {
+  const wait = ['--wait', '--wait-timeout', String(COMPOSE_WAIT_TIMEOUT_SECONDS)];
+  return scale === undefined
+    ? [...base, 'up', '-d', '--no-deps', ...wait, role]
+    : [
+        ...base,
+        'up',
+        '-d',
+        '--no-deps',
+        '--no-recreate',
+        ...wait,
+        '--scale',
+        `${role}=${scale}`,
+        role,
+      ];
+}
+
+interface Failed {
+  readonly code: number;
+  readonly command: readonly string[];
+}
+
+export type StartFirstOutcome =
+  | { readonly ok: true }
+  | ({
+      readonly ok: false;
+      /**
+       * The cleanup of an unhealthy new replica that ALSO failed — its argv names the containers
+       * still running, which the operator now owns. Absent when the cleanup ran clean.
+       */
+      readonly cleanup?: Failed | undefined;
+    } & Failed);
+
+const linesOf = (stdout: string): readonly string[] =>
+  stdout
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0);
+
+/** `docker compose config --hash <role>` prints `<role> <hash>`. */
+const hashOf = (stdout: string): string | undefined => linesOf(stdout)[0]?.split(/\s+/)[1];
+
+/**
+ * The running containers split by the definition they were made from: `current` already runs the
+ * one being deployed (newest first), `stale` anything else. A container `inspect` did not describe
+ * is stale — stopping one too many is a restart, keeping one too many is the old build serving.
+ */
+function partition(
+  running: readonly string[],
+  inspected: string,
+  want: string,
+): { readonly current: readonly string[]; readonly stale: readonly string[] } {
+  const described = new Map(
+    linesOf(inspected).map((line) => {
+      const [id = '', created = '', hash = ''] = line.split(/\s+/);
+      return [id, { created, hash }] as const;
+    }),
+  );
+  const facts = running.map((id) => {
+    const match = [...described].find(([full]) => full === id || full.startsWith(id));
+    return { id, created: match?.[1].created ?? '', hash: match?.[1].hash };
+  });
+  return {
+    current: facts
+      .filter((fact) => fact.hash === want)
+      .sort((a, b) => (a.created < b.created ? 1 : -1))
+      .map((fact) => fact.id),
+    stale: facts.filter((fact) => fact.hash !== want).map((fact) => fact.id),
+  };
+}
+
+/**
+ * Start-first for one role. The containers running now are listed and split by compose's config
+ * hash against the definition being deployed: those already on it are kept (up to `replicas`,
+ * newest first), and only the shortfall is created BESIDE the rest (`--scale running + shortfall
+ * --no-recreate`, so compose builds the extra containers from the new definition and leaves the
+ * others alone). Everything is waited on until its healthcheck — `/readyz` for web and sync, the
+ * scrape port for the others — answers. Only then are the stale containers and any surplus stopped
+ * (SIGTERM, then the `stop_grace_period` compose stamped on each as its stop timeout) and removed.
+ *
+ * Counting the shortfall rather than `running + replicas` is what makes a rerun safe: after a
+ * deploy that brought the new containers up and then failed to stop the old, running it again
+ * creates nothing and finishes the stop.
+ *
+ * A new replica that never turns healthy is stopped and removed, and the old ones keep serving;
+ * a cleanup that fails is reported beside the failure, naming the containers left running. The
+ * same sequence `docker-rollout` performs, because `deploy.update_config.order: start-first` is a
+ * Swarm field `docker compose` ignores.
+ */
+export async function runStartFirst(step: {
+  readonly role: string;
+  readonly base: readonly string[];
+  readonly replicas: number;
+  readonly runner: Runner;
+  readonly options: ExecOptions;
+}): Promise<StartFirstOutcome> {
+  const { role, base, replicas, runner, options } = step;
+  const run = async (command: readonly string[]) => {
+    const result = await runner(command, options);
+    return { result, failed: { ok: false as const, code: result.code, command } };
+  };
+  const listing = await run([...base, 'ps', '-q', role]);
+  if (!listing.result.ok) return listing.failed;
+  const running = linesOf(listing.result.stdout);
+  if (running.length === 0) {
+    const fresh = await run(startFirstUp(base, role, undefined));
+    return fresh.result.ok ? { ok: true } : fresh.failed;
+  }
+  const config = await run([...base, 'config', '--hash', role]);
+  if (!config.result.ok) return config.failed;
+  const inspect = await run(['docker', 'inspect', '--format', INSPECT_FORMAT, ...running]);
+  if (!inspect.result.ok) return inspect.failed;
+  const { current, stale } = partition(
+    running,
+    inspect.result.stdout,
+    hashOf(config.result.stdout) ?? '',
+  );
+  const shortfall = Math.max(0, replicas - current.length);
+  const up = await run(startFirstUp(base, role, running.length + shortfall));
+  if (!up.result.ok) return { ...up.failed, ...(await removeCreated(step, running)) };
+  const retire = [...stale, ...current.slice(replicas)];
+  if (retire.length === 0) return { ok: true };
+  const stopped = await run(['docker', 'stop', ...retire]);
+  if (!stopped.result.ok) return stopped.failed;
+  const removed = await run(['docker', 'rm', ...retire]);
+  return removed.result.ok ? { ok: true } : removed.failed;
+}
+
+/**
+ * After a failed `up`: stop and remove what it created, so the containers that were serving are
+ * the role again. A listing that fails leaves the new replicas to the operator; a stop or rm that
+ * fails is returned as `cleanup`, its argv naming the containers still there.
+ */
+async function removeCreated(
+  step: {
+    readonly role: string;
+    readonly base: readonly string[];
+    readonly runner: Runner;
+    readonly options: ExecOptions;
+  },
+  running: readonly string[],
+): Promise<{ readonly cleanup?: Failed }> {
+  const { role, base, runner, options } = step;
+  const after = await runner([...base, 'ps', '-q', role], options);
+  if (!after.ok) return { cleanup: { code: after.code, command: [...base, 'ps', '-q', role] } };
+  const created = linesOf(after.stdout).filter((id) => !running.includes(id));
+  if (created.length === 0) return {};
+  for (const command of [
+    ['docker', 'stop', ...created],
+    ['docker', 'rm', ...created],
+  ]) {
+    const result = await runner(command, options);
+    if (!result.ok) return { cleanup: { code: result.code, command } };
+  }
+  return {};
+}

package/src/cmd-deploy.ts

@@ -5,6 +5,8 @@
 import { join } from 'node:path';
 import { ERROR_DOCS_URL } from '@ultimat3/core';
 import { requireAppRoot } from './app-root';
+import { type ComposeStrategy, composeStrategies, readComposeFile } from './cmd-deploy-compose';
+import { readHelmDrainOverrides } from './cmd-deploy-drain';
 import {
   type HelmTarget,
   helmUpgradeArgs,
@@ -14,6 +16,7 @@ import {
   readRollout,
 } from './cmd-deploy-helm';
 import { deploySpec } from './cmd-deploy-spec';
+import { runStartFirst, startFirstUp } from './cmd-deploy-start-first';
 import type { CliCommand, CommandContext } from './command';
 import { BadFlagError, UnknownCommandError } from './errors';
 import { msg } from './messages';
@@ -78,17 +81,28 @@ export function readMethod(raw: string | undefined): DeployMethod {
 /** The roles that run to completion and exit, as against the ones that stay up serving. */
 const ONE_SHOT_ROLES: readonly string[] = ['migrate', 'backfill'];
 
+/**
+ * One step. A compose serving role carries its rollout `strategy` (`cmd-deploy-compose.ts`); a
+ * start-first step's `command` is its scale-up as it runs in the steady state, where the
+ * containers running equal the declared replicas — the run counts them first.
+ */
+export interface DeployStep {
+  readonly role: string;
+  readonly command: readonly string[];
+  readonly strategy?: ComposeStrategy | undefined;
+}
+
 export interface DeployPlan {
   readonly image: string;
   /** Ordered: migrate runs to completion before any role that serves traffic starts. */
-  readonly steps: readonly { readonly role: string; readonly command: readonly string[] }[];
+  readonly steps: readonly DeployStep[];
   /**
    * The environment every step runs with — how the COMPOSE method carries the image, because
    * `docker-compose.prod.yml` resolves each service from `${IMAGE:-ultimate-app:latest}` and
    * `docker compose` takes no image argument. Without it `--image` decided nothing: the plan
    * reported the reference the operator asked for while the six steps read `IMAGE` off the
    * ambient environment, or deployed `ultimate-app:latest` where it was unset. Helm carries none
-   * — the chart reads `--set image.repository/tag`, and an env var it never looks at would be a
+   * — the chart reads `--set-string image.repository/tag`, and an env var it never looks at would be a
    * second answer to which image is being deployed.
    */
   readonly env: Readonly<Record<string, string>>;
@@ -96,7 +110,7 @@ export interface DeployPlan {
 
 /**
  * The chart declares `image` as a MAP — `repository`, `tag`, `pullPolicy` — and `_helpers.tpl`
- * renders `printf "%s:%s" .Values.image.repository (default .Chart.AppVersion .Values.image.tag)`.
+ * renders `repository:tag` from it, the tag defaulting to `.Chart.AppVersion`.
  * `--set image=<ref>` replaces that map with a string, so every workload template fails on
  * `.repository` and the deploy that was asked to ship one image ships nothing. The reference is
  * split into the two keys the chart actually reads; a reference with no tag sets only the
@@ -109,34 +123,66 @@ export function helmImageOverrides(image: string): readonly string[] {
   const colon = image.lastIndexOf(':');
   const tag = colon > image.lastIndexOf('/') ? image.slice(colon + 1) : '';
   const repository = tag === '' ? image : image.slice(0, colon);
+  // `--set-string`, never `--set`: `--set` TYPES its value, so `image.tag=1234567` reached the chart
+  // as an int64 and rendered `app:%!s(int64=1234567)` — any all-digit tag (a build number, a date,
+  // a short SHA that happens to be digits) was an ImagePullBackOff behind `--wait`.
   return tag === ''
-    ? ['--set', `image.repository=${repository}`]
-    : ['--set', `image.repository=${repository}`, '--set', `image.tag=${tag}`];
+    ? ['--set-string', `image.repository=${repository}`]
+    : ['--set-string', `image.repository=${repository}`, '--set-string', `image.tag=${tag}`];
 }
 
+/**
+ * The OCI reference grammar (distribution/reference), tag form only — the digest form has its own
+ * refusal in `planDeploy`. `[domain[:port]/]path[:tag]`: a domain of dot-separated host labels, path
+ * components of lower-case alphanumerics joined by `.`, `_`, `__` or dashes, a tag of word
+ * characters, `.` and `-`, at most 128. It holds none of helm's `--set-string` metacharacters (`,`
+ * `=` `\` `{` `}`), so a reference it admits never needs escaping, and one it refuses is never
+ * split into a second value.
+ */
+const LABEL = '[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?';
+const COMPONENT = '[a-z0-9]+(?:(?:[._]|__|-+)[a-z0-9]+)*';
+const OCI_REFERENCE = new RegExp(
+  `^(?:${LABEL}(?:\\.${LABEL})*(?::[0-9]+)?/)?${COMPONENT}(?:/${COMPONENT})*(?::[\\w][\\w.-]{0,127})?$`,
+);
+
 /** What was asked for: the method, and for helm the release it is aimed at. */
 export type DeployRequest =
   | { readonly method: 'compose' }
-  | ({ readonly method: 'helm' } & HelmTarget);
+  | ({ readonly method: 'helm' } & HelmDeployTarget);
+
+/** A helm target plus the `--set` pairs that size the chart's grace periods (`cmd-deploy-drain.ts`). */
+export interface HelmDeployTarget extends HelmTarget {
+  readonly drain?: readonly string[] | undefined;
+}
 
 /**
  * The plan for one method. A helm plan takes its target — release, namespace, timeout — because
  * the release is read off `app.config.ts`, which only the command can import; the compose plan
  * needs none, so its three-argument form is unchanged.
  */
-export function planDeploy(image: string, method: 'compose', root: string): DeployPlan;
+export function planDeploy(
+  image: string,
+  method: 'compose',
+  root: string,
+  strategies?: ReadonlyMap<string, ComposeStrategy>,
+): DeployPlan;
 export function planDeploy(
   image: string,
   method: 'helm',
   root: string,
-  target: HelmTarget,
+  target: HelmDeployTarget,
 ): DeployPlan;
 export function planDeploy(
   image: string,
   method: DeployMethod,
   root: string,
-  target?: HelmTarget,
+  extra?: HelmDeployTarget | ReadonlyMap<string, ComposeStrategy>,
 ): DeployPlan {
+  const target = method === 'helm' ? (extra as HelmDeployTarget | undefined) : undefined;
+  const strategies =
+    method === 'compose' && extra instanceof Map
+      ? (extra as ReadonlyMap<string, ComposeStrategy>)
+      : new Map<string, ComposeStrategy>();
   if (method === 'helm') {
     // A caller from plain JS can still reach this without one; `app` for every app is the
     // defect the target exists to end, so it is refused rather than defaulted back.
@@ -159,10 +205,29 @@ export function planDeploy(
         fix: `x deploy --method helm --image ${image.slice(0, image.lastIndexOf('@'))}:<tag> --json`,
       });
     }
+    // Helm splits a `--set-string` argument on `,` and reads `=` as key/value: an image built from
+    // a git ref name (`app,serviceAccount.create=true:1.2`) set a value nobody asked for. The value
+    // is quoted with JSON in the cause (it may hold a newline) and never echoed into the fix.
+    if (!OCI_REFERENCE.test(image)) {
+      throw new BadFlagError({
+        flag: 'image',
+        command: 'deploy',
+        reason: `${JSON.stringify(image)} is not an OCI image reference ([registry[:port]/]path[:tag], lower-case path), and helm would read its , or = as more --set-string values`,
+        fix: 'x deploy --method helm --image ghcr.io/<org>/<app>:<tag> --json',
+      });
+    }
     return {
       image,
       env: {},
-      steps: [{ role: 'all', command: helmUpgradeArgs(root, target, helmImageOverrides(image)) }],
+      steps: [
+        {
+          role: 'all',
+          command: helmUpgradeArgs(root, target, [
+            ...helmImageOverrides(image),
+            ...(target.drain ?? []),
+          ]),
+        },
+      ],
     };
   }
   return {
@@ -171,27 +236,42 @@ export function planDeploy(
     // header documents `IMAGE=… docker compose …` as the way to run it by hand; this is that line,
     // performed.
     env: { IMAGE: image },
-    steps: DEPLOY_ROLES.map((role) => ({
-      role,
-      command: [
-        'docker',
-        'compose',
-        // Compose interpolates `${SYNC_URL:?…}` and `${POSTGRES_PASSWORD:?…}` from the shell and
-        // `--env-file` only — never from a service's `env_file:`. Without this an operator who put
-        // them in `.env.production`, the one file the compose file tells them to fill, had every
-        // step die on a parse error. Global flag, so it precedes `-f`; the shell still wins over it.
-        '--env-file',
-        join(root, PROD_ENV_FILE),
-        '-f',
-        join(root, 'docker', 'docker-compose.prod.yml'),
-        ONE_SHOT_ROLES.includes(role) ? 'run' : 'up',
-        ONE_SHOT_ROLES.includes(role) ? '--rm' : '-d',
-        role,
-      ],
-    })),
+    steps: DEPLOY_ROLES.map((role) => composeStep(composeBase(root), role, strategies.get(role))),
   };
 }
 
+/**
+ * Every compose command's prefix. Compose interpolates `${SYNC_URL:?…}`, `${APP_URL:?…}` and
+ * `${POSTGRES_PASSWORD:?…}` from the shell and `--env-file` only — never from a service's
+ * `env_file:`. Without it an operator who put them in `.env.production`, the one file the compose
+ * file tells them to fill, had every step die on a parse error. Global flag, so it precedes `-f`;
+ * the shell still wins over it.
+ */
+const composeBase = (root: string): readonly string[] => [
+  'docker',
+  'compose',
+  '--env-file',
+  join(root, PROD_ENV_FILE),
+  '-f',
+  composeFileOf(root),
+];
+
+const composeFileOf = (root: string): string => join(root, 'docker', 'docker-compose.prod.yml');
+
+/** One compose step: a one-shot `run --rm`, a start-first scale-up, or compose's own recreate. */
+function composeStep(
+  base: readonly string[],
+  role: string,
+  strategy: ComposeStrategy | undefined,
+): DeployStep {
+  if (ONE_SHOT_ROLES.includes(role)) return { role, command: [...base, 'run', '--rm', role] };
+  if (strategy?.kind === 'start-first') {
+    const scale = strategy.replicas === 0 ? undefined : strategy.replicas * 2;
+    return { role, command: startFirstUp(base, role, scale), strategy };
+  }
+  return { role, command: [...base, 'up', '-d', role], strategy };
+}
+
 /**
  * One step, as the line an operator would type — the environment first, then the command.
  *
@@ -231,9 +311,23 @@ export async function readDeployRequest(
     release: await readReleaseName(root, flagString(ctx.args, 'release')),
     namespace: namespace === undefined ? undefined : readLabel('namespace', namespace),
     timeout: readHelmTimeout(flagString(ctx.args, 'timeout')),
+    drain: await readHelmDrainOverrides(root),
   };
 }
 
+/**
+ * What a failed start-first roll could not undo. The unhealthy replicas it created are stopped and
+ * removed; when that cleanup fails too, the containers it names are still running beside the old
+ * ones, and saying nothing would leave an operator rerunning a deploy over them unaware.
+ */
+const cleanupNote = (outcome: {
+  readonly ok: false;
+  readonly cleanup?: { readonly code: number; readonly command: readonly string[] } | undefined;
+}): string =>
+  outcome.cleanup === undefined
+    ? ''
+    : `; its cleanup \`${outcome.cleanup.command.map(quoteArg).join(' ')}\` exited ${outcome.cleanup.code}, so the containers it names are still running — stop and remove them before rerunning`;
+
 const stepLine = (env: Readonly<Record<string, string>>, command: readonly string[]): string =>
   [
     ...Object.entries(env).map(([name, value]) => `${name}=${quoteArg(value)}`),
@@ -255,10 +349,17 @@ export const deployCommand: CliCommand = {
     // X_DEPLOY_FAILED, whose fix is the exact command to rerun.
     const method = readMethod(flagString(ctx.args, 'method'));
     const request = await readDeployRequest(ctx, method, root);
+    // Read before the plan, because the plan IS the rollout: which compose roles can start their
+    // new container first is a fact of the app's compose file, and a dry run reports it too.
     const plan =
       request.method === 'helm'
         ? planDeploy(image, 'helm', root, request)
-        : planDeploy(image, 'compose', root);
+        : planDeploy(
+            image,
+            'compose',
+            root,
+            composeStrategies(await readComposeFile(composeFileOf(root))),
+          );
     // What the helm flags resolved to — the release above all, which is read off `app.config.ts`
     // and so is not something the operator typed. Absent on compose, which has no release.
     const target: { readonly [key: string]: JsonValue } =
@@ -278,7 +379,12 @@ export const deployCommand: CliCommand = {
       // Reported, because it is what makes `image` above true on the compose method — a dry run
       // that names an image the steps do not carry is the defect this field closed.
       env: { ...plan.env },
-      steps: plan.steps.map((step) => ({ role: step.role, command: step.command.join(' ') })),
+      steps: plan.steps.map((step) => ({
+        role: step.role,
+        command: step.command.join(' '),
+        ...(step.strategy === undefined ? {} : { strategy: step.strategy.kind }),
+        ...(step.strategy?.kind === 'stop-first' ? { why: step.strategy.why } : {}),
+      })),
     };
     // The roles THIS plan has, never the compose list: on `--method helm` there is one step,
     // `all`, and naming `backfill` there promises an operator a sweep the chart cannot run.
@@ -298,10 +404,26 @@ export const deployCommand: CliCommand = {
     // and revision rather than an exit code standing in for one. Compose has no such record.
     let rollout: { readonly [key: string]: JsonValue } = {};
     for (const step of plan.steps) {
-      const result = await ctx.runner(step.command, { cwd: root, env: plan.env });
-      if (request.method === 'helm')
-        rollout = { rollout: { ...readRollout(request, result.stdout) } };
-      if (!result.ok) {
+      const options = { cwd: root, env: plan.env };
+      // A start-first step is a sequence — list, scale up, wait, stop the old — and reports the
+      // command of the sub-step that failed, so the fix line below is still the one to rerun.
+      const outcome =
+        step.strategy?.kind === 'start-first'
+          ? await runStartFirst({
+              role: step.role,
+              base: composeBase(root),
+              replicas: step.strategy.replicas,
+              runner: ctx.runner,
+              options,
+            })
+          : await ctx.runner(step.command, options).then((result) => {
+              if (request.method === 'helm')
+                rollout = { rollout: { ...readRollout(request, result.stdout) } };
+              return result.ok
+                ? { ok: true as const }
+                : { ok: false as const, code: result.code, command: step.command };
+            });
+      if (!outcome.ok) {
         return {
           ok: false,
           command: 'deploy',
@@ -309,8 +431,8 @@ export const deployCommand: CliCommand = {
           findings: [
             {
               code: 'X_DEPLOY_FAILED',
-              cause: `role "${step.role}" step exited ${result.code}`,
-              fix: `${stepLine(plan.env, step.command)}   # run it directly to see the full output`,
+              cause: `role "${step.role}" step exited ${outcome.code}${cleanupNote(outcome)}`,
+              fix: `${stepLine(plan.env, outcome.command)}   # run it directly to see the full output`,
               docs: ERROR_DOCS_URL,
             },
           ],